diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..65a530a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +build-backend = "flit_core.buildapi" +requires = ["flit_core >=3.2,<4"] + +[project] +name = "yahi" +version = "0.2.11" +description = "Versatile log parser" +readme = "README.md" +python = ">=3.11" +wheel = "*" +authors = [ + { name = "julien tayon", email = "julien@tayon.net" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Python Software Foundation License", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "Programming Language :: Python", +] +dependencies = [ + "repoze.lru", + "archery>=0.3.0", + "httpagentparser==1.9.5", + "pygeoip==0.2.3", + ] + +[project.urls] +Homepage = "https://github.com/jul/yahi" +Documentation = "https://yahi.readthedocs.io/en/latest/index.html" +Repository = "https://github.com/jul/yahi" +Issues = "https://github.com/jul/yahi/issues" +Changelog = "https://github.com/jul/yahi/release-notes/" + +[project.scripts] +speed_shoot = "yahi.scripts.speed_shoot:main" +yahi_all_in_one_maker = "yahi.scripts.yahi_all_in_one_maker:main" + + +[tool.pytest.ini_options] +addopts = [ + "--strict-config", + "--strict-markers", + "--ignore=docs", +] + +[tool.coverage.run] +parallel = true +data_file = "coverage/.coverage" +source = [ + "docs", + "test", + "yahi" +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade +] +ignore = [ + "E501", # line too long, handled by black + "B008", # do not perform function calls in argument defaults + "C901", # too complex + "W191", # indentation contains tabs +] diff --git a/scripts/speed_shoot b/scripts/speed_shoot deleted file mode 100755 index c67aad0..0000000 --- a/scripts/speed_shoot +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -from archery import mdict -from yahi import notch, shoot -from datetime import datetime - - -context=notch() - -date_formater= lambda dt :"%s-%s-%s" % ( dt.year, dt.month, dt.day) -context.output( - shoot( - context, - lambda data : mdict({ - 'by_country': mdict({data['_country']: 1}), - 'date_hit': mdict({data['_date']: 1 }), - 'date_bandwidth': mdict({data['_date']: int(data["bytes"]) }), - 'hour_hit': mdict({data['_datetime'].hour: 1 }), - 'hour_bandwidth': mdict({data['_datetime'].hour: int(data["bytes"]) }), - 'by_os': mdict({data['_platform_name']: 1 }), - 'by_scheme' : mdict( { data["scheme"] : 1 }), - 'by_scheme_by_bandwidth' : mdict( { data["scheme"] : int(data["bytes"])}), - 'by_dist': mdict({data['_dist_name']: 1 }), - 'by_browser': mdict({data['_browser_name']: 1 }), - 'by_bandwidth_by_browser': mdict({data['_browser_name']: int(data["bytes"]) }), - 'by_ip': mdict({data['ip']: 1 }), - 'by_bandwidth_by_ip': mdict({data['ip']: int(data["bytes"]) }), - 'by_status': mdict({data['status']: 1 }), - 'by_url': mdict({data['uri']: 1}), - 'by_agent': mdict({data['agent']: 1}), - 'by_referer': mdict({data['referer']: 1}), - 'date_dayofweek_hit' : mdict({data['_datetime'].weekday(): 1 }), - 'weekday_browser' : mdict({data['_datetime'].weekday(): - mdict({data["_browser_name"] :1 })}), - 'total_line' : 1, - }), - ), -) diff --git a/scripts/yahi_all_in_one_maker b/scripts/yahi_all_in_one_maker deleted file mode 100755 index c831193..0000000 --- a/scripts/yahi_all_in_one_maker +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -All in one maker -================ - -Generates a all in one yahi output web page from the stats given in -its json form as the first parameter. - -Outputs the result in the local file aio.html - -""" -from sys import argv -from yahi.template import template -from os import path -from html import escape - -usage = lambda : print(__doc__) or exit(1) - -data_location="./data.js" -try: - data_location = argv[1] -except: - pass - -try: - with open(data_location) as f: - DATA=f.read() - res = template.replace("{{DATA}}", escape(DATA)) - with open("aio.html", "w", encoding='utf-8') as h: - h.write(res) -except Exception as e: - print(e) - usage() - - diff --git a/setup.py b/setup.py deleted file mode 100644 index afc967e..0000000 --- a/setup.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# -*- coding: "utf-8" -*- - -from setuptools import setup, find_packages -import unittest -import sys - -def test(): - """Specialized Python source builder.""" - loader= unittest.TestLoader() - suite=loader.discover(".", "test_yahi.py") - runner=unittest.TextTestRunner() - result=runner.run(suite) - if not result.wasSuccessful(): - raise Exception( "Test Failed: Aborting install") - -if "sdist" in sys.argv or "bdist_egg" in sys.argv: - - test() - -long_description = open("README.md").read() - -setup( - name='yahi', - version='0.2.10', - author='Julien Tayon, Stephane Bard', - author_email='julien@tayon.net', - packages=['yahi'], - install_requires=[ 'archery>=0.1', 'pygeoip', 'httpagentparser', 'repoze.lru>=0.6' ], - keywords=['log', 'parsing' ], - url='https://github.com/jul/yahi', - scripts=["scripts/speed_shoot", "scripts/yahi_all_in_one_maker"], - license=open('LICENSE.txt').read(), - description='Versatile log parser', - long_description_content_type='text/markdown', - long_description=long_description, - classifiers=[ - 'License :: OSI Approved :: Python Software Foundation License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Programming Language :: Python', - ], -) diff --git a/yahi/__init__.py b/yahi/__init__.py index 29dd477..f71dbdf 100755 --- a/yahi/__init__.py +++ b/yahi/__init__.py @@ -8,7 +8,7 @@ try: import pygeoip except ModuleNotFoundError: - warn("pygeoip required for full feartures" ,ImportWarning) + warn("pygeoip required for full features" ,ImportWarning) try: import httpagentparser except ModuleNotFoundError: diff --git a/yahi/scripts/speed_shoot.py b/yahi/scripts/speed_shoot.py new file mode 100755 index 0000000..0dcc734 --- /dev/null +++ b/yahi/scripts/speed_shoot.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +from archery import mdict +from yahi import notch, shoot +from datetime import datetime + + +context=notch() + + +def main(): + """ + """ + date_formater= lambda dt :"%s-%s-%s" % ( dt.year, dt.month, dt.day) + context.output( + shoot( + context, + lambda data : mdict({ + 'by_country': mdict({data['_country']: 1}), + 'date_hit': mdict({data['_date']: 1 }), + 'date_bandwidth': mdict({data['_date']: int(data["bytes"]) }), + 'hour_hit': mdict({data['_datetime'].hour: 1 }), + 'hour_bandwidth': mdict({data['_datetime'].hour: int(data["bytes"]) }), + 'by_os': mdict({data['_platform_name']: 1 }), + 'by_scheme' : mdict( { data["scheme"] : 1 }), + 'by_scheme_by_bandwidth' : mdict( { data["scheme"] : int(data["bytes"])}), + 'by_dist': mdict({data['_dist_name']: 1 }), + 'by_browser': mdict({data['_browser_name']: 1 }), + 'by_bandwidth_by_browser': mdict({data['_browser_name']: int(data["bytes"]) }), + 'by_ip': mdict({data['ip']: 1 }), + 'by_bandwidth_by_ip': mdict({data['ip']: int(data["bytes"]) }), + 'by_status': mdict({data['status']: 1 }), + 'by_url': mdict({data['uri']: 1}), + 'by_agent': mdict({data['agent']: 1}), + 'by_referer': mdict({data['referer']: 1}), + 'date_dayofweek_hit' : mdict({data['_datetime'].weekday(): 1 }), + 'weekday_browser' : mdict({data['_datetime'].weekday(): + mdict({data["_browser_name"] :1 })}), + 'total_line' : 1, + }), + ), + ) + +if __name__=="__main__": + main() diff --git a/yahi/scripts/yahi_all_in_one_maker.py b/yahi/scripts/yahi_all_in_one_maker.py new file mode 100755 index 0000000..66c5fcb --- /dev/null +++ b/yahi/scripts/yahi_all_in_one_maker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +All in one maker +================ + +Generates a all in one yahi output web page from the stats given in +its json form as the first parameter. + +Outputs the result in the local file aio.html + +""" +from sys import argv +from yahi.template import template +from os import path +from html import escape + +usage = lambda : print(__doc__) or exit(1) + +def main(): + data_location="./data.js" + try: + data_location = argv[1] + except: + pass + + try: + with open(data_location) as f: + DATA=f.read() + res = template.replace("{{DATA}}", escape(DATA)) + with open("aio.html", "w", encoding='utf-8') as h: + h.write(res) + except Exception as e: + print(e) + usage() + +if __name__=="__main__": + main()