diff --git a/sner/server/parser.py b/sner/server/parser.py index 05e6bd60..94630d41 100644 --- a/sner/server/parser.py +++ b/sner/server/parser.py @@ -7,6 +7,7 @@ implement ParserBase interface. """ +import json from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime @@ -16,6 +17,7 @@ from littletable import Table as LittleTable import sner.plugin +from sner.lib import ZipFile, file_from_zip, is_zip REGISTERED_PARSERS = {} @@ -30,6 +32,19 @@ def load_parser_plugins(): REGISTERED_PARSERS[plugin_name] = getattr(module, 'ParserModule') +def auto_detect_parser(path): + """tries automatically detect parser""" + if is_zip(path): + with ZipFile(path) as fzip: + for fname in filter(lambda x: x == 'assignment.json', fzip.namelist()): + try: + return json.loads(file_from_zip(path, fname).decode('utf-8'))['config']['module'] + except KeyError: + return None + + return None + + class ParsedItemBase: # pylint: disable=too-few-public-methods """parsed items base object; shared functions""" diff --git a/sner/server/storage/commands.py b/sner/server/storage/commands.py index 4ca1a809..f9e66fa5 100644 --- a/sner/server/storage/commands.py +++ b/sner/server/storage/commands.py @@ -13,7 +13,7 @@ from sner.lib import format_host_address from sner.server.extensions import db -from sner.server.parser import REGISTERED_PARSERS +from sner.server.parser import REGISTERED_PARSERS, auto_detect_parser from sner.server.storage.core import StorageManager, vuln_export, vuln_report from sner.server.storage.models import Host, Service from sner.server.storage.vulnsearch import sync_vulnsearch @@ -30,21 +30,31 @@ def command(): @with_appcontext @click.option('--dry', is_flag=True, help='do not update database, only print new items') @click.option('--addtag', multiple=True, help='add tag to all imported objects, can be used several times') -@click.argument('parser') +@click.option('--parser', help='specify which parser to use instead of auto detection') @click.argument('path', nargs=-1) def storage_import(path, parser, **kwargs): """import data""" - if parser not in REGISTERED_PARSERS: + is_auto_parser = parser is None + + if parser not in REGISTERED_PARSERS and not is_auto_parser: current_app.logger.error('no such parser') sys.exit(1) - parser_impl = REGISTERED_PARSERS[parser] for item in path: if not Path(item).is_file(): current_app.logger.warning(f'invalid path "{item}"') continue + if is_auto_parser: + parser = auto_detect_parser(item) + + if parser is None: + current_app.logger.error(f'parser was not automatically detected for the file: {item}') + continue + + parser_impl = REGISTERED_PARSERS[parser] + try: if kwargs.get('dry'): StorageManager.import_parsed_dry(parser_impl.parse_path(item)) diff --git a/tests/server/storage/test_commands.py b/tests/server/storage/test_commands.py index da6b4a3a..02042e5d 100644 --- a/tests/server/storage/test_commands.py +++ b/tests/server/storage/test_commands.py @@ -17,29 +17,38 @@ def test_import_command_errorhandling(runner): """test invalid parser""" # invalid parser - result = runner.invoke(command, ['import', 'invalid', 'dummy']) + result = runner.invoke(command, ['import', '--parser', 'invalid', 'dummy']) assert result.exit_code == 1 # parse exception handling - result = runner.invoke(command, ['import', 'nmap', 'sner.yaml.example', 'notexist']) + result = runner.invoke(command, ['import', '--parser', 'nmap', 'sner.yaml.example', 'notexist']) assert result.exit_code == 0 def test_import_command_dryrun(runner): """test import dry run""" - result = runner.invoke(command, ['import', '--dry', 'nmap', 'tests/server/data/parser-nmap-output.xml']) + result = runner.invoke(command, ['import', '--dry', '--parser', 'nmap', 'tests/server/data/parser-nmap-output.xml']) assert result.exit_code == 0 assert 'new host:' in result.output assert 'new service:' in result.output assert 'new note:' in result.output - result = runner.invoke(command, ['import', '--dry', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) + result = runner.invoke(command, ['import', '--dry', '--parser', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) assert result.exit_code == 0 assert 'new host:' in result.output assert 'new service:' in result.output assert 'new vuln:' in result.output + result = runner.invoke(command, ['import', '--dry', 'tests/server/data/parser-jarm-job.zip']) + assert result.exit_code == 0 + assert 'new host:' in result.output + assert 'new service:' in result.output + assert 'new note:' in result.output + + result = runner.invoke(command, ['import', '--dry', 'tests/server/data/parser-dummy-job.zip']) + assert result.exit_code == 0 + def test_flush_command(runner, service, vuln, note): # pylint: disable=unused-argument """flush storage database""" diff --git a/tests/server/storage/test_import_parsers.py b/tests/server/storage/test_import_parsers.py index f63d7877..bed6f8dc 100644 --- a/tests/server/storage/test_import_parsers.py +++ b/tests/server/storage/test_import_parsers.py @@ -12,7 +12,7 @@ def test_import_command_nmap_job(runner): """test import nmap job""" - result = runner.invoke(command, ['import', 'nmap', 'tests/server/data/parser-nmap-job.zip']) + result = runner.invoke(command, ['import', '--parser', 'nmap', 'tests/server/data/parser-nmap-job.zip']) assert result.exit_code == 0 assert Host.query.one() @@ -21,11 +21,11 @@ def test_import_command_nmap_job(runner): def test_import_command_nmap_rawdata(runner): """test nmap parser""" - result = runner.invoke(command, ['import', 'nmap', 'tests/server/data/parser-nmap-output.xml']) + result = runner.invoke(command, ['import', '--parser', 'nmap', 'tests/server/data/parser-nmap-output.xml']) assert result.exit_code == 0 # run twice to check update scheme of the import algorithms - result = runner.invoke(command, ['import', 'nmap', 'tests/server/data/parser-nmap-output.xml']) + result = runner.invoke(command, ['import', '--parser', 'nmap', 'tests/server/data/parser-nmap-output.xml']) assert result.exit_code == 0 host = Host.query.one() @@ -39,11 +39,11 @@ def test_import_command_nmap_rawdata(runner): def test_import_command_nessus(runner): """test nessus parser""" - result = runner.invoke(command, ['import', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) + result = runner.invoke(command, ['import', '--parser', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) assert result.exit_code == 0 # run twice to check update scheme of the import algorithms - result = runner.invoke(command, ['import', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) + result = runner.invoke(command, ['import', '--parser', 'nessus', 'tests/server/data/parser-nessus-simple.xml']) assert result.exit_code == 0 host = Host.query.one() @@ -59,7 +59,7 @@ def test_import_command_nessus(runner): def test_import_command_manymap_job(runner): """test manymap parser; zipfile import""" - result = runner.invoke(command, ['import', 'manymap', 'tests/server/data/parser-manymap-job.zip']) + result = runner.invoke(command, ['import', '--parser', 'manymap', 'tests/server/data/parser-manymap-job.zip']) assert result.exit_code == 0 host = Host.query.one() @@ -70,9 +70,20 @@ def test_import_command_manymap_job(runner): def test_import_command_nc(runner): """test nc parser""" - result = runner.invoke(command, ['import', 'nc', 'tests/server/data/parser-nc.txt']) + result = runner.invoke(command, ['import', '--parser', 'nc', 'tests/server/data/parser-nc.txt']) assert result.exit_code == 0 host = Host.query.one() assert len(host.services) == 2 assert sorted([x.port for x in host.services]) == [21, 22] + + +def test_import_command_auto(runner): + """test automatic detection of parser""" + + result = runner.invoke(command, ['import', 'tests/server/data/parser-six_dns_discover-job.zip']) + assert result.exit_code == 0 + assert Host.query + + result = runner.invoke(command, ['import', 'tests/server/data/parser-nc.txt']) + assert result.exit_code == 0