diff --git a/VERSION b/VERSION index f17f5621..f4476715 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0+2026.04.16T12.07.43.534Z.b0eade5c.berickson.20260406.portfolio.fixes +0.1.0+2026.04.23T20.03.53.409Z.887fd5d9.berickson.20260423.test.fix diff --git a/learning_observer/VERSION b/learning_observer/VERSION index ae344fc5..f4476715 100644 --- a/learning_observer/VERSION +++ b/learning_observer/VERSION @@ -1 +1 @@ -0.1.0+2026.04.06T14.01.45.433Z.6e6d9aca.berickson.20260406.portfolio.fixes +0.1.0+2026.04.23T20.03.53.409Z.887fd5d9.berickson.20260423.test.fix diff --git a/learning_observer/learning_observer/adapters/helpers.py b/learning_observer/learning_observer/adapters/helpers.py index 5a311168..96c60c1b 100644 --- a/learning_observer/learning_observer/adapters/helpers.py +++ b/learning_observer/learning_observer/adapters/helpers.py @@ -18,16 +18,13 @@ def rename_json_keys(source, replacements): >>> source = { ... "event-type": "blog", ... "writing-log": "foobar", - } + ... } >>> replacements = { ... "event-type": "event_type", ... "writing-log": "writing_log", ... } >>> rename_json_keys(source, replacements) - { - "event_type": "blog", - "writing_log": "foobar", - } + {'event_type': 'blog', 'writing_log': 'foobar'} ''' if isinstance(source, dict): for key, value in list(source.items()): diff --git a/learning_observer/learning_observer/auth/events.py b/learning_observer/learning_observer/auth/events.py index e650992a..b862fcf0 100644 --- a/learning_observer/learning_observer/auth/events.py +++ b/learning_observer/learning_observer/auth/events.py @@ -40,6 +40,11 @@ AUTH_METHODS = {} +class TestRequest: + """Simple request stub for doctests.""" + pass + + def register_event_auth(name): ''' Decorator to register a method to authenticate events @@ -135,7 +140,9 @@ async def guest_auth(request, event, source): We assign a cookie on first visit, but we have no guarantee the browser will keep cookies around. - >>> a = asyncio.run(guest_auth(TestRequest(), [], {}, 'org.mitros.test')) + >>> from unittest.mock import AsyncMock, patch + >>> with patch('aiohttp_session.get_session', new=AsyncMock(return_value={})): + ... a = asyncio.run(guest_auth(TestRequest(), {}, 'org.mitros.test')) >>> a['user_id'] = len(a['user_id']) # Different user_id each time, and we want doctest to match exact string. >>> a {'sec': 'none', 'user_id': 32, 'providence': 'guest'} @@ -163,12 +170,16 @@ async def local_storage_auth(request, event, source): unauthenticated (if we don't), or allow for both, with a tag for guest versus non-guest accounts. + >>> from unittest.mock import patch >>> auth_event = {'event': 'local_storage', 'user_tag': 'bob'} - >>> a = asyncio.run(local_storage_auth(TestRequest(), [], auth_event, 'org.mitros.test')) + >>> with patch('learning_observer.auth.events.token_authorize_user', return_value='authenticated'): + ... a = asyncio.run(local_storage_auth(TestRequest(), auth_event, 'org.mitros.test')) >>> a {'sec': 'authenticated', 'user_id': 'ls-bob', 'providence': 'ls'} >>> auth_event['user_tag'] = 'jim' - >>> a = asyncio.run(local_storage_auth(TestRequest(), [auth_event], {}, 'org.mitros.test')) + >>> with patch('learning_observer.auth.events.token_authorize_user', return_value='unauthenticated'): + ... a = asyncio.run(local_storage_auth(TestRequest(), auth_event, 'org.mitros.test')) + >>> a {'sec': 'unauthenticated', 'user_id': 'ls-jim', 'providence': 'ls'} ''' @@ -344,9 +355,6 @@ def check_event_auth_config(): import doctest print("Running tests") - class TestRequest: - pass - session = {} async def get_session(request): diff --git a/learning_observer/learning_observer/communication_protocol/executor.py b/learning_observer/learning_observer/communication_protocol/executor.py index ef59307d..8e4963f7 100644 --- a/learning_observer/learning_observer/communication_protocol/executor.py +++ b/learning_observer/learning_observer/communication_protocol/executor.py @@ -19,7 +19,7 @@ import learning_observer.stream_analytics.fields import learning_observer.stream_analytics.helpers from learning_observer.log_event import debug_log -from learning_observer.util import get_nested_dict_value, clean_json, ensure_async_generator, async_zip +from learning_observer.util import get_nested_dict_value, clean_json, ensure_async_generator, async_zip, async_generator_to_list from learning_observer.communication_protocol.exception import DAGExecutionException @@ -1133,7 +1133,5 @@ async def visit(node_name): if __name__ == "__main__": import doctest - # This function is used by doctests - from learning_observer.util import async_generator_to_list doctest.testmod(optionflags=doctest.ELLIPSIS) diff --git a/learning_observer/learning_observer/doc_processor.py b/learning_observer/learning_observer/doc_processor.py index 9dfe22a1..7069cbcc 100644 --- a/learning_observer/learning_observer/doc_processor.py +++ b/learning_observer/learning_observer/doc_processor.py @@ -17,7 +17,7 @@ import learning_observer.auth.utils import learning_observer.constants -import learning_observer.google +import learning_observer.integrations.google as google_integration import learning_observer.kvs import learning_observer.offline import learning_observer.run @@ -26,13 +26,25 @@ import learning_observer.stream_analytics.helpers as sa_helpers import learning_observer.util -import writing_observer -import writing_observer.awe_nlp -import writing_observer.languagetool -import writing_observer.writing_analysis +try: + import writing_observer + import writing_observer.awe_nlp + import writing_observer.languagetool + import writing_observer.writing_analysis +except ModuleNotFoundError: + writing_observer = None from learning_observer.log_event import debug_log + +def _require_writing_observer(): + if writing_observer is None: + raise RuntimeError( + "writing_observer is required for document processing, " + "but is not installed in this environment." + ) + + pmss.register_field( name='document_processing_delay_seconds', type=pmss.pmsstypes.TYPES.integer, @@ -93,6 +105,7 @@ async def check_recent_mod_and_not_recent_process(doc_id): processing and check whether it is past a specified cutoff time (5 minutes). ''' + _require_writing_observer() cutoff = learning_observer.settings.pmss_settings.document_processing_delay_seconds(types=['modules', 'writing_observer']) student_id = await _determine_student(doc_id) @@ -161,8 +174,9 @@ def fetch_mock_runtime(creds): async def start(): learning_observer.offline.init('creds.yaml') global app, KVS + _require_writing_observer() app = StubApp(asyncio.get_event_loop()) - learning_observer.google.initialize_and_register_routes(app) + google_integration.initialize_and_register_routes(app) KVS = learning_observer.kvs.KVS() # overwrite aiohttp_session.get_session so the Google API diff --git a/learning_observer/learning_observer/integrations/google.py b/learning_observer/learning_observer/integrations/google.py index 4ef51a89..d83a6382 100644 --- a/learning_observer/learning_observer/integrations/google.py +++ b/learning_observer/learning_observer/integrations/google.py @@ -161,11 +161,11 @@ def _force_text_length(text, length): ''' Force text to a given length, either concatenating or padding - >>> force_text_length("Hello", 3) - >>> 'Hel' + >>> _force_text_length("Hello", 3) + 'Hel' - >>> force_text_length("Hello", 13) - >>> 'Hello ' + >>> _force_text_length("Hello", 13) + 'Hello ' ''' return text[:length] + " " * (length - len(text)) diff --git a/learning_observer/learning_observer/integrations/util.py b/learning_observer/learning_observer/integrations/util.py index bf122ea0..a819538b 100644 --- a/learning_observer/learning_observer/integrations/util.py +++ b/learning_observer/learning_observer/integrations/util.py @@ -64,7 +64,7 @@ def extract_parameters_from_format_string(format_string): ''' Extracts parameters from a format string. E.g. - >>> ("hello {hi} my {bye}")] + >>> extract_parameters_from_format_string("hello {hi} my {bye}") ['hi', 'bye'] ''' # The parse returns a lot of context, which we discard. In particular, the diff --git a/learning_observer/learning_observer/merkle_store.py b/learning_observer/learning_observer/merkle_store.py index 1f5fff60..dae1d3fa 100644 --- a/learning_observer/learning_observer/merkle_store.py +++ b/learning_observer/learning_observer/merkle_store.py @@ -68,17 +68,13 @@ import hashlib import json import datetime -from modulefinder import STORE_GLOBAL import os -from pickle import STOP -# These should be abstracted out into a visualization library. -import matplotlib -import networkx -from learning_observer.incoming_student_event import COUNT -import pydot - -from confluent_kafka import Producer, Consumer +try: + from confluent_kafka import Producer, Consumer +except: + Producer = None + Consumer = None def json_dump(obj): @@ -410,6 +406,7 @@ def to_networkx(self): This is used for testing, experimentation, and demonstration. It would never scale with real data. ''' + import networkx G = networkx.DiGraph() for item in self._walk(): print(item) @@ -426,6 +423,7 @@ def to_graphviz(self): This is used for testing, experimentation, and demonstration. It would never scale with real data. ''' + import pydot G = pydot.Dot(graph_type='digraph') for item in self._walk(): node = pydot.Node(item['hash'], label=self._make_label(item)) diff --git a/learning_observer/learning_observer/pubsub/__init__.py b/learning_observer/learning_observer/pubsub/__init__.py index 3528facd..8962be81 100644 --- a/learning_observer/learning_observer/pubsub/__init__.py +++ b/learning_observer/learning_observer/pubsub/__init__.py @@ -18,16 +18,14 @@ TODO this module is no longer being used by the LO system. This should be removed. ''' -import sys - import learning_observer.settings as settings from learning_observer.log_event import debug_log try: PUBSUB = settings.settings['pubsub']['type'] -except KeyError: - print("Pub-sub configuration missing from configuration file.") - sys.exit(-1) +except (TypeError, KeyError): + debug_log("Pub-sub configuration missing from configuration file; defaulting to stub.") + PUBSUB = 'stub' if PUBSUB == 'xmpp': import learning_observer.pubsub.receivexmpp diff --git a/learning_observer/learning_observer/static_data/make_dummy_test_user_tsv.py b/learning_observer/learning_observer/static_data/make_dummy_test_user_tsv.py index a816be42..3bc71a22 100644 --- a/learning_observer/learning_observer/static_data/make_dummy_test_user_tsv.py +++ b/learning_observer/learning_observer/static_data/make_dummy_test_user_tsv.py @@ -1,26 +1,35 @@ import random +from pathlib import Path import names import tsvx -user_id_template = "tsu-ts-test-user-{i}" -w = tsvx.writer(open("class_lists/test_users.tsvx", "w")) -w.title = "Test users" -w.description = "Test user file to go with stream_writing.py" -w.headers = ["user_id", "name", "full_name", "email", "phone"] -w.types = [str, str, str, str, str] +def main(): + user_id_template = "tsu-ts-test-user-{i}" + output_dir = Path("class_lists") + output_dir.mkdir(parents=True, exist_ok=True) + with open(output_dir / "test_users.tsvx", "w") as output_file: + w = tsvx.writer(output_file) + w.title = "Test users" + w.description = "Test user file to go with stream_writing.py" + w.headers = ["user_id", "name", "full_name", "email", "phone"] + w.types = [str, str, str, str, str] -w.write_headers() -for i in range(25): - name = names.get_first_name() - w.write( - user_id_template.format(i=i), - name, - "{fn} {ln}".format(fn=name, ln=names.get_last_name()), - "{name}@school.district.us".format(name=name), - "({pre})-{mid}-{post}".format( - pre=random.randint(200, 999), - mid=random.randint(200, 999), - post=random.randint(1000, 9999)) - ) + w.write_headers() + for i in range(25): + name = names.get_first_name() + w.write( + user_id_template.format(i=i), + name, + "{fn} {ln}".format(fn=name, ln=names.get_last_name()), + "{name}@school.district.us".format(name=name), + "({pre})-{mid}-{post}".format( + pre=random.randint(200, 999), + mid=random.randint(200, 999), + post=random.randint(1000, 9999)) + ) + + +if __name__ == "__main__": + main() diff --git a/learning_observer/learning_observer/stream_analytics/helpers.py b/learning_observer/learning_observer/stream_analytics/helpers.py index 05d5ef1c..99b5e2ba 100644 --- a/learning_observer/learning_observer/stream_analytics/helpers.py +++ b/learning_observer/learning_observer/stream_analytics/helpers.py @@ -52,7 +52,7 @@ def fully_qualified_function_name(func): that function. E.g.: >>> from math import sin - >>> fully_qualified_function_name(math.sin) + >>> fully_qualified_function_name(sin) 'math.sin' This is helpful for then giving unique names to analytics modules. Each module can @@ -147,11 +147,12 @@ def make_key(func, key_dict, state_type): Into a unique string For example: - >>> make_key( - some_module.reducer, - {h.KeyField.STUDENT: 123}, - h.KeyStateType.INTERNAL - ) + >>> from learning_observer.stream_analytics.fields import KeyField, KeyStateType + >>> def reducer(_): + ... return _ + >>> reducer.__module__ = 'some_module' + >>> reducer.__qualname__ = 'reducer' + >>> make_key(reducer, {KeyField.STUDENT: 123}, KeyStateType.INTERNAL) 'Internal,some_module.reducer,STUDENT:123' ''' # pylint: disable=isinstance-second-argument-not-valid-type diff --git a/requirements.txt b/requirements.txt index 9f4a5afb..5562f6e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ dash dash_renderjson docopt dash-bootstrap-components -tsvx @ git+https://github.com/pmitros/tsvx.git@09bf7f33107f66413d929075a8b54c36ca581dae#egg=tsvx +tsvx @ https://codeload.github.com/pmitros/tsvx/tar.gz/f883c63892dc383cc15d1d14c05a33d16fb92317 ipython ipykernel jsonschema