From 1229d8f22eca691a0ce1027a927a8dfde7ac1065 Mon Sep 17 00:00:00 2001 From: anna-singleton-resolver Date: Thu, 5 Mar 2026 14:55:22 +0000 Subject: [PATCH 1/3] feat: configurable e2e test cases --- tests/functional/e2e/test_classify_single.py | 4 ++-- tests/functional/e2e/testcases/parser.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/functional/e2e/test_classify_single.py b/tests/functional/e2e/test_classify_single.py index 06d26a4..9af4cb0 100644 --- a/tests/functional/e2e/test_classify_single.py +++ b/tests/functional/e2e/test_classify_single.py @@ -11,10 +11,10 @@ from resolver_athena_client.client.models import ImageData from tests.functional.e2e.testcases.parser import ( AthenaTestCase, - load_test_cases, + load_test_cases_by_env, ) -TEST_CASES = load_test_cases("integrator_sample") +TEST_CASES = load_test_cases_by_env() FP_ERROR_TOLERANCE = 1e-4 diff --git a/tests/functional/e2e/testcases/parser.py b/tests/functional/e2e/testcases/parser.py index 80d7901..fd81217 100644 --- a/tests/functional/e2e/testcases/parser.py +++ b/tests/functional/e2e/testcases/parser.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path # Path to the shared testcases directory in athena-protobufs @@ -23,6 +24,12 @@ def __init__( self.classification_labels: list[str] = classification_labels +def load_test_cases_by_env() -> list[AthenaTestCase]: + return load_test_cases( + os.getenv("ATHENA_E2E_TESTCASE_DIR", "integrator_sample") + ) + + def load_test_cases(dirname: str = "benign_model") -> list[AthenaTestCase]: with Path.open( Path(TESTCASES_DIR / dirname / "expected_outputs.json"), From f5c7ebc77519f6e725397a3777216fc4b6308adc Mon Sep 17 00:00:00 2001 From: anna-singleton-resolver Date: Thu, 5 Mar 2026 15:08:51 +0000 Subject: [PATCH 2/3] test: load dotenv before loading e2e test cases like other fixtures --- tests/functional/e2e/testcases/parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/e2e/testcases/parser.py b/tests/functional/e2e/testcases/parser.py index fd81217..3d2275a 100644 --- a/tests/functional/e2e/testcases/parser.py +++ b/tests/functional/e2e/testcases/parser.py @@ -2,6 +2,8 @@ import os from pathlib import Path +from dotenv import load_dotenv + # Path to the shared testcases directory in athena-protobufs _REPO_ROOT = Path(__file__).parent.parent.parent.parent.parent TESTCASES_DIR = _REPO_ROOT / "athena-protobufs" / "testcases" @@ -25,6 +27,7 @@ def __init__( def load_test_cases_by_env() -> list[AthenaTestCase]: + _ = load_dotenv() return load_test_cases( os.getenv("ATHENA_E2E_TESTCASE_DIR", "integrator_sample") ) From b8f36a3c9f2b8f44fc6326e1f4e8e6acc24663f1 Mon Sep 17 00:00:00 2001 From: anna-singleton-resolver Date: Thu, 5 Mar 2026 15:38:51 +0000 Subject: [PATCH 3/3] doc: update docs to include information about e2e testing --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9a2d35a..bac9b7c 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,8 @@ ATHENA_NON_EXISTENT_AFFILIATE=non-existent-affiliate-id (default: thisaffiliatedoesnotexist123) - this is used to test error handling. ATHENA_NON_PERMITTED_AFFILIATE=non-permitted-affiliate-id (default: thisaffiliatedoesnothaveathenaenabled) - this is used to test error handling. +ATHENA_E2E_TESTCASE_DIR=test-case-directory (default: integrator_sample) - this is the test case directory to use for the e2e tests. +See E2E Tests section below for more details. ``` Then run the functional tests with: @@ -170,8 +172,18 @@ Then run the functional tests with: pytest -m functional ``` -To exclude the e2e tests, which require usage of the live classifier and -therefore are unsuitable for regular development runs, use: +#### E2E Tests + +The e2e tests assert that the API returns some expected _scores_ rather than +exercising different API paths. As such, they are dependent on the classifier +that you are calling through the API. Right now, there are 2 types of +classifier, benign and live. By default, the tests will run the +`integrator_sample` test set, which uses the live classifier. If you wish to +use the benign classifier instead, you may set the `ATHENA_E2E_TESTCASE_DIR` +environment variable to `benign_model`. + +Alternatively, you may disable these tests altogether, by excluding tests that +have the `e2e` marker, something like this: ```bash pytest -m 'functional and not e2e'