From 1f8cc60c7996cad79c6e75acd0b3add4239eac40 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 26 Jan 2026 08:41:10 +0000 Subject: [PATCH 1/2] Removed adequacy warning catching and made alpha an MR parameter --- .../testing/causal_test_adequacy.py | 20 ++------ .../testing/metamorphic_relation.py | 51 ++++++++++++------- .../test_metamorphic_relations.py | 2 + 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/causal_testing/testing/causal_test_adequacy.py b/causal_testing/testing/causal_test_adequacy.py index 063f8ab8..6dc449c4 100644 --- a/causal_testing/testing/causal_test_adequacy.py +++ b/causal_testing/testing/causal_test_adequacy.py @@ -7,8 +7,6 @@ from itertools import combinations import pandas as pd -from lifelines.exceptions import ConvergenceError -from numpy.linalg import LinAlgError from causal_testing.specification.causal_dag import CausalDAG from causal_testing.testing.causal_test_case import CausalTestCase @@ -104,21 +102,9 @@ def measure_adequacy(self): estimator.df = estimator.df[estimator.df[self.group_by].isin(ids)] else: estimator.df = estimator.df.sample(len(estimator.df), replace=True, random_state=i) - try: - result = self.test_case.execute_test(estimator) - outcomes.append(self.test_case.expected_causal_effect.apply(result)) - results.append(result.effect_estimate.to_df()) - except LinAlgError: - logger.warning("Adequacy LinAlgError") - continue - except ConvergenceError: - logger.warning("Adequacy ConvergenceError") - continue - except ValueError as e: - logger.warning(f"Adequacy ValueError: {e}") - continue - # outcomes = [self.test_case.expected_causal_effect.apply(c) for c in results] - # results = pd.concat([c.effect_estimate.to_df() for c in results]) + result = self.test_case.execute_test(estimator) + outcomes.append(self.test_case.expected_causal_effect.apply(result)) + results.append(result.effect_estimate.to_df()) results = pd.concat(results) results["var"] = results.index results["passed"] = outcomes diff --git a/causal_testing/testing/metamorphic_relation.py b/causal_testing/testing/metamorphic_relation.py index 7c225b63..7f1e2c8d 100644 --- a/causal_testing/testing/metamorphic_relation.py +++ b/causal_testing/testing/metamorphic_relation.py @@ -33,16 +33,13 @@ def __eq__(self, other): same_adjustment_set = set(self.adjustment_vars) == set(other.adjustment_vars) return same_type and same_treatment and same_outcome and same_effect and same_adjustment_set - -class ShouldCause(MetamorphicRelation): - """Class representing a should cause metamorphic relation.""" - def to_json_stub( self, skip: bool = False, estimate_type: str = "coefficient", effect_type: str = "direct", estimator: str = "LinearRegressionEstimator", + alpha: float = 0.05, ) -> dict: """ Convert to a JSON frontend stub string for user customisation. @@ -50,6 +47,7 @@ def to_json_stub( :param effect_type: The type of causal effect to consider (total or direct) :param estimate_type: The estimate type to use when evaluating tests :param estimator: The name of the estimator class to use when evaluating the test + :param alpha: The significance level to use when calculating the confidence intervals """ return { "name": str(self), @@ -57,14 +55,40 @@ def to_json_stub( "estimate_type": estimate_type, "effect": effect_type, "treatment_variable": self.base_test_case.treatment_variable, - "expected_effect": {self.base_test_case.outcome_variable: "SomeEffect"}, "formula": ( f"{self.base_test_case.outcome_variable} ~ " f"{' + '.join([self.base_test_case.treatment_variable] + self.adjustment_vars)}" ), + "alpha": alpha, "skip": skip, } + +class ShouldCause(MetamorphicRelation): + """Class representing a should cause metamorphic relation.""" + + def to_json_stub( + self, + skip: bool = False, + estimate_type: str = "coefficient", + effect_type: str = "direct", + estimator: str = "LinearRegressionEstimator", + alpha: float = 0.05, + ) -> dict: + """ + Convert to a JSON frontend stub string for user customisation. + :param skip: Whether to skip the test (default False). + :param effect_type: The type of causal effect to consider (total or direct) + :param estimate_type: The estimate type to use when evaluating tests + :param estimator: The name of the estimator class to use when evaluating the test + :param alpha: The significance level to use when calculating the confidence intervals + """ + return super().to_json_stub( + skip=skip, estimate_type=estimate_type, effect_type=effect_type, estimator=estimator, alpha=alpha + ) | { + "expected_effect": {self.base_test_case.outcome_variable: "SomeEffect"}, + } + def __str__(self): formatted_str = f"{self.base_test_case.treatment_variable} --> {self.base_test_case.outcome_variable}" if self.adjustment_vars: @@ -81,6 +105,7 @@ def to_json_stub( estimate_type: str = "coefficient", effect_type: str = "direct", estimator: str = "LinearRegressionEstimator", + alpha: float = 0.05, ) -> dict: """ Convert to a JSON frontend stub string for user customisation. @@ -88,20 +113,12 @@ def to_json_stub( :param effect_type: The type of causal effect to consider (total or direct) :param estimate_type: The estimate type to use when evaluating tests :param estimator: The name of the estimator class to use when evaluating the test + :param alpha: The significance level to use when calculating the confidence intervals """ - return { - "name": str(self), - "estimator": estimator, - "estimate_type": estimate_type, - "effect": effect_type, - "treatment_variable": self.base_test_case.treatment_variable, + return super().to_json_stub( + skip=skip, estimate_type=estimate_type, effect_type=effect_type, estimator=estimator, alpha=alpha + ) | { "expected_effect": {self.base_test_case.outcome_variable: "NoEffect"}, - "formula": ( - f"{self.base_test_case.outcome_variable} ~ " - f"{' + '.join([self.base_test_case.treatment_variable] + self.adjustment_vars)}" - ), - "alpha": 0.05, - "skip": skip, } def __str__(self): diff --git a/tests/testing_tests/test_metamorphic_relations.py b/tests/testing_tests/test_metamorphic_relations.py index 75ad6a6d..709a891b 100644 --- a/tests/testing_tests/test_metamorphic_relations.py +++ b/tests/testing_tests/test_metamorphic_relations.py @@ -104,6 +104,7 @@ def test_should_cause_json_stub(self): "formula": "Z ~ X1", "treatment_variable": "X1", "name": "X1 --> Z", + "alpha": 0.05, "skip": False, }, ) @@ -130,6 +131,7 @@ def test_should_cause_logistic_json_stub(self): "formula": "Z ~ X1", "treatment_variable": "X1", "name": "X1 --> Z", + "alpha": 0.05, "skip": False, }, ) From 32fbf151db42563629e4118c401080804183ce4c Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 26 Jan 2026 10:00:36 +0000 Subject: [PATCH 2/2] No to pandas 3 as it breaks tests --- pyproject.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0dd4a911..1bd2ef9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "lifelines~=0.30.0", "networkx>=3.4,<3.5", "numpy>=1.26.0,<=2.2.0", -"pandas>=2.1", +"pandas>=2.1,<3", "scikit_learn~=1.4", "scipy>=1.12.0,<=1.16.2", "statsmodels~=0.14", @@ -100,8 +100,12 @@ skip_missing_interpreters = false # fail if devs don’t have all required Pytho description = "Run pytest under {base_python}" extras = ["dev","test"] deps = ["pytest"] -commands = [["pytest"]] - +commands = [ + [ + "pytest", + "{posargs:tests}", + ], +] # Automatically test for type-checking (TODO: enable type checking in env_list in the future) [tool.tox.env.type] description = "Run type checks with mypy on the codebase"