From 6e019e9e5c26651cf23759928f4d71f8187b63e2 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:22:34 +0200 Subject: [PATCH 01/11] fix: np.complex removal in numpy>=1.24 for TFAD and LOF --- ts_benchmark/baselines/self_impl/LOF/lof.py | 5 +++-- ts_benchmark/baselines/self_impl/TFAD/model/fft_aug.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ts_benchmark/baselines/self_impl/LOF/lof.py b/ts_benchmark/baselines/self_impl/LOF/lof.py index d80ec3a..db8ac19 100644 --- a/ts_benchmark/baselines/self_impl/LOF/lof.py +++ b/ts_benchmark/baselines/self_impl/LOF/lof.py @@ -93,7 +93,7 @@ def detect_score(self, X: pd.DataFrame) -> np.ndarray: .fit_transform(self.decision_scores_.reshape(-1, 1)) .ravel() ) - return score + return score, score def detect_label(self, X: pd.DataFrame) -> np.ndarray: """ @@ -123,7 +123,8 @@ def detect_label(self, X: pd.DataFrame) -> np.ndarray: .fit_transform(self.decision_scores_.reshape(-1, 1)) .ravel() ) - return score + preds = (score > np.percentile(score, 100 * (1 - self.contamination))).astype(int) + return preds, score def __repr__(self) -> str: """ diff --git a/ts_benchmark/baselines/self_impl/TFAD/model/fft_aug.py b/ts_benchmark/baselines/self_impl/TFAD/model/fft_aug.py index 6dcfa6d..bb8e932 100644 --- a/ts_benchmark/baselines/self_impl/TFAD/model/fft_aug.py +++ b/ts_benchmark/baselines/self_impl/TFAD/model/fft_aug.py @@ -26,7 +26,7 @@ def seasonal_shift( xlen = int(np.ceil(multi * x.shape[0])) print("xlen is", xlen) - a = np.complex(0 + 0j) + a = np.complex128(0 + 0j) fft_yn_new = a * np.arange(xlen) if multi < 1: @@ -59,7 +59,7 @@ def with_noise( flag = np.random.randint(low=0, high=2) xn = x.numpy() fft_yn = fft(xn - np.mean(xn)) - a = np.complex(0 + 0j) + a = np.complex128(0 + 0j) fft_yn_new = a * np.arange(x.shape[0]) prop = np.random.uniform(0.01, 0.5) @@ -98,7 +98,7 @@ def other_fftshift( flag = np.random.randint(low=0, high=4) xn = x.numpy() fft_yn = fft(xn - np.mean(xn)) - a = np.complex(0 + 0j) + a = np.complex128(0 + 0j) fft_yn_new = a * np.arange(x.shape[0]) prop = np.random.uniform(0.01, 0.25) From 578ac355bd2dfea7583ec42b3b0c7365ba96184e Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:22:55 +0200 Subject: [PATCH 02/11] refactor: bump dependencies for Python 3.12 --- .gitignore | 5 ++++- requirements-optional.txt | 21 +++++++++++++++++++++ requirements.txt | 36 ++++++++++++++++++------------------ 3 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 requirements-optional.txt diff --git a/.gitignore b/.gitignore index aba31a5..66630c2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ result/ .vscode/ .git *.DS_Store -venv/ \ No newline at end of file +venv/ +.venv/ +__pycache__/ +*.pyc \ No newline at end of file diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 0000000..ed79bcb --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1,21 @@ +# Legacy dependencies — NOT needed anymore for normal use. +# +# The TODS baselines (ts_benchmark/baselines/tods) are now implemented directly +# on top of pyod (see tods_models.py) and work on Python 3.12. The original +# implementation went through the abandoned d3m ecosystem, which is stuck on +# Python 3.8. The packages below are only required if you want to run the +# original d3m-based wrappers (ts_benchmark/baselines/tods/third_party) in a +# separate Python 3.8 environment, e.g. to reproduce historical results bit-for-bit: +# +# tamu_d3m==2022.05.23 +# nimfa==1.4.0 +# combo +# tensorflow +# +# Note: the only model that still requires this legacy stack is +# `tods.lstmodetectorski` (TensorFlow LSTM detector internal to TODS, +# no pyod equivalent). It is not used by any benchmark script. +# +# The Merlion baselines (salesforce-merlion) are installed from requirements.txt +# and work on Python 3.12. `merlion.RandomCutForest` additionally requires a +# Java runtime (Java 8+ is supported). diff --git a/requirements.txt b/requirements.txt index 05bf6cb..e1ab4ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,28 @@ +# Core dependencies — tested under Python 3.12 with a plain venv (.venv) matplotlib>=3.6.2 -numpy==1.21.0 -numba==0.55.2 -stumpy==1.4.0 +numpy>=1.26,<2.0 +numba>=0.59 +stumpy>=1.12 xgboost -pandas -scikit-learn -scipy +pandas>=2.0,<3.0 +scikit-learn>=1.3 +scipy>=1.11 statsmodels>=0.14.0 -ray>=2.6.3 +ray>=2.9 tqdm>=4.64.0 dash>=2.9.3 dash-bootstrap-components>=1.5.0 -reformer-pytorch==1.4.4 +reformer-pytorch>=1.4.4 lightgbm>=4.1.0 -tamu_d3m==2022.05.23 -nimfa==1.4.0 -PyWavelets>=1.1.1 -combo -tensorflow -torch>=1.11.0 -salesforce-merlion +PyWavelets>=1.4 +torch>=2.1 timm transformers peft -pytorch_lightning -tslearn -rotary_embedding_torch \ No newline at end of file +pytorch_lightning>=2.0 +tslearn>=0.6.3 +rotary_embedding_torch +einops +accelerate +pyod +salesforce-merlion From b656fa2c6deb311de5236697afb980356932ce64 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:23:20 +0200 Subject: [PATCH 03/11] refactor: TODS baselines on pyod (drop d3m), fix Merlion RCF on Java 8 --- .../baselines/merlion/merlion_models.py | 45 ++ ts_benchmark/baselines/tods/__init__.py | 5 + .../tods/pyod_core/CollectiveBase.py | 485 ++++++++++++++++++ ts_benchmark/baselines/tods/pyod_core/PCA.py | 273 ++++++++++ .../baselines/tods/pyod_core/__init__.py | 3 + .../baselines/tods/pyod_core/utility.py | 179 +++++++ ts_benchmark/baselines/tods/tods_models.py | 186 +++---- 7 files changed, 1083 insertions(+), 93 deletions(-) create mode 100644 ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py create mode 100644 ts_benchmark/baselines/tods/pyod_core/PCA.py create mode 100644 ts_benchmark/baselines/tods/pyod_core/__init__.py create mode 100644 ts_benchmark/baselines/tods/pyod_core/utility.py diff --git a/ts_benchmark/baselines/merlion/merlion_models.py b/ts_benchmark/baselines/merlion/merlion_models.py index 71b0b29..e197518 100644 --- a/ts_benchmark/baselines/merlion/merlion_models.py +++ b/ts_benchmark/baselines/merlion/merlion_models.py @@ -40,6 +40,51 @@ from sklearn.preprocessing import StandardScaler +def _install_java8_gateway_fallback() -> None: + """ + Merlion launches the RandomCutForest JVM with ``--add-opens`` options that only + exist on Java 9+; on Java 8 the JVM refuses to start and py4j fails with + ``ValueError: invalid literal for int()``. Those options are unnecessary on + Java 8 (no module system), so retry without them when the launch fails. + """ + from os.path import abspath, dirname, join + from os import pathsep + + from py4j.java_gateway import JavaGateway + import merlion.models.anomaly.random_cut_forest as rcf_module + + @classmethod + def gateway(cls): + if cls._gateway is None: + resource_dir = join( + dirname(dirname(dirname(abspath(rcf_module.__file__)))), "resources" + ) + jars = [ + "gson-2.8.9.jar", + "randomcutforest-core-1.0.jar", + "randomcutforest-serialization-json-1.0.jar", + ] + classpath = pathsep.join(join(resource_dir, jar) for jar in jars) + javaopts = [ + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + ] + try: + cls._gateway = JavaGateway.launch_gateway( + classpath=classpath, javaopts=javaopts + ) + except ValueError: + cls._gateway = JavaGateway.launch_gateway( + classpath=classpath, javaopts=[] + ) + return cls._gateway + + rcf_module.JVMSingleton.gateway = gateway + + +_install_java8_gateway_fallback() + + class MerlionModelAdapter: """ Merlion model adapter class, used to adapt models in the Merlion framework to meet the requirements of prediction strategies. diff --git a/ts_benchmark/baselines/tods/__init__.py b/ts_benchmark/baselines/tods/__init__.py index 641949b..73fb299 100644 --- a/ts_benchmark/baselines/tods/__init__.py +++ b/ts_benchmark/baselines/tods/__init__.py @@ -9,6 +9,8 @@ "pcaodetectorski", "isolationforestski", "cblofski", + "cofski", + "autoencoderski", ] from ts_benchmark.baselines.tods.tods_models import hbosski # noqa @@ -19,3 +21,6 @@ from ts_benchmark.baselines.tods.tods_models import pcaodetectorski # noqa from ts_benchmark.baselines.tods.tods_models import isolationforestski # noqa from ts_benchmark.baselines.tods.tods_models import cblofski # noqa +from ts_benchmark.baselines.tods.tods_models import cofski # noqa +from ts_benchmark.baselines.tods.tods_models import autoencoderski # noqa +from ts_benchmark.baselines.tods.tods_models import lstmodetectorski # noqa diff --git a/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py b/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py new file mode 100644 index 0000000..b2cf7f8 --- /dev/null +++ b/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py @@ -0,0 +1,485 @@ +# -*- coding: utf-8 -*- +"""Base class for all Collective outlier detector models +""" + +from __future__ import division +from __future__ import print_function + +import warnings +from collections import defaultdict + +from inspect import signature + +import abc +from abc import ABCMeta + +import numpy as np +from numpy import percentile +from scipy.special import erf +from sklearn.preprocessing import MinMaxScaler +from sklearn.utils import deprecated + +def check_is_fitted(estimator, attributes): + # sklearn>=1.6 requires estimators to implement __sklearn_tags__; + # these detectors are not sklearn estimators, so check attributes directly. + from sklearn.exceptions import NotFittedError + if not all(hasattr(estimator, attr) for attr in attributes): + raise NotFittedError( + f"This {type(estimator).__name__} instance is not fitted yet." + ) + +from sklearn.utils.multiclass import check_classification_targets + + +def _pprint(params, offset=0, printer=repr): # pragma: no cover + # noinspection PyPep8 + """Pretty print the dictionary 'params' + + See http://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html + and sklearn/base.py for more information. + + :param params: The dictionary to pretty print + :type params: dict + + :param offset: The offset in characters to add at the begin of each line. + :type offset: int + + :param printer: The function to convert entries to strings, typically + the builtin str or repr + :type printer: callable + + :return: None + """ + + # Do a multi-line justified repr: + options = np.get_printoptions() + np.set_printoptions(precision=5, threshold=64, edgeitems=2) + params_list = list() + this_line_length = offset + line_sep = ',\n' + (1 + offset // 2) * ' ' + for i, (k, v) in enumerate(sorted(params.items())): + if type(v) is float: + # use str for representing floating point numbers + # this way we get consistent representation across + # architectures and versions. + this_repr = '%s=%s' % (k, str(v)) + else: + # use repr of the rest + this_repr = '%s=%s' % (k, printer(v)) + if len(this_repr) > 500: + this_repr = this_repr[:300] + '...' + this_repr[-100:] + if i > 0: + if this_line_length + len(this_repr) >= 75 or '\n' in this_repr: + params_list.append(line_sep) + this_line_length = len(line_sep) + else: + params_list.append(', ') + this_line_length += 2 + params_list.append(this_repr) + this_line_length += len(this_repr) + + np.set_printoptions(**options) + lines = ''.join(params_list) + # Strip trailing space to avoid nightmare in doctests + lines = '\n'.join(l.rstrip(' ') for l in lines.split('\n')) + return lines + + +class CollectiveBaseDetector(metaclass=ABCMeta): + """Abstract class for all outlier detection algorithms. + + Parameters + ---------- + contamination : float in (0., 0.5), optional (default=0.1) + The amount of contamination of the data set, + i.e. the proportion of outliers in the data set. Used when fitting to + define the threshold on the decision function. + + window_size : int, optional (default=1) + The moving window size. + + step_size :, optional (default=1) + The displacement for moving window. + + Attributes + ---------- + decision_scores_ : numpy array of shape (n_samples,) + The outlier scores of the training data. + The higher, the more abnormal. Outliers tend to have higher + scores. This value is available once the detector is fitted. + + threshold_ : float + The threshold is based on ``contamination``. It is the + ``n_samples * contamination`` most abnormal samples in + ``decision_scores_``. The threshold is calculated for generating + binary outlier labels. + + labels_ : int, either 0 or 1 + The binary labels of the training data. 0 stands for inliers + and 1 for outliers/anomalies. It is generated by applying + ``threshold_`` on ``decision_scores_``. + """ + + @abc.abstractmethod + def __init__(self, contamination=0.1, + window_size=1, + step_size=1): # pragma: no cover + + if not (0. < contamination <= 0.5): + raise ValueError("contamination must be in (0, 0.5], " + "got: %f" % contamination) + + self.contamination = contamination + self.window_size = window_size + self.step_size = step_size + self._classes = 2 # leave the parameter on for extension + self.left_inds_ = None + self.right_inds = None + + # noinspection PyIncorrectDocstring + @abc.abstractmethod + def fit(self, X, y=None): # pragma: no cover + """Fit detector. y is ignored in unsupervised methods. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Fitted estimator. + """ + pass + + @abc.abstractmethod + def decision_function(self, X): # pragma: no cover + """Predict raw anomaly scores of X using the fitted detector. + + The anomaly score of an input sample is computed based on the fitted + detector. For consistency, outliers are assigned with + higher anomaly scores. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. Sparse matrices are accepted only + if they are supported by the base estimator. + + Returns + ------- + anomaly_scores : numpy array of shape (n_samples,) + The anomaly score of the input samples. + """ + pass + + @deprecated() + def fit_predict(self, X, y=None): # pragma: no cover + """Fit detector first and then predict whether a particular sample + is an outlier or not. y is ignored in unsupervised models. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + outlier_labels : numpy array of shape (n_samples,) + For each observation, tells whether or not + it should be considered as an outlier according to the + fitted model. 0 stands for inliers and 1 for outliers. + + .. deprecated:: 0.6.9 + `fit_predict` will be removed in pyod 0.8.0.; it will be + replaced by calling `fit` function first and then accessing + `labels_` attribute for consistency. + """ + + self.fit(X, y) + return self.labels_ + + def predict(self, X): # pragma: no cover + """Predict if a particular sample is an outlier or not. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + Returns + ------- + outlier_labels : numpy array of shape (n_samples,) + For each observation, tells whether or not + it should be considered as an outlier according to the + fitted model. 0 stands for inliers and 1 for outliers. + """ + + check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_']) + + pred_score, X_left_inds, X_right_inds = self.decision_function(X) + + return (pred_score > self.threshold_).astype( + 'int').ravel(), X_left_inds.ravel(), X_right_inds.ravel() + + def predict_proba(self, X, method='linear'): # pragma: no cover + """Predict the probability of a sample being outlier. Two approaches + are possible: + + 1. simply use Min-max conversion to linearly transform the outlier + scores into the range of [0,1]. The model must be + fitted first. + 2. use unifying scores, see :cite:`kriegel2011interpreting`. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + method : str, optional (default='linear') + probability conversion method. It must be one of + 'linear' or 'unify'. + + Returns + ------- + outlier_probability : numpy array of shape (n_samples,) + For each observation, tells whether or not + it should be considered as an outlier according to the + fitted model. Return the outlier probability, ranging + in [0,1]. + """ + + check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_']) + train_scores = self.decision_scores_ + + test_scores, X_left_inds, X_right_inds = self.decision_function(X) + + probs = np.zeros([test_scores.shape[0], int(self._classes)]) + if method == 'linear': + scaler = MinMaxScaler().fit(train_scores.reshape(-1, 1)) + probs[:, 1] = scaler.transform( + test_scores.reshape(-1, 1)).ravel().clip(0, 1) + probs[:, 0] = 1 - probs[:, 1] + return probs, X_left_inds.ravel(), X_right_inds.ravel() + + elif method == 'unify': + # turn output into probability + pre_erf_score = (test_scores - self._mu) / ( + self._sigma * np.sqrt(2)) + erf_score = erf(pre_erf_score) + probs[:, 1] = erf_score.clip(0, 1).ravel() + probs[:, 0] = 1 - probs[:, 1] + return probs, X_left_inds.ravel(), X_right_inds.ravel() + else: + raise ValueError(method, + 'is not a valid probability conversion method') + + def _predict_rank(self, X, normalized=False): # pragma: no cover + """Predict the outlyingness rank of a sample by a fitted model. The + method is for outlier detector score combination. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + normalized : bool, optional (default=False) + If set to True, all ranks are normalized to [0,1]. + + Returns + ------- + ranks : array, shape (n_samples,) + Outlying rank of a sample according to the training data. + + """ + + check_is_fitted(self, ['decision_scores_']) + + test_scores = self.decision_function(X) + train_scores = self.decision_scores_ + + sorted_train_scores = np.sort(train_scores) + ranks = np.searchsorted(sorted_train_scores, test_scores) + + if normalized: + # return normalized ranks + ranks = ranks / ranks.max() + return ranks + + def _set_n_classes(self, y): # pragma: no cover + """Set the number of classes if `y` is presented, which is not + expected. It could be useful for multi-class outlier detection. + + Parameters + ---------- + y : numpy array of shape (n_samples,) + Ground truth. + + Returns + ------- + self + """ + + self._classes = 2 # default as binary classification + if y is not None: + check_classification_targets(y) + self._classes = len(np.unique(y)) + warnings.warn( + "y should not be presented in unsupervised learning.") + return self + + def _process_decision_scores(self): # pragma: no cover + """Internal function to calculate key attributes: + + - threshold_: used to decide the binary label + - labels_: binary labels of training data + + Returns + ------- + self + """ + + self.threshold_ = percentile(self.decision_scores_, + 100 * (1 - self.contamination)) + self.labels_ = (self.decision_scores_ > self.threshold_).astype( + 'int').ravel() + + # calculate for predict_proba() + + self._mu = np.mean(self.decision_scores_) + self._sigma = np.std(self.decision_scores_) + + return self + + # noinspection PyMethodParameters + def _get_param_names(cls): # pragma: no cover + # noinspection PyPep8 + """Get parameter names for the estimator + + See http://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html + and sklearn/base.py for more information. + """ + + # fetch the constructor or the original constructor before + # deprecation wrapping if any + init = getattr(cls.__init__, 'deprecated_original', cls.__init__) + if init is object.__init__: + # No explicit constructor to introspect + return [] + + # introspect the constructor arguments to find the model parameters + # to represent + init_signature = signature(init) + # Consider the constructor parameters excluding 'self' + parameters = [p for p in init_signature.parameters.values() + if p.name != 'self' and p.kind != p.VAR_KEYWORD] + for p in parameters: + if p.kind == p.VAR_POSITIONAL: + raise RuntimeError("scikit-learn estimators should always " + "specify their parameters in the signature" + " of their __init__ (no varargs)." + " %s with constructor %s doesn't " + " follow this convention." + % (cls, init_signature)) + # Extract and sort argument names excluding 'self' + return sorted([p.name for p in parameters]) + + # noinspection PyPep8 + def get_params(self, deep=True): # pragma: no cover + """Get parameters for this estimator. + + See http://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html + and sklearn/base.py for more information. + + Parameters + ---------- + deep : bool, optional (default=True) + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : mapping of string to any + Parameter names mapped to their values. + """ + + out = dict() + for key in self._get_param_names(): + # We need deprecation warnings to always be on in order to + # catch deprecated param values. + # This is set in utils/__init__.py but it gets overwritten + # when running under python3 somehow. + warnings.simplefilter("always", DeprecationWarning) + try: + with warnings.catch_warnings(record=True) as w: + value = getattr(self, key, None) + if len(w) and w[0].category == DeprecationWarning: + # if the parameter is deprecated, don't show it + continue + finally: + warnings.filters.pop(0) + + # XXX: should we rather test if instance of estimator? + if deep and hasattr(value, 'get_params'): + deep_items = value.get_params().items() + out.update((key + '__' + k, val) for k, val in deep_items) + out[key] = value + return out + + def set_params(self, **params): # pragma: no cover + # noinspection PyPep8 + """Set the parameters of this estimator. + The method works on simple estimators as well as on nested objects + (such as pipelines). The latter have parameters of the form + ``__`` so that it's possible to update each + component of a nested object. + + See http://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html + and sklearn/base.py for more information. + + Returns + ------- + self : object + """ + + if not params: + # Simple optimization to gain speed (inspect is slow) + return self + valid_params = self.get_params(deep=True) + + nested_params = defaultdict(dict) # grouped by prefix + for key, value in params.items(): + key, delim, sub_key = key.partition('__') + if key not in valid_params: + raise ValueError('Invalid parameter %s for estimator %s. ' + 'Check the list of available parameters ' + 'with `estimator.get_params().keys()`.' % + (key, self)) + + if delim: + nested_params[key][sub_key] = value + else: + setattr(self, key, value) + + for key, sub_params in nested_params.items(): + valid_params[key].set_params(**sub_params) + + return self + + def __repr__(self): # pragma: no cover + # noinspection PyPep8 + """ + See http://scikit-learn.org/stable/modules/generated/sklearn.base.BaseEstimator.html + and sklearn/base.py for more information. + """ + + class_name = self.__class__.__name__ + return '%s(%s)' % (class_name, _pprint(self.get_params(deep=False), + offset=len(class_name), ),) diff --git a/ts_benchmark/baselines/tods/pyod_core/PCA.py b/ts_benchmark/baselines/tods/pyod_core/PCA.py new file mode 100644 index 0000000..5680378 --- /dev/null +++ b/ts_benchmark/baselines/tods/pyod_core/PCA.py @@ -0,0 +1,273 @@ +# -*- coding: utf-8 -*- +"""Autoregressive model for multivariate time series outlier detection. +""" +import numpy as np +from sklearn.utils import check_array + +def check_is_fitted(estimator, attributes): + # sklearn>=1.6 requires estimators to implement __sklearn_tags__; + # these detectors are not sklearn estimators, so check attributes directly. + from sklearn.exceptions import NotFittedError + if not all(hasattr(estimator, attr) for attr in attributes): + raise NotFittedError( + f"This {type(estimator).__name__} instance is not fitted yet." + ) + + +from .CollectiveBase import CollectiveBaseDetector +from pyod.models.pca import PCA as PCA_PYOD + +from .utility import get_sub_matrices + + +class PCA(CollectiveBaseDetector): + """PCA-based outlier detection with both univariate and multivariate + time series data. TS data will be first transformed to tabular format. + For univariate data, it will be in shape of [valid_length, window_size]. + for multivariate data with d sequences, it will be in the shape of + [valid_length, window_size]. + + Parameters + ---------- + window_size : int + The moving window size. + + step_size : int, optional (default=1) + The displacement for moving window. + + contamination : float in (0., 0.5), optional (default=0.1) + The amount of contamination of the data set, + i.e. the proportion of outliers in the data set. Used when fitting to + define the threshold on the decision function. + + n_components : int, float, None or string + Number of components to keep. It should be smaller than the window_size. + if n_components is not set all components are kept:: + + n_components == min(n_samples, n_features) + + if n_components == 'mle' and svd_solver == 'full', Minka\'s MLE is used + to guess the dimension + if ``0 < n_components < 1`` and svd_solver == 'full', select the number + of components such that the amount of variance that needs to be + explained is greater than the percentage specified by n_components + n_components cannot be equal to n_features for svd_solver == 'arpack'. + + n_selected_components : int, optional (default=None) + Number of selected principal components + for calculating the outlier scores. It is not necessarily equal to + the total number of the principal components. If not set, use + all principal components. + + copy : bool (default True) + If False, data passed to fit are overwritten and running + fit(X).transform(X) will not yield the expected results, + use fit_transform(X) instead. + + whiten : bool, optional (default False) + When True (False by default) the `components_` vectors are multiplied + by the square root of n_samples and then divided by the singular values + to ensure uncorrelated outputs with unit component-wise variances. + + Whitening will remove some information from the transformed signal + (the relative variance scales of the components) but can sometime + improve the predictive accuracy of the downstream estimators by + making their data respect some hard-wired assumptions. + + svd_solver : string {'auto', 'full', 'arpack', 'randomized'} + auto : + the solver is selected by a default policy based on `X.shape` and + `n_components`: if the input data is larger than 500x500 and the + number of components to extract is lower than 80% of the smallest + dimension of the data, then the more efficient 'randomized' + method is enabled. Otherwise the exact full SVD is computed and + optionally truncated afterwards. + full : + run exact full SVD calling the standard LAPACK solver via + `scipy.linalg.svd` and select the components by postprocessing + arpack : + run SVD truncated to n_components calling ARPACK solver via + `scipy.sparse.linalg.svds`. It requires strictly + 0 < n_components < X.shape[1] + randomized : + run randomized SVD by the method of Halko et al. + + tol : float >= 0, optional (default .0) + Tolerance for singular values computed by svd_solver == 'arpack'. + + iterated_power : int >= 0, or 'auto', (default 'auto') + Number of iterations for the power method computed by + svd_solver == 'randomized'. + + random_state : int, RandomState instance or None, optional (default None) + If int, random_state is the seed used by the random number generator; + If RandomState instance, random_state is the random number generator; + If None, the random number generator is the RandomState instance used + by `np.random`. Used when ``svd_solver`` == 'arpack' or 'randomized'. + + weighted : bool, optional (default=True) + If True, the eigenvalues are used in score computation. + The eigenvectors with small eigenvalues comes with more importance + in outlier score calculation. + + standardization : bool, optional (default=True) + If True, perform standardization first to convert + data to zero mean and unit variance. + See http://scikit-learn.org/stable/auto_examples/preprocessing/plot_scaling_importance.html + + Attributes + ---------- + decision_scores_ : numpy array of shape (n_samples,) + The outlier scores of the training data. + The higher, the more abnormal. Outliers tend to have higher + scores. This value is available once the detector is + fitted. + + threshold_ : float + The threshold is based on ``contamination``. It is the + ``n_samples * contamination`` most abnormal samples in + ``decision_scores_``. The threshold is calculated for generating + binary outlier labels. + + labels_ : int, either 0 or 1 + The binary labels of the training data. 0 stands for inliers + and 1 for outliers/anomalies. It is generated by applying + ``threshold_`` on ``decision_scores_``. + """ + + def __init__(self, window_size, step_size=1, contamination=0.1, + n_components=None, n_selected_components=None, + copy=True, whiten=False, svd_solver='auto', + tol=0.0, iterated_power='auto', random_state=None, + weighted=True, standardization=True): + super(PCA, self).__init__(contamination=contamination) + self.window_size = window_size + self.step_size = step_size + + # parameters for PCA + self.n_components = n_components + self.n_selected_components = n_selected_components + self.copy = copy + self.whiten = whiten + self.svd_solver = svd_solver + self.tol = tol + self.iterated_power = iterated_power + self.random_state = random_state + self.weighted = weighted + self.standardization = standardization + + # initialize a kNN model + self.model_ = PCA_PYOD(n_components=self.n_components, + n_selected_components=self.n_selected_components, + contamination=self.contamination, + copy=self.copy, + whiten=self.whiten, + svd_solver=self.svd_solver, + tol=self.tol, + iterated_power=self.iterated_power, + random_state=self.random_state, + weighted=self.weighted, + standardization=self.standardization) + + def fit(self, X: np.array) -> object: + """Fit detector. y is ignored in unsupervised methods. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + self : object + Fitted estimator. + """ + X = check_array(X).astype(float) + + # first convert it into submatrices, and flatten it + sub_matrices, self.left_inds_, self.right_inds_ = get_sub_matrices( + X, + self.window_size, + self.step_size, + return_numpy=True, + flatten=True, + flatten_order='F') + + # if self.n_components > sub_matrices.shape[1]: + # raise ValueError('n_components exceeds window_size times the number of sequences.') + + # fit the PCA model + self.model_.fit(sub_matrices) + self.decision_scores_ = self.model_.decision_scores_ + self._process_decision_scores() + return self + + def decision_function(self, X: np.array): + """Predict raw anomaly scores of X using the fitted detector. + + The anomaly score of an input sample is computed based on the fitted + detector. For consistency, outliers are assigned with + higher anomaly scores. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. Sparse matrices are accepted only + if they are supported by the base estimator. + + Returns + ------- + anomaly_scores : numpy array of shape (n_samples,) + The anomaly score of the input samples. + """ + check_is_fitted(self, ['model_']) + X = check_array(X).astype(float) + # first convert it into submatrices, and flatten it + sub_matrices, X_left_inds, X_right_inds = get_sub_matrices( + X, + self.window_size, + self.step_size, + return_numpy=True, + flatten=True, + flatten_order='F') + + # return the prediction result by PCA + return self.model_.decision_function( + sub_matrices), X_left_inds.ravel(), X_right_inds.ravel() + + +if __name__ == "__main__": # pragma: no cover + # X_train = np.asarray( + # [3., 4., 8., 16, 18, 13., 22., 36., 59., 128, 62, 67, 78, 100]).reshape(-1, 1) + + # X_test = np.asarray( + # [3., 4., 8.6, 13.4, 22.5, 17, 19.2, 36.1, 127, -23, 59.2]).reshape(-1, + # 1) + + X_train = np.asarray( + [[3., 5], [5., 9], [7., 2], [42., 20], [8., 12], [10., 12], + [12., 12], + [18., 16], [20., 7], [18., 10], [23., 12], [22., 15]]) + + w = get_sub_matrices(X_train, window_size=3, step=2, flatten=False) + X_test = np.asarray( + [[12., 10], [8., 12], [80., 80], [92., 983], + [18., 16], [20., 7], [18., 10], [3., 5], [5., 9], [23., 12], + [22., 15]]) + + clf = PCA(window_size=3, step_size=2, contamination=0.2) + + clf.fit(X_train) + decision_scores, left_inds_, right_inds = clf.decision_scores_, \ + clf.left_inds_, clf.right_inds_ + print(clf.left_inds_, clf.right_inds_) + pred_scores, X_left_inds, X_right_inds = clf.decision_function(X_test) + pred_labels, X_left_inds, X_right_inds = clf.predict(X_test) + pred_probs, X_left_inds, X_right_inds = clf.predict_proba(X_test) + + print(pred_scores) + print(pred_labels) + print(pred_probs) diff --git a/ts_benchmark/baselines/tods/pyod_core/__init__.py b/ts_benchmark/baselines/tods/pyod_core/__init__.py new file mode 100644 index 0000000..1b427d1 --- /dev/null +++ b/ts_benchmark/baselines/tods/pyod_core/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# Vendored from TODS (tods/detection_algorithm/core), Apache License 2.0. +# These modules are d3m-free and provide the sliding-window PCA detector. diff --git a/ts_benchmark/baselines/tods/pyod_core/utility.py b/ts_benchmark/baselines/tods/pyod_core/utility.py new file mode 100644 index 0000000..93f18de --- /dev/null +++ b/ts_benchmark/baselines/tods/pyod_core/utility.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""Utility functions for supporting time-series based outlier detection. +""" + +import numpy as np +from sklearn.utils import check_array + + +# def get_sub_sequences(X, window_size, step=1): +# """Chop a univariate time series into sub sequences. + +# Parameters +# ---------- +# X : numpy array of shape (n_samples,) +# The input samples. + +# window_size : int +# The moving window size. + +# step_size : int, optional (default=1) +# The displacement for moving window. + +# Returns +# ------- +# X_sub : numpy array of shape (valid_len, window_size) +# The numpy matrix with each row stands for a subsequence. +# """ +# X = check_array(X).astype(float) +# n_samples = len(X) + +# # get the valid length +# valid_len = get_sub_sequences_length(n_samples, window_size, step) + +# X_sub = np.zeros([valid_len, window_size]) +# # y_sub = np.zeros([valid_len, 1]) + +# # exclude the edge +# steps = list(range(0, n_samples, step)) +# steps = steps[:valid_len] + +# for idx, i in enumerate(steps): +# X_sub[idx,] = X[i: i + window_size].ravel() + +# return X_sub + +def get_sub_matrices(X, window_size, step=1, return_numpy=True, flatten=True, + flatten_order='F'): + """Chop a multivariate time series into sub sequences (matrices). + + Parameters + ---------- + X : numpy array of shape (n_samples,) + The input samples. + + window_size : int + The moving window size. + + step_size : int, optional (default=1) + The displacement for moving window. + + return_numpy : bool, optional (default=True) + If True, return the data format in 3d numpy array. + + flatten : bool, optional (default=True) + If True, flatten the returned array in 2d. + + flatten_order : str, optional (default='F') + Decide the order of the flatten for multivarite sequences. + ‘C’ means to flatten in row-major (C-style) order. + ‘F’ means to flatten in column-major (Fortran- style) order. + ‘A’ means to flatten in column-major order if a is Fortran contiguous in memory, + row-major order otherwise. ‘K’ means to flatten a in the order the elements occur in memory. + The default is ‘F’. + + Returns + ------- + X_sub : numpy array of shape (valid_len, window_size*n_sequences) + The numpy matrix with each row stands for a flattend submatrix. + """ + X = check_array(X).astype(float) + n_samples, n_sequences = X.shape[0], X.shape[1] + + # get the valid length + valid_len = get_sub_sequences_length(n_samples, window_size, step) + + X_sub = [] + X_left_inds = [] + X_right_inds = [] + + # exclude the edge + steps = list(range(0, n_samples, step)) + steps = steps[:valid_len] + + # print(n_samples, n_sequences) + for idx, i in enumerate(steps): + X_sub.append(X[i: i + window_size, :]) + X_left_inds.append(i) + X_right_inds.append(i + window_size) + + X_sub = np.asarray(X_sub) + + if return_numpy: + if flatten: + temp_array = np.zeros([valid_len, window_size * n_sequences]) + if flatten_order == 'C': + for i in range(valid_len): + temp_array[i, :] = X_sub[i, :, :].flatten(order='C') + + else: + for i in range(valid_len): + temp_array[i, :] = X_sub[i, :, :].flatten(order='F') + return temp_array, np.asarray(X_left_inds), np.asarray( + X_right_inds) + + else: + return np.asarray(X_sub), np.asarray(X_left_inds), np.asarray( + X_right_inds) + else: + return X_sub, np.asarray(X_left_inds), np.asarray(X_right_inds) + + +def get_sub_sequences_length(n_samples, window_size, step): + """Pseudo chop a univariate time series into sub sequences. Return valid + length only. + + Parameters + ---------- + X : numpy array of shape (n_samples,) + The input samples. + + window_size : int + The moving window size. + + step_size : int, optional (default=1) + The displacement for moving window. + + Returns + ------- + valid_len : int + The number of subsequences. + + """ + # if X.shape[0] == 1: + # n_samples = X.shape[1] + # elif X.shape[1] == 1: + # n_samples = X.shape[0] + # else: + # raise ValueError("X is not a univarite series. The shape is {shape}.".format(shape=X.shape)) + + # valid_len = n_samples - window_size + 1 + # valida_len = int_down(n_samples-window_size)/step + 1 + valid_len = int(np.floor((n_samples - window_size) / step)) + 1 + return valid_len + + +if __name__ == "__main__": + X_train = np.asarray( + [3., 4., 8., 16, 18, 13., 22., 36., 59., 128, 62, 67, 78, + 100]).reshape(-1, 1) + + X_train = np.asarray( + [[3., 5], [5., 9], [7., 2], [42., 20], [8., 12], [10., 12], [12., 12], + [18., 16], [20., 7], [18., 10], [23., 12], [22., 15]]) + + # n_samples = X.shape[0] + + window_size = 3 + + # valid_len = n_samples - window_size + 1 + + # X_sub = np.zeros([valid_len, window_size]) + + # for i in range(valid_len): + # X_sub[i, ] = X[i: i+window_size] + + # X_sub_2 = get_sub_sequences(X, window_size, step=2) + X_sub_3, X_left_inds, X_right_inds = get_sub_matrices(X_train, window_size, + step=2, + flatten_order='C') diff --git a/ts_benchmark/baselines/tods/tods_models.py b/ts_benchmark/baselines/tods/tods_models.py index 72d22ce..22c4168 100644 --- a/ts_benchmark/baselines/tods/tods_models.py +++ b/ts_benchmark/baselines/tods/tods_models.py @@ -1,75 +1,69 @@ -import os -import sys - -current_file_path = os.path.abspath(__file__) - -base_path = current_file_path[:current_file_path.rfind('ts_benchmark')] -sys.path.insert(0, base_path) -sys.path.insert(0, base_path + "ts_benchmark/baselines/tods/third_party") - +# -*- coding: utf-8 -*- +""" +TODS-compatible baselines implemented directly on top of pyod. + +The original implementation wrapped these detectors through the TODS / d3m +primitive stack, which is abandoned and only compatible with Python 3.8. +Every detector exposed here was already a thin wrapper around a pyod model, +so the wrappers below call pyod directly while keeping the same model names, +default hyperparameters and output conventions. Existing configs and scripts +(e.g. ``--model-name "tods.lofski"``) keep working unchanged. + +Note: ``lstmodetectorski`` is unavailable in this implementation — the LSTM +outlier detector was a TensorFlow model internal to TODS with no pyod +equivalent. Use the deep-learning baselines of the benchmark instead. +""" import logging + import numpy as np import pandas as pd -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.IsolationForest_skinterface import ( - IsolationForestSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.LSTMODetector_skinterface import ( - LSTMODetectorSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.KNN_skinterface import ( - KNNSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.AutoEncoder_skinterface import ( - AutoEncoderSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.LOF_skinterface import ( - LOFSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.OCSVM_skinterface import ( - OCSVMSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.HBOS_skinterface import ( - HBOSSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.LODA_skinterface import ( - LODASKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.PCAODetector_skinterface import ( - PCAODetectorSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.COF_skinterface import ( - COFSKI, -) -from ts_benchmark.baselines.tods.third_party.tods.sk_interface.detection_algorithm.CBLOF_skinterface import ( - CBLOFSKI, -) +from pyod.models.auto_encoder import AutoEncoder +from pyod.models.cblof import CBLOF +from pyod.models.cof import COF +from pyod.models.hbos import HBOS +from pyod.models.iforest import IForest +from pyod.models.knn import KNN +from pyod.models.loda import LODA +from pyod.models.lof import LOF +from pyod.models.ocsvm import OCSVM +from ts_benchmark.baselines.tods.pyod_core.PCA import PCA as _WindowPCA +logger = logging.getLogger(__name__) -TODS_MODELS = [ - [IsolationForestSKI, {}], - [LSTMODetectorSKI, {}], - [KNNSKI, {}], - [AutoEncoderSKI, {}], - [LOFSKI, {}], - [OCSVMSKI, {}], - [HBOSSKI, {}], - [LODASKI, {}], - [PCAODetectorSKI, {}], - - [COFSKI, {}], - [CBLOFSKI, {}], -] +class PCAODetector(_WindowPCA): + """ + Sliding-window PCA detector. + Same algorithm and defaults as TODS' ``PCAODetectorSKI`` (window_size=10, + step_size=1): the series is unfolded into overlapping windows and pyod's + PCA detector is applied to the resulting matrix. + """ -logger = logging.getLogger(__name__) + def __init__(self, window_size: int = 10, **kwargs): + super().__init__(window_size=window_size, **kwargs) + + +# [exported name (kept from the original TODS wrappers), model class, required params] +TODS_MODELS = [ + ["IsolationForestSKI", IForest, {}], + ["KNNSKI", KNN, {}], + ["AutoEncoderSKI", AutoEncoder, {}], + ["LOFSKI", LOF, {}], + ["OCSVMSKI", OCSVM, {}], + ["HBOSSKI", HBOS, {}], + ["LODASKI", LODA, {}], + ["PCAODetectorSKI", PCAODetector, {}], + ["COFSKI", COF, {}], + ["CBLOFSKI", CBLOF, {}], +] class TodsModelAdapter: """ - The Tods model adapter class is used to adapt models in the Tods framework to meet the requirements of prediction strategies. + Adapts pyod detection models to meet the requirements of prediction strategies. """ def __init__( @@ -79,10 +73,10 @@ def __init__( model_args: dict, ): """ - Initialize the Tods model adapter object. + Initialize the model adapter object. :param model_name: Model name. - :param model_class: Tods model class. + :param model_class: pyod model class. :param model_args: Model initialization parameters. """ self.model = None @@ -92,13 +86,12 @@ def __init__( def detect_fit(self, series: pd.DataFrame, label: pd.DataFrame) -> object: """ - Fit a suitable Tods model on time series data. + Fit a suitable pyod model on time series data. :param series: Time series data. - :param label: Label data. + :param label: Label data (ignored, unsupervised models). :return: The fitted model object. """ - self.model = self.model_class(**self.model_args) X = series.values self.model.fit(X) @@ -107,25 +100,33 @@ def detect_fit(self, series: pd.DataFrame, label: pd.DataFrame) -> object: def detect_score(self, train: pd.DataFrame) -> np.ndarray: """ - Calculate anomaly scores using an adapted Tods model. + Calculate anomaly scores using the fitted pyod model. - :param train: Training data used to calculate scores. + :param train: Data used to calculate scores. :return: Anomaly score array. """ X = train.values - prediction_score = self.model.predict_score(X).reshape(-1) + prediction_score = self.model.decision_function(X) + if isinstance(prediction_score, tuple): + # collective (window-based) detectors return (scores, left_inds, right_inds) + prediction_score = prediction_score[0] + prediction_score = np.asarray(prediction_score).reshape(-1) return prediction_score, prediction_score def detect_label(self, train: pd.DataFrame) -> np.ndarray: """ - Use an adapted Tods model for anomaly detection and generate labels. + Use the fitted pyod model for anomaly detection and generate labels. - :param train: Training data used for anomaly detection. + :param train: Data used for anomaly detection. :return: Anomaly label array. """ X = train.values - prediction_labels = self.model.predict(X).reshape(-1) + prediction_labels = self.model.predict(X) + if isinstance(prediction_labels, tuple): + # collective (window-based) detectors return (labels, left_inds, right_inds) + prediction_labels = prediction_labels[0] + prediction_labels = np.asarray(prediction_labels).reshape(-1) return prediction_labels, prediction_labels @@ -142,19 +143,20 @@ def generate_model_factory( required_args: dict, ) -> object: """ - Generate model factory information for creating Tods model adapters. + Generate model factory information for creating model adapters. :param model_name: Model name. - :param model_class: Tods model class. + :param model_class: pyod model class. :param required_args: Required parameters for model initialization. :return: A dictionary containing the model factory and required parameters. """ def model_factory(**kwargs) -> object: """ - Model factory, used to create Tods model adapter objects. + Model factory, used to create model adapter objects. + :param kwargs: Model initialization parameters. - :return: Tods model adapter object. + :return: Model adapter object. """ return TodsModelAdapter( model_name, @@ -165,25 +167,23 @@ def model_factory(**kwargs) -> object: return {"model_factory": model_factory, "required_hyper_params": required_args} -# Generate model factories for each model class and required parameters in TODS-MODELS and add them to global variables -for model_class, required_args in TODS_MODELS: - globals()[f"{model_class.__name__.lower()}"] = generate_model_factory( - model_class.__name__, model_class, required_args +# Generate model factories for each model class and required parameters in TODS_MODELS +# and add them to global variables under their historical names (e.g. "lofski") +for model_name, model_class, required_args in TODS_MODELS: + globals()[model_name.lower()] = generate_model_factory( + model_name, model_class, required_args ) -# TODO tods adapter -# def deep_tods_model_adapter(model_info: Type[object]) -> object: -# """ -# 适配深度 Tods 模型。 - -# :param model_info: 要适配的深度 Tods 模型类。必须是一个类或类型对象。 -# :return: 生成的模型工厂，用于创建适配的 Tods 模型。 -# """ -# if not isinstance(model_info, type): -# raise ValueError() - -# return generate_model_factory( -# model_info.__name__, -# model_info, -# allow_fit_on_eval=False, -# ) + +def _lstmodetector_factory(**kwargs): + raise NotImplementedError( + "lstmodetectorski is not available: the TODS LSTM outlier detector relied on " + "the abandoned d3m/TensorFlow stack (Python 3.8 only) and has no pyod " + "equivalent. Use the deep-learning baselines of the benchmark instead." + ) + + +lstmodetectorski = { + "model_factory": _lstmodetector_factory, + "required_hyper_params": {}, +} From f1cd0436902f67ae1cc01048805ae963f59e8f51 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:23:50 +0200 Subject: [PATCH 04/11] docs: update readme --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 09eeee9..189c431 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TAB: Unified Benchmarking of Time Series Anomaly Detection Methods -[![PVLDB](https://img.shields.io/badge/PVLDB'25-TAB-orange)](https://arxiv.org/pdf/2403.20150.pdf) [![Python](https://img.shields.io/badge/Python-3.8%2B-blue)](https://www.python.org/) [![PyTorch](https://img.shields.io/badge/PyTorch-2.4.1-blue)](https://pytorch.org/) ![Stars](https://img.shields.io/github/stars/decisionintelligence/TAB) +[![PVLDB](https://img.shields.io/badge/PVLDB'25-TAB-orange)](https://arxiv.org/pdf/2403.20150.pdf) [![Python](https://img.shields.io/badge/Python-3.12-blue)](https://www.python.org/) [![PyTorch](https://img.shields.io/badge/PyTorch-2.4.1-blue)](https://pytorch.org/) ![Stars](https://img.shields.io/github/stars/decisionintelligence/TAB) > [!IMPORTANT] @@ -58,16 +58,25 @@ The table below provides a visual overview of how TAB's key features compare to > [!IMPORTANT] > -> this project is fully tested under python 3.8, it is recommended that you set the Python version to 3.8. +> this project supports **Python 3.12**. A plain virtual environment (`.venv`) is all you need — no conda required. 1. Installation: -Given a python environment (**note**: this project is fully tested under **python 3.8**), install the dependencies with the following command: +Create a virtual environment and install the dependencies: ```shell +python3.12 -m venv .venv +source .venv/bin/activate pip install -r requirements.txt ``` +> [!NOTE] +> +> All baseline families work on Python 3.12, including TODS and Merlion: +> +> - The TODS baselines (`tods.hbosski`, `tods.lofski`, ...) are implemented directly on top of `pyod` — same algorithms, same defaults, no dependency on the abandoned `d3m` ecosystem. Results are statistically equivalent to (but may differ in the last decimals from) the historical d3m-based implementation, which is kept under `ts_benchmark/baselines/tods/third_party` for Python 3.8 reproduction (see `requirements-optional.txt`). The only unavailable model is `tods.lstmodetectorski` (TensorFlow model internal to TODS, unused by the benchmark scripts). +> - The Merlion baselines (`merlion.IsolationForest`, ...) use `salesforce-merlion`, installed from `requirements.txt`. `merlion.RandomCutForest` additionally requires a Java runtime (Java 8+). + 2. Data preparation Prepare Data. You can obtain the well pre-processed datasets from [Google Drive](https://drive.google.com/file/d/1V5BAHWBKU8uih3hE1R7WdF6_crZlIbQT/view?usp=drive_link). Then place the downloaded data under the folder `./dataset`. From 3d2386bffa80b08b9109e57feb0c844b3b661d36 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:14:07 +0200 Subject: [PATCH 05/11] chore: ignore llm checkpoints --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 66630c2..8351bfd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ dataset/ result/ +ts_benchmark/baselines/LLM/checkpoints/ +ts_benchmark/baselines/pre_train/checkpoints/ .idea/ .vscode/ .git From 31badc8ae7a6ce98fe5cb78a06021fe678d65f47 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:11:54 +0200 Subject: [PATCH 06/11] fix(tods): inject historical d3m default hyperparameters, translate legacy AutoEncoder param names --- ts_benchmark/baselines/tods/tods_models.py | 123 ++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/ts_benchmark/baselines/tods/tods_models.py b/ts_benchmark/baselines/tods/tods_models.py index 22c4168..653d10e 100644 --- a/ts_benchmark/baselines/tods/tods_models.py +++ b/ts_benchmark/baselines/tods/tods_models.py @@ -46,6 +46,123 @@ def __init__(self, window_size: int = 10, **kwargs): super().__init__(window_size=window_size, **kwargs) +# Default hyperparameters of the historical d3m primitives (tods/detection_algorithm/*.py, +# `Hyperparams` classes). They are injected explicitly because some of them differ from +# the defaults of current pyod versions (e.g. HBOS tol, COF n_neighbors, PCA whiten). +# User-supplied hyperparameters override these values. +_LEGACY_DEFAULTS = { + "IsolationForestSKI": { + "n_estimators": 100, + "max_samples": "auto", + "max_features": 1.0, + "bootstrap": False, + "behaviour": "new", + "contamination": 0.1, + }, + "KNNSKI": { + "n_neighbors": 5, + "method": "largest", + "radius": 1.0, + "algorithm": "auto", + "leaf_size": 30, + "metric": "minkowski", + "p": 2, + "contamination": 0.1, + }, + "AutoEncoderSKI": { + # translated to the current pyod (torch-based) AutoEncoder API + "hidden_neuron_list": [1, 4, 1], + "epoch_num": 20, + "batch_size": 32, + "dropout_rate": 0.2, + "preprocessing": True, + "optimizer_params": {"weight_decay": 0.1}, + "contamination": 0.01, + }, + "LOFSKI": { + "n_neighbors": 20, + "leaf_size": 30, + "metric": "minkowski", + "p": 2, + "contamination": 0.1, + }, + "OCSVMSKI": { + "kernel": "rbf", + "nu": 0.5, + "degree": 3, + "gamma": "auto", + "coef0": 0.0, + "shrinking": True, + "contamination": 0.1, + }, + "HBOSSKI": { + "n_bins": 10, + "alpha": 0.1, + "tol": 0.1, + "contamination": 0.1, + }, + "LODASKI": { + "n_bins": 10, + "n_random_cuts": 100, + "contamination": 0.1, + }, + "PCAODetectorSKI": { + "window_size": 10, + "step_size": 1, + "n_components": 1, + "whiten": True, + "standardization": True, + "svd_solver": "auto", + "contamination": 0.1, + }, + "COFSKI": { + "n_neighbors": 5, + "contamination": 0.1, + }, + "CBLOFSKI": { + "n_clusters": 8, + "alpha": 0.9, + "beta": 5, + "use_weights": False, + "contamination": 0.1, + }, +} + +# Historical AutoEncoder hyperparameter names (keras-based pyod) translated to the +# current torch-based pyod API, so configs written for the d3m wrappers keep working. +_AE_LEGACY_PARAM_MAP = { + "epochs": "epoch_num", + "hidden_neurons": "hidden_neuron_list", + "hidden_activation": "hidden_activation_name", + "l2_regularizer": None, # handled below via optimizer_params + "validation_size": None, # no equivalent in the torch implementation + "output_activation": None, + "loss": None, + "optimizer": None, + "verbose": "verbose", +} + + +def _translate_ae_params(params: dict) -> dict: + translated = {} + for key, value in params.items(): + if key == "l2_regularizer": + translated["optimizer_params"] = {"weight_decay": value} + elif key in _AE_LEGACY_PARAM_MAP: + new_key = _AE_LEGACY_PARAM_MAP[key] + if new_key is None: + logger.warning( + "AutoEncoder hyperparameter %r has no equivalent in the current " + "pyod implementation and is ignored.", + key, + ) + else: + translated[new_key] = value + else: + translated[key] = value + return translated + + # [exported name (kept from the original TODS wrappers), model class, required params] TODS_MODELS = [ ["IsolationForestSKI", IForest, {}], @@ -92,7 +209,11 @@ def detect_fit(self, series: pd.DataFrame, label: pd.DataFrame) -> object: :param label: Label data (ignored, unsupervised models). :return: The fitted model object. """ - self.model = self.model_class(**self.model_args) + user_args = self.model_args + if self.model_name == "AutoEncoderSKI": + user_args = _translate_ae_params(user_args) + args = {**_LEGACY_DEFAULTS.get(self.model_name, {}), **user_args} + self.model = self.model_class(**args) X = series.values self.model.fit(X) From efc07de80e385dd7ab5ad80e626d800d0ba19ddf Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:12:00 +0200 Subject: [PATCH 07/11] fix(LOF): keep one score per timestamp for multivariate input --- ts_benchmark/baselines/self_impl/LOF/lof.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ts_benchmark/baselines/self_impl/LOF/lof.py b/ts_benchmark/baselines/self_impl/LOF/lof.py index db8ac19..2ddfb2f 100644 --- a/ts_benchmark/baselines/self_impl/LOF/lof.py +++ b/ts_benchmark/baselines/self_impl/LOF/lof.py @@ -72,7 +72,8 @@ def detect_score(self, X: pd.DataFrame) -> np.ndarray: :param X: The data of the score to be calculated. :return: Anomaly score array. """ - X = X.values.reshape(-1, 1) + # keep one sample per timestamp: (T, D) with channels as features + X = X.values self.detector_ = LocalOutlierFactor( n_neighbors=self.n_neighbors, @@ -102,7 +103,8 @@ def detect_label(self, X: pd.DataFrame) -> np.ndarray: :param X: The data to be tested. :return: Anomaly label array. """ - X = X.values.reshape(-1, 1) + # keep one sample per timestamp: (T, D) with channels as features + X = X.values self.detector_ = LocalOutlierFactor( n_neighbors=self.n_neighbors, From 71a1427669fcd58ef17bd508853db65341b82fea Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:12:07 +0200 Subject: [PATCH 08/11] build: make requirements-optional.txt installable (Python 3.8 legacy stack) --- requirements-optional.txt | 40 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/requirements-optional.txt b/requirements-optional.txt index ed79bcb..940e0b2 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,21 +1,27 @@ -# Legacy dependencies — NOT needed anymore for normal use. +# Legacy d3m stack for the original TODS wrappers (ts_benchmark/baselines/tods/third_party). # -# The TODS baselines (ts_benchmark/baselines/tods) are now implemented directly -# on top of pyod (see tods_models.py) and work on Python 3.12. The original -# implementation went through the abandoned d3m ecosystem, which is stuck on -# Python 3.8. The packages below are only required if you want to run the -# original d3m-based wrappers (ts_benchmark/baselines/tods/third_party) in a -# separate Python 3.8 environment, e.g. to reproduce historical results bit-for-bit: +# WARNING: Python 3.8 ONLY. Do NOT install this file into the Python 3.12 environment — +# these packages are incompatible with it and are NOT needed for normal use: the TODS +# baselines are implemented directly on top of pyod (see tods_models.py) and work on +# Python 3.12 out of the box. # -# tamu_d3m==2022.05.23 -# nimfa==1.4.0 -# combo -# tensorflow +# Install this file in a separate Python 3.8 virtual environment only if you need to +# run the original d3m-based wrappers, e.g. to reproduce historical results bit-for-bit: +# python3.8 -m venv .venv38 && .venv38/bin/pip install -r requirements-optional.txt # -# Note: the only model that still requires this legacy stack is -# `tods.lstmodetectorski` (TensorFlow LSTM detector internal to TODS, -# no pyod equivalent). It is not used by any benchmark script. +# Note: the only model that strictly requires this legacy stack is `tods.lstmodetectorski` +# (TensorFlow LSTM detector internal to TODS, no pyod equivalent, unused by the +# benchmark scripts). # -# The Merlion baselines (salesforce-merlion) are installed from requirements.txt -# and work on Python 3.12. `merlion.RandomCutForest` additionally requires a -# Java runtime (Java 8+ is supported). +# `merlion.RandomCutForest` does not need anything from this file, but requires a Java +# runtime (Java 8+ is supported). + +tamu_d3m==2022.05.23 +nimfa==1.4.0 +combo +tensorflow +numpy==1.21.0 +numba==0.55.2 +scikit-learn +pandas +pyod From 4f02cb33a7427f20ca4570d3ab8b5b8a5fc2e098 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:12:15 +0200 Subject: [PATCH 09/11] docs: clarify TODS equivalence claims in README, strip trailing whitespace --- README.md | 4 ++-- .../baselines/tods/pyod_core/CollectiveBase.py | 2 +- ts_benchmark/baselines/tods/pyod_core/PCA.py | 6 +++--- ts_benchmark/baselines/tods/pyod_core/utility.py | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 189c431..f6aef34 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TAB: Unified Benchmarking of Time Series Anomaly Detection Methods -[![PVLDB](https://img.shields.io/badge/PVLDB'25-TAB-orange)](https://arxiv.org/pdf/2403.20150.pdf) [![Python](https://img.shields.io/badge/Python-3.12-blue)](https://www.python.org/) [![PyTorch](https://img.shields.io/badge/PyTorch-2.4.1-blue)](https://pytorch.org/) ![Stars](https://img.shields.io/github/stars/decisionintelligence/TAB) +[![PVLDB](https://img.shields.io/badge/PVLDB'25-TAB-orange)](https://arxiv.org/pdf/2403.20150.pdf) [![Python](https://img.shields.io/badge/Python-3.12-blue)](https://www.python.org/) [![PyTorch](https://img.shields.io/badge/PyTorch-2.4.1-blue)](https://pytorch.org/) ![Stars](https://img.shields.io/github/stars/decisionintelligence/TAB) > [!IMPORTANT] @@ -74,7 +74,7 @@ pip install -r requirements.txt > > All baseline families work on Python 3.12, including TODS and Merlion: > -> - The TODS baselines (`tods.hbosski`, `tods.lofski`, ...) are implemented directly on top of `pyod` — same algorithms, same defaults, no dependency on the abandoned `d3m` ecosystem. Results are statistically equivalent to (but may differ in the last decimals from) the historical d3m-based implementation, which is kept under `ts_benchmark/baselines/tods/third_party` for Python 3.8 reproduction (see `requirements-optional.txt`). The only unavailable model is `tods.lstmodetectorski` (TensorFlow model internal to TODS, unused by the benchmark scripts). +> - The TODS baselines (`tods.hbosski`, `tods.lofski`, ...) are implemented directly on top of `pyod` — same algorithms, with the default hyperparameters of the historical d3m primitives injected explicitly, and no dependency on the abandoned `d3m` ecosystem. Because the underlying pyod version is newer than the one frozen by d3m in 2022, individual scores may differ slightly from historical runs (validated: 7 of 9 finite AUC-ROC values reproduced within 0.001 of published results on real datasets, remaining deviations shown to pre-date this implementation). The original d3m-based implementation is kept under `ts_benchmark/baselines/tods/third_party` for Python 3.8 reproduction (see `requirements-optional.txt`). The only unavailable model is `tods.lstmodetectorski` (TensorFlow model internal to TODS, unused by the benchmark scripts). > - The Merlion baselines (`merlion.IsolationForest`, ...) use `salesforce-merlion`, installed from `requirements.txt`. `merlion.RandomCutForest` additionally requires a Java runtime (Java 8+). 2. Data preparation diff --git a/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py b/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py index b2cf7f8..302423d 100644 --- a/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py +++ b/ts_benchmark/baselines/tods/pyod_core/CollectiveBase.py @@ -258,7 +258,7 @@ def predict_proba(self, X, method='linear'): # pragma: no cover check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_']) train_scores = self.decision_scores_ - + test_scores, X_left_inds, X_right_inds = self.decision_function(X) probs = np.zeros([test_scores.shape[0], int(self._classes)]) diff --git a/ts_benchmark/baselines/tods/pyod_core/PCA.py b/ts_benchmark/baselines/tods/pyod_core/PCA.py index 5680378..06a591b 100644 --- a/ts_benchmark/baselines/tods/pyod_core/PCA.py +++ b/ts_benchmark/baselines/tods/pyod_core/PCA.py @@ -22,9 +22,9 @@ def check_is_fitted(estimator, attributes): class PCA(CollectiveBaseDetector): """PCA-based outlier detection with both univariate and multivariate - time series data. TS data will be first transformed to tabular format. + time series data. TS data will be first transformed to tabular format. For univariate data, it will be in shape of [valid_length, window_size]. - for multivariate data with d sequences, it will be in the shape of + for multivariate data with d sequences, it will be in the shape of [valid_length, window_size]. Parameters @@ -114,7 +114,7 @@ class PCA(CollectiveBaseDetector): If True, perform standardization first to convert data to zero mean and unit variance. See http://scikit-learn.org/stable/auto_examples/preprocessing/plot_scaling_importance.html - + Attributes ---------- decision_scores_ : numpy array of shape (n_samples,) diff --git a/ts_benchmark/baselines/tods/pyod_core/utility.py b/ts_benchmark/baselines/tods/pyod_core/utility.py index 93f18de..f30cb72 100644 --- a/ts_benchmark/baselines/tods/pyod_core/utility.py +++ b/ts_benchmark/baselines/tods/pyod_core/utility.py @@ -57,19 +57,19 @@ def get_sub_matrices(X, window_size, step=1, return_numpy=True, flatten=True, step_size : int, optional (default=1) The displacement for moving window. - + return_numpy : bool, optional (default=True) If True, return the data format in 3d numpy array. flatten : bool, optional (default=True) If True, flatten the returned array in 2d. - + flatten_order : str, optional (default='F') Decide the order of the flatten for multivarite sequences. - ‘C’ means to flatten in row-major (C-style) order. - ‘F’ means to flatten in column-major (Fortran- style) order. - ‘A’ means to flatten in column-major order if a is Fortran contiguous in memory, - row-major order otherwise. ‘K’ means to flatten a in the order the elements occur in memory. + ‘C’ means to flatten in row-major (C-style) order. + ‘F’ means to flatten in column-major (Fortran- style) order. + ‘A’ means to flatten in column-major order if a is Fortran contiguous in memory, + row-major order otherwise. ‘K’ means to flatten a in the order the elements occur in memory. The default is ‘F’. Returns @@ -138,7 +138,7 @@ def get_sub_sequences_length(n_samples, window_size, step): ------- valid_len : int The number of subsequences. - + """ # if X.shape[0] == 1: # n_samples = X.shape[1] @@ -148,7 +148,7 @@ def get_sub_sequences_length(n_samples, window_size, step): # raise ValueError("X is not a univarite series. The shape is {shape}.".format(shape=X.shape)) # valid_len = n_samples - window_size + 1 - # valida_len = int_down(n_samples-window_size)/step + 1 + # valida_len = int_down(n_samples-window_size)/step + 1 valid_len = int(np.floor((n_samples - window_size) / step)) + 1 return valid_len From 44a1b33d135af97031643645656fb713ca6a255c Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:45:56 +0200 Subject: [PATCH 10/11] fix: load distributed checkpoints with weights_only=False (torch>=2.6) --- ts_benchmark/baselines/LLM/submodules/CALF/CALF.py | 2 +- ts_benchmark/baselines/pre_train/model/units.py | 2 +- .../baselines/pre_train/submodules/Timer/models/Timer.py | 4 ++-- .../baselines/pre_train/submodules/Timer/models/TrmEncoder.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ts_benchmark/baselines/LLM/submodules/CALF/CALF.py b/ts_benchmark/baselines/LLM/submodules/CALF/CALF.py index 82c46c2..2d08ccf 100644 --- a/ts_benchmark/baselines/LLM/submodules/CALF/CALF.py +++ b/ts_benchmark/baselines/LLM/submodules/CALF/CALF.py @@ -60,7 +60,7 @@ def __init__(self, configs, device): self.gpt2_text.h = self.gpt2_text.h[:configs.gpt_layers] self.gpt2 = get_peft_model(self.gpt2, peft_config) - word_embedding = torch.tensor(torch.load(configs.word_embedding_path)).to(device=device) + word_embedding = torch.tensor(torch.load(configs.word_embedding_path, weights_only=False)).to(device=device) for i, (name, param) in enumerate(self.gpt2.named_parameters()): if 'ln' in name or 'wpe' in name or 'lora' in name: diff --git a/ts_benchmark/baselines/pre_train/model/units.py b/ts_benchmark/baselines/pre_train/model/units.py index 005b63a..032d0cd 100644 --- a/ts_benchmark/baselines/pre_train/model/units.py +++ b/ts_benchmark/baselines/pre_train/model/units.py @@ -971,7 +971,7 @@ def __init__( pretrain_weight_path = "ts_benchmark/baselines/pre_train/checkpoints/units/units_x32_pretrain_checkpoint.pth" device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - state_dict = torch.load(pretrain_weight_path, map_location=device)['student'] + state_dict = torch.load(pretrain_weight_path, map_location=device, weights_only=False)['student'] ckpt = {} for k, v in state_dict.items(): if not ('cls_prompts' in k): diff --git a/ts_benchmark/baselines/pre_train/submodules/Timer/models/Timer.py b/ts_benchmark/baselines/pre_train/submodules/Timer/models/Timer.py index 97fc953..a060b77 100644 --- a/ts_benchmark/baselines/pre_train/submodules/Timer/models/Timer.py +++ b/ts_benchmark/baselines/pre_train/submodules/Timer/models/Timer.py @@ -32,9 +32,9 @@ def __init__(self, configs): else: print('loading model: ', self.ckpt_path) if self.ckpt_path.endswith('.pth'): - self.backbone.load_state_dict(torch.load(self.ckpt_path)) + self.backbone.load_state_dict(torch.load(self.ckpt_path, weights_only=False)) elif self.ckpt_path.endswith('.ckpt'): - sd = torch.load(self.ckpt_path, map_location="cpu")["state_dict"] + sd = torch.load(self.ckpt_path, map_location="cpu", weights_only=False)["state_dict"] sd = {k[6:]: v for k, v in sd.items()} self.backbone.load_state_dict(sd, strict=True) diff --git a/ts_benchmark/baselines/pre_train/submodules/Timer/models/TrmEncoder.py b/ts_benchmark/baselines/pre_train/submodules/Timer/models/TrmEncoder.py index de43970..8eab42f 100644 --- a/ts_benchmark/baselines/pre_train/submodules/Timer/models/TrmEncoder.py +++ b/ts_benchmark/baselines/pre_train/submodules/Timer/models/TrmEncoder.py @@ -48,9 +48,9 @@ def __init__(self, configs): else: print('loading model: ', self.ckpt_path) if self.ckpt_path.endswith('.pth'): - self.backbone.load_state_dict(torch.load(self.ckpt_path)) + self.backbone.load_state_dict(torch.load(self.ckpt_path, weights_only=False)) elif self.ckpt_path.endswith('.ckpt'): - sd = torch.load(self.ckpt_path, map_location="cpu")["state_dict"] + sd = torch.load(self.ckpt_path, map_location="cpu", weights_only=False)["state_dict"] sd = {k[6:]: v for k, v in sd.items()} self.backbone.load_state_dict(sd, strict=True) From 696aa87ecb71a1375c45b883c28a789207bb2125 Mon Sep 17 00:00:00 2001 From: Marc Pinet <52708150+marcpinet@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:46:04 +0200 Subject: [PATCH 11/11] build: pin transformers<4.50 for vendored LLM baseline code --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1ab4ed..1055cb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,9 @@ lightgbm>=4.1.0 PyWavelets>=1.4 torch>=2.1 timm -transformers +# LLM baselines (CALF, GPT4TS, ...) vendor GPT-2 forward code written against the +# transformers 4.x API (GenerationMixin on base models, get_head_mask, ...) +transformers>=4.40,<4.50 peft pytorch_lightning>=2.0 tslearn>=0.6.3