From a28113b80be6d690bc59a07b9f83de6d0ac9952e Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Fri, 6 Mar 2026 14:28:24 +0100 Subject: [PATCH 1/8] Move gitignore to root folder and add docs related ignores --- .gitignore | 7 +++++-- docs/.gitignore | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 docs/.gitignore diff --git a/.gitignore b/.gitignore index 583b596..af350e8 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,9 @@ uv.lock # Quarto docs/_site/ +docs/.quarto/ +docs/**/*.quarto_ipynb* +docs/api/*.qmd +!docs/api/_metadata.yml -# created by quartodoc -docs/api \ No newline at end of file +# created by quartodoc \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index ad29309..0000000 --- a/docs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.quarto/ -**/*.quarto_ipynb From db7bf4bcf6c996e3a41c2a069a3a1d5b16d83432 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Fri, 6 Mar 2026 14:30:09 +0100 Subject: [PATCH 2/8] Move files and add examples --- docs/examples/combining-detectors.qmd | 68 ++++++++++++++ docs/examples/index.qmd | 34 +++++++ docs/examples/quick-start.qmd | 58 ++++++++++++ docs/{ => user-guide}/design.qmd | 80 ++++++++-------- docs/{ => user-guide}/getting-started.qmd | 108 +++++++++++----------- 5 files changed, 256 insertions(+), 92 deletions(-) create mode 100644 docs/examples/combining-detectors.qmd create mode 100644 docs/examples/index.qmd create mode 100644 docs/examples/quick-start.qmd rename docs/{ => user-guide}/design.qmd (97%) rename docs/{ => user-guide}/getting-started.qmd (95%) diff --git a/docs/examples/combining-detectors.qmd b/docs/examples/combining-detectors.qmd new file mode 100644 index 0000000..f80da2d --- /dev/null +++ b/docs/examples/combining-detectors.qmd @@ -0,0 +1,68 @@ +--- +title: Combining detectors +description: Combine multiple detectors for realistic water-domain anomaly detection +jupyter: tsod +--- + +This example demonstrates how to combine detectors for a flow time series with both spikes and flatline behavior. + +## Imports + +```{python} +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from tsod import CombinedDetector, RangeDetector, ConstantValueDetector +``` + +## Create sample data + +```{python} +rng = np.random.default_rng(7) +time = pd.date_range("2025-02-01", periods=300, freq="15min") + +flow = 45 + 6 * np.sin(np.linspace(0, 8 * np.pi, len(time))) + rng.normal(0, 0.8, len(time)) + +# Out-of-range spikes +flow[[80, 210]] = [72, 10] + +# Sensor flatline period +flow[130:145] = flow[129] + +series = pd.Series(flow, index=time, name="flow_m3s") +series.head() +``` + +## Fit if relevant + +```{python} +normal_window = series.iloc[:100] + +range_detector = RangeDetector(quantiles=(0.01, 0.99)) +range_detector.fit(normal_window) + +constant_detector = ConstantValueDetector(window_size=6) + +detector = CombinedDetector([range_detector, constant_detector]) +``` + +## Detect anomalies + +```{python} +anomalies = detector.detect(series) +anomalies.sum() +``` + +## Visualize results + +```{python} +fig, ax = plt.subplots(figsize=(11, 4)) +series.plot(ax=ax, label="Flow") +series[anomalies].plot(ax=ax, linestyle="", marker="o", color="crimson", label="Anomaly") +ax.set_ylabel("m³/s") +ax.set_title("Combined detector output") +ax.legend() +``` + +This draft can later be replaced with a domain-specific case based on observed station data. diff --git a/docs/examples/index.qmd b/docs/examples/index.qmd new file mode 100644 index 0000000..4de6420 --- /dev/null +++ b/docs/examples/index.qmd @@ -0,0 +1,34 @@ +--- +title: Examples +toc: false +--- + +# Examples + +This section contains realistic, water-domain examples for using **tsod**. + +Each example follows the same practical structure: + +- show imports +- create sample data +- demonstrate `detect()` and `fit()` where relevant +- show visualization of results + +## Available examples + +### [Quick start](quick-start.qmd) + +A minimal end-to-end example to get running quickly with a simple detector workflow. + +### [Combining detectors](combining-detectors.qmd) + +Demonstrates how to combine multiple detectors and interpret the merged anomaly signal. + +### [Water level example notebook](https://github.com/DHI/tsod/blob/main/notebooks/Example%20Water%20Level.ipynb) + +A realistic notebook example from the water domain based on observed water-level time series. + +## Planned additions + +Additional examples from different water-related domains will be added over time. + diff --git a/docs/examples/quick-start.qmd b/docs/examples/quick-start.qmd new file mode 100644 index 0000000..1eddc76 --- /dev/null +++ b/docs/examples/quick-start.qmd @@ -0,0 +1,58 @@ +--- +title: Quick start +description: Detect basic anomalies in a synthetic water-level time series +jupyter: tsod +--- + +This example shows a minimal end-to-end workflow on water-level data. + +## Imports + +```{python} +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from tsod import RangeDetector +``` + +## Create sample data + +```{python} +rng = np.random.default_rng(42) +time = pd.date_range("2025-01-01", periods=240, freq="h") + +baseline = 1.5 + 0.2 * np.sin(np.linspace(0, 6 * np.pi, len(time))) +noise = rng.normal(0, 0.03, len(time)) +water_level = baseline + noise + +# Inject a few unrealistic spikes +water_level[[40, 120, 180]] = [2.4, 0.2, 2.6] + +series = pd.Series(water_level, index=time, name="water_level_m") +series.head() +``` + +## Detect anomalies + +```{python} +detector = RangeDetector(min_value=0.8, max_value=2.1) +anomalies = detector.detect(series) + +anomalies.sum() +``` + +## Visualize results + +```{python} +fig, ax = plt.subplots(figsize=(10, 4)) +series.plot(ax=ax, label="Water level") +series[anomalies].plot(ax=ax, linestyle="", marker="o", color="red", label="Anomaly") +ax.set_ylabel("m") +ax.set_title("Quick anomaly screening") +ax.legend() +``` + +## Next step + +Try combining multiple detectors for a more robust signal in [Combining detectors](combining-detectors.qmd). diff --git a/docs/design.qmd b/docs/user-guide/design.qmd similarity index 97% rename from docs/design.qmd rename to docs/user-guide/design.qmd index ce25eb4..411108c 100644 --- a/docs/design.qmd +++ b/docs/user-guide/design.qmd @@ -1,41 +1,41 @@ -# Design philosophy - - -## {{< fa brands python >}} Familiar - -tsod aims to use a syntax familiar to users of scientific computing libraries such as Pandas & sckit-learn. - -## {{< fa download >}} Easy to install - -```bash -$ pip install tsod -``` - - -## {{< fa brands osi >}} Open Source​ -tsod is an open source project licensed under the MIT license. -The software is provided free of charge with the source code available for inspection and modification. - -Contributions are welcome! - -## {{< fa comments >}} Easy to collaborate -By developing tsod on GitHub along with a completely open discussion, we believe that the collaboration between developers and end-users results in a useful library. - -## {{< fa list-ol >}} Reproducible -By providing the historical versions of tsod on PyPI it is possible to reproduce the behaviour of an older existing system, based on an older version. - -**Install specific version** - -```bash -pip install tsod==0.2.0 -``` - -## {{< fa brands github >}} Easy access to new features -Features are being added all the time, by developers at DHI in offices all around the globe as well as external contributors using tsod in their work. -These new features are always available from the [main branch on GitHub](https://github.com/DHI/tsod) and thanks to automated testing, it is always possible to verify that the tests passes before downloading a new development version. - -**Install development version** - -```bash -$ pip install https://github.com/DHI/tsod/archive/main.zip +# Design philosophy + + +## {{< fa brands python >}} Familiar + +tsod aims to use a syntax familiar to users of scientific computing libraries such as Pandas & sckit-learn. + +## {{< fa download >}} Easy to install + +```bash +$ pip install tsod +``` + + +## {{< fa brands osi >}} Open Source​ +tsod is an open source project licensed under the MIT license. +The software is provided free of charge with the source code available for inspection and modification. + +Contributions are welcome! + +## {{< fa comments >}} Easy to collaborate +By developing tsod on GitHub along with a completely open discussion, we believe that the collaboration between developers and end-users results in a useful library. + +## {{< fa list-ol >}} Reproducible +By providing the historical versions of tsod on PyPI it is possible to reproduce the behaviour of an older existing system, based on an older version. + +**Install specific version** + +```bash +pip install tsod==0.2.0 +``` + +## {{< fa brands github >}} Easy access to new features +Features are being added all the time, by developers at DHI in offices all around the globe as well as external contributors using tsod in their work. +These new features are always available from the [main branch on GitHub](https://github.com/DHI/tsod) and thanks to automated testing, it is always possible to verify that the tests passes before downloading a new development version. + +**Install development version** + +```bash +$ pip install https://github.com/DHI/tsod/archive/main.zip ``` \ No newline at end of file diff --git a/docs/getting-started.qmd b/docs/user-guide/getting-started.qmd similarity index 95% rename from docs/getting-started.qmd rename to docs/user-guide/getting-started.qmd index 5c2a9c0..5273396 100644 --- a/docs/getting-started.qmd +++ b/docs/user-guide/getting-started.qmd @@ -1,53 +1,57 @@ -Getting started -=============== - -![](https://raw.githubusercontent.com/DHI/tsod/main/images/anomaly.png) - -Sensors often provide faulty or missing observations. These anomalies must be detected automatically and replaced with more feasible values before feeding the data to numerical simulation engines as boundary conditions or real time decision systems. - -This package aims to provide examples and algorithms for detecting anomalies in time series data specifically tailored to DHI users and the water domain. It is simple to install and deploy operationally and is accessible to everyone (open-source). - -`tsod` is library for timeseries data. The format of a timeseries is always a [](`pandas.Series`) and in some cases with a [](`pandas.DatetimeIndex`) - -1. Get data in the form of a a [](`pandas.Series`) (see Data formats below) -2. Select one or more detectors e.g. [](`~tsod.RangeDetector`) or [](`~tsod.ConstantValueDetector`) -3. Define parameters (e.g. min/max, max rate of change) or... -4. Fit parameters based on normal data, i.e. without outliers -5. Detect outliers in any dataset - -Example -------- - -```{python} -import pandas as pd -from tsod import RangeDetector -rd = RangeDetector(max_value=2.0) -data = pd.Series([0.0, 1.0, 3.0]) # 3.0 is out of range i.e. an anomaly -anom = rd.detect(data) -anom -``` - -```{python} -data[anom] # get anomalous data -``` - -```{python} -data[~anom] # get normal data -``` - - -Saving and loading ------------------- -Save a configured detector -```python -cd = CombinedDetector([ConstantValueDetector(), RangeDetector()]) -cd.fit(normal_data) -cd.save("detector.joblib") -``` - -... and then later load it from disk -```python -my_detector = tsod.load("detector.joblib") -my_detector.detect(some_data) -``` +--- +title: Getting started +execute: + enabled: false +--- + + +![](https://raw.githubusercontent.com/DHI/tsod/main/images/anomaly.png) + +Sensors often provide faulty or missing observations. These anomalies must be detected automatically and replaced with more feasible values before feeding the data to numerical simulation engines as boundary conditions or real time decision systems. + +This package aims to provide examples and algorithms for detecting anomalies in time series data specifically tailored to DHI users and the water domain. It is simple to install and deploy operationally and is accessible to everyone (open-source). + +`tsod` is library for timeseries data. The format of a timeseries is always a [](`pandas.Series`) and in some cases with a [](`pandas.DatetimeIndex`) + +1. Get data in the form of a a [](`pandas.Series`) (see Data formats below) +2. Select one or more detectors e.g. [](`~tsod.RangeDetector`) or [](`~tsod.ConstantValueDetector`) +3. Define parameters (e.g. min/max, max rate of change) or... +4. Fit parameters based on normal data, i.e. without outliers +5. Detect outliers in any dataset + +Example +------- + +```{python} +import pandas as pd +from tsod import RangeDetector +rd = RangeDetector(max_value=2.0) +data = pd.Series([0.0, 1.0, 3.0]) # 3.0 is out of range i.e. an anomaly +anom = rd.detect(data) +anom +``` + +```{python} +data[anom] # get anomalous data +``` + +```{python} +data[~anom] # get normal data +``` + + +Saving and loading +------------------ +Save a configured detector +```python +cd = CombinedDetector([ConstantValueDetector(), RangeDetector()]) +cd.fit(normal_data) +cd.save("detector.joblib") +``` + +... and then later load it from disk +```python +my_detector = tsod.load("detector.joblib") +my_detector.detect(some_data) +``` \ No newline at end of file From 878559046b9e197e99f3fe084a3977c7b7c3c2d8 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Fri, 6 Mar 2026 14:30:29 +0100 Subject: [PATCH 3/8] Update quarto file with new structure --- docs/_quarto.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 34bab29..007db88 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -2,13 +2,15 @@ project: type: website website: - title: "tsod" + title: "" page-footer: "© 2025 DHI Group" repo-url: https://github.com/DHI/tsod repo-actions: [edit] repo-subdir: docs + page-navigation: true navbar: + logo: https://raw.githubusercontent.com/DHI/tsod/main/images/logo/tsod.png tools: - icon: github menu: @@ -17,13 +19,27 @@ website: - text: Report a Bug url: https://github.com/DHI/tsod/issues left: - - href: index.qmd - text: Home - - href: getting-started.qmd - text: Getting Started - - href: design.qmd - - href: api/index.qmd - text: API Reference + - text: Home + href: index.qmd + - text: User Guide + href: user-guide/getting-started.qmd + - text: Examples + href: examples/index.qmd + - text: API Reference + href: api/index.qmd + + sidebar: + - title: "User Guide" + style: docked + contents: + - user-guide/getting-started.qmd + - user-guide/design.qmd + - title: "Examples" + style: docked + contents: + - examples/index.qmd + - examples/quick-start.qmd + - examples/combining-detectors.qmd filters: - interlinks From 615fac402608522ae9110afe244b2c5b77eff221 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Fri, 6 Mar 2026 14:31:20 +0100 Subject: [PATCH 4/8] Add metadata in api and Makefile Followed the structure in mikeio --- docs/Makefile | 3 ++- docs/api/_metadata.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 docs/api/_metadata.yml diff --git a/docs/Makefile b/docs/Makefile index 7d837ac..f1c20a6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,5 @@ preview: api uv run quarto preview clean: - rm -rf _site api objects.json + rm -rf _site objects.json + rm -f api/*.qmd diff --git a/docs/api/_metadata.yml b/docs/api/_metadata.yml new file mode 100644 index 0000000..e21e9fa --- /dev/null +++ b/docs/api/_metadata.yml @@ -0,0 +1 @@ +repo-actions: false From cd888c69846100b1fb4829d9a1146cef7b28ded0 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Mon, 9 Mar 2026 11:32:11 +0100 Subject: [PATCH 5/8] Add links in getting-started as in mikeio docs --- docs/examples/index.qmd | 2 +- docs/index.qmd | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/examples/index.qmd b/docs/examples/index.qmd index 4de6420..84f9854 100644 --- a/docs/examples/index.qmd +++ b/docs/examples/index.qmd @@ -5,7 +5,7 @@ toc: false # Examples -This section contains realistic, water-domain examples for using **tsod**. +This section contains water-domain examples for using **tsod**. Each example follows the same practical structure: diff --git a/docs/index.qmd b/docs/index.qmd index 5577cf2..9806e30 100644 --- a/docs/index.qmd +++ b/docs/index.qmd @@ -20,12 +20,13 @@ format-links: false Install **tsod** with [`pip`](https://pypi.org/project/tsod/) and get up and running in minutes - +[**Getting started**](user-guide/getting-started.qmd) ## {{< fa brands python >}} **It's just Python** Use familiar Python workflows to integrate anomaly detection into your models and pipelines +[**API Reference**](api/index.qmd) ::: @@ -40,6 +41,7 @@ Choose from detectors like `RangeDetector` and `ConstantValueDetector` to identi **tsod** is licensed under MIT and the source code is available on [GitHub](https://github.com/DHI/tsod) +[**Design philosophy**](user-guide/design.qmd) ::: From 311e5347f50e8a5f0197883fc6c30622fc1ab860 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Mon, 9 Mar 2026 11:44:36 +0100 Subject: [PATCH 6/8] Remove _metadata file to keep "edit this page" button --- docs/api/_metadata.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/api/_metadata.yml diff --git a/docs/api/_metadata.yml b/docs/api/_metadata.yml deleted file mode 100644 index e21e9fa..0000000 --- a/docs/api/_metadata.yml +++ /dev/null @@ -1 +0,0 @@ -repo-actions: false From f97d14332e1a8d52ef1c39b63da5d5c7482f3d4b Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Fri, 13 Mar 2026 10:14:37 +0100 Subject: [PATCH 7/8] Add script for automated generation of notebooks instead of using qmd files and some small changes to fix the structure --- .gitignore | 4 + Makefile | 10 +- docs/Makefile | 15 +- docs/_quarto.yml | 12 +- docs/api/index.qmd | 17 ++ docs/custom.css | 7 + docs/examples/combining-detectors.qmd | 68 ------ docs/examples/index.qmd | 31 +-- docs/examples/quick-start.qmd | 58 ------ .../generate_examples_from_notebooks.py | 194 ++++++++++++++++++ 10 files changed, 253 insertions(+), 163 deletions(-) create mode 100644 docs/api/index.qmd create mode 100644 docs/custom.css delete mode 100644 docs/examples/combining-detectors.qmd delete mode 100644 docs/examples/quick-start.qmd create mode 100644 docs/scripts/generate_examples_from_notebooks.py diff --git a/.gitignore b/.gitignore index af350e8..4dfc8f2 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,10 @@ docs/_site/ docs/.quarto/ docs/**/*.quarto_ipynb* docs/api/*.qmd +!docs/api/index.qmd !docs/api/_metadata.yml +# Generated from notebooks/ by docs/scripts/generate_examples_from_notebooks.py +docs/examples/*.ipynb +docs/examples/*_files/ # created by quartodoc \ No newline at end of file diff --git a/Makefile b/Makefile index 460f4cf..2127f69 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ LIB = src/tsod -.PHONY: check build lint format test coverage docs clean +.PHONY: check build lint format test coverage docs examples clean check: lint test @@ -19,13 +19,15 @@ test: coverage: uv run pytest --cov-report html --cov=$(LIB) tests/ +examples: + $(MAKE) -C docs examples + docs: - cd docs && uv run quartodoc build - uv run quarto render docs + $(MAKE) -C docs build clean: rm -rf .pytest_cache rm -rf .mypy_cache rm -rf .coverage rm -rf dist - rm -rf docs/_build + $(MAKE) -C docs clean diff --git a/docs/Makefile b/docs/Makefile index f1c20a6..04ea590 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,7 +1,7 @@ # Minimal makefile for Quarto documentation # -.PHONY: help api build preview clean +.PHONY: help api examples build preview clean help: @echo "Please use 'make ' where is one of:" @@ -13,12 +13,17 @@ help: api: uv run quartodoc build -build: api +examples: + uv run python scripts/generate_examples_from_notebooks.py + +build: api examples uv run quarto render -preview: api +preview: api examples uv run quarto preview clean: - rm -rf _site objects.json - rm -f api/*.qmd + rm -rf _site .quarto objects.json + find api -name "*.qmd" ! -name "index.qmd" -delete + rm -f examples/*.ipynb + rm -rf examples/*_files diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 007db88..cff62f4 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -38,8 +38,11 @@ website: style: docked contents: - examples/index.qmd - - examples/quick-start.qmd - - examples/combining-detectors.qmd + # BEGIN_GENERATED_EXAMPLES — managed by docs/scripts/generate_examples_from_notebooks.py + - examples/getting-started.ipynb + - examples/example-water-level.ipynb + - examples/detect-on-dataframes.ipynb + # END_GENERATED_EXAMPLES filters: - interlinks @@ -79,6 +82,7 @@ quartodoc: format: html: theme: cosmo + css: custom.css toc: true - ipynb: - toc: true + # ipynb: + # toc: true diff --git a/docs/api/index.qmd b/docs/api/index.qmd new file mode 100644 index 0000000..0857ce4 --- /dev/null +++ b/docs/api/index.qmd @@ -0,0 +1,17 @@ +# API Reference {.doc .doc-index} + +## tsod + + + +| | | +| --- | --- | +| [RangeDetector](RangeDetector.qmd#tsod.RangeDetector) | Detect values outside range. | +| [ConstantValueDetector](ConstantValueDetector.qmd#tsod.ConstantValueDetector) | Detect contiguous periods of constant values within a configurable time window. | +| [ConstantGradientDetector](ConstantGradientDetector.qmd#tsod.ConstantGradientDetector) | Detect constant gradients. | +| [GradientDetector](GradientDetector.qmd#tsod.GradientDetector) | Detect abrupt changes in time series data. | +| [DiffDetector](DiffDetector.qmd#tsod.DiffDetector) | Detect sudden shifts in data, irrespective of time axis. | +| [RollingStandardDeviationDetector](RollingStandardDeviationDetector.qmd#tsod.RollingStandardDeviationDetector) | Detect large variations. | +| [CombinedDetector](CombinedDetector.qmd#tsod.CombinedDetector) | Combine detectors. | +| [HampelDetector](HampelDetector.qmd#tsod.HampelDetector) | Hampel filter implementation that works on numpy arrays, implemented with numba. | +| [load](load.qmd#tsod.load) | Load a saved model from disk saved with `Detector.save` | \ No newline at end of file diff --git a/docs/custom.css b/docs/custom.css new file mode 100644 index 0000000..4470ead --- /dev/null +++ b/docs/custom.css @@ -0,0 +1,7 @@ +#quarto-content.page-layout-full main.content.column-body > #title-block-header + p { + display: none; +} + +#quarto-content.page-layout-full main.content.column-body { + max-width: min(1600px, calc(100vw - 4rem)); +} \ No newline at end of file diff --git a/docs/examples/combining-detectors.qmd b/docs/examples/combining-detectors.qmd deleted file mode 100644 index f80da2d..0000000 --- a/docs/examples/combining-detectors.qmd +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: Combining detectors -description: Combine multiple detectors for realistic water-domain anomaly detection -jupyter: tsod ---- - -This example demonstrates how to combine detectors for a flow time series with both spikes and flatline behavior. - -## Imports - -```{python} -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt - -from tsod import CombinedDetector, RangeDetector, ConstantValueDetector -``` - -## Create sample data - -```{python} -rng = np.random.default_rng(7) -time = pd.date_range("2025-02-01", periods=300, freq="15min") - -flow = 45 + 6 * np.sin(np.linspace(0, 8 * np.pi, len(time))) + rng.normal(0, 0.8, len(time)) - -# Out-of-range spikes -flow[[80, 210]] = [72, 10] - -# Sensor flatline period -flow[130:145] = flow[129] - -series = pd.Series(flow, index=time, name="flow_m3s") -series.head() -``` - -## Fit if relevant - -```{python} -normal_window = series.iloc[:100] - -range_detector = RangeDetector(quantiles=(0.01, 0.99)) -range_detector.fit(normal_window) - -constant_detector = ConstantValueDetector(window_size=6) - -detector = CombinedDetector([range_detector, constant_detector]) -``` - -## Detect anomalies - -```{python} -anomalies = detector.detect(series) -anomalies.sum() -``` - -## Visualize results - -```{python} -fig, ax = plt.subplots(figsize=(11, 4)) -series.plot(ax=ax, label="Flow") -series[anomalies].plot(ax=ax, linestyle="", marker="o", color="crimson", label="Anomaly") -ax.set_ylabel("m³/s") -ax.set_title("Combined detector output") -ax.legend() -``` - -This draft can later be replaced with a domain-specific case based on observed station data. diff --git a/docs/examples/index.qmd b/docs/examples/index.qmd index 84f9854..db871fc 100644 --- a/docs/examples/index.qmd +++ b/docs/examples/index.qmd @@ -1,34 +1,17 @@ --- title: Examples +page-layout: full toc: false --- # Examples -This section contains water-domain examples for using **tsod**. +This page is auto-generated from notebooks in `notebooks/`. -Each example follows the same practical structure: +## Available notebook examples -- show imports -- create sample data -- demonstrate `detect()` and `fit()` where relevant -- show visualization of results - -## Available examples - -### [Quick start](quick-start.qmd) - -A minimal end-to-end example to get running quickly with a simple detector workflow. - -### [Combining detectors](combining-detectors.qmd) - -Demonstrates how to combine multiple detectors and interpret the merged anomaly signal. - -### [Water level example notebook](https://github.com/DHI/tsod/blob/main/notebooks/Example%20Water%20Level.ipynb) - -A realistic notebook example from the water domain based on observed water-level time series. - -## Planned additions - -Additional examples from different water-related domains will be added over time. +- [Getting started](getting-started.ipynb) +- [Example Water Level](example-water-level.ipynb) +- [Detect on DataFrames](detect-on-dataframes.ipynb) +Regenerate with `make examples`. diff --git a/docs/examples/quick-start.qmd b/docs/examples/quick-start.qmd deleted file mode 100644 index 1eddc76..0000000 --- a/docs/examples/quick-start.qmd +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: Quick start -description: Detect basic anomalies in a synthetic water-level time series -jupyter: tsod ---- - -This example shows a minimal end-to-end workflow on water-level data. - -## Imports - -```{python} -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt - -from tsod import RangeDetector -``` - -## Create sample data - -```{python} -rng = np.random.default_rng(42) -time = pd.date_range("2025-01-01", periods=240, freq="h") - -baseline = 1.5 + 0.2 * np.sin(np.linspace(0, 6 * np.pi, len(time))) -noise = rng.normal(0, 0.03, len(time)) -water_level = baseline + noise - -# Inject a few unrealistic spikes -water_level[[40, 120, 180]] = [2.4, 0.2, 2.6] - -series = pd.Series(water_level, index=time, name="water_level_m") -series.head() -``` - -## Detect anomalies - -```{python} -detector = RangeDetector(min_value=0.8, max_value=2.1) -anomalies = detector.detect(series) - -anomalies.sum() -``` - -## Visualize results - -```{python} -fig, ax = plt.subplots(figsize=(10, 4)) -series.plot(ax=ax, label="Water level") -series[anomalies].plot(ax=ax, linestyle="", marker="o", color="red", label="Anomaly") -ax.set_ylabel("m") -ax.set_title("Quick anomaly screening") -ax.legend() -``` - -## Next step - -Try combining multiple detectors for a more robust signal in [Combining detectors](combining-detectors.qmd). diff --git a/docs/scripts/generate_examples_from_notebooks.py b/docs/scripts/generate_examples_from_notebooks.py new file mode 100644 index 0000000..464cd47 --- /dev/null +++ b/docs/scripts/generate_examples_from_notebooks.py @@ -0,0 +1,194 @@ +import json +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +NOTEBOOKS_DIR = REPO_ROOT / "notebooks" +EXAMPLES_DIR = REPO_ROOT / "docs" / "examples" +QUARTO_YML = REPO_ROOT / "docs" / "_quarto.yml" + +_SIDEBAR_BEGIN = " # BEGIN_GENERATED_EXAMPLES — managed by docs/scripts/generate_examples_from_notebooks.py" +_SIDEBAR_END = " # END_GENERATED_EXAMPLES" +_EXAMPLE_ORDER = { + "getting-started.ipynb": 0, + "example-water-level.ipynb": 1, + "detect-on-dataframes.ipynb": 2, +} + + +def slugify(name: str) -> str: + """Create a filesystem-friendly slug from a notebook stem.""" + pieces: list[str] = [] + current: list[str] = [] + + for char in name.lower(): + if char.isalnum(): + current.append(char) + continue + + if current: + pieces.append("".join(current)) + current = [] + + if current: + pieces.append("".join(current)) + + slug = "-".join(pieces) + return slug or "example" + + +def sort_entries(entries: list[tuple[str, str]]) -> list[tuple[str, str]]: + """Keep generated example pages in a stable, user-defined order.""" + return sorted( + entries, + key=lambda item: (_EXAMPLE_ORDER.get(item[1], len(_EXAMPLE_ORDER)), item[0].lower()), + ) + + +def title_from_notebook(notebook: dict, fallback: str) -> str: + metadata_title = notebook.get("metadata", {}).get("title") + if isinstance(metadata_title, str) and metadata_title.strip(): + return metadata_title.strip() + return fallback + + +def rewrite_notebook_relative_paths(source: str) -> str: + """Rewrite paths that are valid in notebooks/ to paths valid in docs/examples/.""" + return source.replace("../tests/", "../../tests/") + + +def rewrite_cell_source_paths(notebook: dict) -> None: + """Rewrite relative paths in markdown and code cell sources.""" + for cell in notebook.get("cells", []): + source = cell.get("source") + if isinstance(source, str): + cell["source"] = rewrite_notebook_relative_paths(source) + continue + if isinstance(source, list): + cell["source"] = [rewrite_notebook_relative_paths(line) for line in source] + + +def notebook_front_matter_source(title: str, notebook_name: str) -> str: + lines = [ + "---", + f"title: {title}", + f"description: Auto-generated from notebooks/{notebook_name}", + "jupyter: tsod", + "page-layout: full", + "---", + "", + "", + ] + return "\n".join(lines) + "\n" + + +def apply_front_matter_cell(notebook: dict, title: str, notebook_name: str) -> None: + source = notebook_front_matter_source(title=title, notebook_name=notebook_name) + front_matter_cell = { + "cell_type": "markdown", + "metadata": {"language": "markdown", "tags": ["remove-cell"]}, + "source": source, + } + + cells = notebook.setdefault("cells", []) + if not cells: + cells.append(front_matter_cell) + return + + first_cell = cells[0] + first_source = first_cell.get("source") + lines = first_source if isinstance(first_source, list) else [str(first_source or "")] + first_line = lines[0].strip() if lines else "" + + if first_cell.get("cell_type") == "markdown" and first_line == "---": + first_cell["source"] = source + return + + cells.insert(0, front_matter_cell) + + +def copy_notebook_to_examples(notebook_path: Path) -> tuple[str, str]: + notebook = json.loads(notebook_path.read_text(encoding="utf-8")) + stem = notebook_path.stem + title = title_from_notebook(notebook, fallback=stem) + slug = slugify(stem) + ipynb_path = EXAMPLES_DIR / f"{slug}.ipynb" + + rewrite_cell_source_paths(notebook) + apply_front_matter_cell(notebook, title=title, notebook_name=notebook_path.name) + + ipynb_path.write_text(json.dumps(notebook, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return title, ipynb_path.name + + +def write_index(entries: list[tuple[str, str]]) -> None: + index_lines = [ + "---", + "title: Examples", + "page-layout: full", + "toc: false", + "---", + "", + "# Examples", + "", + "This page is auto-generated from notebooks in `notebooks/`.", + "", + "## Available notebook examples", + "", + ] + + for title, rel_path in sort_entries(entries): + index_lines.append(f"- [{title}]({rel_path})") + + index_lines.append("") + index_lines.append("Regenerate with `make examples`.") + + (EXAMPLES_DIR / "index.qmd").write_text("\n".join(index_lines) + "\n", encoding="utf-8") + + +def update_quarto_sidebar(entries: list[tuple[str, str]]) -> None: + """Keep the Examples sidebar in _quarto.yml in sync with generated notebooks.""" + content = QUARTO_YML.read_text(encoding="utf-8") + + lines = [_SIDEBAR_BEGIN] + for _title, ipynb_name in sort_entries(entries): + lines.append(f" - examples/{ipynb_name}") + lines.append(_SIDEBAR_END) + new_block = "\n".join(lines) + + updated = re.sub( + re.escape(_SIDEBAR_BEGIN) + r".*?" + re.escape(_SIDEBAR_END), + new_block, + content, + flags=re.DOTALL, + ) + QUARTO_YML.write_text(updated, encoding="utf-8") + + +def main() -> None: + EXAMPLES_DIR.mkdir(parents=True, exist_ok=True) + + # Remove previously generated files to avoid stale pages. + for ipynb_file in EXAMPLES_DIR.glob("*.ipynb"): + ipynb_file.unlink() + + for qmd_file in EXAMPLES_DIR.glob("*.qmd"): + if qmd_file.name != "index.qmd": + qmd_file.unlink() + + for quarto_ipynb in EXAMPLES_DIR.glob("*.quarto_ipynb"): + quarto_ipynb.unlink() + + notebook_files = sorted(NOTEBOOKS_DIR.glob("*.ipynb")) + entries: list[tuple[str, str]] = [] + + for notebook_path in notebook_files: + title, ipynb_name = copy_notebook_to_examples(notebook_path) + entries.append((title, ipynb_name)) + + write_index(entries) + update_quarto_sidebar(entries) + + +if __name__ == "__main__": + main() From 6101c4b4ad9cee9b539818ae11f0e2f1362fbca1 Mon Sep 17 00:00:00 2001 From: adihellstrom Date: Mon, 16 Mar 2026 09:34:35 +0100 Subject: [PATCH 8/8] Trim notebook generation code (e.g. skip changing file names) --- docs/_quarto.yml | 6 ++-- docs/examples/index.qmd | 6 ++-- .../generate_examples_from_notebooks.py | 34 ++++--------------- 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/docs/_quarto.yml b/docs/_quarto.yml index cff62f4..735ff17 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -39,9 +39,9 @@ website: contents: - examples/index.qmd # BEGIN_GENERATED_EXAMPLES — managed by docs/scripts/generate_examples_from_notebooks.py - - examples/getting-started.ipynb - - examples/example-water-level.ipynb - - examples/detect-on-dataframes.ipynb + - examples/Getting started.ipynb + - examples/Example Water Level.ipynb + - examples/Detect on DataFrames.ipynb # END_GENERATED_EXAMPLES filters: diff --git a/docs/examples/index.qmd b/docs/examples/index.qmd index db871fc..76c938c 100644 --- a/docs/examples/index.qmd +++ b/docs/examples/index.qmd @@ -10,8 +10,8 @@ This page is auto-generated from notebooks in `notebooks/`. ## Available notebook examples -- [Getting started](getting-started.ipynb) -- [Example Water Level](example-water-level.ipynb) -- [Detect on DataFrames](detect-on-dataframes.ipynb) +- [Getting started](Getting%20started.ipynb) +- [Example Water Level](Example%20Water%20Level.ipynb) +- [Detect on DataFrames](Detect%20on%20DataFrames.ipynb) Regenerate with `make examples`. diff --git a/docs/scripts/generate_examples_from_notebooks.py b/docs/scripts/generate_examples_from_notebooks.py index 464cd47..11c9591 100644 --- a/docs/scripts/generate_examples_from_notebooks.py +++ b/docs/scripts/generate_examples_from_notebooks.py @@ -1,6 +1,7 @@ import json import re from pathlib import Path +from urllib.parse import quote REPO_ROOT = Path(__file__).resolve().parents[2] NOTEBOOKS_DIR = REPO_ROOT / "notebooks" @@ -10,33 +11,12 @@ _SIDEBAR_BEGIN = " # BEGIN_GENERATED_EXAMPLES — managed by docs/scripts/generate_examples_from_notebooks.py" _SIDEBAR_END = " # END_GENERATED_EXAMPLES" _EXAMPLE_ORDER = { - "getting-started.ipynb": 0, - "example-water-level.ipynb": 1, - "detect-on-dataframes.ipynb": 2, + "Getting started.ipynb": 0, + "Example Water Level.ipynb": 1, + "Detect on DataFrames.ipynb": 2, } -def slugify(name: str) -> str: - """Create a filesystem-friendly slug from a notebook stem.""" - pieces: list[str] = [] - current: list[str] = [] - - for char in name.lower(): - if char.isalnum(): - current.append(char) - continue - - if current: - pieces.append("".join(current)) - current = [] - - if current: - pieces.append("".join(current)) - - slug = "-".join(pieces) - return slug or "example" - - def sort_entries(entries: list[tuple[str, str]]) -> list[tuple[str, str]]: """Keep generated example pages in a stable, user-defined order.""" return sorted( @@ -111,8 +91,7 @@ def copy_notebook_to_examples(notebook_path: Path) -> tuple[str, str]: notebook = json.loads(notebook_path.read_text(encoding="utf-8")) stem = notebook_path.stem title = title_from_notebook(notebook, fallback=stem) - slug = slugify(stem) - ipynb_path = EXAMPLES_DIR / f"{slug}.ipynb" + ipynb_path = EXAMPLES_DIR / notebook_path.name rewrite_cell_source_paths(notebook) apply_front_matter_cell(notebook, title=title, notebook_name=notebook_path.name) @@ -138,7 +117,8 @@ def write_index(entries: list[tuple[str, str]]) -> None: ] for title, rel_path in sort_entries(entries): - index_lines.append(f"- [{title}]({rel_path})") + encoded_path = quote(rel_path, safe="/") + index_lines.append(f"- [{title}]({encoded_path})") index_lines.append("") index_lines.append("Regenerate with `make examples`.")