From 51ab1055fd438989acf93d2b2a5a22b1253a944a Mon Sep 17 00:00:00 2001 From: zolizoli Date: Mon, 29 Jun 2026 17:17:24 +0200 Subject: [PATCH] Docs overhaul: contract docstrings + rebuilt quickstart/tutorials/troubleshooting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render fix (package-wide): Google docstring example sections used the singular "Example:" keyword, which griffe does not recognise as a code section — every example rendered as nested blockquotes and tripped mkdocs-autorefs ("Could not find cross-reference target") on bracketed subscripts like g["cat"]["sat"]. Renaming to "Examples:" makes them render as highlighted code blocks and clears all three strict-mode warnings. Docs: - Enable mkdocs strict mode; add markdown_extensions (admonition, pymdownx highlight/superfences/snippets) for a richer site and to embed example scripts. - Rewrite quickstart as a runnable, end-to-end walkthrough (verified in a venv). - New tutorial: word-association graphs + concept-to-concept pathfinding via networkx (no new public API), with a conceptual note on human-norm comparison. - New troubleshooting page from the real failure modes (spaCy model, Python.h / Cython headers, pickle safety, stopword ValueErrors, empty graphs, NodeNotFound). - Convert example pages from link stubs to embedded source (snippets) + real output. - Add missing API page for get_stopwords; complete the reference. - Promote the load_graph pickle-safety caveat to a prominent Warning admonition. - gitignore the mkdocs site/ build output. make ci green; mkdocs build --strict warning-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 + docs/api/stopwords.md | 3 + docs/examples/comparing_two_texts.md | 39 +++++++-- docs/examples/multilingual_analysis.md | 48 +++++++++-- docs/examples/news_article_analysis.md | 50 +++++++++-- docs/index.md | 8 +- docs/quickstart.md | 76 +++++++++++++++-- docs/troubleshooting.md | 102 +++++++++++++++++++++++ docs/tutorials/word_association_graph.md | 98 ++++++++++++++++++++++ mkdocs.yml | 34 +++++++- src/kenon/backbone.py | 8 +- src/kenon/cooccurrence.py | 4 +- src/kenon/embeddings.py | 6 +- src/kenon/graphs.py | 13 ++- src/kenon/stopwords.py | 2 +- src/kenon/tokenizer.py | 8 +- 16 files changed, 451 insertions(+), 51 deletions(-) create mode 100644 docs/api/stopwords.md create mode 100644 docs/troubleshooting.md create mode 100644 docs/tutorials/word_association_graph.md diff --git a/.gitignore b/.gitignore index 82d863f..c95b5ba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ wheels/ # Virtual environments .venv +# Built documentation (mkdocs output) +site/ + # Experiment data (large, generated) experiments/google_and_the_mind/data/ experiments/google_and_the_mind/graphs/ diff --git a/docs/api/stopwords.md b/docs/api/stopwords.md new file mode 100644 index 0000000..091a934 --- /dev/null +++ b/docs/api/stopwords.md @@ -0,0 +1,3 @@ +# Stopwords + +::: kenon.stopwords.get_stopwords diff --git a/docs/examples/comparing_two_texts.md b/docs/examples/comparing_two_texts.md index 004d606..4c6b63a 100644 --- a/docs/examples/comparing_two_texts.md +++ b/docs/examples/comparing_two_texts.md @@ -1,9 +1,38 @@ -# Comparing Two Texts +# Comparing two texts -See `examples/comparing_two_texts.py` for the full source. +Build a co-occurrence graph for each of two texts on different topics, then +quantify how much vocabulary they share. This is the building block for any +graph-to-graph comparison (including comparing a text-derived network against +human association norms — see the +[word-association tutorial](../tutorials/word_association_graph.md)). -This example demonstrates: +## What it shows 1. Building co-occurrence graphs for two different texts -2. Computing Jaccard similarity of node sets -3. Identifying vocabulary gaps between texts +2. Computing the Jaccard similarity of their node sets +3. Identifying the vocabulary gap (terms unique to each text) + +## Run it + +```bash +python examples/comparing_two_texts.py +``` + +```text +Text A graph: 28 nodes, 55 edges +Text B graph: 30 nodes, 59 edges + +Jaccard similarity of node sets: 0.018 + +Shared nodes (1): ['natural'] +Only in Text A (27): ['algorithm', 'architecture', ...] +Only in Text B (29): ['biodiversity', 'cap', ...] + +Vocabulary gap: 56 unique terms out of 57 total +``` + +## Source + +```python title="examples/comparing_two_texts.py" +--8<-- "examples/comparing_two_texts.py" +``` diff --git a/docs/examples/multilingual_analysis.md b/docs/examples/multilingual_analysis.md index 3c063b3..491c74c 100644 --- a/docs/examples/multilingual_analysis.md +++ b/docs/examples/multilingual_analysis.md @@ -1,9 +1,45 @@ -# Multilingual Analysis +# Multilingual analysis -See `examples/multilingual_analysis.py` for the full source. +kenon is language-agnostic: tokenisation is driven by whichever spaCy model you +load, and stopwords can come from NLTK (many languages) or scikit-learn (English +only). This example processes English and German text and compares graph density. -This example demonstrates: +## What it shows -1. Tokenizing English and German texts -2. Building co-occurrence graphs for each language -3. Comparing graph density across languages +1. Tokenising English and German texts with different spaCy models +2. Selecting stopword sources per language (`sources=["nltk"]` for German) +3. Building co-occurrence graphs for each and comparing density + +## Prerequisites + +The German half needs the German spaCy model. If it is missing, the script +prints a clear message and skips that section rather than crashing: + +```bash +python -m spacy download de_core_news_sm +``` + +## Run it + +```bash +python examples/multilingual_analysis.py +``` + +```text +Multilingual Co-occurrence Graph Analysis +================================================== + +--- English --- + Tokens: 37, After filtering: 23 + Graph: 22 nodes, 43 edges + Density: 0.1861 + Top 5 nodes by degree: [('growth', 8), ('finding', 4), ('prestigious', 4), ('scientific', 4), ('journal', 4)] + +German analysis skipped: spaCy model 'de_core_news_sm' is not installed. +``` + +## Source + +```python title="examples/multilingual_analysis.py" +--8<-- "examples/multilingual_analysis.py" +``` diff --git a/docs/examples/news_article_analysis.md b/docs/examples/news_article_analysis.md index a7b8933..1af3854 100644 --- a/docs/examples/news_article_analysis.md +++ b/docs/examples/news_article_analysis.md @@ -1,12 +1,46 @@ -# News Article Analysis +# News article analysis -See `examples/news_article_analysis.py` for the full source. +A complete walk-through of the core kenon workflow on a single news article: +tokenise → remove stopwords → build a co-occurrence graph → extract the backbone +→ rank words by centrality → compare two semantic-graph embedders. -This example demonstrates: +## What it shows -1. Tokenizing a news article with lemmatization +1. Tokenising a news article with lemmatisation 2. Removing stopwords -3. Building a co-occurrence graph with window=3 -4. Extracting the backbone -5. Finding top nodes by degree centrality -6. Building semantic graphs with CountVectorizer and TF-IDF +3. Building a co-occurrence graph (`window=3`) +4. Extracting the backbone with the disparity filter +5. Finding the top nodes by degree centrality +6. Building semantic graphs with `CountVectorizerEmbedder` and `TfidfEmbedder` + +## Run it + +```bash +python examples/news_article_analysis.py +``` + +```text +Total tokens: 158 +Tokens after stopword removal: 95 + +Co-occurrence graph: 78 nodes, 274 edges +Backbone: 4 nodes, 6 edges + +Top 10 nodes by degree centrality (backbone): + growth: 1.000 + sluggish: 1.000 + recent: 1.000 + quarter: 1.000 + +--- CountVectorizer Semantic Graph --- +Nodes: 81, Edges: 555 + +--- TF-IDF Semantic Graph --- +Nodes: 81, Edges: 555 +``` + +## Source + +```python title="examples/news_article_analysis.py" +--8<-- "examples/news_article_analysis.py" +``` diff --git a/docs/index.md b/docs/index.md index a7352a6..f1e0779 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,9 +17,11 @@ No neural models or external training data required. ## Quick links -- [Quickstart](quickstart.md) -- [API Reference](api/tokenizer.md) -- [Examples](examples/news_article_analysis.md) +- [Quickstart](quickstart.md) — clean install to backbone graph +- [Tutorial: word-association graphs](tutorials/word_association_graph.md) — find paths between concepts +- [Examples](examples/news_article_analysis.md) — full, runnable scripts +- [Troubleshooting](troubleshooting.md) — common errors and fixes +- [API Reference](api/tokenizer.md) — every public function and its contract --- diff --git a/docs/quickstart.md b/docs/quickstart.md index d2204f2..0dda15a 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,5 +1,8 @@ # Quickstart +This page takes you from a clean install to a backbone graph in one sitting. +Every snippet below is runnable as-is. + ## Installation ```bash @@ -7,19 +10,78 @@ uv add kenon python -m spacy download en_core_web_sm ``` -## Basic usage +kenon uses a spaCy model for tokenisation. `en_core_web_sm` is the small English +model used throughout the docs; the download step is required the first time. + +!!! note "Building from source" + kenon depends on [`chronowords`](https://pypi.org/project/chronowords/), + which compiles a Cython extension. If `uv add` fails with + `fatal error: Python.h: No such file or directory`, see the + [Troubleshooting](troubleshooting.md) page. + +## Build a co-occurrence graph + +A co-occurrence graph connects tokens that appear near each other within a +sliding window. Nodes are tokens; edge weights are normalised co-occurrence +frequencies. ```python from kenon import Tokenizer, get_stopwords, build_cooccurrence_graph, extract_backbone +import networkx as nx -# Tokenize +text = ( + "The central bank held interest rates steady on Thursday. " + "Inflation in the euro zone was declining but remained too high. " + "Financial markets reacted positively as bond yields fell and stocks rose. " + "Analysts said the rate-hiking cycle had likely peaked." +) + +# 1. Tokenise (lemmatise so "rates"/"rate" collapse to one node) tokenizer = Tokenizer("en_core_web_sm", lemmatize=True) -tokens = tokenizer.flat_tokens("Your text here.") +tokens = tokenizer.flat_tokens(text) -# Build graph +# 2. Build the co-occurrence graph, dropping stopwords first stopwords = get_stopwords("english") -graph = build_cooccurrence_graph(tokens, window=2, stopwords=stopwords) +graph = build_cooccurrence_graph(tokens, window=3, stopwords=stopwords) +print(f"co-occurrence graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges") +``` + +## Extract the backbone + +Real co-occurrence graphs are dense and noisy. The **disparity filter** +([Serrano et al. 2009](https://arxiv.org/pdf/0904.2389.pdf)) keeps only edges +that are statistically significant relative to each node's other connections. -# Extract backbone -backbone = extract_backbone(graph, min_alpha_ptile=0.3, min_degree=2) +```python +# 3. Keep the statistically significant edges +backbone = extract_backbone(graph, min_alpha_ptile=0.3, min_degree=1) +print(f"backbone: {backbone.number_of_nodes()} nodes, {backbone.number_of_edges()} edges") + +# 4. A backbone graph is a plain networkx.Graph — use the whole networkx toolkit +top = sorted(nx.degree_centrality(backbone).items(), key=lambda kv: kv[1], reverse=True)[:5] +for word, score in top: + print(f" {word}: {score:.2f}") +``` + +Running the two snippets above prints something like: + +```text +co-occurrence graph: 27 nodes, 78 edges +backbone: 6 nodes, 6 edges + bank: 0.40 + central: 0.40 + hold: 0.40 + cycle: 0.40 + likely: 0.40 ``` + +!!! tip "`min_alpha_ptile` controls aggressiveness" + Higher values prune more edges. On short texts like this one, start low + (`0.3`) and `min_degree=1`; on book-length corpora, raise both. + +## Where to next + +- [Word-association graphs tutorial](tutorials/word_association_graph.md) — find + paths between two concepts. +- [Examples](examples/news_article_analysis.md) — full, runnable scripts. +- [API Reference](api/tokenizer.md) — every public function and its contract. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..b815ad0 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,102 @@ +# Troubleshooting + +Common errors and how to resolve them. Each entry shows the failure, why it +happens, and the fix. + +## `RuntimeError: spaCy model '...' is not installed` + +```text +RuntimeError: spaCy model 'en_core_web_sm' is not installed. +Run: python -m spacy download en_core_web_sm +``` + +`Tokenizer` loads its spaCy model lazily on first use, so this surfaces the +first time you call `flat_tokens`, `tokenize`, or `sentencize` — not at +construction time. Install the model named in the message: + +```bash +python -m spacy download en_core_web_sm +``` + +For other languages, pass the model name to `Tokenizer(...)` and download the +matching model (e.g. `de_core_news_sm` for German). + +## `fatal error: Python.h: No such file or directory` + +This appears while installing kenon, during the build of its transitive +dependency [`chronowords`](https://pypi.org/project/chronowords/), which +compiles a Cython extension and needs the Python development headers. System +Pythons on most Linux distros don't ship them by default. + +Either install the headers for your system Python: + +```bash +sudo apt install python3.12-dev # Debian/Ubuntu; match your version +``` + +…or use a uv-managed Python, which bundles them: + +```bash +uv python install 3.11 +uv sync --all-extras --python 3.11 +``` + +## `ImportError: chronowords is required for PMIEmbedder` + +`PMIEmbedder` depends on `chronowords` (it ships as a core dependency, but can be +absent in a stripped environment). Install it: + +```bash +uv add chronowords +``` + +`CountVectorizerEmbedder` and `TfidfEmbedder` have no such dependency — use them +if you only need count- or TF-IDF-based embeddings. + +## `ValueError: sklearn stopwords are only available for English` + +```python +get_stopwords("german") # ValueError +``` + +scikit-learn ships an English-only stopword list. For other languages, restrict +the sources to NLTK: + +```python +get_stopwords("german", sources=["nltk"]) +``` + +## `ValueError: Unsupported stopword sources` + +`get_stopwords(..., sources=[...])` accepts only `"nltk"` and `"sklearn"`. Any +other name (typo, wrong list) raises. Pass one or both of the supported names. + +## An empty or tiny backbone + +`extract_backbone` returns an **empty graph** when the input has no edges, and a +small backbone is expected for short inputs — the disparity filter is designed to +discard everything that isn't statistically significant. If you get fewer nodes +than you want: + +- Lower `min_alpha_ptile` (e.g. `0.3`) to keep more edges. +- Lower `min_degree` to `1` so weakly connected nodes survive. +- Use a larger corpus — the filter needs enough edges per node to judge + significance. + +## `NodeNotFound` when finding paths + +`networkx.shortest_path(graph, src, dst)` raises `NodeNotFound` if either word is +absent from the graph — usually because it was removed as a stopword, never +occurred, or fell below `min_weight`. Guard before querying: + +```python +if src in graph and dst in graph and nx.has_path(graph, src, dst): + path = nx.shortest_path(graph, src, dst, weight="distance") +``` + +## Loading a graph fails or behaves oddly + +`save_graph` / `load_graph` must use the **same `fmt`** on both ends. `graphml` +and `gml` round-trip attributes safely and are human-readable; +`pickle` is fastest but **executes arbitrary code on load** — never load a +pickle file from an untrusted source. Prefer `graphml` for anything you share. diff --git a/docs/tutorials/word_association_graph.md b/docs/tutorials/word_association_graph.md new file mode 100644 index 0000000..b03e7a7 --- /dev/null +++ b/docs/tutorials/word_association_graph.md @@ -0,0 +1,98 @@ +# Tutorial: word-association graphs + +A co-occurrence graph built from text is a kind of **word-association network**: +words that habitually appear together become connected. This tutorial builds one +from a small corpus, then walks the network to answer a concrete question: + +> How does the corpus "get from" one concept to another? + +Everything here uses the graph directly via [networkx](https://networkx.org/) — +a kenon graph *is* a `networkx.Graph`, so the entire networkx toolkit applies. + +## 1. Build the association graph + +```python +import networkx as nx +from kenon import Tokenizer, get_stopwords, build_cooccurrence_graph + +corpus = """ +Coffee contains caffeine, a stimulant that increases alertness and focus. +Caffeine blocks adenosine receptors in the brain, reducing the feeling of fatigue. +Many students drink coffee to stay awake while studying for exams. +Sleep deprivation harms memory and concentration, so studying tired is ineffective. +A good night of sleep consolidates memory and restores focus for the next day. +""" + +tokenizer = Tokenizer("en_core_web_sm", lemmatize=True) +tokens = tokenizer.flat_tokens(corpus) +stopwords = get_stopwords("english") + +graph = build_cooccurrence_graph(tokens, window=4, stopwords=stopwords) +print(f"graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges") +``` + +```text +graph: 32 nodes, 135 edges +``` + +## 2. Turn association strength into distance + +Co-occurrence edge weights are **association strength** — a *higher* weight means +two words are *more* closely associated. Shortest-path algorithms, on the other +hand, minimise total *distance*. Invert the weight so that strong associations +become short hops: + +```python +for _u, _v, data in graph.edges(data=True): + data["distance"] = 1.0 / data["weight"] +``` + +!!! warning "Don't pass raw co-occurrence weight to `shortest_path`" + Using `weight="weight"` would treat the *strongest* associations as the + *longest* detours. Always convert to a distance first. + +## 3. Find the path between two concepts + +```python +src, dst = "coffee", "memory" + +path = nx.shortest_path(graph, src, dst, weight="distance") +print("shortest association path:", " -> ".join(path)) + +print("a few alternative paths (<= 4 hops):") +for p in list(nx.all_simple_paths(graph, src, dst, cutoff=4))[:3]: + print(" ", " -> ".join(p)) +``` + +```text +shortest association path: coffee -> exam -> memory +a few alternative paths (<= 4 hops): + coffee -> contain -> caffeine -> focus -> memory + coffee -> contain -> stimulant -> focus -> memory + coffee -> contain -> increase -> focus -> memory +``` + +The shortest path is the corpus's most direct line of reasoning from `coffee` to +`memory`; the simple paths expose the alternative routes (via `caffeine` and +`focus`). On a denser corpus, run `extract_backbone` first to strip noise edges +before pathfinding, so the paths follow only statistically significant links. + +!!! note "Missing nodes raise `NodeNotFound`" + `nx.shortest_path` raises if a concept was filtered out as a stopword, never + occurred, or fell below `min_weight`. Guard with `src in graph and dst in graph`, + and check `nx.has_path(graph, src, dst)` before asking for a path. + +## 4. Comparing against human association norms + +A natural next question is whether a *text-derived* association network matches +how *people* associate words — as captured by human free-association norms such +as the [Nelson norms](http://w3.usf.edu/FreeAssociation/) or the +[Small World of Words](https://smallworldofwords.org/) (SWOW) project. The +comparison is conceptually straightforward: build the kenon graph, load the norm +graph, align the shared vocabulary, and compare neighbourhoods (e.g. rank +correlation of edge weights, or overlap of each word's top associates). + +kenon does not yet ship a loader for these external datasets or a built-in graph +comparison helper — both are tracked as proposed features (see `CHANGES_SUMMARY.md` +in the repository). For now you can compare two graphs with networkx directly, as +shown in the [Comparing two texts](../examples/comparing_two_texts.md) example. diff --git a/mkdocs.yml b/mkdocs.yml index 6b6cf4f..f4c10c0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,10 @@ site_url: https://kenon.readthedocs.io repo_url: https://github.com/crow-intelligence/kenon repo_name: crow-intelligence/kenon +# Fail the build on any warning (broken links, missing autodoc targets, dead +# snippet paths). Keeps the published docs honest. +strict: true + extra: social: - icon: fontawesome/solid/globe @@ -14,6 +18,9 @@ theme: name: material logo: assets/kenon.svg favicon: assets/kenon.svg + features: + - navigation.sections + - content.code.copy plugins: - search @@ -24,17 +31,36 @@ plugins: show_source: true show_root_heading: true +markdown_extensions: + - admonition + - attr_list + - tables + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.superfences + # Pull the verified example scripts into the docs so they never drift. + - pymdownx.snippets: + base_path: ["."] + check_paths: true + nav: - Getting Started: - index.md - quickstart.md + - Tutorials: + - tutorials/word_association_graph.md + - Examples: + - examples/news_article_analysis.md + - examples/comparing_two_texts.md + - examples/multilingual_analysis.md + - Troubleshooting: troubleshooting.md - API Reference: - api/tokenizer.md + - api/stopwords.md - api/embeddings.md - api/cooccurrence.md - api/graphs.md - api/backbone.md - - Examples: - - examples/news_article_analysis.md - - examples/comparing_two_texts.md - - examples/multilingual_analysis.md diff --git a/src/kenon/backbone.py b/src/kenon/backbone.py index 4e19f6e..901b951 100644 --- a/src/kenon/backbone.py +++ b/src/kenon/backbone.py @@ -31,7 +31,7 @@ def disparity_integral(x: float, k: float) -> float: - ``x`` must not equal 1.0 (division by zero). - ``k`` must not equal 1.0 (division by zero). - Example: + Examples: >>> abs(disparity_integral(0.5, 3.0) - disparity_integral(0.0, 3.0)) > 0 True """ @@ -52,7 +52,7 @@ def get_disparity_significance(norm_weight: float, degree: float) -> float: - If ``degree`` <= 1, returns 0.0. - Result is clipped to [0, 1]. - Example: + Examples: >>> alpha = get_disparity_significance(0.5, 3.0) >>> 0.0 <= alpha <= 1.0 True @@ -86,7 +86,7 @@ def apply_disparity_filter(graph: SemanticGraph) -> list[float]: - Every edge gets ``norm_weight``, ``alpha``, and ``alpha_ptile`` attributes. - Every node gets a ``strength`` attribute. - Example: + Examples: >>> import networkx as nx >>> g = nx.Graph() >>> g.add_edge("a", "b", weight=0.8) @@ -162,7 +162,7 @@ def extract_backbone( - Result has <= nodes and <= edges compared to the original. - All remaining nodes satisfy ``degree >= min_degree``. - Example: + Examples: >>> import networkx as nx >>> g = nx.path_graph(5) >>> for u, v in g.edges(): diff --git a/src/kenon/cooccurrence.py b/src/kenon/cooccurrence.py index 250bea4..31ba267 100644 --- a/src/kenon/cooccurrence.py +++ b/src/kenon/cooccurrence.py @@ -45,7 +45,7 @@ def build_cooccurrence_graph( - All edge weights are positive. - Stopword filtering happens before counting. - Example: + Examples: >>> tokens = ["cat", "sat", "mat", "cat", "mat"] >>> g = build_cooccurrence_graph(tokens, window=1) >>> g.has_node("cat") @@ -116,7 +116,7 @@ def detect_collocations( - Returns at most ``top_n`` tuples. - Each tuple has length ``n``. - Example: + Examples: >>> tokens = ["new", "york", "city", "new", "york", "times"] * 10 >>> colls = detect_collocations(tokens, n=2, top_n=5) >>> ("new", "york") in colls diff --git a/src/kenon/embeddings.py b/src/kenon/embeddings.py index ece39c6..a582dd5 100644 --- a/src/kenon/embeddings.py +++ b/src/kenon/embeddings.py @@ -54,7 +54,7 @@ class CountVectorizerEmbedder: - ``vocabulary`` raises ``RuntimeError`` if accessed before ``fit``. - Output matrix dtype is always float64. - Example: + Examples: >>> emb = CountVectorizerEmbedder() >>> mat = emb.fit_transform(["the cat sat", "the dog ran"]) >>> mat.shape[0] @@ -136,7 +136,7 @@ class TfidfEmbedder: - ``vocabulary`` raises ``RuntimeError`` if accessed before ``fit``. - Output matrix dtype is always float64. - Example: + Examples: >>> emb = TfidfEmbedder() >>> mat = emb.fit_transform(["the cat sat", "the dog ran"]) >>> mat.shape[0] @@ -235,7 +235,7 @@ class returns word-level embeddings (vocab x n_components), not ``(len(vocabulary), n_components)``. - The embedder is serialisable via ``pickle``. - Example: + Examples: >>> emb = PMIEmbedder(n_components=50, window=3) >>> corpus = ["the cat sat on the mat", "the dog ran on the road"] >>> mat = emb.fit_transform(corpus) diff --git a/src/kenon/graphs.py b/src/kenon/graphs.py index a7c844d..bf6eb07 100644 --- a/src/kenon/graphs.py +++ b/src/kenon/graphs.py @@ -60,7 +60,7 @@ def build_semantic_graph( - All edge weights are in [0, 1]. - Node labels are vocabulary token strings. - Example: + Examples: >>> from kenon.embeddings import TfidfEmbedder >>> emb = TfidfEmbedder() >>> corpus = ["cat mat sat", "dog ran fast", "cat ran fast"] * 5 @@ -133,7 +133,7 @@ def cosine_similarity_matrix( - Matrix is symmetric. - All values are in [-1, 1]. - Example: + Examples: >>> from kenon.embeddings import TfidfEmbedder >>> emb = TfidfEmbedder() >>> corpus = ["cat mat", "dog ran"] * 3 @@ -174,7 +174,7 @@ def save_graph( - The file is written atomically (or as atomically as the format allows). - ``"graphml"`` and ``"gml"`` produce human-readable output. - Example: + Examples: >>> import tempfile, os, networkx as nx >>> g = nx.Graph(); g.add_edge("a", "b", weight=0.5) >>> with tempfile.NamedTemporaryFile(suffix=".graphml", delete=False) as f: @@ -203,6 +203,11 @@ def load_graph( ) -> SemanticGraph: """Load a graph from disk. + Warning: + The ``"pickle"`` format executes arbitrary code on load and must + **never** be used with files from an untrusted source. Prefer + ``"graphml"`` or ``"gml"`` for any graph you did not write yourself. + Args: path: Source file path. fmt: Format string. Must match the format used when saving. @@ -217,7 +222,7 @@ def load_graph( - Loaded graph preserves all node and edge attributes from the original. - ``"pickle"`` format is not safe for untrusted files. - Example: + Examples: >>> import tempfile, networkx as nx >>> g = nx.Graph(); g.add_edge("x", "y", weight=0.9) >>> with tempfile.NamedTemporaryFile(suffix=".graphml", delete=False) as f: diff --git a/src/kenon/stopwords.py b/src/kenon/stopwords.py index b1a4cbe..91c8187 100644 --- a/src/kenon/stopwords.py +++ b/src/kenon/stopwords.py @@ -45,7 +45,7 @@ def get_stopwords( - ``extra`` words are always present in the result. - NLTK data is auto-downloaded if missing. - Example: + Examples: >>> sw = get_stopwords("english") >>> "the" in sw True diff --git a/src/kenon/tokenizer.py b/src/kenon/tokenizer.py index 15809d2..fd4bf8e 100644 --- a/src/kenon/tokenizer.py +++ b/src/kenon/tokenizer.py @@ -41,7 +41,7 @@ class Tokenizer: - All methods accept ``str`` inputs only, never file paths. - Pure whitespace and punctuation tokens are excluded by default. - Example: + Examples: >>> t = Tokenizer("en_core_web_sm") >>> sents = t.sentencize("The cat sat. The dog ran.") >>> len(sents) @@ -150,7 +150,7 @@ def sentencize(self, text: str) -> list[str]: - Never returns empty strings in the output list. - Sentence boundaries are determined by spaCy's sentence segmenter. - Example: + Examples: >>> t = Tokenizer("en_core_web_sm") >>> sents = t.sentencize("Hello world. Goodbye world.") >>> len(sents) == 2 @@ -179,7 +179,7 @@ def tokenize(self, text: str, *, keep_punct: bool = False) -> Document: - Whitespace-only tokens are always excluded. - Punctuation tokens excluded unless ``keep_punct=True``. - Example: + Examples: >>> t = Tokenizer("en_core_web_sm", lemmatize=True) >>> doc = t.tokenize("The cats were running.") >>> "cat" in doc[0] @@ -217,7 +217,7 @@ def flat_tokens(self, text: str, *, keep_punct: bool = False) -> list[Token]: - All returned tokens are substrings of the original text (possibly lowercased or lemmatised). - Example: + Examples: >>> t = Tokenizer("en_core_web_sm") >>> tokens = t.flat_tokens("The cat sat on the mat.") >>> "cat" in tokens