diff --git a/README.md b/README.md
index 5686e6f..4fddcca 100644
--- a/README.md
+++ b/README.md
@@ -16,10 +16,16 @@
```bash
-pip install cognis-tokenmeter
+pip install "git+https://github.com/cognis-digital/tokenmeter.git"
tokenmeter scan . # → prioritized findings in seconds
```
+
+## What is this?
+
+Tokenmeter is a command-line tool that tells you how many tokens your text will use and how much it will cost when sent to an AI model like GPT-4 or Claude. You give it text or a file, choose a model, and it instantly calculates the token count and estimated price in US dollars. It also lets you set a budget limit so your scripts or CI pipelines automatically fail if a request would cost too much. It is designed for developers who want to track and control AI API spending without surprises.
+
+
## Contents
- [Why tokenmeter?](#why) · [Features](#features) · [Quick start](#quick-start) · [Example](#example) · [Architecture](#architecture) · [AI stack](#ai-stack) · [How it compares](#how-it-compares) · [Integrations](#integrations) · [Install anywhere](#install-anywhere) · [Related](#related) · [Contributing](#contributing)
@@ -49,10 +55,56 @@ AI cost control
+
+## Domains
+
+**Primary domain:** Finance & Quant · **JTF MERIDIAN division:** BLACKBOOK · ORACLE
+
+**Topics:** `cognis` `finance` `fintech` `quant`
+
+Part of the **Cognis Neural Suite** — 300+ source-available tools organized across 12 domains under the JTF MERIDIAN command structure. See the [suite on GitHub](https://github.com/cognis-digital) and [jtf-meridian](https://github.com/cognis-digital/jtf-meridian) for how the pieces fit together.
+
+
+
+## Install
+
+`tokenmeter` is source-available (not published to PyPI) — every method below installs
+straight from GitHub. Pick whichever you prefer; the one-line scripts auto-detect
+the best tool available on your machine.
+
+**One-liner (Linux / macOS):**
+```sh
+curl -fsSL https://raw.githubusercontent.com/cognis-digital/tokenmeter/HEAD/install.sh | sh
+```
+
+**One-liner (Windows PowerShell):**
+```powershell
+irm https://raw.githubusercontent.com/cognis-digital/tokenmeter/HEAD/install.ps1 | iex
+```
+
+**Or install manually — any one of:**
+```sh
+pipx install "git+https://github.com/cognis-digital/tokenmeter.git" # isolated (recommended)
+uv tool install "git+https://github.com/cognis-digital/tokenmeter.git" # uv
+pip install "git+https://github.com/cognis-digital/tokenmeter.git" # pip
+```
+
+**From source:**
+```sh
+git clone https://github.com/cognis-digital/tokenmeter.git
+cd tokenmeter && pip install .
+```
+
+Then run:
+```sh
+tokenmeter --help
+```
+
+
## Quick start
```bash
-pip install cognis-tokenmeter
+pip install "git+https://github.com/cognis-digital/tokenmeter.git"
tokenmeter --version
tokenmeter scan . # scan current project
tokenmeter scan . --format json # machine-readable
diff --git a/install.ps1 b/install.ps1
new file mode 100644
index 0000000..385300c
--- /dev/null
+++ b/install.ps1
@@ -0,0 +1,29 @@
+# Comprehensive installer for cognis-digital/tokenmeter (Windows PowerShell).
+# Tries: pipx -> uv -> pip (git+https) -> from source.
+# tokenmeter is source-available and not on PyPI; all paths install from GitHub.
+$ErrorActionPreference = "Stop"
+$Repo = "tokenmeter"
+$Url = "git+https://github.com/cognis-digital/tokenmeter.git"
+$Git = "https://github.com/cognis-digital/tokenmeter.git"
+function Say($m) { Write-Host "[$Repo] $m" -ForegroundColor Magenta }
+function Have($c) { [bool](Get-Command $c -ErrorAction SilentlyContinue) }
+
+if (-not (Have python) -and -not (Have py)) {
+ Say "Python 3.9+ is required but was not found. Install Python first."; exit 1
+}
+if (Have pipx) {
+ Say "Installing with pipx (isolated, recommended)..."
+ pipx install $Url; if ($LASTEXITCODE -eq 0) { Say "Done. Run: tokenmeter"; exit 0 }
+}
+if (Have uv) {
+ Say "Installing with uv..."
+ uv tool install $Url; if ($LASTEXITCODE -eq 0) { Say "Done. Run: tokenmeter"; exit 0 }
+}
+if (Have pip) {
+ Say "Installing with pip (user site)..."
+ pip install --user $Url; if ($LASTEXITCODE -eq 0) { Say "Done. Run: tokenmeter"; exit 0 }
+}
+Say "No packaging tool worked; falling back to a source clone."
+$Tmp = Join-Path $env:TEMP "$Repo-src"
+git clone --depth 1 $Git $Tmp
+Say "Cloned to $Tmp - run: cd $Tmp; python -m pip install ."
diff --git a/install.sh b/install.sh
index 6d623eb..cfa2309 100644
--- a/install.sh
+++ b/install.sh
@@ -1,10 +1,34 @@
-#!/usr/bin/env sh
-# Universal installer for tokenmeter. Prefers uv > pipx > pip; installs from the repo.
-set -e
-SRC="git+https://github.com/cognis-digital/tokenmeter.git"
-echo "Installing tokenmeter ..."
-if command -v uv >/dev/null 2>&1; then uv tool install "$SRC"
-elif command -v pipx >/dev/null 2>&1; then pipx install "$SRC"
-elif command -v python3 >/dev/null 2>&1; then python3 -m pip install --user "$SRC"
-else echo "Need uv, pipx, or python3+pip"; exit 1; fi
-echo "Done. Run: tokenmeter --help"
+#!/usr/bin/env sh
+# Comprehensive installer for cognis-digital/tokenmeter (Linux / macOS).
+# Tries the best available method: pipx -> uv -> pip (git+https) -> from source.
+# tokenmeter is source-available and not on PyPI; all paths install from GitHub.
+set -eu
+
+REPO="tokenmeter"
+URL="git+https://github.com/cognis-digital/tokenmeter.git"
+GITURL="https://github.com/cognis-digital/tokenmeter.git"
+
+say() { printf '\033[1;35m[%s]\033[0m %s\n' "$REPO" "$1"; }
+have() { command -v "$1" >/dev/null 2>&1; }
+
+if ! have python3 && ! have python; then
+ say "Python 3.9+ is required but was not found. Install Python first."; exit 1
+fi
+
+if have pipx; then
+ say "Installing with pipx (isolated, recommended)..."
+ pipx install "$URL" && { say "Done. Run: tokenmeter"; exit 0; }
+fi
+if have uv; then
+ say "Installing with uv..."
+ uv tool install "$URL" && { say "Done. Run: tokenmeter"; exit 0; }
+fi
+if have pip3 || have pip; then
+ PIP="$(command -v pip3 || command -v pip)"
+ say "Installing with pip (user site)..."
+ "$PIP" install --user "$URL" && { say "Done. Run: tokenmeter"; exit 0; }
+fi
+
+say "No packaging tool worked; falling back to a source clone."
+TMP="$(mktemp -d)"; git clone --depth 1 "$GITURL" "$TMP/$REPO"
+say "Cloned to $TMP/$REPO — run: cd $TMP/$REPO && python3 -m pip install ."
diff --git a/integrations/webhook.py b/integrations/webhook.py
index 91e0211..3fdf48f 100644
--- a/integrations/webhook.py
+++ b/integrations/webhook.py
@@ -5,21 +5,50 @@
Usage: scan . --format json | python integrations/webhook.py --url URL
"""
from __future__ import annotations
-import argparse, json, sys, urllib.request
+import argparse
+import sys
+import urllib.request
def main() -> int:
- ap = argparse.ArgumentParser()
- ap.add_argument("--url", required=True)
+ ap = argparse.ArgumentParser(
+ description="Forward JSON findings from stdin to a webhook URL."
+ )
+ ap.add_argument("--url", required=True, help="Destination URL (http/https)")
ap.add_argument("--header", action="append", default=[], help="Key: Value")
+ ap.add_argument(
+ "--timeout",
+ type=float,
+ default=15.0,
+ help="Request timeout in seconds (default: 15)",
+ )
args = ap.parse_args()
+
+ if not args.url.startswith(("http://", "https://")):
+ print(
+ f"error: --url must start with http:// or https://, got {args.url!r}",
+ file=sys.stderr,
+ )
+ return 2
+
+ if args.timeout <= 0:
+ print("error: --timeout must be > 0", file=sys.stderr)
+ return 2
+
payload = sys.stdin.read().encode("utf-8")
+ if not payload:
+ print("error: no data on stdin — nothing to post", file=sys.stderr)
+ return 2
+
req = urllib.request.Request(args.url, data=payload, method="POST")
req.add_header("Content-Type", "application/json")
for h in args.header:
k, _, v = h.partition(":")
+ if not k.strip():
+ print(f"error: malformed --header value {h!r}", file=sys.stderr)
+ return 2
req.add_header(k.strip(), v.strip())
try:
- with urllib.request.urlopen(req, timeout=15) as r:
+ with urllib.request.urlopen(req, timeout=args.timeout) as r:
print(f"posted {len(payload)} bytes -> {r.status}")
return 0
except Exception as e:
diff --git a/layman.md b/layman.md
new file mode 100644
index 0000000..7092bab
--- /dev/null
+++ b/layman.md
@@ -0,0 +1 @@
+Tokenmeter is a command-line tool that tells you how many tokens your text will use and how much it will cost when sent to an AI model like GPT-4 or Claude. You give it text or a file, choose a model, and it instantly calculates the token count and estimated price in US dollars. It also lets you set a budget limit so your scripts or CI pipelines automatically fail if a request would cost too much. It is designed for developers who want to track and control AI API spending without surprises.
diff --git a/tests/test_smoke.py b/tests/test_smoke.py
index 7b8417f..053e9cd 100644
--- a/tests/test_smoke.py
+++ b/tests/test_smoke.py
@@ -1,16 +1,105 @@
-"""Smoke tests for TOKENMETER."""
-from tokenmeter.core import scan, TOOL_NAME, TOOL_VERSION
-
-def test_identity():
- assert TOOL_NAME and TOOL_VERSION
-
-def test_scan_runs(tmp_path):
- f = tmp_path / "x.txt"
- f.write_text("a TODO here\nFIXME later\n")
- res = scan(str(tmp_path))
- assert res.score >= 0
- assert any("TODO" in fi.title or "FIXME" in fi.title for fi in res.findings)
-
-def test_cli_importable():
- from tokenmeter.cli import main
- assert callable(main)
+"""Smoke tests for TOKENMETER."""
+import contextlib
+import io
+import pytest
+
+from tokenmeter.core import TOOL_NAME, TOOL_VERSION, add_model, count_tokens, estimate
+from tokenmeter import cli
+
+
+def test_identity():
+ assert TOOL_NAME and TOOL_VERSION
+
+
+def test_count_tokens_basic(tmp_path):
+ # Write a file with known text and confirm token counting works on the text.
+ f = tmp_path / "x.txt"
+ f.write_text("hello world this is a test\n")
+ text = f.read_text()
+ result = count_tokens(text)
+ assert result >= 1
+ assert isinstance(result, int)
+
+
+def test_cli_importable():
+ assert callable(cli.main)
+
+
+# ---------------------------------------------------------------------------
+# Hardening: edge cases and invalid inputs
+# ---------------------------------------------------------------------------
+
+def _run_cli(argv):
+ """Run CLI and capture stdout; return (exit_code, stdout_text)."""
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ code = cli.main(argv)
+ return code, out.getvalue()
+
+
+def test_count_tokens_none_safe():
+ # count_tokens must not crash on None; treat as empty string.
+ assert count_tokens(None) == 0 # type: ignore[arg-type]
+
+
+def test_count_tokens_empty():
+ assert count_tokens("") == 0
+
+
+def test_add_model_negative_price_raises():
+ with pytest.raises(ValueError, match="input_per_1k"):
+ add_model("bad-price", -0.001, 0.002, 4096)
+
+
+def test_add_model_zero_context_raises():
+ with pytest.raises(ValueError, match="context_window"):
+ add_model("bad-ctx", 0.001, 0.002, 0)
+
+
+def test_add_model_empty_name_raises():
+ with pytest.raises(ValueError, match="name"):
+ add_model("", 0.001, 0.002, 4096)
+
+
+def test_estimate_negative_output_tokens_raises():
+ with pytest.raises(ValueError, match="output_tokens"):
+ estimate("hello", model="claude-sonnet", output_tokens=-1)
+
+
+def test_estimate_negative_input_tokens_raises():
+ with pytest.raises(ValueError, match="input_tokens"):
+ estimate("", model="claude-sonnet", input_tokens=-5)
+
+
+def test_cli_missing_file_returns_exit_2(tmp_path):
+ # Pointing --file at a nonexistent path should exit 2 (not traceback).
+ missing = str(tmp_path / "does_not_exist.txt")
+ with pytest.raises(SystemExit) as exc_info:
+ _run_cli(["count", "-f", missing])
+ assert exc_info.value.code == 2
+
+
+def test_cli_negative_output_tokens_returns_exit_2():
+ code, _ = _run_cli(["count", "-t", "hello", "-o", "-5"])
+ assert code == 2
+
+
+def test_cli_budget_negative_max_cost_returns_exit_2():
+ code, _ = _run_cli(["budget", "-t", "hello", "--max-cost", "-1.0"])
+ assert code == 2
+
+
+def test_cli_budget_negative_max_tokens_returns_exit_2():
+ code, _ = _run_cli(["budget", "-t", "hello", "--max-tokens", "-10"])
+ assert code == 2
+
+
+def test_cli_unknown_model_returns_exit_2():
+ code, _ = _run_cli(["count", "-t", "hello", "-m", "not-a-real-model"])
+ assert code == 2
+
+
+def test_mcp_server_importable():
+ # mcp_server must import cleanly (the broken scan/to_json import is fixed).
+ from tokenmeter import mcp_server # noqa: F401
+ assert callable(mcp_server.serve)
diff --git a/tokenmeter/cli.py b/tokenmeter/cli.py
index 3bf09f8..edf4f19 100644
--- a/tokenmeter/cli.py
+++ b/tokenmeter/cli.py
@@ -12,7 +12,6 @@
aggregate,
check_budget,
estimate,
- get_pricing,
list_models,
)
@@ -21,8 +20,12 @@ def _read_input(args: argparse.Namespace) -> str:
if getattr(args, "text", None) is not None:
return args.text
if getattr(args, "file", None):
- with open(args.file, "r", encoding="utf-8", errors="replace") as fh:
- return fh.read()
+ try:
+ with open(args.file, "r", encoding="utf-8", errors="replace") as fh:
+ return fh.read()
+ except OSError as exc:
+ print(f"error: cannot read file {args.file!r}: {exc}", file=sys.stderr)
+ raise SystemExit(2) from exc
# No explicit source: read stdin if piped.
if not sys.stdin.isatty():
return sys.stdin.read()
@@ -35,7 +38,10 @@ def _emit(payload: dict, fmt: str, rows: Optional[List[tuple]] = None) -> None:
return
# table
if rows is not None:
- widths = [max(len(str(r[i])) for r in rows) for i in range(len(rows[0]))]
+ if not rows:
+ return
+ ncols = len(rows[0])
+ widths = [max(len(str(r[i])) for r in rows) for i in range(ncols)]
for r in rows:
print(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(r)))
return
@@ -44,6 +50,9 @@ def _emit(payload: dict, fmt: str, rows: Optional[List[tuple]] = None) -> None:
def _cmd_count(args: argparse.Namespace) -> int:
+ if args.output_tokens < 0:
+ print("error: --output-tokens must be >= 0", file=sys.stderr)
+ return 2
text = _read_input(args)
est = estimate(
text,
@@ -67,6 +76,15 @@ def _cmd_count(args: argparse.Namespace) -> int:
def _cmd_budget(args: argparse.Namespace) -> int:
+ if args.output_tokens < 0:
+ print("error: --output-tokens must be >= 0", file=sys.stderr)
+ return 2
+ if args.max_cost is not None and args.max_cost < 0:
+ print("error: --max-cost must be >= 0", file=sys.stderr)
+ return 2
+ if args.max_tokens is not None and args.max_tokens < 0:
+ print("error: --max-tokens must be >= 0", file=sys.stderr)
+ return 2
text = _read_input(args)
est = estimate(text, model=args.model, output_tokens=args.output_tokens)
result = check_budget(
diff --git a/tokenmeter/core.py b/tokenmeter/core.py
index a5acc72..7b40d64 100644
--- a/tokenmeter/core.py
+++ b/tokenmeter/core.py
@@ -17,6 +17,9 @@
from dataclasses import dataclass, field
from typing import Dict, Iterable, List, Optional
+TOOL_NAME: str = "tokenmeter"
+TOOL_VERSION: str = "0.7.7"
+
@dataclass(frozen=True)
class ModelPricing:
@@ -51,7 +54,18 @@ def add_model(
name: str, input_per_1k: float, output_per_1k: float, context_window: int = 8192
) -> ModelPricing:
"""Register or override a model's pricing at runtime."""
- p = ModelPricing(name, float(input_per_1k), float(output_per_1k), int(context_window))
+ if not name or not isinstance(name, str):
+ raise ValueError("model name must be a non-empty string")
+ in_price = float(input_per_1k)
+ out_price = float(output_per_1k)
+ ctx = int(context_window)
+ if in_price < 0:
+ raise ValueError(f"input_per_1k must be >= 0, got {in_price}")
+ if out_price < 0:
+ raise ValueError(f"output_per_1k must be >= 0, got {out_price}")
+ if ctx <= 0:
+ raise ValueError(f"context_window must be > 0, got {ctx}")
+ p = ModelPricing(name, in_price, out_price, ctx)
MODELS[name] = p
return p
@@ -84,6 +98,8 @@ def count_tokens(text: str) -> int:
BPE tends to split numbers), min 1.
* Single punctuation/symbol: 1 token.
"""
+ if text is None:
+ return 0
if not text:
return 0
@@ -145,7 +161,11 @@ def estimate(
"""
pricing = get_pricing(model)
in_tok = count_tokens(text) if input_tokens is None else int(input_tokens)
+ if in_tok < 0:
+ raise ValueError(f"input_tokens must be >= 0, got {in_tok}")
out_tok = int(output_tokens)
+ if out_tok < 0:
+ raise ValueError(f"output_tokens must be >= 0, got {out_tok}")
in_cost = in_tok / 1000.0 * pricing.input_per_1k
out_cost = out_tok / 1000.0 * pricing.output_per_1k
used = in_tok + out_tok
diff --git a/tokenmeter/mcp_server.py b/tokenmeter/mcp_server.py
index e7e3c78..6990ad4 100644
--- a/tokenmeter/mcp_server.py
+++ b/tokenmeter/mcp_server.py
@@ -1,6 +1,7 @@
-"""TOKENMETER MCP server — exposes scan() as an MCP tool for Cognis.Studio."""
+"""TOKENMETER MCP server — exposes estimate() as an MCP tool for Cognis.Studio."""
from __future__ import annotations
-from tokenmeter.core import scan, to_json
+import json
+import sys
def serve() -> int:
"""Start an MCP stdio server. Requires the optional 'mcp' extra:
@@ -8,15 +9,28 @@ def serve() -> int:
"""
try:
from mcp.server.fastmcp import FastMCP
- except Exception:
- print("Install the MCP extra: pip install 'cognis-tokenmeter[mcp]'")
+ except ImportError:
+ print(
+ "error: MCP extra not installed. Run: pip install 'cognis-tokenmeter[mcp]'",
+ file=sys.stderr,
+ )
return 1
+ from tokenmeter.core import estimate
+
app = FastMCP("tokenmeter")
@app.tool()
- def tokenmeter_scan(target: str) -> str:
- """Token and cost counter / budgeter for LLM apps, CI-ready. Returns JSON findings."""
- return to_json(scan(target))
+ def tokenmeter_estimate(
+ text: str,
+ model: str = "claude-sonnet",
+ output_tokens: int = 0,
+ ) -> str:
+ """Estimate token count and cost for the given text. Returns JSON."""
+ try:
+ est = estimate(text, model=model, output_tokens=output_tokens)
+ except (KeyError, ValueError) as exc:
+ return json.dumps({"error": str(exc)})
+ return json.dumps(est.to_dict())
app.run()
return 0