diff --git a/.github/workflows/ingest-smoke.yml b/.github/workflows/ingest-smoke.yml new file mode 100644 index 0000000..c235068 --- /dev/null +++ b/.github/workflows/ingest-smoke.yml @@ -0,0 +1,75 @@ +name: Ingest smoke (dev S3) + +# Separate from PR CI: proves GitHub OIDC → AWS dev bucket access. +# Requires GitHub Environment "dev" with repository variables (see docs/mvp/guides/cloud-foundation.md). + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" + +permissions: + id-token: write + contents: read + +jobs: + ingest-smoke: + runs-on: ubuntu-latest + environment: dev + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Write smoke marker to dev lake raw zone + env: + DEV_LAKE_BUCKET: ${{ vars.DEV_LAKE_BUCKET }} + DEV_LAKE_PREFIX: ${{ vars.DEV_LAKE_PREFIX }} + run: | + set -euo pipefail + if [ -z "${DEV_LAKE_BUCKET:-}" ]; then + echo "Missing DEV_LAKE_BUCKET. See docs/mvp/guides/cloud-foundation.md" + exit 1 + fi + prefix="${DEV_LAKE_PREFIX:-}" + key="${prefix}raw/_smoke/ingest-smoke.txt" + echo "smartwealthai ingest-smoke $(date -u +%Y-%m-%dT%H:%M:%SZ)" > smoke.txt + aws s3 cp smoke.txt "s3://${DEV_LAKE_BUCKET}/${key}" + + - name: Validate smoke object + env: + DEV_LAKE_BUCKET: ${{ vars.DEV_LAKE_BUCKET }} + DEV_LAKE_PREFIX: ${{ vars.DEV_LAKE_PREFIX }} + run: | + set -euo pipefail + prefix="${DEV_LAKE_PREFIX:-}" + key="${prefix}raw/_smoke/ingest-smoke.txt" + aws s3api head-object --bucket "${DEV_LAKE_BUCKET}" --key "${key}" + aws s3 cp "s3://${DEV_LAKE_BUCKET}/${key}" - + echo + echo "ingest-smoke OK: s3://${DEV_LAKE_BUCKET}/${key}" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Poetry + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a + with: + virtualenvs-in-project: true + + - name: Validate lake root URI parsing (no live S3 I/O) + run: | + poetry install --with dev + poetry run python -c " + from smartwealthai.lake_root import resolve_lake_root + root = resolve_lake_root('s3://${{ vars.DEV_LAKE_BUCKET }}/${{ vars.DEV_LAKE_PREFIX }}') + assert root.backend == 's3' + print('lake root:', root.uri, 'bucket=', root.s3_bucket) + " diff --git a/CONTEXT.md b/CONTEXT.md index abc9705..dcd9c09 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -77,12 +77,40 @@ _Avoid_: dividend yield, earnings/price without EV _Avoid_: market cap alone as “value” **Combined rank**: -Sum of ROC rank and EY rank; lower is better. Used for portfolio selection and sell-watch opportunity cost. -_Avoid_: average of ranks, z-score blend (not MVP) +Sum of ROC rank and EY rank; lower is better. **Benchmark only** (Magic Formula replica) — not production portfolio selection after Phase 2. Sell-watch opportunity-cost triggers on MF path are superseded by QV triggers in production. +_Avoid_: production ranking term post–Phase 2; average of ranks, z-score blend **Magic Formula replica**: -Canonical benchmark portfolio using the same ROC, EY, combined rank, universe, and annual rebalance as production. Used to gate backtest pass vs strategy Sharpe. -_Avoid_: live Greenblatt fund, generic “value factor” +Canonical **benchmark** portfolio using ROC, EY, combined rank, universe, and annual rebalance. Used to gate backtest pass vs strategy Sharpe. **Not** the Phase 2+ production scoring path. +_Avoid_: live Greenblatt fund, generic “value factor”, conflating with Quantitative Value production + +**Quantitative Value (QV) funnel**: +Phase 2+ **production** scoring: universe → forensic hard exclusion (incl. Beneish bottom-5%) → EBIT/TEV value decile → FS-Score quality screen → ~50-name equal-weight model portfolio. Spec: `docs/mvp/features/quantitative-value.md`. +_Avoid_: Magic Formula path, ROC+EY combined rank for production + +**Quality (production)**: +**FS-Score** composite (0–10, Gray/Carlisle variant) on the EBIT/TEV value pool — not ROC rank alone. +_Avoid_: ROC rank as production quality after Phase 2; ESG or subjective moat + +**Cheap (production)**: +Membership in the **EBIT/TEV value pool** (top decile among forensic survivors) — not EY rank alone. +_Avoid_: EY rank as production cheapness after Phase 2; low P/E without EV + +**EBIT/TEV**: +`EBIT / Enterprise Value`; value-screen metric for QV production. Same EV definition as **Earnings yield**; ranked within forensic survivors to form the value pool. Spec: `docs/mvp/features/quantitative-value.md`. +_Avoid_: MF EY rank, market cap alone + +**FS-Score**: +Ten binary financial-strength components (profitability, stability, recent operational improvements) summed to 0–10. Production quality factor after Phase 2. `formula_version` on every scored row. +_Avoid_: Piotroski F-Score (different formula), ROC as production quality + +**QV funnel rank**: +Order within the value pool after the FS-Score quality screen; determines portfolio membership. Supersedes **combined rank** for production. +_Avoid_: combined rank, ROC rank + EY rank for production selection + +**Forensic evaluator**: +Hard `exclude` / `pass` before value or quality scoring; distress and fraud rules from permanent-loss filter plus Beneish M-Score bottom-5% gate. Every exclusion carries `rule_id`, `rule_version`, `triggered_value`, `threshold`, `explanation`. +_Avoid_: soft penalty, scoring before forensics **Cross-sectional rank**: Rank across all passing companies on one run date. Not comparable across dates without re-running the pipeline. @@ -93,7 +121,7 @@ Version id for ROC, EY, or filter rules so runs and backtests stay reproducible. _Avoid_: “latest formula”, implicit default **Model portfolio**: -Target long-only holdings from the pipeline; **June 30 demo:** top 30 names by combined rank, equal-weight only, market-cap tie-break on ranks. No watchlist in demo slice. Paper-traded in full MVP (phase 2). +Target long-only holdings from the pipeline. **June 30 demo:** top 30 by MF combined rank, equal-weight, market-cap tie-break. **Phase 2+ production (QV):** ~50 names by FS-Score within the EBIT/TEV value pool, equal-weight, market-cap tie-break (configurable cap). Paper-traded in full MVP (phase 2). _Avoid_: personal portfolio, watchlist (demo slice) **Watchlist**: @@ -130,7 +158,9 @@ _Avoid_: ad-hoc snapshot without run id ## Relationships -- A **run date** drives **universe** → **permanent loss filter** → **ROC** and **EY** ranks → **combined rank** → **model portfolio** +- **Demo:** **run date** → **universe** → **ROC** and **EY** ranks → **combined rank** → **model portfolio** (top 30) +- **Phase 2+ production:** **run date** → **universe** → **forensic evaluator** → **EBIT/TEV value pool** → **FS-Score** → **QV funnel rank** → **model portfolio** (~50) +- **Benchmark (all phases):** MF replica path (ROC + EY + combined rank) for backtest Sharpe gate — parallel to production, not mixed into QV funnel - **As-of date** tags each fundamental row; PIT queries filter `as_of_date <= run_date` - **Watchlist** superset of names that may enter the **model portfolio** on rebalance - **Magic Formula replica** is the strategy’s primary benchmark comparator for Sharpe pass/fail @@ -142,11 +172,12 @@ Resolved scope cuts (see ADRs and [`docs/mvp/demo-slice.md`](docs/mvp/demo-slice - **June 30 demo MVP:** SimFin bulk US → raw → normalizer → **universe (US market)** → ROC/EY → combined rank → top-30 EW model portfolio → Streamlit dashboard. No permanent loss filter, backtest, sell-watch, or paper trading in this slice. - SEC ETL spike (`sec_client`, `edgartools_client`, `download-fundamentals`) is **frozen** in repo for phase 2; demo pipeline uses SimFin bulk for fundamentals and run-date prices (`shareprices/latest`). -- **Phase 2 (Quantitative Value):** will need multi-period fundamentals (not only TTM snapshots)—lake design should not block adding annual/quarterly income history later. +- **Phase 2 (Quantitative Value):** production scoring follows the **QV funnel** (`docs/mvp/features/quantitative-value.md`); requires multi-period fundamentals for FS-Score YoY deltas. Terminology reminders: -- “Cheap” means high **EY**, not low P/E—use **EY rank** in issues and code names. -- “Quality” means high **ROC**, not ESG or subjective moat—use **ROC rank**. +- **Demo / MF benchmark:** “cheap” = high **EY**; “quality” = high **ROC** — use **EY rank** and **ROC rank** in MF code paths. +- **Production (Phase 2+):** “cheap” = **EBIT/TEV value pool** membership; “quality” = **FS-Score** — do not use ROC/EY ranks for production portfolio selection. +- **Combined rank** is **benchmark-only** after Phase 2; production uses **QV funnel rank**. - “Value trap” in specs means negative EBIT routed to **review queue**, not a separate score. - MVP specs in `docs/mvp/` remain canonical until an ADR or architecture decision supersedes them; update `CONTEXT.md` when `/grill-with-docs` resolves a term conflict. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cbceddd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# Pipeline batch image for ingest, normalize, and score-universe on AWS or locally. +# Build: make docker-build +# Run: docker run --rm -e LAKE_ROOT_URI=file:///lake -v "$PWD/data:/lake" smartwealthai-pipeline score-universe --help + +FROM python:3.11-slim-bookworm + +WORKDIR /app + +ENV POETRY_VERSION=2.1.1 \ + POETRY_VIRTUALENVS_CREATE=false \ + POETRY_NO_INTERACTION=1 \ + LAKE_ROOT_URI=file:///lake + +RUN pip install --no-cache-dir "poetry==${POETRY_VERSION}" + +COPY pyproject.toml poetry.lock ./ +COPY src ./src +COPY config ./config + +RUN poetry install --only main + +ENTRYPOINT ["score-universe"] +CMD ["--help"] diff --git a/Makefile b/Makefile index 665216f..b625ba9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install lint test download-fundamentals +.PHONY: install lint test download-fundamentals docker-build install: poetry install --with dev @@ -10,6 +10,9 @@ lint: test: poetry run pytest --cov=smartwealthai --cov-report=term-missing +docker-build: + docker build -f Dockerfile -t smartwealthai-pipeline . + # Requires SEC_IDENTITY in the environment. See docs/mvp/guides/download-fundamentals.md download-fundamentals: poetry run download-fundamentals --universe dow30 diff --git a/README.md b/README.md index d69378d..e6ca6f0 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,19 @@ Canonical specs: [`docs/mvp/`](docs/mvp/) · Ubiquitous language: [`CONTEXT.md`] Guide: [`docs/mvp/guides/download-fundamentals.md`](docs/mvp/guides/download-fundamentals.md). +## Cloud foundation (Phase 2a) + +Pipeline batch work targets AWS with a **configurable lake root**; the Streamlit dashboard stays local for now. + +| Concern | Local dev | AWS integration | +| --- | --- | --- | +| Lake root | `LAKE_ROOT_URI=file://…` or `--data-dir data` | `LAKE_ROOT_URI=s3://dev-bucket/prefix/` (I/O slice: file backend in pytest) | +| PR CI | `make lint` + `make test` on fixtures | No credentials | +| Ingest-smoke | N/A | [`.github/workflows/ingest-smoke.yml`](.github/workflows/ingest-smoke.yml) via OIDC | +| Pipeline image | `make docker-build` | ECR/ECS in [#95](https://github.com/JLaborda/SmartWealthAI/issues/95) | + +Guide: [`docs/mvp/guides/cloud-foundation.md`](docs/mvp/guides/cloud-foundation.md). + ## Roadmap See [`docs/mvp/demo-slice.md`](docs/mvp/demo-slice.md) for the **June 30, 2026** delivery target and [`docs/mvp/architecture/architecture.md`](docs/mvp/architecture/architecture.md) for the full MVP north star. diff --git a/docs/mvp/demo-slice.md b/docs/mvp/demo-slice.md index 0572334..5430467 100644 --- a/docs/mvp/demo-slice.md +++ b/docs/mvp/demo-slice.md @@ -90,9 +90,15 @@ flowchart LR ## After the demo (phase 2 order) -1. Historical S&P 500 universe + permanent loss filter -2. Minimal backtest (annual rebalance, 20 years) — custom pandas/DuckDB loop, not Zipline -3. Walk-forward, Monte Carlo, benchmark gate -4. Sell-watch + paper trading -5. SEC EDGAR normalizer (optional PIT upgrade) -6. Quantitative Value metrics (multi-period fundamentals from raw SimFin archives) +Per [`prds/phase2/prd.md`](prds/phase2/prd.md) — **Quantitative Value is production scoring**; Magic Formula (ROC + EY + combined rank) remains **benchmark-only**. + +1. **QV feature spec** — [`quantitative-value.md`](features/quantitative-value.md) (canonical funnel; blocks scoring implementation) +2. Multi-period fundamentals + daily prices (FS-Score YoY deltas, backtest NAV) +3. Forensic evaluator + Beneish bottom-5% gate (extends [`permanent-loss-filter.md`](features/permanent-loss-filter.md)) +4. **QV funnel** — EBIT/TEV value decile → FS-Score → ~50-name EW model portfolio +5. Light backtest (5–10 years, annual rebalance) vs S&P 500 CW + MF replica +6. Cloud pipeline (S3 lake, ECS, MLflow S3 artifacts) +7. Sell-watch with QV-adapted triggers + dashboard +8. Historical S&P 500 universe (delisted names) + full backtest (20+ years, walk-forward, Sharpe gate) +9. SEC EDGAR normalizer (optional PIT upgrade) +10. Paper trading + broker (after passing full backtest) diff --git a/docs/mvp/features/quantitative-value.md b/docs/mvp/features/quantitative-value.md new file mode 100644 index 0000000..2ee0456 --- /dev/null +++ b/docs/mvp/features/quantitative-value.md @@ -0,0 +1,248 @@ +# Feature: Quantitative Value (QV funnel) + +## Implementation status + +**done** (spec only) — canonical QV feature spec authored in [#86](https://github.com/JLaborda/SmartWealthAI/issues/86). Scoring code deferred to [#89](https://github.com/JLaborda/SmartWealthAI/issues/89)–[#93](https://github.com/JLaborda/SmartWealthAI/issues/93). + +## Objective + +Define the **production scoring path** for Phase 2+: the *Quantitative Value* funnel from Wesley R. Gray and Tobias Carlisle — forensic hard exclusion, EBIT/TEV value screen, FS-Score quality screen, and a concentrated equal-weight **model portfolio**. The June 30 demo **Magic Formula** path (ROC + EY + combined rank → top 30) remains a **benchmark-only** replica for backtest comparison, not production scoring. + +Parent PRD: [`docs/mvp/prds/phase2/prd.md`](../prds/phase2/prd.md). + +## MVP scope + +- **Sequential funnel** with monotonic shrinkage: each stage only removes names; stage input/output counts are auditable and logged to MLflow. +- **Stage 1 — Universe pass:** investable tickers from [`universe-construction.md`](universe-construction.md) (sector hard exclusions upstream). +- **Stage 2 — Forensic hard exclusion:** evaluate all rules in [`permanent-loss-filter.md`](permanent-loss-filter.md) plus **Beneish M-Score** with a QVAL-style **bottom-5% cross-sectional gate** per forensic model among survivors. Emit `exclude` or `pass` with `rule_id`, `rule_version`, `triggered_value`, `threshold`, `explanation`. +- **Stage 3 — Value screen:** rank forensic survivors by **EBIT/TEV** descending (same EV definition as [`cheap-stocks.md`](cheap-stocks.md)); keep the top **decile (~10%)**, configurable via `value_decile_fraction`. +- **Stage 4 — Quality screen:** compute **FS-Score** (Gray/Carlisle 10-component variant) on the value pool; rank by FS-Score descending; keep the top **N** names up to the portfolio cap (default **50**). +- **Stage 5 — Model portfolio:** equal-weight long-only holdings; **market-cap tie-break** on rank ties (ascending, same convention as demo MF). +- **Configuration:** versioned YAML under `config/quantitative_value/` (funnel parameters) and `config/permanent_loss/` (forensic thresholds, Beneish coefficients). +- **Outputs:** per-stage parquet tables, final `model_portfolio` rows, MLflow metrics (`qv_stage_*_count`, config hashes, `git_sha`). +- **Explainability:** dashboard shows stage membership, EBIT/TEV rank, FS-Score components, and forensic pass/exclusion reasons per ticker. + +## Out of MVP scope + +- Implementing forensic evaluator, FS-Score calculator, QV orchestrator, or scoring CLI wiring ([#89](https://github.com/JLaborda/SmartWealthAI/issues/89)–[#91](https://github.com/JLaborda/SmartWealthAI/issues/91)). +- Multi-period ETL and daily prices ([#87](https://github.com/JLaborda/SmartWealthAI/issues/87), [#88](https://github.com/JLaborda/SmartWealthAI/issues/88)) — prerequisite data work, not part of this spec's implementation slice. +- Cloud deploy, ECS, S3 lake root ([#94](https://github.com/JLaborda/SmartWealthAI/issues/94), [#95](https://github.com/JLaborda/SmartWealthAI/issues/95)). +- Changing demo Magic Formula scoring behavior (ROC/EY/combined rank path preserved for benchmark). +- Moat "pre-flight checklist" and full forensic model zoo beyond Beneish + spec distress rules. +- Score-weighted and risk-parity portfolio weighting (equal-weight only in Phase 2a). +- Paper trading, broker execution, auto-execution of sell signals. + +## Inputs + +| Input | Source | Notes | +| --- | --- | --- | +| Universe pass list | `curated/universe` | `pass` rows only; banks/insurers/utilities already excluded. | +| PIT fundamentals (multi-period) | `curated/fundamentals` | `as_of_date <= run_date`; annual/quarterly history required for FS-Score YoY deltas ([#87](https://github.com/JLaborda/SmartWealthAI/issues/87)). | +| Run-date prices | `curated/prices` | Market cap and EV components at `run_date`. | +| Forensic rules + Beneish config | `config/permanent_loss/` | Thresholds and coefficient sets are versioned. | +| QV funnel config | `config/quantitative_value/` | `value_decile_fraction`, `portfolio_size`, `formula_version`, tie-break policy. | +| Run date | Pipeline parameter | Decision date for PIT queries and cross-sectional ranks. | +| Permanent loss exclusions (reference) | [`permanent-loss-filter.md`](permanent-loss-filter.md) | Bankruptcy/distress + fraud rules; extended here with Beneish. | + +## Outputs + +| Output | Path / target | +| --- | --- | +| Stage counts (MLflow) | `qv_stage_universe_count`, `qv_stage_forensic_pass_count`, `qv_stage_value_pool_count`, `qv_stage_quality_pool_count`, `qv_stage_portfolio_count` | +| Forensic exclusions | `curated/permanent_loss/run_date=/exclusions.parquet` — columns `cik, ticker, subfilter, rule_id, rule_version, triggered_value, threshold, percentile, as_of_date, explanation` | +| Value pool scores | `curated/scores/qv_value/run_date=/scores.parquet` — `cik, ticker, ebit, ev, ebit_tev, value_rank, formula_version, as_of_date` | +| Quality scores | `curated/scores/qv_quality/run_date=/scores.parquet` — `cik, ticker, fs_score, fs_*` component columns, `formula_version`, `as_of_date` | +| Funnel audit | `curated/scores/qv_funnel/run_date=/funnel.parquet` — per-ticker stage reached, exclusion reason if any | +| Model portfolio (QV) | `curated/portfolio/qv/run_date=/portfolio.parquet` — `cik, ticker, qv_funnel_rank, fs_score, ebit_tev, weight, formula_version` | +| Review queue | `curated/issues/run_date=/qv.parquet` — missing FS-Score or forensic inputs | + +## Funnel stages + +| Stage | Name | Action | Typical shrinkage | +| --- | --- | --- | --- | +| 0 | Universe | Sector-filtered investable set | Baseline count | +| 1 | Forensic | Hard `exclude` on any distress/fraud rule **or** Beneish bottom 5% | Largest drop | +| 2 | Value | Top decile by EBIT/TEV among survivors | ~90% removed | +| 3 | Quality | FS-Score rank within value pool; select top N by score | To ~50 names | +| 4 | Portfolio | Equal-weight; market-cap tie-break on rank ties | Final holdings | + +**Monotonicity contract:** a name excluded at stage *k* never appears in stages *k+1* … portfolio. Stage counts must satisfy `count[k] >= count[k+1]` for all adjacent stages. + +## Forensic stage (extends permanent loss filter) + +Forensic evaluation runs **before** any value or quality score. It **references and extends** [`permanent-loss-filter.md`](permanent-loss-filter.md): + +| Source | Rules | MVP behavior | +| --- | --- | --- | +| Bankruptcy / distress | `BK_ALTMAN_Z`, `BK_INT_COVERAGE`, `BK_NETDEBT_EBITDA`, `BK_NEGATIVE_EQUITY`, `BK_DELISTED` | Hard `exclude` if any rule fires | +| Fraud | `FRD_RESTATEMENT_RECENT`, `FRD_AUDITOR_CHANGE_REPEATED`, `FRD_LATE_FILER` | Hard `exclude` if any rule fires; EDGAR-dependent rules may be `unavailable` until wired | +| Manipulation (QV) | `FRD_BENEISH_M` | Compute Beneish M-Score; **exclude names in the bottom 5%** of the cross-sectional M-Score distribution among forensic-stage candidates | + +### Beneish M-Score bottom-5% gate + +- **Rule id:** `FRD_BENEISH_M` +- **Config:** `config/permanent_loss/beneish.yaml` (`rule_version`, coefficient set, minimum input coverage) +- **Gate:** among companies with a computable M-Score at `run_date`, exclude those at or below the **5th percentile** (QVAL-style safety screen per forensic model). +- **Audit columns:** `rule_id`, `rule_version`, `triggered_value` (M-Score), `threshold` (5th percentile cutoff), `percentile`, `explanation` +- **Missing inputs:** route to review queue; do not silently pass (same policy as permanent-loss spec). + +Distress and fraud rules from the permanent-loss spec retain their existing thresholds in `config/permanent_loss/`; Beneish coefficients and percentile gate live in the same namespace with separate version ids. + +## Value screen (EBIT/TEV) + +Production **cheapness** is membership in the **EBIT/TEV value pool**, not MF **EY rank** alone. + +``` +EBIT/TEV = EBIT / Enterprise Value +EV = Market Cap + Total Debt + Preferred Equity + Minority Interest - Cash +``` + +- Same `EBIT` and EV component definitions as [`cheap-stocks.md`](cheap-stocks.md) (`formula_version` shared where components overlap). +- Cross-sectional rank **within forensic survivors** by descending EBIT/TEV. +- Keep top `value_decile_fraction` (default **0.10** ≈ decile). +- `EBIT <= 0` or invalid EV → review queue, not value pool. +- Output metric is **EBIT/TEV** and `value_rank`; this path is separate from MF `ey_rank`. + +## FS-Score (Gray/Carlisle variant) + +Production **quality** is the **FS-Score** composite on the value pool, not ROC rank alone. Ten binary components (1 = good, 0 = bad); sum to integer **0–10**. Reference: Gray & Carlisle, *Quantitative Value* (2013), Ch. 6; [Alpha Architect FS-Score article](https://alphaarchitect.com/2015/05/value-investing-research-simple-methods-to-improve-the-piotroski-f-score/). + +`formula_version` (initial: `fs_v1`) is recorded on every scored row. + +### Current profitability + +| Component id | Definition | Score = 1 when | +| --- | --- | --- | +| `FS_ROA` | ROA = net income before extraordinary items / total assets (most recent fiscal year) | ROA > 0 | +| `FS_FCFTA` | FCFTA = free cash flow / total assets | FCFTA > 0 | +| `FS_ACCRUAL` | Accrual quality signal | FCFTA > ROA | + +### Stability + +| Component id | Definition | Score = 1 when | +| --- | --- | --- | +| `FS_ΔLEVER` | Change in long-term debt / total assets | Leverage ratio **decreased** YoY | +| `FS_ΔLIQUID` | Change in current ratio (current assets / current liabilities) | Liquidity ratio **increased** YoY | +| `FS_NEQISS` | Net equity issuance = repurchases − issuances | Repurchases **exceed** issuances | + +### Recent operational improvements + +| Component id | Definition | Score = 1 when | +| --- | --- | --- | +| `FS_ΔROA` | Current ROA − prior ROA | ΔROA > 0 | +| `FS_ΔFCFTA` | Current FCFTA − prior FCFTA | ΔFCFTA > 0 | +| `FS_ΔMARGIN` | Current gross margin − prior gross margin | ΔMARGIN > 0 | +| `FS_ΔTURN` | Current asset turnover − prior asset turnover | ΔTURN > 0 | + +**FS-Score** = sum of all ten components. **QV funnel rank** within the value pool orders by FS-Score descending; ties break by ascending market cap (configurable in `config/quantitative_value/tie_break.yaml`). + +Missing prior-year inputs for a component → component scores 0 and row is flagged in review queue if coverage falls below configured minimum. + +## Model portfolio (QV) + +| Topic | Decision (closed) | +| --- | --- | +| Default size | **50** names | +| Configurable cap | `portfolio_size` in `config/quantitative_value/funnel.yaml` | +| Weighting | Equal-weight only (Phase 2a) | +| Selection | Top N by FS-Score within value pool | +| Tie-break | Ascending market cap on rank ties | +| Long/short | Long-only | + +This supersedes the demo slice top-30 MF portfolio for **production** scoring only; the MF replica may use a different N for benchmark comparison. + +## Key interfaces (spec contracts) + +These are **documentation contracts** for downstream implementation issues; no code in this issue. + +### QV funnel orchestrator + +- **Input:** `run_date`, lake root, QV + forensic config hashes. +- **Behavior:** run stages 0→4 sequentially; delegate to forensic evaluator, value ranker, FS-Score ranker, portfolio constructor; emit stage-count metrics. +- **Output:** `ScoringResult`-like object with per-stage tables, final portfolio, MLflow-ready metrics. + +### Forensic evaluator + +- **Input:** universe pass list + PIT fundamentals (+ optional filing flags). +- **Output:** `exclude` or `pass` per `(cik, run_date)` with full audit columns. + +### FS-Score calculator + +- **Input:** multi-period PIT income/balance/cashflow for one ticker at `decision_date`. +- **Output:** component dict, total 0–10, `formula_version`. + +### Value metrics + +- Reuse EV/EBIT building blocks from cheap-stocks module; separate production path from MF `EY rank`. + +### Magic Formula benchmark path + +- Preserve ROC + EY + **combined rank** unchanged; invoke only from backtest benchmark builder and dedicated CI tests. + +## Configuration namespace + +| Path | Contents | +| --- | --- | +| `config/quantitative_value/funnel.yaml` | `portfolio_size` (default 50), `value_decile_fraction` (default 0.10), stage ordering | +| `config/quantitative_value/fs_score.yaml` | `formula_version`, minimum input coverage, fiscal-year alignment rules | +| `config/quantitative_value/tie_break.yaml` | Market-cap tie-break policy | +| `config/permanent_loss/*.yaml` | Distress/fraud thresholds (existing) + `beneish.yaml` (coefficients, bottom-percentile gate) | + +Every MLflow run logs config file hashes and `formula_version` values. + +## Mermaid diagram + +```mermaid +flowchart TD + Uni["Stage 0: Universe pass"] --> Forensic["Stage 1: Forensic evaluator"] + Forensic --> Distress{"Distress / fraud rule fired?"} + Distress -->|Yes| Excl["Hard exclude + audit row"] + Distress -->|No| Beneish{"Beneish M-Score bottom 5%?"} + Beneish -->|Yes| Excl + Beneish -->|No| Value["Stage 2: EBIT/TEV rank → top decile"] + Value --> Quality["Stage 3: FS-Score on value pool"] + Quality --> Port["Stage 4: Top N EW portfolio (~50)"] + Excl --> Audit["exclusions.parquet + MLflow metrics"] + Port --> Out["portfolio.parquet + funnel audit"] + Uni -.-> MLflow["qv_stage_*_count"] + Forensic -.-> MLflow + Value -.-> MLflow + Quality -.-> MLflow + Port -.-> MLflow +``` + +## Expected flow + +1. Load universe pass list for `run_date`; log `qv_stage_universe_count`. +2. Run forensic evaluator (permanent-loss rules + Beneish bottom-5% gate); write exclusions; log `qv_stage_forensic_pass_count`. +3. Rank forensic survivors by EBIT/TEV; keep top decile; log `qv_stage_value_pool_count`. +4. Compute FS-Score for value-pool names; rank by total score; select top `portfolio_size`; log `qv_stage_quality_pool_count`. +5. Build equal-weight portfolio with market-cap tie-break; log `qv_stage_portfolio_count`. +6. Persist parquet artifacts and MLflow run (params, stage metrics, portfolio artifact, `git_sha`). + +## Acceptance criteria + +- [x] Spec exists with objective, MVP scope, inputs/outputs, funnel stages, Mermaid diagram, acceptance criteria, and risks ([#86](https://github.com/JLaborda/SmartWealthAI/issues/86)). +- [x] FS-Score documents all **10 binary components** with Gray/Carlisle definitions and `formula_version`. +- [x] Forensic stage references [`permanent-loss-filter.md`](permanent-loss-filter.md) and documents Beneish **bottom-5%** gate with audit columns. +- [x] Portfolio size default **~50** recorded as a closed decision with configurable cap. +- [ ] Funnel monotonic shrinkage: each stage only removes names; stage counts auditable in MLflow (implementation [#91](https://github.com/JLaborda/SmartWealthAI/issues/91)). +- [ ] Forensic hard exclusion runs before any value/quality score (implementation [#89](https://github.com/JLaborda/SmartWealthAI/issues/89)). +- [ ] Value screen keeps top decile by EBIT/TEV among survivors (implementation [#90](https://github.com/JLaborda/SmartWealthAI/issues/90)). +- [ ] Quality screen ranks FS-Score within value pool; portfolio cap applied last (implementation [#91](https://github.com/JLaborda/SmartWealthAI/issues/91)). +- [ ] MF **combined rank** remains benchmark-only; demo ROC/EY path unchanged on fixtures (regression in CI). +- [ ] Same `(universe, run_date, config hashes)` produces byte-identical funnel outputs when implementation lands. + +## Open questions + +- Beneish M-Score: use classic 8-variable model or include optional 9th/10th variables when data exists? **Recommendation:** start with 8-variable `M-Score` per Beneish (1999); document coefficient set in `beneish.yaml`; extend in a new `rule_version` if coverage improves. +- FS-Score fiscal alignment: match on fiscal year-end or trailing four quarters? **Recommendation:** fiscal year pairs for YoY deltas; flag mismatched fiscal calendars in review queue. +- Value decile: fixed 10% fraction vs fixed count? **Recommendation:** fraction (`value_decile_fraction = 0.10`) so pool scales with universe size. + +## Risks + +- **Missing multi-period data** shrinks FS-Score coverage; depends on [#87](https://github.com/JLaborda/SmartWealthAI/issues/87) landing first. +- **Beneish false positives** on growth firms with high DSRI; bottom-5% gate mitigates but does not eliminate FP review burden. +- **Survivorship bias** in Phase 2a light backtest (current SimFin US universe) — must be labeled in reports per PRD. +- **Compute cost:** full funnel × rebalance dates in backtests; plan DuckDB pushdown or materialized stage tables early. +- **Vocabulary drift:** production terms (FS-Score, EBIT/TEV value pool, QV funnel rank) must stay distinct from MF benchmark terms in code and `CONTEXT.md`. diff --git a/docs/mvp/guides/cloud-foundation.md b/docs/mvp/guides/cloud-foundation.md new file mode 100644 index 0000000..cf8f6bb --- /dev/null +++ b/docs/mvp/guides/cloud-foundation.md @@ -0,0 +1,97 @@ +# Cloud foundation (Phase 2a) + +Operator guide for **lake root URI**, the pipeline Docker image, and the **ingest-smoke** GitHub Actions workflow. Implements GitHub issue [#94](https://github.com/JLaborda/SmartWealthAI/issues/94). ECS Fargate deploy is [#95](https://github.com/JLaborda/SmartWealthAI/issues/95). + +## Lake root URI + +All ingest and scoring CLIs resolve storage from a single **lake root**: + +| Source | Precedence | +| --- | --- | +| `--lake-root-uri` | 1 (highest) | +| `--data-dir` | 2 | +| `LAKE_ROOT_URI` env var | 3 | +| `SMARTWEALTHAI_DATA_DIR` env var | 4 | +| default `data/` | 5 | + +Supported URI forms: + +- `file:///absolute/path` or bare local path (`data`, `/tmp/lake`) +- `s3://bucket/prefix/` (parsed for cloud workflows; parquet I/O in this slice is **file-backend only**) + +Example (local mirror): + +```bash +export LAKE_ROOT_URI="file://$(pwd)/data" +poetry run score-universe --run-date 2026-06-18 +``` + +Backward compatible: + +```bash +poetry run score-universe --data-dir data --run-date 2026-06-18 +``` + +## Pipeline Docker image + +Build and smoke-test locally (no AWS): + +```bash +make docker-build +docker run --rm smartwealthai-pipeline --help +docker run --rm \ + -e LAKE_ROOT_URI=file:///lake \ + -v "$(pwd)/tests/fixtures/lake:/lake:ro" \ + smartwealthai-pipeline \ + --run-date 2026-06-18 --portfolio-size 3 --quiet +``` + +The image entrypoint is `score-universe`. Override the command for other CLIs by installing Poetry in a dev image or extending the Dockerfile entrypoint in [#95](https://github.com/JLaborda/SmartWealthAI/issues/95). + +## Ingest-smoke workflow + +Workflow: [`.github/workflows/ingest-smoke.yml`](../../.github/workflows/ingest-smoke.yml) + +- Triggers: manual `workflow_dispatch` and weekly cron (Mondays 06:00 UTC) +- Uses GitHub Environment **`dev`** and **OIDC** (no long-lived AWS access keys) +- Writes `raw/_smoke/ingest-smoke.txt` to the dev bucket and validates with `head-object` + +### Human prerequisites (one-time AWS setup) + +If buckets and OIDC are not yet provisioned in your AWS account: + +1. **S3 dev bucket** — e.g. `smartwealthai-dev-lake` with optional prefix `data/`. Same zone layout as local: `raw/`, `curated/`, `curated/issues/`. +2. **GitHub OIDC provider** in IAM (issuer `token.actions.githubusercontent.com`, audience `sts.amazonaws.com`). +3. **IAM role** trusted by the repo and `environment:dev`, with `s3:PutObject`, `s3:GetObject`, `s3:ListBucket` on the dev bucket prefix. +4. **GitHub Environment `dev`** repository variables: + + | Variable | Example | + | --- | --- | + | `AWS_ROLE_ARN` | `arn:aws:iam::123456789012:role/github-actions-dev` | + | `AWS_REGION` | `us-east-1` | + | `DEV_LAKE_BUCKET` | `smartwealthai-dev-lake` | + | `DEV_LAKE_PREFIX` | `data/` (or empty string) | + +5. **Runtime secrets (ECS, #95)** — `SIMFIN_API_KEY` in AWS Secrets Manager; not required for ingest-smoke marker upload. + +Terraform or CloudFormation for the above is out of scope for #94; document and provision manually or in a future infra issue. + +### Run ingest-smoke manually + +GitHub → Actions → **Ingest smoke (dev S3)** → Run workflow (environment: `dev`). + +## Three-tier testing model + +| Tier | What runs | AWS / network | +| --- | --- | --- | +| **PR CI** | `make lint`, `pytest` on fixtures | None | +| **Ingest-smoke** | OIDC S3 marker + lake URI parse | Dev bucket only | +| **Deploy (#95)** | ECR push, ECS Fargate pipeline | Dev/prod on merge | + +PR CI must stay hermetic: no SimFin, yfinance, or live S3. + +## Phase 2a scope + +- **Pipeline on AWS** (batch ingest + score) — this guide + Docker image + ingest-smoke +- **Dashboard local** — `poetry run run-dashboard` reads `SMARTWEALTHAI_DATA_DIR` or local `data/` +- Full scheduled ECS deploy, MLflow on S3, and Secrets Manager task injection: [#95](https://github.com/JLaborda/SmartWealthAI/issues/95) diff --git a/src/smartwealthai/build_universe.py b/src/smartwealthai/build_universe.py index d22d110..15997ac 100644 --- a/src/smartwealthai/build_universe.py +++ b/src/smartwealthai/build_universe.py @@ -8,18 +8,13 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.simfin_industry_exclusions import REFERENCE_PATH from smartwealthai.universe_builder import build_universe @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--run-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -40,16 +35,22 @@ help="Industry exclusions reference CSV.", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, run_date: datetime, snapshot_date: datetime | None, exclusions_file: Path, ) -> None: """Build curated universe and exclusion artifacts for one run date.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + decision_date = run_date.date() snapshot = snapshot_date.date() if snapshot_date is not None else decision_date result = build_universe( - data_dir, + lake_path, run_date=decision_date, snapshot_date=snapshot, exclusions_path=exclusions_file, diff --git a/src/smartwealthai/cli_lake.py b/src/smartwealthai/cli_lake.py new file mode 100644 index 0000000..2f0aa81 --- /dev/null +++ b/src/smartwealthai/cli_lake.py @@ -0,0 +1,57 @@ +"""Shared lake root resolution for pipeline CLIs.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import click + +from smartwealthai.lake_root import LakeRoot, resolve_lake_root + + +def lake_root_options(fn: Callable[..., Any]) -> Callable[..., Any]: + """Attach ``--lake-root-uri`` and ``--data-dir`` options to a Click command.""" + fn = click.option( + "--lake-root-uri", + envvar="LAKE_ROOT_URI", + default=None, + help="Lake root URI (file:// or s3://). Overrides --data-dir.", + )(fn) + fn = click.option( + "--data-dir", + type=click.Path(path_type=Path, file_okay=False), + default=None, + help="Local data lake root (backward compatible; default data/).", + )(fn) + return fn + + +def resolve_cli_lake_root( + *, + lake_root_uri: str | None, + data_dir: Path | None, +) -> LakeRoot: + """Resolve lake root with CLI/env precedence. + + Order: ``--lake-root-uri`` (includes ``LAKE_ROOT_URI`` via Click) → + ``--data-dir`` → ``SMARTWEALTHAI_DATA_DIR`` → ``data``. + """ + if lake_root_uri: + return resolve_lake_root(lake_root_uri) + if data_dir is not None: + return resolve_lake_root(str(data_dir)) + if uri := os.environ.get("SMARTWEALTHAI_DATA_DIR"): + return resolve_lake_root(uri) + return resolve_lake_root("data") + + +def resolve_cli_data_dir( + *, + lake_root_uri: str | None, + data_dir: Path | None, +) -> Path: + """Return a local :class:`Path` for file-backend pipeline stages.""" + return resolve_cli_lake_root(lake_root_uri=lake_root_uri, data_dir=data_dir).as_path() diff --git a/src/smartwealthai/compute_metrics.py b/src/smartwealthai/compute_metrics.py index 16a9a58..ffd444f 100644 --- a/src/smartwealthai/compute_metrics.py +++ b/src/smartwealthai/compute_metrics.py @@ -8,6 +8,7 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.magic_formula_metrics import MetricsResult from smartwealthai.pit_fundamentals import MetricsInputError, compute_metrics_for_ticker @@ -18,13 +19,7 @@ required=True, help="Ticker symbol (e.g. AAPL).", ) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--as-of-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -32,12 +27,22 @@ show_default="today (UTC)", help="Decision date (PIT fundamentals and price snapshot).", ) -def main(ticker: str, data_dir: Path, as_of_date: datetime) -> None: +def main( + ticker: str, + lake_root_uri: str | None, + data_dir: Path | None, + as_of_date: datetime, +) -> None: """Compute ROC and EY for one ticker using curated fundamentals and prices.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + decision_date = as_of_date.date() try: result = compute_metrics_for_ticker( - data_dir, + lake_path, ticker=ticker.upper(), as_of_date=decision_date, ) diff --git a/src/smartwealthai/download_fundamentals.py b/src/smartwealthai/download_fundamentals.py index b027cd9..c6f25c2 100644 --- a/src/smartwealthai/download_fundamentals.py +++ b/src/smartwealthai/download_fundamentals.py @@ -24,6 +24,7 @@ import click +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.edgartools_client import EdgartoolsClientError, download_statements from smartwealthai.lake_paths import ( STATEMENT_NAMES, @@ -189,13 +190,7 @@ def run_download( type=click.Path(path_type=Path, dir_okay=False), help="Path to a universe CSV with columns ticker,cik.", ) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--periods", type=int, @@ -217,18 +212,24 @@ def run_download( def main( universe: str | None, universe_file: Path | None, - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, periods: int, as_of_date: datetime | None, force: bool, ) -> None: """Download SEC companyfacts and edgartools annual statements for a universe.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + snapshot_date = as_of_date.date() if as_of_date is not None else None raise SystemExit( run_download( universe=universe, universe_file=universe_file, - data_dir=data_dir, + data_dir=lake_path, periods=periods, as_of_date=snapshot_date, force=force, diff --git a/src/smartwealthai/download_prices.py b/src/smartwealthai/download_prices.py index c023e82..5eb2c14 100644 --- a/src/smartwealthai/download_prices.py +++ b/src/smartwealthai/download_prices.py @@ -10,6 +10,7 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.lake_paths import price_ingest_errors_path from smartwealthai.price_ingest import run_price_ingest @@ -17,13 +18,7 @@ @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--run-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -42,19 +37,25 @@ help="Rebuild curated snapshot even when it already exists.", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, run_date: datetime, snapshot_date: datetime | None, force: bool, ) -> None: """Build curated run-date prices from SimFin bulk shareprices/latest.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") decision_date = run_date.date() snapshot = snapshot_date.date() if snapshot_date is not None else decision_date try: ingest_run = run_price_ingest( - data_dir=data_dir, + data_dir=lake_path, run_date=decision_date, snapshot_date=snapshot, force=force, @@ -77,7 +78,7 @@ def main( {"ticker": ticker, "error": "no SimFin share price on or before run_date"} for ticker in ingest_run.missing_tickers ] - error_file = price_ingest_errors_path(data_dir, run_date=decision_date) + error_file = price_ingest_errors_path(lake_path, run_date=decision_date) error_file.parent.mkdir(parents=True, exist_ok=True) error_file.write_text(json.dumps(payload, indent=2)) click.echo(f"Wrote error summary: {error_file}") diff --git a/src/smartwealthai/download_simfin.py b/src/smartwealthai/download_simfin.py index 96d065d..8738601 100644 --- a/src/smartwealthai/download_simfin.py +++ b/src/smartwealthai/download_simfin.py @@ -28,6 +28,7 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.lake_paths import simfin_bulk_path, simfin_errors_path from smartwealthai.simfin_client import configure_simfin, fetch_dataset_csv @@ -181,13 +182,7 @@ def run_download( @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--as-of-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -207,16 +202,22 @@ def run_download( help="Re-download and overwrite lake copies regardless of age.", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, as_of_date: datetime | None, refresh_days: int, force: bool, ) -> None: """Download SimFin bulk US fundamentals into the raw lake.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + snapshot_date = as_of_date.date() if as_of_date is not None else datetime.now(UTC).date() raise SystemExit( run_download( - data_dir=data_dir, + data_dir=lake_path, as_of_date=snapshot_date, refresh_days=refresh_days, force=force, diff --git a/src/smartwealthai/lake_root.py b/src/smartwealthai/lake_root.py new file mode 100644 index 0000000..1a72c6d --- /dev/null +++ b/src/smartwealthai/lake_root.py @@ -0,0 +1,107 @@ +"""Lake root URI resolution and storage-backend I/O.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Literal +from urllib.parse import urlparse + +import pandas as pd + +from smartwealthai import lake_paths + +Backend = Literal["file", "s3"] + + +@dataclass(frozen=True) +class LakeRoot: + """Configurable data lake root with zone-relative path builders.""" + + backend: Backend + uri: str + _path: Path | None = None + s3_bucket: str | None = None + s3_prefix: str | None = None + + def as_path(self) -> Path: + """Return the local filesystem root (file backend only).""" + if self.backend != "file": + msg = f"S3 lake root has no local Path; use lake I/O helpers: {self.uri}" + raise TypeError(msg) + if self._path is None: + msg = f"File lake root is missing a path: {self.uri}" + raise ValueError(msg) + return self._path + + def curated_portfolio_path(self, *, run_date: date) -> Path: + """Build the curated portfolio parquet path for a run date.""" + if self.backend != "file": + msg = "curated_portfolio_path is implemented for the file backend in this slice" + raise NotImplementedError(msg) + return lake_paths.curated_portfolio_path(self.as_path(), run_date=run_date) + + def write_parquet(self, path: Path, frame: pd.DataFrame) -> None: + """Write a parquet artifact under the lake root.""" + if self.backend != "file": + msg = "S3 parquet writes are not implemented in this slice; use file:// locally" + raise NotImplementedError(msg) + path.parent.mkdir(parents=True, exist_ok=True) + frame.to_parquet(path, index=False) + + def read_parquet(self, path: Path) -> pd.DataFrame: + """Read a parquet artifact from the lake root.""" + if self.backend != "file": + msg = "S3 parquet reads are not implemented in this slice; use file:// locally" + raise NotImplementedError(msg) + return pd.read_parquet(path) + + +def resolve_lake_root(uri: str | None) -> LakeRoot: + """Parse a lake root URI into a :class:`LakeRoot`. + + Supported forms: + - ``file:///absolute/path`` or ``file://relative/path`` + - bare local path (``data``, ``/tmp/lake``) + - ``s3://bucket/prefix`` (parsed for cloud workflows; file I/O stays local here) + """ + if uri is None or not str(uri).strip(): + msg = "Lake root URI is required" + raise ValueError(msg) + + raw = str(uri).strip() + if raw.startswith("s3://"): + return _resolve_s3(raw) + if raw.startswith("file://"): + return _resolve_file(raw) + if "://" in raw: + scheme = urlparse(raw).scheme + msg = f"Unsupported lake root URI scheme: {scheme!r}" + raise ValueError(msg) + return _resolve_file(raw) + + +def _resolve_file(uri: str) -> LakeRoot: + if uri.startswith("file://"): + parsed = urlparse(uri) + if parsed.netloc: + path = Path(f"/{parsed.netloc}{parsed.path}") + else: + path = Path(parsed.path) + else: + path = Path(uri) + resolved = path.expanduser().resolve() + return LakeRoot(backend="file", uri=uri, _path=resolved) + + +def _resolve_s3(uri: str) -> LakeRoot: + parsed = urlparse(uri) + bucket = parsed.netloc + if not bucket: + msg = f"Invalid S3 lake root URI: {uri!r}" + raise ValueError(msg) + prefix = parsed.path.lstrip("/") + if prefix and not prefix.endswith("/"): + prefix = f"{prefix}/" + return LakeRoot(backend="s3", uri=uri, s3_bucket=bucket, s3_prefix=prefix or "") diff --git a/src/smartwealthai/normalize_simfin.py b/src/smartwealthai/normalize_simfin.py index 4190e3e..72e2de0 100644 --- a/src/smartwealthai/normalize_simfin.py +++ b/src/smartwealthai/normalize_simfin.py @@ -9,6 +9,7 @@ import pandas as pd from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.lake_paths import curated_universe_path from smartwealthai.simfin_normalizer import DEFAULT_MAPPING_PATH, normalize_simfin @@ -61,13 +62,7 @@ def resolve_normalize_tickers( @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--snapshot-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -105,7 +100,8 @@ def resolve_normalize_tickers( help="Suppress progress bars.", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, snapshot_date: datetime, universe_run_date: datetime | None, mapping: Path, @@ -114,15 +110,20 @@ def main( quiet: bool, ) -> None: """Normalize raw SimFin bulk snapshots into curated PIT fundamentals.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + snapshot = snapshot_date.date() universe_date = universe_run_date.date() if universe_run_date is not None else None ticker_set = resolve_normalize_tickers( - data_dir, + lake_path, universe_run_date=universe_date, explicit_tickers=tickers, ) result = normalize_simfin( - data_dir, + lake_path, snapshot_date=snapshot, mapping_path=mapping, tickers=ticker_set, diff --git a/src/smartwealthai/run_demo_pipeline.py b/src/smartwealthai/run_demo_pipeline.py index a0772c9..62cac15 100644 --- a/src/smartwealthai/run_demo_pipeline.py +++ b/src/smartwealthai/run_demo_pipeline.py @@ -11,6 +11,7 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.download_simfin import run_download from smartwealthai.magic_formula_ranking import ScoringResult, score_universe from smartwealthai.normalize_simfin import resolve_normalize_tickers @@ -234,13 +235,7 @@ def format_pipeline_summary(result: PipelineResult) -> str: @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--run-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -289,7 +284,8 @@ def format_pipeline_summary(result: PipelineResult) -> str: help="Skip SimFin bulk download (use existing raw lake).", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, run_date: datetime, snapshot_date: datetime | None, tickers: tuple[str, ...], @@ -300,13 +296,18 @@ def main( skip_download: bool, ) -> None: """Run the demo slice: SimFin ingest, universe, normalize, prices, score-universe.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") decision_date = run_date.date() snapshot = snapshot_date.date() if snapshot_date is not None else decision_date try: result = run_demo_pipeline( - data_dir, + lake_path, run_date=decision_date, snapshot_date=snapshot, tickers=tickers, diff --git a/src/smartwealthai/score_universe.py b/src/smartwealthai/score_universe.py index 7551dc8..fb556a7 100644 --- a/src/smartwealthai/score_universe.py +++ b/src/smartwealthai/score_universe.py @@ -8,6 +8,7 @@ import click from click.testing import CliRunner +from smartwealthai.cli_lake import lake_root_options, resolve_cli_data_dir from smartwealthai.magic_formula_ranking import score_universe from smartwealthai.normalize_simfin import resolve_show_progress @@ -40,13 +41,7 @@ def format_scoring_summary(result: object) -> str: @click.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--data-dir", - type=click.Path(path_type=Path, file_okay=False), - default=Path("data"), - show_default=True, - help="Data lake root.", -) +@lake_root_options @click.option( "--run-date", type=click.DateTime(formats=["%Y-%m-%d"]), @@ -72,17 +67,23 @@ def format_scoring_summary(result: object) -> str: help="Suppress progress bars.", ) def main( - data_dir: Path, + lake_root_uri: str | None, + data_dir: Path | None, run_date: datetime, portfolio_size: int, progress: bool, quiet: bool, ) -> None: """Score the demo universe and write ranking + portfolio parquets.""" + try: + lake_path = resolve_cli_data_dir(lake_root_uri=lake_root_uri, data_dir=data_dir) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + decision_date = run_date.date() try: result = score_universe( - data_dir, + lake_path, run_date=decision_date, portfolio_size=portfolio_size, show_progress=resolve_show_progress(progress=progress, quiet=quiet), diff --git a/tests/test_lake_root.py b/tests/test_lake_root.py new file mode 100644 index 0000000..a764b9d --- /dev/null +++ b/tests/test_lake_root.py @@ -0,0 +1,171 @@ +"""Hermetic tests for lake root URI resolution and file-backend I/O.""" + +from __future__ import annotations + +import shutil +from datetime import date +from pathlib import Path + +import pandas as pd +import pytest +from click.testing import CliRunner + +from smartwealthai.cli_lake import resolve_cli_data_dir, resolve_cli_lake_root +from smartwealthai.lake_paths import curated_portfolio_path +from smartwealthai.lake_root import resolve_lake_root +from smartwealthai.score_universe import cli_run as score_cli_run + +RUN_DATE = date(2026, 6, 18) +FIXTURE_LAKE = Path("tests/fixtures/lake") + + +def test_resolve_lake_root_file_uri_returns_usable_root(tmp_path: Path) -> None: + lake_dir = tmp_path / "lake" + lake_dir.mkdir() + root = resolve_lake_root(f"file://{lake_dir}") + expected = curated_portfolio_path(lake_dir, run_date=RUN_DATE) + assert root.curated_portfolio_path(run_date=RUN_DATE) == expected + assert root.as_path() == lake_dir.resolve() + + +def test_resolve_lake_root_bare_path_equivalent_to_file_uri(tmp_path: Path) -> None: + lake_dir = tmp_path / "lake" + lake_dir.mkdir() + from_uri = resolve_lake_root(f"file://{lake_dir}") + from_bare = resolve_lake_root(str(lake_dir)) + assert from_uri.as_path() == from_bare.as_path() + + +def test_lake_root_parquet_round_trip(tmp_path: Path) -> None: + root = resolve_lake_root(str(tmp_path / "lake")) + path = root.curated_portfolio_path(run_date=RUN_DATE) + frame = pd.DataFrame({"ticker": ["AAPL"], "weight": [1.0]}) + root.write_parquet(path, frame) + loaded = root.read_parquet(path) + pd.testing.assert_frame_equal(loaded, frame) + + +def test_resolve_lake_root_rejects_unsupported_scheme() -> None: + with pytest.raises(ValueError, match="Unsupported lake root URI scheme"): + resolve_lake_root("ftp://example.com/lake") + + +def test_resolve_lake_root_rejects_empty_uri() -> None: + with pytest.raises(ValueError, match="Lake root URI is required"): + resolve_lake_root(None) + with pytest.raises(ValueError, match="Lake root URI is required"): + resolve_lake_root(" ") + + +def test_resolve_lake_root_rejects_invalid_s3_uri() -> None: + with pytest.raises(ValueError, match="Invalid S3 lake root URI"): + resolve_lake_root("s3://") + + +def test_resolve_cli_lake_root_precedence(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + flag_path = tmp_path / "from-flag" + data_dir_path = tmp_path / "from-data-dir" + swai_path = tmp_path / "from-swai" + for path in (flag_path, data_dir_path, swai_path): + path.mkdir() + + monkeypatch.setenv("SMARTWEALTHAI_DATA_DIR", str(swai_path)) + + flag_root = resolve_cli_lake_root( + lake_root_uri=f"file://{flag_path}", + data_dir=data_dir_path, + ) + assert flag_root.as_path() == flag_path.resolve() + + data_root = resolve_cli_lake_root(lake_root_uri=None, data_dir=data_dir_path) + assert data_root.as_path() == data_dir_path.resolve() + + env_root = resolve_cli_lake_root(lake_root_uri=None, data_dir=None) + assert env_root.as_path() == swai_path.resolve() + + monkeypatch.delenv("SMARTWEALTHAI_DATA_DIR", raising=False) + default_root = resolve_cli_lake_root(lake_root_uri=None, data_dir=None) + assert default_root.as_path() == Path("data").resolve() + + +def test_resolve_cli_data_dir_returns_path(tmp_path: Path) -> None: + lake_dir = tmp_path / "lake" + lake_dir.mkdir() + path = resolve_cli_data_dir(lake_root_uri=f"file://{lake_dir}", data_dir=None) + assert path == lake_dir.resolve() + + +@pytest.fixture +def scored_lake(tmp_path: Path) -> Path: + """Minimal prepared lake for score-universe CLI regression.""" + from smartwealthai.normalize_simfin import cli_run as normalize_cli_run + from smartwealthai.price_ingest import run_price_ingest + from smartwealthai.universe_builder import build_universe + + shutil.copytree(FIXTURE_LAKE, tmp_path / "lake") + root = tmp_path / "lake" + build_universe(root, run_date=RUN_DATE, snapshot_date=RUN_DATE) + normalize_cli_run( + [ + "--data-dir", + str(root), + "--snapshot-date", + RUN_DATE.isoformat(), + "--universe-run-date", + RUN_DATE.isoformat(), + ] + ) + run_price_ingest(data_dir=root, run_date=RUN_DATE, snapshot_date=RUN_DATE) + return root + + +def test_cli_score_universe_accepts_lake_root_uri_flag(scored_lake: Path) -> None: + exit_code = score_cli_run( + [ + "--lake-root-uri", + f"file://{scored_lake}", + "--run-date", + RUN_DATE.isoformat(), + "--portfolio-size", + "3", + ] + ) + assert exit_code == 0 + + +def test_cli_score_universe_data_dir_still_works(scored_lake: Path) -> None: + exit_code = score_cli_run( + [ + "--data-dir", + str(scored_lake), + "--run-date", + RUN_DATE.isoformat(), + "--portfolio-size", + "3", + ] + ) + assert exit_code == 0 + + +def test_cli_score_universe_rejects_malformed_lake_root_uri(scored_lake: Path) -> None: + runner = CliRunner() + result = runner.invoke( + __import__("smartwealthai.score_universe", fromlist=["main"]).main, + [ + "--lake-root-uri", + "ftp://bad", + "--run-date", + RUN_DATE.isoformat(), + ], + ) + assert result.exit_code != 0 + assert "Unsupported lake root URI scheme" in result.output + + +def test_s3_lake_root_parses_without_local_path() -> None: + root = resolve_lake_root("s3://smartwealthai-dev-lake/data/") + assert root.backend == "s3" + assert root.s3_bucket == "smartwealthai-dev-lake" + assert root.s3_prefix == "data/" + with pytest.raises(TypeError, match="local Path"): + root.as_path() diff --git a/tests/test_quantitative_value_spec.py b/tests/test_quantitative_value_spec.py new file mode 100644 index 0000000..41ff75c --- /dev/null +++ b/tests/test_quantitative_value_spec.py @@ -0,0 +1,22 @@ +"""Tracer bullet: quantitative-value.md spec structure (issue #86).""" + +from pathlib import Path + +SPEC_PATH = Path("docs/mvp/features/quantitative-value.md") + +REQUIRED_HEADINGS = ( + "## Objective", + "## MVP scope", + "## Acceptance criteria", +) + + +def test_quantitative_value_spec_exists_with_required_sections() -> None: + assert SPEC_PATH.is_file(), f"missing spec: {SPEC_PATH}" + + text = SPEC_PATH.read_text(encoding="utf-8") + + for heading in REQUIRED_HEADINGS: + assert heading in text, f"missing heading: {heading}" + + assert "```mermaid" in text, "spec must include at least one Mermaid diagram"