Taskfile.yaml

version: '3'

# Only targets that compose, inject env, or encode a non-obvious default.
# Pure CLI wrappers (uv/sqlmesh/dagster) live in docs/commands.md.

vars: { VENV_DIR: .venv }
env: { PYTHONPATH: . }

tasks:
  install:
    desc: "Bootstrap .venv, copy .env, uv sync, install pre-commit hook"
    cmds:
      - test -d {{.VENV_DIR}} || uv venv {{.VENV_DIR}}
      - test -f .env || cp .env.example .env
      - uv sync
      - ./scripts/setup_pre_commit.sh
      # Route SQLMesh's hard-coded `logs/` dir into `.logs/sqlmesh/` so the
      # repo keeps one log root. No equivalent env var / config knob exists
      # in SQLMesh — the symlink is the least-ugly option.
      - mkdir -p .logs/sqlmesh
      - test -e transforms/main/logs || ln -s ../../.logs/sqlmesh transforms/main/logs
    sources: [pyproject.toml, uv.lock, scripts/setup_pre_commit.sh]

  ci:
    desc: "Every CI gate: ruff + mypy + pytest + secret scan + staging drift"
    deps: [install]
    cmds:
      - "{{.VENV_DIR}}/bin/ruff check ."
      - "{{.VENV_DIR}}/bin/ruff format --check ."
      - "{{.VENV_DIR}}/bin/mypy packages/"
      - "{{.VENV_DIR}}/bin/pytest"
      - python scripts/check_secrets.py .
      - python scripts/generate_staging.py --check
      - python scripts/generate_platform_health.py --check

  docs:build:
    desc: "Generate dictionary, build MkDocs (strict)"
    deps: [install]
    cmds:
      - "{{.VENV_DIR}}/bin/python scripts/generate_docs.py"
      - "{{.VENV_DIR}}/bin/mkdocs build --strict"

  plan:dev:
    desc: "SQLMesh plan against the dev virtual env (interactive)"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan dev {{.CLI_ARGS}}"]
  plan:prod:
    desc: "SQLMesh plan against prod — fails if prod is ahead of dev"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan prod {{.CLI_ARGS}}"]
  promote:
    desc: "Auto-apply dev → prod promotion (assumes dev verified)"
    dir: transforms/main
    cmds: ["../../{{.VENV_DIR}}/bin/sqlmesh plan prod --auto-apply {{.CLI_ARGS}}"]
  verify:dev:
    desc: "Run every Soda contract against the __dev schemas"
    cmds: ["{{.VENV_DIR}}/bin/python scripts/verify_dev.py"]

  full-refresh:
    desc: "Every dlt source + SQLMesh + Soda via Dagster — logs to .logs/"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster" }
    cmds: ["./scripts/run-logged.sh full-refresh -- {{.VENV_DIR}}/bin/dagster asset materialize --select '*' -m databox.orchestration.definitions"]
  verify:
    desc: "Smoke full-refresh — DATABOX_SMOKE=1 caps each source to 5 items — logs to .logs/"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster", DATABOX_SMOKE: "1" }
    cmds: ["./scripts/run-logged.sh verify -- {{.VENV_DIR}}/bin/dagster asset materialize --select '*' -m databox.orchestration.definitions"]
  dagster:dev:
    desc: "Launch Dagster UI — discovers workspace.yaml at repo root"
    env: { DAGSTER_HOME: "{{.USER_WORKING_DIR}}/.dagster" }
    cmds: ["{{.VENV_DIR}}/bin/dagster dev"]

  streamlit:
    desc: "Launch Databox Explorer"
    dir: app
    cmds: ["../{{.VENV_DIR}}/bin/streamlit run main.py"]
  init:
    desc: "Rebrand a fresh fork from scaffold.yaml (see docs/template.md)"
    cmds: ["python scripts/bootstrap.py {{.CLI_ARGS}}"]
  new-source:
    desc: "Scaffold a new dlt source (see docs/new-source.md)"
    cmds: ["python scripts/new_source.py {{.CLI_ARGS}}"]

  db:reset:
    desc: "Delete local DuckDB files (MotherDuck dbs must be dropped manually)"
    cmds: ["rm -f data/databox.duckdb data/sqlmesh_state.duckdb data/raw_ebird.duckdb data/raw_noaa.duckdb data/raw_usgs.duckdb"]
  clean:
    desc: "Remove build + test + cache artifacts"
    cmds:
      - rm -rf build/ dist/ *.egg-info/ .pytest_cache/ .coverage htmlcov/
      - find . -type d -name __pycache__ -exec rm -rf {} +
  clean-all:
    desc: "Clean + drop .venv, data/, .dlt_state/"
    deps: [clean]
    cmds: ["rm -rf {{.VENV_DIR}} data .dlt_state/"]