diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8d4cda8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 + +[*.py] +indent_size = 4 + +[Makefile] +indent_style = tab diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 267f5b8..b9f887b 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,15 +1 @@ -# These are supported funding model platforms - -github: 508-dev # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -polar: # Replace with a single Polar username -buy_me_a_coffee: # Replace with a single Buy Me a Coffee username -thanks_dev: # Replace with a single thanks.dev username -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] +github: 508-dev diff --git a/.github/ISSUE_TEMPLATE/docs_request.yml b/.github/ISSUE_TEMPLATE/docs_request.yml new file mode 100644 index 0000000..3bb77bc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/docs_request.yml @@ -0,0 +1,29 @@ +name: Documentation Request +description: Request new or improved documentation. +title: "docs: " +labels: + - documentation +body: + - type: textarea + id: gap + attributes: + label: Documentation Gap + description: What is missing, unclear, stale, or hard to find? + validations: + required: true + - type: textarea + id: audience + attributes: + label: Audience + description: Who needs this documentation? + placeholder: API consumers, scraper operators, contributors, agents, etc. + validations: + required: false + - type: textarea + id: suggested-location + attributes: + label: Suggested Location + description: Where should this live? + placeholder: README.md, docs/USAGE.md, docs/ARCHITECTURE.md, CONTRIBUTING.md + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..db1f135 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,41 @@ +name: Feature Request +description: Propose a new scraper capability or workflow. +title: "feature: " +labels: + - enhancement +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What user, scraper, or downstream-consumer problem should this solve? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed Solution + description: What should change? + validations: + required: true + - type: textarea + id: output-contract + attributes: + label: Output Contract + description: Describe any expected JSON shape, field additions, or compatibility constraints. + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: What other approaches are available? + validations: + required: false + - type: textarea + id: context + attributes: + label: Additional Context + description: Links, sanitized payloads, screenshots, examples, constraints, or non-goals. + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 24feb28..157e7ba 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,21 @@ - +## Summary - +- -## Description +## Scraper Impact - +- [ ] Saved-list extraction +- [ ] Place-page extraction +- [ ] CLI or public API +- [ ] Documentation/tooling only -## Related Issue +## Validation - +- [ ] Ran the narrowest relevant checks. +- [ ] Added or updated focused tests for parser, scraper, CLI, or public API changes. +- [ ] Updated docs or examples when behavior changed. -## How Has This Been Tested? +## Risk - +- [ ] Low risk, isolated change. +- [ ] Needs reviewer attention for schema drift, browser behavior, LLM repair, dependency changes, or output-contract compatibility. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dc0475..1211097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,12 +5,22 @@ on: push: branches: - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@v6 @@ -19,6 +29,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + enable-cache: true - name: Sync dependencies run: uv sync --locked --dev @@ -30,6 +42,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@v6 @@ -38,6 +52,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + enable-cache: true - name: Sync dependencies run: uv sync --locked --dev @@ -49,6 +65,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@v6 @@ -57,9 +75,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + enable-cache: true - name: Sync dependencies run: uv sync --locked --dev - name: Test - run: uv run python -m unittest discover -s tests + run: ./scripts/test.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04834a2..3ae84a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,16 @@ Run the local gates before opening or updating a PR: ```bash ./scripts/lint.sh ./scripts/typecheck.sh -uv run python -m unittest discover -s tests +./scripts/test.sh +./scripts/check-all.sh ``` +`./scripts/check-all.sh` is the same local gate in one command. It runs linting, +type checking, and unit tests. + +Dependency changes should follow [docs/supply-chain.md](docs/supply-chain.md): +keep `uv.lock` committed and prefer locked installs. + ## Pull Request Expectations - Keep changes scoped to the scraper behavior being fixed. diff --git a/README.md b/README.md index d15d9dc..aec45f5 100644 --- a/README.md +++ b/README.md @@ -196,8 +196,12 @@ the CLI, when a refresh only needs overview facts. LLM setup, cache behavior, and downstream refresh examples - [Architecture](docs/ARCHITECTURE.md): extraction layers, diagnostics, LLM repair, translation memory, and session/concurrency design +- [Supply Chain](docs/supply-chain.md): dependency cooldowns, locked installs, + and local artifact handling - [Contributing](CONTRIBUTING.md): local development, tests, PR expectations, and translation-memory promotion +- [Security](SECURITY.md): vulnerability reporting, secret handling, and + dependency policy ## Development @@ -205,5 +209,11 @@ the CLI, when a refresh only needs overview facts. uv sync --dev ./scripts/lint.sh ./scripts/typecheck.sh -uv run python -m unittest discover -s tests +./scripts/test.sh +``` + +Run the complete local gate with: + +```bash +./scripts/check-all.sh ``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6fa7ed0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Reporting Vulnerabilities + +Do not open public issues for vulnerabilities or leaked secrets. + +Report security concerns to the project maintainers through the repository's +private security advisory flow or another private channel configured by the +maintainers. + +## Secret Handling + +- Keep secrets in environment variables or local ignored files. +- Never commit real `.env` files, tokens, private keys, credentials, Google Maps + session state, browser profiles, or production data. +- Use documented examples for configuration only. +- Keep debug dumps, screenshots, and browser artifacts out of committed docs + unless they are intentionally sanitized fixtures. + +## Dependency Policy + +This project uses `uv` with dependency cooldowns and locked installs: + +- `pyproject.toml` sets `exclude-newer = "7 days"` for `uv` and `uv pip`. +- CI should use `uv sync --locked --dev`. +- Dependency changes should keep `uv.lock` committed and reviewable. + +## GitHub Actions + +Workflows should use least-privilege permissions, pinned or reviewed action +versions, and `persist-credentials: false` where practical. diff --git a/docs/supply-chain.md b/docs/supply-chain.md new file mode 100644 index 0000000..4ef0daf --- /dev/null +++ b/docs/supply-chain.md @@ -0,0 +1,33 @@ +# Dependency Supply-Chain Policy + +This project is a Python package managed with `uv`. + +## uv + +Dependency resolution uses a seven-day cooldown: + +```toml +[tool.uv] +exclude-newer = "7 days" + +[tool.uv.pip] +exclude-newer = "7 days" +``` + +Keep `uv.lock` committed. CI and release workflows should install with: + +```bash +uv sync --locked --dev +``` + +## Runtime Dependencies + +Treat dependency changes as executable-code changes. Before adding or upgrading +packages, review the package purpose, release recency, lockfile diff, and any +native or browser automation surface area. + +## Local Artifacts + +Do not commit browser profiles, session directories, raw Google Maps payload +dumps, LLM cache files, learned translation-memory files, or screenshots unless +they are intentionally sanitized fixtures. diff --git a/pyproject.toml b/pyproject.toml index 4ce2115..037999a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,9 @@ dev = [ [tool.uv] exclude-newer = "7 days" +[tool.uv.pip] +exclude-newer = "7 days" + [tool.hatch.build.targets.wheel] packages = ["src/gmaps_scraper"] @@ -38,6 +41,7 @@ strict = true [tool.ruff] line-length = 100 target-version = "py314" +src = ["src", "tests"] [tool.ruff.lint] select = ["B", "E", "F", "I", "UP"] diff --git a/scripts/check-all.sh b/scripts/check-all.sh new file mode 100755 index 0000000..b22ae93 --- /dev/null +++ b/scripts/check-all.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +./scripts/lint.sh +./scripts/typecheck.sh +./scripts/test.sh diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..631553a --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +uv run python -m unittest discover -s tests