diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 209834e20..000000000 --- a/.flake8 +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -max-line-length = 88 -exclude = - */migrations/*, - scripts/*, - adhoc/*, - node_modules/*, - .venv/*, - venv/*, - .tox/*, - __pycache__/*, - mediafiles/*, - .git/*, - *.json, diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2a90a1bab..9185df6c7 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -33,21 +33,23 @@ suggestions and reviews match the architecture instead of fighting it. Deeper op (name the bad instance), not the tolerance. - **Every caught exception is persisted, once (ADR 0019 + 0001).** Errors live in the `AppError` table, - not just stdout. Use the two-arm dedup pattern so a single failure is logged once as it unwinds: + not just stdout. `persist_app_error` is idempotent — it marks the exception and returns the existing + row on any later call — so a single failure is one row as it unwinds, no wrapper needed: ```python - from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error try: operation() - except AlreadyLoggedException: - raise # already persisted upstream — pass through unchanged except Exception as exc: - err = persist_app_error(exc) # MANDATORY - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) # idempotent — one AppError row per failure + raise ``` + A handler that *converts* the exception must chain the cause (`raise ValueError(...) from exc`), or the + converted failure earns a second row (pylint `W0707` enforces this). At the HTTP boundary, read the id + with `app_error_for(exc)` for `error_id` (ADR 0013) and map the status from the exception's real type. + - **Backend owns data; frontend owns presentation (ADR 0020).** Anything involving the DB, business rules, or external systems is backend. Static UI constants, layout, and ergonomics are frontend. The boundary is the *kind of value*, not the layer of code. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ba974121..4b29d51ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: cache-dependency-path: poetry.lock - name: Set up Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: 22 cache: 'npm' @@ -162,7 +162,7 @@ jobs: working-directory: frontend steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: 22 cache: 'npm' @@ -179,7 +179,7 @@ jobs: working-directory: frontend steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: 22 cache: 'npm' @@ -195,7 +195,7 @@ jobs: working-directory: frontend steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: 22 cache: 'npm' @@ -215,7 +215,7 @@ jobs: VITE_API_BASE_URL: http://localhost:8000 steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: 22 cache: 'npm' diff --git a/.gitignore b/.gitignore index 0d5c84a69..c6e5710e7 100644 --- a/.gitignore +++ b/.gitignore @@ -57,8 +57,11 @@ instance/ # Sphinx documentation docs/_build/ -# Planning artefacts (Claude Code plan-mode files; local-only) -docs/plans/ +# Planning artefacts (Claude Code plan-mode files; local-only). +# The pattern must match the contents, not the directory: git never descends +# into an excluded directory, which would leave the negation unreachable. +docs/plans/* +!docs/plans/_template.md # PyBuilder target/ @@ -275,3 +278,6 @@ session-replays/ docs/.codesight/ frontend/.codesight/ *.tsv + +# Local state for scripts/write_google_doc.py (per-instance runtime data) +scripts/google_doc_manifest.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbeb3ad94..3235214dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,17 +105,34 @@ repos: types: [python] exclude: '.*/migrations/.*\.py$' - # pylint - check for common bugs and merge issues + # pylint - check for common bugs and merge issues. + # + # Deliberately an allowlist, not full pylint. Full pylint reports ~4300 + # findings here, ~72% of which are noise or actively fight this codebase: + # line-too-long duplicates Black, abstract-method is a DRF false positive, + # duplicate-code is cross-module (so it cannot baseline stably), and + # broad-exception-caught fires on the handler pattern ADR 0019 mandates. + # Rules below earn their place; each was verified against real findings. + # Rejected after testing, all high false-positive on Django/SDK dynamism: + # E1111/E1121 (SDK property monkeypatching), E0203 (Django FK _id + # descriptors), E0606, E1120, C0325. + # # E0102: function-redefined (duplicate methods) # E0108: duplicate-argument-name # E0118: used-prior-to-assignment + # E0701: bad-except-order (subclass handler shadowed - found a live bug) + # E1123: unexpected-keyword-arg (found a live TypeError) + # W0101: unreachable # W0104: pointless-statement (no effect) # W0199: assert-on-tuple (always true) # W0221: arguments-differ (override signature mismatch) + # W0612: unused-variable # W0622: redefined-builtin (redefining id, type, etc) + # W0707: raise-missing-from (exception chaining, per ADR 0001) + # R1710: inconsistent-return-statements - id: pylint-bugs name: Check for common bugs with pylint - entry: poetry run python -m pylint --disable=all --enable=E0102,E0108,E0118,W0104,W0199,W0221,W0622 --ignore-patterns='migrations/.*\.py' + entry: poetry run python -m pylint --disable=all --enable=E0102,E0108,E0118,E0701,E1123,W0101,W0104,W0199,W0221,W0612,W0622,W0707,R1710 --ignore-patterns='migrations/.*\.py' language: system files: '^apps/.+\.py$' types: [python] diff --git a/.vscode/tasks.json b/.vscode/tasks.json index b90360605..2ca3ff660 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -28,24 +28,24 @@ } }, { - "label": "Frontend Manual Dev Server", + "label": "Frontend Preview (build)", "type": "process", "command": "${env:HOME}/.nvm/nvm-exec", - "args": ["npm", "run", "manual:dev"], + "args": ["npm", "run", "preview:e2e"], "hide": true, "options": { "cwd": "${workspaceFolder}/frontend" }, "isBackground": true, "problemMatcher": { - "owner": "vitepress", + "owner": "vite-preview", "pattern": { "regexp": "^$" }, "background": { "activeOnStart": true, - "beginsPattern": "vitepress", - "endsPattern": "http://localhost" + "beginsPattern": ".", + "endsPattern": "Local:" } }, "presentation": { @@ -75,6 +75,34 @@ "panel": "dedicated" } }, + { + "label": "Django (runserver)", + "type": "process", + "command": "${workspaceFolder}/.venv/bin/python", + "args": ["${workspaceFolder}/manage.py", "runserver", "--noreload"], + "options": { + "env": { + "PYTHONPATH": "${workspaceFolder}" + } + }, + "hide": true, + "isBackground": true, + "problemMatcher": { + "owner": "django", + "pattern": { + "regexp": "^$" + }, + "background": { + "activeOnStart": true, + "beginsPattern": ".", + "endsPattern": "Starting development server" + } + }, + "presentation": { + "reveal": "always", + "panel": "dedicated" + } + }, { "label": "Celery Worker", "type": "shell", @@ -123,7 +151,19 @@ "label": "Start Dev Environment", "dependsOn": [ "Frontend Dev Server", - "Frontend Manual Dev Server", + "Ngrok Tunnels", + "Celery Worker", + "Celery Beat" + ], + "dependsOrder": "parallel", + "problemMatcher": [] + }, + { + "label": "Start E2E Environment", + "detail": "Serves the production BUILD on :5173 + plain Django on :8000 (no debugger) for E2E runs, fast over ngrok/LAN. Stop 'Start Dev Environment' AND the Run>Debug 'Django' session first — both bind :5173/:8000.", + "dependsOn": [ + "Frontend Preview (build)", + "Django (runserver)", "Ngrok Tunnels", "Celery Worker", "Celery Beat" diff --git a/CLAUDE.md b/CLAUDE.md index fdd93400e..2f4a35543 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,10 +7,11 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Major architectural decisions are recorded in [`docs/adr/`](docs/adr/README.md). Read the ADR index before non-trivial work — the codebase deviates from typical Django/Vue defaults in deliberate ways that aren't reconstructable from the code alone. The most operationally consequential ADRs: - **0017** — Zero backwards compatibility: when a name, URL, or shape changes, every caller changes in the same PR. No deprecation aliases, no `getattr` shims, no "for safety" columns. -- **0019 + 0001** — Every exception is persisted to `AppError` (0019); nested handlers re-raise via the `AlreadyLoggedException` two-arm dedup pattern (0001). +- **0019 + 0001** — Every exception is persisted to `AppError` (0019); `persist_app_error` is idempotent, so handlers just persist and re-raise — one row per failure by construction, no wrapper (0001). - **0015** — When a consumer finds malformed data, fix the data (migration). Consumers stay strict; never add a read-side fallback. - **0020** — Backend owns data, calculations, and external systems; frontend owns presentation. The boundary is the kind of value, not the layer of code. - **0021** — Frontend reads/writes the API only via the generated client; raw `fetch`/`axios` is forbidden. +- **0032** — Less code is better: prefer a maintained library over a homegrown implementation. Writing your own for something a library provides needs an explicit, recorded reason it isn't a library. CLAUDE.md is the operational layer (session behaviour, code-style gotchas, architecture facts). ADRs explain *why*. @@ -120,11 +121,17 @@ ADJUSTMENT entries (kind='adjust'): fixes when that is the pragmatic path, but direct-to-main commits are banned. - Before committing, check the current branch. If it is `main`, create or switch to a branch first. -- Do not leave uncommitted changes behind at the end of a task. If the change is - complete and scoped, commit it on the current branch. If the scope is unclear, - mixed with unrelated work, or the user may not want it committed, ask before - committing. +- Commits are all-or-nothing for the worktree: either do not commit at all, or + commit every tracked change together. Never use a path-limited commit. +- Do not leave uncommitted changes behind at the end of a task. Finish and + verify incomplete work before committing the whole worktree. +- `docs/plans/` is ephemeral scratch — one plan per piece of work, gitignored + (except `_template.md`). Delete a plan when its PR is opened, having first + migrated anything durable to its real home: open work → the Jira ticket, tools + → `scripts/`, decisions → an ADR. Never leave a non-plan artifact (script, data + file) sitting in `docs/plans/` — it goes through that same migrate-or-delete gate. - Run focused tests for touched code when useful. Do not manually run expensive hook commands like `bash scripts/check_mypy.sh`, `npm run test:unit`, `npm run lint`, `npm run type-check`, or frontend builds unless diagnosing a hook failure; they run automatically during `git commit`/`git push`. +- Tests must protect enduring behaviour, invariants, or algorithms. Never assert the implementation's own text — `assertIn` on source code, a CLI flag or log string, or source line ordering — which mirrors the code, breaks on every refactor, and catches no bug. Execute the code path and assert the observable outcome: return value, exit code, output, or resulting state. ### Code Style and Quality @@ -159,24 +166,33 @@ ADR 0015 (fix data, not fallback) and ADR 0017 (zero backwards compatibility) ar ### Mandatory error persistence -Every exception handler persists once via `persist_app_error(exc)` (ADR 0019) and re-raises through the two-arm dedup pattern (ADR 0001). +A `try` needs a strong reason: you are going to **handle** the failure — reshape it (domain error, or an HTTP status at the boundary), or persist it from the layer that understands it well enough to add business context. Otherwise let it raise. + +Every handler you do write persists via `persist_app_error(exc)` (ADR 0019) and re-raises. `persist_app_error` is idempotent — it marks the exception and returns the existing row on any later call — so one failure is one `AppError` row no matter how many layers catch it (ADR 0001). No wrapper type, no pass-through arm. ```python -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error try: operation() -except AlreadyLoggedException: - raise # already persisted upstream — pass through unchanged except Exception as exc: - err = persist_app_error(exc) # MANDATORY - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc, job_id=job.id) # the context is why this handler exists + raise ``` +A handler that *converts* the exception must chain the cause, or the converted failure earns a second row (pylint `W0707` enforces this): + +```python +except Job.DoesNotExist as exc: + persist_app_error(exc) + raise ValueError(f"Job {job_id} not found") from exc +``` + +At the HTTP boundary, read the persisted id with `app_error_for(exc)` to include `error_id` in the response (ADR 0013), and map the status from the exception's real type. + ## Environment Configuration -See `.env.example` for required environment variables. Key integrations: Xero API, Dropbox, PostgreSQL. Frontend tooling reads `APP_DOMAIN` from the backend `.env` at `../.env` and derives URLs from it (see ADR 0008's Consequences). Deploy uses `scripts/server/deploy.sh` (per-instance `-`); it also runs on boot via systemd so a cold machine catches up to `production`. Servers only ever run the `production` branch — `main` is the integration branch and is never deployed (ADR 0029). +See `.env.example` for required environment variables. Key integrations: Xero API, Dropbox, PostgreSQL. Frontend tooling reads `APP_DOMAIN` from the backend `.env` at `../.env` and derives URLs from it (see ADR 0008's Consequences). Deploy uses `scripts/server/deploy.sh` (per-instance `-`); it also runs on boot via systemd so a cold machine catches up to `production`. Servers run the `production` branch by default; `main` is the integration branch — never deployed to production, but deployed to UAT as a release candidate via `deploy.sh --ref` / `instance.sh create --ref` (ADR 0029). ## Migration Management @@ -197,8 +213,6 @@ See ADR 0020. Backend owns data, calculations, and external systems; frontend ow `npm run test:e2e` (from `frontend/`) runs Playwright tests against live HTTP endpoints. The suite takes ~20–25 min. -**CRITICAL: The backend serving `APP_DOMAIN` must run with `XERO_READONLY=True`.** The flag swaps in a provider that suppresses all Xero writes (contacts, invoices, quotes, attachments, history notes) while reads and token refresh stay live; without it the suite writes real entities into the connected Xero tenant. Global-setup enforces this via `/api/xero/ping/` (`xero_readonly` field) and aborts otherwise. The flag is process-scoped: any celery worker/beat sharing the DB must also run with `XERO_READONLY=True`, or the hourly `xero_regular_sync_task` will push local `[TEST]` stock to Xero — global-setup cannot verify a worker's environment. - **CRITICAL: The global teardown (`global-teardown.ts`) MUST always run to completion.** It restores the database from backup, saves/reinjects Xero tokens, removes the lock file, and runs integrity checks. If the bash process is killed (timeout, SIGTERM, etc.) the teardown never executes and the database is left polluted with `[TEST]` data. - **Never set a bash timeout on the E2E command.** A timeout (or SIGTERM) kills the node process before Playwright calls `globalTeardown`, leaving the DB polluted with `[TEST]` data and a stale lock file. The teardown is NOT a signal handler — it only fires on normal exit. diff --git a/apps/accounting/services/core.py b/apps/accounting/services/core.py index a62a9fd84..5157d6db1 100644 --- a/apps/accounting/services/core.py +++ b/apps/accounting/services/core.py @@ -17,19 +17,12 @@ from apps.accounts.utils import get_displayable_staff, get_payroll_excluded_staff_ids from apps.job.models import Job from apps.job.models.costing import CostLine -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults from apps.workflow.services.error_persistence import persist_app_error logger = getLogger(__name__) -def _persist_and_raise(exception: Exception, **context) -> None: - """Persist an exception and re-raise as AlreadyLoggedException.""" - app_error = persist_app_error(exception, **context) - raise AlreadyLoggedException(exception, app_error.id) - - class KPIService: """ Service responsible for calculating and providing KPI metrics for reports. @@ -790,11 +783,9 @@ def get_job_aging_data(include_archived: bool = False) -> Dict[str, Any]: jobs_query = jobs_query.exclude(status="archived") jobs = jobs_query.order_by("-created_at") - except AlreadyLoggedException: - raise except Exception as exc: logger.error(f"Database error fetching jobs: {str(exc)}") - _persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "fetch_jobs_for_aging_report", @@ -802,6 +793,7 @@ def get_job_aging_data(include_archived: bool = False) -> Dict[str, Any]: "query_filters": "active_jobs_with_company_and_cost_data", }, ) + raise job_data = [] for job in jobs: @@ -970,12 +962,6 @@ def _get_timing_data(job: Job) -> Dict[str, Any]: last_activity = JobAgingService._get_last_activity(job) if last_activity: timing_data.update(last_activity) - except AlreadyLoggedException as exc: - logger.warning( - "Error getting last activity for job %s: %s", - job.job_number, - exc.original, - ) except Exception as exc: logger.warning( f"Error getting last activity for job {job.job_number}: {str(exc)}" @@ -1086,7 +1072,7 @@ def _get_last_activity(job: Job) -> Dict[str, Any]: ) logger.error(message) logged_exc = ValueError(message) - app_error = persist_app_error( + persist_app_error( logged_exc, job_id=job.id, additional_context={ @@ -1095,9 +1081,7 @@ def _get_last_activity(job: Job) -> Dict[str, Any]: "staff_id": cost_line.staff_id, }, ) - raise AlreadyLoggedException( - logged_exc, app_error.id - ) from exc + raise logged_exc from exc activities.append( { @@ -1106,8 +1090,6 @@ def _get_last_activity(job: Job) -> Dict[str, Any]: "description": description, } ) - except AlreadyLoggedException: - raise except Exception as exc: logger.error( ( @@ -1115,7 +1097,7 @@ def _get_last_activity(job: Job) -> Dict[str, Any]: f"{job.job_number}: {str(exc)}" ) ) - _persist_and_raise( + persist_app_error( exc, job_id=job.id, additional_context={ @@ -1123,6 +1105,7 @@ def _get_last_activity(job: Job) -> Dict[str, Any]: "job_number": job.job_number, }, ) + raise # Find the most recent activity if activities: @@ -1242,11 +1225,9 @@ def get_staff_performance_data( "period_summary": period_summary, } - except AlreadyLoggedException: - raise except Exception as exc: logger.error(f"Error getting staff performance data: {str(exc)}") - _persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "staff_performance_data", @@ -1255,6 +1236,7 @@ def get_staff_performance_data( "staff_id": staff_id, }, ) + raise @staticmethod def _calculate_staff_metrics( diff --git a/apps/accounting/services/payroll_reconciliation_service.py b/apps/accounting/services/payroll_reconciliation_service.py index 57255c795..2e70c4058 100644 --- a/apps/accounting/services/payroll_reconciliation_service.py +++ b/apps/accounting/services/payroll_reconciliation_service.py @@ -6,11 +6,9 @@ from apps.accounts.models import Staff from apps.job.models import CostLine -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models.company_defaults import CompanyDefaults from apps.workflow.models.xero_payroll import XeroPayRun - -from .core import _persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -105,13 +103,12 @@ def get_reconciliation_data(start_date: date, end_date: date) -> dict[str, Any]: "diff_pct": round(grand_pct, 1), }, } - except AlreadyLoggedException: - raise except Exception as exc: - _persist_and_raise( + persist_app_error( exc, additional_context={"operation": "payroll_reconciliation"}, ) + raise @staticmethod def get_aligned_date_range(start_date: date, end_date: date) -> dict[str, date]: diff --git a/apps/accounting/services/rdti_spend_service.py b/apps/accounting/services/rdti_spend_service.py index 2b43e41b9..b6386209d 100644 --- a/apps/accounting/services/rdti_spend_service.py +++ b/apps/accounting/services/rdti_spend_service.py @@ -8,7 +8,6 @@ from apps.job.enums import RDTIType from apps.job.models.costing import CostLine -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -19,21 +18,13 @@ ] + [("unclassified", "Unclassified")] -def _persist_and_raise(exception: Exception, **context: Any) -> None: - """Persist an exception and re-raise as AlreadyLoggedException.""" - app_error = persist_app_error(exception, **context) - raise AlreadyLoggedException(exception, app_error.id) - - class RDTISpendService: @staticmethod def get_rdti_spend_data(start_date: date, end_date: date) -> dict[str, Any]: try: return RDTISpendService._build_report(start_date, end_date) - except AlreadyLoggedException: - raise except Exception as exc: - _persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "rdti_spend_report", @@ -41,7 +32,7 @@ def get_rdti_spend_data(start_date: date, end_date: date) -> dict[str, Any]: "end_date": str(end_date), }, ) - raise # unreachable but keeps type checker happy + raise @staticmethod def _build_report(start_date: date, end_date: date) -> dict[str, Any]: diff --git a/apps/accounting/services/wip_service.py b/apps/accounting/services/wip_service.py index 0f11e087a..7570720cb 100644 --- a/apps/accounting/services/wip_service.py +++ b/apps/accounting/services/wip_service.py @@ -7,7 +7,6 @@ from apps.accounting.models import Invoice from apps.job.models import CostLine, Job -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -22,12 +21,6 @@ VALID_INVOICE_STATUSES = ["DRAFT", "SUBMITTED", "AUTHORISED", "PAID"] -def _persist_and_raise(exception: Exception, **context: Any) -> None: - """Persist an exception and re-raise as AlreadyLoggedException.""" - app_error = persist_app_error(exception, **context) - raise AlreadyLoggedException(exception, app_error.id) - - class WIPService: """ Calculates Work In Progress as at a given date. @@ -63,10 +56,8 @@ def get_wip_data(report_date: date, method: str) -> dict[str, Any]: .select_related("latest_actual", "company") .order_by("job_number") ) - except AlreadyLoggedException: - raise except Exception as exc: - _persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "wip_fetch_jobs", @@ -74,6 +65,7 @@ def get_wip_data(report_date: date, method: str) -> dict[str, Any]: "method": method, }, ) + raise wip_jobs: list[dict[str, Any]] = [] archived_jobs: list[dict[str, Any]] = [] diff --git a/apps/accounting/views/job_aging_view.py b/apps/accounting/views/job_aging_view.py index 5e2ab07af..46c0288a9 100644 --- a/apps/accounting/views/job_aging_view.py +++ b/apps/accounting/views/job_aging_view.py @@ -14,8 +14,8 @@ StandardErrorSerializer, ) from apps.accounting.services import JobAgingService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import ( + app_error_for, extract_request_context, persist_app_error, ) @@ -95,15 +95,15 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("Job Aging API Error: %s", exc.original) - return _build_standard_error_response( - message=f"Error obtaining job aging data: {exc.original}", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error(f"Job Aging API Error: {str(exc)}") + if app_error_for(exc) is not None: + return _build_standard_error_response( + message=f"Error obtaining job aging data: {str(exc)}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + request_context = extract_request_context(request) app_error = persist_app_error( exc, diff --git a/apps/accounting/views/rdti_spend_view.py b/apps/accounting/views/rdti_spend_view.py index 48ad4b33e..79cfc889b 100644 --- a/apps/accounting/views/rdti_spend_view.py +++ b/apps/accounting/views/rdti_spend_view.py @@ -14,8 +14,8 @@ RDTISpendResponseSerializer, ) from apps.accounting.services.rdti_spend_service import RDTISpendService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import ( + app_error_for, extract_request_context, persist_app_error, ) @@ -86,15 +86,15 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("RDTI Spend API Error: %s", exc.original) - return _build_standard_error_response( - message=f"Error obtaining RDTI spend data: {exc.original}", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error(f"RDTI Spend API Error: {str(exc)}") + if app_error_for(exc) is not None: + return _build_standard_error_response( + message=f"Error obtaining RDTI spend data: {str(exc)}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + request_context = extract_request_context(request) app_error = persist_app_error( exc, diff --git a/apps/accounting/views/sales_pipeline_view.py b/apps/accounting/views/sales_pipeline_view.py index 843270075..53b72c2d8 100644 --- a/apps/accounting/views/sales_pipeline_view.py +++ b/apps/accounting/views/sales_pipeline_view.py @@ -16,7 +16,6 @@ StandardErrorSerializer, ) from apps.accounting.services import SalesPipelineService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import ( extract_request_context, persist_app_error, @@ -105,16 +104,6 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: response_serializer.is_valid(raise_exception=True) return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("Sales Pipeline API Error: %s", exc.original) - details: dict[str, Any] | None = None - if exc.app_error_id is not None: - details = {"error_id": str(exc.app_error_id)} - return _build_standard_error_response( - message=f"Error obtaining sales pipeline data: {exc.original}", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - details=details, - ) except Exception as exc: logger.error(f"Sales Pipeline API Error: {str(exc)}") request_context = extract_request_context(request) diff --git a/apps/accounting/views/staff_performance_views.py b/apps/accounting/views/staff_performance_views.py index 98effcd3e..0bd4187aa 100644 --- a/apps/accounting/views/staff_performance_views.py +++ b/apps/accounting/views/staff_performance_views.py @@ -14,8 +14,7 @@ StaffPerformanceResponseSerializer, ) from apps.accounting.services import StaffPerformanceService -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_app_error +from apps.workflow.services.error_persistence import app_error_for, persist_app_error logger = getLogger(__name__) @@ -115,17 +114,16 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("Error in staff performance summary API: %s", exc.original) - return _build_staff_error_response( - message=( - "Internal server error occurred while generating staff " - "performance report" - ), - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error(f"Error in staff performance summary API: {str(exc)}") + if app_error_for(exc) is not None: + return _build_staff_error_response( + message=( + "Internal server error occurred while generating staff " + "performance report" + ), + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) app_error = persist_app_error( exc, additional_context={ @@ -229,17 +227,16 @@ def get( return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("Error in staff performance detail API: %s", exc.original) - return _build_staff_error_response( - message=( - "Internal server error occurred while generating staff " - "performance report" - ), - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error(f"Error in staff performance detail API: {str(exc)}") + if app_error_for(exc) is not None: + return _build_staff_error_response( + message=( + "Internal server error occurred while generating staff " + "performance report" + ), + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) app_error = persist_app_error( exc, additional_context={ diff --git a/apps/accounting/views/wip_view.py b/apps/accounting/views/wip_view.py index 16451489d..3d602e5e4 100644 --- a/apps/accounting/views/wip_view.py +++ b/apps/accounting/views/wip_view.py @@ -15,8 +15,8 @@ WIPResponseSerializer, ) from apps.accounting.services.wip_service import WIPService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import ( + app_error_for, extract_request_context, persist_app_error, ) @@ -99,15 +99,15 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("WIP Report API Error: %s", exc.original) - return _build_standard_error_response( - message=f"Error generating WIP report: {exc.original}", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error("WIP Report API Error: %s", exc) + if app_error_for(exc) is not None: + return _build_standard_error_response( + message=f"Error generating WIP report: {exc}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + request_context = extract_request_context(request) app_error = persist_app_error( exc, diff --git a/apps/accounts/__init__.py b/apps/accounts/__init__.py index 4754efe70..8ed6cda3d 100644 --- a/apps/accounts/__init__.py +++ b/apps/accounts/__init__.py @@ -10,10 +10,8 @@ from .models import Staff, StaffManager from .permissions import CanManageTimesheets, IsStaff, IsSuperuser from .serializers import ( - BaseStaffSerializer, CustomTokenObtainPairSerializer, EmptySerializer, - GenericStaffMethodsMixin, KanbanStaffSerializer, StaffCreateSerializer, StaffSerializer, @@ -34,11 +32,9 @@ __all__ = [ "AccountsConfig", - "BaseStaffSerializer", "CanManageTimesheets", "CustomTokenObtainPairSerializer", "EmptySerializer", - "GenericStaffMethodsMixin", "IsStaff", "IsSuperuser", "KanbanStaffSerializer", diff --git a/apps/accounts/apps.py b/apps/accounts/apps.py index a988be53e..e8b3a07d3 100644 --- a/apps/accounts/apps.py +++ b/apps/accounts/apps.py @@ -7,5 +7,7 @@ class AccountsConfig(AppConfig): verbose_name = "User Accounts" def ready(self) -> None: - # Import here to avoid AppRegistryNotReady during Django startup + # Imported for its import-time side effects only, so the name is + # deliberately unused (F401). Deferred to ready() because importing it + # at module level raises AppRegistryNotReady during Django startup. import apps.workflow.extensions # noqa: F401 diff --git a/apps/accounts/models.py b/apps/accounts/models.py index 817ba5a86..fbe85628f 100644 --- a/apps/accounts/models.py +++ b/apps/accounts/models.py @@ -109,9 +109,10 @@ class Staff(AbstractBaseUser, PermissionsMixin): ] # Internal fields not exposed via API (write-only or internal use). + # `icon` is deliberately absent: it is written only by the dedicated icon + # upload endpoint and read via the icon_url property. STAFF_INTERNAL_FIELDS = [ "password", - "icon", # Raw ImageField - use icon_url property for API ] # Computed properties exposed via API (read-only). diff --git a/apps/accounts/serializers.py b/apps/accounts/serializers.py index 1c5440eff..b62fc113c 100644 --- a/apps/accounts/serializers.py +++ b/apps/accounts/serializers.py @@ -1,5 +1,4 @@ import logging -from decimal import Decimal from typing import Any, Dict, Optional from django.contrib.auth import authenticate @@ -11,18 +10,20 @@ logger = logging.getLogger(__name__) -def _build_icon_url(staff: Staff, context: Optional[Dict[str, Any]]) -> Optional[str]: - """ - Build an absolute icon URL when possible, otherwise fall back to the stored path. +def _build_icon_url(staff: Staff) -> Optional[str]: + """Return the icon path relative to the site root. + + Deliberately relative. The browser resolves it against its own origin, so + the same value is correct behind ngrok in dev and behind the proxy in + UAT/production. Building an absolute URL from the request instead leaks the + internal host (http://localhost:8000/...) wherever the forwarded-host + headers aren't trusted, and the browser then blocks the image as a + cross-origin request to the loopback address space. """ if not staff.icon: return None - request = (context or {}).get("request") - try: - return request.build_absolute_uri(staff.icon.url) if request else staff.icon.url - except Exception: - return staff.icon.url + return staff.icon.url class EmptySerializer(serializers.Serializer): @@ -68,62 +69,11 @@ def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]: raise serializers.ValidationError("Invalid credentials") -class GenericStaffMethodsMixin: - """ - Utilitary methods shared between StaffSerializer and StaffCreateSerializer - - Normalises arrays received as "" (groups, user_permissions) - - Normalises decimal fields received as "" to "0.00" (wage_rate, hours_*) - """ - - ARRAY_FIELDS = ["groups", "user_permissions"] - DECIMAL_FIELDS = [ - "base_wage_rate", - "hours_mon", - "hours_tue", - "hours_wed", - "hours_thu", - "hours_fri", - "hours_sat", - "hours_sun", - ] - - def to_internal_value(self, data: Any) -> Dict[str, Any]: - is_querydict = hasattr(data, "getlist") - if is_querydict: - data = data.copy() - - for field in self.ARRAY_FIELDS: - if is_querydict: - values = data.getlist(field) - if values == [""] or values == [] or not values: - data.setlist(field, []) - else: - value = data.get(field) - if value in ("", None): - data[field] = [] - - for field in self.DECIMAL_FIELDS: - if is_querydict: - value = data.get(field) - if value == "": - data[field] = "0.00" - else: - if field in data and data[field] == "": - data[field] = str(Decimal("0.00")) - - return super().to_internal_value(data) - - -class BaseStaffSerializer(GenericStaffMethodsMixin, serializers.ModelSerializer): - """Base serializer for Staff model with shared logic for create and update operations.""" - - -class StaffSerializer(BaseStaffSerializer): - icon = serializers.ImageField(required=False, allow_null=True, write_only=True) +class StaffSerializer(serializers.ModelSerializer[Staff]): icon_url = serializers.SerializerMethodField(read_only=True) def get_icon_url(self, obj: Staff) -> Optional[str]: - return _build_icon_url(obj, self.context) + return _build_icon_url(obj) def update(self, instance: Staff, validated_data: Dict[str, Any]) -> Staff: password = validated_data.pop("password", None) @@ -153,16 +103,14 @@ class Meta: "preferred_name": {"required": False}, "xero_user_id": {"required": False}, "date_left": {"required": False}, - "icon": {"required": False, "write_only": True}, } -class StaffCreateSerializer(BaseStaffSerializer): - icon = serializers.ImageField(required=False, allow_null=True, write_only=True) +class StaffCreateSerializer(serializers.ModelSerializer[Staff]): icon_url = serializers.SerializerMethodField(read_only=True) def get_icon_url(self, obj: Staff) -> Optional[str]: - return _build_icon_url(obj, self.context) + return _build_icon_url(obj) def create(self, validated_data: Dict[str, Any]) -> Staff: password = validated_data.pop("password", None) @@ -199,7 +147,6 @@ class Meta: "xero_user_id": {"required": False}, "date_left": {"required": False}, "password_needs_reset": {"required": False}, - "icon": {"required": False, "write_only": True}, } @@ -211,7 +158,7 @@ class KanbanStaffSerializer(serializers.ModelSerializer): icon_url = serializers.SerializerMethodField() def get_icon_url(self, obj: Staff) -> Optional[str]: - return _build_icon_url(obj, self.context) + return _build_icon_url(obj) class Meta: model = Staff diff --git a/apps/accounts/tests/test_staff_api.py b/apps/accounts/tests/test_staff_api.py index 9286b9971..e143d9ebf 100644 --- a/apps/accounts/tests/test_staff_api.py +++ b/apps/accounts/tests/test_staff_api.py @@ -1,12 +1,28 @@ +import datetime +import io +import os +import tempfile +from typing import Any, ClassVar + from django.contrib.auth.models import Group +from django.core.files.uploadedfile import SimpleUploadedFile from django.db import connection -from django.test.utils import CaptureQueriesContext +from django.test.utils import CaptureQueriesContext, override_settings +from PIL import Image +from rest_framework.response import Response from rest_framework.test import APIClient from apps.accounts.models import Staff from apps.testing import BaseTestCase +def _png_bytes(size: int = 8) -> bytes: + """Build a real PNG so ImageField validation has something to accept.""" + buffer = io.BytesIO() + Image.new("RGB", (size, size), color="red").save(buffer, format="PNG") + return buffer.getvalue() + + class StaffListCreateAPIViewTests(BaseTestCase): def test_staff_list_prefetches_groups_for_serializer(self): office_user = Staff.objects.create_user( @@ -50,3 +66,308 @@ def test_staff_list_prefetches_groups_for_serializer(self): and "accounts_staff_groups" in query["sql"].lower() ] self.assertEqual(len(group_queries), 1) + + +class StaffDetailAPIViewTests(BaseTestCase): + def test_staff_cannot_be_deleted_via_api(self) -> None: + """Staff are offboarded by setting date_left, never deleted. The detail + endpoint must reject DELETE so a hard delete (which would orphan or be + blocked by protected time entries) can't be reintroduced.""" + office_user = Staff.objects.create_user( + email="office@example.test", + password="testpass", + first_name="Office", + last_name="User", + is_office_staff=True, + ) + target = Staff.objects.create_user( + email="leaver@example.test", + password="testpass", + first_name="Depa", + last_name="Rting", + ) + + client = APIClient() + client.force_authenticate(user=office_user) + + response = client.delete(f"/api/accounts/staff/{target.id}/") + + self.assertEqual(response.status_code, 405) + + +class StaffJSONContractTests(BaseTestCase): + """The staff resource is JSON-only. + + These pin the shapes the admin staff form actually sends. They exist + because the endpoints previously accepted only multipart, which cannot + express null, numbers, or arrays — every value arrived as a string and a + serializer mixin hand-rebuilt the types. A blank Date Left was serialised + as the literal text "null" and rejected, breaking staff create and edit. + """ + + def setUp(self) -> None: + super().setUp() + self.admin = Staff.objects.create_user( + email="admin@example.test", + password="testpass", + first_name="Admin", + last_name="User", + is_office_staff=True, + ) + self.client_api = APIClient() + self.client_api.force_authenticate(user=self.admin) + + def test_create_leaves_a_new_staff_member_active(self) -> None: + """A new staff member has no leaving date, so they are current.""" + response = self.client_api.post( + "/api/accounts/staff/", + { + "email": "newstarter@example.test", + "first_name": "New", + "last_name": "Starter", + "password": "TestPassword123!", + "base_wage_rate": 32.5, + "date_left": None, + }, + format="json", + ) + + self.assertEqual(response.status_code, 201, response.content) + created = Staff.objects.get(email="newstarter@example.test") + self.assertIsNone(created.date_left) + self.assertTrue(created.is_currently_active) + + def test_setting_date_left_offboards_a_staff_member(self) -> None: + target = Staff.objects.create_user( + email="leaving@example.test", + password="testpass", + first_name="Going", + last_name="Away", + ) + + response = self.client_api.patch( + f"/api/accounts/staff/{target.id}/", + {"date_left": "2026-07-01"}, + format="json", + ) + + self.assertEqual(response.status_code, 200, response.content) + target.refresh_from_db() + self.assertEqual(target.date_left, datetime.date(2026, 7, 1)) + + def test_clearing_date_left_reinstates_an_offboarded_staff_member(self) -> None: + """Clearing the Date Left field brings someone back onto the books. + + This is why date_left is always sent rather than omitted — omitting a + field cannot clear it on a PATCH. + """ + target = Staff.objects.create_user( + email="returning@example.test", + password="testpass", + first_name="Back", + last_name="Again", + ) + target.date_left = datetime.date(2026, 1, 31) + target.save(update_fields=["date_left"]) + + response = self.client_api.patch( + f"/api/accounts/staff/{target.id}/", + {"date_left": None}, + format="json", + ) + + self.assertEqual(response.status_code, 200, response.content) + target.refresh_from_db() + self.assertIsNone(target.date_left) + self.assertTrue(target.is_currently_active) + + def test_groups_round_trip_as_a_json_array(self) -> None: + """Permissions arrive as a real array, not a comma-joined string.""" + group = Group.objects.create(name="Estimators") + target = Staff.objects.create_user( + email="grouped@example.test", + password="testpass", + first_name="Group", + last_name="Member", + ) + + response = self.client_api.patch( + f"/api/accounts/staff/{target.id}/", + {"groups": [group.id]}, + format="json", + ) + + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(list(target.groups.values_list("id", flat=True)), [group.id]) + + def test_empty_groups_array_clears_membership(self) -> None: + group = Group.objects.create(name="Temporary") + target = Staff.objects.create_user( + email="ungrouped@example.test", + password="testpass", + first_name="No", + last_name="Groups", + ) + target.groups.add(group) + + response = self.client_api.patch( + f"/api/accounts/staff/{target.id}/", + {"groups": []}, + format="json", + ) + + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(list(target.groups.all()), []) + + +class StaffIconAPIViewTests(BaseTestCase): + """Profile pictures upload through their own endpoint. + + The staff resource is JSON, which cannot carry a file, so the icon has a + dedicated multipart endpoint — the same split already used for company + logos and job files. + + MEDIA_ROOT is redirected to a temporary directory: these tests write real + files, and the default root is the developer's own mediafiles/ tree. + """ + + _media: ClassVar[tempfile.TemporaryDirectory] # type: ignore[type-arg] # py3.12 stub is ungeneric + _media_override: ClassVar[Any] + + @classmethod + def setUpClass(cls) -> None: + cls._media = tempfile.TemporaryDirectory(prefix="staff-icons-test-") + cls._media_override = override_settings(MEDIA_ROOT=cls._media.name) + # Enabled before super() so the base fixture copying also lands in the + # temporary tree rather than the real one. + cls._media_override.enable() + super().setUpClass() + + @classmethod + def tearDownClass(cls) -> None: + super().tearDownClass() + cls._media_override.disable() + cls._media.cleanup() + + def setUp(self) -> None: + super().setUp() + self.admin = Staff.objects.create_user( + email="iconadmin@example.test", + password="testpass", + first_name="Icon", + last_name="Admin", + is_office_staff=True, + ) + self.target = Staff.objects.create_user( + email="photographed@example.test", + password="testpass", + first_name="Photo", + last_name="Subject", + ) + self.client_api = APIClient() + self.client_api.force_authenticate(user=self.admin) + + def _upload(self, upload: SimpleUploadedFile, staff_id: object = None) -> Response: + return self.client_api.post( + f"/api/accounts/staff/{staff_id or self.target.id}/icon/", + {"file": upload}, + format="multipart", + ) + + def test_upload_sets_the_icon_and_returns_its_url(self) -> None: + response = self._upload( + SimpleUploadedFile("face.png", _png_bytes(), content_type="image/png") + ) + + self.assertEqual(response.status_code, 200, response.content) + self.target.refresh_from_db() + self.assertTrue(self.target.icon) + + # Relative on purpose: the browser resolves it against its own origin. + # An absolute URL built from the request would embed the internal host + # and be blocked as a cross-origin request wherever the app is proxied. + icon_url = response.data["icon_url"] + self.assertTrue(icon_url.startswith("/"), icon_url) + + def test_replacing_an_icon_removes_the_previous_file(self) -> None: + """Repeated uploads must not leave orphaned images on disk.""" + self._upload( + SimpleUploadedFile("first.png", _png_bytes(), content_type="image/png") + ) + self.target.refresh_from_db() + first_path = self.target.icon.path + self.assertTrue(os.path.exists(first_path)) + + self._upload( + SimpleUploadedFile("second.png", _png_bytes(16), content_type="image/png") + ) + self.target.refresh_from_db() + + self.assertNotEqual(self.target.icon.path, first_path) + self.assertFalse(os.path.exists(first_path)) + self.assertTrue(os.path.exists(self.target.icon.path)) + + def test_upload_rejects_a_non_image_extension(self) -> None: + response = self._upload( + SimpleUploadedFile("resume.txt", b"not an image", content_type="text/plain") + ) + + self.assertEqual(response.status_code, 400) + self.target.refresh_from_db() + self.assertFalse(self.target.icon) + + def test_upload_rejects_a_file_over_the_size_limit(self) -> None: + oversized = SimpleUploadedFile( + "huge.png", b"x" * (5 * 1024 * 1024 + 1), content_type="image/png" + ) + + response = self._upload(oversized) + + self.assertEqual(response.status_code, 400) + + def test_upload_requires_a_file(self) -> None: + response = self.client_api.post( + f"/api/accounts/staff/{self.target.id}/icon/", {}, format="multipart" + ) + + self.assertEqual(response.status_code, 400) + + def test_removing_a_picture_clears_it_and_deletes_the_file(self) -> None: + """Removing a photo must not leave the image behind on disk.""" + self._upload( + SimpleUploadedFile("face.png", _png_bytes(), content_type="image/png") + ) + self.target.refresh_from_db() + path = self.target.icon.path + self.assertTrue(os.path.exists(path)) + + response = self.client_api.delete(f"/api/accounts/staff/{self.target.id}/icon/") + + self.assertEqual(response.status_code, 200, response.content) + self.target.refresh_from_db() + self.assertFalse(self.target.icon) + self.assertFalse(os.path.exists(path)) + self.assertIsNone(response.data["icon_url"]) + + def test_removing_an_absent_picture_succeeds(self) -> None: + """Idempotent: the requested end state (no photo) already holds.""" + response = self.client_api.delete(f"/api/accounts/staff/{self.target.id}/icon/") + + self.assertEqual(response.status_code, 200, response.content) + self.target.refresh_from_db() + self.assertFalse(self.target.icon) + + def test_removing_a_picture_for_an_unknown_staff_member_is_not_found(self) -> None: + response = self.client_api.delete( + "/api/accounts/staff/00000000-0000-0000-0000-000000000000/icon/" + ) + + self.assertEqual(response.status_code, 404) + + def test_upload_to_an_unknown_staff_member_is_not_found(self) -> None: + response = self._upload( + SimpleUploadedFile("face.png", _png_bytes(), content_type="image/png"), + staff_id="00000000-0000-0000-0000-000000000000", + ) + + self.assertEqual(response.status_code, 404) diff --git a/apps/accounts/urls.py b/apps/accounts/urls.py index b21128259..9554a9737 100644 --- a/apps/accounts/urls.py +++ b/apps/accounts/urls.py @@ -4,8 +4,9 @@ from apps.accounts.views.password_views import SecurityPasswordChangeView from apps.accounts.views.staff_api import ( StaffListCreateAPIView, - StaffRetrieveUpdateDestroyAPIView, + StaffRetrieveUpdateAPIView, ) +from apps.accounts.views.staff_icon_api import StaffIconAPIView from apps.accounts.views.staff_views import ( StaffListAPIView, get_staff_rates, @@ -39,7 +40,12 @@ path("staff/", StaffListCreateAPIView.as_view(), name="api_staff_list_create"), path( "staff//", - StaffRetrieveUpdateDestroyAPIView.as_view(), + StaffRetrieveUpdateAPIView.as_view(), name="api_staff_detail", ), + path( + "staff//icon/", + StaffIconAPIView.as_view(), + name="api_staff_icon", + ), ] diff --git a/apps/accounts/utils.py b/apps/accounts/utils.py index 96178b593..cb344c615 100644 --- a/apps/accounts/utils.py +++ b/apps/accounts/utils.py @@ -132,7 +132,7 @@ def get_staff_from_nickname(name: str, *, include_inactive: bool = False): ) # scored entries: (match_string, score, index) best_by_staff: dict = {} - for match_str, score, idx in scored: + for _match_str, score, idx in scored: staff = candidates[idx][1] if score > best_by_staff.get(staff.pk, (0,))[0]: best_by_staff[staff.pk] = (score, staff) diff --git a/apps/accounts/views/__init__.py b/apps/accounts/views/__init__.py index 6d24435e9..1f630bb3a 100644 --- a/apps/accounts/views/__init__.py +++ b/apps/accounts/views/__init__.py @@ -10,7 +10,8 @@ from django.apps import apps if apps.ready: - from .staff_api import StaffListCreateAPIView, StaffRetrieveUpdateDestroyAPIView + from .staff_api import StaffListCreateAPIView, StaffRetrieveUpdateAPIView + from .staff_icon_api import StaffIconAPIView except (ImportError, RuntimeError): # Django not ready or circular import, skip conditional imports pass @@ -21,8 +22,9 @@ "GetCurrentUserAPIView", "LogoutUserAPIView", "SecurityPasswordChangeView", + "StaffIconAPIView", "StaffListAPIView", "StaffListCreateAPIView", - "StaffRetrieveUpdateDestroyAPIView", + "StaffRetrieveUpdateAPIView", "get_staff_rates", ] diff --git a/apps/accounts/views/staff_api.py b/apps/accounts/views/staff_api.py index 8ba3aeec0..fba54e30c 100644 --- a/apps/accounts/views/staff_api.py +++ b/apps/accounts/views/staff_api.py @@ -3,7 +3,7 @@ from drf_spectacular.utils import OpenApiExample, extend_schema from rest_framework import generics, status from rest_framework.exceptions import ValidationError -from rest_framework.parsers import FormParser, MultiPartParser +from rest_framework.parsers import JSONParser from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response @@ -17,7 +17,7 @@ @extend_schema( summary="List and create staff members", description="API endpoint for listing all staff members and creating new staff members. " - "Supports multipart/form data for file uploads (e.g., profile pictures).", + "Profile pictures are uploaded separately via the staff icon endpoint.", tags=["Staff Management"], examples=[ OpenApiExample( @@ -46,13 +46,13 @@ class StaffListCreateAPIView(generics.ListCreateAPIView): """API endpoint for listing and creating staff members. Supports both GET (list all staff) and POST (create new staff) operations. - Requires authentication and staff permissions. Handles multipart/form data - for file uploads (e.g., profile pictures). + Requires authentication and staff permissions. Accepts JSON only; profile + pictures are uploaded separately via StaffIconAPIView. """ queryset = Staff.objects.all() permission_classes = [IsAuthenticated, IsStaff] - parser_classes = [MultiPartParser, FormParser] + parser_classes = [JSONParser] def get_queryset(self): return Staff.objects.prefetch_related( @@ -68,7 +68,7 @@ def get_serializer_class(self): @extend_schema( summary="Create a new staff member", description="Create a new staff member with the provided details. " - "Supports multipart/form data for file uploads (e.g., profile pictures).", + "Profile pictures are uploaded separately via the staff icon endpoint.", tags=["Staff Management"], request=StaffCreateSerializer, responses={201: StaffSerializer}, @@ -87,10 +87,12 @@ def post(self, request, *args, **kwargs): @extend_schema( - summary="Retrieve, update, or delete staff member", - description="API endpoint for retrieving, updating, and deleting individual staff members. " - "Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. " - "Includes comprehensive logging for update operations and handles multipart/form data for file uploads.", + summary="Retrieve or update staff member", + description="API endpoint for retrieving and updating individual staff members. " + "Supports GET (retrieve) and PUT/PATCH (update). " + "Includes comprehensive logging for update operations. " + "Profile pictures are uploaded separately via the staff icon endpoint. " + "Staff are not deleted; offboarding is done by setting date_left.", tags=["Staff Management"], examples=[ OpenApiExample( @@ -110,18 +112,21 @@ def post(self, request, *args, **kwargs): ), ], ) -class StaffRetrieveUpdateDestroyAPIView(generics.RetrieveUpdateDestroyAPIView): - """API endpoint for retrieving, updating, and deleting individual staff members. +class StaffRetrieveUpdateAPIView(generics.RetrieveUpdateAPIView[Staff]): + """API endpoint for retrieving and updating individual staff members. - Supports GET (retrieve), PUT/PATCH (update), and DELETE operations on - specific staff members. Includes comprehensive logging for update operations - and handles multipart/form data for file uploads. + Supports GET (retrieve) and PUT/PATCH (update) on specific staff members. + Includes comprehensive logging for update operations. Accepts JSON only; + profile pictures are uploaded separately via StaffIconAPIView. + + Staff are never deleted (their time entries are protected); offboarding is + done by setting date_left. """ queryset = Staff.objects.all() serializer_class = StaffSerializer permission_classes = [IsAuthenticated, IsStaff] - parser_classes = [MultiPartParser, FormParser] + parser_classes = [JSONParser] def get_queryset(self): return Staff.objects.prefetch_related( diff --git a/apps/accounts/views/staff_icon_api.py b/apps/accounts/views/staff_icon_api.py new file mode 100644 index 000000000..891c0bb1c --- /dev/null +++ b/apps/accounts/views/staff_icon_api.py @@ -0,0 +1,95 @@ +import logging +import os + +from drf_spectacular.utils import extend_schema +from rest_framework.parsers import FormParser, MultiPartParser +from rest_framework.permissions import IsAuthenticated +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from apps.accounts.models import Staff +from apps.accounts.permissions import IsStaff +from apps.accounts.serializers import StaffSerializer +from apps.workflow.views.company_defaults_logo_api import ( + ALLOWED_EXTENSIONS, + MAX_UPLOAD_SIZE, +) + +logger = logging.getLogger(__name__) + + +class StaffIconAPIView(APIView): + """Upload or remove the profile picture for a single staff member. + + Errors are persisted by the project-wide DRF exception handler, which has + the request context (user, session replay id, path). Persisting here first + would win the "first write wins" race in persist_app_error and record the + failure without any of it, so this view deliberately has no try/except. + """ + + serializer_class = StaffSerializer + parser_classes = [MultiPartParser, FormParser] + permission_classes = [IsAuthenticated, IsStaff] + + @extend_schema( + summary="Upload a staff profile picture", + description="Replace a staff member's profile picture. This is a " + "separate endpoint because the staff resource itself is JSON-only — a " + "file cannot ride inside a JSON body.", + tags=["Staff Management"], + request={ + "multipart/form-data": { + "type": "object", + "properties": {"file": {"type": "string", "format": "binary"}}, + "required": ["file"], + } + }, + responses={200: StaffSerializer}, + ) + def post(self, request: Request, pk: str) -> Response: + staff = Staff.objects.filter(pk=pk).first() + if staff is None: + return Response({"error": "Staff member not found"}, status=404) + + file = request.data.get("file") + if not file: + return Response({"error": "No file provided"}, status=400) + + if file.size > MAX_UPLOAD_SIZE: + return Response({"error": "File too large (max 5MB)"}, status=400) + + ext = os.path.splitext(file.name)[1].lower() + if ext not in ALLOWED_EXTENSIONS: + return Response({"error": f"Unsupported file type: {ext}"}, status=400) + + # Drop the previous image so replacing a picture does not orphan a file. + # Every staff icon is a user upload under MEDIA_ROOT/staff_icons, so + # unlike company logos there is no shipped baseline asset to protect. + staff.icon.delete(save=False) + staff.icon = file + staff.save(update_fields=["icon"]) + + logger.info(f"[StaffIcon] Updated icon for Staff ID: {pk}") + return Response(StaffSerializer(staff, context={"request": request}).data) + + @extend_schema( + summary="Remove a staff profile picture", + description="Clear a staff member's profile picture and delete the " + "image from disk. Idempotent: removing an absent picture succeeds, " + "because the requested end state already holds.", + tags=["Staff Management"], + responses={200: StaffSerializer}, + ) + def delete(self, request: Request, pk: str) -> Response: + staff = Staff.objects.filter(pk=pk).first() + if staff is None: + return Response({"error": "Staff member not found"}, status=404) + + # FieldFile.delete() returns early when there is no file and clears the + # field itself, so this needs no guard and is idempotent. + staff.icon.delete(save=False) + staff.save(update_fields=["icon"]) + + logger.info(f"[StaffIcon] Removed icon for Staff ID: {pk}") + return Response(StaffSerializer(staff, context={"request": request}).data) diff --git a/apps/accounts/views/user_profile_view.py b/apps/accounts/views/user_profile_view.py index bfe7b6af6..1b39a58b9 100644 --- a/apps/accounts/views/user_profile_view.py +++ b/apps/accounts/views/user_profile_view.py @@ -13,7 +13,6 @@ from rest_framework.views import APIView from apps.accounts.serializers import UserProfileSerializer -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.request import get_client_ip @@ -96,8 +95,6 @@ def post(self, request: Request) -> Response: return response - except AlreadyLoggedException: - raise except Exception as e: app_error = persist_app_error(e) return Response( diff --git a/apps/company/services/company_rest_service.py b/apps/company/services/company_rest_service.py index 05d6286cb..fc6e040a4 100644 --- a/apps/company/services/company_rest_service.py +++ b/apps/company/services/company_rest_service.py @@ -36,11 +36,7 @@ from apps.company.utils import date_to_datetime from apps.crm.tasks import rematch_phone_calls_task from apps.workflow.accounting.registry import get_provider -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import ( - persist_and_raise, - persist_app_error, -) +from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.search_telemetry import SearchTelemetryService COMPANY_SEARCH_TOKEN_RE = re.compile(r"[a-z0-9]+") @@ -85,10 +81,9 @@ def get_all_companies() -> List[Dict[str, Any]]: } for company in companies ] - except AlreadyLoggedException: - raise except Exception as exc: - persist_and_raise(exc) + persist_app_error(exc) + raise @staticmethod def search_companies(query: str, limit: int = 10) -> List[Dict[str, Any]]: @@ -118,10 +113,9 @@ def search_companies(query: str, limit: int = 10) -> List[Dict[str, Any]]: companies = CompanyRestService._execute_company_search(query, limit) return CompanyRestService._format_company_search_results(companies) - except AlreadyLoggedException: - raise except Exception as exc: - persist_and_raise(exc, additional_context={"query": query, "limit": limit}) + persist_app_error(exc, additional_context={"query": query, "limit": limit}) + raise @staticmethod def list_companies( @@ -198,12 +192,10 @@ def list_companies( "total_pages": total_pages, } - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "query": query, @@ -211,6 +203,7 @@ def list_companies( "page_size": page_size, }, ) + raise @staticmethod def get_company_by_id(company_id: UUID) -> Dict[str, Any]: @@ -237,20 +230,19 @@ def get_company_by_id(company_id: UUID) -> Dict[str, Any]: .get(id=company_id) ) return CompanyRestService._format_company_detail(company) - except Company.DoesNotExist: - raise ValueError(f"Company with id {company_id} not found") - except AlreadyLoggedException: - raise + except Company.DoesNotExist as exc: + raise ValueError(f"Company with id {company_id} not found") from exc except ValueError: raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "get_company_by_id", "company_id": str(company_id), }, ) + raise @staticmethod def create_company(data: Dict[str, Any]) -> Company: @@ -287,16 +279,15 @@ def create_company(data: Dict[str, Any]) -> Company: ) return company - except AlreadyLoggedException: - raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "create_company", "payload_keys": list(data.keys()), }, ) + raise @staticmethod def update_company( @@ -407,12 +398,10 @@ def update_company( .get(id=updated_company.id) ) return updated_with_phone - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "update_company", @@ -420,6 +409,7 @@ def update_company( "payload_keys": list(data.keys()), }, ) + raise @staticmethod def get_company_contacts(company_id: UUID) -> List[Dict[str, Any]]: @@ -453,16 +443,15 @@ def get_company_contacts(company_id: UUID) -> List[Dict[str, Any]]: for link in links ] - except AlreadyLoggedException: - raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "get_company_contacts", "company_id": str(company_id), }, ) + raise @staticmethod def get_job_person(job_id: UUID) -> Dict[str, Any]: @@ -483,18 +472,17 @@ def get_job_person(job_id: UUID) -> Dict[str, Any]: try: job = Job.objects.select_related("person").get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") - except AlreadyLoggedException: - raise + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "get_job_person", "job_id": str(job_id), }, ) + raise if not job.person: # Documented business validation failure should not be persisted @@ -507,16 +495,15 @@ def get_job_person(job_id: UUID) -> Dict[str, Any]: "name": person.name, "email": person.email, } - except AlreadyLoggedException: - raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "serialize_job_person", "job_id": str(job_id), }, ) + raise @staticmethod def update_job_person( @@ -543,8 +530,8 @@ def update_job_person( try: job = Job.objects.select_related("company", "person").get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc person_id = person_data.get("id") if not person_id: @@ -574,12 +561,10 @@ def update_job_person( "email": person.email, } - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "update_job_person", @@ -587,6 +572,7 @@ def update_job_person( "person_id": person_data.get("id"), }, ) + raise @staticmethod def _execute_company_search(query: str, limit: int): @@ -1010,7 +996,7 @@ def _update_company_in_xero( token = provider.get_valid_token() if not token: exc = RuntimeError("Accounting provider authentication required") - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "_update_company_in_xero", @@ -1018,6 +1004,7 @@ def _update_company_in_xero( "provider": provider.provider_name, }, ) + raise exc # Update local fields first with transaction.atomic(): @@ -1052,7 +1039,7 @@ def _update_company_in_xero( exc = RuntimeError( f"Failed to update company in {provider.provider_name}: {result.error}" ) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "_update_company_in_xero", @@ -1061,6 +1048,7 @@ def _update_company_in_xero( "provider_error": result.error, }, ) + raise exc logger.info( f"Company {company.id} updated locally and in {provider.provider_name}", diff --git a/apps/company/services/person_merge_service.py b/apps/company/services/person_merge_service.py index b08a7e77f..9e054d470 100644 --- a/apps/company/services/person_merge_service.py +++ b/apps/company/services/person_merge_service.py @@ -8,7 +8,6 @@ from apps.accounts.models import Staff from apps.company.models import CompanyPersonLink, ContactMethod, Person -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error @@ -166,10 +165,8 @@ def merge_people( "contact_methods_moved": methods_moved, "contact_methods_collapsed": methods_collapsed, } - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - error = persist_app_error(exc) - raise AlreadyLoggedException(exc, error.id) from exc + persist_app_error(exc) + raise diff --git a/apps/company/tests/test_company_invoice_summary.py b/apps/company/tests/test_company_invoice_summary.py index e684610ea..3ca54c1d9 100644 --- a/apps/company/tests/test_company_invoice_summary.py +++ b/apps/company/tests/test_company_invoice_summary.py @@ -16,7 +16,6 @@ from apps.company.views.company_rest_views import CompanyCreateRestView from apps.testing import BaseTestCase from apps.workflow.accounting.types import ContactResult -from apps.workflow.exceptions import AlreadyLoggedException def _make_company(name: str) -> Company: @@ -194,7 +193,7 @@ def test_create_company_cleans_up_local_row_when_xero_create_fails(self): "apps.company.services.company_rest_service.get_provider", return_value=provider, ): - with pytest.raises(AlreadyLoggedException, match="RemoteDisconnected"): + with pytest.raises(ValueError, match="RemoteDisconnected"): CompanyRestService.create_company( { "name": "Failed Xero Company", diff --git a/apps/company/tests/test_contact_methods.py b/apps/company/tests/test_contact_methods.py index f1f159c4c..714ee9834 100644 --- a/apps/company/tests/test_contact_methods.py +++ b/apps/company/tests/test_contact_methods.py @@ -20,8 +20,8 @@ from apps.crm.services.phone_call_service import rematch_calls_for_numbers from apps.testing import BaseAPITestCase, BaseTestCase from apps.workflow.accounting.types import ContactResult -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError +from apps.workflow.services.error_persistence import persist_app_error def _link(company: Company, name: str, email: str | None = None) -> CompanyPersonLink: @@ -919,19 +919,12 @@ def test_already_logged_update_failure_is_not_persisted_again(self) -> None: company.xero_contact_id = "xero-contact-id" company.save() before = AppError.objects.count() - app_error = AppError.objects.create( - message="upstream failure", - app="company", - file="company_rest_service.py", - function="_update_company_in_xero", - ) + upstream_exc = RuntimeError("upstream failure") + app_error = persist_app_error(upstream_exc) with patch( "apps.company.services.company_rest_service.get_provider", - side_effect=AlreadyLoggedException( - RuntimeError("upstream failure"), - app_error.id, - ), + side_effect=upstream_exc, ): response = self.client.patch( self._update_url(company.id), @@ -946,7 +939,7 @@ def test_already_logged_update_failure_is_not_persisted_again(self) -> None: class CompanyUpdateProviderFailureTests(BaseTestCase): """Service-level guard for ADR 0019 on update_company: Xero/system - failures must persist to AppError and surface as AlreadyLoggedException + failures must persist to AppError and surface as RuntimeError at the service boundary, never ride the user-facing ValueError (400) path. Complements the HTTP-level 500/error_id tests above.""" @@ -972,7 +965,7 @@ def test_failed_provider_push_persists_app_error(self) -> None: "apps.company.services.company_rest_service.get_provider", return_value=provider, ): - with self.assertRaises(AlreadyLoggedException) as ctx: + with self.assertRaises(RuntimeError) as ctx: CompanyRestService.update_company(company.id, {"name": "Acme Renamed"}) self.assertIn("Failed to update company", str(ctx.exception)) @@ -990,7 +983,7 @@ def test_missing_provider_token_persists_app_error(self) -> None: "apps.company.services.company_rest_service.get_provider", return_value=provider, ): - with self.assertRaises(AlreadyLoggedException) as ctx: + with self.assertRaises(RuntimeError) as ctx: CompanyRestService.update_company(company.id, {"name": "Acme Renamed"}) self.assertIn("authentication required", str(ctx.exception)) @@ -1045,7 +1038,9 @@ def test_create_with_conflicting_phone_rolls_back_company(self) -> None: ) provider = self._provider() - with self.assertRaises(AlreadyLoggedException) as ctx: + # _apply_company_phone_change converts the model's ValidationError to + # ValueError, which the REST boundary maps to 400. + with self.assertRaises(ValueError) as ctx: self._create(provider, phone="09 777 7777") self.assertIn("already belongs", str(ctx.exception)) diff --git a/apps/company/tests/test_person_merge_service.py b/apps/company/tests/test_person_merge_service.py index d90dead1f..f7d3edaa7 100644 --- a/apps/company/tests/test_person_merge_service.py +++ b/apps/company/tests/test_person_merge_service.py @@ -8,7 +8,6 @@ from apps.crm.models import PhoneCallRecord from apps.job.models import Job from apps.testing import BaseTestCase -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError @@ -147,7 +146,7 @@ def test_unexpected_failure_rolls_back_and_persists_once(self) -> None: "apps.company.services.person_merge_service._merge_contact_methods", side_effect=RuntimeError("merge failed"), ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): merge_people(source.id, destination.id, self.test_staff) source_link.refresh_from_db() diff --git a/apps/company/views/company_rest_views.py b/apps/company/views/company_rest_views.py index 37d16f688..30c291807 100644 --- a/apps/company/views/company_rest_views.py +++ b/apps/company/views/company_rest_views.py @@ -42,7 +42,6 @@ JobPersonUpdateSerializer, ) from apps.company.services.company_rest_service import CompanyRestService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -55,19 +54,12 @@ def _build_server_error_response( status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR, ) -> Response: """Serialize an error response while ensuring exceptions persist only once.""" - if isinstance(exc, AlreadyLoggedException): - root_exc = exc.original - error_id = exc.app_error_id - else: - root_exc = exc - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) - - logger.error("%s: %s", message, root_exc) - - payload: Dict[str, Any] = {"error": message, "details": str(root_exc)} - if error_id: - payload["error_id"] = str(error_id) + app_error = persist_app_error(exc) + + logger.error("%s: %s", message, exc) + + payload: Dict[str, Any] = {"error": message, "details": str(exc)} + payload["error_id"] = str(app_error.id) serializer = CompanyErrorResponseSerializer(data=payload) serializer.is_valid(raise_exception=True) diff --git a/apps/company/views/supplier_search_alias_views.py b/apps/company/views/supplier_search_alias_views.py index 632004646..7d1fce6bc 100644 --- a/apps/company/views/supplier_search_alias_views.py +++ b/apps/company/views/supplier_search_alias_views.py @@ -16,26 +16,18 @@ SupplierSearchAliasCreateSerializer, SupplierSearchAliasSerializer, ) -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) def _build_server_error_response(*, message: str, exc: Exception) -> Response: - if isinstance(exc, AlreadyLoggedException): - root_exc = exc.original - error_id = exc.app_error_id - else: - root_exc = exc - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) - - logger.error("%s: %s", message, root_exc) - - payload: dict[str, Any] = {"error": message, "details": str(root_exc)} - if error_id: - payload["error_id"] = str(error_id) + app_error = persist_app_error(exc) + + logger.error("%s: %s", message, exc) + + payload: dict[str, Any] = {"error": message, "details": str(exc)} + payload["error_id"] = str(app_error.id) return Response(payload, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/apps/crm/services/phone_call_service.py b/apps/crm/services/phone_call_service.py index 6c344d1a8..3f8f5895c 100644 --- a/apps/crm/services/phone_call_service.py +++ b/apps/crm/services/phone_call_service.py @@ -23,7 +23,6 @@ PhoneEndpoint, PhoneProviderSettings, ) -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error if TYPE_CHECKING: @@ -131,8 +130,6 @@ def sync_call_history( call=call, recording=recording, ) - except AlreadyLoggedException: - raise except Exception as exc: persist_app_error(exc) recording.archive_error = str(exc) diff --git a/apps/crm/tasks.py b/apps/crm/tasks.py index 21dafee4f..459857018 100644 --- a/apps/crm/tasks.py +++ b/apps/crm/tasks.py @@ -5,7 +5,6 @@ from django.db import close_old_connections from apps.crm.models import PhoneProviderSettings -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error scheduler_logger = logging.getLogger("apps.crm.tasks") @@ -34,12 +33,10 @@ def sync_phone_calls_task() -> None: result.recordings_seen, result.recordings_archived, ) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error("Error during phone call sync: %s", exc, exc_info=True) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @shared_task(name="apps.crm.tasks.delete_archived_phone_recordings_task") @@ -67,16 +64,14 @@ def delete_archived_phone_recordings_task(limit: int = 100) -> None: result.deleted, result.failed, ) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during phone recording provider cleanup: %s", exc, exc_info=True, ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise class RematchPhoneCallsTask(Protocol): @@ -100,16 +95,14 @@ def _rematch_phone_calls_task(numbers: list[str]) -> None: from apps.crm.services.phone_call_service import rematch_calls_for_numbers rematch_calls_for_numbers(numbers) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during phone call rematch: %s", exc, exc_info=True, ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise rematch_phone_calls_task = cast( diff --git a/apps/crm/tests/test_phone_call_service.py b/apps/crm/tests/test_phone_call_service.py index 5f962d779..e7b9cdd63 100644 --- a/apps/crm/tests/test_phone_call_service.py +++ b/apps/crm/tests/test_phone_call_service.py @@ -1,3 +1,4 @@ +import hashlib import tempfile import uuid from pathlib import Path @@ -883,6 +884,69 @@ def test_download_streams_archived_recording_without_provider_settings( body = b"".join(response.streaming_content) self.assertEqual(body, payload) + def test_download_revalidates_with_etag_instead_of_resending(self) -> None: + """Replaying a recording must not transfer the audio a second time.""" + storage_path = "2026/06/02/etag-playback.mp3" + payload = b"\xff\xe3\x28\xc4recorded audio" + full_path = Path(self.storage_root.name) / storage_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_bytes(payload) + recording = PhoneCallRecording.objects.create( + call=self.call, + provider_recording_id="etag-playback", + account_code="account", + filename="etag-playback.mp3", + storage_path=storage_path, + content_type="audio/mpeg", + byte_size=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + archived_at=timezone.now(), + ) + url = f"/api/crm/phone-call-recordings/{recording.id}/download/" + + first = self.api.get(url) + + self.assertEqual(first.status_code, 200) + etag = first["ETag"] + + second = self.api.get(url, headers={"if-none-match": etag}) + + self.assertEqual(second.status_code, 304) + self.assertEqual(second.content, b"") + # The conditional response carries the validator it was matched on, so + # the client can revalidate again without re-reading the body. + self.assertEqual(second["ETag"], etag) + + def test_download_404s_a_missing_file_even_when_the_client_revalidates( + self, + ) -> None: + """A vanished file must surface, not hide behind a 304.""" + storage_path = "2026/06/02/vanished.mp3" + payload = b"\xff\xe3\x28\xc4recorded audio" + full_path = Path(self.storage_root.name) / storage_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_bytes(payload) + recording = PhoneCallRecording.objects.create( + call=self.call, + provider_recording_id="vanished", + account_code="account", + filename="vanished.mp3", + storage_path=storage_path, + content_type="audio/mpeg", + byte_size=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + archived_at=timezone.now(), + ) + url = f"/api/crm/phone-call-recordings/{recording.id}/download/" + etag = self.api.get(url)["ETag"] + + # Lost out of band: the row keeps its digest, the bytes are gone. + full_path.unlink() + + response = self.api.get(url, headers={"if-none-match": etag}) + + self.assertEqual(response.status_code, 404) + def test_only_office_staff_can_read_recording_downloads(self) -> None: self.api.force_authenticate(user=self.workshop_staff) recording = PhoneCallRecording.objects.create( diff --git a/apps/crm/views/phone_call_views.py b/apps/crm/views/phone_call_views.py index c45e2ede4..0e8edfe85 100644 --- a/apps/crm/views/phone_call_views.py +++ b/apps/crm/views/phone_call_views.py @@ -4,7 +4,7 @@ from uuid import UUID from django.db.models import Q, QuerySet -from django.http import FileResponse +from django.http import FileResponse, HttpResponseNotModified from django.shortcuts import get_object_or_404 from django.utils.dateparse import parse_date from drf_spectacular.types import OpenApiTypes @@ -50,7 +50,6 @@ from apps.crm.tasks import rematch_phone_calls_task from apps.job.permissions import IsOfficeStaff from apps.workflow.api.pagination import PageSizePagination -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error @@ -312,11 +311,9 @@ def _call_operation_response( {"status": "error", "message": str(exc)}, status=status.HTTP_400_BAD_REQUEST, ) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise response_serializer = self.get_serializer(call) return Response(response_serializer.data, status=status.HTTP_200_OK) @@ -414,26 +411,39 @@ def download( self, request: Request, pk: str | None = None, - ) -> FileResponse | Response: + ) -> FileResponse | HttpResponseNotModified | Response: recording = get_object_or_404(self.get_queryset(), pk=pk) + # Recordings are content-addressed, so the stored digest is a strong ETag. + etag = f'"{recording.sha256}"' if recording.sha256 else None + try: full_path = recording_file_path(recording) - response = FileResponse(open(full_path, "rb")) + # Opened before the conditional is answered: a row whose digest + # outlives its file must 404, not 304 the clients holding a stale copy. + handle = open(full_path, "rb") + + if etag and request.headers.get("If-None-Match") == etag: + handle.close() + not_modified = HttpResponseNotModified() + not_modified["ETag"] = etag # RFC 9110: a 304 repeats the validator + return not_modified + + response = FileResponse(handle) content_type, _ = mimetypes.guess_type(full_path) if content_type: response["Content-Type"] = content_type response["Content-Disposition"] = f'inline; filename="{recording.filename}"' + if etag: + response["ETag"] = etag return response except FileNotFoundError: return Response( {"status": "error", "message": "Recording file not found on disk"}, status=status.HTTP_404_NOT_FOUND, ) - except AlreadyLoggedException: - raise except Exception as exc: - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @extend_schema( operation_id="deleteLocalPhoneCallRecording", @@ -450,11 +460,9 @@ def delete_local_file( try: delete_local_recording(recording) return Response(status=status.HTTP_204_NO_CONTENT) - except AlreadyLoggedException: - raise except Exception as exc: - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @extend_schema( operation_id="deleteProviderPhoneCallRecording", @@ -471,11 +479,9 @@ def delete_provider_file( try: provider_delete_recording(recording) return Response(status=status.HTTP_204_NO_CONTENT) - except AlreadyLoggedException: - raise except Exception as exc: - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise class PhoneEndpointViewSet(viewsets.ModelViewSet[PhoneEndpoint]): diff --git a/apps/job/importers/google_sheets.py b/apps/job/importers/google_sheets.py index 3db6fadbf..6590290e6 100644 --- a/apps/job/importers/google_sheets.py +++ b/apps/job/importers/google_sheets.py @@ -101,7 +101,7 @@ def _svc(api: str, version: str) -> Any: return service except Exception as e: - raise RuntimeError(f"Failed to create {api} {version} service: {str(e)}") + raise RuntimeError(f"Failed to create {api} {version} service: {str(e)}") from e def extract_file_id(url_or_id: str) -> str: @@ -178,9 +178,11 @@ def create_folder(name: str, parent_id: Optional[str] = None) -> str: return str(folder_id) if folder_id else "" except HttpError as e: - raise RuntimeError(f"Failed to create folder '{name}': {e.reason}") + raise RuntimeError(f"Failed to create folder '{name}': {e.reason}") from e except Exception as e: - raise RuntimeError(f"Unexpected error creating folder '{name}': {str(e)}") + raise RuntimeError( + f"Unexpected error creating folder '{name}': {str(e)}" + ) from e def copy_file( @@ -238,9 +240,9 @@ def copy_file( return str(copied_id) if copied_id else "" except HttpError as e: - raise RuntimeError(f"Failed to copy file '{name}': {e.reason}") + raise RuntimeError(f"Failed to copy file '{name}': {e.reason}") from e except Exception as e: - raise RuntimeError(f"Unexpected error copying file '{name}': {str(e)}") + raise RuntimeError(f"Unexpected error copying file '{name}': {str(e)}") from e def _set_public_edit_permissions(file_id: str) -> None: @@ -360,10 +362,10 @@ def fetch_sheet_df(sheet_id: str, sheet_range: str = "Primary Details") -> pd.Da except HttpError as e: logger.error(f"❌ Google Sheets API error: {e.reason}") - raise RuntimeError(f"Failed to fetch sheet data: {e.reason}") + raise RuntimeError(f"Failed to fetch sheet data: {e.reason}") from e except Exception as e: logger.error(f"❌ Unexpected error fetching sheet data: {str(e)}") - raise RuntimeError(f"Unexpected error fetching sheet data: {str(e)}") + raise RuntimeError(f"Unexpected error fetching sheet data: {str(e)}") from e def copy_template_for_job(job: Job) -> tuple[str, str]: @@ -416,7 +418,7 @@ def copy_template_for_job(job: Job) -> tuple[str, str]: except Exception as e: logger.error(f"Failed to copy template for job {job.job_number}: {str(e)}") - raise RuntimeError(f"Failed to copy template: {str(e)}") + raise RuntimeError(f"Failed to copy template: {str(e)}") from e def _get_or_create_docketworks_folder() -> str: @@ -459,7 +461,7 @@ def _get_or_create_docketworks_folder() -> str: except Exception as e: logger.error(f"Failed to get/create DocketWorks folder: {str(e)}") - raise RuntimeError(f"Failed to access DocketWorks folder: {str(e)}") + raise RuntimeError(f"Failed to access DocketWorks folder: {str(e)}") from e def populate_sheet_from_costset(sheet_id: str, costset: CostSet) -> None: @@ -574,7 +576,7 @@ def populate_sheet_from_costset(sheet_id: str, costset: CostSet) -> None: except Exception as e: logger.error(f"Failed to populate sheet {sheet_id}: {str(e)}") - raise RuntimeError(f"Failed to populate sheet: {str(e)}") + raise RuntimeError(f"Failed to populate sheet: {str(e)}") from e def export_sheet_as_xlsx(sheet_id: str, file_path: str) -> None: @@ -591,8 +593,8 @@ def export_sheet_as_xlsx(sheet_id: str, file_path: str) -> None: downloader = MediaIoBaseDownload(f, request) done = False while not done: - status, done = downloader.next_chunk() + _status, done = downloader.next_chunk() logger.info(f"Exported Google Sheet {sheet_id} to {file_path}") except Exception as e: logger.error(f"Failed to export Google Sheet {sheet_id} as xlsx: {str(e)}") - raise RuntimeError(f"Failed to export Google Sheet as xlsx: {str(e)}") + raise RuntimeError(f"Failed to export Google Sheet as xlsx: {str(e)}") from e diff --git a/apps/job/importers/quote_spreadsheet.py b/apps/job/importers/quote_spreadsheet.py index 25f5c8c4b..47d570534 100644 --- a/apps/job/importers/quote_spreadsheet.py +++ b/apps/job/importers/quote_spreadsheet.py @@ -1,6 +1,4 @@ -# flake8: noqa -# pylint: skip-file -# This entire file is AI slop and will be rewritten - no point fixing linting issues +# This entire file is AI slop and is slated for a rewrite. import logging from dataclasses import dataclass @@ -444,10 +442,10 @@ def parse_xlsx( return draft_lines, validation_report - except FileNotFoundError: - raise FileNotFoundError(f"Excel file not found: {path}") + except FileNotFoundError as exc: + raise FileNotFoundError(f"Excel file not found: {path}") from exc except Exception as e: - raise Exception(f"Error parsing Excel file: {str(e)}") + raise Exception(f"Error parsing Excel file: {str(e)}") from e def find_validation_cells(df, labour_col): @@ -967,7 +965,7 @@ def _validate_pricing_consistency(path: str, df) -> List[ValidationError]: labour_cost_row = None margin_row = None - for idx, row in pricing_df.iterrows(): + for _idx, row in pricing_df.iterrows(): desc = str(row.get("Description", "")).strip().lower() if "labour cost" in desc: labour_cost_row = row diff --git a/apps/job/management/commands/create_shop_jobs.py b/apps/job/management/commands/create_shop_jobs.py index b325751ca..819756016 100644 --- a/apps/job/management/commands/create_shop_jobs.py +++ b/apps/job/management/commands/create_shop_jobs.py @@ -1,4 +1,4 @@ -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from apps.accounts.models import Staff from apps.job.models import Job @@ -8,7 +8,7 @@ class Command(BaseCommand): help = "Create shop jobs for internal purposes" - def handle(self, *args, **kwargs): + def handle(self, *args: object, **kwargs: object) -> None: # Define shop job details shop_jobs = [ { @@ -49,21 +49,32 @@ def handle(self, *args, **kwargs): company_defaults = CompanyDefaults.get_solo() shop_company = company_defaults.shop_company - # Iterate through the shop jobs and create them + created = 0 + updated = 0 automation_user = Staff.get_automation_user() for job_details in shop_jobs: - # Create the job instance - job = Job( + matches = Job.objects.filter( name=job_details["name"], company=shop_company, - description="", - status="special", - shop_job=True, # Changed from shop_job to is_shop_job - job_is_valid=True, - paid=False, ) + if matches.count() > 1: + raise CommandError( + f"Multiple shop jobs named '{job_details['name']}' already exist." + ) + job = matches.first() + if job is None: + job = Job(name=job_details["name"], company=shop_company) + created += 1 + else: + updated += 1 + job.description = job_details["description"] + job.status = "special" + job.job_is_valid = True + job.paid = False job.save(staff=automation_user) self.stdout.write( - self.style.SUCCESS("Shop jobs have been successfully created.") + self.style.SUCCESS( + f"Shop jobs ready: {created} created, {updated} updated." + ) ) diff --git a/apps/job/management/commands/test_gemini_chat.py b/apps/job/management/commands/test_gemini_chat.py index 43c9c6c0d..bad1ad637 100644 --- a/apps/job/management/commands/test_gemini_chat.py +++ b/apps/job/management/commands/test_gemini_chat.py @@ -13,6 +13,7 @@ from apps.job.models import Job from apps.job.services.chat_service import ChatService +from apps.workflow.services.error_persistence import persist_app_error # Configure basic logging logger = logging.getLogger(__name__) @@ -125,13 +126,16 @@ def handle(self, *args, **options): ) ) - except Job.DoesNotExist: - raise CommandError(f'Job with ID "{job_id}" does not exist.') + except Job.DoesNotExist as exc: + persist_app_error(exc) + raise CommandError(f'Job with ID "{job_id}" does not exist.') from exc except ValueError as e: # Catches configuration errors from the service, e.g., missing API key - raise CommandError(f"Configuration Error: {e}") + persist_app_error(e) + raise CommandError(f"Configuration Error: {e}") from e except Exception as e: logger.exception("An unexpected error occurred during the chat test.") - raise CommandError(f"An unexpected error occurred: {e}") + persist_app_error(e) + raise CommandError(f"An unexpected error occurred: {e}") from e self.stdout.write(self.style.SUCCESS("--- Test Completed Successfully ---")) diff --git a/apps/job/serializers/costing_serializer.py b/apps/job/serializers/costing_serializer.py index bd9100049..474dfb7df 100644 --- a/apps/job/serializers/costing_serializer.py +++ b/apps/job/serializers/costing_serializer.py @@ -300,8 +300,10 @@ def save(self, **kwargs: Any) -> CostLine: f"unit_cost={unit_cost}, unit_rev={unit_rev}" ) - except Staff.DoesNotExist: - raise serializers.ValidationError(f"Staff not found: {staff_id}") + except Staff.DoesNotExist as exc: + raise serializers.ValidationError( + f"Staff not found: {staff_id}" + ) from exc except Exception as e: logger.error(f"Error calculating unit_cost: {e}") raise diff --git a/apps/job/serializers/job_serializer.py b/apps/job/serializers/job_serializer.py index e30951303..c50a8f724 100644 --- a/apps/job/serializers/job_serializer.py +++ b/apps/job/serializers/job_serializer.py @@ -1127,8 +1127,8 @@ def validate_company_id(self, value): if value: try: Company.objects.get(id=value) - except Company.DoesNotExist: - raise serializers.ValidationError("Company not found") + except Company.DoesNotExist as exc: + raise serializers.ValidationError("Company not found") from exc return value def validate_person_id(self, value): diff --git a/apps/job/serializers/kanban_serializer.py b/apps/job/serializers/kanban_serializer.py index 5df3dc9e0..92e114fcb 100644 --- a/apps/job/serializers/kanban_serializer.py +++ b/apps/job/serializers/kanban_serializer.py @@ -85,7 +85,11 @@ class KanbanJobPersonSerializer(serializers.Serializer): id = serializers.UUIDField() display_name = serializers.CharField() - icon_url = serializers.URLField(allow_null=True) + # CharField, not URLField: icon URLs are site-root-relative (/media/...) so + # the browser resolves them against its own origin. URLField would declare + # format: uri, which the generated client turns into a z.string().url() + # check that a relative path fails. + icon_url = serializers.CharField(allow_null=True) class KanbanJobSerializer(serializers.Serializer): diff --git a/apps/job/services/data_integrity_service.py b/apps/job/services/data_integrity_service.py index 7a555a103..14432aca7 100644 --- a/apps/job/services/data_integrity_service.py +++ b/apps/job/services/data_integrity_service.py @@ -41,7 +41,7 @@ SupplierProduct, ) from apps.workflow.models import AppError, XeroAccount -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error class DataIntegrityService: @@ -64,7 +64,8 @@ def scan_all_relationships() -> dict[str, Any]: "summary": {}, # Will be populated by view } except Exception as exc: - persist_and_raise(exc) + persist_app_error(exc) + raise @staticmethod def _check_all_fk_references() -> list[dict[str, Any]]: diff --git a/apps/job/services/delivery_docket_service.py b/apps/job/services/delivery_docket_service.py index 34a1a80c1..5ac78c8ae 100644 --- a/apps/job/services/delivery_docket_service.py +++ b/apps/job/services/delivery_docket_service.py @@ -10,8 +10,7 @@ from apps.accounts.models import Staff from apps.job.models import Job, JobEvent, JobFile from apps.job.services.workshop_pdf_service import create_delivery_docket_pdf -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -97,7 +96,5 @@ def generate_delivery_docket(job: Job, staff: Staff) -> tuple[BytesIO, JobFile]: logger.error( f"Failed to generate delivery docket for job {job.job_number}: {exc}" ) - try: - persist_and_raise(exc) - except AlreadyLoggedException: - raise + persist_app_error(exc) + raise diff --git a/apps/job/services/import_quote_service.py b/apps/job/services/import_quote_service.py index 5b1c66706..08abc9fa2 100644 --- a/apps/job/services/import_quote_service.py +++ b/apps/job/services/import_quote_service.py @@ -418,7 +418,9 @@ def import_quote_from_file( # Step 1: Parse and validate the spreadsheet if skip_validation: # Use simple parser without validation - draft_lines, validation_issues = parse_xlsx(file_path, skip_validation=True) + draft_lines, _validation_issues = parse_xlsx( + file_path, skip_validation=True + ) validation_report = None else: # Use full validation diff --git a/apps/job/services/job_rest_service.py b/apps/job/services/job_rest_service.py index 39cdb772d..3e4a74ab0 100644 --- a/apps/job/services/job_rest_service.py +++ b/apps/job/services/job_rest_service.py @@ -34,7 +34,7 @@ ) from apps.job.services.delta_checksum import compute_job_delta_checksum, normalise_value from apps.workflow.models import CompanyDefaults, XeroPayItem -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -256,8 +256,8 @@ def create_job(data: Dict[str, Any], user: Staff) -> Job: try: company = Company.objects.get(id=data["company_id"]) - except Company.DoesNotExist: - raise ValueError("Company not found") + except Company.DoesNotExist as exc: + raise ValueError("Company not found") from exc if not company.allow_jobs: raise ValueError( @@ -470,7 +470,8 @@ def _record_delta_rejection( request_ip=request_ip, ) except Exception as exc: # pragma: no cover - defensive persistence - persist_and_raise(exc) + persist_app_error(exc) + raise @staticmethod def _collect_soft_fail_context( @@ -669,8 +670,8 @@ def get_job_for_edit(job_id: UUID, request) -> Dict[str, Any]: """ try: job = Job.objects.select_related("company").get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc # Serialise main data job_data = JobSerializer(job, context={"request": request}).data @@ -715,8 +716,8 @@ def get_job_summary(job_id: UUID, request) -> Dict[str, Any]: """ try: job = Job.objects.select_related("company").get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc job_data = JobSummarySerializer(job, context={"request": request}).data @@ -732,26 +733,28 @@ def get_job_summary(job_id: UUID, request) -> Dict[str, Any]: } @staticmethod - def get_job_quote(job_id: UUID) -> list[Dict[str, Any]]: + def get_job_quote(job_id: UUID) -> Dict[str, Any] | None: """ - Fetches quotes for a specific job. + Fetches the quote for a specific job. Args: job_id: Job UUID Returns: - List of quote data + Serialised quote data, or None when the job has no quote yet. Raises: ValueError: If job is not found """ try: job = Job.objects.get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc + + if not job.quoted: + return None - if job.quoted: - return QuoteSerializer(job.quote).data + return QuoteSerializer(job.quote).data @staticmethod def get_job_invoices(job_id: UUID) -> list[Dict[str, Any]]: @@ -769,8 +772,8 @@ def get_job_invoices(job_id: UUID) -> list[Dict[str, Any]]: """ try: job = Job.objects.get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc invoices = job.invoices.all().order_by("-date") return InvoiceSerializer(invoices, many=True).data @@ -791,8 +794,8 @@ def get_job_basic_information(job_id: UUID) -> Dict[str, Any]: """ try: job = Job.objects.get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc return { "description": job.description or "", @@ -1145,10 +1148,8 @@ def update_job( job.save(staff=user, update_fields=["priority", "updated_at"]) result_job = job - except PreconditionFailed: - if soft_fail_context: - JobRestService._record_delta_rejection(**soft_fail_context) - raise + # DeltaValidationError subclasses PreconditionFailed, so it must be + # caught first or this arm is unreachable. except DeltaValidationError as exc: JobRestService._record_delta_rejection( job=job, @@ -1163,6 +1164,10 @@ def update_job( request_etag=delta_payload.etag or if_match, ) raise + except PreconditionFailed: + if soft_fail_context: + JobRestService._record_delta_rejection(**soft_fail_context) + raise except ValueError as exc: context = getattr(exc, "delta_rejection_context", None) if context is not None: @@ -1378,10 +1383,10 @@ def add_job_event(job_id: UUID, description: str, user: Staff) -> Dict[str, Any] "duplicate_prevented": not created, } - except Job.DoesNotExist: + except Job.DoesNotExist as exc: error_msg = f"Job {job_id} not found" logger.error(error_msg) - raise ValueError(error_msg) + raise ValueError(error_msg) from exc except (ValidationError, IntegrityError) as e: # Handle duplicate constraint violations @@ -1390,11 +1395,13 @@ def add_job_event(job_id: UUID, description: str, user: Staff) -> Dict[str, Any] ) # If we can't find existing event, re-raise - raise ValueError("Unable to create event due to duplicate constraint") + raise ValueError( + "Unable to create event due to duplicate constraint" + ) from e except Exception as e: # Persist error for debugging - persist_and_raise( + persist_app_error( exception=e, app="JobRestService", file=__file__, @@ -1407,6 +1414,7 @@ def add_job_event(job_id: UUID, description: str, user: Staff) -> Dict[str, Any] "operation": "add_job_event", }, ) + raise @staticmethod def delete_job( @@ -1502,8 +1510,8 @@ def get_job_timeline(job_id: UUID) -> list[Dict[str, Any]]: """ try: job = Job.objects.get(id=job_id) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc timeline_entries = [] @@ -1589,19 +1597,21 @@ def get_job_timeline(job_id: UUID) -> list[Dict[str, Any]]: # FAIL EARLY: Invalid staff_id indicates data corruption try: staff_ids.add(UUID(str(staff_id))) - except (ValueError, TypeError): + except (ValueError, TypeError) as exc: error_msg = ( f"Invalid staff_id in cost_line {cost_line.id}: {staff_id}" ) logger.error(error_msg) - persist_and_raise( - ValueError(error_msg), + error = ValueError(error_msg) + persist_app_error( + error, additional_context={ "cost_line_id": str(cost_line.id), "staff_id": staff_id, "job_id": str(job.id), }, ) + raise error from exc # Fetch all staff members in bulk staff_map = Staff.objects.in_bulk(staff_ids) if staff_ids else {} diff --git a/apps/job/services/job_service.py b/apps/job/services/job_service.py index 041101016..335ac1bf8 100644 --- a/apps/job/services/job_service.py +++ b/apps/job/services/job_service.py @@ -10,8 +10,7 @@ from apps.accounting.models.invoice import Invoice from apps.accounts.models import Staff from apps.job.models import Job -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -130,10 +129,8 @@ def recalculate_job_invoicing_state(job_id: str, staff) -> None: logger.error("Provided job id doesn't exist") raise except Exception as e: - try: - persist_and_raise(e) - except AlreadyLoggedException: - raise + persist_app_error(e) + raise class JobStaffService: diff --git a/apps/job/services/workshop_pdf_service.py b/apps/job/services/workshop_pdf_service.py index 412db2a4d..594e5ae49 100644 --- a/apps/job/services/workshop_pdf_service.py +++ b/apps/job/services/workshop_pdf_service.py @@ -26,9 +26,8 @@ from apps.crm.models import PhoneEndpoint from apps.job.enums import SpeedQualityTradeoff from apps.job.models import CostLine, CostSet, Job, JobFile -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -737,12 +736,10 @@ def create_workshop_pdf(job: Job) -> BytesIO: pdf_files = [f for f in files_to_print if f.mime_type == "application/pdf"] return process_attachments(main_buffer, image_files, pdf_files) - except AlreadyLoggedException: - raise except Exception as exc: logger.error("Error creating workshop PDF: %s", exc) - persist_and_raise(exc, job_id=str(job.id)) - raise AssertionError("persist_and_raise returned unexpectedly") + persist_app_error(exc, job_id=str(job.id)) + raise def create_delivery_docket_pdf(job: Job) -> BytesIO: @@ -1611,7 +1608,4 @@ def merge_pdfs(pdf_sources: list[Union[BytesIO, str]]) -> BytesIO: buffer.close() except Exception as e: logger.error(f"Error closing buffer: {str(e)}") - try: - persist_and_raise(e) - except AlreadyLoggedException: - pass + persist_app_error(e) diff --git a/apps/job/tasks.py b/apps/job/tasks.py index 9778b1fb3..db04e2fb8 100644 --- a/apps/job/tasks.py +++ b/apps/job/tasks.py @@ -13,8 +13,7 @@ from django.core.cache import caches from django.db import close_old_connections, connection, transaction -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger("apps.job.tasks") @@ -52,9 +51,6 @@ def _queue_job_summary_pdf_refresh(countdown: int | None = None) -> None: ) try: _schedule_job_summary_pdf_refresh(scheduled_countdown) - except AlreadyLoggedException: - cache.delete(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY) - raise except Exception as exc: cache.delete(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY) logger.error( @@ -62,13 +58,14 @@ def _queue_job_summary_pdf_refresh(countdown: int | None = None) -> None: exc, exc_info=True, ) - persist_and_raise( + persist_app_error( exc, additional_context={ "countdown": scheduled_countdown, "limit": JOB_SUMMARY_PDF_REFRESH_BATCH_SIZE, }, ) + raise else: logger.debug("JobSummary.pdf refresh is already queued.") @@ -107,8 +104,6 @@ def create_job_file_thumbnail_task(job_file_id: str) -> None: create_thumbnail(source_path, thumb_path) logger.info("Created thumbnail for job file %s.", job_file_id) - except AlreadyLoggedException: - raise except Exception as exc: logger.error( "Error creating thumbnail for job file %s: %s", @@ -116,7 +111,8 @@ def create_job_file_thumbnail_task(job_file_id: str) -> None: exc, exc_info=True, ) - persist_and_raise(exc, additional_context={"job_file_id": job_file_id}) + persist_app_error(exc, additional_context={"job_file_id": job_file_id}) + raise def _refresh_job_summary_pdfs_task( @@ -148,9 +144,6 @@ def _refresh_job_summary_pdfs_task( follow_up_required = remaining or bool( cache.get(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY) ) - except AlreadyLoggedException: - cache.delete(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY) - raise except Exception as exc: cache.delete(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY) logger.error( @@ -158,7 +151,8 @@ def _refresh_job_summary_pdfs_task( exc, exc_info=True, ) - persist_and_raise(exc) + persist_app_error(exc) + raise finally: cache.delete(JOB_SUMMARY_PDF_REFRESH_RUNNING_KEY) @@ -198,11 +192,10 @@ def set_paid_flag_task() -> None: result.missing_invoices, result.duration_seconds, ) - except AlreadyLoggedException: - raise except Exception as exc: logger.error("Error during set_paid_flag_task: %s", exc, exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise @shared_task(name="apps.job.tasks.auto_archive_completed_jobs_task") @@ -224,10 +217,9 @@ def auto_archive_completed_jobs_task() -> None: result.jobs_archived, result.duration_seconds, ) - except AlreadyLoggedException: - raise except Exception as exc: logger.error( "Error during auto_archive_completed_jobs_task: %s", exc, exc_info=True ) - persist_and_raise(exc) + persist_app_error(exc) + raise diff --git a/apps/job/tests/_pdf_golden_fixtures.py b/apps/job/tests/_pdf_golden_fixtures.py index 9a2e8b5f7..6aa2b5b07 100644 --- a/apps/job/tests/_pdf_golden_fixtures.py +++ b/apps/job/tests/_pdf_golden_fixtures.py @@ -25,6 +25,7 @@ from apps.accounts.models import Staff from apps.company.models import Company, Person +from apps.crm.models import PhoneEndpoint from apps.job.models import CostLine, Job, JobEvent, JobFile, LabourSubtype from apps.workflow.models import CompanyDefaults, XeroPayItem @@ -73,6 +74,17 @@ def build_golden_job(test_staff: Staff) -> Job: company.starting_job_number = STARTING_JOB_NUMBER company.save() + # The letterhead prints the shop's main-line number + # (workshop_pdf_service._primary_company_endpoint_number). Create it here + # so PDF output stays byte-identical regardless of which seed fixtures a + # caller happens to load — this builder is the single source of truth for + # every field that influences the rendered PDF. + PhoneEndpoint.objects.create( + number="+6496365131", + label="Main line", + endpoint_type=PhoneEndpoint.EndpointType.MAIN_LINE, + ) + company = Company.objects.create( name="ACME Engineering Ltd", xero_last_modified=FROZEN_NOW, diff --git a/apps/job/tests/test_event_deduplication.py b/apps/job/tests/test_event_deduplication.py index 77347ac56..52177ea0d 100644 --- a/apps/job/tests/test_event_deduplication.py +++ b/apps/job/tests/test_event_deduplication.py @@ -153,7 +153,7 @@ def test_automatic_events_not_affected(self): } ] } - for i in range(3): + for _ in range(3): JobEvent.objects.create( job=self.job, staff=self.user, diff --git a/apps/job/tests/test_job_rest_error_mapping.py b/apps/job/tests/test_job_rest_error_mapping.py new file mode 100644 index 000000000..f31740b54 --- /dev/null +++ b/apps/job/tests/test_job_rest_error_mapping.py @@ -0,0 +1,127 @@ +"""Status-code mapping for service-layer exceptions. + +The raising site chooses the exception *type* — the semantic claim. The +boundary chooses the *number*. Dispatch is by isinstance so a subclass +answers its base's status: AllocationDeletionError subclasses ValueError +and must be a 400, not a 500. Under the previous name-string dispatch it +was a 500, which is the regression these tests pin. + +Bodies are asserted alongside statuses because the frontend consumes them +through the generated client, where `error` is a required key. +""" + +from django.db import IntegrityError +from django.http import Http404 +from rest_framework import status +from rest_framework.exceptions import NotFound + +from apps.accounting.services.invoice_calculation import InvoiceCalculationError +from apps.job.services.job_rest_service import DeltaValidationError, PreconditionFailed +from apps.job.views.job_rest_views import BaseJobRestView +from apps.purchasing.services.allocation_service import AllocationDeletionError +from apps.testing import BaseTestCase +from apps.workflow.models import AppError + + +class HandleServiceErrorMappingTests(BaseTestCase): + def setUp(self) -> None: + super().setUp() + self.view = BaseJobRestView() + + def assert_maps_to( + self, error: Exception, expected_status: int, expected_body: dict[str, str] + ) -> None: + response = self.view.handle_service_error(error) + self.assertEqual(response.status_code, expected_status) + self.assertEqual(response.data, expected_body) + + def test_value_error_is_a_bad_request(self) -> None: + self.assert_maps_to( + ValueError("Job with id abc not found"), + status.HTTP_400_BAD_REQUEST, + {"error": "Job with id abc not found"}, + ) + + def test_value_error_subclasses_are_bad_requests_not_server_errors(self) -> None: + """The regression: name-string dispatch sent these to 500.""" + for error in ( + AllocationDeletionError("cannot delete allocation"), + InvoiceCalculationError("bad invoice total"), + ): + with self.subTest(error=type(error).__name__): + self.assert_maps_to( + error, + status.HTTP_400_BAD_REQUEST, + {"error": str(error)}, + ) + + def test_precondition_failed_is_412(self) -> None: + self.assert_maps_to( + PreconditionFailed("etag mismatch"), + status.HTTP_412_PRECONDITION_FAILED, + { + "error": ( + "Precondition failed (ETag mismatch). Reload the job and retry." + ) + }, + ) + + def test_delta_validation_error_is_412_not_shadowed(self) -> None: + """DeltaValidationError subclasses PreconditionFailed; it must not + fall through to the generic ValueError or default arm.""" + self.assert_maps_to( + DeltaValidationError("checksum mismatch"), + status.HTTP_412_PRECONDITION_FAILED, + { + "error": ( + "Precondition failed (ETag mismatch). Reload the job and retry." + ) + }, + ) + + def test_not_found_variants_are_404(self) -> None: + for error in (Http404("gone"), NotFound("gone")): + with self.subTest(error=type(error).__name__): + self.assert_maps_to( + error, + status.HTTP_404_NOT_FOUND, + {"error": "Resource not found"}, + ) + + def test_integrity_error_is_409(self) -> None: + self.assert_maps_to( + IntegrityError("duplicate key"), + status.HTTP_409_CONFLICT, + {"error": "Duplicate event prevented by database constraint"}, + ) + + def test_permission_error_is_403(self) -> None: + self.assert_maps_to( + PermissionError("not allowed"), + status.HTTP_403_FORBIDDEN, + {"error": "not allowed"}, + ) + + def test_unmapped_exception_is_500_without_leaking_the_message(self) -> None: + self.assert_maps_to( + RuntimeError("internal detail that should not ship"), + status.HTTP_500_INTERNAL_SERVER_ERROR, + {"error": "Internal server error"}, + ) + + +class HandleServiceErrorPersistenceTests(BaseTestCase): + def test_every_handled_error_is_persisted_exactly_once(self) -> None: + BaseJobRestView().handle_service_error(ValueError("boom")) + + self.assertEqual(AppError.objects.count(), 1) + + def test_an_error_persisted_upstream_is_not_persisted_again(self) -> None: + from apps.workflow.services.error_persistence import persist_app_error + + error = ValueError("boom") + persist_app_error(error) + + BaseJobRestView().handle_service_error(error) + + self.assertEqual(AppError.objects.count(), 1) diff --git a/apps/job/tests/test_job_rest_service.py b/apps/job/tests/test_job_rest_service.py index 17fec110d..8175cb221 100644 --- a/apps/job/tests/test_job_rest_service.py +++ b/apps/job/tests/test_job_rest_service.py @@ -1,13 +1,14 @@ from decimal import Decimal +from uuid import uuid4 from django.utils import timezone from apps.company.models import Company, Person -from apps.job.models import Job, JobEvent +from apps.job.models import Job, JobDeltaRejection, JobEvent from apps.job.models.costing import CostLine -from apps.job.services.job_rest_service import JobRestService +from apps.job.services.job_rest_service import DeltaValidationError, JobRestService from apps.testing import BaseTestCase -from apps.workflow.models import XeroPayItem +from apps.workflow.models import CompanyDefaults, XeroPayItem class JobRestServiceCreateJobTests(BaseTestCase): @@ -68,3 +69,45 @@ def test_get_job_for_edit_serializes_event_staff(self): result = JobRestService.get_job_for_edit(job.id, request=None) self.assertEqual(result["events"][0]["staff"], "Test Staff") + + +class JobRestServiceDeltaRejectionRecordingTests(BaseTestCase): + def test_hard_checksum_mismatch_records_the_rejection(self) -> None: + """A refused delta must leave a JobDeltaRejection explaining why. + + ``DeltaValidationError`` subclasses ``PreconditionFailed``. If the + broader handler is ordered first it swallows the specific one, and + because ``soft_fail_context`` is still None that early in the update + the rejection is dropped entirely — the only record of why a client's + edit was refused disappears. + """ + defaults = CompanyDefaults.get_solo() + defaults.job_delta_soft_fail = False + defaults.save(update_fields=["job_delta_soft_fail"]) + + company = Company.objects.create( + name="Delta Reject Company", + xero_last_modified=timezone.now(), + ) + job = Job.objects.create( + name="Rejectable Job", + company=company, + created_by=self.test_staff, + default_xero_pay_item=XeroPayItem.get_ordinary_time(), + staff=self.test_staff, + ) + + payload = { + "change_id": str(uuid4()), + "fields": ["description"], + "before": {"description": job.description}, + "after": {"description": "Edited elsewhere"}, + "before_checksum": "stale-checksum-from-an-older-read", + } + + with self.assertRaises(DeltaValidationError): + JobRestService.update_job(job.id, payload, self.test_staff) + + rejection = JobDeltaRejection.objects.get() + self.assertIn("checksum mismatch", rejection.reason.lower()) + self.assertEqual(str(rejection.change_id), payload["change_id"]) diff --git a/apps/job/tests/test_job_summary_pdf_service.py b/apps/job/tests/test_job_summary_pdf_service.py index 728fd9e12..07119a15e 100644 --- a/apps/job/tests/test_job_summary_pdf_service.py +++ b/apps/job/tests/test_job_summary_pdf_service.py @@ -4,7 +4,7 @@ from io import BytesIO from pathlib import Path from unittest.mock import patch -from uuid import UUID, uuid4 +from uuid import UUID from django.core.cache import caches from django.test import TestCase, override_settings @@ -21,8 +21,8 @@ request_job_summary_pdf_refresh, ) from apps.testing import BaseTestCase -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError +from apps.workflow.services.error_persistence import persist_app_error class JobSummaryPdfServiceTests(BaseTestCase): @@ -137,7 +137,7 @@ def test_request_refresh_persists_dispatch_failure_and_clears_queue( "apps.job.tasks.refresh_job_summary_pdfs_task.apply_async", side_effect=RuntimeError("broker unavailable"), ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): with self.captureOnCommitCallbacks(execute=True): request_job_summary_pdf_refresh() @@ -155,7 +155,7 @@ def test_refresh_task_persists_failure_once(self) -> None: ), patch("apps.job.tasks._schedule_job_summary_pdf_refresh") as schedule, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): refresh_job_summary_pdfs_task() self.assertEqual(AppError.objects.count(), before + 1) @@ -164,19 +164,17 @@ def test_refresh_task_persists_failure_once(self) -> None: schedule.assert_not_called() def test_refresh_task_passes_prelogged_failure_without_duplicate(self) -> None: - before = AppError.objects.count() cache = caches["shared"] cache.set(JOB_SUMMARY_PDF_REFRESH_QUEUED_KEY, True, timeout=60) - prelogged = AlreadyLoggedException( - RuntimeError("summary render failed"), - uuid4(), - ) + prelogged = RuntimeError("summary render failed") + persist_app_error(prelogged) + before = AppError.objects.count() with patch( "apps.job.services.job_summary_pdf_service.JobSummaryPdfService.refresh_stale", side_effect=prelogged, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): refresh_job_summary_pdfs_task() self.assertEqual(AppError.objects.count(), before) diff --git a/apps/job/tests/test_recreate_jobfiles.py b/apps/job/tests/test_recreate_jobfiles.py new file mode 100644 index 000000000..5890f1c78 --- /dev/null +++ b/apps/job/tests/test_recreate_jobfiles.py @@ -0,0 +1,71 @@ +import importlib.util +import os +import tempfile +import types +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase +from PIL import Image + +SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "recreate_jobfiles.py" + + +def load_recreate_jobfiles() -> types.ModuleType: + spec = importlib.util.spec_from_file_location("recreate_jobfiles", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError("Could not load recreate_jobfiles.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class RecreateJobFilesTests(SimpleTestCase): + def test_image_placeholder_accepts_quotes_in_job_name(self) -> None: + recreate_jobfiles = load_recreate_jobfiles() + + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "placeholder.png" + recreate_jobfiles.create_dummy_file( + str(output), + 'COLOURSTEEL "CLOUD" POWDERCOATING', + 97380, + "placeholder.png", + ) + + with Image.open(output) as image: + self.assertEqual(image.size, (400, 200)) + + def test_pdf_runs_pandoc_in_a_writable_cwd(self) -> None: + """pandoc must never run in the inherited cwd (the read-only release dir). + + Guards the UAT regression where pandoc's intermediate temp write hit + `openTempFile: permission denied` on the immutable release dir. + """ + recreate_jobfiles = load_recreate_jobfiles() + + captured_cwd: str | None = None + cwd_was_writable = False + + def fake_run(cmd: list[str], **kwargs: object) -> types.SimpleNamespace: + nonlocal captured_cwd, cwd_was_writable + cwd = kwargs.get("cwd") + assert isinstance(cwd, str) + captured_cwd = cwd + # The tempdir is deleted after the call, so check writability now. + cwd_was_writable = os.access(cwd, os.W_OK) + return types.SimpleNamespace(returncode=0, stderr="") + + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "placeholder.pdf" + with mock.patch.object( + recreate_jobfiles.subprocess, "run", side_effect=fake_run + ): + recreate_jobfiles.create_dummy_file( + str(output), "Job Name", 97380, "placeholder.pdf" + ) + + self.assertIsNotNone(captured_cwd) + self.assertTrue(cwd_was_writable) + assert captured_cwd is not None + self.assertNotEqual(Path(captured_cwd).resolve(), Path.cwd().resolve()) diff --git a/apps/job/views/data_integrity_views.py b/apps/job/views/data_integrity_views.py index e66896d0b..4737c61ca 100644 --- a/apps/job/views/data_integrity_views.py +++ b/apps/job/views/data_integrity_views.py @@ -17,8 +17,7 @@ DataIntegrityResponseSerializer, ) from apps.job.services.data_integrity_service import DataIntegrityService -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -73,17 +72,11 @@ def get(self, request) -> Response: return Response(serializer.data, status=status.HTTP_200_OK) except Exception as exc: logger.error(f"Error running data integrity scan: {exc}", exc_info=True) - try: - persist_and_raise(exc) - except AlreadyLoggedException as logged_exc: - return Response( - { - "error": f"Failed to run data integrity scan: {str(exc)}", - "error_id": ( - str(logged_exc.app_error_id) - if logged_exc.app_error_id - else None - ), - }, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error(exc) + return Response( + { + "error": f"Failed to run data integrity scan: {str(exc)}", + "error_id": str(app_error.id), + }, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/apps/job/views/data_quality_report_views.py b/apps/job/views/data_quality_report_views.py index 63c2d0607..65ddcc8fa 100644 --- a/apps/job/views/data_quality_report_views.py +++ b/apps/job/views/data_quality_report_views.py @@ -23,7 +23,6 @@ DuplicatePhonesResponseSerializer, ) from apps.job.services.data_quality_report import ArchivedJobsComplianceService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error @@ -59,11 +58,9 @@ def get(self, request) -> Response: # Return the data directly without wrapping return Response(serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException: - raise # already persisted upstream — pass through unchanged except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise class DuplicatePhonesView(APIView): @@ -93,11 +90,9 @@ def get(self, request: Request) -> Response: serializer.is_valid(raise_exception=True) return Response(serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException: - raise # already persisted upstream — pass through unchanged except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise class DuplicateIdentitiesView(APIView): @@ -121,8 +116,6 @@ def get(self, request: Request) -> Response: serializer = DuplicateIdentitiesResponseSerializer(data=result) serializer.is_valid(raise_exception=True) return Response(serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise diff --git a/apps/job/views/job_file_detail_view.py b/apps/job/views/job_file_detail_view.py index dd5b1d062..d794cd42a 100644 --- a/apps/job/views/job_file_detail_view.py +++ b/apps/job/views/job_file_detail_view.py @@ -30,8 +30,7 @@ JobFileSerializer, JobFileUpdateSuccessResponseSerializer, ) -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -93,25 +92,25 @@ def get(self, request, job_id, file_id): return response except Exception as e: logger.exception("Error serving file %s", file_id) - try: - persist_and_raise( - e, - job_id=str(job.id), - user_id=( - str(request.user.id) - if getattr(request.user, "is_authenticated", False) - else None - ), - additional_context={"file_id": str(file_id)}, - ) - except AlreadyLoggedException as logged_exc: - payload = {"status": "error", "message": str(e)} - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response( - payload, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error( + e, + job_id=str(job.id), + user_id=( + str(request.user.id) + if getattr(request.user, "is_authenticated", False) + else None + ), + additional_context={"file_id": str(file_id)}, + ) + payload = { + "status": "error", + "message": str(e), + "error_id": str(app_error.id), + } + return Response( + payload, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) @extend_schema( operation_id="updateJobFile", @@ -200,25 +199,25 @@ def put(self, request, job_id, file_id): except Exception as e: logger.exception("Error updating file %s", file_id) - try: - persist_and_raise( - e, - job_id=str(job.id), - user_id=( - str(request.user.id) - if getattr(request.user, "is_authenticated", False) - else None - ), - additional_context={"file_id": str(file_id)}, - ) - except AlreadyLoggedException as logged_exc: - payload = {"status": "error", "message": str(e)} - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response( - payload, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error( + e, + job_id=str(job.id), + user_id=( + str(request.user.id) + if getattr(request.user, "is_authenticated", False) + else None + ), + additional_context={"file_id": str(file_id)}, + ) + payload = { + "status": "error", + "message": str(e), + "error_id": str(app_error.id), + } + return Response( + payload, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) @extend_schema( operation_id="deleteJobFile", @@ -263,22 +262,22 @@ def delete(self, request, job_id, file_id): except Exception as e: logger.exception("Error deleting file %s", file_id) - try: - persist_and_raise( - e, - job_id=str(job.id), - user_id=( - str(request.user.id) - if getattr(request.user, "is_authenticated", False) - else None - ), - additional_context={"file_id": str(file_id)}, - ) - except AlreadyLoggedException as logged_exc: - payload = {"status": "error", "message": str(e)} - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response( - payload, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error( + e, + job_id=str(job.id), + user_id=( + str(request.user.id) + if getattr(request.user, "is_authenticated", False) + else None + ), + additional_context={"file_id": str(file_id)}, + ) + payload = { + "status": "error", + "message": str(e), + "error_id": str(app_error.id), + } + return Response( + payload, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/apps/job/views/job_file_thumbnail_view.py b/apps/job/views/job_file_thumbnail_view.py index c6f3d4455..88df521d0 100644 --- a/apps/job/views/job_file_thumbnail_view.py +++ b/apps/job/views/job_file_thumbnail_view.py @@ -23,8 +23,7 @@ from apps.job.serializers.job_file_serializer import ( JobFileThumbnailErrorResponseSerializer, ) -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -70,22 +69,22 @@ def get(self, request, job_id, file_id): return FileResponse(open(thumb_path, "rb"), content_type="image/jpeg") except Exception as e: logger.exception("Error serving thumbnail %s", file_id) - try: - persist_and_raise( - e, - job_id=str(job.id), - user_id=( - str(request.user.id) - if getattr(request.user, "is_authenticated", False) - else None - ), - additional_context={"file_id": str(file_id)}, - ) - except AlreadyLoggedException as logged_exc: - payload = {"status": "error", "message": "Could not serve thumbnail"} - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response( - payload, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error( + e, + job_id=str(job.id), + user_id=( + str(request.user.id) + if getattr(request.user, "is_authenticated", False) + else None + ), + additional_context={"file_id": str(file_id)}, + ) + payload = { + "status": "error", + "message": "Could not serve thumbnail", + "error_id": str(app_error.id), + } + return Response( + payload, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/apps/job/views/job_files_collection_view.py b/apps/job/views/job_files_collection_view.py index 1ec95016c..06d34c67d 100644 --- a/apps/job/views/job_files_collection_view.py +++ b/apps/job/views/job_files_collection_view.py @@ -29,11 +29,7 @@ JobFileUploadSuccessResponseSerializer, ) from apps.job.tasks import create_job_file_thumbnail_task -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import ( - persist_and_raise, - persist_app_error, -) +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -97,11 +93,6 @@ def save_file(self, job, file_obj, print_on_jobsheet, request): if file_obj.content_type and file_obj.content_type.startswith("image/"): try: create_job_file_thumbnail_task.delay(str(job_file.id)) - except AlreadyLoggedException: - logger.exception( - "Thumbnail generation failed after upload for file %s", - job_file.id, - ) except Exception as thumb_exc: logger.exception( "Could not enqueue thumbnail generation for file %s", @@ -122,7 +113,7 @@ def save_file(self, job, file_obj, print_on_jobsheet, request): except Exception as e: logger.error("Error saving file %s: %s", file_obj.name, str(e)) - persist_and_raise( + persist_app_error( e, job_id=str(job.id), user_id=( @@ -132,6 +123,7 @@ def save_file(self, job, file_obj, print_on_jobsheet, request): ), additional_context={"filename": file_obj.name}, ) + raise @extend_schema( operation_id="uploadJobFiles", diff --git a/apps/job/views/job_profitability_report_views.py b/apps/job/views/job_profitability_report_views.py index a5db0b326..14ab901c3 100644 --- a/apps/job/views/job_profitability_report_views.py +++ b/apps/job/views/job_profitability_report_views.py @@ -14,8 +14,7 @@ JobProfitabilityReportResponseSerializer, ) from apps.job.services.job_profitability_report import JobProfitabilityReportService -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -69,17 +68,11 @@ def get(self, request) -> Response: logger.error( f"Error generating job profitability report: {exc}", exc_info=True ) - try: - persist_and_raise(exc) - except AlreadyLoggedException as logged_exc: - return Response( - { - "error": f"Failed to generate job profitability report: {str(exc)}", - "error_id": ( - str(logged_exc.app_error_id) - if logged_exc.app_error_id - else None - ), - }, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + app_error = persist_app_error(exc) + return Response( + { + "error": f"Failed to generate job profitability report: {str(exc)}", + "error_id": str(app_error.id), + }, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/apps/job/views/job_quote_chat_views.py b/apps/job/views/job_quote_chat_views.py index cb877e036..6f320e29d 100644 --- a/apps/job/views/job_quote_chat_views.py +++ b/apps/job/views/job_quote_chat_views.py @@ -29,6 +29,7 @@ JobQuoteChatUpdateSerializer, ) from apps.job.serializers.job_quote_chat_serializer import JobQuoteChatCreateSerializer +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -58,7 +59,14 @@ def get_message_or_404(self, job, message_id): return JobQuoteChat.objects.get(job=job, message_id=message_id) def handle_error(self, error: Exception) -> Response: - """Handle errors and return appropriate response using match-case.""" + """Handle errors and return appropriate response using match-case. + + Persists here because this method returns a Response rather than + re-raising, so the exception never reaches the DRF boundary that + would otherwise record it. + """ + persist_app_error(error) + match error: case ParseError(): return Response( diff --git a/apps/job/views/job_rest_views.py b/apps/job/views/job_rest_views.py index e83e06f32..75c9f5f86 100644 --- a/apps/job/views/job_rest_views.py +++ b/apps/job/views/job_rest_views.py @@ -14,13 +14,15 @@ from uuid import UUID from django.core.cache import cache -from django.http import JsonResponse +from django.db import IntegrityError +from django.http import Http404, JsonResponse from django.shortcuts import get_object_or_404 from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import OpenApiParameter, OpenApiTypes, extend_schema from rest_framework import status from rest_framework.decorators import api_view, permission_classes +from rest_framework.exceptions import NotFound from rest_framework.permissions import SAFE_METHODS, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView @@ -52,10 +54,13 @@ QuoteSerializer, WeeklyMetricsSerializer, ) -from apps.job.services.job_rest_service import DeltaValidationError, JobRestService -from apps.workflow.exceptions import AlreadyLoggedException +from apps.job.services.job_rest_service import ( + DeltaValidationError, + JobRestService, + PreconditionFailed, +) from apps.workflow.models import CompanyDefaults -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.utils import parse_pagination_params logger = logging.getLogger(__name__) @@ -91,63 +96,60 @@ def parse_json_body(self, request) -> Dict[str, Any]: try: return json.loads(request.body) except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON: {str(e)}") + raise ValueError(f"Invalid JSON: {str(e)}") from e def handle_service_error(self, error: Exception) -> Response: """ Centralise service layer error handling with error persistence. + + Dispatch is by isinstance, not by type name, so subclasses resolve to + their base's status — AllocationDeletionError and friends subclass + ValueError and must answer 400, not 500. Order is specific-before-general. """ - try: - # Persist error for debugging - persist_and_raise(error) - except AlreadyLoggedException as logged_exc: - logger.error( - f"[JOB-REST-VIEW] Handled and persisted error {str(error)} " - f"(error_id={logged_exc.app_error_id})" - ) - except Exception as persist_error: - logger.error(f"Failed to persist error: {persist_error}") + app_error = persist_app_error(error) + logger.error( + f"[JOB-REST-VIEW] Handled and persisted error {str(error)} " + f"(error_id={app_error.id})" + ) error_message = str(error) - match type(error).__name__: - case "ValueError": - error_response = {"error": error_message} - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response( - error_serializer.data, status=status.HTTP_400_BAD_REQUEST - ) - case "PermissionError": - error_response = {"error": error_message} - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response(error_serializer.data, status=status.HTTP_403_FORBIDDEN) - case "IntegrityError": - # Handle database constraint violations (duplicates) - error_response = { - "error": "Duplicate event prevented by database constraint" - } - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response(error_serializer.data, status=status.HTTP_409_CONFLICT) - case "NotFound" | "Http404": - error_response = {"error": "Resource not found"} - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response(error_serializer.data, status=status.HTTP_404_NOT_FOUND) - case "PreconditionFailed" | "DeltaValidationError": - # ETag mismatch -> Optimistic concurrency conflict - error_response = { - "error": "Precondition failed (ETag mismatch). Reload the job and retry." - } - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response( - error_serializer.data, status=status.HTTP_412_PRECONDITION_FAILED - ) - case _: - logger.exception(f"Unhandled error: {error}") - error_response = {"error": "Internal server error"} - error_serializer = JobRestErrorResponseSerializer(error_response) - return Response( - error_serializer.data, status=status.HTTP_500_INTERNAL_SERVER_ERROR - ) + # PreconditionFailed covers DeltaValidationError, which subclasses it. + if isinstance(error, PreconditionFailed): + # ETag mismatch -> Optimistic concurrency conflict + error_response = { + "error": "Precondition failed (ETag mismatch). Reload the job and retry." + } + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response( + error_serializer.data, status=status.HTTP_412_PRECONDITION_FAILED + ) + if isinstance(error, (Http404, NotFound)): + error_response = {"error": "Resource not found"} + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response(error_serializer.data, status=status.HTTP_404_NOT_FOUND) + if isinstance(error, IntegrityError): + # Handle database constraint violations (duplicates) + error_response = { + "error": "Duplicate event prevented by database constraint" + } + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response(error_serializer.data, status=status.HTTP_409_CONFLICT) + if isinstance(error, PermissionError): + error_response = {"error": error_message} + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response(error_serializer.data, status=status.HTTP_403_FORBIDDEN) + if isinstance(error, ValueError): + error_response = {"error": error_message} + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response(error_serializer.data, status=status.HTTP_400_BAD_REQUEST) + + logger.exception(f"Unhandled error: {error}") + error_response = {"error": "Internal server error"} + error_serializer = JobRestErrorResponseSerializer(error_response) + return Response( + error_serializer.data, status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) def check_request_debounce( self, request, operation_key: str, debounce_seconds: int = 2 @@ -916,8 +918,8 @@ def get(self, request, job_id): resp = self._set_etag(resp, current_etag) return resp - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as e: return self.handle_service_error(e) @@ -958,8 +960,8 @@ def get(self, request, job_id): resp = Response(serializer.data, status=status.HTTP_200_OK) return self._set_etag(resp, current_etag) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as e: return self.handle_service_error(e) @@ -999,8 +1001,8 @@ def get(self, request, job_id): resp = Response(quote, status=status.HTTP_200_OK) return self._set_etag(resp, current_etag) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as e: return self.handle_service_error(e) @@ -1125,8 +1127,8 @@ def calculate_profit_margin(cost_set): resp = Response(serializer.data, status=status.HTTP_200_OK) return self._set_etag(resp, current_etag) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as e: return self.handle_service_error(e) @@ -1194,8 +1196,8 @@ def get(self, request, job_id): serializer = JobEventsResponseSerializer({"events": events}) return Response(serializer.data, status=status.HTTP_200_OK) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc except Exception as e: return self.handle_service_error(e) @@ -1232,8 +1234,8 @@ class JobDeltaRejectionListRestView(BaseJobRestView): def get(self, request, job_id: UUID): try: job = get_object_or_404(Job.objects.only("id"), id=job_id) - except Exception: - raise ValueError(f"Job with id {job_id} not found") + except Http404 as exc: + raise ValueError(f"Job with id {job_id} not found") from exc try: limit, offset = parse_pagination_params(request) @@ -1438,8 +1440,8 @@ def get(self, request, job_id): try: job = Job.objects.only("id", "updated_at").get(id=job_id) current_etag = self._gen_job_etag(job) - except Job.DoesNotExist: - raise ValueError(f"Job with id {job_id} not found") + except Job.DoesNotExist as exc: + raise ValueError(f"Job with id {job_id} not found") from exc if_none_match = self._get_if_none_match(request) if if_none_match and self._normalize_etag(current_etag) == if_none_match: diff --git a/apps/job/views/kanban_view_api.py b/apps/job/views/kanban_view_api.py index bc237261c..9c1bb9a88 100644 --- a/apps/job/views/kanban_view_api.py +++ b/apps/job/views/kanban_view_api.py @@ -27,15 +27,12 @@ KanbanSuccessResponseSerializer, ) from apps.job.services.kanban_service import KanbanService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) def _persist_unexpected_error(exc: Exception) -> None: - if isinstance(exc, AlreadyLoggedException): - return persist_app_error(exc) diff --git a/apps/job/views/workshop_pdf_view.py b/apps/job/views/workshop_pdf_view.py index 89cd70e46..fbed69b31 100644 --- a/apps/job/views/workshop_pdf_view.py +++ b/apps/job/views/workshop_pdf_view.py @@ -10,8 +10,7 @@ from apps.job.models import Job from apps.job.serializers.job_serializer import WorkshopPDFResponseSerializer from apps.job.services.workshop_pdf_service import create_workshop_pdf -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -55,32 +54,23 @@ def get(self, request, job_id): return response - except AlreadyLoggedException as exc: + except Exception as exc: logger.exception("Error generating workshop PDF for job %s", job_id) - payload = {"status": "error", "message": str(exc.original)} - if exc.app_error_id: - payload["error_id"] = str(exc.app_error_id) + app_error = persist_app_error( + exc, + job_id=str(job_id), + user_id=( + str(request.user.id) + if getattr(request.user, "is_authenticated", False) + else None + ), + ) + payload = { + "status": "error", + "message": str(exc), + "error_id": str(app_error.id), + } return Response( payload, status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - except Exception as exc: - logger.exception("Error generating workshop PDF for job %s", job_id) - try: - persist_and_raise( - exc, - job_id=str(job_id), - user_id=( - str(request.user.id) - if getattr(request.user, "is_authenticated", False) - else None - ), - ) - except AlreadyLoggedException as logged_exc: - payload = {"status": "error", "message": str(logged_exc.original)} - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response( - payload, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) diff --git a/apps/purchasing/models.py b/apps/purchasing/models.py index 015ca492b..5f5f49ad3 100644 --- a/apps/purchasing/models.py +++ b/apps/purchasing/models.py @@ -616,7 +616,7 @@ def retail_rate(self, value): except (ValueError, TypeError, InvalidOperation) as e: raise ValueError( f"Invalid retail rate value: {value} (type: {type(value)}). Error: {e}" - ) + ) from e else: raise ValueError( "Unit cost must be set and greater than zero to set retail rate" diff --git a/apps/purchasing/services/allocation_service.py b/apps/purchasing/services/allocation_service.py index 44c80140d..24ceda806 100644 --- a/apps/purchasing/services/allocation_service.py +++ b/apps/purchasing/services/allocation_service.py @@ -10,8 +10,7 @@ from apps.job.models import CostLine from apps.purchasing.models import PurchaseOrder, PurchaseOrderLine, Stock -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -96,11 +95,9 @@ def delete_allocation( except AllocationDeletionError as exc: logger.error("Allocation deletion validation error: %s", exc) raise - except AlreadyLoggedException: - raise except Exception as exc: logger.error("Unexpected error during allocation deletion: %s", exc) - persist_and_raise( + persist_app_error( exc, additional_context={ "po_id": str(po_id), @@ -109,6 +106,7 @@ def delete_allocation( "allocation_id": str(allocation_id), }, ) + raise @staticmethod def get_allocation_details( @@ -152,11 +150,9 @@ def get_allocation_details( "unit_cost": float(cost_line.unit_cost), "unit_revenue": float(cost_line.unit_rev), } - except AlreadyLoggedException: - raise except Exception as exc: logger.error("Error getting allocation details: %s", exc) - persist_and_raise( + persist_app_error( exc, additional_context={ "po_id": str(po_id), @@ -164,13 +160,14 @@ def get_allocation_details( "allocation_id": str(allocation_id), }, ) + raise @staticmethod def _get_po_or_error(po_id: str) -> PurchaseOrder: try: return PurchaseOrder.objects.get(id=po_id) - except PurchaseOrder.DoesNotExist: - raise AllocationDeletionError(f"Purchase Order {po_id} not found") + except PurchaseOrder.DoesNotExist as exc: + raise AllocationDeletionError(f"Purchase Order {po_id} not found") from exc @staticmethod def _get_stock_or_error(po: PurchaseOrder, stock_id: str) -> Stock: @@ -180,10 +177,10 @@ def _get_stock_or_error(po: PurchaseOrder, stock_id: str) -> Stock: source="purchase_order", source_purchase_order_line__purchase_order=po, ) - except Stock.DoesNotExist: + except Stock.DoesNotExist as exc: raise AllocationDeletionError( f"Stock allocation {stock_id} not found or not from PO {po.id}" - ) + ) from exc @staticmethod def _get_costline_or_error(po: PurchaseOrder, cost_line_id: str) -> CostLine: @@ -204,10 +201,10 @@ def _get_costline_or_error(po: PurchaseOrder, cost_line_id: str) -> CostLine: ) .get(id=cost_line_id, po_id=str(po.id)) ) - except CostLine.DoesNotExist: + except CostLine.DoesNotExist as exc: raise AllocationDeletionError( f"Job allocation {cost_line_id} not found or not from PO {po.id}" - ) + ) from exc if not line.po_line_id: raise AllocationDeletionError( @@ -231,10 +228,10 @@ def _resolve_allocation_or_error( po_line = PurchaseOrderLine.objects.get( id=line.ext_refs["purchase_order_line_id"], purchase_order=po ) - except PurchaseOrderLine.DoesNotExist: + except PurchaseOrderLine.DoesNotExist as exc: raise AllocationDeletionError( f"Purchase Order Line referenced by allocation {allocation_id} not found" - ) + ) from exc return po_line, line @staticmethod diff --git a/apps/purchasing/services/purchase_order_pdf_service.py b/apps/purchasing/services/purchase_order_pdf_service.py index a1e725aaf..622b61cda 100644 --- a/apps/purchasing/services/purchase_order_pdf_service.py +++ b/apps/purchasing/services/purchase_order_pdf_service.py @@ -271,7 +271,7 @@ def add_line_items_table(self, y_position): lines_table.setStyle(table_style) # Check if table fits on current page - table_width, table_height = lines_table.wrap(CONTENT_WIDTH, PAGE_HEIGHT) + _table_width, table_height = lines_table.wrap(CONTENT_WIDTH, PAGE_HEIGHT) if y_position - table_height < MARGIN + 50: # 50 is space for footer # Start new page if needed self.pdf.showPage() diff --git a/apps/purchasing/services/quote_to_po_service.py b/apps/purchasing/services/quote_to_po_service.py index 3bfcfb840..f66cf7d49 100644 --- a/apps/purchasing/services/quote_to_po_service.py +++ b/apps/purchasing/services/quote_to_po_service.py @@ -25,7 +25,6 @@ PurchaseOrderSupplierQuote, ) from apps.workflow.enums import AIProviderTypes -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AIProvider from apps.workflow.services.error_persistence import persist_app_error @@ -415,12 +414,10 @@ def extract_data_from_supplier_quote( # Return the extracted data return quote_data, None - except AlreadyLoggedException: - raise except Exception as exc: logger.exception(f"Error extracting data from supplier quote: {exc}") - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def read_file_content(file_path: str) -> Optional[bytes]: @@ -709,19 +706,17 @@ def extract_data_from_supplier_quote_gemini( return quote_data, None - except AlreadyLoggedException: - raise except json.JSONDecodeError as exc: logger.exception(f"Error decoding JSON from Gemini response: {exc}") invalid_json_error = ValueError(f"Invalid JSON response from Gemini: {exc!s}") - err = persist_app_error(invalid_json_error) - raise AlreadyLoggedException(invalid_json_error, err.id) from exc + persist_app_error(invalid_json_error) + raise invalid_json_error from exc except Exception as exc: logger.exception( f"Error extracting data from supplier quote with Gemini: {exc}" ) - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def create_po_from_quote( @@ -761,7 +756,7 @@ def create_po_from_quote( err_msg = f"Invalid AI provider received: {ai_provider}." logger.error(err_msg) error = err_msg - except AlreadyLoggedException as exc: + except Exception as exc: logger.exception(f"Quote extraction failed for {quote.filename}") return None, str(exc) diff --git a/apps/purchasing/services/stock_search_service.py b/apps/purchasing/services/stock_search_service.py index aedcbc14c..8ba8cb0cc 100644 --- a/apps/purchasing/services/stock_search_service.py +++ b/apps/purchasing/services/stock_search_service.py @@ -11,8 +11,7 @@ from apps.job.models.costing import CostLine from apps.purchasing.models import Stock from apps.purchasing.serializers import StockItemSerializer -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.search import apply_text_search logger = logging.getLogger(__name__) @@ -438,12 +437,11 @@ def search_stock(query: str, limit: int = 10) -> List[Dict[str, Any]]: ) return _serialize(results, usage_counts) - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - persist_and_raise(exc, additional_context={"query": query, "limit": limit}) + persist_app_error(exc, additional_context={"query": query, "limit": limit}) + raise def list_stock( @@ -505,12 +503,10 @@ def list_stock( "total_pages": total_pages, } - except AlreadyLoggedException: - raise except ValueError: raise except Exception as exc: - persist_and_raise( + persist_app_error( exc, additional_context={ "query": query, @@ -518,3 +514,4 @@ def list_stock( "page_size": page_size, }, ) + raise diff --git a/apps/purchasing/tasks.py b/apps/purchasing/tasks.py index 7c5917827..eb58300ce 100644 --- a/apps/purchasing/tasks.py +++ b/apps/purchasing/tasks.py @@ -10,7 +10,6 @@ from apps.purchasing.models import Stock from apps.quoting.services.stock_parser import auto_parse_stock_item -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger("apps.purchasing.tasks") @@ -99,8 +98,6 @@ def _parse_stock_item_task(stock_id: str, force: bool = False) -> None: return else: auto_parse_stock_item(stock, force=force) - except AlreadyLoggedException: - raise except Exception as exc: logger.error( "Error parsing stock metadata for %s: %s", @@ -108,8 +105,8 @@ def _parse_stock_item_task(stock_id: str, force: bool = False) -> None: exc, exc_info=True, ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise parse_stock_item_task = cast( @@ -141,16 +138,14 @@ def _parse_unparsed_stock_items_task(limit: int = 50) -> None: for stock_id in stock_ids: parse_stock_item_task.delay(str(stock_id)) logger.info("Queued %s stock metadata parse tasks.", len(stock_ids)) - except AlreadyLoggedException: - raise except Exception as exc: logger.error( "Error queueing stock metadata parse batch: %s", exc, exc_info=True, ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise parse_unparsed_stock_items_task = cast( diff --git a/apps/purchasing/tests/test_quote_to_po_service.py b/apps/purchasing/tests/test_quote_to_po_service.py index 29fe2b57f..155c1e2b9 100644 --- a/apps/purchasing/tests/test_quote_to_po_service.py +++ b/apps/purchasing/tests/test_quote_to_po_service.py @@ -9,6 +9,7 @@ import pytest from django.conf import settings from django.utils import timezone +from pydantic import ValidationError from apps.company.models import Company from apps.job.enums import MetalType @@ -21,7 +22,6 @@ extract_data_from_supplier_quote, ) from apps.workflow.enums import AIProviderTypes -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AIProvider @@ -186,7 +186,7 @@ def test_extract_rejects_malformed_quote_payload_shape( "apps.purchasing.services.quote_to_po_service.anthropic.Anthropic", return_value=mock_client, ): - with pytest.raises(AlreadyLoggedException) as exc_info: + with pytest.raises(ValidationError) as exc_info: extract_data_from_supplier_quote(str(quote_path), content_type="text/plain") assert error_text in str(exc_info.value) @@ -318,7 +318,7 @@ def test_create_po_from_quote_returns_already_logged_extraction_error( file_path="quote.txt", mime_type="text/plain", ) - logged_error = AlreadyLoggedException(ValueError("Invalid quote payload"), "err-1") + logged_error = ValueError("Invalid quote payload") with patch( "apps.purchasing.services.quote_to_po_service.extract_data_from_supplier_quote", diff --git a/apps/purchasing/tests/test_stock_metadata_tasks.py b/apps/purchasing/tests/test_stock_metadata_tasks.py index 04e2e2c47..f8a387e6c 100644 --- a/apps/purchasing/tests/test_stock_metadata_tasks.py +++ b/apps/purchasing/tests/test_stock_metadata_tasks.py @@ -27,7 +27,6 @@ ) from apps.quoting.services.stock_parser import auto_parse_stock_item from apps.workflow.api.xero.transforms import transform_stock -from apps.workflow.exceptions import AlreadyLoggedException _transform_stock = cast(Callable[[Any, str], tuple[Stock, str]], transform_stock) @@ -244,7 +243,7 @@ def test_auto_parse_stock_item_does_not_record_attempt_on_parser_exception() -> parser_class.return_value.parse_product.side_effect = TimeoutError( "Gemini timeout" ) - with pytest.raises(AlreadyLoggedException): + with pytest.raises(TimeoutError): auto_parse_stock_item(stock) stock.refresh_from_db() diff --git a/apps/purchasing/views/purchasing_rest_views.py b/apps/purchasing/views/purchasing_rest_views.py index f82229e81..0e37b1267 100644 --- a/apps/purchasing/views/purchasing_rest_views.py +++ b/apps/purchasing/views/purchasing_rest_views.py @@ -145,7 +145,7 @@ def get(self, request): total_products = None changes_last_update = None if pls: - latest_id, latest_file, latest_dt = pls[0] + latest_id, _latest_file, _latest_dt = pls[0] # Count all supplier products linked to this supplier (across price lists) total_products = SupplierProduct.objects.filter( supplier_id=s.id diff --git a/apps/purchasing/views/stock_search_rest_view.py b/apps/purchasing/views/stock_search_rest_view.py index c511dd3d2..d60fb310e 100644 --- a/apps/purchasing/views/stock_search_rest_view.py +++ b/apps/purchasing/views/stock_search_rest_view.py @@ -23,7 +23,6 @@ from apps.purchasing.serializers import StockSearchResponseSerializer from apps.purchasing.services.stock_search_service import MAX_PAGE_SIZE, list_stock -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.search_telemetry import SearchTelemetryService @@ -31,19 +30,12 @@ def _build_server_error_response(*, message: str, exc: Exception) -> Response: - if isinstance(exc, AlreadyLoggedException): - root_exc = exc.original - error_id = exc.app_error_id - else: - root_exc = exc - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) - - logger.error("%s: %s", message, root_exc) - - payload: Dict[str, Any] = {"error": message, "details": str(root_exc)} - if error_id: - payload["error_id"] = str(error_id) + app_error = persist_app_error(exc) + + logger.error("%s: %s", message, exc) + + payload: Dict[str, Any] = {"error": message, "details": str(exc)} + payload["error_id"] = str(app_error.id) return Response(payload, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/apps/purchasing/views/supplier_search_rest_view.py b/apps/purchasing/views/supplier_search_rest_view.py index 53ef30906..ffd57eaff 100644 --- a/apps/purchasing/views/supplier_search_rest_view.py +++ b/apps/purchasing/views/supplier_search_rest_view.py @@ -20,26 +20,18 @@ MAX_PAGE_SIZE, list_suppliers, ) -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) def _build_server_error_response(*, message: str, exc: Exception) -> Response: - if isinstance(exc, AlreadyLoggedException): - root_exc = exc.original - error_id = exc.app_error_id - else: - root_exc = exc - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) + app_error = persist_app_error(exc) - logger.error("%s: %s", message, root_exc) + logger.error("%s: %s", message, exc) - payload: dict[str, Any] = {"error": message, "details": str(root_exc)} - if error_id: - payload["error_id"] = str(error_id) + payload: dict[str, Any] = {"error": message, "details": str(exc)} + payload["error_id"] = str(app_error.id) return Response(payload, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/apps/quoting/scrapers/base.py b/apps/quoting/scrapers/base.py index b42335172..5b3091531 100644 --- a/apps/quoting/scrapers/base.py +++ b/apps/quoting/scrapers/base.py @@ -12,7 +12,6 @@ from selenium.webdriver.chrome.options import Options from apps.quoting.services.product_parser import create_mapping_record -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error @@ -226,8 +225,6 @@ def run(self): populate_all_mappings_with_llm() self.logger.info("LLM parsing completed") - except AlreadyLoggedException as exc: - self.logger.error("LLM parsing failed: %s", exc.original, exc_info=True) except Exception as e: persist_app_error(e) self.logger.error(f"LLM parsing failed: {e}", exc_info=True) diff --git a/apps/quoting/services/ai_price_extraction.py b/apps/quoting/services/ai_price_extraction.py index da7dc371d..df1dee301 100644 --- a/apps/quoting/services/ai_price_extraction.py +++ b/apps/quoting/services/ai_price_extraction.py @@ -1,51 +1,38 @@ -import abc import logging from typing import Any, Dict, Optional, Tuple from apps.workflow.enums import AIProviderTypes -from .providers.gemini_provider import GeminiPriceExtractionProvider +from .providers.base import PriceExtractionProvider +from .providers.gemini_provider import ( + GEMINI_FLASH_MODEL, + GeminiPriceExtractionProvider, +) # from .providers.claude_provider import ClaudePriceExtractionProvider -from .providers.mistral_provider import MistralPriceExtractionProvider +from .providers.mistral_provider import ( + MISTRAL_OCR_MODEL, + MistralPriceExtractionProvider, +) logger = logging.getLogger(__name__) -class PriceExtractionProvider(abc.ABC): - """Abstract base class for AI price extraction providers.""" - - provider_name: str - - @abc.abstractmethod - def extract_price_data( - self, file_path: str, content_type: Optional[str] = None - ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: - """ - Extract price data from a supplier price list file. - - Args: - file_path: Path to the price list file - content_type: MIME type of the file - - Returns: - Tuple containing extracted data dict and error message if any - """ - - class PriceExtractionFactory: """Factory for creating AI price extraction providers.""" @staticmethod def create_provider( - provider_type: str, api_key: str, model_name: str = None + provider_type: str, api_key: str, model_name: str | None = None ) -> PriceExtractionProvider: """Create a provider instance based on type.""" if provider_type == AIProviderTypes.MISTRAL: - return MistralPriceExtractionProvider(api_key) + return MistralPriceExtractionProvider( + api_key, model_name or MISTRAL_OCR_MODEL + ) elif provider_type == AIProviderTypes.GOOGLE: return GeminiPriceExtractionProvider( - api_key, model_name or "gemini-2.0-flash-exp" + api_key, model_name or GEMINI_FLASH_MODEL ) # elif provider_type == AIProviderTypes.ANTHROPIC: # return ClaudePriceExtractionProvider(api_key) diff --git a/apps/quoting/services/pdf_data_validation.py b/apps/quoting/services/pdf_data_validation.py index f50f21800..a85e58f73 100644 --- a/apps/quoting/services/pdf_data_validation.py +++ b/apps/quoting/services/pdf_data_validation.py @@ -16,9 +16,9 @@ class PDFDataValidationService: and duplicate detection for supplier products. """ - def __init__(self): - self.validation_errors = [] - self.warnings = [] + def __init__(self) -> None: + self.validation_errors: List[str] = [] + self.warnings: List[str] = [] def validate_extracted_data( self, data: Dict[str, Any] @@ -260,14 +260,14 @@ def _normalize_price(self, price: Any) -> Optional[float]: price_str = price_str[:-1] try: return float(price_str) / 100.0 - except ValueError: - raise ValueError(f"Invalid percentage format: {price}") + except ValueError as exc: + raise ValueError(f"Invalid percentage format: {price}") from exc # Try to convert to float try: return float(price_str) - except ValueError: - raise ValueError(f"Cannot parse price: {price}") + except ValueError as exc: + raise ValueError(f"Cannot parse price: {price}") from exc def check_duplicates( self, products: List[Dict[str, Any]], supplier_name: str diff --git a/apps/quoting/services/providers/base.py b/apps/quoting/services/providers/base.py new file mode 100644 index 000000000..cd9970be8 --- /dev/null +++ b/apps/quoting/services/providers/base.py @@ -0,0 +1,24 @@ +import abc +from typing import Any, Dict, Optional, Tuple + + +class PriceExtractionProvider(abc.ABC): + """Abstract base class for AI price extraction providers.""" + + provider_name: str + model_name: str + + @abc.abstractmethod + def extract_price_data( + self, file_path: str, content_type: Optional[str] = None + ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + Extract price data from a supplier price list file. + + Args: + file_path: Path to the price list file + content_type: MIME type of the file + + Returns: + Tuple containing extracted data dict and error message if any + """ diff --git a/apps/quoting/services/providers/gemini_provider.py b/apps/quoting/services/providers/gemini_provider.py index 289771b17..0e52526f7 100644 --- a/apps/quoting/services/providers/gemini_provider.py +++ b/apps/quoting/services/providers/gemini_provider.py @@ -12,17 +12,20 @@ from apps.workflow.models import CompanyDefaults +from .base import PriceExtractionProvider from .common import clean_json_response, create_extraction_prompt, log_token_usage logger = logging.getLogger(__name__) +GEMINI_FLASH_MODEL = "gemini-flash-latest" -class GeminiPriceExtractionProvider: + +class GeminiPriceExtractionProvider(PriceExtractionProvider): """Gemini AI provider for price extraction from PDF documents.""" provider_name = "Gemini" - def __init__(self, api_key: str, model_name: str = "gemini-2.5-flash"): + def __init__(self, api_key: str, model_name: str = GEMINI_FLASH_MODEL): self.api_key = api_key self.model_name = model_name @@ -30,7 +33,7 @@ def extract_price_data( self, file_path: str, content_type: Optional[str] = None ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: """ - Extract price data from a supplier price list PDF using Gemini 2.5 Flash. + Extract price data from a supplier price list PDF using Gemini Flash. Args: file_path: Path to the PDF file @@ -140,7 +143,7 @@ def _process_extracted_data( "total_lines": len(str(raw_data).split("\n")), "items_found": len(processed_items), "pages_processed": 1, # Gemini processes the entire PDF at once - "extraction_method": "Gemini 2.5 Flash", + "extraction_method": f"Gemini ({self.model_name})", }, } @@ -420,7 +423,9 @@ def _extract_from_multiple_pages( "total_lines": len(str(all_items).split("\n")), "items_found": len(all_items), "pages_processed": num_pages, - "extraction_method": "Gemini 2.5 Flash (Page-by-page)", + "extraction_method": ( + f"Gemini ({self.model_name}, page-by-page)" + ), }, } logger.info( diff --git a/apps/quoting/services/providers/mistral_provider.py b/apps/quoting/services/providers/mistral_provider.py index a864725c5..15714db7f 100644 --- a/apps/quoting/services/providers/mistral_provider.py +++ b/apps/quoting/services/providers/mistral_provider.py @@ -8,8 +8,34 @@ from mistralai.client.sdk import Mistral +from .base import PriceExtractionProvider + logger = logging.getLogger(__name__) +MISTRAL_OCR_MODEL = "mistral-ocr-latest" + + +def _format_dimensions(parsed: Dict[str, Optional[str]]) -> str: + """Render parsed dimensions as the display string the importer stores. + + PDFDataValidationService writes `dimensions` straight to a text column via + _clean_text, which stringifies whatever it is given — so anything but a + string here lands in the database as its Python repr. + """ + sized = [ + part + for part in (parsed["thickness"], parsed["width"], parsed["length"]) + if part + ] + if sized: + return " x ".join(sized) + + # Nothing sheet- or tube-shaped was parsed; round stock carries a diameter. + diameter = parsed["diameter"] + if not diameter: + return "" + return f"dia {diameter}" + def encode_pdf(pdf_path): """Encode the PDF file to base64.""" @@ -21,13 +47,14 @@ def encode_pdf(pdf_path): return None -class MistralPriceExtractionProvider: +class MistralPriceExtractionProvider(PriceExtractionProvider): """Mistral AI provider for price extraction using OCR""" provider_name = "Mistral" - def __init__(self, api_key: str): + def __init__(self, api_key: str, model_name: str = MISTRAL_OCR_MODEL): self.api_key = api_key + self.model_name = model_name def _extract_supplier_from_text(self, text: str) -> str: """Extract supplier name from the OCR text.""" @@ -173,19 +200,19 @@ def _extract_products_from_markdown_tables( # Create variant ID from description variant_id = description.replace(" ", "_").replace("/", "_")[:100] + # Field names here are a contract with + # PDFDataValidationService._sanitize_single_product — it + # reads item_no, price_unit and a string dimensions, and + # silently drops anything named differently. product = { "description": description, - "supplier_item_code": item_code, + "item_no": item_code, "variant_id": variant_id, "unit_price": unit_price, + "price_unit": "each", "category": current_category, "specifications": dimensions["specifications"], - "dimensions": { - "width": dimensions["width"], - "length": dimensions["length"], - "thickness": dimensions.get("thickness"), - "diameter": dimensions.get("diameter"), - }, + "dimensions": _format_dimensions(dimensions), "product_name": ( f"{current_category} - {description}" if current_category @@ -305,7 +332,7 @@ def extract_price_data( raise ValueError("Failed to encode PDF file") # Process the document with OCR ocr_response = client.ocr.process( - model="mistral-ocr-latest", + model=self.model_name, document={ "type": "document_url", "document_url": f"data:application/pdf;base64,{base64_pdf}", diff --git a/apps/quoting/services/stock_parser.py b/apps/quoting/services/stock_parser.py index 6ff3402a1..c497767dd 100644 --- a/apps/quoting/services/stock_parser.py +++ b/apps/quoting/services/stock_parser.py @@ -8,7 +8,6 @@ from apps.job.enums import MetalType from apps.purchasing.models import Stock from apps.quoting.services.product_parser import ProductParser -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -200,9 +199,7 @@ def auto_parse_stock_item(stock_instance: Stock, *, force: bool = False) -> None ) logger.warning("Failed to parse stock item %s", stock_instance.id) - except AlreadyLoggedException: - raise except Exception as exc: logger.exception("Error parsing stock item %s: %s", stock_instance.id, exc) - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise diff --git a/apps/quoting/tasks.py b/apps/quoting/tasks.py index cc5fe28b1..eb18b3725 100644 --- a/apps/quoting/tasks.py +++ b/apps/quoting/tasks.py @@ -10,7 +10,6 @@ from django.core.management import call_command from django.db import close_old_connections -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger("apps.quoting.tasks") @@ -28,9 +27,7 @@ def run_all_scrapers_task() -> None: close_old_connections() call_command("run_scrapers", refresh_old=True) logger.info("Successfully completed scheduled scraper run.") - except AlreadyLoggedException: - raise except Exception as exc: logger.error("Error during scheduled scraper run: %s", exc, exc_info=True) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise diff --git a/apps/quoting/tests/test_ai_price_extraction.py b/apps/quoting/tests/test_ai_price_extraction.py new file mode 100644 index 000000000..3d40e734d --- /dev/null +++ b/apps/quoting/tests/test_ai_price_extraction.py @@ -0,0 +1,72 @@ +from django.test import SimpleTestCase + +from apps.quoting.services.ai_price_extraction import PriceExtractionFactory +from apps.quoting.services.providers.base import PriceExtractionProvider +from apps.quoting.services.providers.gemini_provider import ( + GEMINI_FLASH_MODEL, + GeminiPriceExtractionProvider, +) +from apps.quoting.services.providers.mistral_provider import ( + MISTRAL_OCR_MODEL, + MistralPriceExtractionProvider, +) +from apps.workflow.enums import AIProviderTypes + + +class GeminiModelSelectionTests(SimpleTestCase): + def test_provider_defaults_to_rolling_flash_alias(self) -> None: + provider = GeminiPriceExtractionProvider("test-api-key") + + self.assertEqual(provider.model_name, GEMINI_FLASH_MODEL) + + def test_factory_uses_rolling_flash_alias_when_model_is_not_configured( + self, + ) -> None: + provider = PriceExtractionFactory.create_provider( + AIProviderTypes.GOOGLE, + "test-api-key", + "", + ) + + self.assertIsInstance(provider, GeminiPriceExtractionProvider) + self.assertEqual(provider.model_name, GEMINI_FLASH_MODEL) + + def test_factory_preserves_an_explicit_gemini_model(self) -> None: + provider = PriceExtractionFactory.create_provider( + AIProviderTypes.GOOGLE, + "test-api-key", + "gemini-pro-latest", + ) + + self.assertEqual(provider.model_name, "gemini-pro-latest") + + +class ProviderContractTests(SimpleTestCase): + """Every provider the factory can return honours the shared contract.""" + + def test_factory_returns_a_provider_with_a_model_name(self) -> None: + for provider_type in (AIProviderTypes.GOOGLE, AIProviderTypes.MISTRAL): + with self.subTest(provider_type=provider_type): + provider = PriceExtractionFactory.create_provider( + provider_type, + "test-api-key", + "", + ) + + self.assertIsInstance(provider, PriceExtractionProvider) + self.assertTrue(provider.provider_name) + self.assertTrue(provider.model_name) + + def test_mistral_defaults_to_the_rolling_ocr_alias(self) -> None: + provider = MistralPriceExtractionProvider("test-api-key") + + self.assertEqual(provider.model_name, MISTRAL_OCR_MODEL) + + def test_factory_preserves_an_explicit_mistral_model(self) -> None: + provider = PriceExtractionFactory.create_provider( + AIProviderTypes.MISTRAL, + "test-api-key", + "mistral-ocr-2505", + ) + + self.assertEqual(provider.model_name, "mistral-ocr-2505") diff --git a/apps/quoting/tests/test_ocr_fixtures.py b/apps/quoting/tests/test_ocr_fixtures.py index a7f98531c..a0ff3d19c 100644 --- a/apps/quoting/tests/test_ocr_fixtures.py +++ b/apps/quoting/tests/test_ocr_fixtures.py @@ -2,6 +2,7 @@ from types import SimpleNamespace from unittest.mock import Mock, patch +from apps.quoting.services.pdf_data_validation import PDFDataValidationService from apps.quoting.services.providers.mistral_provider import ( MistralPriceExtractionProvider, ) @@ -24,6 +25,26 @@ def _ocr_response(self): ) return SimpleNamespace(pages=[page]) + def _ocr_response_with_item_code(self) -> SimpleNamespace: + """As above, but with a supplier item code in the description. + + The main fixture's descriptions carry no code, so item_no is legitimately + empty there and cannot show whether the code survives the import. + """ + page = SimpleNamespace( + markdown=( + "Customer: | Morris Sheetmetal |\n" + "Date: | 2026-05-22 |\n\n" + "# Aluminium Sheet\n\n" + "| Description | Price |\n" + "| --- | --- |\n" + "| UA1130 1.2mm x 1200 x 2400 5005 Sheet | $71.07 |\n\n" + "**WM Aluminium Ltd**" + ), + text="", + ) + return SimpleNamespace(pages=[page]) + @patch("apps.quoting.services.providers.mistral_provider.Mistral") def test_price_parsing_with_mocked_ocr_response(self, mock_mistral_class): """Catches OCR parser drift without making a live Mistral API call.""" @@ -71,21 +92,55 @@ def test_price_parsing_with_mocked_ocr_response(self, mock_mistral_class): first_item, { "description": "1.2mm x 1200 x 2400 5005 Sheet", - "supplier_item_code": "", + "item_no": "", "variant_id": "1.2mm_x_1200_x_2400_5005_Sheet", "unit_price": 71.07, + "price_unit": "each", "category": "Aluminium Sheet", "specifications": "1.2mm x 1200 x 2400 5005 Sheet", - "dimensions": { - "width": "1200", - "length": "2400", - "thickness": "1.2mm", - "diameter": None, - }, + "dimensions": "1.2mm x 1200 x 2400", "product_name": ("Aluminium Sheet - 1.2mm x 1200 x 2400 5005 Sheet"), }, ) + @patch("apps.quoting.services.providers.mistral_provider.Mistral") + def test_extracted_items_survive_the_import_sanitiser( + self, mock_mistral_class: Mock + ) -> None: + """The import sanitiser must keep the fields Mistral extracts. + + The provider names its fields for + PDFDataValidationService._sanitize_single_product, and a mismatch is + silent — the field is simply absent from the sanitised product. So this + asserts across that boundary, on the last hop before import, rather + than on the provider's own dict, which would agree with itself after a + rename. + """ + mock_client = Mock() + mock_client.ocr.process.return_value = self._ocr_response_with_item_code() + mock_mistral_class.return_value = mock_client + provider = MistralPriceExtractionProvider(api_key="dummy_key_for_testing") + + with ( + patch( + "apps.quoting.services.providers.mistral_provider.os.path.exists", + return_value=True, + ), + patch( + "apps.quoting.services.providers.mistral_provider.encode_pdf", + return_value="mock_base64", + ), + ): + result, error = provider.extract_price_data("mock_file_path.pdf") + + self.assertIsNone(error) + assert result is not None + + sanitised = PDFDataValidationService().sanitize_product_data(result["items"]) + + self.assertEqual(sanitised[0]["item_no"], "UA1130") + self.assertEqual(sanitised[0]["dimensions"], "1.2mm x 1200 x 2400") + if __name__ == "__main__": unittest.main() diff --git a/apps/quoting/tests/test_pdf_data_validation.py b/apps/quoting/tests/test_pdf_data_validation.py index d650238a8..013b2e028 100644 --- a/apps/quoting/tests/test_pdf_data_validation.py +++ b/apps/quoting/tests/test_pdf_data_validation.py @@ -53,7 +53,7 @@ def test_validate_extracted_data_missing_supplier(self): ], } - is_valid, errors, warnings = self.service.validate_extracted_data(data) + is_valid, errors, _warnings = self.service.validate_extracted_data(data) self.assertFalse(is_valid) self.assertIn("Missing supplier name", errors) @@ -62,7 +62,7 @@ def test_validate_extracted_data_no_items(self): """Test validation with no items.""" data = {"supplier": {"name": "Test Supplier"}, "items": []} - is_valid, errors, warnings = self.service.validate_extracted_data(data) + is_valid, _errors, warnings = self.service.validate_extracted_data(data) self.assertTrue(is_valid) # No items is valid, just a warning self.assertIn("No items found in extracted data", warnings) @@ -79,7 +79,7 @@ def test_validate_extracted_data_invalid_items(self): ], } - is_valid, errors, warnings = self.service.validate_extracted_data(data) + is_valid, errors, _warnings = self.service.validate_extracted_data(data) self.assertFalse(is_valid) self.assertIn("No valid items found", errors) diff --git a/apps/quoting/views.py b/apps/quoting/views.py index 04be86738..8b97cf95f 100644 --- a/apps/quoting/views.py +++ b/apps/quoting/views.py @@ -16,7 +16,6 @@ from apps.quoting.services.ai_price_extraction import extract_price_data from apps.quoting.services.pdf_data_validation import PDFDataValidationService from apps.quoting.services.pdf_import_service import PDFImportService -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger(__name__) @@ -271,19 +270,6 @@ def extract_supplier_price_list_data_view(request): logger.info(f"Processing completed successfully: {import_stats}") return JsonResponse(results) - except AlreadyLoggedException as exc: - logger.exception( - "Error in extract_supplier_price_list_data_view: %s", exc.original - ) - return JsonResponse( - { - "success": False, - "error": f"An unexpected error occurred: {str(exc.original)}", - "stage": "processing", - "error_id": exc.app_error_id, - }, - status=500, - ) except Exception as exc: app_error = persist_app_error(exc) logger.exception("Error in extract_supplier_price_list_data_view: %s", exc) diff --git a/apps/timesheet/management/commands/create_leave_entries.py b/apps/timesheet/management/commands/create_leave_entries.py index 29eacd878..fd9e278d6 100644 --- a/apps/timesheet/management/commands/create_leave_entries.py +++ b/apps/timesheet/management/commands/create_leave_entries.py @@ -16,6 +16,7 @@ from apps.accounts.models import Staff from apps.job.models import CostLine, CostSet, Job +from apps.workflow.services.error_persistence import persist_app_error # --- Entry batches --- # IMPORTANT: NEVER edit or remove existing batches. Only APPEND new ones. @@ -123,10 +124,11 @@ def handle(self, *args, **options): for key, job_name in LEAVE_JOB_NAMES.items(): try: job = Job.objects.get(name=job_name, status="special") - except Job.DoesNotExist: + except Job.DoesNotExist as exc: + persist_app_error(exc) raise CommandError( f"Leave job '{job_name}' not found with status='special'" - ) + ) from exc if not job.default_xero_pay_item: raise CommandError( f"Leave job '{job_name}' has no default_xero_pay_item set" @@ -134,8 +136,11 @@ def handle(self, *args, **options): leave_jobs[key] = job try: leave_cost_sets[key] = CostSet.objects.get(job_id=job.id, kind="actual") - except CostSet.DoesNotExist: - raise CommandError(f"No 'actual' CostSet found for job '{job_name}'") + except CostSet.DoesNotExist as exc: + persist_app_error(exc) + raise CommandError( + f"No 'actual' CostSet found for job '{job_name}'" + ) from exc if not ENTRIES: self.stdout.write("No entries to create. Edit ENTRIES in the command file.") diff --git a/apps/timesheet/management/commands/create_overtime_entries.py b/apps/timesheet/management/commands/create_overtime_entries.py index 1ebecf0e3..2c9d26f79 100644 --- a/apps/timesheet/management/commands/create_overtime_entries.py +++ b/apps/timesheet/management/commands/create_overtime_entries.py @@ -36,6 +36,7 @@ get_xero_hours_by_staff_week, ) from apps.workflow.models import XeroPayItem +from apps.workflow.services.error_persistence import persist_app_error DESC_PREFIX = "Retrospectively added" @@ -309,19 +310,21 @@ def _do_apply(self, csv_path: str): if staff_id not in staff_cache: try: staff_cache[staff_id] = Staff.objects.get(id=staff_id) - except Staff.DoesNotExist: - raise CommandError(f"Row {i}: Staff not found: {staff_id}") + except Staff.DoesNotExist as exc: + persist_app_error(exc) + raise CommandError(f"Row {i}: Staff not found: {staff_id}") from exc if job_id not in cost_set_cache: try: cost_set_cache[job_id] = CostSet.objects.get( job_id=job_id, kind="actual" ) - except CostSet.DoesNotExist: + except CostSet.DoesNotExist as exc: + persist_app_error(exc) raise CommandError( f"Row {i}: No 'actual' CostSet for job {job_id} " f"({row.get('job_name', '?')})" - ) + ) from exc staff = staff_cache[staff_id] cost_set = cost_set_cache[job_id] @@ -505,5 +508,6 @@ def _parse_date(value: str) -> date: def _parse_decimal(value: str) -> Decimal: try: return Decimal(value.strip()) - except (InvalidOperation, ValueError): - raise CommandError(f"Invalid decimal value: {value}") + except (InvalidOperation, ValueError) as exc: + persist_app_error(exc) + raise CommandError(f"Invalid decimal value: {value}") from exc diff --git a/apps/timesheet/management/commands/reclassify_overtime_entries.py b/apps/timesheet/management/commands/reclassify_overtime_entries.py index 353f5a3b4..ca68e7731 100644 --- a/apps/timesheet/management/commands/reclassify_overtime_entries.py +++ b/apps/timesheet/management/commands/reclassify_overtime_entries.py @@ -29,6 +29,7 @@ get_xero_hours_by_staff_week, ) from apps.workflow.models import XeroPayItem +from apps.workflow.services.error_persistence import persist_app_error # Leave job names to exclude from candidate selection LEAVE_JOB_NAMES = { @@ -290,8 +291,11 @@ def _do_apply(self, csv_path: str): costline = CostLine.objects.select_related( "cost_set", "cost_set__job" ).get(id=costline_id) - except CostLine.DoesNotExist: - raise CommandError(f"Row {i}: CostLine not found: {costline_id}") + except CostLine.DoesNotExist as exc: + persist_app_error(exc) + raise CommandError( + f"Row {i}: CostLine not found: {costline_id}" + ) from exc if action == "split" and remaining_hours <= 0: raise CommandError( @@ -465,5 +469,6 @@ def _parse_date(value: str) -> date: def _parse_decimal(value: str) -> Decimal: try: return Decimal(value.strip()) - except (InvalidOperation, ValueError): - raise CommandError(f"Invalid decimal value: {value}") + except (InvalidOperation, ValueError) as exc: + persist_app_error(exc) + raise CommandError(f"Invalid decimal value: {value}") from exc diff --git a/apps/timesheet/services/payroll_employee_sync.py b/apps/timesheet/services/payroll_employee_sync.py index ae88d7988..b2aaea76b 100644 --- a/apps/timesheet/services/payroll_employee_sync.py +++ b/apps/timesheet/services/payroll_employee_sync.py @@ -19,6 +19,7 @@ setup_employee_tax, ) from apps.workflow.api.xero.payroll import ( + coerce_xero_date, create_payroll_employee, get_employee_salary_and_wages, get_employee_working_patterns, @@ -459,13 +460,16 @@ def import_staff_from_xero( xero_employees = get_employees() - # Filter to active employees (no end_date or end_date in future) + # Filter to active employees (no end_date or end_date in future). + # Xero hands back end_date as a datetime, so normalize before comparing. today = date.today() - active_employees = [ - emp - for emp in xero_employees - if not getattr(emp, "end_date", None) or emp.end_date > today - ] + active_employees = [] + for emp in xero_employees: + end = coerce_xero_date(getattr(emp, "end_date", None)) + if end and end <= today: + continue + else: + active_employees.append(emp) summary: Dict[str, Any] = { "total_xero_employees": len(xero_employees), diff --git a/apps/timesheet/tests/test_payroll_employee_sync_import.py b/apps/timesheet/tests/test_payroll_employee_sync_import.py new file mode 100644 index 000000000..44e8442c7 --- /dev/null +++ b/apps/timesheet/tests/test_payroll_employee_sync_import.py @@ -0,0 +1,296 @@ +"""Tests for importing Staff from Xero Payroll employees.""" + +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional +from unittest.mock import MagicMock, patch + +from apps.accounts.models import Staff +from apps.testing import BaseTestCase +from apps.timesheet.services.payroll_employee_sync import PayrollEmployeeSyncService +from apps.workflow.api.xero.payroll import ( + coerce_xero_date, + get_employee_working_patterns, +) +from apps.workflow.models import CompanyDefaults + +PAYROLL = "apps.workflow.api.xero.payroll" +SYNC = "apps.timesheet.services.payroll_employee_sync" + + +@dataclass +class FakeXeroEmployee: + """Stand-in for a Xero Payroll NZ employee. + + ``end_date`` is a ``datetime`` because that is what the Xero SDK deserializes + it to — the distinction this test exists to protect. + """ + + employee_id: str + first_name: str + last_name: str + email: str + end_date: Optional[datetime] + + +@dataclass +class FakeSalaryAndWage: + status: str = "Active" + rate_per_unit: Decimal = Decimal("32.50") + + +@dataclass +class FakeWorkingWeek: + monday: float = 8.0 + tuesday: float = 8.0 + wednesday: float = 8.0 + thursday: float = 8.0 + friday: float = 6.0 + saturday: float = 0.0 + sunday: float = 0.0 + + +@dataclass +class FakePatternSummary: + """Mirrors EmployeeWorkingPattern: identifiers and dates only, no hours.""" + + payee_working_pattern_id: str + effective_from: Optional[datetime] + + +@dataclass +class FakePatternsResponse: + """Mirrors EmployeeWorkingPatternsObject.""" + + payee_working_patterns: Optional[List[FakePatternSummary]] + + +@dataclass +class FakePatternWithWeeks: + """Mirrors EmployeeWorkingPatternWithWorkingWeeks.""" + + working_weeks: List[FakeWorkingWeek] = field(default_factory=list) + + +@dataclass +class FakePatternDetail: + """Mirrors EmployeeWorkingPatternWithWorkingWeeksObject.""" + + payee_working_pattern: FakePatternWithWeeks + + +def _patched_payroll_api(patterns_response: Any, detail_response: Any) -> MagicMock: + api = MagicMock() + api.get_employee_working_patterns.return_value = patterns_response + api.get_employee_working_pattern.return_value = detail_response + return api + + +class GetEmployeeWorkingPatternsTests(BaseTestCase): + """Xero returns pattern IDs from the list call and hours from a second call. + + Reading hours off the list response returned nothing and raised — the import + could never seed a new hire's weekly hours. + """ + + EMPLOYEE_ID = "11111111-1111-1111-1111-111111111111" + + def _call(self, patterns_response: Any, detail_response: Any = None) -> Any: + api = _patched_payroll_api(patterns_response, detail_response) + with ( + patch(f"{PAYROLL}.get_tenant_id", return_value="tenant-1"), + patch(f"{PAYROLL}.PayrollNzApi", return_value=api), + patch(f"{PAYROLL}.time.sleep"), + ): + return get_employee_working_patterns(self.EMPLOYEE_ID), api + + def test_returns_hours_from_the_detail_call(self) -> None: + summaries = [ + FakePatternSummary( + payee_working_pattern_id="pattern-1", + effective_from=datetime.now() - timedelta(days=10), + ) + ] + detail = FakePatternDetail( + payee_working_pattern=FakePatternWithWeeks( + working_weeks=[FakeWorkingWeek()] + ) + ) + + result, api = self._call(FakePatternsResponse(summaries), detail) + + self.assertEqual( + result, + [ + { + "monday": 8.0, + "tuesday": 8.0, + "wednesday": 8.0, + "thursday": 8.0, + "friday": 6.0, + "saturday": 0.0, + "sunday": 0.0, + } + ], + ) + api.get_employee_working_pattern.assert_called_once() + self.assertEqual( + api.get_employee_working_pattern.call_args.kwargs[ + "employee_working_pattern_id" + ], + "pattern-1", + ) + + def test_picks_the_latest_pattern_already_in_effect(self) -> None: + """effective_from arrives as a datetime; ordering must not trust the list.""" + now = datetime.now() + summaries = [ + FakePatternSummary("old", now - timedelta(days=400)), + FakePatternSummary("future", now + timedelta(days=30)), + FakePatternSummary("current", now - timedelta(days=5)), + ] + detail = FakePatternDetail( + payee_working_pattern=FakePatternWithWeeks( + working_weeks=[FakeWorkingWeek()] + ) + ) + + _, api = self._call(FakePatternsResponse(summaries), detail) + + self.assertEqual( + api.get_employee_working_pattern.call_args.kwargs[ + "employee_working_pattern_id" + ], + "current", + ) + + def test_alternating_multi_week_pattern_returns_empty(self) -> None: + """No single-week representation exists, so the caller seeds defaults.""" + summaries = [FakePatternSummary("p", datetime.now() - timedelta(days=1))] + detail = FakePatternDetail( + payee_working_pattern=FakePatternWithWeeks( + working_weeks=[FakeWorkingWeek(), FakeWorkingWeek(monday=4.0)] + ) + ) + + result, _ = self._call(FakePatternsResponse(summaries), detail) + + self.assertEqual(result, []) + + def test_no_patterns_returns_empty(self) -> None: + result, api = self._call(FakePatternsResponse(None)) + + self.assertEqual(result, []) + api.get_employee_working_pattern.assert_not_called() + + def test_only_future_dated_patterns_returns_empty(self) -> None: + summaries = [FakePatternSummary("f", datetime.now() + timedelta(days=7))] + + result, api = self._call(FakePatternsResponse(summaries)) + + self.assertEqual(result, []) + api.get_employee_working_pattern.assert_not_called() + + +class ImportStaffFromXeroActiveFilterTests(BaseTestCase): + """The active-employee filter must survive Xero's datetime end_dates.""" + + def setUp(self) -> None: + super().setUp() + company = CompanyDefaults.get_solo() + company.xero_payroll_calendar_name = "Weekly" + company.save(update_fields=["xero_payroll_calendar_name"]) + + now = datetime.now() + self.current_employee = FakeXeroEmployee( + employee_id="11111111-1111-1111-1111-111111111111", + first_name="Current", + last_name="Worker", + email="current@example.com", + end_date=None, + ) + self.leaving_employee = FakeXeroEmployee( + employee_id="22222222-2222-2222-2222-222222222222", + first_name="Leaving", + last_name="Worker", + email="leaving@example.com", + end_date=now + timedelta(days=30), + ) + self.departed_employee = FakeXeroEmployee( + employee_id="33333333-3333-3333-3333-333333333333", + first_name="Departed", + last_name="Worker", + email="departed@example.com", + end_date=now - timedelta(days=30), + ) + + def _import(self, dry_run: bool = True) -> Dict[str, Any]: + employees = [ + self.current_employee, + self.leaving_employee, + self.departed_employee, + ] + # A realistic pattern, not []: the empty stub previously let these tests + # pass over a function that could only raise. + hours = [ + { + "monday": 8.0, + "tuesday": 8.0, + "wednesday": 8.0, + "thursday": 8.0, + "friday": 6.0, + "saturday": 0.0, + "sunday": 0.0, + } + ] + with ( + patch(f"{SYNC}.get_employees", return_value=employees), + patch( + f"{SYNC}.get_employee_salary_and_wages", + return_value=[FakeSalaryAndWage()], + ), + patch(f"{SYNC}.get_employee_working_patterns", return_value=hours), + ): + return PayrollEmployeeSyncService.import_staff_from_xero( + dry_run=dry_run, + initial_password="import-test-password", + ) + + def test_datetime_end_date_does_not_break_the_filter(self) -> None: + """A terminated employee carries a datetime end_date; comparing it to + date.today() raised TypeError and made --import-staff unusable on any + tenant with past employees.""" + summary = self._import() + + self.assertEqual(summary["total_xero_employees"], 3) + self.assertEqual(summary["active_employees"], 2) + + def test_departed_employee_is_not_imported(self) -> None: + summary = self._import() + + imported = {item["employee"] for item in summary["created"]} + self.assertIn("Current Worker (current@example.com)", imported) + self.assertIn("Leaving Worker (leaving@example.com)", imported) + self.assertNotIn("Departed Worker (departed@example.com)", imported) + self.assertEqual(summary["errors"], []) + + def test_imported_staff_carry_the_xero_working_hours(self) -> None: + """The point of reading the pattern at all: the new hire's roster + expectation drives timesheet visibility and workshop capacity.""" + self._import(dry_run=False) + + staff = Staff.objects.get(email="current@example.com") + self.assertEqual(float(staff.hours_mon), 8.0) + self.assertEqual(float(staff.hours_fri), 6.0) + self.assertEqual(float(staff.hours_sat), 0.0) + self.assertEqual(staff.xero_user_id, self.current_employee.employee_id) + + +class CoerceXeroDateTests(BaseTestCase): + """The shared normalizer is the contract the pattern selection depends on.""" + + def test_datetime_becomes_date(self) -> None: + self.assertEqual( + coerce_xero_date(datetime(2026, 3, 4, 15, 30)), date(2026, 3, 4) + ) diff --git a/apps/timesheet/views/api.py b/apps/timesheet/views/api.py index 789836806..8c3dc75d0 100644 --- a/apps/timesheet/views/api.py +++ b/apps/timesheet/views/api.py @@ -50,7 +50,6 @@ validate_pay_items_for_week, ) from apps.workflow.api.xero.sync import sync_all_xero_data -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults, XeroPayRun from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.utils import build_xero_payroll_url @@ -83,18 +82,12 @@ def build_internal_error_response( """ Construct a consistent error response while ensuring exceptions are persisted once. """ - if isinstance(exc, AlreadyLoggedException): - root_exception = exc.original - error_id = exc.app_error_id - else: - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) - root_exception = exc + app_error = persist_app_error(exc) - logger.error(f"{message}: {root_exception}", exc_info=True) + logger.error(f"{message}: {exc}", exc_info=True) payload = {"error": message} - details_text = str(root_exception) + details_text = str(exc) if staff_only_details and request is not None: payload["details"] = ( details_text if request.user.is_office_staff else "Internal server error" @@ -102,8 +95,7 @@ def build_internal_error_response( else: payload["details"] = details_text - if error_id: - payload["error_id"] = str(error_id) + payload["error_id"] = str(app_error.id) return Response(payload, status=status_code) diff --git a/apps/workflow/__init__.py b/apps/workflow/__init__.py index a76b6b2c7..405847d8b 100644 --- a/apps/workflow/__init__.py +++ b/apps/workflow/__init__.py @@ -1,9 +1,8 @@ # This file is autogenerated by update_init.py script from .apps import WorkflowConfig, check_company_defaults_field_sections -from .enums import AIProviderTypes +from .enums import AIProviderTypes, NotebookLmRestriction from .exceptions import ( - AlreadyLoggedException, NoValidXeroTokenError, XeroQuotaFloorReached, XeroSyncAlreadyRunningError, @@ -47,6 +46,7 @@ GroupedAppErrorSerializer, GroupedErrorResolveRequestSerializer, GroupedErrorResolveResponseSerializer, + NotebookLmLinkSerializer, SessionReplayChunkCreateSerializer, SessionReplayChunkSerializer, SessionReplayEventsResponseSerializer, @@ -103,7 +103,6 @@ "AIProviderSerializer", "AIProviderTypes", "AccessLoggingMiddleware", - "AlreadyLoggedException", "AppErrorDetailResponseSerializer", "AppErrorListResponseSerializer", "AppErrorSerializer", @@ -120,6 +119,8 @@ "JWTAuthentication", "LoginRequiredMiddleware", "NoValidXeroTokenError", + "NotebookLmLinkSerializer", + "NotebookLmRestriction", "PasswordStrengthMiddleware", "SearchTelemetryClickRequestSerializer", "SearchTelemetryClickResponseSerializer", diff --git a/apps/workflow/accounting/provider.py b/apps/workflow/accounting/provider.py index d203975a5..0e12f4406 100644 --- a/apps/workflow/accounting/provider.py +++ b/apps/workflow/accounting/provider.py @@ -18,6 +18,7 @@ InvoicePayload, POPayload, QuotePayload, + QuotePdfDocument, ) logger = logging.getLogger(__name__) @@ -44,7 +45,7 @@ def get_valid_token(self) -> dict | None: """Return a valid token, refreshing if needed. Returns None only when not connected or missing stored refresh material. - Refresh attempt failures may raise AlreadyLoggedException. + Refresh attempt failures propagate to the caller. """ ... @@ -52,7 +53,7 @@ def refresh_token(self) -> dict | None: """Force-refresh the current token. Returns None only when no refresh can be attempted. Refresh attempt - failures may raise AlreadyLoggedException. + failures propagate to the caller. """ ... @@ -96,6 +97,10 @@ def delete_quote(self, external_id: str) -> DocumentResult: """Delete/void a quote in the accounting system.""" ... + def download_quote_pdf(self, external_id: str) -> QuotePdfDocument: + """Download the provider-rendered quote PDF for inspection.""" + ... + def create_purchase_order(self, payload: POPayload) -> DocumentResult: """Create a purchase order in the accounting system.""" ... diff --git a/apps/workflow/accounting/quote_pdf_service.py b/apps/workflow/accounting/quote_pdf_service.py new file mode 100644 index 000000000..9e4158a17 --- /dev/null +++ b/apps/workflow/accounting/quote_pdf_service.py @@ -0,0 +1,68 @@ +"""Inspection of provider-rendered quote PDFs.""" + +from __future__ import annotations + +from dataclasses import dataclass +from uuid import UUID + +from pypdf import PdfReader + +from apps.workflow.accounting.registry import get_provider +from apps.workflow.models import CompanyDefaults + + +@dataclass(frozen=True) +class QuotePdfInspection: + """Structured evidence from a provider-rendered quote PDF.""" + + quote_id: str + remote_branding_theme_id: str | None + configured_branding_theme_id: str | None + page_count: int + contains_expected_text: bool + + +def inspect_quote_pdf( + quote_id: UUID, + expected_text: str, +) -> QuotePdfInspection: + """Inspect the real provider PDF without exposing its customer text.""" + normalized_expected_text = " ".join(expected_text.split()) + if not normalized_expected_text: + raise ValueError("Expected quote PDF text must not be empty") + + provider = get_provider() + document = provider.download_quote_pdf(str(quote_id)) + reader = PdfReader(document.temporary_file_path) + page_text: list[str] = [] + for page in reader.pages: + extracted = page.extract_text() + # A blank or image-only page extracts to "" — keep only pages with real + # text so an all-blank PDF raises below rather than reporting the marker + # absent and deleting the diagnostic file. + if extracted is not None and extracted.strip(): + page_text.append(extracted) + + if not page_text: + raise ValueError(f"Quote {quote_id} PDF contains no extractable text") + + normalized_document_text = " ".join("\n".join(page_text).split()) + compact_expected_text = "".join(normalized_expected_text.split()) + compact_document_text = "".join(normalized_document_text.split()) + configured_theme_id = CompanyDefaults.get_solo().xero_sales_branding_theme_id + inspection = QuotePdfInspection( + quote_id=document.external_id, + remote_branding_theme_id=document.document_theme_external_id, + configured_branding_theme_id=( + str(configured_theme_id) if configured_theme_id is not None else None + ), + page_count=len(reader.pages), + contains_expected_text=( + normalized_expected_text in normalized_document_text + or compact_expected_text in compact_document_text + ), + ) + # Only on the success path: a failure above leaves the file for inspection, + # and losing a temp file matters less than losing the error that caused it. + document.temporary_file_path.unlink(missing_ok=True) + return inspection diff --git a/apps/workflow/accounting/types.py b/apps/workflow/accounting/types.py index d62f05063..de79e1b95 100644 --- a/apps/workflow/accounting/types.py +++ b/apps/workflow/accounting/types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from decimal import Decimal +from pathlib import Path @dataclass(frozen=True) @@ -15,6 +16,18 @@ class DocumentTheme: is_default: bool +@dataclass(frozen=True) +class QuotePdfDocument: + """A provider-rendered quote PDF and its presentation metadata. + + ``temporary_file_path`` is owned by the caller and must be removed after use. + """ + + external_id: str + document_theme_external_id: str | None + temporary_file_path: Path + + @dataclass class DocumentLineItem: """A single line item on an invoice, quote, or purchase order.""" @@ -53,6 +66,7 @@ class QuotePayload: date: date expiry_date: date document_theme_external_id: str + terms: str currency_code: str = "NZD" reference: str | None = None status: str = "DRAFT" diff --git a/apps/workflow/accounting/xero/provider.py b/apps/workflow/accounting/xero/provider.py index d36ff60d0..316a1d9e2 100644 --- a/apps/workflow/accounting/xero/provider.py +++ b/apps/workflow/accounting/xero/provider.py @@ -4,12 +4,12 @@ import logging from datetime import datetime +from pathlib import Path from typing import TYPE_CHECKING from uuid import UUID from apps.workflow.accounting.registry import register_provider from apps.workflow.api.xero.transforms import process_xero_data -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error if TYPE_CHECKING: @@ -24,6 +24,7 @@ InvoicePayload, POPayload, QuotePayload, + QuotePdfDocument, ) logger = logging.getLogger("xero") @@ -207,11 +208,9 @@ def list_document_themes(self) -> list[DocumentTheme]: ) return [theme for _sort_order, theme in sorted(ranked_themes)] - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def create_invoice(self, payload: InvoicePayload) -> DocumentResult: from xero_python.accounting.models import Contact, Invoice @@ -306,6 +305,7 @@ def create_quote(self, payload: QuotePayload) -> DocumentResult: status=payload.status, reference=payload.reference, branding_theme_id=payload.document_theme_external_id, + terms=payload.terms, ) response = api.create_quotes( @@ -355,6 +355,55 @@ def delete_quote(self, external_id: str) -> DocumentResult: persist_app_error(exc) return self._make_error_result(exc) + def download_quote_pdf(self, external_id: str) -> QuotePdfDocument: + """Download Xero's native quote PDF and report the applied theme.""" + from apps.workflow.accounting.types import QuotePdfDocument + + try: + quote_id = str(UUID(external_id)) + api, tenant_id = self._get_api() + response = api.get_quote(tenant_id, quote_id) + if len(response.quotes) != 1: + raise ValueError( + f"Xero returned {len(response.quotes)} quotes for {quote_id}" + ) + + quote = response.quotes[0] + if not isinstance(quote.quote_id, str): + raise ValueError(f"Xero quote {quote_id} is missing its identifier") + returned_quote_id = str(UUID(quote.quote_id)) + if returned_quote_id != quote_id: + raise ValueError( + f"Xero returned quote {returned_quote_id} for requested {quote_id}" + ) + + if quote.branding_theme_id is None: + document_theme_external_id = None + elif isinstance(quote.branding_theme_id, str): + document_theme_external_id = str(UUID(quote.branding_theme_id)) + else: + raise ValueError( + f"Xero quote {quote_id} has an invalid branding theme identifier" + ) + + downloaded_path = api.get_quote_as_pdf(tenant_id, quote_id) + if not isinstance(downloaded_path, str): + raise TypeError( + f"Xero quote PDF download returned {type(downloaded_path).__name__}" + ) + temporary_file_path = Path(downloaded_path) + if not temporary_file_path.is_file(): + raise FileNotFoundError(temporary_file_path) + + return QuotePdfDocument( + external_id=returned_quote_id, + document_theme_external_id=document_theme_external_id, + temporary_file_path=temporary_file_path, + ) + except Exception as exc: + persist_app_error(exc) + raise + def _create_or_update_purchase_order(self, payload: POPayload) -> DocumentResult: """Shared implementation for PO create and update.""" from xero_python.accounting.models import Contact, PurchaseOrder diff --git a/apps/workflow/accounting/xero/readonly_provider.py b/apps/workflow/accounting/xero/readonly_provider.py index 5d6db4cd4..82485f537 100644 --- a/apps/workflow/accounting/xero/readonly_provider.py +++ b/apps/workflow/accounting/xero/readonly_provider.py @@ -29,6 +29,7 @@ InvoicePayload, POPayload, QuotePayload, + QuotePdfDocument, ) logger = logging.getLogger("xero") @@ -154,6 +155,11 @@ def delete_quote(self, external_id: str) -> DocumentResult: _log_suppressed("delete_quote", external_id) return DocumentResult(success=True, external_id=external_id) + def download_quote_pdf(self, external_id: str) -> QuotePdfDocument: + raise RuntimeError( + "XERO_READONLY: a native Xero quote PDF cannot be downloaded" + ) + def _create_or_update_purchase_order(self, payload: POPayload) -> DocumentResult: raise RuntimeError( "XERO_READONLY: real Xero PO helper reached — a write override is missing" diff --git a/apps/workflow/api/__init__.py b/apps/workflow/api/__init__.py index a66c07cdb..8116d61c9 100644 --- a/apps/workflow/api/__init__.py +++ b/apps/workflow/api/__init__.py @@ -1,7 +1,5 @@ # This file is autogenerated by update_init.py script -from .enums import get_enum_choices - # Conditional imports (only when Django is ready) try: from django.apps import apps @@ -15,5 +13,4 @@ __all__ = [ "FiftyPerPagePagination", "PageSizePagination", - "get_enum_choices", ] diff --git a/apps/workflow/api/enums.py b/apps/workflow/api/enums.py deleted file mode 100644 index 3941e5048..000000000 --- a/apps/workflow/api/enums.py +++ /dev/null @@ -1,93 +0,0 @@ -import importlib -import inspect -import logging - -from django.http import HttpRequest, JsonResponse -from django.views.decorators.http import require_http_methods - -logger = logging.getLogger(__name__) - - -@require_http_methods(["GET"]) -def get_enum_choices(request: HttpRequest, enum_name: str) -> JsonResponse: - """ - API endpoint to get enum choices. - Returns the choices for the specified enum as a JSON object. - - Args: - request: The HTTP request - enum_name: The name of the enum to get choices for (e.g., 'MetalType') - - Returns: - JsonResponse with the enum choices - """ - if not request.user.is_authenticated: - return JsonResponse({"error": "Authentication required"}, status=401) - - try: - # First check the job enums module - try: - enums_module = importlib.import_module("apps.job.enums") - - if hasattr(enums_module, enum_name): - enum_class = getattr(enums_module, enum_name) - - if hasattr(enum_class, "choices"): - choices = [ - {"value": value, "display_name": display_name} - for value, display_name in enum_class.choices - ] - return JsonResponse({"choices": choices}) - except (ImportError, AttributeError): - # Fall back to workflow enums - pass - - # Fall back to workflow.enums if not found in job.enums - enums_module = importlib.import_module("apps.workflow.enums") - - if hasattr(enums_module, enum_name): - enum_class = getattr(enums_module, enum_name) - - if hasattr(enum_class, "choices"): - choices = [ - {"value": value, "display_name": display_name} - for value, display_name in enum_class.choices - ] - return JsonResponse({"choices": choices}) - else: - return JsonResponse( - { - "error": f'"{enum_name}" does not appear to be a valid Django Choices enum' - }, - status=400, - ) - else: - # List available enums from both modules - job_enums_module = importlib.import_module("job.enums") - workflow_enums_module = importlib.import_module("workflow.enums") - - job_enums = [ - name - for name, obj in inspect.getmembers(job_enums_module) - if inspect.isclass(obj) and hasattr(obj, "choices") - ] - - workflow_enums = [ - name - for name, obj in inspect.getmembers(workflow_enums_module) - if inspect.isclass(obj) and hasattr(obj, "choices") - ] - - return JsonResponse( - { - "error": f'Enum "{enum_name}" not found', - "available_enums": {"job": job_enums, "workflow": workflow_enums}, - }, - status=404, - ) - - except Exception as e: - logger.exception(f"Unexpected error getting enum choices: {e}") - return JsonResponse( - {"error": "An unexpected server error occurred."}, status=500 - ) diff --git a/apps/workflow/api/reports/payroll_reconciliation.py b/apps/workflow/api/reports/payroll_reconciliation.py index 3b549da4c..0ad5dc834 100644 --- a/apps/workflow/api/reports/payroll_reconciliation.py +++ b/apps/workflow/api/reports/payroll_reconciliation.py @@ -16,8 +16,7 @@ from apps.accounting.services.payroll_reconciliation_service import ( PayrollReconciliationService, ) -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.services.error_persistence import persist_app_error +from apps.workflow.services.error_persistence import app_error_for, persist_app_error logger = logging.getLogger(__name__) @@ -102,17 +101,18 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(response_serializer.data, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: - logger.error("Payroll reconciliation error: %s", exc.original) - return _build_error_response( - message=( - "Internal server error occurred while generating " - "payroll reconciliation report" - ), - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error("Payroll reconciliation error: %s", exc) + if app_error_for(exc) is not None: + # Persisted upstream — the response omits error_id, as it + # always has for failures logged deeper in the stack. + return _build_error_response( + message=( + "Internal server error occurred while generating " + "payroll reconciliation report" + ), + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) app_error = persist_app_error( exc, additional_context={"operation": "payroll_reconciliation"}, diff --git a/apps/workflow/api/xero/__init__.py b/apps/workflow/api/xero/__init__.py index 4b1935ef1..b2f17c197 100644 --- a/apps/workflow/api/xero/__init__.py +++ b/apps/workflow/api/xero/__init__.py @@ -25,6 +25,7 @@ from .client import RateLimitedRESTClient, quota_floor_breached from .payroll import ( DraftPayRunBlocksLeaveDeletion, + coerce_xero_date, create_employee_leave, create_pay_run, create_payroll_employee, @@ -133,7 +134,6 @@ create_expense_entries, create_project, create_time_entries, - get_projects, get_xero_items, update_expense_entries, update_project, @@ -152,6 +152,7 @@ "bind_token_callbacks", "bulk_create_contacts_in_xero", "clean_json", + "coerce_xero_date", "create_company_contact_in_xero", "create_default_task", "create_employee_leave", @@ -190,7 +191,6 @@ "get_pay_slips_for_sync", "get_payroll_calendar_id", "get_payroll_calendars", - "get_projects", "get_sync_cursor", "get_tenant_id", "get_tenant_id_from_connections", diff --git a/apps/workflow/api/xero/active_app.py b/apps/workflow/api/xero/active_app.py index d9215a1f3..1ffa2f819 100644 --- a/apps/workflow/api/xero/active_app.py +++ b/apps/workflow/api/xero/active_app.py @@ -21,7 +21,6 @@ from django.db import transaction from apps.workflow.api.xero.constants import TENANT_ID_CACHE_KEY -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import XeroApp from apps.workflow.services.error_persistence import persist_app_error @@ -102,11 +101,9 @@ def _restart_sibling_workers() -> None: stderr=subprocess.DEVNULL, ) logger.info(f"Dispatched detached restart for: {', '.join(units)}") - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def wipe_tokens_and_quota(app: XeroApp) -> None: diff --git a/apps/workflow/api/xero/auth.py b/apps/workflow/api/xero/auth.py index 0d7e660db..e265d669c 100644 --- a/apps/workflow/api/xero/auth.py +++ b/apps/workflow/api/xero/auth.py @@ -26,7 +26,6 @@ from apps.workflow.api.xero.client import RateLimitedRESTClient from apps.workflow.api.xero.constants import TENANT_ID_CACHE_KEY, XERO_SCOPES -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults, XeroApp from apps.workflow.services.error_persistence import persist_app_error @@ -173,11 +172,9 @@ def _save(token: Dict[str, Any]) -> None: logger.info( f"Stored token on XeroApp {app_id}, expires {expires_at.isoformat()}" ) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise return _save @@ -214,11 +211,9 @@ def refresh_token() -> Optional[Dict[str, Any]]: # explicitly via set_oauth2_token, which calls the saver. api_client.set_oauth2_token(refreshed) return refreshed - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def _payload_needs_refresh(payload: Dict[str, Any]) -> bool: @@ -234,7 +229,7 @@ def get_valid_token() -> Optional[Dict[str, Any]]: ``None`` means the installation is not connected, or the active row lacks the stored token material required to refresh. Refresh attempts that reach - Xero and fail propagate as ``AlreadyLoggedException`` so callers do not + Xero and fail propagate to the caller so callers do not mistake operational failures for an unconnected install. """ try: @@ -282,8 +277,8 @@ def get_authentication_url(state: str) -> str: try: app = XeroApp.objects.get(is_active=True) except XeroApp.DoesNotExist as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise params = { "response_type": "code", @@ -330,8 +325,8 @@ def exchange_code_for_token( try: app = XeroApp.objects.get(is_active=True) except XeroApp.DoesNotExist as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise logger.debug( f"Exchanging code for token. Code: {code}, State: {state}, " @@ -356,11 +351,9 @@ def exchange_code_for_token( # Write to the same row that issued the request. _make_token_saver(app.id)(token) return token - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def get_tenant_id() -> str: @@ -387,8 +380,8 @@ def get_tenant_id() -> str: "No Xero tenant ID configured in company defaults. " "Please set this up first." ) - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise exc tenant_id = company_defaults.xero_tenant_id cache.set(TENANT_ID_CACHE_KEY, tenant_id) diff --git a/apps/workflow/api/xero/payroll.py b/apps/workflow/api/xero/payroll.py index b49ec718b..0c6b3afbf 100644 --- a/apps/workflow/api/xero/payroll.py +++ b/apps/workflow/api/xero/payroll.py @@ -15,6 +15,7 @@ Employee, EmployeeLeaveSetup, EmployeeTax, + EmployeeWorkingPattern, EmployeeWorkingPatternWithWorkingWeeksRequest, Employment, PaymentMethod, @@ -28,12 +29,8 @@ from apps.workflow.api.xero.auth import api_client, get_tenant_id from apps.workflow.api.xero.transforms import transform_pay_run -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults, XeroPayItem, XeroPayRun -from apps.workflow.services.error_persistence import ( - persist_and_raise, - persist_app_error, -) +from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger("xero.payroll") @@ -130,7 +127,8 @@ def get_employees() -> List[Employee]: return employees except Exception as exc: logger.error(f"Failed to get Xero Payroll employees: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise def create_payroll_employee(employee_data: Dict[str, Any]) -> Employee: @@ -297,13 +295,14 @@ def create_payroll_employee(employee_data: Dict[str, Any]) -> Employee: exc, exc_info=True, ) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "create_payroll_employee", "email": employee_data.get("email"), }, ) + raise def update_employee_name(employee_id: str, first_name: str, last_name: str) -> None: @@ -615,19 +614,25 @@ def get_employee_salary_and_wages(employee_id: str) -> List[SalaryAndWage]: exc, exc_info=True, ) - persist_and_raise(exc) + persist_app_error(exc) + raise def get_employee_working_patterns(employee_id: str) -> List[Dict[str, float]]: """ - Get working pattern (weekly hours per day) for a Xero Payroll employee. + Get the employee's current working pattern (weekly hours per day). + + Xero splits this across two endpoints: the list call returns pattern + identifiers and their effective dates only, and the weekly hours live behind + a per-pattern fetch. Returns: - List of dicts with keys: monday, tuesday, ..., sunday (float hours). - Typically one active pattern per employee. + A single-element list of dicts keyed monday..sunday (float hours), or an + empty list when Xero holds no usable pattern. Empty is a legitimate + answer: the caller seeds CompanyDefaults hours instead. Raises: - Exception: If API call fails + Exception: If an API call fails """ tenant_id = get_tenant_id() if not tenant_id: @@ -643,29 +648,75 @@ def get_employee_working_patterns(employee_id: str) -> List[Dict[str, float]]: ) time.sleep(SLEEP_TIME) - patterns = [] - if response and response.working_patterns: - for pattern in response.working_patterns: - if not pattern.working_weeks: - continue - # Use the first (typically only) working week - week = pattern.working_weeks[0] - patterns.append( - { - "monday": float(week.monday or 0), - "tuesday": float(week.tuesday or 0), - "wednesday": float(week.wednesday or 0), - "thursday": float(week.thursday or 0), - "friday": float(week.friday or 0), - "saturday": float(week.saturday or 0), - "sunday": float(week.sunday or 0), - } - ) + summaries = response.payee_working_patterns if response else None + if not summaries: + logger.info("Employee %s has no working pattern in Xero", employee_id) + return [] + + # Patterns are effective-dated history; the current one is the latest + # that has already taken effect. Do not trust list ordering. + today = date.today() + effective: List[Tuple[date, EmployeeWorkingPattern]] = [] + for summary in summaries: + effective_from = coerce_xero_date(summary.effective_from) + if not effective_from or effective_from > today: + continue + else: + effective.append((effective_from, summary)) + if not effective: + logger.info( + "Employee %s has %d working pattern(s), none yet effective", + employee_id, + len(summaries), + ) + return [] + + current = max(effective, key=lambda pair: pair[0])[1] + + detail = payroll_api.get_employee_working_pattern( + xero_tenant_id=tenant_id, + employee_id=employee_id, + employee_working_pattern_id=current.payee_working_pattern_id, + ) + time.sleep(SLEEP_TIME) + + weeks = detail.payee_working_pattern.working_weeks + if not weeks: + logger.info( + "Working pattern %s for employee %s has no working weeks", + current.payee_working_pattern_id, + employee_id, + ) + return [] + if len(weeks) > 1: + # An alternating roster has no single-week representation in Staff's + # hours_mon..hours_sun. Seed defaults and let an admin correct it. + logger.warning( + "Employee %s has an alternating %d-week pattern; cannot represent " + "as weekly hours, falling back to company defaults", + employee_id, + len(weeks), + ) + return [] + + week = weeks[0] + patterns = [ + { + "monday": float(week.monday or 0), + "tuesday": float(week.tuesday or 0), + "wednesday": float(week.wednesday or 0), + "thursday": float(week.thursday or 0), + "friday": float(week.friday or 0), + "saturday": float(week.saturday or 0), + "sunday": float(week.sunday or 0), + } + ] logger.info( - "Retrieved %d working patterns for employee %s", - len(patterns), + "Retrieved working pattern %s for employee %s (%.1f hrs/week)", + current.payee_working_pattern_id, employee_id, + sum(patterns[0].values()), ) return patterns except Exception as exc: @@ -675,7 +726,8 @@ def get_employee_working_patterns(employee_id: str) -> List[Dict[str, float]]: exc, exc_info=True, ) - persist_and_raise(exc) + persist_app_error(exc) + raise def get_payroll_calendars() -> List[Dict[str, Any]]: @@ -716,7 +768,8 @@ def get_payroll_calendars() -> List[Dict[str, Any]]: return calendars except Exception as exc: logger.error(f"Failed to get Xero Payroll calendars: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise def get_pay_runs() -> List[Dict[str, Any]]: @@ -762,7 +815,8 @@ def get_pay_runs() -> List[Dict[str, Any]]: return pay_runs except Exception as exc: logger.error(f"Failed to get Xero Payroll pay runs: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise def get_pay_run(pay_run_id: str): @@ -795,16 +849,17 @@ def get_pay_run(pay_run_id: str): return None except Exception as exc: logger.error(f"Failed to get Xero pay run {pay_run_id}: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise def _pay_run_payload_from_object(pay_run: Any, *, status: str) -> PayRun: return PayRun( pay_run_id=str(pay_run.pay_run_id), payroll_calendar_id=str(pay_run.payroll_calendar_id), - period_start_date=_coerce_xero_date(pay_run.period_start_date), - period_end_date=_coerce_xero_date(pay_run.period_end_date), - payment_date=_coerce_xero_date(pay_run.payment_date), + period_start_date=coerce_xero_date(pay_run.period_start_date), + period_end_date=coerce_xero_date(pay_run.period_end_date), + payment_date=coerce_xero_date(pay_run.payment_date), pay_run_status=status, pay_run_type=getattr(pay_run, "pay_run_type", None), ) @@ -842,7 +897,7 @@ def _update_pay_run(pay_run_id: str, pay_run: PayRun) -> Any: ) except Exception as exc: logger.error("Failed to update Xero pay run %s: %s", pay_run_id, exc) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "update_pay_run", @@ -850,6 +905,7 @@ def _update_pay_run(pay_run_id: str, pay_run: PayRun) -> Any: "pay_run_status": pay_run.pay_run_status, }, ) + raise def _find_same_week_draft_pay_run(week_start_date: date) -> Any | None: @@ -878,8 +934,8 @@ def _find_same_week_draft_pay_run(week_start_date: date) -> Any | None: response = payroll_api.get_pay_runs(xero_tenant_id=tenant_id, status="Draft") for pay_run in getattr(response, "pay_runs", []) or []: if ( - _coerce_xero_date(pay_run.period_start_date) == week_start_date - and _coerce_xero_date(pay_run.period_end_date) == week_end_date + coerce_xero_date(pay_run.period_start_date) == week_start_date + and coerce_xero_date(pay_run.period_end_date) == week_end_date and getattr(pay_run, "pay_run_status", None) == "Draft" ): return pay_run @@ -1087,7 +1143,8 @@ def get_leave_types() -> List[Dict[str, Any]]: return leave_types except Exception as exc: logger.error(f"Failed to get Xero Payroll leave types: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise def get_earnings_rates() -> List[Dict[str, Any]]: @@ -1142,7 +1199,8 @@ def get_earnings_rates() -> List[Dict[str, Any]]: return earnings_rates except Exception as exc: logger.error(f"Failed to get Xero Payroll earnings rates: {exc}", exc_info=True) - persist_and_raise(exc) + persist_app_error(exc) + raise # Cache for earnings rate lookups (populated once per session) @@ -1239,7 +1297,7 @@ def get_leave_type_id_by_name(name: str) -> str: return _leave_type_cache[name] -def _coerce_xero_date(value: Any) -> Optional[date]: +def coerce_xero_date(value: Any) -> Optional[date]: """Normalize Xero date or datetime payloads (strings, datetimes, dates) into date objects.""" if value is None: return None @@ -1337,8 +1395,8 @@ def create_pay_run( if not response or not response.pay_run: raise Exception("Failed to create pay run") - actual_start_date = _coerce_xero_date(response.pay_run.period_start_date) - actual_end_date = _coerce_xero_date(response.pay_run.period_end_date) + actual_start_date = coerce_xero_date(response.pay_run.period_start_date) + actual_end_date = coerce_xero_date(response.pay_run.period_end_date) if actual_start_date != week_start_date or actual_end_date != week_end_date: # Xero creates the calendar's next unprocessed period regardless of # the dates we asked for. The pay run now exists in Xero, so mirror @@ -1369,13 +1427,14 @@ def create_pay_run( logger.error( f"Failed to create pay run for week {week_start_date}: {exc}", exc_info=True ) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "create_pay_run", "week": str(week_start_date), }, ) + raise def get_payroll_calendar_id() -> str: @@ -1641,7 +1700,7 @@ def post_timesheet( f"Failed to post timesheet for employee {employee_id}: {exc}", exc_info=True, ) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "post_timesheet", @@ -1649,6 +1708,7 @@ def post_timesheet( "week_start_date": week_start_date.isoformat(), }, ) + raise def create_employee_leave( @@ -1744,7 +1804,7 @@ def create_employee_leave( f"Failed to create leave for employee {employee_id}: {exc}", exc_info=True, ) - persist_and_raise( + persist_app_error( exc, additional_context={ "operation": "create_employee_leave", @@ -1752,6 +1812,7 @@ def create_employee_leave( "leave_type_id": leave_type_id, }, ) + raise # ============================================================================= @@ -1943,7 +2004,7 @@ def post_staff_week_to_xero( Raises: ValueError: If inputs are invalid - AlreadyLoggedException: If Xero API call fails + Exception: If Xero API call fails """ from apps.accounts.models import Staff from apps.job.models.costing import CostLine @@ -2079,20 +2140,18 @@ def post_staff_week_to_xero( "errors": [], } - except AlreadyLoggedException: - raise except Exception as exc: logger.error( f"Failed to post timesheet for staff {staff_id}: {exc}", exc_info=True ) - app_error = persist_app_error( + persist_app_error( exc, additional_context={ "staff_id": str(staff_id), "week_start_date": week_start_date.isoformat(), }, ) - raise AlreadyLoggedException(exc, app_error.id) + raise def _categorize_entries(entries: List) -> tuple: @@ -2211,8 +2270,8 @@ def _delete_existing_leave_for_week( deleted_count = 0 for leave in response.leave: - leave_start_date = _coerce_xero_date(leave.start_date) - leave_end_date = _coerce_xero_date(leave.end_date) + leave_start_date = coerce_xero_date(leave.start_date) + leave_end_date = coerce_xero_date(leave.end_date) if leave_start_date is None or leave_end_date is None: raise ValueError( f"Xero leave {leave.leave_id} has invalid date range: " @@ -2367,8 +2426,8 @@ def reconcile_leave_for_staff_week( kept_leave_ids = [] for leave in existing_leaves: - leave_start_date = _coerce_xero_date(leave.start_date) - leave_end_date = _coerce_xero_date(leave.end_date) + leave_start_date = coerce_xero_date(leave.start_date) + leave_end_date = coerce_xero_date(leave.end_date) if leave_start_date is None or leave_end_date is None: raise ValueError( f"Xero leave {leave.leave_id} has invalid date range: " @@ -2633,7 +2692,7 @@ def sync_xero_pay_items() -> Dict[str, Any]: else: leave_multiplier = Decimal("1.00") - pay_item, created = XeroPayItem.objects.update_or_create( + _pay_item, created = XeroPayItem.objects.update_or_create( name=lt["name"], uses_leave_api=True, defaults={ @@ -2656,7 +2715,7 @@ def sync_xero_pay_items() -> Dict[str, Any]: if multiplier is not None: multiplier = Decimal(str(multiplier)) - pay_item, created = XeroPayItem.objects.update_or_create( + _pay_item, created = XeroPayItem.objects.update_or_create( name=rate["name"], uses_leave_api=False, defaults={ diff --git a/apps/workflow/api/xero/push.py b/apps/workflow/api/xero/push.py index 6f28a1a3f..5b1d5dbf1 100644 --- a/apps/workflow/api/xero/push.py +++ b/apps/workflow/api/xero/push.py @@ -259,11 +259,11 @@ def map_costline_to_time_entry(costline, task_id: str) -> TimeEntryCreateOrUpdat try: Staff.objects.get(id=staff_id) - except Staff.DoesNotExist: + except Staff.DoesNotExist as exc: error = ValueError( f"CostLine {costline.id} references non-existent staff {staff_id}" ) - raise error + raise error from exc # Convert hours to minutes (Xero uses minutes) if costline.quantity is None: diff --git a/apps/workflow/api/xero/reprocess_xero.py b/apps/workflow/api/xero/reprocess_xero.py index 935a58d5d..cfd6ce6c7 100644 --- a/apps/workflow/api/xero/reprocess_xero.py +++ b/apps/workflow/api/xero/reprocess_xero.py @@ -22,7 +22,6 @@ SupplierPickupAddress, ) from apps.crm.tasks import rematch_phone_calls_task -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import XeroAccount from apps.workflow.services.error_persistence import ( persist_app_error, @@ -78,15 +77,13 @@ def sync_xero_phone_methods(company: Company) -> list[str]: "source": ContactMethod.Source.IMPORTED, }, ) - except AlreadyLoggedException: - raise except ValidationError as exc: error = ValidationError( f"Xero phone sync for company '{company.name}' ({company.id}) " f"rejected number '{value}' ({normalized}): " f"{'; '.join(exc.messages)}" ) - app_error = persist_app_error( + persist_app_error( error, additional_context={ "operation": "sync_xero_phone_methods_duplicate_owner", @@ -94,7 +91,7 @@ def sync_xero_phone_methods(company: Company) -> list[str]: "normalized_number": normalized, }, ) - raise AlreadyLoggedException(error, app_error.id) from exc + raise error from exc if created: created_numbers.append(normalized) return created_numbers @@ -222,7 +219,7 @@ def set_invoice_or_bill_fields(document, document_type, new_from_xero=False): # Sync the line item using dynamic field name kwargs = {document_field: document, "xero_line_id": xero_line_id} - line_item, created = LineItemModel.objects.update_or_create( + _line_item, _created = LineItemModel.objects.update_or_create( **kwargs, defaults={ "quantity": quantity, diff --git a/apps/workflow/api/xero/sync.py b/apps/workflow/api/xero/sync.py index d614c8ad2..0e8bf3184 100644 --- a/apps/workflow/api/xero/sync.py +++ b/apps/workflow/api/xero/sync.py @@ -13,35 +13,16 @@ from apps.accounting.models import Bill, CreditNote, Invoice, Quote from apps.company.models import Company from apps.purchasing.models import PurchaseOrder, Stock +from apps.workflow.accounting.registry import is_accounting_enabled from apps.workflow.api.xero.auth import api_client, get_tenant_id, get_valid_token from apps.workflow.api.xero.client import quota_floor_breached from apps.workflow.api.xero.payroll import ( get_all_pay_slips_for_sync, get_pay_runs_for_sync, ) -from apps.workflow.api.xero.push import ( # noqa: F401 - bulk_create_contacts_in_xero, - create_company_contact_in_xero, - get_all_xero_contacts, - map_costline_to_expense_entry, - map_costline_to_time_entry, - sync_company_to_xero, - sync_costlines_to_xero, - sync_expense_entries_bulk, - sync_job_to_xero, - sync_time_entries_bulk, -) -from apps.workflow.api.xero.seed import ( # noqa: F401 - seed_companies_to_xero, - seed_jobs_to_xero, - sync_single_contact, - sync_single_invoice, - sync_single_pay_run, -) -from apps.workflow.api.xero.transforms import process_xero_data # noqa: F401 -from apps.workflow.api.xero.transforms import sync_companies # noqa: F401 from apps.workflow.api.xero.transforms import ( sync_accounts, + sync_companies, sync_entities, transform_bill, transform_credit_note, @@ -54,7 +35,6 @@ ) from apps.workflow.api.xero.xero import get_xero_items from apps.workflow.exceptions import ( - AlreadyLoggedException, NoValidXeroTokenError, XeroQuotaFloorReached, XeroValidationError, @@ -66,7 +46,9 @@ XeroPaySlip, XeroSyncCursor, ) +from apps.workflow.services.e2e_artifacts import drop_e2e_artifacts from apps.workflow.services.error_persistence import ( + app_error_for, persist_app_error, persist_xero_error, ) @@ -82,6 +64,7 @@ class XeroSyncEvent(TypedDict, total=False): entity: str severity: str message: str + status: str progress: float | None recordsUpdated: int @@ -142,15 +125,15 @@ def process_xero_item(item, sync_function, entity_type): "message": str(exc), "progress": None, } - except AlreadyLoggedException as exc: - # Already persisted upstream — report the failure without re-logging. - return False, { - "datetime": timezone.now().isoformat(), - "severity": "error", - "message": str(exc), - "progress": None, - } except Exception as exc: + if app_error_for(exc) is not None: + # Already persisted upstream — report the failure without re-logging. + return False, { + "datetime": timezone.now().isoformat(), + "severity": "error", + "message": str(exc), + "progress": None, + } persist_app_error(exc) return False, { "datetime": timezone.now().isoformat(), @@ -273,19 +256,29 @@ def sync_xero_data( if not items: break - try: - sync_function(items) - total_processed += len(items) - except XeroValidationError as exc: - persist_xero_error(exc) - raise - except AlreadyLoggedException: - raise # already persisted upstream — pass through unchanged - except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc - - # Track the max updated_date_utc across all pages for cursor update + # Drop objects a finished E2E run created in Xero. Filtered here rather + # than inside the sync functions so a suppressed contact and the + # documents referencing it are dropped together, and pagination still + # terminates on the fetched page rather than the filtered one. + items_to_sync = drop_e2e_artifacts(items, our_entity_type) + + if items_to_sync: + try: + sync_function(items_to_sync) + total_processed += len(items_to_sync) + except XeroValidationError as exc: + persist_xero_error(exc) + raise + except Exception as exc: + persist_app_error(exc) + raise + else: + pass # Whole page suppressed; the cursor below still advances past it. + + # Track the max updated_date_utc across all pages for cursor update. + # Deliberately over the fetched items, not the filtered ones, so the + # cursor advances past suppressed objects instead of refetching them + # from Xero every hour. for item in items: item_updated = getattr(item, "updated_date_utc", None) if item_updated and ( @@ -297,9 +290,9 @@ def sync_xero_data( "datetime": timezone.now().isoformat(), "entity": our_entity_type, "severity": "info", - "message": f"Processed {len(items)} {our_entity_type}", + "message": f"Processed {len(items_to_sync)} {our_entity_type}", "progress": None, - "recordsUpdated": len(items), + "recordsUpdated": len(items_to_sync), } # Check if done @@ -454,24 +447,22 @@ def sync_all_xero_data( force: bool = False, ) -> Iterator[XeroSyncEvent]: """Sync Xero data - either using latest timestamps or looking back N days.""" + # Safety net: don't sync until seeding is complete (prod IDs cleared, dev IDs set). + # Targeted syncs (e.g. --entity accounts) during setup can pass force=True. + if not force and not is_accounting_enabled(): + logger.warning( + "Xero sync not ready: enable_xero_sync is False. " + "In DEV: Run 'python manage.py seed_xero_from_database' first. " + "In Prod: Set using the gui" + ) + return + token = get_valid_token() if not token: message = "No valid Xero token found" logger.warning(message) raise NoValidXeroTokenError(message) - # Safety net: don't sync until seeding is complete (prod IDs cleared, dev IDs set). - # Targeted syncs (e.g. --entity accounts) during setup can pass force=True. - if not force: - company = CompanyDefaults.get_solo() - if not company.enable_xero_sync: - logger.warning( - "Xero sync not ready: enable_xero_sync is False. " - "In DEV: Run 'python manage.py seed_xero_from_database' first. " - "In Prod: Set using the gui" - ) - return - if entities is None: entities = list(ENTITY_CONFIGS.keys()) @@ -494,7 +485,7 @@ def sync_all_xero_data( ( xero_type, our_type, - model, + _model, api_method, sync_func, params, @@ -580,14 +571,18 @@ def sync_local_stock_to_xero(): } -def one_way_sync_all_xero_data(entities=None, force=False): +def one_way_sync_all_xero_data( + entities: Sequence[str] | None = None, force: bool = False +) -> Iterator[XeroSyncEvent]: """Normal sync using latest timestamps""" yield from sync_all_xero_data( use_latest_timestamps=True, entities=entities, force=force ) -def deep_sync_xero_data(days_back=30, entities=None): +def deep_sync_xero_data( + days_back: int = 30, entities: Sequence[str] | None = None +) -> Iterator[XeroSyncEvent]: """Perform a deep synchronisation over a time window. Args: @@ -602,15 +597,26 @@ def deep_sync_xero_data(days_back=30, entities=None): ) -def synchronise_xero_data(delay_between_requests=1): +def synchronise_xero_data() -> Iterator[XeroSyncEvent]: """Yield progress events while performing a full Xero synchronisation.""" from apps.workflow.api.xero.payroll import sync_xero_pay_items + company_defaults = CompanyDefaults.get_solo() + if not company_defaults.enable_xero_sync: + logger.info("Xero sync skipped: enable_xero_sync is False") + yield { + "datetime": timezone.now().isoformat(), + "entity": "sync", + "severity": "warning", + "message": "Xero sync skipped: enable_xero_sync is False", + } + return + # `sync_xero_pay_items` runs before any per-page gate and isn't itself # gated; without this orchestrator-level check it would 429 below the # floor on every breached sync. The per-page gate in `sync_xero_data` # never gets a chance because the orchestrator crashes first. - floor = CompanyDefaults.get_solo().xero_automated_day_floor + floor = company_defaults.xero_automated_day_floor if quota_floor_breached(floor): # Raise rather than yield-and-return: the latter would let # `XeroSyncService.run_sync` finish normally and emit @@ -629,7 +635,6 @@ def synchronise_xero_data(delay_between_requests=1): return try: - company_defaults = CompanyDefaults.get_solo() now = timezone.now() # Sync pay items (leave types + earnings rates) - lightweight, 2 API calls. diff --git a/apps/workflow/api/xero/xero.py b/apps/workflow/api/xero/xero.py index 40757b413..6c4941656 100644 --- a/apps/workflow/api/xero/xero.py +++ b/apps/workflow/api/xero/xero.py @@ -13,7 +13,6 @@ ) from apps.workflow.api.xero.auth import api_client, get_tenant_id -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error logger = logging.getLogger("xero") @@ -61,46 +60,6 @@ def get_xero_items(if_modified_since: Optional[datetime] = None) -> Any: raise -def get_projects(if_modified_since: Optional[datetime] = None) -> Any: - """ - Fetches Xero Projects using the Projects API. - Handles rate limiting and other API errors. - """ - logger.info(f"Fetching Xero Projects. If modified since: {if_modified_since}") - - tenant_id = get_tenant_id() - projects_api = ProjectApi(api_client) - logger.info(f"Using tenant ID: {tenant_id}") - - # Convert string to datetime if needed - if isinstance(if_modified_since, str): - if_modified_since = datetime.fromisoformat( - if_modified_since.replace("Z", "+00:00") - ) - - try: - match if_modified_since: - case None: - logger.info("No 'if_modified_since' provided, fetching all projects.") - projects = projects_api.get_projects(xero_tenant_id=tenant_id) - case datetime(): - logger.info( - f"'if_modified_since' provided: {if_modified_since.isoformat()}" - ) - projects = projects_api.get_projects( - xero_tenant_id=tenant_id, if_modified_since=if_modified_since - ) - case _: - raise ValueError( - f"Invalid type for 'if_modified_since': {type(if_modified_since)}. Expected datetime or None." - ) - logger.info(f"Successfully fetched {len(projects.items)} Xero Projects.") - return projects.items - except Exception as e: - logger.error(f"Error fetching Xero Projects: {e}", exc_info=True) - raise - - def create_project(project_data: Dict[str, Any]) -> Any: """ Creates a new Xero Project using the Projects API. @@ -235,11 +194,9 @@ def create_default_task(project_id: str) -> Any: try: workshop_rate = LabourSubtype.default_workshop().default_charge_out_rate - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise rate_amount = Amount(currency=CurrencyCode.NZD, value=float(workshop_rate)) @@ -255,15 +212,13 @@ def create_default_task(project_id: str) -> Any: ) logger.info(f"Successfully created default Labor task for Project {project_id}") return created_task - except AlreadyLoggedException: - raise except Exception as exc: logger.error( f"Error creating default task for Project {project_id}: {exc}", exc_info=True, ) - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def create_expense_entries( diff --git a/apps/workflow/apps.py b/apps/workflow/apps.py index cb1031105..b50113e19 100644 --- a/apps/workflow/apps.py +++ b/apps/workflow/apps.py @@ -60,5 +60,5 @@ def ready(self) -> None: @staticmethod def _register_accounting_providers() -> None: """Import accounting provider modules so they auto-register.""" - import apps.workflow.accounting.xero.provider # noqa: F401 + import apps.workflow.accounting.xero.provider import apps.workflow.accounting.xero.readonly_provider # noqa: F401 diff --git a/apps/workflow/authentication.py b/apps/workflow/authentication.py index d58ba91c1..ecf2383ac 100644 --- a/apps/workflow/authentication.py +++ b/apps/workflow/authentication.py @@ -73,7 +73,7 @@ def authenticate(self, request): has_refresh_cookie, ) return None - user, token = result + user, _token = result if not user.is_currently_active: raise exceptions.AuthenticationFailed( "User is inactive.", code="user_inactive" @@ -131,8 +131,8 @@ def authenticate(self, request): # so we return the service key object as the "user" for authorization checks return (service_key, None) - except ServiceAPIKey.DoesNotExist: - raise AuthenticationFailed("Invalid API key") + except ServiceAPIKey.DoesNotExist as exc: + raise AuthenticationFailed("Invalid API key") from exc def authenticate_header(self, request): """ diff --git a/apps/workflow/enums.py b/apps/workflow/enums.py index 14aae7002..3d54921b7 100644 --- a/apps/workflow/enums.py +++ b/apps/workflow/enums.py @@ -6,3 +6,8 @@ class AIProviderTypes(models.TextChoices): GOOGLE = "Gemini" MISTRAL = "Mistral" OPENAI = "OpenAI" + + +class NotebookLmRestriction(models.TextChoices): + NONE = "none", "All staff" + SUPERUSER = "superuser", "Superusers only" diff --git a/apps/workflow/exception_handlers.py b/apps/workflow/exception_handlers.py index 09afbc650..f5d2523c2 100644 --- a/apps/workflow/exception_handlers.py +++ b/apps/workflow/exception_handlers.py @@ -13,7 +13,6 @@ from rest_framework.response import Response from rest_framework.views import exception_handler -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error auth_logger = logging.getLogger("auth") @@ -24,7 +23,7 @@ def custom_exception_handler(exc: Exception, context: dict) -> Optional[Response Custom exception handler that persists unlogged errors and logs permission denied. Catches all exceptions that reach DRF's exception handler. Persists any that - haven't already been persisted upstream (detected via AlreadyLoggedException). + haven't already been persisted upstream (persist_app_error is idempotent). """ response = exception_handler(exc, context) request = context.get("request") @@ -41,13 +40,12 @@ def custom_exception_handler(exc: Exception, context: dict) -> Optional[Response "session_replay_id": session_replay_id, } - if not isinstance(exc, AlreadyLoggedException): - persist_app_error( - exc, - user_id=user_id, - session_replay_id=session_replay_id, - additional_context=additional_context, - ) + persist_app_error( + exc, + user_id=user_id, + session_replay_id=session_replay_id, + additional_context=additional_context, + ) if isinstance(exc, PermissionDenied): view = context.get("view") diff --git a/apps/workflow/exceptions.py b/apps/workflow/exceptions.py index b725abb48..d081ec4c2 100644 --- a/apps/workflow/exceptions.py +++ b/apps/workflow/exceptions.py @@ -1,5 +1,4 @@ from typing import List, Optional -from uuid import UUID class XeroValidationError(Exception): @@ -21,19 +20,6 @@ def __init__( super().__init__(message) -class AlreadyLoggedException(Exception): - """Exception that indicates the wrapped exception was already persisted.""" - - def __init__( - self, - original_exception: Exception, - app_error_id: Optional[str | UUID] = None, - ) -> None: - self.original = original_exception - self.app_error_id = str(app_error_id) if app_error_id is not None else None - super().__init__(str(original_exception)) - - class XeroQuotaFloorReached(Exception): """Raised when an automated Xero call cannot proceed because the day-quota is at or below CompanyDefaults.xero_automated_day_floor. diff --git a/apps/workflow/fixtures/ai_providers.json.example b/apps/workflow/fixtures/ai_providers.json.example index b7d23eb04..1237c1fa1 100644 --- a/apps/workflow/fixtures/ai_providers.json.example +++ b/apps/workflow/fixtures/ai_providers.json.example @@ -6,7 +6,7 @@ "name": "Claude", "provider_type": "Claude", "api_key": "sk-ant-YOUR_ANTHROPIC_KEY_HERE", - "model_name": "claude-sonnet-4-20250514", + "model_name": "claude-sonnet-5", "default": true } }, @@ -17,7 +17,7 @@ "name": "Gemini", "provider_type": "Gemini", "api_key": "YOUR_GEMINI_KEY_HERE", - "model_name": "gemini-2.5-flash", + "model_name": "gemini-flash-latest", "default": false } }, diff --git a/apps/workflow/fixtures/company_defaults.json b/apps/workflow/fixtures/company_defaults.json index e9007ac1d..47770951e 100644 --- a/apps/workflow/fixtures/company_defaults.json +++ b/apps/workflow/fixtures/company_defaults.json @@ -42,9 +42,10 @@ "gdrive_sops_folder_id": null, "gdrive_reference_library_folder_id": null, "accounting_provider": "xero", - "xero_tenant_id": null, + "xero_tenant_id": "00000000-0000-0000-0000-000000000000", "xero_shortcode": null, "xero_sales_branding_theme_id": null, + "xero_quote_terms": "Terms of trade can be found on our website: https://www.democompany.example.com/terms-of-trade", "enable_xero_sync": false, "xero_automated_day_floor": 100, "job_delta_soft_fail": true, @@ -89,21 +90,5 @@ "logo": "app_images/docketworks_logo.png", "logo_wide": "app_images/docketworks_logo_wide.png" } - }, - { - "model": "crm.phoneendpoint", - "pk": "00000000-0000-0000-0000-000000000101", - "fields": { - "number": "+6496365131", - "normalized_number": "+6496365131", - "label": "Main line", - "endpoint_type": "main_line", - "staff": null, - "provider_account_code": "", - "provider_metadata": {}, - "is_active": true, - "created_at": "2024-01-01T00:00:00Z", - "updated_at": "2024-01-01T00:00:00Z" - } } ] diff --git a/apps/workflow/fixtures/company_defaults_prospect.json b/apps/workflow/fixtures/company_defaults_prospect.json index ba9a0c9d8..0f481bb06 100644 --- a/apps/workflow/fixtures/company_defaults_prospect.json +++ b/apps/workflow/fixtures/company_defaults_prospect.json @@ -45,6 +45,7 @@ "xero_tenant_id": null, "xero_shortcode": null, "xero_sales_branding_theme_id": null, + "xero_quote_terms": "Terms of trade can be found on our website: __URL__/terms-of-trade", "enable_xero_sync": false, "xero_automated_day_floor": 100, "job_delta_soft_fail": true, diff --git a/apps/workflow/fixtures/initial_data.json b/apps/workflow/fixtures/initial_data.json index 69fae2b95..75219be6d 100644 --- a/apps/workflow/fixtures/initial_data.json +++ b/apps/workflow/fixtures/initial_data.json @@ -1,28 +1,15 @@ [ - { - "model": "company.company", - "pk": "00000000-0000-0000-0000-000000000001", - "fields": { - "xero_contact_id": null, - "name": "Demo Company Shop", - "email": "demo@example.com", - "address": null, - "is_account_customer": false, - "raw_json": {}, - "xero_last_modified": "2024-01-01T00:00:00Z", - "django_created_at": "2024-01-01T00:00:00Z", - "django_updated_at": "2024-01-01T00:00:00Z" - } - }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000001", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "charles.baker@example.com", "first_name": "Charles", "last_name": "Baker", "preferred_name": "Charlie", - "wage_rate": "35.80", + "base_wage_rate": "35.80", + "wage_rate": "42.96", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -32,18 +19,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe38" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000002", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "alex.cooper@example.com", "first_name": "Alex", "last_name": "Cooper", "preferred_name": null, - "wage_rate": "33.50", + "base_wage_rate": "33.50", + "wage_rate": "40.20", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -53,18 +42,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe39" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000003", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "nathan.chen@example.com", "first_name": "Nathan", "last_name": "Chen", "preferred_name": "Nate", - "wage_rate": "36.25", + "base_wage_rate": "36.25", + "wage_rate": "43.50", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -74,18 +65,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe40" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000004", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "robert.irwin@example.com", "first_name": "Robert", "last_name": "Irwin", "preferred_name": "Rob", - "wage_rate": "37.50", + "base_wage_rate": "37.50", + "wage_rate": "45.00", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -95,18 +88,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe41" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000005", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "peter.johnson@example.com", "first_name": "Peter", "last_name": "Johnson", "preferred_name": "Pete", - "wage_rate": "34.75", + "base_wage_rate": "34.75", + "wage_rate": "41.70", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -116,18 +111,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe42" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000006", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "james.kennedy@example.com", "first_name": "James", "last_name": "Kennedy", "preferred_name": "Jim", - "wage_rate": "36.00", + "base_wage_rate": "36.00", + "wage_rate": "43.20", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -137,18 +134,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe43" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000007", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "andrew.mitchell@example.com", "first_name": "Andrew", "last_name": "Mitchell", "preferred_name": "Andy", - "wage_rate": "35.25", + "base_wage_rate": "35.25", + "wage_rate": "42.30", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -158,18 +157,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe44" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000008", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "anthony.parker@example.com", "first_name": "Anthony", "last_name": "Parker", "preferred_name": "Tony", - "wage_rate": "33.75", + "base_wage_rate": "33.75", + "wage_rate": "40.50", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -179,18 +180,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe45" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000009", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "thomas.richards@example.com", "first_name": "Thomas", "last_name": "Richards", "preferred_name": "Tom", - "wage_rate": "34.50", + "base_wage_rate": "34.50", + "wage_rate": "41.40", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -200,18 +203,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe46" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000010", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "matthew.robinson@example.com", "first_name": "Matthew", "last_name": "Robinson", "preferred_name": "Matt", - "wage_rate": "36.50", + "base_wage_rate": "36.50", + "wage_rate": "43.80", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -221,18 +226,20 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe47" + "xero_user_id": null } }, { "model": "accounts.staff", "pk": "10000000-0000-0000-0000-000000000011", "fields": { + "password": "pbkdf2_sha256$1200000$r38oa8JEHbS2mwPAOUuXvP$3I5jBqODtIZEnfrWoV0ONxu7R2JBNCu2tidiV9ChlYY=", "email": "patrick.xu@example.com", "first_name": "Patrick", "last_name": "Xu", "preferred_name": null, - "wage_rate": "35.00", + "base_wage_rate": "35.00", + "wage_rate": "42.00", "hours_mon": "8.0", "hours_tue": "8.0", "hours_wed": "8.0", @@ -242,30 +249,21 @@ "hours_sun": "0.00", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z", - "xero_user_id": "4ac80a91-1aee-434b-88a9-c509b48bfe48" + "xero_user_id": null } }, { - "model": "accounts.staff", - "pk": "10000000-0000-0000-0000-000000000012", + "model": "crm.phoneendpoint", + "pk": "00000000-0000-0000-0000-000000000101", "fields": { - "password": "pbkdf2_sha256$870000$5Nw3RUuFaZZPCkeyVOm4kx$Attep1SqGF6ymdwm44LOte4wwszqte0W5ey3xcENFAI=", - "email": "defaultadmin@example.com", - "first_name": "Default", - "last_name": "Admin", - "preferred_name": null, - "wage_rate": "40.00", - "hours_mon": "8.0", - "hours_tue": "8.0", - "hours_wed": "8.0", - "hours_thu": "8.0", - "hours_fri": "8.0", - "hours_sat": "0.00", - "hours_sun": "0.00", + "number": "+6496365131", + "normalized_number": "+6496365131", + "label": "Main line", + "endpoint_type": "main_line", + "staff": null, + "provider_account_code": "", + "provider_metadata": {}, "is_active": true, - "is_office_staff": true, - "is_superuser": true, - "date_joined": "2024-01-01T00:00:00Z", "created_at": "2024-01-01T00:00:00Z", "updated_at": "2024-01-01T00:00:00Z" } diff --git a/apps/workflow/management/commands/backport_data_backup.py b/apps/workflow/management/commands/backport_data_backup.py index 4132558cc..86a8ab5ad 100644 --- a/apps/workflow/management/commands/backport_data_backup.py +++ b/apps/workflow/management/commands/backport_data_backup.py @@ -8,7 +8,6 @@ from django.conf import settings from django.core.management.base import BaseCommand -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services import db_scrubber from apps.workflow.services.error_persistence import persist_app_error @@ -39,10 +38,11 @@ def add_arguments(self, parser): def handle(self, *args, **options): if options.get("analyze_fields"): - return self.analyze_fields( + self.analyze_fields( sample_size=options["sample_size"], model_filter=options.get("model_filter"), ) + return default_db = settings.DATABASES["default"] scrub_db = settings.DATABASES["scrub"] @@ -167,11 +167,9 @@ def handle(self, *args, **options): self.stdout.write( self.style.SUCCESS(f"Scrubbed dump written: {scrubbed_dump}") ) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise def _run(self, cmd, env=None): subprocess.run(cmd, check=True, env=env, capture_output=True, text=True) diff --git a/apps/workflow/management/commands/e2e_cleanup.py b/apps/workflow/management/commands/e2e_cleanup.py index 9666e77cd..72153aaee 100644 --- a/apps/workflow/management/commands/e2e_cleanup.py +++ b/apps/workflow/management/commands/e2e_cleanup.py @@ -19,11 +19,13 @@ from apps.company.models import Company, CompanyPersonLink, Person from apps.job.models import Job, QuoteSpreadsheet from apps.purchasing.models import PurchaseOrder, PurchaseOrderLine +from apps.workflow.services.e2e_artifacts import ( + TEST_COMPANY_NAME, + TEST_DATA_PREFIX, +) logger = logging.getLogger(__name__) -TEST_DATA_PREFIX = "[TEST]" -TEST_COMPANY_NAME = "ABC Carpet Cleaning TEST IGNORE" LEGACY_E2E_PREFIXES = ["E2E Test Client", "E2E Modal Client", "E2E Test Supplier"] diff --git a/apps/workflow/management/commands/finalize_instance_onboarding.py b/apps/workflow/management/commands/finalize_instance_onboarding.py new file mode 100644 index 000000000..6f7f994de --- /dev/null +++ b/apps/workflow/management/commands/finalize_instance_onboarding.py @@ -0,0 +1,32 @@ +"""Management command for the post-OAuth instance onboarding workflow.""" + +from django.core.management.base import BaseCommand, CommandParser + +from apps.workflow.models import CompanyDefaults +from apps.workflow.services.error_persistence import persist_app_error +from apps.workflow.services.instance_onboarding import finalize_instance_onboarding + + +class Command(BaseCommand): + help = "Finalize Xero onboarding and enable synchronization for a new instance" + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument( + "--seed-xero", + action="store_true", + help="Create missing demo-only Xero configuration and employees", + ) + + def handle(self, *args: object, **options: object) -> None: + try: + finalize_instance_onboarding(seed_xero=bool(options["seed_xero"])) + except Exception as exc: + CompanyDefaults.set_xero_sync_enabled(enabled=False) + persist_app_error(exc) + raise + + self.stdout.write( + self.style.SUCCESS( + "Instance onboarding complete; automated Xero sync is enabled." + ) + ) diff --git a/apps/workflow/management/commands/inspect_xero_quote_pdf.py b/apps/workflow/management/commands/inspect_xero_quote_pdf.py new file mode 100644 index 000000000..8d16009fe --- /dev/null +++ b/apps/workflow/management/commands/inspect_xero_quote_pdf.py @@ -0,0 +1,33 @@ +"""Inspect a native Xero quote PDF for an expected text marker.""" + +from __future__ import annotations + +import json +from argparse import ArgumentParser +from dataclasses import asdict +from uuid import UUID + +from django.core.management.base import BaseCommand, CommandError, CommandParser + +from apps.workflow.accounting.quote_pdf_service import inspect_quote_pdf + + +class Command(BaseCommand): + """Expose quote PDF inspection as a structured operational command.""" + + help = "Inspect a provider-rendered quote PDF for expected text" + + def add_arguments(self, parser: ArgumentParser | CommandParser) -> None: + parser.add_argument("quote_id", type=UUID) + parser.add_argument("--expected-text", required=True) + + def handle(self, *args: object, **options: object) -> None: + quote_id = options["quote_id"] + expected_text = options["expected_text"] + if not isinstance(quote_id, UUID): + raise CommandError("quote_id must be a UUID") + if not isinstance(expected_text, str) or not expected_text.strip(): + raise CommandError("--expected-text must not be empty") + + inspection = inspect_quote_pdf(quote_id, expected_text) + self.stdout.write(json.dumps(asdict(inspection), sort_keys=True)) diff --git a/apps/workflow/management/commands/relabel_client_app.py b/apps/workflow/management/commands/relabel_client_app.py deleted file mode 100644 index 2e58139bf..000000000 --- a/apps/workflow/management/commands/relabel_client_app.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Relabel the historical `client` app to `company` (KAN-278 one-time surgery). - -Runs before `migrate` (deploy.sh calls it for every instance). Idempotent: -keys off django_migrations rows still recorded under the old label. - -TEMPORARY KAN-278: remove this command and its deploy hook after every -production instance has completed the cutover and produced a verified -company-schema backup. - -The historic pre-squash rows (0001_initial .. 0023_drop_scalar_phone_fields) -are DELETED rather than relabelled: with the squash's `replaces` lists gone, -a blanket relabel would strand ("company", "0001_initial")-style ghost rows -that collide with any future company migration reusing one of those names -and can never be pruned. Only the baseline row carries state worth keeping. -""" - -from django.core.management.base import BaseCommand, CommandError -from django.db import connection, transaction - -TABLE_RENAMES = [ - ("client_client", "company_client"), - ("client_clientcontact", "company_clientcontact"), - ("client_clientcontactmethod", "company_clientcontactmethod"), - ("client_suppliersearchalias", "company_suppliersearchalias"), - ("client_supplierpickupaddress", "company_supplierpickupaddress"), -] - - -class Command(BaseCommand): - help = ( - "Relabel the 'client' app to 'company' in django_migrations/" - "content types/tables." - ) - - def handle( - self, - *args: object, # object: Django's untyped pass-through args; unused here - **options: object, # object: Django's untyped pass-through args; unused here - ) -> None: - # Atomic so a crash mid-surgery rolls back everything. The idempotence - # guard keys off django_migrations, so a half-applied state (UPDATE done, - # ALTERs not) must be impossible; Postgres DDL is transactional. - with transaction.atomic(), connection.cursor() as cursor: - cursor.execute( - "SELECT COUNT(*) FROM django_migrations WHERE app = 'client'" - ) - row = cursor.fetchone() - if row is None: - raise RuntimeError("django_migrations COUNT(*) query returned no row") - (stale_rows,) = row - if stale_rows == 0: - self.stdout.write("Already relabelled; nothing to do.") - return - cursor.execute( - "SELECT COUNT(*) FROM django_migrations " - "WHERE app = 'client' AND name = '0001_baseline'" - ) - baseline_row = cursor.fetchone() - if baseline_row is None: - raise RuntimeError("django_migrations COUNT(*) query returned no row") - (baseline_count,) = baseline_row - if baseline_count == 0: - raise CommandError( - "django_migrations has app='client' rows but no " - "('client', '0001_baseline') row - this database predates " - "the migration squash. Deploy a pre-squash release, run " - "migrate, then retry this deploy." - ) - cursor.execute( - "DELETE FROM django_migrations " - "WHERE app = 'client' AND name <> '0001_baseline'" - ) - cursor.execute( - "UPDATE django_migrations SET app = 'company' WHERE app = 'client'" - ) - cursor.execute( - "UPDATE django_content_type SET app_label = 'company' " - "WHERE app_label = 'client'" - ) - renamed_tables = 0 - for old, new in TABLE_RENAMES: - cursor.execute("SELECT to_regclass(%s)", [old]) - if cursor.fetchone()[0] is None: - continue - cursor.execute(f'ALTER TABLE "{old}" RENAME TO "{new}"') - renamed_tables += 1 - self.stdout.write( - self.style.SUCCESS( - f"Relabelled the client app: dropped {stale_rows - 1} historic " - "ledger rows, kept 0001_baseline, renamed content types and " - f"{renamed_tables} tables." - ) - ) diff --git a/apps/workflow/management/commands/seed_xero_from_database.py b/apps/workflow/management/commands/seed_xero_from_database.py index 97502bbde..a21b39ac5 100644 --- a/apps/workflow/management/commands/seed_xero_from_database.py +++ b/apps/workflow/management/commands/seed_xero_from_database.py @@ -175,9 +175,7 @@ def handle(self, *args, **options): self.stdout.write("Dry run complete - no changes made") else: # Enable Xero sync now that prod IDs are cleared and dev IDs are seeded - company = CompanyDefaults.get_solo() - company.enable_xero_sync = True - company.save() + CompanyDefaults.set_xero_sync_enabled(enabled=True) self.stdout.write("Xero seeding complete! enable_xero_sync is now True.") def process_accounts(self, dry_run): diff --git a/apps/workflow/management/commands/xero.py b/apps/workflow/management/commands/xero.py index 06db25c6d..9d27414e1 100644 --- a/apps/workflow/management/commands/xero.py +++ b/apps/workflow/management/commands/xero.py @@ -17,7 +17,11 @@ resolve_sales_branding_theme, ) from apps.workflow.accounting.registry import get_provider -from apps.workflow.api.xero.auth import api_client, get_tenant_id, get_valid_token +from apps.workflow.api.xero.auth import ( + api_client, + get_tenant_id, + get_valid_token, +) from apps.workflow.api.xero.payroll import ( get_earnings_rates, get_employees, @@ -27,6 +31,7 @@ ) from apps.workflow.models import XeroApp from apps.workflow.models.company_defaults import CompanyDefaults +from apps.workflow.services.error_persistence import persist_app_error def get_employees_simple_dev(): @@ -83,6 +88,11 @@ def add_arguments(self, parser): "and payroll calendar" ), ) + parser.add_argument( + "--seed-xero", + action="store_true", + help="Create missing demo-only Xero configuration during --setup", + ) parser.add_argument( "--no-set", action="store_true", @@ -184,20 +194,27 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): + try: + self._handle(*args, **options) + except Exception as exc: + persist_app_error(exc) + raise + + def _handle(self, *args: object, **options: object) -> None: # First check we have a valid token token = get_valid_token() if not token: - self.stdout.write( - self.style.ERROR( - "No valid Xero token found.\n" - "Connect to Xero via Admin > Xero Settings in the web app first." - ) + raise CommandError( + "No valid Xero token found. Connect to Xero via Admin > " + "Xero Settings in the web app first." ) - return # Handle specific flags + if options["seed_xero"] and not options["setup"]: + raise CommandError("--seed-xero is only valid with --setup.") + if options["setup"]: - self.run_setup() + self.run_setup(seed_xero=bool(options["seed_xero"])) return if options["users"]: @@ -205,7 +222,7 @@ def handle(self, *args, **options): return if options["payroll_employees"]: - self.get_payroll_employees(use_raw_api=options.get("raw_api", False)) + self.get_payroll_employees(use_raw_api=bool(options["raw_api"])) return if options["payroll_rates"]: @@ -228,8 +245,8 @@ def handle(self, *args, **options): self.configure_payroll() return - import_staff_requested = options["import_staff"] or options.get( - "import_staff_dry_run" + import_staff_requested = ( + options["import_staff"] or options["import_staff_dry_run"] ) if import_staff_requested: self.import_staff(options) @@ -237,8 +254,8 @@ def handle(self, *args, **options): link_staff_requested = ( options["link_staff"] - or bool(options.get("link_staff_dry_run")) - or bool(options.get("link_staff_emails")) + or bool(options["link_staff_dry_run"]) + or bool(options["link_staff_emails"]) ) if link_staff_requested: @@ -247,8 +264,8 @@ def handle(self, *args, **options): create_staff_requested = ( options["create_staff"] - or bool(options.get("create_staff_dry_run")) - or bool(options.get("create_staff_emails")) + or bool(options["create_staff_dry_run"]) + or bool(options["create_staff_emails"]) ) if create_staff_requested: @@ -263,7 +280,7 @@ def handle(self, *args, **options): self.get_tenants(options) def get_tenants(self, options): - """Get available Xero tenant IDs and names""" + """Get available Xero tenant IDs and names.""" identity_api = IdentityApi(api_client) connections = identity_api.get_connections() @@ -274,27 +291,19 @@ def get_tenants(self, options): self.stdout.write(f"Name: {conn.tenant_name}") self.stdout.write("-----------------------------") - # If only one tenant and --no-set not specified, automatically set it if len(connections) == 1 and not options["no_set"]: tenant_id = connections[0].tenant_id tenant_name = connections[0].tenant_name - - try: - company_defaults = CompanyDefaults.get_solo() - company_defaults.xero_tenant_id = tenant_id - company_defaults.save() - - self.stdout.write( - self.style.SUCCESS( - f"Automatically set tenant ID to {tenant_id} " - f"({tenant_name}) in CompanyDefaults" - ) - ) - except Exception as e: - self.stdout.write( - self.style.ERROR(f"Failed to set tenant ID in CompanyDefaults: {e}") + company_defaults = CompanyDefaults.get_solo() + company_defaults.xero_tenant_id = tenant_id + company_defaults.save(update_fields=["xero_tenant_id"]) + self.stdout.write( + self.style.SUCCESS( + f"Automatically set tenant ID to {tenant_id} " + f"({tenant_name}) in CompanyDefaults" ) - elif len(connections) == 1 and options["no_set"]: + ) + elif len(connections) == 1: self.stdout.write( self.style.WARNING( "Single tenant found but --no-set specified, " @@ -309,6 +318,36 @@ def get_tenants(self, options): ) ) + def _validate_production_xero_items(self, calendar_name: str) -> None: + """Require production payroll configuration without creating Xero data.""" + from apps.workflow.models import XeroPayItem + + if not calendar_name: + raise CommandError("Production requires xero_payroll_calendar_name.") + calendars = get_payroll_calendars() + if not any(calendar["name"] == calendar_name for calendar in calendars): + raise CommandError( + f"Payroll calendar '{calendar_name}' does not exist in the production Xero tenant." + ) + + pay_items = list(XeroPayItem.objects.all()) + if not pay_items: + raise CommandError( + "No required XeroPayItem records are configured locally." + ) + earnings_names = {rate["name"] for rate in get_earnings_rates()} + leave_names = {leave["name"] for leave in get_leave_types()} + missing = [ + item.name + for item in pay_items + if item.name not in (leave_names if item.uses_leave_api else earnings_names) + ] + if missing: + raise CommandError( + "Production Xero is missing required pay items: " + + ", ".join(sorted(missing)) + ) + def _ensure_demo_xero_items_exist(self, calendar_name: str, tenant_id: str) -> None: """Create any Xero payroll items missing from the demo org (e.g. after a demo org reset).""" from xero_python.payrollnz.models import EarningsRate, LeaveType @@ -384,12 +423,9 @@ def _ensure_demo_xero_items_exist(self, calendar_name: str, tenant_id: str) -> N None, ) if not expense_account_id: - self.stdout.write( - self.style.ERROR( - "Cannot create earnings rates: no existing rate has an expense_account_id." - ) + raise CommandError( + "Cannot create demo earnings rates: no existing rate has an expense_account_id." ) - return for item in pay_items: if item.uses_leave_api: @@ -434,7 +470,7 @@ def _ensure_demo_xero_items_exist(self, calendar_name: str, tenant_id: str) -> N ) ) - def run_setup(self): + def run_setup(self, *, seed_xero: bool = False) -> None: """Configure Xero tenant, theme, shortcode, and payroll calendar.""" self.stdout.write("Setting up Xero connection...") @@ -447,15 +483,13 @@ def run_setup(self): raise if not connections: - self.stdout.write( - self.style.ERROR( - "No Xero organisations connected.\n" - "Please connect an organisation in Xero first." - ) + raise CommandError( + "No Xero organisations connected. Please connect an organisation " + "in Xero first." ) - return - # Step 2: Use first connected organisation + # Step 2: Use first connected organisation. This intentionally rebinds + # CompanyDefaults after Xero's recurring demo-tenant resets. connection = connections[0] tenant_id = connection.tenant_id tenant_name = connection.tenant_name @@ -487,9 +521,12 @@ def run_setup(self): # Always update cache to prevent stale tenant ID from being used cache.set("xero_tenant_id", tenant_id) - self._ensure_demo_xero_items_exist( - company.xero_payroll_calendar_name, tenant_id - ) + if seed_xero: + self._ensure_demo_xero_items_exist( + company.xero_payroll_calendar_name, tenant_id + ) + else: + self._validate_production_xero_items(company.xero_payroll_calendar_name) # Step 4: Fetch organisation shortcode for deep linking accounting_api = AccountingApi(api_client) @@ -504,9 +541,24 @@ def run_setup(self): shortcode = org_response.organisations[0].short_code # Step 5: Select a live sales branding theme for this tenant - sales_branding_theme = resolve_sales_branding_theme( - get_provider(), company.xero_sales_branding_theme_id - ) + if not seed_xero: + configured_theme_id = company.xero_sales_branding_theme_id + if configured_theme_id is None: + raise CommandError( + "Production requires an explicitly selected Xero sales branding theme." + ) + sales_branding_theme = next( + ( + theme + for theme in get_provider().list_document_themes() + if theme.external_id == str(configured_theme_id) + ), + None, + ) + else: + sales_branding_theme = resolve_sales_branding_theme( + get_provider(), company.xero_sales_branding_theme_id + ) if sales_branding_theme is None: raise CommandError( "Xero returned no branding themes. Create a branding theme in " @@ -516,28 +568,18 @@ def run_setup(self): # Step 6: Fetch payroll calendar ID calendar_name = company.xero_payroll_calendar_name if not calendar_name: - self.stdout.write( - self.style.WARNING( - "xero_payroll_calendar_name not configured in CompanyDefaults. " - "Skipping payroll calendar setup." - ) - ) - payroll_calendar_id = None - else: - calendars = get_payroll_calendars() - matching_calendar = next( - (c for c in calendars if c["name"] == calendar_name), None + raise CommandError("xero_payroll_calendar_name is required.") + calendars = get_payroll_calendars() + matching_calendar = next( + (c for c in calendars if c["name"] == calendar_name), None + ) + if not matching_calendar: + available = [c["name"] for c in calendars] + raise CommandError( + f"Payroll calendar '{calendar_name}' not found in Xero. " + f"Available calendars: {available}" ) - if not matching_calendar: - available = [c["name"] for c in calendars] - self.stdout.write( - self.style.ERROR( - f"Payroll calendar '{calendar_name}' not found in Xero.\n" - f"Available calendars: {available}" - ) - ) - return - payroll_calendar_id = matching_calendar["id"] + payroll_calendar_id = matching_calendar["id"] # Step 7: Save to CompanyDefaults company.xero_shortcode = shortcode @@ -809,8 +851,8 @@ def link_staff(self, options): Includes all staff (active and inactive) so that historical timesheets can be posted for departed employees. """ - emails_option = options.get("link_staff_emails") - dry_run = options.get("link_staff_dry_run", False) + emails_option = options["link_staff_emails"] + dry_run = options["link_staff_dry_run"] # Only include staff with wage rates (excludes admin-only users) queryset = Staff.objects.filter(wage_rate__gt=0) @@ -858,8 +900,8 @@ def link_staff(self, options): def create_staff(self, options): """Create Xero Payroll employees for specified staff members.""" - emails_option = options.get("create_staff_emails") - dry_run = options.get("create_staff_dry_run", False) + emails_option = options["create_staff_emails"] + dry_run = options["create_staff_dry_run"] if not emails_option: self.stdout.write( @@ -1038,13 +1080,12 @@ def _prompt_for_leave_type_name(self, label, leave_types, current_value): return type_name - def import_staff(self, options): + def import_staff(self, options: dict[str, object]) -> None: """Import employees from Xero Payroll as local Staff records.""" - dry_run = options.get("import_staff_dry_run", False) - force = options.get("force", False) - initial_password = options.get( - "import_staff_password", "Default-staff-password" - ) + # Defaults live in add_arguments(); argparse always populates these keys. + dry_run = bool(options["import_staff_dry_run"]) + force = bool(options["force"]) + initial_password = str(options["import_staff_password"]) # Guard against double-import existing_staff = Staff.objects.filter(base_wage_rate__gt=0).count() diff --git a/apps/workflow/middleware.py b/apps/workflow/middleware.py index 518aabc25..9bffbf0de 100644 --- a/apps/workflow/middleware.py +++ b/apps/workflow/middleware.py @@ -11,7 +11,7 @@ from django.urls import reverse from apps.workflow.authentication import JWTAuthentication -from apps.workflow.services.error_persistence import persist_and_raise +from apps.workflow.services.error_persistence import persist_app_error # Get access logger configured in Django settings access_logger = logging.getLogger("access") @@ -95,7 +95,8 @@ def __call__(self, request: HttpRequest) -> HttpResponse: except Exception as e: # Log any errors that occur during logging access_logger.error(f"Error logging access: {e}") - persist_and_raise(e) + persist_app_error(e) + raise return response diff --git a/apps/workflow/migrations/0012_use_latest_gemini_flash_model.py b/apps/workflow/migrations/0012_use_latest_gemini_flash_model.py new file mode 100644 index 000000000..1da7e4cb0 --- /dev/null +++ b/apps/workflow/migrations/0012_use_latest_gemini_flash_model.py @@ -0,0 +1,41 @@ +from django.apps.registry import Apps +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor + +DEPRECATED_GEMINI_MODELS = ( + "gemini-2.0-flash-exp", + "gemini-2.5-flash", +) +GEMINI_FLASH_MODEL = "gemini-flash-latest" + + +def use_latest_gemini_flash_model( + apps: Apps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + AIProvider = apps.get_model("workflow", "AIProvider") + AIProvider.objects.filter( + provider_type="Gemini", + model_name__in=DEPRECATED_GEMINI_MODELS, + ).update(model_name=GEMINI_FLASH_MODEL) + + +class Migration(migrations.Migration): + dependencies = [ + ("workflow", "0011_companydefaults_xero_sales_branding_theme_id"), + ] + + operations = [ + migrations.AlterField( + model_name="aiprovider", + name="model_name", + field=models.CharField( + blank=True, + help_text="Model name (e.g., gemini-flash-latest)", + max_length=100, + ), + ), + migrations.RunPython( + use_latest_gemini_flash_model, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/apps/workflow/migrations/0013_notebooklmlink.py b/apps/workflow/migrations/0013_notebooklmlink.py new file mode 100644 index 000000000..bfdcc83b1 --- /dev/null +++ b/apps/workflow/migrations/0013_notebooklmlink.py @@ -0,0 +1,56 @@ +# Generated by Django 6.0.7 on 2026-07-23 00:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("workflow", "0012_use_latest_gemini_flash_model"), + ] + + operations = [ + migrations.CreateModel( + name="NotebookLmLink", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(help_text="Menu item name", max_length=100)), + ("url", models.URLField(help_text="NotebookLM notebook URL")), + ( + "enabled", + models.BooleanField( + default=True, help_text="Show this link in the training menu" + ), + ), + ( + "restriction", + models.CharField( + choices=[ + ("none", "All staff"), + ("superuser", "Superusers only"), + ], + default="none", + help_text="Who may see this link in the menu", + max_length=20, + ), + ), + ( + "order", + models.IntegerField(default=0, help_text="Menu display order"), + ), + ], + options={ + "verbose_name": "NotebookLM Link", + "verbose_name_plural": "NotebookLM Links", + "ordering": ["order", "name"], + }, + ), + ] diff --git a/apps/workflow/migrations/0014_companydefaults_xero_quote_terms.py b/apps/workflow/migrations/0014_companydefaults_xero_quote_terms.py new file mode 100644 index 000000000..ceb325550 --- /dev/null +++ b/apps/workflow/migrations/0014_companydefaults_xero_quote_terms.py @@ -0,0 +1,63 @@ +from django.db import migrations, models + +# Kept as a literal so this migration stays self-contained; the model's copy of +# the same starting text may drift without changing what was written here. +DEFAULT_QUOTE_TERMS = "Terms of trade can be found on our website." + + +def populate_default_quote_terms(apps, schema_editor): + """Point rows with a known website at its terms-of-trade page. + + AddField has already given every row DEFAULT_QUOTE_TERMS, so this only + upgrades the ones where a company URL makes a more specific sentence possible. + """ + CompanyDefaults = apps.get_model("workflow", "CompanyDefaults") + for defaults in CompanyDefaults.objects.exclude(company_url__isnull=True): + if not defaults.company_url: + continue + company_url = defaults.company_url.rstrip("/") + defaults.xero_quote_terms = ( + "Terms of trade can be found on our website: " + f"{company_url}/terms-of-trade" + ) + defaults.save(update_fields=["xero_quote_terms"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("workflow", "0013_notebooklmlink"), + ] + + operations = [ + migrations.AlterField( + model_name="companydefaults", + name="xero_sales_branding_theme_id", + field=models.UUIDField( + blank=True, + help_text=( + "Controls the layout and presentation of every quote and sales " + "invoice created in Xero. It is configured during Xero setup and " + "required before sales documents can be created." + ), + null=True, + verbose_name="Xero sales branding theme", + ), + ), + migrations.AddField( + model_name="companydefaults", + name="xero_quote_terms", + field=models.TextField( + default=DEFAULT_QUOTE_TERMS, + help_text=( + "Terms sent on every quote created by DocketWorks. Required — " + "Xero does not apply its own Terms (Quotes) default to quotes " + "created through the API. Copy the same text to Xero's Terms " + "(Quotes) setting so quotes created directly in Xero during an " + "outage use the same terms." + ), + max_length=4000, + verbose_name="Xero quote terms", + ), + ), + migrations.RunPython(populate_default_quote_terms, migrations.RunPython.noop), + ] diff --git a/apps/workflow/models/__init__.py b/apps/workflow/models/__init__.py index f839fbda0..78b1ae453 100644 --- a/apps/workflow/models/__init__.py +++ b/apps/workflow/models/__init__.py @@ -3,6 +3,7 @@ from .ai_provider import AIProvider from .app_error import AppError, XeroError from .company_defaults import CompanyDefaults +from .notebook_lm_link import NotebookLmLink from .search_telemetry_event import SearchTelemetryEvent from .service_api_key import ServiceAPIKey from .session_replay import SessionReplayChunk, SessionReplayRecording @@ -16,6 +17,7 @@ "AIProvider", "AppError", "CompanyDefaults", + "NotebookLmLink", "SearchTelemetryEvent", "ServiceAPIKey", "SessionReplayChunk", diff --git a/apps/workflow/models/ai_provider.py b/apps/workflow/models/ai_provider.py index f8f4dfb31..82a7291bb 100644 --- a/apps/workflow/models/ai_provider.py +++ b/apps/workflow/models/ai_provider.py @@ -13,7 +13,7 @@ class AIProvider(models.Model): ) model_name = models.CharField( max_length=100, - help_text="Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17)", + help_text="Model name (e.g., gemini-flash-latest)", blank=True, ) provider_type = models.CharField( diff --git a/apps/workflow/models/company_defaults.py b/apps/workflow/models/company_defaults.py index 6b03ba5d5..9fc58ce54 100644 --- a/apps/workflow/models/company_defaults.py +++ b/apps/workflow/models/company_defaults.py @@ -1,8 +1,14 @@ +from collections.abc import Iterable from decimal import Decimal from django.db import models +from django.db.models.base import ModelBase from solo.models import SingletonModel +# Starting point for an installation that has not had its terms written yet. +# Real wording is seeded per client by the fixtures and edited in Company Settings. +DEFAULT_XERO_QUOTE_TERMS = "Terms of trade can be found on our website." + class CompanyDefaults(SingletonModel): company_name = models.CharField(max_length=255) @@ -130,10 +136,20 @@ class CompanyDefaults(SingletonModel): blank=True, verbose_name="Xero sales branding theme", help_text=( - "Branding theme applied to every quote and sales invoice created in " - "Xero. Select a theme containing the required terms and conditions; " - "it is configured during Xero setup and required before sales " - "documents can be created." + "Controls the layout and presentation of every quote and sales invoice " + "created in Xero. It is configured during Xero setup and required " + "before sales documents can be created." + ), + ) + xero_quote_terms = models.TextField( + max_length=4000, + default=DEFAULT_XERO_QUOTE_TERMS, + verbose_name="Xero quote terms", + help_text=( + "Terms sent on every quote created by DocketWorks. Required — Xero does " + "not apply its own Terms (Quotes) default to quotes created through the " + "API. Copy the same text to Xero's Terms (Quotes) setting so quotes " + "created directly in Xero during an outage use the same terms." ), ) enable_xero_sync = models.BooleanField( @@ -344,7 +360,24 @@ class Meta: ), ] - def save(self, *args, **kwargs): + @classmethod + def set_xero_sync_enabled(cls, *, enabled: bool) -> None: + """Persist the Xero sync gate and refresh django-solo's shared cache.""" + company_defaults = cls.objects.get(pk=cls.singleton_instance_id) + company_defaults.enable_xero_sync = enabled + company_defaults.save(update_fields=["enable_xero_sync"]) + + # Variadic to stay substitutable for SingletonModel.save, which is variadic; + # a fixed signature here trips pylint's arguments-differ. + def save( + self, + *args: object, + force_insert: bool | tuple[ModelBase, ...] = False, + force_update: bool = False, + using: str | None = None, + update_fields: Iterable[str] | None = None, + **kwargs: object, + ) -> None: # Check if annual_leave_loading changed - if so, recompute all staff wage_rates loading_changed = False if self.pk: @@ -354,14 +387,19 @@ def save(self, *args, **kwargs): except CompanyDefaults.DoesNotExist: pass - result = super().save(*args, **kwargs) + super().save( + *args, + force_insert=force_insert, + force_update=force_update, + using=using, + update_fields=update_fields, + **kwargs, + ) if loading_changed: self._recompute_all_staff_wage_rates() - return result - - def _recompute_all_staff_wage_rates(self): + def _recompute_all_staff_wage_rates(self) -> None: """Bulk-recompute wage_rate for all staff based on current annual_leave_loading.""" from apps.accounts.models import Staff diff --git a/apps/workflow/models/notebook_lm_link.py b/apps/workflow/models/notebook_lm_link.py new file mode 100644 index 000000000..49e8fab9e --- /dev/null +++ b/apps/workflow/models/notebook_lm_link.py @@ -0,0 +1,33 @@ +from django.db import models + +from apps.workflow.enums import NotebookLmRestriction + + +class NotebookLmLink(models.Model): + """A NotebookLM notebook link shown in the app's training menu. + + Per-instance and admin-managed: each client configures their own rows. + `restriction` decides which staff see the link in the navbar; it is a UX + filter, not an access boundary (NotebookLM access is enforced by Drive ACLs). + """ + + name = models.CharField(max_length=100, help_text="Menu item name") + url = models.URLField(help_text="NotebookLM notebook URL") + enabled = models.BooleanField( + default=True, help_text="Show this link in the training menu" + ) + restriction = models.CharField( + max_length=20, + choices=NotebookLmRestriction, + default=NotebookLmRestriction.NONE, + help_text="Who may see this link in the menu", + ) + order = models.IntegerField(default=0, help_text="Menu display order") + + def __str__(self) -> str: + return self.name + + class Meta: + ordering = ["order", "name"] + verbose_name = "NotebookLM Link" + verbose_name_plural = "NotebookLM Links" diff --git a/apps/workflow/models/settings_metadata.py b/apps/workflow/models/settings_metadata.py index 0637d43ca..15b85b593 100644 --- a/apps/workflow/models/settings_metadata.py +++ b/apps/workflow/models/settings_metadata.py @@ -142,6 +142,7 @@ def get_section_info(cls, key: str) -> tuple[str, str, int] | None: "xero_tenant_id": "xero", "xero_shortcode": "xero", "xero_sales_branding_theme_id": "xero", + "xero_quote_terms": "xero", "enable_xero_sync": "xero", "xero_automated_day_floor": "xero", "xero_payroll_calendar_name": "xero", diff --git a/apps/workflow/serializers.py b/apps/workflow/serializers.py index d52346c03..dee7a5053 100644 --- a/apps/workflow/serializers.py +++ b/apps/workflow/serializers.py @@ -9,6 +9,7 @@ AIProvider, AppError, CompanyDefaults, + NotebookLmLink, XeroAccount, XeroApp, XeroError, @@ -18,17 +19,36 @@ from .models.settings_metadata import COMPANY_DEFAULTS_READ_ONLY_FIELDS -def _build_logo_url( - instance: CompanyDefaults, field_name: str, context: dict[str, Any] | None -) -> str | None: - """Build an absolute logo URL when possible, fall back to relative URL without request.""" +def _build_logo_url(instance: CompanyDefaults, field_name: str) -> str | None: + """Return the logo path relative to the site root. + + Deliberately relative, for the same reason as the staff icon URL: the + browser resolves it against its own origin, so one value is correct behind + ngrok in dev and behind the proxy in UAT/production. Building an absolute + URL from the request leaks the internal host wherever the forwarded-host + headers aren't trusted, and the browser then blocks the image. + + PDF rendering does not use this — it reads company.logo.path directly. + """ field_file = getattr(instance, field_name, None) if not field_file: return None - request = (context or {}).get("request") - if not request: - return field_file.url - return request.build_absolute_uri(field_file.url) + return field_file.url + + +class NotebookLmLinkSerializer(serializers.ModelSerializer[NotebookLmLink]): + """Serializer for NotebookLM training-menu links (read + write).""" + + class Meta: + model = NotebookLmLink + fields = ( + "id", + "name", + "url", + "enabled", + "restriction", + "order", + ) class AIProviderSerializer(serializers.ModelSerializer): @@ -53,6 +73,11 @@ class CompanyDefaultsSerializer(serializers.ModelSerializer): logo_wide = serializers.ImageField(required=False, allow_null=True, write_only=True) logo_url = serializers.SerializerMethodField(read_only=True) logo_wide_url = serializers.SerializerMethodField(read_only=True) + xero_quote_terms = serializers.CharField( + required=False, + max_length=4000, + trim_whitespace=False, + ) optional_url_fields = ( "master_quote_template_url", "gdrive_quotes_folder_url", @@ -60,10 +85,10 @@ class CompanyDefaultsSerializer(serializers.ModelSerializer): ) def get_logo_url(self, obj: CompanyDefaults) -> str | None: - return _build_logo_url(obj, "logo", self.context) + return _build_logo_url(obj, "logo") def get_logo_wide_url(self, obj: CompanyDefaults) -> str | None: - return _build_logo_url(obj, "logo_wide", self.context) + return _build_logo_url(obj, "logo_wide") def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: for field_name in self.optional_url_fields: @@ -73,6 +98,16 @@ def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: pass return attrs + def validate_xero_quote_terms(self, value: str) -> str: + """Reject whitespace-only terms. + + DRF rejects null and "" from the field itself, but trim_whitespace=False + means its blank check is a bare ``value == ""`` and lets " \\n " through. + """ + if not value.strip(): + raise serializers.ValidationError("Xero quote terms must not be blank.") + return value + class Meta: model = CompanyDefaults fields = "__all__" diff --git a/apps/workflow/services/__init__.py b/apps/workflow/services/__init__.py index ca12a449c..653b969c4 100644 --- a/apps/workflow/services/__init__.py +++ b/apps/workflow/services/__init__.py @@ -7,6 +7,13 @@ if apps.ready: from .db_scrubber import scrub from .dev_demo_export_scrubber import ScrubResult, scrub_dev_demo_export + from .e2e_artifacts import ( + InboundXeroObject, + XeroContactLike, + drop_e2e_artifacts, + get_closed_e2e_windows, + is_test_company_name, + ) from .error_grouping import ( list_grouped_app_errors, list_grouped_xero_errors, @@ -20,13 +27,14 @@ mark_xero_error_group_unresolved_by_fingerprint, ) from .error_persistence import ( + app_error_for, extract_job_context, extract_request_context, list_app_errors, - persist_and_raise, persist_app_error, persist_xero_error, ) + from .instance_onboarding import finalize_instance_onboarding from .llm_service import LLMService, quick_completion, quick_json_completion from .request import get_client_ip from .search import apply_text_search @@ -50,17 +58,24 @@ pass __all__ = [ + "InboundXeroObject", "LLMService", "ScrubResult", "SearchTelemetryService", + "XeroContactLike", "XeroSyncService", "XeroSyncStartResult", + "app_error_for", "append_chunk", "apply_text_search", "create_recording", + "drop_e2e_artifacts", "extract_job_context", "extract_request_context", + "finalize_instance_onboarding", "get_client_ip", + "get_closed_e2e_windows", + "is_test_company_name", "list_app_errors", "list_grouped_app_errors", "list_grouped_xero_errors", @@ -74,7 +89,6 @@ "mark_xero_error_group_unresolved", "mark_xero_error_group_unresolved_by_fingerprint", "normalize_search_query", - "persist_and_raise", "persist_app_error", "persist_xero_error", "purge_old_recordings", diff --git a/apps/workflow/services/db_scrubber.py b/apps/workflow/services/db_scrubber.py index 9a93ad4e6..d189fb9ad 100644 --- a/apps/workflow/services/db_scrubber.py +++ b/apps/workflow/services/db_scrubber.py @@ -32,7 +32,6 @@ from apps.accounts.models import SYSTEM_AUTOMATION_EMAIL, Staff from apps.accounts.staff_anonymization import create_staff_profile from apps.company.models import Company, ContactMethod, Person -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults from apps.workflow.services.error_persistence import persist_app_error @@ -324,8 +323,6 @@ def scrub() -> None: _delete_unlinked_accounting() _truncate_excluded_tables() _assert_private_config_removed() - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise diff --git a/apps/workflow/services/e2e_artifacts.py b/apps/workflow/services/e2e_artifacts.py new file mode 100644 index 000000000..7fe4859cc --- /dev/null +++ b/apps/workflow/services/e2e_artifacts.py @@ -0,0 +1,147 @@ +"""Ignore Xero objects created by finished E2E runs. Development only. + +E2E runs against a live Xero organisation with writes enabled, so the contacts, +invoices and quotes a run creates persist in Xero after the run's database has +been restored from backup. The hourly Xero sync then re-imports them into the +clean database, which is what this module exists to prevent. + +An inbound Xero object is skipped only when *both* hold: + +1. its timestamp falls inside a closed E2E run window, and +2. it belongs to a test company. + +Neither is sufficient alone. The window alone would also discard genuine Xero +changes that happened to land inside it. The name alone is equally true while a +run is executing, and would blind the inbound path the run exists to exercise. + +Run windows live in a temp file written by the E2E harness, not in the database: +teardown restores the database from a backup taken before the run, so database +state cannot describe a run that has just finished. Written by +frontend/tests/scripts/e2e-sync-windows.ts. +""" + +import json +import logging +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Protocol + +from django.conf import settings + +logger = logging.getLogger("xero") + + +class XeroContactLike(Protocol): + """The embedded contact a Xero document carries.""" + + name: str | None + + +class InboundXeroObject(Protocol): + """Structural view of the Xero SDK objects the sync feeds through here. + + The SDK types share no base class, and only some carry a company: contacts + hold their own `name`, documents hold a `contact`, and accounts and stock + hold neither. This declares the one attribute they all have; the rest are + read defensively. + """ + + updated_date_utc: datetime | None + + +# Fixed path so the backend and the E2E harness agree without one telling the +# other; same convention as the E2E lock file. Absent on any machine that has +# never run E2E, which is what keeps this module inert. +E2E_SYNC_WINDOWS_FILE = ( + Path(tempfile.gettempdir()) / "docketworks-e2e-sync-windows.json" +) + +# Names reserved for E2E test data. TEST_DATA_PREFIX marks companies, jobs and +# people a run creates; TEST_COMPANY_NAME is the standing fixture company that +# test jobs — and so the invoices and quotes raised against them — hang off. +# Mirrored in frontend/tests/scripts/db-backup-utils.ts. +TEST_DATA_PREFIX = "[TEST]" +TEST_COMPANY_NAME = "ABC Carpet Cleaning TEST IGNORE" + + +def is_test_company_name(name: str | None) -> bool: + """True when a company name is reserved for E2E test data.""" + if not name: + return False + return name.startswith(TEST_DATA_PREFIX) or name == TEST_COMPANY_NAME + + +def get_closed_e2e_windows() -> list[tuple[datetime, datetime]]: + """Finished E2E runs, as (started_at, ended_at) pairs. + + A run still in progress has no `ended_at` and is skipped: while tests + execute, inbound Xero data must behave exactly as it does in production, + because that round trip is what the run exercises. + """ + if not E2E_SYNC_WINDOWS_FILE.exists(): + return [] + + return [ + ( + datetime.fromisoformat(entry["started_at"]), + datetime.fromisoformat(entry["ended_at"]), + ) + for entry in json.loads(E2E_SYNC_WINDOWS_FILE.read_text()) + if entry["ended_at"] + ] + + +def _owning_company_name(item: InboundXeroObject) -> str | None: + """Name of the company an inbound Xero object belongs to. + + A contact carries its own name. Documents — invoices, quotes, bills, + purchase orders, credit notes — carry an embedded contact. Entities with no + company at all, such as accounts and stock, return None and are never + skipped. + """ + contact: XeroContactLike | None = getattr(item, "contact", None) + if contact is not None: + return contact.name + + name: str | None = getattr(item, "name", None) + return name + + +def _is_e2e_artifact( + item: InboundXeroObject, closed_windows: list[tuple[datetime, datetime]] +) -> bool: + """True when this Xero object was created by a finished E2E run.""" + updated_at: datetime | None = getattr(item, "updated_date_utc", None) + if updated_at is None: + return False + + if not is_test_company_name(_owning_company_name(item)): + return False + + return any(start <= updated_at <= end for start, end in closed_windows) + + +def drop_e2e_artifacts( + items: list[InboundXeroObject], entity_type: str +) -> list[InboundXeroObject]: + """Remove objects created by finished E2E runs from an inbound batch. + + Filtering the batch before it reaches the sync function is deliberate: the + invoice, quote and purchase-order importers resolve their contact through + ``resolve_company_from_xero_contact``, which raises rather than skips when a + company cannot be synced. Dropping a suppressed contact and the documents + that reference it together keeps that path from ever being reached. + """ + if settings.PRODUCTION_LIKE: + return items + + closed_windows = get_closed_e2e_windows() + if not closed_windows: + return items + + kept = [item for item in items if not _is_e2e_artifact(item, closed_windows)] + dropped = len(items) - len(kept) + if dropped: + logger.info("Skipped %d %s created by a finished E2E run", dropped, entity_type) + return kept diff --git a/apps/workflow/services/error_persistence.py b/apps/workflow/services/error_persistence.py index cdb394a42..9e9160a58 100644 --- a/apps/workflow/services/error_persistence.py +++ b/apps/workflow/services/error_persistence.py @@ -2,15 +2,45 @@ import logging import traceback from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Optional from uuid import UUID from django.db.models import QuerySet from django.http import HttpRequest -from apps.workflow.exceptions import AlreadyLoggedException, XeroValidationError +from apps.workflow.exceptions import XeroValidationError from apps.workflow.models import AppError, SessionReplayRecording, XeroError +# Set on an exception instance once it has been persisted. BaseException always +# carries a __dict__, so this is safe to set on any exception. The marker is +# metadata *about* the exception, deliberately not a wrapper type — wrapping +# would destroy the type the HTTP boundary needs to choose a status code. +_APP_ERROR_ATTR = "__app_error__" + + +def _mark_persisted(exception: Exception, app_error: AppError) -> None: + setattr(exception, _APP_ERROR_ATTR, app_error) + + +def _existing_app_error(exception: Exception) -> Optional[AppError]: + """Find the AppError for this failure, following the ``__cause__`` chain. + + A handler that converts (``raise ValueError(...) from exc``) produces a new + exception object, so the marker lives on the cause rather than on what the + boundary finally sees. Walking the chain keeps one failure to one row across + conversions. This is why ``raise ... from exc`` is mandatory (pylint W0707) — + an unchained conversion severs the link and earns a second row. + """ + seen: set[int] = set() + current: Optional[BaseException] = exception + while current is not None and id(current) not in seen: + seen.add(id(current)) + app_error: object = getattr(current, _APP_ERROR_ATTR, None) + if isinstance(app_error, AppError): + return app_error + current = current.__cause__ + return None + def _make_json_serializable(obj: Any) -> Any: """Convert non-JSON-serializable objects (like UUIDs) to strings.""" @@ -91,8 +121,8 @@ def persist_xero_error(exc: XeroValidationError): def _clean_session_replay_id( *, - session_replay_id: str | None, - user_id: str | None, + session_replay_id: str | UUID | None, + user_id: str | UUID | None, context_data: dict[str, Any], ) -> UUID | None: if not session_replay_id: @@ -122,14 +152,14 @@ def _clean_session_replay_id( def persist_app_error( exception: Exception, - app: str = None, - file: str = None, - function: str = None, + app: Optional[str] = None, + file: Optional[str] = None, + function: Optional[str] = None, severity: int = logging.ERROR, - job_id: str = None, - user_id: str = None, - session_replay_id: str = None, - additional_context: dict = None, + job_id: Optional[str | UUID] = None, + user_id: Optional[str | UUID] = None, + session_replay_id: Optional[str | UUID] = None, + additional_context: Optional[Dict[str, Any]] = None, ) -> AppError: """Create and save an AppError with enhanced context. @@ -148,8 +178,15 @@ def persist_app_error( additional_context: Additional context data to store in JSON field Returns: - Created AppError instance + The AppError for this exception — newly created, or the existing one + if it has already been persisted. """ + already_persisted = _existing_app_error(exception) + if already_persisted is not None: + # Persisted deeper in the stack, where the context was richer. One + # failure is one row; the first write wins. + return already_persisted + # Auto-extract caller context if not provided caller_context = _extract_caller_context() @@ -166,7 +203,7 @@ def persist_app_error( context_data=context_data, ) - return AppError.objects.create( + app_error = AppError.objects.create( message=str(exception), data=context_data, app=app or caller_context["app"], @@ -177,6 +214,17 @@ def persist_app_error( user_id=user_id, session_replay_id=clean_session_replay_id, ) + _mark_persisted(exception, app_error) + return app_error + + +def app_error_for(exception: Exception) -> Optional[AppError]: + """Return the AppError this exception was persisted as, if any. + + Response builders use this to surface ``error_id`` (ADR 0013) without + needing the exception to have been wrapped in a marker type. + """ + return _existing_app_error(exception) def list_app_errors( @@ -224,15 +272,3 @@ def list_app_errors( "previous": prev_offset, "results": results, } - - -def persist_and_raise(exception: Exception, **context: Any) -> None: - """ - Persist the exception via ``persist_app_error`` and raise AlreadyLoggedException. - - Args: - exception: The original exception to persist. - context: Additional keyword arguments forwarded to ``persist_app_error``. - """ - app_error = persist_app_error(exception, **context) - raise AlreadyLoggedException(exception, app_error.id) diff --git a/apps/workflow/services/instance_onboarding.py b/apps/workflow/services/instance_onboarding.py new file mode 100644 index 000000000..c24f056ba --- /dev/null +++ b/apps/workflow/services/instance_onboarding.py @@ -0,0 +1,108 @@ +"""Finalise a freshly created instance after its Xero OAuth connection exists.""" + +from django.core.management import call_command + +from apps.accounts.models import Staff +from apps.job.models import Job +from apps.timesheet.services import PayrollEmployeeSyncService +from apps.workflow.api.xero.auth import get_valid_token +from apps.workflow.api.xero.payroll import sync_xero_pay_items +from apps.workflow.api.xero.sync import one_way_sync_all_xero_data +from apps.workflow.models import CompanyDefaults, XeroAccount + +CANONICAL_SHOP_JOB_NAMES = ( + "Annual Leave", + "Bench - busy work", + "Bereavement Leave", + "Business Development", + "Office Admin", + "Sick Leave", + "Training", + "Travel", + "Worker Admin", +) + + +def _sync_accounts() -> None: + errors = [ + event.get("message", "Unknown account sync error") + for event in one_way_sync_all_xero_data(entities=["accounts"], force=True) + if event.get("severity") == "error" + ] + if errors: + raise RuntimeError("Xero account sync failed: " + "; ".join(errors)) + if not XeroAccount.objects.exists(): + raise RuntimeError( + "Xero account sync completed without importing any accounts." + ) + + +def _sync_staff(*, seed_xero: bool) -> None: + if seed_xero: + staff = Staff.objects.filter(base_wage_rate__gt=0) + summary = PayrollEmployeeSyncService.sync_staff( + staff, + dry_run=False, + allow_create=True, + ) + if summary["missing"]: + raise RuntimeError("Demo staff could not all be linked to Xero Payroll.") + else: + summary = PayrollEmployeeSyncService.import_staff_from_xero( + dry_run=False, + initial_password="Default-staff-password", + ) + if summary["errors"]: + messages = [item["reason"] for item in summary["errors"]] + raise RuntimeError("Xero staff import failed: " + "; ".join(messages)) + + wage_staff = Staff.objects.filter(base_wage_rate__gt=0) + if not wage_staff.exists(): + raise RuntimeError("No wage-earning staff were configured during onboarding.") + staff_without_xero = wage_staff.filter( + xero_user_id__isnull=True, + ) + if staff_without_xero.exists(): + raise RuntimeError("One or more wage-earning staff are not linked to Xero.") + + +def _validate_completion() -> CompanyDefaults: + company = CompanyDefaults.get_solo() + required_xero_values = { + "xero_tenant_id": company.xero_tenant_id, + "xero_shortcode": company.xero_shortcode, + "xero_sales_branding_theme_id": company.xero_sales_branding_theme_id, + "xero_payroll_calendar_id": company.xero_payroll_calendar_id, + } + missing = [name for name, value in required_xero_values.items() if not value] + if missing: + raise RuntimeError( + "Xero onboarding left required CompanyDefaults unset: " + ", ".join(missing) + ) + shop_job_count = Job.objects.filter( + company=company.shop_company, + status="special", + name__in=CANONICAL_SHOP_JOB_NAMES, + ).count() + if shop_job_count != 9: + raise RuntimeError(f"Expected 9 canonical shop jobs, found {shop_job_count}.") + return company + + +def finalize_instance_onboarding(*, seed_xero: bool = False) -> None: + """Complete Xero-dependent setup and enable automated sync last.""" + CompanyDefaults.set_xero_sync_enabled(enabled=False) + + if not get_valid_token(): + raise RuntimeError("Complete Xero OAuth before finalising instance onboarding.") + xero_setup_args = ["--setup"] + if seed_xero: + xero_setup_args.append("--seed-xero") + call_command("xero", *xero_setup_args) + sync_xero_pay_items() + _sync_accounts() + _sync_staff(seed_xero=seed_xero) + call_command("create_shop_jobs") + + _validate_completion() + CompanyDefaults.set_xero_sync_enabled(enabled=True) diff --git a/apps/workflow/services/llm_service.py b/apps/workflow/services/llm_service.py index e8707feaf..6b2bbcd56 100644 --- a/apps/workflow/services/llm_service.py +++ b/apps/workflow/services/llm_service.py @@ -220,7 +220,7 @@ def _parse_json_response(self, text: str) -> dict: except json.JSONDecodeError as e: logger.error(f"Failed to parse JSON response: {e}") logger.debug(f"Raw response: {text[:500]}") - raise ValueError(f"LLM returned invalid JSON: {e}") + raise ValueError(f"LLM returned invalid JSON: {e}") from e def get_text_response( self, diff --git a/apps/workflow/services/validation.py b/apps/workflow/services/validation.py index 5467849bb..8afe91ee1 100644 --- a/apps/workflow/services/validation.py +++ b/apps/workflow/services/validation.py @@ -14,8 +14,8 @@ def to_decimal(value, *, field_label: str) -> Decimal: """ try: d = Decimal(str(value)) - except (InvalidOperation, TypeError): - raise ValueError(f"Invalid decimal format for {field_label}.") + except (InvalidOperation, TypeError) as exc: + raise ValueError(f"Invalid decimal format for {field_label}.") from exc if d < 0: raise ValueError(f"Negative value not allowed for {field_label}.") return d diff --git a/apps/workflow/services/xero_sync_service.py b/apps/workflow/services/xero_sync_service.py index 715cf0c57..067e88344 100644 --- a/apps/workflow/services/xero_sync_service.py +++ b/apps/workflow/services/xero_sync_service.py @@ -109,7 +109,7 @@ def start_sync() -> XeroSyncStartResult: except Exception: # Broker unavailable — release the lock so the next attempt can # try. Don't persist here; the caller (Beat task or view) owns - # the AlreadyLoggedException pattern. + # error persistence. _sync_cache.delete(SYNC_STATUS_KEY) raise diff --git a/apps/workflow/services/xero_sync_worker.py b/apps/workflow/services/xero_sync_worker.py index cd6c726c1..56696acd0 100644 --- a/apps/workflow/services/xero_sync_worker.py +++ b/apps/workflow/services/xero_sync_worker.py @@ -14,7 +14,8 @@ from django.core.cache import caches from django.utils import timezone -from apps.workflow.exceptions import AlreadyLoggedException, XeroQuotaFloorReached +from apps.workflow.accounting.registry import is_accounting_enabled +from apps.workflow.exceptions import XeroQuotaFloorReached from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.xero_sync_constants import SYNC_STATUS_KEY @@ -87,6 +88,23 @@ def xero_sync_task(task_id: str) -> None: overall_key = f"xero_sync_overall_progress_{task_id}" try: + if not is_accounting_enabled(): + msgs = _sync_cache.get(messages_key, []) + msgs.append( + { + "datetime": timezone.now().isoformat(), + "entity": "sync", + "severity": "warning", + "message": "Xero sync skipped: enable_xero_sync is False", + "progress": None, + "task_id": task_id, + "sync_status": "aborted", + } + ) + _sync_cache.set(messages_key, msgs, timeout=86400) + scheduler_logger.info("Xero sync task %s skipped: sync disabled", task_id) + return + provider = get_provider() msgs = _sync_cache.get(messages_key, []) processed = 0 @@ -150,17 +168,6 @@ def xero_sync_task(task_id: str) -> None: # "the task ran and decided to abort cleanly". except Exception as exc: - if isinstance(exc, AlreadyLoggedException): - app_error_id = exc.app_error_id - _append_sync_failure_messages( - messages_key=messages_key, - task_id=task_id, - message=f"Error during sync: {exc}", - sync_status="error", - app_error_id=app_error_id, - ) - raise - err = persist_app_error(exc) _append_sync_failure_messages( messages_key=messages_key, @@ -169,7 +176,7 @@ def xero_sync_task(task_id: str) -> None: sync_status="error", app_error_id=str(err.id), ) - raise AlreadyLoggedException(exc, err.id) from exc + raise finally: _sync_cache.delete(current_key) diff --git a/apps/workflow/tasks.py b/apps/workflow/tasks.py index d325869d8..ca15f24bd 100644 --- a/apps/workflow/tasks.py +++ b/apps/workflow/tasks.py @@ -12,11 +12,9 @@ from django.conf import settings from django.db import close_old_connections +from apps.workflow.accounting.registry import is_accounting_enabled from apps.workflow.api.xero.client import quota_floor_breached -from apps.workflow.api.xero.sync import sync_single_contact, sync_single_invoice -from apps.workflow.exceptions import ( - AlreadyLoggedException, -) +from apps.workflow.api.xero.seed import sync_single_contact, sync_single_invoice from apps.workflow.models import CompanyDefaults from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.xero_sync_service import XeroSyncService @@ -55,6 +53,9 @@ def process_xero_webhook_event(tenant_id: str, event: Dict[str, Any]) -> None: from process state. Write-side: callers do not read a return value. """ company_defaults = CompanyDefaults.get_solo() + if not company_defaults.enable_xero_sync: + return + if quota_floor_breached(company_defaults.xero_automated_day_floor): logger.warning( "Xero day quota at floor (%s) — skipping webhook event %s", @@ -79,9 +80,6 @@ def process_xero_webhook_event(tenant_id: str, event: Dict[str, Any]) -> None: ) return - if not company_defaults.enable_xero_sync: - return - try: sync_service = XeroSyncService(tenant_id=tenant_id) if event_category == "CONTACT": @@ -92,11 +90,9 @@ def process_xero_webhook_event(tenant_id: str, event: Dict[str, Any]) -> None: sync_single_invoice(sync_service, resource_id) else: logger.warning("Unknown webhook event category: %s", event_category) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise @shared_task(name="apps.workflow.tasks.xero_heartbeat_task") @@ -113,14 +109,12 @@ def xero_heartbeat_task() -> None: scheduler_logger.info("Xero API token refreshed successfully.") else: scheduler_logger.error("No Xero token available to refresh.") - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during Xero Heartbeat task: %s", exc, exc_info=True ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @shared_task(name="apps.workflow.tasks.xero_regular_sync_task") @@ -133,6 +127,11 @@ def xero_regular_sync_task() -> None: scheduler_logger.info("Running Xero Regular Sync task.") try: close_old_connections() + if not is_accounting_enabled(): + scheduler_logger.info( + "Xero regular sync skipped: enable_xero_sync is False" + ) + return result = XeroSyncService.start_sync() if result.reason == "already_running": scheduler_logger.info( @@ -147,14 +146,12 @@ def xero_regular_sync_task() -> None: scheduler_logger.info( "Xero regular sync dispatched (task_id=%s)", result.task_id ) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during Xero Regular Sync task: %s", exc, exc_info=True ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @shared_task(name="apps.workflow.tasks.xero_30_day_sync_task") @@ -165,6 +162,9 @@ def xero_30_day_sync_task() -> None: scheduler_logger.info("Running Xero 30-Day Sync task.") try: close_old_connections() + if not is_accounting_enabled(): + scheduler_logger.info("Xero 30-day sync skipped: enable_xero_sync is False") + return result = XeroSyncService.start_sync() if result.reason == "already_running": scheduler_logger.info( @@ -179,14 +179,12 @@ def xero_30_day_sync_task() -> None: scheduler_logger.info( "Xero 30-day sync dispatched (task_id=%s)", result.task_id ) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during Xero 30-Day Sync task: %s", exc, exc_info=True ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise @shared_task(name="apps.workflow.tasks.purge_old_session_replays_task") @@ -204,11 +202,9 @@ def purge_old_session_replays_task() -> None: deleted, retention_days, ) - except AlreadyLoggedException: - raise except Exception as exc: scheduler_logger.error( "Error during session replay purge: %s", exc, exc_info=True ) - app_error = persist_app_error(exc) - raise AlreadyLoggedException(exc, app_error.id) from exc + persist_app_error(exc) + raise diff --git a/apps/workflow/tests/test_api_schema_coverage.py b/apps/workflow/tests/test_api_schema_coverage.py index 301c516f3..9b82dc953 100644 --- a/apps/workflow/tests/test_api_schema_coverage.py +++ b/apps/workflow/tests/test_api_schema_coverage.py @@ -31,8 +31,6 @@ "api/xero/webhook/", # AWS instance management (internal ops, not frontend) "api/aws/", - # Enum endpoint (internal, values embedded in schema) - "api/enums/", # DRF router roots (meta-endpoints listing sub-routes, not real APIs) "api/workflow/", "api/companies/", diff --git a/apps/workflow/tests/test_backup_scripts.py b/apps/workflow/tests/test_backup_scripts.py index ab60f6d5c..72f885eaf 100644 --- a/apps/workflow/tests/test_backup_scripts.py +++ b/apps/workflow/tests/test_backup_scripts.py @@ -8,10 +8,11 @@ from unittest import mock from django.core.management import call_command -from django.test import SimpleTestCase +from django.test import SimpleTestCase, TestCase -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.management.commands.backport_data_backup import Command +from apps.workflow.models import AppError +from apps.workflow.services.error_persistence import persist_app_error REPO_ROOT = Path(__file__).resolve().parents[3] CLEANUP_BACKUPS = REPO_ROOT / "scripts" / "cleanup_backups.py" @@ -132,7 +133,7 @@ def test_prod_pull_success_preserves_verified_archive_and_cleans_remote( [event.split(":", 1)[0] for event in events], ["ssh", "scp", "verify", "sha256sum", "ssh"], ) - self.assertIn("--allow-legacy-client-baseline", events[2]) + self.assertNotIn("--allow-legacy-client-baseline", events[2]) self.assertIn("backport_data_backup", events[0]) self.assertIn("rm -f", events[-1]) @@ -340,10 +341,12 @@ def test_file_backup_script_is_incremental_and_scoped(self) -> None: self.assertIn("refusing to back up symlinked directory", content) -class BackportCommandErrorPersistenceTests(SimpleTestCase): +class BackportCommandErrorPersistenceTests(TestCase): def test_prelogged_scrub_failure_is_not_persisted_again(self) -> None: command = Command() - failure = AlreadyLoggedException(RuntimeError("scrub failed"), "error-123") + failure = RuntimeError("scrub failed") + persist_app_error(failure) + before = AppError.objects.count() with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "backup.dump" with ( @@ -353,17 +356,14 @@ def test_prelogged_scrub_failure_is_not_persisted_again(self) -> None: "apps.workflow.management.commands.backport_data_backup.db_scrubber.scrub", side_effect=failure, ), - mock.patch( - "apps.workflow.management.commands.backport_data_backup.persist_app_error" - ) as persist_app_error, - self.assertRaises(AlreadyLoggedException) as raised, + self.assertRaises(RuntimeError) as raised, ): call_command(command, output=str(output)) self.assertIs(raised.exception, failure) - persist_app_error.assert_not_called() + self.assertEqual(AppError.objects.count(), before) - def test_new_command_failure_is_persisted_and_wrapped(self) -> None: + def test_new_command_failure_is_persisted_once_and_reraised(self) -> None: command = Command() failure = RuntimeError("backup failed") with tempfile.TemporaryDirectory() as tmp: @@ -372,12 +372,10 @@ def test_new_command_failure_is_persisted_and_wrapped(self) -> None: mock.patch.object(command, "_run", side_effect=failure), mock.patch( "apps.workflow.management.commands.backport_data_backup.persist_app_error" - ) as persist_app_error, - self.assertRaises(AlreadyLoggedException) as raised, + ) as persist_app_error_mock, + self.assertRaises(RuntimeError) as raised, ): - persist_app_error.return_value.id = "error-456" call_command(command, output=str(output)) - self.assertIs(raised.exception.original, failure) - self.assertEqual(raised.exception.app_error_id, "error-456") - persist_app_error.assert_called_once_with(failure) + self.assertIs(raised.exception, failure) + persist_app_error_mock.assert_called_once_with(failure) diff --git a/apps/workflow/tests/test_check_ai_providers.py b/apps/workflow/tests/test_check_ai_providers.py new file mode 100644 index 000000000..ea876e25f --- /dev/null +++ b/apps/workflow/tests/test_check_ai_providers.py @@ -0,0 +1,55 @@ +import importlib.util +import types +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock, patch + +from django.test import TestCase + +from apps.workflow.enums import AIProviderTypes +from apps.workflow.models import AIProvider + +SCRIPT = ( + Path(__file__).resolve().parents[3] + / "scripts" + / "restore_checks" + / "check_ai_providers.py" +) + + +def load_check_ai_providers() -> types.ModuleType: + spec = importlib.util.spec_from_file_location("check_ai_providers", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError("Could not load check_ai_providers.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class CheckAIProvidersTests(TestCase): + check: ClassVar[types.ModuleType] + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.check = load_check_ai_providers() + + def test_missing_provider_fails_validation(self) -> None: + with self.assertRaisesRegex(RuntimeError, "Claude: Not configured"): + self.check.validate_chat_provider(AIProviderTypes.ANTHROPIC) + + def test_provider_error_propagates(self) -> None: + AIProvider.objects.create( + name="Claude", + provider_type=AIProviderTypes.ANTHROPIC, + api_key="test-key", + model_name="test-model", + ) + service = MagicMock() + service.get_text_response.side_effect = RuntimeError("provider unavailable") + + with ( + patch.object(self.check, "LLMService", return_value=service), + self.assertRaisesRegex(RuntimeError, "provider unavailable"), + ): + self.check.validate_chat_provider(AIProviderTypes.ANTHROPIC) diff --git a/apps/workflow/tests/test_company_defaults_api.py b/apps/workflow/tests/test_company_defaults_api.py index 08970a506..c3ad6c02c 100644 --- a/apps/workflow/tests/test_company_defaults_api.py +++ b/apps/workflow/tests/test_company_defaults_api.py @@ -79,3 +79,79 @@ def test_patch_persists_and_clears_xero_sales_branding_theme(self) -> None: self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertIsNone(payload["xero_sales_branding_theme_id"]) + + def test_patch_persists_multiline_xero_quote_terms_exactly(self) -> None: + terms = "First line\n\n Indented final line " + + response = self.client.patch( + "/api/company-defaults/", + {"xero_quote_terms": terms}, + format="json", + ) + payload = response.json() + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(payload["xero_quote_terms"], terms) + self.assertEqual(CompanyDefaults.get_solo().xero_quote_terms, terms) + + def test_patch_of_other_xero_field_round_trips_existing_terms(self) -> None: + """The settings form PATCHes every field in a section, terms included. + + A round-trip of the stored terms alongside another field must not be + mistaken for an attempt to clear them, or no Xero setting is editable. + """ + terms = CompanyDefaults.get_solo().xero_quote_terms + self.assertTrue(terms) + + response = self.client.patch( + "/api/company-defaults/", + {"xero_quote_terms": terms, "xero_shortcode": "ABC123"}, + format="json", + ) + payload = response.json() + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(payload["xero_shortcode"], "ABC123") + self.assertEqual(payload["xero_quote_terms"], terms) + + def test_patch_rejects_blank_xero_quote_terms(self) -> None: + response = self.client.patch( + "/api/company-defaults/", + {"xero_quote_terms": " \n "}, + format="json", + ) + payload = response.json() + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + payload["xero_quote_terms"], + ["Xero quote terms must not be blank."], + ) + + def test_patch_rejects_null_xero_quote_terms(self) -> None: + response = self.client.patch( + "/api/company-defaults/", + {"xero_quote_terms": None}, + format="json", + ) + payload = response.json() + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + payload["xero_quote_terms"], + ["This field may not be null."], + ) + + def test_patch_rejects_xero_quote_terms_over_4000_characters(self) -> None: + response = self.client.patch( + "/api/company-defaults/", + {"xero_quote_terms": "x" * 4001}, + format="json", + ) + payload = response.json() + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + payload["xero_quote_terms"], + ["Ensure this field has no more than 4000 characters."], + ) diff --git a/apps/workflow/tests/test_company_defaults_schema.py b/apps/workflow/tests/test_company_defaults_schema.py index 79b510188..a85f4c5c9 100644 --- a/apps/workflow/tests/test_company_defaults_schema.py +++ b/apps/workflow/tests/test_company_defaults_schema.py @@ -242,4 +242,14 @@ def test_xero_section_exposes_sales_branding_theme_selector(self) -> None: self.assertEqual(theme_field["type"], "xero_branding_theme") self.assertEqual(theme_field["label"], "Xero Sales Branding Theme") self.assertFalse(theme_field["read_only"]) - self.assertIn("terms and conditions", theme_field["help_text"]) + self.assertIn("layout and presentation", theme_field["help_text"]) + + terms_field = next( + field + for field in xero_section["fields"] + if field["key"] == "xero_quote_terms" + ) + self.assertEqual(terms_field["type"], "textarea") + self.assertEqual(terms_field["label"], "Xero Quote Terms") + self.assertFalse(terms_field["read_only"]) + self.assertIn("Terms (Quotes)", terms_field["help_text"]) diff --git a/apps/workflow/tests/test_db_scrubber.py b/apps/workflow/tests/test_db_scrubber.py index e7514cd67..cfe747129 100644 --- a/apps/workflow/tests/test_db_scrubber.py +++ b/apps/workflow/tests/test_db_scrubber.py @@ -7,16 +7,17 @@ from unittest.mock import MagicMock, patch -from django.test import SimpleTestCase +from django.test import SimpleTestCase, TestCase from apps.company.models import ContactMethod -from apps.workflow.exceptions import AlreadyLoggedException +from apps.workflow.models import AppError from apps.workflow.services.db_scrubber import ( _PRIVATE_CONFIG_TABLES, _assert_private_config_removed, _unique_scrub_value, scrub, ) +from apps.workflow.services.error_persistence import persist_app_error PHONE = ContactMethod.MethodType.PHONE @@ -88,45 +89,43 @@ def test_private_config_postcondition_reports_counts_not_values( self.assertNotIn("api_key", str(raised.exception)) -class ScrubErrorPersistenceTests(SimpleTestCase): +class ScrubErrorPersistenceTests(TestCase): @patch("apps.workflow.services.db_scrubber._assert_scrub_alias_is_safe") @patch("apps.workflow.services.db_scrubber.transaction.atomic") @patch("apps.workflow.services.db_scrubber._scrub_staff") @patch("apps.workflow.services.db_scrubber.persist_app_error") - def test_new_failure_is_persisted_and_wrapped_once( + def test_new_failure_is_persisted_once_and_reraised( self, - persist_app_error: MagicMock, + persist_app_error_mock: MagicMock, scrub_staff: MagicMock, _atomic: MagicMock, _assert_safe: MagicMock, ) -> None: failure = RuntimeError("scrub failed") scrub_staff.side_effect = failure - persist_app_error.return_value.id = "error-123" - with self.assertRaises(AlreadyLoggedException) as raised: + with self.assertRaises(RuntimeError) as raised: scrub() - self.assertIs(raised.exception.original, failure) - self.assertEqual(raised.exception.app_error_id, "error-123") - persist_app_error.assert_called_once_with(failure) + self.assertIs(raised.exception, failure) + persist_app_error_mock.assert_called_once_with(failure) @patch("apps.workflow.services.db_scrubber._assert_scrub_alias_is_safe") @patch("apps.workflow.services.db_scrubber.transaction.atomic") @patch("apps.workflow.services.db_scrubber._scrub_staff") - @patch("apps.workflow.services.db_scrubber.persist_app_error") - def test_prelogged_failure_passes_through_unchanged( + def test_prelogged_failure_gets_no_second_app_error( self, - persist_app_error: MagicMock, scrub_staff: MagicMock, _atomic: MagicMock, _assert_safe: MagicMock, ) -> None: - failure = AlreadyLoggedException(RuntimeError("scrub failed"), "error-123") + failure = RuntimeError("scrub failed") + persist_app_error(failure) scrub_staff.side_effect = failure + before = AppError.objects.count() - with self.assertRaises(AlreadyLoggedException) as raised: + with self.assertRaises(RuntimeError) as raised: scrub() self.assertIs(raised.exception, failure) - persist_app_error.assert_not_called() + self.assertEqual(AppError.objects.count(), before) diff --git a/apps/workflow/tests/test_e2e_artifacts.py b/apps/workflow/tests/test_e2e_artifacts.py new file mode 100644 index 000000000..ba68860ba --- /dev/null +++ b/apps/workflow/tests/test_e2e_artifacts.py @@ -0,0 +1,152 @@ +"""Tests for ignoring Xero objects created by finished E2E runs. + +E2E runs write real Contacts, Invoices and Quotes to a development Xero org. +Those survive the post-run database restore, and the hourly Xero sync would +otherwise replay them into the clean database. These tests pin the two +conditions that together decide the skip, and the gate that stops it ever +running outside dev. +""" + +import json +from datetime import datetime, timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import TestCase, override_settings +from django.utils import timezone + +from apps.workflow.services.e2e_artifacts import ( + TEST_COMPANY_NAME, + TEST_DATA_PREFIX, + drop_e2e_artifacts, +) + + +def _contact(name: str, updated_at: datetime) -> SimpleNamespace: + """An inbound Xero contact: carries its own company name.""" + return SimpleNamespace(name=name, updated_date_utc=updated_at) + + +def _invoice(contact_name: str, updated_at: datetime) -> SimpleNamespace: + """An inbound Xero document: carries its company via an embedded contact.""" + return SimpleNamespace( + contact=SimpleNamespace(name=contact_name), + updated_date_utc=updated_at, + ) + + +class E2EWindowFileTestCase(TestCase): + """Base that points the module at a temporary windows file.""" + + def setUp(self) -> None: + self.now = timezone.now() + self.run_start = self.now - timedelta(minutes=30) + self.run_end = self.now - timedelta(minutes=5) + self.during_run = self.now - timedelta(minutes=20) + + self._tmp = TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.windows_file = Path(self._tmp.name) / "sync-windows.json" + + patcher = patch( + "apps.workflow.services.e2e_artifacts.E2E_SYNC_WINDOWS_FILE", + self.windows_file, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def write_window(self, *, ended: bool) -> None: + self.windows_file.write_text( + json.dumps( + [ + { + "run_id": "testrun1", + "started_at": self.run_start.isoformat(), + "ended_at": self.run_end.isoformat() if ended else None, + } + ] + ) + ) + + +class DropE2EArtifactsTests(E2EWindowFileTestCase): + def test_test_company_inside_closed_window_is_dropped(self) -> None: + self.write_window(ended=True) + item = _contact(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), []) + + @override_settings(PRODUCTION_LIKE=True) + def test_production_like_never_drops_anything(self) -> None: + """Every server is production_like; only DJANGO_ENV=local is not. + + Discarding inbound Xero data is only ever correct against a development + org, so a stray windows file must not be enough to cause it. + """ + self.write_window(ended=True) + item = _contact(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), [item]) + + def test_fixture_company_inside_closed_window_is_dropped(self) -> None: + """The standing fixture company is test data too. + + Test jobs hang off it, so the invoices and quotes a run raises are its + documents, not those of a [TEST]-prefixed company. + """ + self.write_window(ended=True) + item = _contact(TEST_COMPANY_NAME, self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), []) + + def test_ordinary_company_inside_closed_window_is_kept(self) -> None: + """The window alone must never suppress; this is the over-reach guard. + + A real Xero edit landing inside a run's window would otherwise be + discarded, and because the sync cursor still advances it would never be + fetched again. + """ + self.write_window(ended=True) + item = _contact("Morris Sheetmetal", self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), [item]) + + def test_test_company_outside_any_window_is_kept(self) -> None: + self.write_window(ended=True) + item = _contact(f"{TEST_DATA_PREFIX} Company 123", self.now) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), [item]) + + def test_open_window_suppresses_nothing(self) -> None: + """The mid-run guarantee. + + While a run executes, inbound Xero data must behave exactly as it does + in production — that round trip is what the run exercises. Fails if the + ended_at condition is ever dropped. + """ + self.write_window(ended=False) + item = _contact(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), [item]) + + def test_document_is_dropped_with_its_contact(self) -> None: + """A suppressed contact's documents must go with it. + + The invoice, quote and purchase-order importers resolve their contact + through resolve_company_from_xero_contact, which raises rather than + skips when the company cannot be synced. Leaving the document behind + would abort the whole sync run. + """ + self.write_window(ended=True) + test_contact = _contact(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + its_invoice = _invoice(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + + self.assertEqual(drop_e2e_artifacts([test_contact, its_invoice], "mixed"), []) + + def test_absent_windows_file_suppresses_nothing(self) -> None: + """The ordinary state of any machine that has never run E2E.""" + item = _contact(f"{TEST_DATA_PREFIX} Company 123", self.during_run) + + self.assertEqual(drop_e2e_artifacts([item], "contacts"), [item]) diff --git a/apps/workflow/tests/test_error_persistence.py b/apps/workflow/tests/test_error_persistence.py new file mode 100644 index 000000000..8cdf12e7e --- /dev/null +++ b/apps/workflow/tests/test_error_persistence.py @@ -0,0 +1,119 @@ +"""Unit tests for idempotent error persistence. + +The system guarantee is one AppError row per failure. That guarantee is +enforced by persist_app_error itself — it marks the exception it persists +and returns the existing row on any subsequent call — rather than by every +handler in the codebase remembering a two-arm ritual. + +The propagation test is the regression that motivates the design: an +exception persisted deep in a service and re-raised through intermediate +handlers up to the DRF boundary must still produce exactly one row. +""" + +from apps.testing import BaseTestCase +from apps.workflow.models import AppError +from apps.workflow.services.error_persistence import ( + app_error_for, + persist_app_error, +) + + +class PersistAppErrorIdempotencyTests(BaseTestCase): + def test_persisting_twice_creates_one_row(self) -> None: + exc = ValueError("boom") + + first = persist_app_error(exc) + second = persist_app_error(exc) + + self.assertEqual(AppError.objects.count(), 1) + self.assertEqual(first.id, second.id) + + def test_distinct_exceptions_each_get_a_row(self) -> None: + persist_app_error(ValueError("one")) + persist_app_error(ValueError("two")) + + self.assertEqual(AppError.objects.count(), 2) + + def test_survives_propagation_through_nested_handlers(self) -> None: + """A failure persisted in a service and re-raised through two + intermediate handlers to the boundary yields one row, not three.""" + + def service() -> None: + raise ValueError("deep failure") + + def middle() -> None: + try: + service() + except Exception as exc: + persist_app_error(exc) + raise + + def boundary() -> None: + try: + middle() + except Exception as exc: + persist_app_error(exc) + raise + + with self.assertRaises(ValueError): + try: + boundary() + except Exception as exc: + persist_app_error(exc) + raise + + self.assertEqual(AppError.objects.count(), 1) + + def test_conversion_with_from_exc_does_not_earn_a_second_row(self) -> None: + """A handler that converts the exception type still represents one + failure. `raise ... from exc` links them, so persistence follows the + cause chain rather than counting objects.""" + original = KeyError("missing") + persist_app_error(original) + + try: + raise ValueError("Job not found") from original + except ValueError as converted: + persist_app_error(converted) + + self.assertEqual(AppError.objects.count(), 1) + + def test_unchained_conversion_is_a_separate_failure(self) -> None: + """Without `from exc` there is no link, so the new exception is a + genuinely separate record — the cost of severing the chain.""" + persist_app_error(KeyError("missing")) + + try: + raise ValueError("Job not found") + except ValueError as converted: + persist_app_error(converted) + + self.assertEqual(AppError.objects.count(), 2) + + def test_context_from_the_first_persist_is_kept(self) -> None: + """The innermost handler has the richest context (it knows the job), + so the first write wins and later calls must not overwrite it.""" + exc = ValueError("boom") + + persist_app_error(exc, app="inner", additional_context={"detail": "rich"}) + persist_app_error(exc, app="outer") + + stored = AppError.objects.get() + self.assertEqual(stored.app, "inner") + stored_data = stored.data + assert stored_data is not None # narrows for mypy + self.assertEqual(stored_data["detail"], "rich") + + +class AppErrorForTests(BaseTestCase): + def test_returns_the_row_for_a_persisted_exception(self) -> None: + exc = ValueError("boom") + err = persist_app_error(exc) + + found = app_error_for(exc) + self.assertIsNotNone(found) + assert found is not None # narrows for mypy + self.assertEqual(found.id, err.id) + + def test_returns_none_for_an_unpersisted_exception(self) -> None: + self.assertIsNone(app_error_for(ValueError("never persisted"))) diff --git a/apps/workflow/tests/test_instance_onboarding.py b/apps/workflow/tests/test_instance_onboarding.py new file mode 100644 index 000000000..a70dfa2e5 --- /dev/null +++ b/apps/workflow/tests/test_instance_onboarding.py @@ -0,0 +1,76 @@ +from unittest.mock import Mock, patch + +from django.core.management import call_command + +from apps.job.models import Job +from apps.testing import BaseTestCase +from apps.workflow.models import CompanyDefaults +from apps.workflow.services.instance_onboarding import finalize_instance_onboarding + + +class FinalizeInstanceOnboardingTests(BaseTestCase): + @patch("apps.workflow.services.instance_onboarding._validate_completion") + @patch("apps.workflow.services.instance_onboarding._sync_staff") + @patch("apps.workflow.services.instance_onboarding._sync_accounts") + @patch("apps.workflow.services.instance_onboarding.sync_xero_pay_items") + @patch("apps.workflow.services.instance_onboarding.call_command") + @patch( + "apps.workflow.services.instance_onboarding.get_valid_token", + return_value={"access_token": "token"}, + ) + def test_enables_sync_only_after_every_onboarding_step_succeeds( + self, + _mock_token: Mock, + _mock_call_command: Mock, + _mock_pay_items: Mock, + _mock_accounts: Mock, + _mock_staff: Mock, + mock_validate: Mock, + ) -> None: + company = CompanyDefaults.get_solo() + company.enable_xero_sync = False + company.save(update_fields=["enable_xero_sync"]) + mock_validate.return_value = company + + finalize_instance_onboarding(seed_xero=True) + + company.refresh_from_db() + self.assertTrue(company.enable_xero_sync) + _mock_call_command.assert_any_call("xero", "--setup", "--seed-xero") + _mock_staff.assert_called_once_with(seed_xero=True) + + @patch("apps.workflow.services.instance_onboarding._sync_accounts") + @patch("apps.workflow.services.instance_onboarding.sync_xero_pay_items") + @patch("apps.workflow.services.instance_onboarding.call_command") + @patch( + "apps.workflow.services.instance_onboarding.get_valid_token", + return_value={"access_token": "token"}, + ) + def test_failure_leaves_sync_disabled( + self, + _mock_token: Mock, + _mock_call_command: Mock, + _mock_pay_items: Mock, + mock_accounts: Mock, + ) -> None: + company = CompanyDefaults.get_solo() + company.enable_xero_sync = True + company.save(update_fields=["enable_xero_sync"]) + mock_accounts.side_effect = RuntimeError("account sync failed") + + with self.assertRaisesRegex(RuntimeError, "account sync failed"): + finalize_instance_onboarding() + + company.refresh_from_db() + self.assertFalse(company.enable_xero_sync) + + +class CreateShopJobsTests(BaseTestCase): + def test_command_is_idempotent(self) -> None: + call_command("create_shop_jobs", verbosity=0) + call_command("create_shop_jobs", verbosity=0) + + company = CompanyDefaults.get_solo() + jobs = Job.objects.filter(company=company.shop_company, status="special") + self.assertEqual(jobs.count(), 9) + self.assertEqual(jobs.filter(name="Training").count(), 1) diff --git a/apps/workflow/tests/test_latest_gemini_model_migration.py b/apps/workflow/tests/test_latest_gemini_model_migration.py new file mode 100644 index 000000000..6e4d01af0 --- /dev/null +++ b/apps/workflow/tests/test_latest_gemini_model_migration.py @@ -0,0 +1,61 @@ +from typing import ClassVar + +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.test import TransactionTestCase + + +class LatestGeminiModelMigrationTests(TransactionTestCase): + migrate_from: ClassVar[tuple[tuple[str, str], ...]] = ( + ("workflow", "0011_companydefaults_xero_sales_branding_theme_id"), + ) + migrate_to: ClassVar[tuple[tuple[str, str], ...]] = ( + ("workflow", "0012_use_latest_gemini_flash_model"), + ) + + def setUp(self) -> None: + super().setUp() + self.executor = MigrationExecutor(connection) + self.executor.migrate(self.migrate_from) + self.old_apps = self.executor.loader.project_state(self.migrate_from).apps + + def tearDown(self) -> None: + self.executor.loader.build_graph() + self.executor.migrate(self.executor.loader.graph.leaf_nodes()) + super().tearDown() + + def test_deprecated_gemini_models_move_to_rolling_alias(self) -> None: + AIProvider = self.old_apps.get_model("workflow", "AIProvider") + obsolete_stable = AIProvider.objects.create( + name="Gemini Stable", + provider_type="Gemini", + model_name="gemini-2.5-flash", + ) + obsolete_preview = AIProvider.objects.create( + name="Gemini Preview", + provider_type="Gemini", + model_name="gemini-2.0-flash-exp", + ) + explicit_alias = AIProvider.objects.create( + name="Gemini Pro", + provider_type="Gemini", + model_name="gemini-pro-latest", + ) + + self.executor.loader.build_graph() + self.executor.migrate(self.migrate_to) + new_apps = self.executor.loader.project_state(self.migrate_to).apps + AIProvider = new_apps.get_model("workflow", "AIProvider") + + self.assertEqual( + AIProvider.objects.get(pk=obsolete_stable.pk).model_name, + "gemini-flash-latest", + ) + self.assertEqual( + AIProvider.objects.get(pk=obsolete_preview.pk).model_name, + "gemini-flash-latest", + ) + self.assertEqual( + AIProvider.objects.get(pk=explicit_alias.pk).model_name, + "gemini-pro-latest", + ) diff --git a/apps/workflow/tests/test_localdate_regression.py b/apps/workflow/tests/test_localdate_regression.py index 73670ce23..5e1d43d64 100644 --- a/apps/workflow/tests/test_localdate_regression.py +++ b/apps/workflow/tests/test_localdate_regression.py @@ -45,8 +45,9 @@ class FreezeTimeSanityTests(TestCase): def test_utc_and_nz_disagree_on_the_chosen_moment(self): with freeze_time(FROZEN_UTC_MOMENT): - # noqa: localdate intentional — this assertion exists to prove the - # UTC/NZ-disagree premise that every other test in this file relies on. + # This assertion exists to prove the UTC/NZ-disagree premise that + # every other test in this file relies on. The suppression that + # actually silences the checker is on the assertEqual line below. self.assertEqual( timezone.now().date(), UTC_DATE ) # noqa: localdate test fixture asserts UTC date deliberately @@ -334,7 +335,8 @@ def test_build_payload_uses_nz_local_date(self): patch.object(manager, "get_line_items", return_value=[]), ): payload = manager.build_payload( - document_theme_external_id=DOCUMENT_THEME_ID + document_theme_external_id=DOCUMENT_THEME_ID, + terms="Client-approved quote terms", ) self.assertEqual(payload.date, NZ_DATE) diff --git a/apps/workflow/tests/test_notebook_lm_link_api.py b/apps/workflow/tests/test_notebook_lm_link_api.py new file mode 100644 index 000000000..1aa6b81c7 --- /dev/null +++ b/apps/workflow/tests/test_notebook_lm_link_api.py @@ -0,0 +1,86 @@ +"""Tests for /api/workflow/notebook-lm-links/ — menu filtering + CRUD permissions.""" + +from rest_framework import status +from rest_framework.test import APIClient, APITestCase + +from apps.accounts.models import Staff +from apps.workflow.enums import NotebookLmRestriction +from apps.workflow.models import NotebookLmLink + +LIST_URL = "/api/workflow/notebook-lm-links/" +MENU_URL = "/api/workflow/notebook-lm-links/menu/" + + +def _staff(email: str, *, office: bool = False, superuser: bool = False) -> Staff: + return Staff.objects.create_user( + email=email, + password="x", + first_name="Test", + last_name="User", + is_office_staff=office, + is_superuser=superuser, + ) + + +def _link( + name: str, + *, + enabled: bool = True, + restriction: str = NotebookLmRestriction.NONE, + order: int = 0, +) -> NotebookLmLink: + return NotebookLmLink.objects.create( + name=name, + url=f"https://nb.test/{name}", + enabled=enabled, + restriction=restriction, + order=order, + ) + + +class NotebookLmMenuTests(APITestCase): + def setUp(self) -> None: + _link("Training", order=1) + _link("HS", order=2) + _link("Admin", restriction=NotebookLmRestriction.SUPERUSER, order=3) + _link("Disabled", enabled=False, order=0) + + def test_menu_hides_restricted_and_disabled_for_regular_staff(self) -> None: + client = APIClient() + client.force_authenticate(_staff("worker@example.test")) + resp = client.get(MENU_URL) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + names = [row["name"] for row in resp.json()] + self.assertEqual(names, ["Training", "HS"]) + + def test_menu_includes_restricted_for_superuser(self) -> None: + client = APIClient() + client.force_authenticate(_staff("root@example.test", superuser=True)) + resp = client.get(MENU_URL) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + names = [row["name"] for row in resp.json()] + self.assertEqual(names, ["Training", "HS", "Admin"]) + + def test_menu_requires_authentication(self) -> None: + resp = self.client.get(MENU_URL) + self.assertIn(resp.status_code, (401, 403)) + + +class NotebookLmCrudPermissionTests(APITestCase): + def test_non_office_staff_cannot_create(self) -> None: + client = APIClient() + client.force_authenticate(_staff("worker@example.test")) + resp = client.post( + LIST_URL, {"name": "X", "url": "https://nb.test/x"}, format="json" + ) + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) + self.assertFalse(NotebookLmLink.objects.filter(name="X").exists()) + + def test_office_staff_can_create(self) -> None: + client = APIClient() + client.force_authenticate(_staff("office@example.test", office=True)) + resp = client.post( + LIST_URL, {"name": "X", "url": "https://nb.test/x"}, format="json" + ) + self.assertEqual(resp.status_code, status.HTTP_201_CREATED) + self.assertTrue(NotebookLmLink.objects.filter(name="X").exists()) diff --git a/apps/workflow/tests/test_relabel_client_app.py b/apps/workflow/tests/test_relabel_client_app.py deleted file mode 100644 index caf556abc..000000000 --- a/apps/workflow/tests/test_relabel_client_app.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Tests for the KAN-278 relabel_client_app one-time DB surgery. - -The fully-migrated test DB is post-rename (tables are ``company_company`` -etc., ledger rows say app='company'), so each test stages the legacy state -it needs with raw DDL/DML and restores the live table names afterwards - -the surgery's net DDL is deliberately NOT zero (it only does the app-label -half of the table renames; the rename migration does the model half). -""" - -from django.core.management import call_command -from django.core.management.base import CommandError -from django.db import connection -from django.test import TransactionTestCase - -# The pre-squash client migration names every production ledger carried -# (verified against the dev ledger, which mirrored prod at the squash). -HISTORIC_CLIENT_MIGRATIONS = [ - "0001_initial", - "0002_clientcontact", - "0003_add_xero_merge_tracking", - "0004_populate_merge_fields", - "0005_client_is_supplier", - "0006_alter_client_name", - "0007_delete_empty_name_contacts", - "0008_merge_duplicate_contacts", - "0009_clientcontact_unique_client_contact_name", - "0010_add_is_active_to_clientcontact", - "0011_convert_empty_strings_to_null", - "0012_supplierpickupaddress", - "0013_add_google_fields_to_pickup_address", - "0014_add_suburb_to_pickup_address", - "0015_populate_xero_addresses", - "0016_alter_client_table_alter_clientcontact_table_and_more", - "0017_reassign_stranded_merged_client_fks", - "0018_client_allow_jobs", - "0019_client_name_fts_index", - "0020_suppliersearchalias_and_more", - "0021_clientcontactmethod", - "0022_client_name_trgm_index", - "0023_drop_scalar_phone_fields", -] - -# live table name -> pre-surgery (legacy) table name -LEGACY_TABLE_NAMES = [ - ("company_company", "client_client"), - ("company_suppliersearchalias", "client_suppliersearchalias"), - ("company_supplierpickupaddress", "client_supplierpickupaddress"), -] - - -def _insert_ghost_rows(cursor) -> None: - for name in HISTORIC_CLIENT_MIGRATIONS: - cursor.execute( - "INSERT INTO django_migrations (app, name, applied) " - "VALUES ('client', %s, NOW())", - [name], - ) - - -class RelabelClientAppTests(TransactionTestCase): - def test_noop_on_fresh_db(self) -> None: - # The migrated test DB has no app='client' rows: the command must - # no-op without touching tables. - call_command("relabel_client_app") - with connection.cursor() as cursor: - cursor.execute( - "SELECT COUNT(*) FROM django_migrations WHERE app = 'client'" - ) - self.assertEqual(cursor.fetchone()[0], 0) - cursor.execute("SELECT to_regclass('company_company')") - self.assertIsNotNone(cursor.fetchone()[0]) - - def test_relabels_postsquash_ledger(self) -> None: - # Stage the real pre-deploy state: legacy table names, the baseline - # ledger row under app='client', 23 historic ghost rows, content - # types under app_label='client'. - with connection.cursor() as cursor: - for live, legacy in LEGACY_TABLE_NAMES: - cursor.execute(f'ALTER TABLE "{live}" RENAME TO "{legacy}"') - cursor.execute( - "UPDATE django_migrations SET app = 'client' " - "WHERE app = 'company' AND name = '0001_baseline'" - ) - _insert_ghost_rows(cursor) - cursor.execute( - "UPDATE django_content_type SET app_label = 'client' " - "WHERE app_label = 'company'" - ) - - try: - call_command("relabel_client_app") - with connection.cursor() as cursor: - cursor.execute( - "SELECT COUNT(*) FROM django_migrations WHERE app = 'client'" - ) - self.assertEqual(cursor.fetchone()[0], 0) - cursor.execute( - "SELECT COUNT(*) FROM django_migrations " - "WHERE app = 'company' AND name = '0001_baseline'" - ) - self.assertEqual(cursor.fetchone()[0], 1) - # The historic ghosts are deleted, not relabelled. - cursor.execute( - "SELECT COUNT(*) FROM django_migrations " - "WHERE app = 'company' AND name = ANY(%s)", - [HISTORIC_CLIENT_MIGRATIONS], - ) - self.assertEqual(cursor.fetchone()[0], 0) - cursor.execute( - "SELECT COUNT(*) FROM django_content_type " - "WHERE app_label = 'client'" - ) - self.assertEqual(cursor.fetchone()[0], 0) - cursor.execute( - "SELECT COUNT(*) FROM django_content_type " - "WHERE app_label = 'company'" - ) - self.assertGreater(cursor.fetchone()[0], 0) - # Surgery does the label half only: client_client ends up at - # company_client; the rename migration owns the model half. - cursor.execute("SELECT to_regclass('company_client')") - self.assertIsNotNone(cursor.fetchone()[0]) - cursor.execute("SELECT to_regclass('client_client')") - self.assertIsNone(cursor.fetchone()[0]) - - # Second run must be a clean no-op. - call_command("relabel_client_app") - with connection.cursor() as cursor: - cursor.execute( - "SELECT COUNT(*) FROM django_migrations WHERE app = 'client'" - ) - self.assertEqual(cursor.fetchone()[0], 0) - finally: - # Restore the live schema for subsequent tests: only the main - # table sits at its label-half name (company_client). - with connection.cursor() as cursor: - cursor.execute("SELECT to_regclass('company_client')") - if cursor.fetchone()[0] is not None: - cursor.execute( - "ALTER TABLE company_client RENAME TO company_company" - ) - - def test_aborts_on_presquash_ledger(self) -> None: - # app='client' rows without a ('client', '0001_baseline') row mean a - # pre-squash database: the command must abort loudly and leave every - # staged row in place (its transaction rolls back atomically). - with connection.cursor() as cursor: - _insert_ghost_rows(cursor) - try: - with self.assertRaises(CommandError): - call_command("relabel_client_app") - with connection.cursor() as cursor: - cursor.execute( - "SELECT COUNT(*) FROM django_migrations WHERE app = 'client'" - ) - self.assertEqual(cursor.fetchone()[0], len(HISTORIC_CLIENT_MIGRATIONS)) - cursor.execute("SELECT to_regclass('company_company')") - self.assertIsNotNone(cursor.fetchone()[0]) - finally: - with connection.cursor() as cursor: - cursor.execute("DELETE FROM django_migrations WHERE app = 'client'") diff --git a/apps/workflow/tests/test_restore_migration_state.py b/apps/workflow/tests/test_restore_migration_state.py deleted file mode 100644 index 1609aa85a..000000000 --- a/apps/workflow/tests/test_restore_migration_state.py +++ /dev/null @@ -1,58 +0,0 @@ -import importlib.util -import types -from pathlib import Path -from typing import ClassVar - -from django.test import TestCase - -REPO_ROOT = Path(__file__).resolve().parents[3] -POST_CHECK = REPO_ROOT / "scripts" / "restore_checks" / "check_post_migration_state.py" - - -def load_post_check() -> types.ModuleType: - spec = importlib.util.spec_from_file_location( - "check_post_migration_state", POST_CHECK - ) - if spec is None or spec.loader is None: - raise RuntimeError("Could not load check_post_migration_state.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -class RestoreMigrationStateTests(TestCase): - check: ClassVar[types.ModuleType] - - @classmethod - def setUpClass(cls) -> None: - super().setUpClass() - cls.check = load_post_check() - - def counts(self) -> dict[str, int]: - return { - "companies": 10, - "contacts": 8, - "contact_methods": 7, - "jobs": 20, - "calls": 5, - "jobs_with_contact": 12, - "calls_with_contact": 4, - } - - def test_count_comparison_accepts_preservation_and_cleanup(self) -> None: - before = self.counts() - after = {**before, "contacts": 6, "contact_methods": 5} - - self.assertEqual(self.check.comparison_errors(before, after), []) - - def test_count_comparison_reports_lost_business_references(self) -> None: - before = self.counts() - after = {**before, "jobs_with_contact": 11} - - self.assertEqual( - self.check.comparison_errors(before, after), - ["jobs_with_contact: before=12, after=11"], - ) - - def test_current_test_schema_satisfies_structural_invariants(self) -> None: - self.assertEqual(self.check.structural_errors(), []) diff --git a/apps/workflow/tests/test_sync_clients.py b/apps/workflow/tests/test_sync_clients.py index c223ca7d9..350b4ae06 100644 --- a/apps/workflow/tests/test_sync_clients.py +++ b/apps/workflow/tests/test_sync_clients.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from unittest.mock import patch +from django.core.exceptions import ValidationError from django.db import connection from django.test import TestCase from django.test.utils import CaptureQueriesContext @@ -16,7 +17,6 @@ set_company_fields, sync_xero_phone_methods, ) -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError @@ -195,7 +195,7 @@ def test_archived_contact_creates_separate_record( merged_to=self.active_xero_id, ) - from apps.workflow.api.xero.sync import sync_companies + from apps.workflow.api.xero.transforms import sync_companies result = sync_companies([archived_contact]) @@ -228,7 +228,7 @@ def test_active_contact_name_collision_still_raises( status="ACTIVE", ) - from apps.workflow.api.xero.sync import sync_companies + from apps.workflow.api.xero.transforms import sync_companies with self.assertRaises(ValueError) as ctx: sync_companies([conflicting_contact]) @@ -251,7 +251,7 @@ def test_archived_contact_with_existing_xero_id_updates_in_place( status="ARCHIVED", ) - from apps.workflow.api.xero.sync import sync_companies + from apps.workflow.api.xero.transforms import sync_companies result = sync_companies([same_id_contact]) @@ -331,7 +331,7 @@ def test_duplicate_phone_owner_crashes_sync_and_persists_app_error(self) -> None before = AppError.objects.count() with self.assertRaisesRegex( - AlreadyLoggedException, + ValidationError, "already belongs to.*Existing Phone Owner", ): sync_xero_phone_methods(imported) diff --git a/apps/workflow/tests/test_tasks.py b/apps/workflow/tests/test_tasks.py index e49ad3b98..bcbb5a28d 100644 --- a/apps/workflow/tests/test_tasks.py +++ b/apps/workflow/tests/test_tasks.py @@ -1,9 +1,9 @@ """Unit tests for the Xero Celery task pipeline. xero_sync_task: a failure inside the sync body persists AppError exactly -once and surfaces as AlreadyLoggedException, so Celery records FAILURE -with a populated traceback in TaskResult — the bug PR #273 originally -introduced (a detached thread crash recorded SUCCESS in TaskResult). +once and propagates unchanged, so Celery records FAILURE with a populated +traceback in TaskResult — the bug PR #273 originally introduced (a +detached thread crash recorded SUCCESS in TaskResult). xero_regular_sync_task: when start_sync() reports the lock is held (another sync already in progress), the Beat task body returns cleanly @@ -15,20 +15,22 @@ from unittest.mock import MagicMock, patch from django.core.cache import caches -from django.test import TestCase +from django.test import override_settings -from apps.workflow.exceptions import ( - AlreadyLoggedException, - NoValidXeroTokenError, - XeroQuotaFloorReached, -) -from apps.workflow.models import AppError +from apps.testing import BaseTestCase +from apps.workflow.exceptions import NoValidXeroTokenError, XeroQuotaFloorReached +from apps.workflow.models import AppError, CompanyDefaults +from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.services.xero_sync_constants import SYNC_STATUS_KEY from apps.workflow.services.xero_sync_service import ( XeroSyncService, XeroSyncStartResult, ) -from apps.workflow.tasks import xero_regular_sync_task, xero_sync_task +from apps.workflow.tasks import ( + xero_30_day_sync_task, + xero_regular_sync_task, + xero_sync_task, +) # Sync state lives on the "shared" alias (Redis in prod, LocMem in tests via # settings_test). Test fixtures must seed/inspect it on the same alias the @@ -36,18 +38,44 @@ _shared = caches["shared"] -class XeroSyncTaskFailureTests(TestCase): +@override_settings(SOLO_CACHE="shared") +class XeroSyncGateCacheTests(BaseTestCase): + def setUp(self) -> None: + CompanyDefaults.clear_cache() + + def tearDown(self) -> None: + CompanyDefaults.clear_cache() + + def test_gate_update_replaces_a_stale_shared_cache_value(self) -> None: + company = CompanyDefaults.get_solo() + company.enable_xero_sync = True + company.save(update_fields=["enable_xero_sync"]) + + CompanyDefaults.objects.filter(pk=company.pk).update(enable_xero_sync=False) + self.assertTrue(CompanyDefaults.get_solo().enable_xero_sync) + + CompanyDefaults.set_xero_sync_enabled(enabled=False) + + cached = _shared.get(CompanyDefaults.get_cache_key()) + self.assertIsNotNone(cached) + self.assertFalse(cached.enable_xero_sync) + company.refresh_from_db() + self.assertFalse(company.enable_xero_sync) + + +class XeroSyncTaskFailureTests(BaseTestCase): """A failure inside the sync body must persist AppError exactly once - and surface as AlreadyLoggedException so Celery records FAILURE with - a populated traceback in TaskResult.""" + and propagate unchanged so Celery records FAILURE with a populated + traceback in TaskResult.""" def setUp(self) -> None: _shared.delete(SYNC_STATUS_KEY) + CompanyDefaults.set_xero_sync_enabled(enabled=True) def tearDown(self) -> None: _shared.delete(SYNC_STATUS_KEY) - def test_inner_sync_failure_persists_and_raises_already_logged(self) -> None: + def test_inner_sync_failure_persists_and_reraises(self) -> None: task_id = "test-task-failure" _shared.set(f"xero_sync_messages_{task_id}", [], timeout=60) _shared.set(SYNC_STATUS_KEY, task_id, timeout=60) @@ -65,7 +93,7 @@ def boom() -> Iterator[dict[str, object]]: "apps.workflow.accounting.registry.get_provider", return_value=provider, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): xero_sync_task(task_id) # Exactly one AppError row — no double-persist from a wrapping handler. @@ -86,17 +114,15 @@ def test_prelogged_sync_failure_surfaces_without_duplicate_app_error(self) -> No _shared.set(f"xero_sync_messages_{task_id}", [], timeout=60) _shared.set(SYNC_STATUS_KEY, task_id, timeout=60) - persisted = AppError.objects.create( - message="No Xero tenants found.", - app="workflow", - ) + prelogged = RuntimeError("No Xero tenants found.") + persisted = persist_app_error(prelogged) + # `yield from ()` keeps this a generator function — so the error + # surfaces on iteration, not at call time — without dead code after + # the raise. def boom() -> Iterator[dict[str, object]]: - raise AlreadyLoggedException( - RuntimeError("No Xero tenants found."), - persisted.id, - ) - yield + yield from () + raise prelogged provider = MagicMock() provider.run_full_sync.return_value = boom() @@ -107,7 +133,7 @@ def boom() -> Iterator[dict[str, object]]: "apps.workflow.accounting.registry.get_provider", return_value=provider, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): xero_sync_task(task_id) self.assertEqual(AppError.objects.count(), before) @@ -126,8 +152,8 @@ def test_quota_floor_aborts_without_app_error(self) -> None: _shared.set(SYNC_STATUS_KEY, task_id, timeout=60) def abort() -> Iterator[dict[str, object]]: + yield from () raise XeroQuotaFloorReached("Skipping sync: Xero day quota at floor (100)") - yield provider = MagicMock() provider.run_full_sync.return_value = abort() @@ -146,10 +172,25 @@ def abort() -> Iterator[dict[str, object]]: self.assertEqual(messages[-1]["sync_status"], "aborted") self.assertIsNone(_shared.get(SYNC_STATUS_KEY)) + def test_disabled_gate_skips_queued_worker_before_provider_access(self) -> None: + task_id = "test-disabled-sync" + _shared.set(f"xero_sync_messages_{task_id}", [], timeout=60) + _shared.set(SYNC_STATUS_KEY, task_id, timeout=60) + CompanyDefaults.set_xero_sync_enabled(enabled=False) -class XeroSyncStartResultTests(TestCase): + with patch("apps.workflow.accounting.registry.get_provider") as get_provider: + xero_sync_task(task_id) + + get_provider.assert_not_called() + messages = _shared.get(f"xero_sync_messages_{task_id}", []) + self.assertEqual(messages[-1]["sync_status"], "aborted") + self.assertIsNone(_shared.get(SYNC_STATUS_KEY)) + + +class XeroSyncStartResultTests(BaseTestCase): def setUp(self) -> None: _shared.delete(SYNC_STATUS_KEY) + CompanyDefaults.set_xero_sync_enabled(enabled=True) def tearDown(self) -> None: _shared.delete(SYNC_STATUS_KEY) @@ -185,16 +226,15 @@ def test_start_sync_reports_no_valid_token_without_exception(self) -> None: def test_start_sync_releases_lock_when_token_refresh_raises(self) -> None: provider = MagicMock() - provider.get_valid_token.side_effect = AlreadyLoggedException( - RuntimeError("refresh failed"), - "app-error-id", - ) + prelogged = RuntimeError("refresh failed") + persist_app_error(prelogged) + provider.get_valid_token.side_effect = prelogged with patch( "apps.workflow.services.xero_sync_service.get_provider", return_value=provider, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): XeroSyncService.start_sync() self.assertIsNone(_shared.get(SYNC_STATUS_KEY)) @@ -227,7 +267,8 @@ def test_sync_all_xero_data_raises_when_token_missing(self) -> None: list(sync_all_xero_data()) -class XeroRegularSyncSkipTests(TestCase): +@override_settings(SOLO_CACHE="shared") +class XeroRegularSyncSkipTests(BaseTestCase): """When start_sync() reports the lock is held (started=False), the Beat task must not dispatch xero_sync_task and must return cleanly so its own TaskResult is SUCCESS — the sync didn't run, but the *decision* @@ -235,9 +276,11 @@ class XeroRegularSyncSkipTests(TestCase): def setUp(self) -> None: _shared.delete(SYNC_STATUS_KEY) + CompanyDefaults.set_xero_sync_enabled(enabled=True) def tearDown(self) -> None: _shared.delete(SYNC_STATUS_KEY) + CompanyDefaults.clear_cache() def test_lock_held_skips_dispatch_and_returns_cleanly(self) -> None: # Pre-acquire the lock with some other task_id, simulating a sync @@ -248,6 +291,38 @@ def test_lock_held_skips_dispatch_and_returns_cleanly(self) -> None: timeout=60, ) - with patch("apps.workflow.tasks.xero_sync_task") as mock_task: + with ( + patch("apps.workflow.tasks.close_old_connections"), + patch("apps.workflow.tasks.xero_sync_task") as mock_task, + ): xero_regular_sync_task() mock_task.delay.assert_not_called() + + def test_disabled_gate_skips_all_periodic_sync_dispatch(self) -> None: + CompanyDefaults.set_xero_sync_enabled(enabled=False) + + with ( + patch("apps.workflow.tasks.close_old_connections"), + patch.object(XeroSyncService, "start_sync") as start_sync, + ): + xero_regular_sync_task() + xero_30_day_sync_task() + + start_sync.assert_not_called() + + def test_disabled_gate_stops_before_quota_and_pay_item_calls(self) -> None: + from apps.workflow.api.xero.sync import synchronise_xero_data + + CompanyDefaults.set_xero_sync_enabled(enabled=False) + + with ( + patch("apps.workflow.api.xero.sync.quota_floor_breached") as quota, + patch( + "apps.workflow.api.xero.payroll.sync_xero_pay_items" + ) as sync_pay_items, + ): + messages = list(synchronise_xero_data()) + + quota.assert_not_called() + sync_pay_items.assert_not_called() + self.assertEqual(messages[-1]["severity"], "warning") diff --git a/apps/workflow/tests/test_verify_scrubbed_backup.py b/apps/workflow/tests/test_verify_scrubbed_backup.py index ec3758cd2..3e9ba7d8d 100644 --- a/apps/workflow/tests/test_verify_scrubbed_backup.py +++ b/apps/workflow/tests/test_verify_scrubbed_backup.py @@ -98,7 +98,7 @@ def pg_restore( @patch.object(Path, "is_file", return_value=True) @patch("subprocess.run") - def test_legacy_baseline_requires_temporary_cutover_flag( + def test_rejects_legacy_client_baseline( self, run: MagicMock, _is_file: object ) -> None: def pg_restore( @@ -113,14 +113,9 @@ def pg_restore( run.side_effect = pg_restore - with self.assertRaisesRegex(RuntimeError, "temporary pre-KAN-278"): + with self.assertRaisesRegex(RuntimeError, "obsolete client migration label"): self.verifier.verify_backup(self.archive) - self.verifier.verify_backup( - self.archive, - allow_legacy_client_baseline=True, - ) - @patch.object(Path, "is_file", return_value=True) @patch("subprocess.run") def test_rejects_mixed_client_and_company_baselines( @@ -142,10 +137,7 @@ def pg_restore( run.side_effect = pg_restore with self.assertRaisesRegex(RuntimeError, "mixed client/company"): - self.verifier.verify_backup( - self.archive, - allow_legacy_client_baseline=True, - ) + self.verifier.verify_backup(self.archive) @patch.object(Path, "is_file", return_value=True) @patch("subprocess.run") diff --git a/apps/workflow/tests/test_xero_app_active.py b/apps/workflow/tests/test_xero_app_active.py index e0a030118..32945c5f3 100644 --- a/apps/workflow/tests/test_xero_app_active.py +++ b/apps/workflow/tests/test_xero_app_active.py @@ -10,8 +10,8 @@ from rest_framework.test import APIRequestFactory, force_authenticate from apps.accounts.models import Staff -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError, XeroApp +from apps.workflow.services.error_persistence import persist_app_error def _row(**overrides: object) -> XeroApp: @@ -83,7 +83,7 @@ def test_swap_invalidates_tenant_id_cache(self) -> None: from apps.workflow.api.xero.active_app import swap_active from apps.workflow.api.xero.constants import TENANT_ID_CACHE_KEY - a = _row(client_id="a1", is_active=True) # noqa: F841 + _row(client_id="a1", is_active=True) b = _row(client_id="b1", is_active=False) cache.set(TENANT_ID_CACHE_KEY, "tenant-from-a") with patch("apps.workflow.api.xero.active_app._restart_sibling_workers"): @@ -416,7 +416,7 @@ def test_refresh_failure_is_persisted_and_propagated(self) -> None: before = AppError.objects.count() with patch.object(TokenApi, "refresh_token", side_effect=RuntimeError("boom")): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): auth.get_valid_token() self.assertEqual(AppError.objects.count(), before + 1) @@ -474,7 +474,9 @@ def test_ping_returns_500_for_prelogged_refresh_failure(self) -> None: is_office_staff=True, ) force_authenticate(request, user=user) - exc = AlreadyLoggedException(RuntimeError("refresh failed"), "err-123") + exc = RuntimeError("refresh failed") + app_error = persist_app_error(exc) + before = AppError.objects.count() with patch( "apps.workflow.views.xero.xero_view.get_valid_token", @@ -484,4 +486,5 @@ def test_ping_returns_500_for_prelogged_refresh_failure(self) -> None: self.assertEqual(response.status_code, 500) self.assertEqual(response.data["connected"], False) - self.assertEqual(response.data["error_id"], "err-123") + self.assertEqual(response.data["error_id"], str(app_error.id)) + self.assertEqual(AppError.objects.count(), before) diff --git a/apps/workflow/tests/test_xero_branding_themes.py b/apps/workflow/tests/test_xero_branding_themes.py index 2147f8818..2b73efaba 100644 --- a/apps/workflow/tests/test_xero_branding_themes.py +++ b/apps/workflow/tests/test_xero_branding_themes.py @@ -28,6 +28,7 @@ from apps.workflow.views.xero.xero_view import XeroAuthenticationResult THEME_ID = "11111111-2222-3333-4444-555555555555" +QUOTE_TERMS = "Client-approved quote terms" class XeroBrandingThemeProviderTests(BaseTestCase): @@ -76,7 +77,7 @@ def test_invoice_create_payload_includes_branding_theme_id( @patch("apps.workflow.accounting.xero.provider.process_xero_data", return_value={}) @patch.object(XeroAccountingProvider, "_get_api") - def test_quote_create_payload_includes_branding_theme_without_terms( + def test_quote_create_payload_includes_branding_theme_and_terms( self, mock_get_api: Mock, _mock_process: Mock ) -> None: api = Mock() @@ -104,6 +105,7 @@ def test_quote_create_payload_includes_branding_theme_without_terms( date=date(2026, 7, 16), expiry_date=date(2026, 8, 15), document_theme_external_id=THEME_ID, + terms=QUOTE_TERMS, ) result = XeroAccountingProvider().create_quote(payload) @@ -111,7 +113,7 @@ def test_quote_create_payload_includes_branding_theme_without_terms( self.assertTrue(result.success) sent = api.create_quotes.call_args.kwargs["quotes"]["Quotes"][0] self.assertEqual(sent["BrandingThemeID"], THEME_ID) - self.assertNotIn("Terms", sent) + self.assertEqual(sent["Terms"], QUOTE_TERMS) @patch.object(XeroAccountingProvider, "_get_api") def test_list_document_themes_preserves_xero_order_and_default( @@ -147,7 +149,8 @@ class XeroBrandingThemeConfigurationTests(BaseTestCase): def setUp(self) -> None: defaults = CompanyDefaults.get_solo() CompanyDefaults.objects.filter(pk=defaults.pk).update( - xero_sales_branding_theme_id=None + xero_sales_branding_theme_id=None, + xero_quote_terms=QUOTE_TERMS, ) CompanyDefaults.clear_cache() @@ -219,6 +222,50 @@ def test_configured_theme_does_not_add_a_xero_read(self) -> None: self.assertEqual(selected_id, THEME_ID) manager.provider.list_document_themes.assert_not_called() + def test_quote_creation_stops_when_terms_are_unconfigured(self) -> None: + defaults = CompanyDefaults.get_solo() + CompanyDefaults.objects.filter(pk=defaults.pk).update( + xero_sales_branding_theme_id=uuid.UUID(THEME_ID), + xero_quote_terms="", + ) + CompanyDefaults.clear_cache() + manager = XeroQuoteManager( + company=self.company, job=self.job, staff=self.test_staff + ) + manager.provider = Mock() + + result = manager.create_document() + + self.assertFalse(result["success"]) + self.assertEqual(result["status"], 400) + self.assertEqual(result["error_type"], "configuration_error") + error = result["error"] + assert error is not None + self.assertIn("Configure Xero quote terms", error) + manager.provider.create_quote.assert_not_called() + + def test_quote_creation_stops_when_terms_exceed_xero_limit(self) -> None: + defaults = CompanyDefaults.get_solo() + CompanyDefaults.objects.filter(pk=defaults.pk).update( + xero_sales_branding_theme_id=uuid.UUID(THEME_ID), + xero_quote_terms="x" * 4001, + ) + CompanyDefaults.clear_cache() + manager = XeroQuoteManager( + company=self.company, job=self.job, staff=self.test_staff + ) + manager.provider = Mock() + + result = manager.create_document() + + self.assertFalse(result["success"]) + self.assertEqual(result["status"], 400) + self.assertEqual(result["error_type"], "configuration_error") + error = result["error"] + assert error is not None + self.assertIn("no more than 4000 characters", error) + manager.provider.create_quote.assert_not_called() + class SalesBrandingThemeResolutionTests(BaseTestCase): """Migration and setup share one provider-order selection contract.""" diff --git a/apps/workflow/tests/test_xero_default_task.py b/apps/workflow/tests/test_xero_default_task.py index 27fb46cc4..deff09cab 100644 --- a/apps/workflow/tests/test_xero_default_task.py +++ b/apps/workflow/tests/test_xero_default_task.py @@ -2,14 +2,13 @@ from apps.job.models import LabourSubtype from apps.testing import BaseTestCase -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models.app_error import AppError class CreateDefaultTaskTests(BaseTestCase): """create_default_task looks up the Workshop charge-out rate before the external Xero call. A missing Workshop subtype must not crash silently - (ADR 0019): it persists an AppError and re-raises AlreadyLoggedException.""" + (ADR 0019): it persists an AppError and re-raises the ValueError.""" def test_missing_workshop_subtype_persists_app_error(self) -> None: from apps.workflow.api.xero import xero @@ -23,7 +22,7 @@ def test_missing_workshop_subtype_persists_app_error(self) -> None: patch.object(xero, "get_tenant_id", return_value="tenant"), patch.object(xero, "ProjectApi") as mock_project_api, ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(ValueError): xero.create_default_task("project-id") # The Xero task must never be attempted once the rate lookup fails. diff --git a/apps/workflow/tests/test_xero_document_error_handling.py b/apps/workflow/tests/test_xero_document_error_handling.py index 5591e6d04..1ecba3a96 100644 --- a/apps/workflow/tests/test_xero_document_error_handling.py +++ b/apps/workflow/tests/test_xero_document_error_handling.py @@ -1,9 +1,10 @@ """Error contract for Xero document managers. The managers are service objects, not the HTTP boundary (ADR 0001): an -unexpected exception is persisted once and re-raised as AlreadyLoggedException -for the view to convert into a 500 carrying ``error_id``. Only *expected* -business outcomes come back as ``success: False`` dicts. +unexpected exception is persisted once and re-raised unchanged, so the view +can convert it into a 500 carrying ``error_id`` looked up via +``app_error_for``. Only *expected* business outcomes come back as +``success: False`` dicts. """ import uuid @@ -15,8 +16,8 @@ from apps.company.models import Company from apps.job.models import Job from apps.testing import BaseTestCase -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import AppError, CompanyDefaults +from apps.workflow.services.error_persistence import app_error_for, persist_app_error from apps.workflow.views.xero.xero_invoice_manager import XeroInvoiceManager from apps.workflow.views.xero.xero_quote_manager import XeroQuoteManager @@ -29,7 +30,8 @@ class XeroDocumentManagerErrorContractTests(BaseTestCase): def setUp(self) -> None: defaults = CompanyDefaults.get_solo() CompanyDefaults.objects.filter(pk=defaults.pk).update( - xero_sales_branding_theme_id=uuid.UUID(THEME_ID) + xero_sales_branding_theme_id=uuid.UUID(THEME_ID), + xero_quote_terms="Client-approved quote terms", ) CompanyDefaults.clear_cache() @@ -64,52 +66,52 @@ def _quote_manager(self) -> tuple[XeroQuoteManager, Mock]: # --- create ----------------------------------------------------------- - def test_invoice_create_reraises_already_logged(self) -> None: - """A Xero blow-up must reach the view as AlreadyLoggedException, not a dict.""" + def test_invoice_create_reraises_original(self) -> None: + """A Xero blow-up must reach the view as the raised error, not a dict.""" manager, provider = self._invoice_manager() provider.create_invoice.side_effect = RuntimeError("Xero exploded") with patch.object(XeroInvoiceManager, "build_payload", return_value=Mock()): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.create_document(total_amount=Decimal("100")) - self.assertIsNotNone(caught.exception.app_error_id) + self.assertIsNotNone(app_error_for(caught.exception)) self.assertEqual(str(caught.exception), "Xero exploded") self.assertEqual(AppError.objects.count(), 1) - def test_quote_create_reraises_already_logged(self) -> None: + def test_quote_create_reraises_original(self) -> None: manager, provider = self._quote_manager() provider.create_quote.side_effect = RuntimeError("Xero exploded") with patch.object(XeroQuoteManager, "build_payload", return_value=Mock()): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.create_document() - self.assertIsNotNone(caught.exception.app_error_id) + self.assertIsNotNone(app_error_for(caught.exception)) self.assertEqual(AppError.objects.count(), 1) # --- delete ----------------------------------------------------------- - def test_invoice_delete_reraises_already_logged(self) -> None: + def test_invoice_delete_reraises_original(self) -> None: manager, provider = self._invoice_manager() provider.delete_invoice.side_effect = RuntimeError("Xero exploded") with patch.object(XeroInvoiceManager, "get_xero_id", return_value="xero-1"): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.delete_document() - self.assertIsNotNone(caught.exception.app_error_id) + self.assertIsNotNone(app_error_for(caught.exception)) self.assertEqual(AppError.objects.count(), 1) - def test_quote_delete_reraises_already_logged(self) -> None: + def test_quote_delete_reraises_original(self) -> None: manager, provider = self._quote_manager() provider.delete_quote.side_effect = RuntimeError("Xero exploded") with patch.object(XeroQuoteManager, "get_xero_id", return_value="xero-1"): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.delete_document() - self.assertIsNotNone(caught.exception.app_error_id) + self.assertIsNotNone(app_error_for(caught.exception)) self.assertEqual(AppError.objects.count(), 1) # --- dedup regression guard ------------------------------------------ @@ -117,31 +119,33 @@ def test_quote_delete_reraises_already_logged(self) -> None: def test_invoice_delete_does_not_double_persist(self) -> None: """The delete path once lacked the pass-through arm and re-persisted.""" original = RuntimeError("Persisted upstream") - already_logged = AlreadyLoggedException(original, uuid.uuid4()) + upstream_error = persist_app_error(original) manager, provider = self._invoice_manager() - provider.delete_invoice.side_effect = already_logged + provider.delete_invoice.side_effect = original with patch.object(XeroInvoiceManager, "get_xero_id", return_value="xero-1"): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.delete_document() - self.assertIs(caught.exception, already_logged) - self.assertEqual(AppError.objects.count(), 0) + self.assertIs(caught.exception, original) + self.assertEqual(app_error_for(caught.exception), upstream_error) + self.assertEqual(AppError.objects.count(), 1) def test_quote_delete_does_not_double_persist(self) -> None: original = RuntimeError("Persisted upstream") - already_logged = AlreadyLoggedException(original, uuid.uuid4()) + upstream_error = persist_app_error(original) manager, provider = self._quote_manager() - provider.delete_quote.side_effect = already_logged + provider.delete_quote.side_effect = original with patch.object(XeroQuoteManager, "get_xero_id", return_value="xero-1"): - with self.assertRaises(AlreadyLoggedException) as caught: + with self.assertRaises(RuntimeError) as caught: manager.delete_document() - self.assertIs(caught.exception, already_logged) - self.assertEqual(AppError.objects.count(), 0) + self.assertIs(caught.exception, original) + self.assertEqual(app_error_for(caught.exception), upstream_error) + self.assertEqual(AppError.objects.count(), 1) # --- expected failures still return a dict ---------------------------- diff --git a/apps/workflow/tests/test_xero_document_raw_json.py b/apps/workflow/tests/test_xero_document_raw_json.py index bf8359d1e..0d2183346 100644 --- a/apps/workflow/tests/test_xero_document_raw_json.py +++ b/apps/workflow/tests/test_xero_document_raw_json.py @@ -49,7 +49,8 @@ def setUp(self): # provider call; without it create_document stops at the config guard. defaults = CompanyDefaults.get_solo() CompanyDefaults.objects.filter(pk=defaults.pk).update( - xero_sales_branding_theme_id=uuid.uuid4() + xero_sales_branding_theme_id=uuid.uuid4(), + xero_quote_terms="Client-approved quote terms", ) CompanyDefaults.clear_cache() diff --git a/apps/workflow/tests/test_xero_instance_templates.py b/apps/workflow/tests/test_xero_instance_templates.py index 5d1aaecd8..24e14c20c 100644 --- a/apps/workflow/tests/test_xero_instance_templates.py +++ b/apps/workflow/tests/test_xero_instance_templates.py @@ -1,14 +1,20 @@ import json import subprocess import tempfile +from decimal import Decimal from pathlib import Path +from uuid import UUID -from django.test import SimpleTestCase +from django.core import serializers +from django.core.management import call_command +from django.test import SimpleTestCase, TestCase + +from apps.accounts.models import Staff +from apps.crm.models import PhoneEndpoint +from apps.workflow.models import CompanyDefaults, XeroApp REPO_ROOT = Path(__file__).resolve().parents[3] -CREDENTIALS_TEMPLATE = ( - REPO_ROOT / "scripts" / "server" / "templates" / "credentials-instance.template" -) +COMMON_SCRIPT = REPO_ROOT / "scripts" / "server" / "common.sh" XERO_APPS_TEMPLATE = ( REPO_ROOT / "scripts" / "server" / "templates" / "xero-apps.json.template" ) @@ -19,79 +25,82 @@ / "templates" / "phone-provider-settings.json.template" ) -INSTANCE_SCRIPT = REPO_ROOT / "scripts" / "server" / "instance.sh" -DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "server" / "deploy.sh" -PREDEPLOY_BACKUP_SCRIPT = REPO_ROOT / "scripts" / "predeploy_backup.sh" -BACKUP_DB_SCRIPT = REPO_ROOT / "scripts" / "backup_db.sh" -COMMON_SCRIPT = REPO_ROOT / "scripts" / "server" / "common.sh" -SERVER_SETUP_SCRIPT = REPO_ROOT / "scripts" / "server" / "server-setup.sh" -SERVER_README = REPO_ROOT / "scripts" / "server" / "README.md" -PRODUCTION_SETUP_DOC = REPO_ROOT / "docs" / "instance-setup-production.md" -DEMO_SETUP_DOC = REPO_ROOT / "docs" / "instance-setup-demo.md" -DW_RUN_SCRIPT = REPO_ROOT / "scripts" / "server" / "dw-run.sh" -RELEASE_UTILS = REPO_ROOT / "scripts" / "server" / "release-utils.sh" -GUNICORN_TEMPLATE = ( - REPO_ROOT - / "scripts" - / "server" - / "templates" - / "gunicorn-instance.service.template" -) -CELERY_WORKER_TEMPLATE = ( - REPO_ROOT - / "scripts" - / "server" - / "templates" - / "celery-worker-instance.service.template" -) -CELERY_BEAT_TEMPLATE = ( - REPO_ROOT - / "scripts" - / "server" - / "templates" - / "celery-beat-instance.service.template" -) -NGINX_TEMPLATE = ( - REPO_ROOT / "scripts" / "server" / "templates" / "nginx-instance.conf.template" +COMPANY_DEFAULTS_FIXTURE = ( + REPO_ROOT / "apps" / "workflow" / "fixtures" / "company_defaults.json" ) -BACKUP_TEMPLATE = ( - REPO_ROOT - / "scripts" - / "server" - / "templates" - / "backup-db-instance.service.template" -) -BACKUP_FILES_TEMPLATE = ( - REPO_ROOT - / "scripts" - / "server" - / "templates" - / "backup-files-instance.service.template" +INITIAL_DATA_FIXTURE = ( + REPO_ROOT / "apps" / "workflow" / "fixtures" / "initial_data.json" ) -SETTINGS_FILE = REPO_ROOT / "docketworks" / "settings.py" -class XeroInstanceTemplateTests(SimpleTestCase): - def test_credentials_template_includes_xero_oauth_env_vars(self): - content = CREDENTIALS_TEMPLATE.read_text() +class DemoSeedFixtureTests(TestCase): + def test_demo_seed_fixtures_load_current_demo_contract(self) -> None: + """Schema changes must not break demo creation or its advertised logins.""" + call_command( + "loaddata", + str(COMPANY_DEFAULTS_FIXTURE), + str(INITIAL_DATA_FIXTURE), + verbosity=0, + ) + + demo_staff = Staff.objects.filter(email__endswith="@example.com") + self.assertEqual(demo_staff.count(), 11) + self.assertFalse( + Staff.objects.filter(email="defaultadmin@example.com").exists() + ) + self.assertTrue( + all(staff.check_password("Default-staff-password") for staff in demo_staff) + ) + self.assertFalse(demo_staff.exclude(xero_user_id__isnull=True).exists()) + + charles = demo_staff.get(email="charles.baker@example.com") + self.assertEqual(charles.base_wage_rate, Decimal("35.80")) + self.assertEqual(charles.wage_rate, Decimal("42.96")) + + defaults = CompanyDefaults.objects.get(pk=1) + self.assertEqual(defaults.company_name, "Demo Company") + self.assertEqual( + defaults.xero_quote_terms, + ( + "Terms of trade can be found on our website: " + "https://www.democompany.example.com/terms-of-trade" + ), + ) + self.assertEqual( + str(defaults.shop_company_id), + "00000000-0000-0000-0000-000000000001", + ) + + main_line = PhoneEndpoint.objects.get(label="Main line") + self.assertEqual(main_line.number, "+6496365131") + self.assertEqual(main_line.endpoint_type, PhoneEndpoint.EndpointType.MAIN_LINE) - self.assertIn("XERO_DEFAULT_USER_ID=", content) - self.assertIn("XERO_CLIENT_ID=", content) - self.assertIn("XERO_CLIENT_SECRET=", content) - self.assertIn("XERO_WEBHOOK_KEY=", content) - self.assertIn("XERO_REDIRECT_URI=", content) + def test_seed_template_satisfies_instance_sh_validator_contract(self) -> None: + """`prepare-config --seed` copies this fixture to the operator config, which + `instance.sh validate_company_defaults_config` then requires to be exactly one + Company and one CompanyDefaults with a valid-UUID xero_tenant_id and sync + disabled. Drift here silently breaks `create --seed`.""" + text = COMPANY_DEFAULTS_FIXTURE.read_text() + records = json.loads(text) + + self.assertEqual( + sorted(record["model"] for record in records), + ["company.company", "workflow.companydefaults"], + ) + self.assertNotIn("__", text) - def test_credentials_template_includes_phone_provider_env_vars(self) -> None: - content = CREDENTIALS_TEMPLATE.read_text() + defaults = next( + record["fields"] + for record in records + if record["model"] == "workflow.companydefaults" + ) + UUID(defaults["xero_tenant_id"]) + self.assertIs(defaults["enable_xero_sync"], False) - self.assertIn("PHONE_PROVIDER_DOWNLOADS_ENABLED=false", content) - self.assertIn("PHONE_PROVIDER_RECORDING_DELETION_ENABLED=false", content) - self.assertIn("PHONE_PROVIDER_BASE_URL=", content) - self.assertIn("PHONE_PROVIDER_USERNAME=", content) - self.assertIn("PHONE_PROVIDER_PASSWORD=", content) - self.assertIn("PHONE_PROVIDER_ACCOUNT_CODE=", content) - def test_xero_apps_template_renders_to_valid_json(self): +class XeroInstanceTemplateTests(SimpleTestCase): + def test_xero_apps_template_renders_to_valid_json(self) -> None: + """Model field removals must not leave provisioning fixtures unreadable.""" rendered = ( XERO_APPS_TEMPLATE.read_text() .replace("__INSTANCE__", "msm-uat") @@ -104,15 +113,16 @@ def test_xero_apps_template_renders_to_valid_json(self): ) ) - payload = json.loads(rendered) - self.assertEqual(len(payload), 1) - fields = payload[0]["fields"] - self.assertEqual(fields["label"], "msm-uat xero") - self.assertEqual(fields["client_id"], "client-id") - self.assertEqual(fields["client_secret"], "client-secret") - self.assertEqual(fields["webhook_key"], "webhook-key") + deserialized = list(serializers.deserialize("json", rendered)) + self.assertEqual(len(deserialized), 1) + xero_app = deserialized[0].object + assert isinstance(xero_app, XeroApp) + self.assertEqual(xero_app.label, "msm-uat xero") + self.assertEqual(xero_app.client_id, "client-id") + self.assertEqual(xero_app.client_secret, "client-secret") + self.assertEqual(xero_app.webhook_key, "webhook-key") self.assertEqual( - fields["redirect_uri"], + xero_app.redirect_uri, "https://msm-uat.docketworks.site/api/xero/oauth/callback/", ) @@ -142,281 +152,7 @@ def test_phone_provider_settings_template_renders_to_valid_json(self) -> None: self.assertEqual(fields["password"], "phone-secret") self.assertEqual(fields["account_code"], "15539090") - def test_instance_script_requires_and_loads_xero_app_fixture(self): - content = INSTANCE_SCRIPT.read_text() - - self.assertIn('[[ -z "${XERO_CLIENT_ID:-}" ]]', content) - self.assertIn('[[ -z "${XERO_CLIENT_SECRET:-}" ]]', content) - self.assertIn('[[ -z "${XERO_WEBHOOK_KEY:-}" ]]', content) - self.assertIn('[[ -z "${XERO_REDIRECT_URI:-}" ]]', content) - - self.assertIn("xero-apps.json.template", content) - self.assertIn( - "call_command('loaddata', '$XERO_APPS_FIXTURE')", - content, - ) - self.assertIn( - "XeroApp already configured; skipping xero_apps.json load", content - ) - self.assertIn("if XeroApp.objects.exists()", content) - self.assertNotIn("XeroApp.objects.filter", content) - self.assertNotIn(".delete()", content) - self.assertNotIn( - "python manage.py loaddata apps/workflow/fixtures/xero_apps.json", content - ) - self.assertNotIn( - 'rm -f "$INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json"', content - ) - - def test_instance_script_requires_xero_default_user_id(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn('[[ -z "${XERO_DEFAULT_USER_ID:-}" ]]', content) - self.assertIn('MISSING+=("XERO_DEFAULT_USER_ID")', content) - self.assertNotIn("UNCONFIGURED_XERO_DEFAULT_USER_ID", content) - - def test_instance_script_exposes_reconfigure_as_convergent_command(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn("instance.sh reconfigure ", content) - self.assertIn("do_reconfigure()", content) - self.assertIn("do_configure false reconfigure", content) - self.assertIn("reconfigure) do_reconfigure", content) - - def test_instance_script_rerenders_env_preserving_generated_values(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn("render_instance_env()", content) - self.assertIn( - 'db_password="$(read_env_value "$env_file" DB_PASSWORD)"', content - ) - self.assertIn( - 'test_db_password="$(read_env_value "$env_file" TEST_DB_PASSWORD)"', - content, - ) - self.assertIn('secret_key="$(read_env_value "$env_file" SECRET_KEY)"', content) - self.assertIn( - 'bearer_secret="$(read_env_value "$env_file" BEARER_SECRET)"', - content, - ) - self.assertIn('tmp_env="$(mktemp "$instance_dir/.env.tmp.XXXXXX")"', content) - self.assertIn('mv "$tmp_env" "$env_file"', content) - self.assertNotIn(".env already exists — skipping", content) - self.assertIn( - 'DB_PASSWORD="$(read_env_value "$INSTANCE_DIR/.env" DB_PASSWORD)"', - content, - ) - self.assertNotIn('DB_PASSWORD="$(. "$INSTANCE_DIR/.env"', content) - - def test_instance_script_only_seeds_missing_db_config(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn( - "AIProvider already configured; skipping ai_providers.json load", content - ) - self.assertIn("if AIProvider.objects.exists()", content) - self.assertIn( - "XeroApp already configured; skipping xero_apps.json load", content - ) - self.assertIn("if XeroApp.objects.exists()", content) - self.assertIn( - "PhoneProviderSettings already configured; skipping phone_provider_settings.json load", - content, - ) - self.assertIn("PhoneProviderSettings.get_solo()", content) - self.assertNotIn( - "python manage.py loaddata apps/workflow/fixtures/ai_providers.json", - content, - ) - - def test_instance_script_rejects_seed_for_existing_checkout(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn('[[ "$IS_EXISTING" == "true" && "$SEED" == "true" ]]', content) - self.assertIn("--seed is only valid when creating a new instance", content) - - def test_instance_script_rejects_config_without_release_link(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn( - '[[ -f "$INSTANCE_DIR/.env" && ! -L "$INSTANCE_DIR/app" && ! -L "$INSTANCE_DIR/current" ]]', - content, - ) - self.assertIn("has config but no app/current release link", content) - self.assertIn("Restore or recreate the instance", content) - self.assertLess( - content.index("has config but no app/current release link"), - content.index('TARGET_SHA="$(resolve_release_ref origin/production)"'), - ) - - def test_instance_script_uses_shared_releases_not_instance_checkouts(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn('source "$SCRIPT_DIR/release-utils.sh"', content) - self.assertIn('ensure_release "$TARGET_SHA"', content) - self.assertIn('switch_instance_release "$INSTANCE" "$TARGET_SHA"', content) - self.assertNotIn('git -C "$INSTANCE_DIR" init', content) - self.assertNotIn('git -C "$INSTANCE_DIR" checkout', content) - self.assertNotIn("Building frontend for instance", content) - - def test_instance_secret_fixtures_are_instance_private(self) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn('local fixture_dir="$instance_dir/.fixtures"', content) - self.assertIn( - 'local AI_PROVIDERS_FIXTURE="$INSTANCE_DIR/.fixtures/ai_providers.json"', - content, - ) - self.assertIn( - 'local XERO_APPS_FIXTURE="$INSTANCE_DIR/.fixtures/xero_apps.json"', content - ) - self.assertIn( - 'local PHONE_PROVIDER_SETTINGS_FIXTURE="$INSTANCE_DIR/.fixtures/phone_provider_settings.json"', - content, - ) - self.assertNotIn( - "$instance_dir/apps/workflow/fixtures/ai_providers.json", - content, - ) - self.assertNotIn( - "$instance_dir/apps/workflow/fixtures/xero_apps.json", - content, - ) - self.assertNotIn( - "$instance_dir/apps/workflow/fixtures/phone_provider_settings.json", - content, - ) - - def test_deploy_prepares_one_shared_release_for_all_targets(self) -> None: - content = DEPLOY_SCRIPT.read_text() - - self.assertIn('TARGET_REF="origin/production"', content) - self.assertIn('TARGET_SHA="$(resolve_release_ref "$TARGET_REF")"', content) - self.assertIn('ensure_release "$TARGET_SHA"', content) - self.assertIn('switch_instance_release "$instance" "$TARGET_SHA"', content) - self.assertNotIn("Updating shared Python dependencies", content) - self.assertNotIn("Updating shared node_modules", content) - self.assertNotIn("Building frontend", content) - self.assertNotIn('git -C "$inst_dir" pull', content) - - def test_release_utils_builds_immutable_release_artifacts(self) -> None: - content = RELEASE_UTILS.read_text() - - self.assertIn("RELEASES_DIR", content) - self.assertIn("git -C '$LOCAL_REPO' archive '$sha'", content) - self.assertIn("printf '%s\\n' '$sha' > '$release_dir/.release-sha'", content) - self.assertIn("python3.12 -m venv '$release_dir/.venv'", content) - self.assertIn("npm run check:typed-router", content) - self.assertIn("npm run build", content) - self.assertIn("npm run manual:build", content) - self.assertIn("rm -rf node_modules", content) - self.assertIn("touch '$release_dir/.complete'", content) - - def test_runtime_templates_use_app_release(self) -> None: - for template in [ - GUNICORN_TEMPLATE, - CELERY_WORKER_TEMPLATE, - CELERY_BEAT_TEMPLATE, - ]: - content = template.read_text() - self.assertIn( - "WorkingDirectory=/opt/docketworks/instances/__INSTANCE__/app", - content, - ) - self.assertIn( - "/opt/docketworks/instances/__INSTANCE__/app/.venv/bin/", content - ) - self.assertIn("Environment=PYTHONDONTWRITEBYTECODE=1", content) - self.assertNotIn("/opt/docketworks/.venv/bin/", content) - - nginx = NGINX_TEMPLATE.read_text() - self.assertIn( - "/opt/docketworks/instances/__INSTANCE__/app/frontend/dist", nginx - ) - self.assertIn("/opt/docketworks/instances/__INSTANCE__/mediafiles/", nginx) - - backup = BACKUP_TEMPLATE.read_text() - self.assertIn( - "ExecStart=/opt/docketworks/instances/__INSTANCE__/app/scripts/backup_db.sh __INSTANCE__", - backup, - ) - backup_files = BACKUP_FILES_TEMPLATE.read_text() - self.assertIn( - "ExecStart=/opt/docketworks/instances/__INSTANCE__/app/scripts/backup_instance_files.sh __INSTANCE__", - backup_files, - ) - - def test_dw_run_uses_app_release_and_instance_env(self) -> None: - content = DW_RUN_SCRIPT.read_text() - - self.assertIn('APP_DIR="$INSTANCE_DIR/app"', content) - self.assertIn("source '$APP_DIR/.venv/bin/activate'", content) - self.assertIn("source '$INSTANCE_DIR/.env'", content) - self.assertIn("cd '$APP_DIR'", content) - self.assertIn("PYTHONDONTWRITEBYTECODE=1", content) - - def test_build_id_reads_release_sha_before_git(self) -> None: - content = SETTINGS_FILE.read_text() - - self.assertIn('os.environ.get("DOCKETWORKS_BUILD_SHA"', content) - self.assertIn('release_sha_file = BASE_DIR / ".release-sha"', content) - self.assertIn('["git", "rev-parse", "HEAD"]', content) - - def test_credentials_file_stays_root_owned_before_root_source(self) -> None: - common_content = COMMON_SCRIPT.read_text() - instance_content = INSTANCE_SCRIPT.read_text() - deploy_content = DEPLOY_SCRIPT.read_text() - server_setup_content = SERVER_SETUP_SCRIPT.read_text() - - self.assertIn("require_root_owned_credentials_file()", common_content) - self.assertIn("stat -c '%u:%g:%a' \"$creds_file\"", common_content) - self.assertIn('"0:0:600"', common_content) - self.assertIn('[[ -L "$creds_file" ]]', common_content) - self.assertIn("ensure_config_dir()", common_content) - self.assertIn("stat -c '%u:%g:%a' \"$config_dir\"", common_content) - self.assertIn('"0:0:755"', common_content) - self.assertIn('[[ -L "$CONFIG_DIR" ]]', common_content) - self.assertIn('[[ -L "$config_dir" ]]', common_content) - - self.assertIn('chown root:root "$CREDS_FILE"', instance_content) - self.assertIn( - 'require_root_owned_credentials_file "$creds_file"', - instance_content, - ) - self.assertIn( - 'require_root_owned_credentials_file "$CREDS_FILE"', - instance_content, - ) - self.assertNotIn( - 'chown "$INSTANCE_USER:$INSTANCE_USER" "$CREDS_FILE"', - instance_content, - ) - - self.assertIn( - 'require_root_owned_credentials_file "$creds_file"', - deploy_content, - ) - self.assertIn("chown root:root /opt/docketworks/config", server_setup_content) - self.assertIn("chmod 755 /opt/docketworks/config", server_setup_content) - def test_node_major_parsing_accepts_patch_versions(self) -> None: - common_content = COMMON_SCRIPT.read_text() - release_utils_content = RELEASE_UTILS.read_text() - server_setup_content = SERVER_SETUP_SCRIPT.read_text() - - self.assertIn("node_major_from_nvmrc()", common_content) - self.assertIn("node_major_from_nvmrc()", server_setup_content) - self.assertIn( - "sed -nE 's/^[[:space:]]*v?([0-9]+).*/\\1/p'", - common_content, - ) - self.assertIn( - "sed -nE 's/^[[:space:]]*v?([0-9]+).*/\\1/p' .nvmrc", - release_utils_content, - ) - self.assertNotIn("tr -d 'v[:space:]'", release_utils_content) - self.assertNotIn("tr -d 'v[:space:]'", server_setup_content) - for nvmrc_value in ["18", "v18", "18.2.0", "v18.2.0", " v18.2.0"]: with tempfile.NamedTemporaryFile("w", encoding="utf-8") as nvmrc: nvmrc.write(nvmrc_value) @@ -438,165 +174,66 @@ def test_node_major_parsing_accepts_patch_versions(self) -> None: self.assertEqual(result.stdout.strip(), "18") - def test_instance_mediafiles_are_owned_for_app_writes_and_nginx_reads( - self, - ) -> None: - content = INSTANCE_SCRIPT.read_text() - - self.assertIn( - 'chown "$INSTANCE_USER:www-data" "$INSTANCE_DIR/mediafiles"', - content, - ) - self.assertIn('chmod 750 "$INSTANCE_DIR/mediafiles"', content) - - def test_instance_backups_are_owned_for_backup_timer_writes(self) -> None: - instance_content = INSTANCE_SCRIPT.read_text() - common_content = COMMON_SCRIPT.read_text() - predeploy_backup_content = PREDEPLOY_BACKUP_SCRIPT.read_text() - backup_content = BACKUP_DB_SCRIPT.read_text() - - self.assertIn("ensure_instance_backup_dir()", common_content) - self.assertIn( - 'ensure_instance_backup_dir "$INSTANCE" "$INSTANCE_USER"', - instance_content, - ) - self.assertIn( - 'ensure_instance_backup_dir "$INSTANCE" "$INST_USER"', - predeploy_backup_content, - ) - self.assertIn( - 'chown "$instance_user:$instance_user" "$backup_dir"', - common_content, - ) - self.assertIn('chmod 700 "$backup_dir"', common_content) - self.assertIn('if [[ ! -w "$BACKUP_DIR" ]]; then', backup_content) - self.assertIn( - 'RELEASE_SHA_FILE="$INSTANCE_DIR/app/.release-sha"', backup_content - ) - self.assertIn('DAILY_SHA="$BACKUP_DIR/daily_$TODAY.sha"', backup_content) - self.assertIn('MONTHLY_SHA="$BACKUP_DIR/monthly_$MONTH.sha"', backup_content) - - def test_backup_rclone_config_supports_shared_drive(self) -> None: - credentials_content = CREDENTIALS_TEMPLATE.read_text() - common_content = COMMON_SCRIPT.read_text() - deploy_content = DEPLOY_SCRIPT.read_text() - instance_content = INSTANCE_SCRIPT.read_text() - - self.assertIn("BACKUP_GDRIVE_TEAM_DRIVE_ID=", credentials_content) - self.assertIn('local team_drive_id="${4:-}"', common_content) - self.assertIn("team_drive = $team_drive_id", common_content) - self.assertIn( - 'backup_team_drive_id="$(read_env_value "$creds_file" ' - 'BACKUP_GDRIVE_TEAM_DRIVE_ID)"', - deploy_content, - ) - self.assertIn('"${BACKUP_GDRIVE_TEAM_DRIVE_ID:-}"', instance_content) - - def test_instance_file_backup_timer_is_rendered_and_enabled(self) -> None: - common_content = COMMON_SCRIPT.read_text() - deploy_content = DEPLOY_SCRIPT.read_text() - instance_content = INSTANCE_SCRIPT.read_text() + def test_prod_ref_guard_refuses_non_production_ref_on_prod_only(self) -> None: + def run( + instance: str, ref: str, allow: str + ) -> "subprocess.CompletedProcess[str]": + return subprocess.run( + [ + "bash", + "-c", + 'source "$1"; require_production_ref_or_ack "$2" "$3" "$4"', + "_", + str(COMMON_SCRIPT), + instance, + ref, + allow, + ], + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + ) - self.assertIn("backup-files-instance.service.template", common_content) - self.assertIn("backup-files-instance.timer.template", common_content) - self.assertIn( - 'systemctl enable --now "backup-files-$instance.timer"', deploy_content - ) - self.assertIn( - 'systemctl enable --now "backup-files-$INSTANCE.timer"', instance_content - ) - self.assertIn("backup_instance_files.sh", BACKUP_FILES_TEMPLATE.read_text()) + # non-prod instance: any ref is fine + self.assertEqual(run("msm-uat", "origin/main", "false").returncode, 0) + # prod instance on the production ref: fine + self.assertEqual(run("msm-prod", "origin/production", "false").returncode, 0) + # prod instance on a candidate ref, not acknowledged (no tty): refused + self.assertNotEqual(run("msm-prod", "origin/main", "false").returncode, 0) + # prod instance on a candidate ref, explicitly acknowledged: allowed + self.assertEqual(run("msm-prod", "origin/main", "true").returncode, 0) - def test_xero_default_user_id_docs_match_required_create_time_workflow( + def test_require_root_owned_credentials_file_rejects_bad_owner_and_symlink( self, ) -> None: - docs = "\n".join( - [ - CREDENTIALS_TEMPLATE.read_text(), - SERVER_README.read_text(), - PRODUCTION_SETUP_DOC.read_text(), - DEMO_SETUP_DOC.read_text(), - ] - ) - - self.assertIn("XERO_DEFAULT_USER_ID must be present", docs) - self.assertIn("required before `instance.sh create`", docs) - self.assertNotIn("leave blank for now", docs) - self.assertNotIn("Create the instance first", docs) - self.assertNotIn("copy that UUID", docs) - self.assertNotIn("Copy the relevant user ID into credentials.env", docs) - self.assertNotIn("then run `instance.sh reconfigure`", docs) - - def test_release_cleanup_is_deploy_integrated_and_reachability_based(self) -> None: - deploy_content = DEPLOY_SCRIPT.read_text() - release_utils_content = RELEASE_UTILS.read_text() - - self.assertIn("--cleanup-releases", deploy_content) - self.assertIn("cleanup_incomplete_releases", deploy_content) - self.assertIn('cleanup_unreferenced_releases "$TARGET_SHA"', deploy_content) - self.assertIn("release_is_referenced()", release_utils_content) - self.assertIn( - 'read_env_value "$instance_dir/deploy-state.env" PREVIOUS_SHA', - release_utils_content, - ) - self.assertIn("Removing unreferenced release", release_utils_content) - - def test_deploy_uses_only_shared_release_rollback(self) -> None: - content = DEPLOY_SCRIPT.read_text() - - self.assertIn('if [[ -n "$previous_sha" ]]; then', content) - self.assertIn( - 'sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance $(short_release_sha "$previous_sha")', - content, - ) - self.assertIn("if [[ $DO_BACKUP -eq 1 ]]; then", content) - self.assertLess( - content.index("if [[ $DO_BACKUP -eq 1 ]]; then"), - content.index( - 'sudo $SCRIPT_DIR/../predeploy_rollback.sh $instance $(short_release_sha "$previous_sha")' - ), - ) - self.assertIn( - "--no-backup was used; no pre-deploy rollback backup was created", - content, - ) - self.assertIn( - 'if [[ ! -L "$local_dir/app" && ! -L "$local_dir/current" ]]; then', - content, - ) - self.assertNotIn("is_legacy_" + "checkout", content) - self.assertNotIn("legacy_" + "rollback.sh", content) - self.assertNotIn("--allow-" + "dirty", content) - self.assertNotIn("dirty legacy working " + "tree", content) - self.assertNotIn(".git", content) - - def test_predeploy_backups_use_instance_backup_dir_only(self) -> None: - content = PREDEPLOY_BACKUP_SCRIPT.read_text() - - self.assertIn('BACKUP_DIR="$INSTANCE_DIR/backups"', content) - self.assertIn('OUT="$BACKUP_DIR/predeploy_${TS}_${HASH}.sql.gz"', content) - self.assertNotIn("legacy-" + "rollbacks", content) - self.assertNotIn("LEGACY_MANIFEST", content) - self.assertNotIn('OUT_DIR="$ROLLBACK_DIR"', content) - - def test_server_readme_documents_shared_release_rollback_only(self) -> None: - content = SERVER_README.read_text() + """The guard must reject a credentials file whose config dir is not + root:root, or is reached through a symlink. (The root-owned pass path + needs root and is covered by E2E, not here.)""" + + def run(creds_file: str) -> "subprocess.CompletedProcess[str]": + return subprocess.run( + [ + "bash", + "-c", + 'source "$1"; require_root_owned_credentials_file "$2"', + "_", + str(COMMON_SCRIPT), + creds_file, + ], + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + ) - self.assertIn( - "predeploy_rollback.sh", - content, - ) - self.assertIn("unless `--no-backup` was used", content) - self.assertNotIn("legacy_" + "rollback.sh", content) - self.assertNotIn("first legacy checkout " + "cutover", content) + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + creds = base / "inst.credentials.env" + creds.write_text("") - def test_typed_router_drift_is_checked_in_release_build(self) -> None: - deploy_content = DEPLOY_SCRIPT.read_text() - release_utils_content = RELEASE_UTILS.read_text() + # config dir is owned by the test user, not root:root 755 → rejected + self.assertNotEqual(run(str(creds)).returncode, 0) - self.assertIn("npm run check:typed-router", release_utils_content) - self.assertNotIn( - "server generated a different frontend/src/typed-router.d.ts", - deploy_content, - ) - self.assertNotIn("frontend/src/typed-router.d.ts", deploy_content) + # config dir reached via a symlink → rejected + link = base / "linkdir" + link.symlink_to(base) + self.assertNotEqual(run(str(link / "inst.credentials.env")).returncode, 0) diff --git a/apps/workflow/tests/test_xero_quota_floor.py b/apps/workflow/tests/test_xero_quota_floor.py index e962dfdf5..13f3c207f 100644 --- a/apps/workflow/tests/test_xero_quota_floor.py +++ b/apps/workflow/tests/test_xero_quota_floor.py @@ -556,8 +556,15 @@ def _gen(): provider.get_sync_entity_count.return_value = 5 appserror_before = AppError.objects.count() - with patch( - "apps.workflow.accounting.registry.get_provider", return_value=provider + with ( + patch( + "apps.workflow.accounting.registry.get_provider", + return_value=provider, + ), + patch( + "apps.workflow.services.xero_sync_worker.is_accounting_enabled", + return_value=True, + ), ): xero_sync_task(task_id) diff --git a/apps/workflow/tests/test_xero_quote_pdf.py b/apps/workflow/tests/test_xero_quote_pdf.py new file mode 100644 index 000000000..1374e1a2a --- /dev/null +++ b/apps/workflow/tests/test_xero_quote_pdf.py @@ -0,0 +1,201 @@ +"""Tests for native Xero quote PDF inspection.""" + +from __future__ import annotations + +import json +import tempfile +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch +from uuid import UUID, uuid4 + +from django.core.management import call_command +from django.test import SimpleTestCase +from pypdf.errors import PdfReadError +from reportlab.pdfgen import canvas + +from apps.testing import BaseTestCase +from apps.workflow.accounting.quote_pdf_service import ( + QuotePdfInspection, + inspect_quote_pdf, +) +from apps.workflow.accounting.types import QuotePdfDocument +from apps.workflow.accounting.xero.provider import XeroAccountingProvider +from apps.workflow.models import CompanyDefaults + +EXPECTED_TERMS = "Terms of trade can be found" +REMOTE_THEME_ID = "11111111-2222-3333-4444-555555555555" + + +def _write_pdf(text_lines: list[str]) -> Path: + temporary = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) + temporary.close() + pdf_path = Path(temporary.name) + document = canvas.Canvas(str(pdf_path)) + vertical_position = 800 + for line in text_lines: + document.drawString(40, vertical_position, line) + vertical_position -= 20 + document.save() + return pdf_path + + +class XeroQuotePdfProviderTests(SimpleTestCase): + """The provider must use Xero's rendered PDF, not recreate a local document.""" + + @patch.object(XeroAccountingProvider, "_get_api") + def test_download_quote_pdf_returns_xero_file_and_theme( + self, mock_get_api: Mock + ) -> None: + quote_id = str(uuid4()) + pdf_path = _write_pdf([EXPECTED_TERMS]) + api = Mock() + api.get_quote.return_value = SimpleNamespace( + quotes=[ + SimpleNamespace( + quote_id=quote_id, + branding_theme_id=REMOTE_THEME_ID, + ) + ] + ) + api.get_quote_as_pdf.return_value = str(pdf_path) + mock_get_api.return_value = (api, "tenant-id") + + result = XeroAccountingProvider().download_quote_pdf(quote_id) + + self.assertEqual(result.external_id, quote_id) + self.assertEqual(result.document_theme_external_id, REMOTE_THEME_ID) + self.assertEqual(result.temporary_file_path, pdf_path) + api.get_quote.assert_called_once_with("tenant-id", quote_id) + api.get_quote_as_pdf.assert_called_once_with("tenant-id", quote_id) + pdf_path.unlink() + + +class QuotePdfInspectionTests(BaseTestCase): + """PDF rendering can regress despite a correct BrandingThemeID payload.""" + + def setUp(self) -> None: + defaults = CompanyDefaults.get_solo() + CompanyDefaults.objects.filter(pk=defaults.pk).update( + xero_sales_branding_theme_id=UUID(REMOTE_THEME_ID) + ) + CompanyDefaults.clear_cache() + + def _provider_for_pdf(self, quote_id: UUID, pdf_path: Path) -> Mock: + provider = Mock() + provider.download_quote_pdf.return_value = QuotePdfDocument( + external_id=str(quote_id), + document_theme_external_id=REMOTE_THEME_ID, + temporary_file_path=pdf_path, + ) + return provider + + @patch("apps.workflow.accounting.quote_pdf_service.get_provider") + def test_terms_marker_survives_pdf_line_wrapping( + self, mock_get_provider: Mock + ) -> None: + quote_id = uuid4() + pdf_path = _write_pdf(["Terms of trade", "can be found online"]) + mock_get_provider.return_value = self._provider_for_pdf(quote_id, pdf_path) + + result = inspect_quote_pdf(quote_id, EXPECTED_TERMS) + + self.assertTrue(result.contains_expected_text) + self.assertEqual(result.page_count, 1) + self.assertEqual(result.remote_branding_theme_id, REMOTE_THEME_ID) + self.assertEqual(result.configured_branding_theme_id, REMOTE_THEME_ID) + self.assertFalse(pdf_path.exists()) + + @patch("apps.workflow.accounting.quote_pdf_service.get_provider") + def test_missing_or_differently_cased_terms_marker_is_red( + self, mock_get_provider: Mock + ) -> None: + quote_id = uuid4() + pdf_path = _write_pdf(["TERMS OF TRADE CAN BE FOUND online"]) + mock_get_provider.return_value = self._provider_for_pdf(quote_id, pdf_path) + + result = inspect_quote_pdf(quote_id, EXPECTED_TERMS) + + self.assertFalse(result.contains_expected_text) + self.assertFalse(pdf_path.exists()) + + @patch("apps.workflow.accounting.quote_pdf_service.get_provider") + def test_terms_marker_survives_xero_text_layer_without_word_spaces( + self, mock_get_provider: Mock + ) -> None: + quote_id = uuid4() + pdf_path = _write_pdf(["Termsoftradecanbefoundonline"]) + mock_get_provider.return_value = self._provider_for_pdf(quote_id, pdf_path) + + result = inspect_quote_pdf(quote_id, EXPECTED_TERMS) + + self.assertTrue(result.contains_expected_text) + self.assertFalse(pdf_path.exists()) + + @patch("apps.workflow.accounting.quote_pdf_service.get_provider") + def test_unreadable_pdf_raises_and_keeps_the_download( + self, mock_get_provider: Mock + ) -> None: + """The read error is what the operator needs, not a tidy temp directory.""" + quote_id = uuid4() + temporary = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) + temporary.write(b"not a PDF") + temporary.close() + pdf_path = Path(temporary.name) + self.addCleanup(pdf_path.unlink, True) + mock_get_provider.return_value = self._provider_for_pdf(quote_id, pdf_path) + + with self.assertRaises(PdfReadError): + inspect_quote_pdf(quote_id, EXPECTED_TERMS) + + self.assertTrue(pdf_path.exists()) + + @patch("apps.workflow.accounting.quote_pdf_service.get_provider") + def test_blank_pages_raise_rather_than_reporting_terms_absent( + self, mock_get_provider: Mock + ) -> None: + """An all-blank PDF is a failed render, not a quote missing its terms.""" + quote_id = uuid4() + pdf_path = _write_pdf([]) + self.addCleanup(pdf_path.unlink, True) + mock_get_provider.return_value = self._provider_for_pdf(quote_id, pdf_path) + + with self.assertRaisesRegex(ValueError, "no extractable text"): + inspect_quote_pdf(quote_id, EXPECTED_TERMS) + + self.assertTrue(pdf_path.exists()) + + +class InspectXeroQuotePdfCommandTests(SimpleTestCase): + """The E2E subprocess contract must remain structured and parseable.""" + + @patch("apps.workflow.management.commands.inspect_xero_quote_pdf.inspect_quote_pdf") + def test_command_emits_one_json_result(self, mock_inspect: Mock) -> None: + quote_id = uuid4() + mock_inspect.return_value = QuotePdfInspection( + quote_id=str(quote_id), + remote_branding_theme_id=REMOTE_THEME_ID, + configured_branding_theme_id=REMOTE_THEME_ID, + page_count=2, + contains_expected_text=False, + ) + output = StringIO() + + call_command( + "inspect_xero_quote_pdf", + str(quote_id), + expected_text=EXPECTED_TERMS, + stdout=output, + ) + + self.assertEqual( + json.loads(output.getvalue()), + { + "configured_branding_theme_id": REMOTE_THEME_ID, + "contains_expected_text": False, + "page_count": 2, + "quote_id": str(quote_id), + "remote_branding_theme_id": REMOTE_THEME_ID, + }, + ) diff --git a/apps/workflow/tests/test_xero_readonly_provider.py b/apps/workflow/tests/test_xero_readonly_provider.py index e90f57316..26f43f2c0 100644 --- a/apps/workflow/tests/test_xero_readonly_provider.py +++ b/apps/workflow/tests/test_xero_readonly_provider.py @@ -123,7 +123,8 @@ def setUp(self) -> None: # provider call; without it create_document stops at the config guard. defaults = CompanyDefaults.get_solo() CompanyDefaults.objects.filter(pk=defaults.pk).update( - xero_sales_branding_theme_id=uuid.uuid4() + xero_sales_branding_theme_id=uuid.uuid4(), + xero_quote_terms="Client-approved quote terms", ) CompanyDefaults.clear_cache() diff --git a/apps/workflow/tests/test_xero_setup_command.py b/apps/workflow/tests/test_xero_setup_command.py index 8e33923b6..a33ff0cd4 100644 --- a/apps/workflow/tests/test_xero_setup_command.py +++ b/apps/workflow/tests/test_xero_setup_command.py @@ -188,12 +188,12 @@ def test_run_setup_always_calls_demo_item_provisioning( mock_identity_api_cls, mock_accounting_api_cls, mock_get_payroll_calendars, - _mock_cache_set, + mock_cache_set, mock_get_provider, mock_resolve_theme, ): company = SimpleNamespace( - xero_tenant_id=None, + xero_tenant_id="stale-tenant", xero_payroll_calendar_name="Weekly Testing", xero_shortcode=None, xero_sales_branding_theme_id=None, @@ -224,7 +224,7 @@ def test_run_setup_always_calls_demo_item_provisioning( cmd = Command() cmd._ensure_demo_xero_items_exist = Mock() - cmd.run_setup() + cmd.run_setup(seed_xero=True) cmd._ensure_demo_xero_items_exist.assert_called_once_with( "Weekly Testing", "tenant-123" @@ -234,6 +234,60 @@ def test_run_setup_always_calls_demo_item_provisioning( company.xero_sales_branding_theme_id, UUID(selected_theme.external_id), ) + self.assertEqual(company.xero_tenant_id, "tenant-123") + mock_cache_set.assert_called_once_with("xero_tenant_id", "tenant-123") + + @patch("apps.workflow.management.commands.xero.get_provider") + @patch("apps.workflow.management.commands.xero.cache.set") + @patch("apps.workflow.management.commands.xero.AccountingApi") + @patch("apps.workflow.management.commands.xero.IdentityApi") + @patch("apps.workflow.management.commands.xero.CompanyDefaults.get_solo") + def test_production_setup_validates_without_creating_xero_items( + self, + mock_get_solo: Mock, + mock_identity_api_cls: Mock, + mock_accounting_api_cls: Mock, + _mock_cache_set: Mock, + mock_get_provider: Mock, + ) -> None: + theme_id = UUID("11111111-2222-3333-4444-555555555555") + company = SimpleNamespace( + xero_tenant_id=None, + xero_payroll_calendar_name="Weekly", + xero_shortcode=None, + xero_sales_branding_theme_id=theme_id, + xero_payroll_calendar_id=None, + save=Mock(), + ) + mock_get_solo.return_value = company + mock_identity_api_cls.return_value.get_connections.return_value = [ + SimpleNamespace(tenant_id="tenant-123", tenant_name="Live Company") + ] + mock_accounting_api_cls.return_value.get_organisations.return_value = ( + SimpleNamespace(organisations=[SimpleNamespace(short_code="LIVE")]) + ) + mock_get_provider.return_value.list_document_themes.return_value = [ + DocumentTheme( + external_id=str(theme_id), + name="Invoices", + is_default=False, + ) + ] + + cmd = Command() + with ( + patch.object(cmd, "_ensure_demo_xero_items_exist") as mock_ensure_demo, + patch.object(cmd, "_validate_production_xero_items") as mock_validate_prod, + patch( + "apps.workflow.management.commands.xero.get_payroll_calendars", + return_value=[{"name": "Weekly", "id": "calendar-live"}], + ), + ): + cmd.run_setup() + + mock_ensure_demo.assert_not_called() + mock_validate_prod.assert_called_once_with("Weekly") + self.assertEqual(company.xero_tenant_id, "tenant-123") self.assertEqual( company.save.call_args_list[-1].kwargs["update_fields"], [ @@ -248,6 +302,14 @@ def test_removed_create_missing_xero_items_flag_is_rejected(self): with self.assertRaises(CommandError): parser.parse_args(["--setup", "--create-missing-xero-items"]) + def test_seed_xero_requires_setup(self) -> None: + with patch( + "apps.workflow.management.commands.xero.get_valid_token", + return_value={"access_token": "token"}, + ): + with self.assertRaisesRegex(CommandError, "only valid with --setup"): + Command()._handle(seed_xero=True, setup=False) + class CompanyDefaultsBrandingThemeFixtureTests(TestCase): def test_shared_fixtures_do_not_ship_tenant_specific_theme_ids(self) -> None: diff --git a/apps/workflow/tests/test_xero_webhooks.py b/apps/workflow/tests/test_xero_webhooks.py index 603b83899..d55434469 100644 --- a/apps/workflow/tests/test_xero_webhooks.py +++ b/apps/workflow/tests/test_xero_webhooks.py @@ -24,8 +24,7 @@ from django.test import RequestFactory, TestCase, TransactionTestCase, override_settings from django.urls import reverse -from apps.workflow.exceptions import AlreadyLoggedException -from apps.workflow.models import XeroApp +from apps.workflow.models import AppError, XeroApp from apps.workflow.tasks import process_xero_webhook_event from apps.workflow.xero_webhooks import XeroWebhookView from docketworks.celery import app as celery_app @@ -245,21 +244,15 @@ def test_no_webhook_key_returns_503_and_persists_app_error(self) -> None: content_type="application/json", HTTP_X_XERO_SIGNATURE=_sign(body), ) - with ( - patch("apps.workflow.xero_webhooks.persist_app_error") as mock_persist, - patch.object(process_xero_webhook_event, "delay") as mock_delay, - ): - mock_persist.return_value = SimpleNamespace(id="ae-test-1") + with patch.object(process_xero_webhook_event, "delay") as mock_delay: response = cast(HttpResponse, XeroWebhookView.as_view()(request)) self.assertEqual(response.status_code, 503) self.assertIn(b"webhook_key", response.content) - self.assertIn(b"ae-test-1", response.content) - mock_persist.assert_called_once() - # The persisted exception is the actual config-error message, - # not a wrapping AlreadyLoggedException. - (persisted_exc,), _ = mock_persist.call_args - self.assertIsInstance(persisted_exc, RuntimeError) - self.assertIn("webhook_key", str(persisted_exc)) + # Exactly one row for the one config failure, and the 503 body + # carries its id so the operator can look it up. + app_error = AppError.objects.get() + self.assertIn("webhook_key", app_error.message) + self.assertIn(str(app_error.id).encode(), response.content) # Events must NOT be processed when validation can't even run. mock_delay.assert_not_called() @@ -280,11 +273,9 @@ def test_no_webhook_key_with_blank_rows_still_returns_503(self) -> None: content_type="application/json", HTTP_X_XERO_SIGNATURE=_sign(body), ) - with patch("apps.workflow.xero_webhooks.persist_app_error") as mock_persist: - mock_persist.return_value = SimpleNamespace(id="ae-test-2") - response = XeroWebhookView.as_view()(request) + response = XeroWebhookView.as_view()(request) self.assertEqual(response.status_code, 503) - mock_persist.assert_called_once() + self.assertEqual(AppError.objects.count(), 1) class ProcessXeroWebhookEventTaskTests(TestCase): @@ -294,13 +285,16 @@ class ProcessXeroWebhookEventTaskTests(TestCase): """ def _patch_company_defaults( - self, configured_tenant_id: str = TENANT_ID + self, + configured_tenant_id: str = TENANT_ID, + *, + enable_xero_sync: bool = True, ) -> AbstractContextManager[MagicMock]: return patch( "apps.workflow.tasks.CompanyDefaults.get_solo", return_value=SimpleNamespace( xero_tenant_id=configured_tenant_id, - enable_xero_sync=True, + enable_xero_sync=enable_xero_sync, xero_automated_day_floor=100, ), ) @@ -349,6 +343,19 @@ def test_wrong_tenant_skips_sync(self) -> None: mock_invoice.assert_not_called() mock_contact.assert_not_called() + def test_disabled_gate_skips_before_quota_or_sync_access(self) -> None: + with ( + self._patch_company_defaults(enable_xero_sync=False), + self._patch_sync_service() as mock_svc, + patch("apps.workflow.tasks.quota_floor_breached") as quota, + patch("apps.workflow.tasks.sync_single_invoice") as mock_invoice, + ): + process_xero_webhook_event(TENANT_ID, _event()) + + quota.assert_not_called() + mock_svc.assert_not_called() + mock_invoice.assert_not_called() + def test_unknown_event_category_does_not_dispatch(self) -> None: with ( self._patch_company_defaults(), @@ -367,7 +374,7 @@ def test_missing_required_fields_skips_without_raising(self) -> None: process_xero_webhook_event(TENANT_ID, bad_event) mock_svc.assert_not_called() - def test_sync_exception_persists_and_raises_already_logged(self) -> None: + def test_sync_exception_persists_and_reraises(self) -> None: from apps.workflow.models import AppError before = AppError.objects.count() @@ -380,7 +387,7 @@ def boom(sync_service: MagicMock, invoice_id: str) -> None: self._patch_sync_service(), patch("apps.workflow.tasks.sync_single_invoice", side_effect=boom), ): - with self.assertRaises(AlreadyLoggedException): + with self.assertRaises(RuntimeError): process_xero_webhook_event(TENANT_ID, _event()) after = AppError.objects.count() diff --git a/apps/workflow/urls.py b/apps/workflow/urls.py index 23d5783f1..f17f7ba25 100644 --- a/apps/workflow/urls.py +++ b/apps/workflow/urls.py @@ -7,7 +7,6 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter -from apps.workflow.api.enums import get_enum_choices from apps.workflow.views.ai_provider_viewset import AIProviderViewSet from apps.workflow.views.app_error_grouped_view import ( AppErrorGroupedListView, @@ -28,6 +27,7 @@ from apps.workflow.views.company_defaults_logo_api import CompanyDefaultsLogoAPIView from apps.workflow.views.company_defaults_schema_api import CompanyDefaultsSchemaAPIView from apps.workflow.views.data_versions_view import DataVersionsAPIView +from apps.workflow.views.notebook_lm_link_viewset import NotebookLmLinkViewSet from apps.workflow.views.search_telemetry_view import SearchTelemetryClickAPIView from apps.workflow.views.session_replay_view import ( SessionReplayChunkCreateView, @@ -46,6 +46,7 @@ # --------------------------------------------------------------------------- router = DefaultRouter() router.register("ai-providers", AIProviderViewSet, basename="ai-provider") +router.register("notebook-lm-links", NotebookLmLinkViewSet, basename="notebook-lm-link") router.register("app-errors", AppErrorViewSet, basename="app-error") router.register("xero-pay-items", XeroPayItemViewSet, basename="xero-pay-item") router.register("xero-apps", XeroAppViewSet, basename="xero-app") @@ -83,7 +84,6 @@ SessionReplayFrontendErrorView.as_view(), name="session-replay-frontend-error", ), - path("enums//", get_enum_choices, name="get_enum_choices"), path( "xero/authenticate/", xero_view.xero_authenticate, diff --git a/apps/workflow/utils.py b/apps/workflow/utils.py index c110c84d1..317a14d65 100644 --- a/apps/workflow/utils.py +++ b/apps/workflow/utils.py @@ -83,8 +83,8 @@ def parse_pagination_params(request) -> tuple[int, int]: try: limit = int(request.query_params.get("limit", "50")) offset = int(request.query_params.get("offset", "0")) - except (TypeError, ValueError): - raise ValueError("Invalid pagination parameters") + except (TypeError, ValueError) as exc: + raise ValueError("Invalid pagination parameters") from exc return limit, offset diff --git a/apps/workflow/views/app_error_view.py b/apps/workflow/views/app_error_view.py index 58463b85b..5cf36d842 100644 --- a/apps/workflow/views/app_error_view.py +++ b/apps/workflow/views/app_error_view.py @@ -132,8 +132,8 @@ def _cast_uuid(value: str | None, field: str) -> str | None: return None try: return str(UUID(str(value))) - except ValueError: - raise ValueError(f"Invalid {field} parameter") + except ValueError as exc: + raise ValueError(f"Invalid {field} parameter") from exc try: job_id = _cast_uuid(request.query_params.get("job_id"), "job_id") diff --git a/apps/workflow/views/data_versions_view.py b/apps/workflow/views/data_versions_view.py index 04761d090..deb836e16 100644 --- a/apps/workflow/views/data_versions_view.py +++ b/apps/workflow/views/data_versions_view.py @@ -27,7 +27,6 @@ from apps.crm.models import PhoneCallRecord, PhoneCallRecording from apps.job.models import Job from apps.purchasing.models import Stock -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error @@ -98,11 +97,9 @@ def get(self, request: Request) -> Response: payload = { key: provider() for key, provider in DATASET_VERSION_PROVIDERS.items() } - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise response = Response(payload) response["Cache-Control"] = "no-store" diff --git a/apps/workflow/views/notebook_lm_link_viewset.py b/apps/workflow/views/notebook_lm_link_viewset.py new file mode 100644 index 000000000..6e8b69008 --- /dev/null +++ b/apps/workflow/views/notebook_lm_link_viewset.py @@ -0,0 +1,40 @@ +from drf_spectacular.utils import extend_schema +from rest_framework import permissions, viewsets +from rest_framework.decorators import action +from rest_framework.request import Request +from rest_framework.response import Response + +from apps.accounts.models import Staff +from apps.job.permissions import IsOfficeStaff +from apps.workflow.enums import NotebookLmRestriction +from apps.workflow.models import NotebookLmLink +from apps.workflow.serializers import NotebookLmLinkSerializer + + +class NotebookLmLinkViewSet(viewsets.ModelViewSet[NotebookLmLink]): + """CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + """ + + permission_classes = [permissions.IsAuthenticated, IsOfficeStaff] + queryset = NotebookLmLink.objects.all() + serializer_class = NotebookLmLinkSerializer + + @extend_schema(responses={200: NotebookLmLinkSerializer(many=True)}) + @action( + detail=False, + methods=["get"], + permission_classes=[permissions.IsAuthenticated], + ) + def menu(self, request: Request) -> Response: + user = request.user + include_restricted = isinstance(user, Staff) and user.is_superuser + links = NotebookLmLink.objects.filter(enabled=True) + if not include_restricted: + links = links.exclude(restriction=NotebookLmRestriction.SUPERUSER) + serializer = self.get_serializer(links, many=True) + return Response(serializer.data) diff --git a/apps/workflow/views/search_telemetry_view.py b/apps/workflow/views/search_telemetry_view.py index c4e9d03d7..a164627c3 100644 --- a/apps/workflow/views/search_telemetry_view.py +++ b/apps/workflow/views/search_telemetry_view.py @@ -8,7 +8,6 @@ from rest_framework.response import Response from rest_framework.views import APIView -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.search_telemetry_serializers import ( SearchTelemetryClickRequestSerializer, SearchTelemetryClickResponseSerializer, @@ -20,19 +19,17 @@ def _build_server_error_response(*, message: str, exc: Exception) -> Response: - if isinstance(exc, AlreadyLoggedException): - root_exc = exc.original - error_id = exc.app_error_id - else: - root_exc = exc - app_error = persist_app_error(exc) - error_id = getattr(app_error, "id", None) + # Idempotent: returns the existing row if this failure was persisted deeper + # in the stack, so there is always exactly one id to report. + app_error = persist_app_error(exc) - logger.error("%s: %s", message, root_exc) + logger.error("%s: %s", message, exc) - payload: Dict[str, Any] = {"error": message, "details": str(root_exc)} - if error_id: - payload["error_id"] = str(error_id) + payload: Dict[str, Any] = { + "error": message, + "details": str(exc), + "error_id": str(app_error.id), + } return Response(payload, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/apps/workflow/views/xero/xero_base_manager.py b/apps/workflow/views/xero/xero_base_manager.py index fad2a5d80..768877f6e 100644 --- a/apps/workflow/views/xero/xero_base_manager.py +++ b/apps/workflow/views/xero/xero_base_manager.py @@ -20,7 +20,7 @@ class XeroDocumentResponse(TypedDict, total=False): Only *expected* outcomes travel as a value: success, or a business failure the caller renders as a 4xx. Unexpected exceptions are persisted once and - re-raised as ``AlreadyLoggedException`` (ADR 0001) — they never appear here. + re-raised unchanged (ADR 0001) — they never appear here. ``success`` is present on every response. """ @@ -126,6 +126,14 @@ def get_xero_sales_branding_theme_id() -> str | None: return None return str(theme_id) + @staticmethod + def get_xero_quote_terms() -> str | None: + """Return the terms explicitly sent on API-created Xero quotes.""" + terms = CompanyDefaults.get_solo().xero_quote_terms + if not terms.strip(): + return None + return terms + def validate_company(self): """ Ensures the company exists and is synced with Xero. diff --git a/apps/workflow/views/xero/xero_invoice_manager.py b/apps/workflow/views/xero/xero_invoice_manager.py index ce1101255..b68b06328 100644 --- a/apps/workflow/views/xero/xero_invoice_manager.py +++ b/apps/workflow/views/xero/xero_invoice_manager.py @@ -16,7 +16,6 @@ from apps.job.models.costing import CostSet from apps.job.services.workshop_pdf_service import create_workshop_pdf from apps.workflow.accounting.types import DocumentLineItem, InvoicePayload -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error # Import base class and helpers @@ -326,15 +325,13 @@ def create_document( result_dict["messages"] = messages_list return result_dict - except AlreadyLoggedException: - raise except Exception as exc: job_id = self.job.id if self.job else "Unknown" logger.exception( f"Unexpected error during invoice creation for job {job_id}" ) - err = persist_app_error(exc, job_id=str(job_id)) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc, job_id=str(job_id)) + raise def delete_document(self) -> XeroDocumentResponse: """Deletes an invoice via the provider and removes the local record.""" @@ -386,12 +383,10 @@ def delete_document(self) -> XeroDocumentResponse: "message": "Invoice deleted successfully.", } - except AlreadyLoggedException: - raise except Exception as exc: job_id = self.job.id if self.job else "Unknown" logger.exception( f"Unexpected error during invoice deletion for job {job_id}" ) - err = persist_app_error(exc, job_id=str(job_id)) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc, job_id=str(job_id)) + raise diff --git a/apps/workflow/views/xero/xero_quote_manager.py b/apps/workflow/views/xero/xero_quote_manager.py index 9187960e4..82194a704 100644 --- a/apps/workflow/views/xero/xero_quote_manager.py +++ b/apps/workflow/views/xero/xero_quote_manager.py @@ -17,7 +17,6 @@ from apps.accounting.models import Quote from apps.job.models.costing import CostSet from apps.workflow.accounting.types import DocumentLineItem, QuotePayload -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.services.error_persistence import persist_app_error # Import base class and helpers @@ -133,6 +132,7 @@ def build_payload( breakdown: bool = True, *, document_theme_external_id: str, + terms: str, ) -> QuotePayload: """Build a provider-agnostic quote payload from the job and company.""" if not self.job: @@ -148,6 +148,7 @@ def build_payload( date=today, expiry_date=today + timedelta(days=30), document_theme_external_id=document_theme_external_id, + terms=terms, reference=( self.job.order_number if hasattr(self.job, "order_number") and self.job.order_number @@ -181,9 +182,34 @@ def create_document(self, breakdown: bool = True) -> XeroDocumentResponse: "status": 400, } + terms = self.get_xero_quote_terms() + if terms is None: + return { + "success": False, + "error": ( + "Configure Xero quote terms in Company Settings before " + "creating a quote. Xero does not apply its quote terms " + "default to API-created quotes." + ), + "error_type": "configuration_error", + "status": 400, + } + + if len(terms) > 4000: + return { + "success": False, + "error": ( + "Xero quote terms must be no more than 4000 characters. " + "Shorten them in Company Settings before creating a quote." + ), + "error_type": "configuration_error", + "status": 400, + } + payload = self.build_payload( breakdown=breakdown, document_theme_external_id=document_theme_external_id, + terms=terms, ) result = self.provider.create_quote(payload) @@ -242,13 +268,11 @@ def create_document(self, breakdown: bool = True) -> XeroDocumentResponse: "online_url": result.online_url, } - except AlreadyLoggedException: - raise except Exception as exc: job_id = self.job.id if self.job else "Unknown" logger.exception(f"Unexpected error during quote creation for job {job_id}") - err = persist_app_error(exc, job_id=str(job_id)) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc, job_id=str(job_id)) + raise def delete_document(self) -> XeroDocumentResponse: """Deletes a quote via the provider and removes the local record.""" @@ -310,10 +334,8 @@ def delete_document(self) -> XeroDocumentResponse: "messages": ["Quote deleted successfully."], } - except AlreadyLoggedException: - raise except Exception as exc: job_id = self.job.id if self.job else "Unknown" logger.exception(f"Unexpected error during quote deletion for job {job_id}") - err = persist_app_error(exc, job_id=str(job_id)) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc, job_id=str(job_id)) + raise diff --git a/apps/workflow/views/xero/xero_view.py b/apps/workflow/views/xero/xero_view.py index 169c92f5d..59ffc2d63 100644 --- a/apps/workflow/views/xero/xero_view.py +++ b/apps/workflow/views/xero/xero_view.py @@ -57,9 +57,6 @@ refresh_token, ) from apps.workflow.api.xero.sync import ENTITY_CONFIGS -from apps.workflow.exceptions import ( - AlreadyLoggedException, -) from apps.workflow.models import XeroError, XeroPayItem from apps.workflow.serializers import ( XeroAuthenticationErrorResponseSerializer, @@ -76,7 +73,7 @@ XeroTriggerSyncResponseSerializer, ) from apps.workflow.services.error_persistence import ( - persist_and_raise, + app_error_for, persist_app_error, ) from apps.workflow.services.xero_sync_service import XeroSyncService @@ -125,7 +122,8 @@ def _build_xero_error_payload( """ Generate a consistent error payload and persist the exception if needed. """ - error_id = exc.app_error_id if isinstance(exc, AlreadyLoggedException) else None + app_error = app_error_for(exc) + error_id = app_error.id if app_error is not None else None payload: dict[str, object] = {"success": False, "error": message} if error_id: @@ -442,19 +440,12 @@ def list_xero_branding_themes(request: Request) -> Response: XeroBrandingThemeSerializer(theme).data for theme in themes ] return Response(serialized_themes, status=status.HTTP_200_OK) - except AlreadyLoggedException as exc: + except Exception as exc: + persist_app_error(exc, user_id=str(request.user.id)) return Response( _build_xero_error_payload(exc), status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - except Exception as exc: - try: - persist_and_raise(exc, user_id=str(request.user.id)) - except AlreadyLoggedException as logged_exc: - return Response( - _build_xero_error_payload(logged_exc), - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) @csrf_exempt @@ -539,21 +530,21 @@ def create_xero_invoice(request: Request, job_id: uuid.UUID) -> Response: } messages.error(request, str(exc)) return Response(error_data, status=status.HTTP_400_BAD_REQUEST) - except AlreadyLoggedException as exc: - error_data = _build_xero_error_payload( - exc, - message=f"An unexpected error occurred ({exc}) while creating " - "the invoice. Please contact support to check the data sent.", - ) - messages.error(request, "An unexpected error occurred while creating invoice.") - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: - try: - persist_and_raise(exc, job_id=str(job_id)) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) - messages.error(request, f"An unexpected error occurred: {str(exc)}") + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload( + exc, + message=f"An unexpected error occurred ({exc}) while creating " + "the invoice. Please contact support to check the data sent.", + ) + messages.error( + request, "An unexpected error occurred while creating invoice." + ) return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + persist_app_error(exc, job_id=str(job_id)) + error_data = _build_xero_error_payload(exc) + messages.error(request, f"An unexpected error occurred: {str(exc)}") + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @csrf_exempt @@ -620,37 +611,35 @@ def create_xero_purchase_order( error_type = result_data.get("error_type", "unknown") status_code = result_data.get("status", 400) - try: - persist_and_raise( - ValueError(error_msg), - user_id=( - str(request.user.id) if request.user.is_authenticated else None - ), - additional_context={ - "purchase_order_id": str(purchase_order_id), - "error_type": error_type, - "result_data": result_data, - "request_path": request.path, - "request_method": request.method, - }, - ) - except AlreadyLoggedException as logged_exc: - messages.error(request, f"Failed to sync Purchase Order: {error_msg}") - logger.warning( - f"Failed to sync PO {purchase_order_id}: {error_msg}", - extra={ - "purchase_order_id": str(purchase_order_id), - "error_message": error_msg, - "error_type": error_type, - "result_data": result_data, - }, - ) - serializer = XeroDocumentErrorResponseSerializer(data=result_data) - serializer.is_valid(raise_exception=True) - payload = serializer.data - if logged_exc.app_error_id: - payload["error_id"] = str(logged_exc.app_error_id) - return Response(payload, status=status_code) + logged_error = persist_app_error( + ValueError(error_msg), + user_id=( + str(request.user.id) if request.user.is_authenticated else None + ), + additional_context={ + "purchase_order_id": str(purchase_order_id), + "error_type": error_type, + "result_data": result_data, + "request_path": request.path, + "request_method": request.method, + }, + ) + messages.error(request, f"Failed to sync Purchase Order: {error_msg}") + logger.warning( + f"Failed to sync PO {purchase_order_id}: {error_msg}", + extra={ + "purchase_order_id": str(purchase_order_id), + "error_message": error_msg, + "error_type": error_type, + "result_data": result_data, + }, + ) + serializer = XeroDocumentErrorResponseSerializer(data=result_data) + serializer.is_valid(raise_exception=True) + payload = serializer.data + if logged_error.id: + payload["error_id"] = str(logged_error.id) + return Response(payload, status=status_code) # Handle happy case messages.success(request, "Purchase Order synced successfully with Xero.") @@ -682,21 +671,6 @@ def create_xero_purchase_order( } messages.error(request, error_data["error"]) return Response(error_data, status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException as exc: - logger.exception( - f"Unexpected error in create_xero_purchase_order view for PO {purchase_order_id}", - extra={ - "purchase_order_id": str(purchase_order_id), - "user_id": ( - str(request.user.id) if request.user.is_authenticated else None - ), - "error_type": type(exc.original).__name__, - "error_message": str(exc.original), - }, - ) - error_data = _build_xero_error_payload(exc) - messages.error(request, "An unexpected error occurred while syncing PO.") - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: logger.exception( f"Unexpected error in create_xero_purchase_order view for PO {purchase_order_id}", @@ -709,20 +683,22 @@ def create_xero_purchase_order( "error_message": str(exc), }, ) - try: - persist_and_raise( - exc, - user_id=str(request.user.id) if request.user.is_authenticated else None, - additional_context={ - "purchase_order_id": str(purchase_order_id), - "request_path": request.path, - "request_method": request.method, - }, - ) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) - messages.error(request, f"An unexpected error occurred: {str(exc)}") + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload(exc) + messages.error(request, "An unexpected error occurred while syncing PO.") return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + persist_app_error( + exc, + user_id=str(request.user.id) if request.user.is_authenticated else None, + additional_context={ + "purchase_order_id": str(purchase_order_id), + "request_path": request.path, + "request_method": request.method, + }, + ) + error_data = _build_xero_error_payload(exc) + messages.error(request, f"An unexpected error occurred: {str(exc)}") + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @csrf_exempt @@ -781,19 +757,17 @@ def create_xero_quote(request: Request, job_id: uuid.UUID) -> Response: except Job.DoesNotExist: error_data = {"success": False, "error": f"Job with ID {job_id} not found."} return Response(error_data, status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException as exc: - error_data = _build_xero_error_payload( - exc, - message=f"An unexpected error occurred ({exc}) while creating " - "the quote. Please contact support.", - ) - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: - try: - persist_and_raise(exc, job_id=str(job_id)) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload( + exc, + message=f"An unexpected error occurred ({exc}) while creating " + "the quote. Please contact support.", + ) return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + persist_app_error(exc, job_id=str(job_id)) + error_data = _build_xero_error_payload(exc) + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @csrf_exempt @@ -868,19 +842,17 @@ def delete_xero_invoice(request: Request, job_id: uuid.UUID) -> Response: "error": f"Invoice with Xero ID {xero_invoice_id} not found for this job.", } return Response(error_data, status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException as exc: - error_data = _build_xero_error_payload( - exc, - message=f"An unexpected error occurred ({exc}) while deleting " - "the invoice. Please contact support.", - ) - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: - try: - persist_and_raise(exc, job_id=str(job_id)) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload( + exc, + message=f"An unexpected error occurred ({exc}) while deleting " + "the invoice. Please contact support.", + ) return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + persist_app_error(exc, job_id=str(job_id)) + error_data = _build_xero_error_payload(exc) + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @csrf_exempt @@ -927,23 +899,22 @@ def delete_xero_quote(request: Request, job_id: uuid.UUID) -> Response: error_data = {"success": False, "error": f"Job with ID {job_id} not found."} messages.error(request, error_data["error"]) return Response(error_data, status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException as exc: - logger.exception(f"Error in delete_xero_quote view for job {job_id}") - error_data = _build_xero_error_payload( - exc, - message=f"An unexpected error occurred ({exc}) while deleting " - "the quote. Please contact support.", - ) - messages.error(request, "An unexpected error occurred while deleting quote.") - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: logger.exception(f"Error in delete_xero_quote view for job {job_id}") - try: - persist_and_raise(exc, job_id=str(job_id)) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) - messages.error(request, f"An unexpected error occurred: {str(exc)}") + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload( + exc, + message=f"An unexpected error occurred ({exc}) while deleting " + "the quote. Please contact support.", + ) + messages.error( + request, "An unexpected error occurred while deleting quote." + ) return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + persist_app_error(exc, job_id=str(job_id)) + error_data = _build_xero_error_payload(exc) + messages.error(request, f"An unexpected error occurred: {str(exc)}") + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @csrf_exempt @@ -994,27 +965,21 @@ def delete_xero_purchase_order( } messages.error(request, error_data["error"]) return Response(error_data, status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException as exc: - logger.exception( - f"Error in delete_xero_purchase_order view for PO {purchase_order_id}" - ) - error_data = _build_xero_error_payload(exc) - messages.error(request, "An unexpected error occurred while deleting PO.") - return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as exc: logger.exception( f"Error in delete_xero_purchase_order view for PO {purchase_order_id}" ) - try: - persist_and_raise( - exc, - additional_context={"purchase_order_id": str(purchase_order_id)}, - ) - except AlreadyLoggedException as logged_exc: - error_data = _build_xero_error_payload(logged_exc) - messages.error(request, f"An unexpected error occurred: {str(exc)}") + if app_error_for(exc) is not None: + error_data = _build_xero_error_payload(exc) + messages.error(request, "An unexpected error occurred while deleting PO.") return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - raise AssertionError("persist_and_raise returned without raising") from exc + persist_app_error( + exc, + additional_context={"purchase_order_id": str(purchase_order_id)}, + ) + error_data = _build_xero_error_payload(exc) + messages.error(request, f"An unexpected error occurred: {str(exc)}") + return Response(error_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @extend_schema( @@ -1047,8 +1012,6 @@ def xero_disconnect(request): _xero_ping_payload(connected=False), status=status.HTTP_200_OK, ) - except AlreadyLoggedException: - raise except Exception as exc: persist_app_error(exc) logger.error(f"Error disconnecting from Xero: {str(exc)}") @@ -1240,16 +1203,6 @@ def xero_ping(request): _xero_ping_payload(connected=is_connected), status=status.HTTP_200_OK, ) - except AlreadyLoggedException as exc: - logger.error("Error in xero_ping: %s", exc) - return Response( - { - "connected": False, - "error": str(exc), - "error_id": exc.app_error_id, - }, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) except Exception as exc: logger.error("Error in xero_ping: %s", exc) app_error = persist_app_error(exc) diff --git a/apps/workflow/views/xero_apps_view.py b/apps/workflow/views/xero_apps_view.py index f22756579..15569d036 100644 --- a/apps/workflow/views/xero_apps_view.py +++ b/apps/workflow/views/xero_apps_view.py @@ -22,7 +22,6 @@ swap_active, wipe_tokens_and_quota, ) -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import CompanyDefaults, XeroApp from apps.workflow.serializers import XeroAppCreateSerializer, XeroAppSerializer from apps.workflow.services.error_persistence import persist_app_error @@ -119,11 +118,9 @@ def activate(self, request, pk=None): target = swap_active(pk) except XeroApp.DoesNotExist: return Response(status=status.HTTP_404_NOT_FOUND) - except AlreadyLoggedException: - raise except Exception as exc: - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise # swap_active dispatched a detached `systemctl restart` for the # worker units — gunicorn (this process) included. The HTTP response # gets out before systemd kills us; the operator's next request diff --git a/apps/workflow/xero_webhooks.py b/apps/workflow/xero_webhooks.py index 9d622345a..c630e6192 100644 --- a/apps/workflow/xero_webhooks.py +++ b/apps/workflow/xero_webhooks.py @@ -19,7 +19,6 @@ from django.views import View from django.views.decorators.csrf import csrf_exempt -from apps.workflow.exceptions import AlreadyLoggedException from apps.workflow.models import XeroApp from apps.workflow.services.error_persistence import persist_app_error from apps.workflow.tasks import process_xero_webhook_event @@ -39,7 +38,7 @@ def validate_webhook_signature(request: HttpRequest) -> bool: the operator hasn't deleted it in the Xero portal, that's fine: we process them. Cleaning up orphan apps in Xero is the operator's job. - Raises ``AlreadyLoggedException`` (after persisting an AppError) if + Raises ``RuntimeError`` (after persisting an AppError) if no XeroApp row has a webhook_key set. That state is a deploy-time misconfiguration, not a request-time bad-signature event — surfacing it via AppError gets it in front of an operator instead of leaving @@ -59,8 +58,8 @@ def validate_webhook_signature(request: HttpRequest) -> bool: "signatures. Set webhook_key via the Xero Apps admin UI or " "the per-install fixture and redeploy." ) - err = persist_app_error(exc) - raise AlreadyLoggedException(exc, err.id) from exc + persist_app_error(exc) + raise exc body = request.body for key in keys: @@ -81,15 +80,18 @@ class XeroWebhookView(View): def post(self, request: HttpRequest) -> HttpResponse: try: valid = validate_webhook_signature(request) - except AlreadyLoggedException as exc: + except RuntimeError as exc: # Config error already persisted as AppError. Return 503 so # Xero treats this as a transient failure and retries — by # the time the operator notices and fixes the config, the # backlog of redelivered events still gets processed. A 4xx # would tell Xero "stop trying", which is the wrong signal # for a fixable config bug. + # Idempotent — validate_webhook_signature already persisted this, + # so this returns that same row rather than writing a second. + err = persist_app_error(exc) return HttpResponse( - f"Service Unavailable: {exc} (error_id={exc.app_error_id})", + f"Service Unavailable: {exc} (error_id={err.id})", status=503, ) if not valid: diff --git a/docketworks/settings_test.py b/docketworks/settings_test.py index 50fada16d..7089a4880 100644 --- a/docketworks/settings_test.py +++ b/docketworks/settings_test.py @@ -9,6 +9,9 @@ import os +# Django settings modules are consumed by attribute lookup, so the star-import +# is the mechanism, not a shortcut: every base setting must land in this +# module's namespace before the overrides below narrow the DB credentials. from .settings import * # noqa: F401, F403 from .settings import DATABASES diff --git a/docs/adr/0001-exception-already-logged-dedup.md b/docs/adr/0001-exception-already-logged-dedup.md index 5c33ce188..6cf69eb57 100644 --- a/docs/adr/0001-exception-already-logged-dedup.md +++ b/docs/adr/0001-exception-already-logged-dedup.md @@ -1,26 +1,45 @@ -# 0001 — Exception deduplication via AlreadyLoggedException +# 0001 — Idempotent error persistence -Wrap once-persisted exceptions in `AlreadyLoggedException`; nested handlers re-raise unchanged instead of re-persisting. +`persist_app_error` marks the exception it persists and returns the existing row on any later call, so one failure is one `AppError` row no matter how many handlers catch it. ## Problem -Exceptions travel integration → service → view → scheduler. Every layer has its own `try/except` that calls `persist_app_error()`. Without a marker, the same failure gets persisted 4–5 times — one bug, four duplicate `AppError` rows. Scheduler jobs sit outside request middleware, so any "just centralise it in one outer handler" plan cannot reach them. +Exceptions travel integration → service → view → scheduler. Every layer has its own `try/except` that calls `persist_app_error()`. Without deduplication the same failure gets persisted 4–5 times — one bug, four duplicate `AppError` rows. Scheduler jobs and management commands sit outside request middleware, so any "just centralise it in one outer handler" plan cannot reach them. ## Decision -`AlreadyLoggedException` (in `apps/workflow/exceptions.py`) wraps the original exception plus the persisted `AppError.id`. Every handler is two-arm: re-raise `AlreadyLoggedException` unchanged; otherwise persist once, wrap, re-raise. `persist_app_error()` returns the `AppError` instance so callers can carry the id forward. +`persist_app_error()` records the created `AppError` on the exception instance (a `__app_error__` marker) and, on any subsequent call for the same failure, returns that existing row instead of writing a new one. The dedup is idempotency in the function, not a discipline every handler has to follow. A handler is one arm: -The chain terminates at the HTTP boundary: the outermost view handler catches `AlreadyLoggedException` and converts it to a response (carrying `error_id`, per ADR 0013) instead of re-raising. Service objects invoked by views always re-raise; they never shape responses. A service object returns a failure value only for an *expected* business outcome, never for an unexpected exception. +```python +try: + operation() +except Exception as exc: + persist_app_error(exc, job_id=job.id) # the context is why this handler exists + raise +``` + +Lookup walks the `__cause__` chain, so a handler that *converts* the exception keeps the failure to one row as long as it chains the cause: + +```python +except Job.DoesNotExist as exc: + persist_app_error(exc) + raise ValueError(f"Job {job_id} not found") from exc +``` + +The `from exc` is load-bearing: it is what links the converted exception back to the persisted one (pylint `W0707` enforces it repo-wide). Without the link the two are distinct objects and the second earns its own row. + +The chain terminates at the HTTP boundary. The outermost handler chooses the response *status* from the exception's real type (`isinstance`, most-specific first) and reads the persisted id with `app_error_for(exc)` to include `error_id` in the body (ADR 0013). Service objects invoked by views always re-raise; they never shape responses. A service returns a failure value only for an *expected* business outcome, never for an unexpected exception. ## Why -A marker exception is in-band — it works identically in views, services, schedulers, and management commands, so every handler in the codebase follows the same two-arm template. Reviewers and new handlers have one rule to remember. The id carries forward so an outer handler can correlate without re-querying. +Idempotency in the function means the guarantee holds no matter how many handlers a failure passes through — the caller cannot get it wrong by forgetting an arm. The exception keeps its real type all the way to the boundary, which is what lets the boundary map it to 404 / 409 / 412 / 503 rather than a blanket 500. It works identically in views, services, schedulers, and management commands, since it depends on nothing request-scoped. ## Alternatives considered -- **Centralise persistence in middleware / a single outer handler.** Standard for request-scoped flows. Rejected: cannot reach scheduler jobs (no middleware), and the layer-local context that made the persisted message useful is gone. +- **A marker wrapper (`AlreadyLoggedException`), the previous decision here.** Every handler was two-arm: re-raise the wrapper unchanged, else persist-wrap-raise. Rejected: wrapping conflated *metadata about* an exception ("already persisted") with the exception's *identity*, so carrying the marker destroyed the type the boundary needs to pick a status code. It also required every one of ~900 handlers to remember the pattern — and in practice most did not: the majority of call sites persisted and re-raised without wrapping, so the same failure double-logged anyway. Marking the exception instead of replacing it gives the same one-row guarantee without either cost. +- **Centralise persistence in middleware / a single outer handler.** Standard for request-scoped flows. Rejected: cannot reach scheduler jobs and management commands (no middleware), and the layer-local context that made the persisted message useful is gone. - **Suppress duplicates at the DB layer via a content hash.** Common in heavy-throughput logging systems. Rejected: loses signal — which layer caught it, what message that layer produced — and does nothing for the scheduler-persistence gap. ## Consequences -One `AppError` row per real failure; scheduler errors survive log rotation. Every new `try/except` must know the `AlreadyLoggedException` arm or risk double-persisting. +One `AppError` row per real failure, guaranteed by `persist_app_error` rather than by every handler remembering a ritual; scheduler and management-command errors survive log rotation. `raise ... from exc` is mandatory on any handler that converts an exception, or the converted failure is persisted a second time. The exception's type reaches the boundary intact, so status-code mapping is `isinstance`-based, not name-string-based. diff --git a/docs/adr/0019-mandatory-error-persistence.md b/docs/adr/0019-mandatory-error-persistence.md index 275906545..f626903ed 100644 --- a/docs/adr/0019-mandatory-error-persistence.md +++ b/docs/adr/0019-mandatory-error-persistence.md @@ -1,6 +1,6 @@ # 0019 — Every exception is persisted to AppError -Every `except` block in the codebase calls `persist_app_error(exc)` — errors live in postgres, not stdout — before re-raising via the dedup pattern in ADR 0001. +Every `except` block in the codebase calls `persist_app_error(exc)` — errors live in postgres, not stdout — before re-raising. `persist_app_error` is idempotent (ADR 0001), so re-raising directly cannot double-persist. ## Problem @@ -8,7 +8,9 @@ Errors logged to stdout/stderr survive only as long as log retention. A schedule ## Decision -Every `except` block calls `persist_app_error(exc)`, which stores the message, traceback, request context, and a UUID id in the `AppError` table. The handler then re-raises through the two-arm dedup pattern (ADR 0001) so the same failure isn't persisted twice as it travels up the stack. Continuation without re-raise is allowed only when business logic explicitly requires it. +Every `except` block calls `persist_app_error(exc)`, which stores the message, traceback, request context, and a UUID id in the `AppError` table. The handler then re-raises directly. `persist_app_error` is idempotent — it marks the exception it persists and returns the existing row on any later call (ADR 0001) — so the same failure isn't persisted twice as it travels up the stack, even though every layer calls it. Continuation without re-raise is allowed only when business logic explicitly requires it. + +This governs `except` blocks that exist; it is not an instruction to introduce them. A `try` needs a strong reason: you are going to **handle** the failure. Converting its shape is handling — into a domain error, or into an HTTP status at the boundary. Being the layer that understands the failure well enough to persist it with real business context is handling. Absent one of those, let it raise to a layer that has one. ## Why diff --git a/docs/adr/0029-servers-run-the-production-branch.md b/docs/adr/0029-servers-run-the-production-branch.md index 990c4099a..9a92893bb 100644 --- a/docs/adr/0029-servers-run-the-production-branch.md +++ b/docs/adr/0029-servers-run-the-production-branch.md @@ -1,6 +1,6 @@ # 0029 — Servers run the production branch -Servers only ever deploy `production`; `main` is the integration branch, and releasing is an explicit merge from `main` into `production`. +Servers deploy `production` by default; `main` is the integration branch, released by an explicit merge from `main` into `production` and verified on UAT as a candidate (`deploy.sh --ref`) beforehand. ## Problem @@ -26,3 +26,4 @@ Separating "integrated" from "released" makes the deployable state an explicit, - Hotfixes must be back-merged to `main` in the same working session; a fix that exists only on `production` is a regression waiting to be re-released. - `production` carries the same branch protections as `main`. - Anything that assumes servers track `main` (docs, boot-time catch-up units, operator habit) must say `production` instead. +- `deploy.sh --ref` and `instance.sh create --ref` deploy a candidate ref (e.g. `origin/main`) to UAT; `instance.sh status ` reports which tracked ref a running instance matches. A non-production ref on a `*-prod` instance is refused unless acknowledged — interactive confirm, or `--allow-prod-ref` non-interactively. Nothing persists the ref per instance, so boot-time catch-up still returns UAT to `production`. diff --git a/docs/adr/0031-single-logging-gate-debug-namespaces.md b/docs/adr/0031-single-logging-gate-debug-namespaces.md new file mode 100644 index 000000000..7cac61e6b --- /dev/null +++ b/docs/adr/0031-single-logging-gate-debug-namespaces.md @@ -0,0 +1,43 @@ +# 0031 — One logging gate: the `debug` library with namespaces + +All frontend and E2E diagnostic logging flows through the `debug` library under a `:` namespace; there is one gate and one enable mechanism. + +## Status + +Accepted + +## Context + +The Vue app and the Playwright suite carried two uncoordinated logging mechanisms: a homegrown `src/utils/debug.ts` wrapper on the app side and ad-hoc `console.log` plus per-test `page.on('console')` handlers on the test side. Enabling diagnostics was global — a single on/off flag with no way to select one feature — so turning logging on drowned the signal, and dev-mode E2E runs were buried under app narration surfaced by scattered console forwarders. The homegrown wrapper reimplemented, worse, exactly what the `debug` library already provides: per-namespace selection, wildcard enabling, and zero cost when disabled. Retiring it in favour of the library is a first application of ADR 0032 (prefer libraries over homegrown implementations). + +## Decision + +Diagnostic logging goes through the `debug` library and nothing else, on both the app and test sides. + +- Each module declares one namespaced logger: `import debug from 'debug'; const log = debug('job:autosave')`. Namespaces are `:`, lower-kebab, colon depth ≤ 2, feature-scoped — files serving one feature share a namespace. App domains in use: `app auth api job kanban po timesheet xero quote company person cost report workshop staff ai session search settings admin`. The test side uses `e2e:`. +- The homegrown `src/utils/debug.ts` (`debugLog`) is deleted and every call site migrated to `debug` in one PR — no alias, no shim, no catch-all namespace (ADR 0017). +- Logging is silent by default. Enable in the browser with `localStorage.debug='job:*'`; enable for node E2E with `DEBUG=e2e:kanban`. +- App→test log surfacing goes through one gated, namespaced forwarder (`frontend/tests/fixtures/debug-forwarder.ts`, wired into the `page` fixture in `frontend/tests/fixtures/auth.ts`), opt-in via `DEBUG=e2e:`. Per-test `page.on('console')` handlers are not used. + +Existing and legacy log statements follow a three-way rule: + +1. **Never delete genuine feature narration.** Gate it behind a namespace so it is silent-but-preserved. +2. **Delete a log** only when it is redundant with a neighbouring assertion or the failure trace. +3. **Keep ungated** the bad-state / error-branch / skip-notice logs — they only fire when something is already wrong. + +## Why + +One gate with per-feature selectivity is the whole point: a developer enables exactly the namespace they are debugging and sees only that, in the browser or in an E2E run, without editing code. Collapsing two mechanisms into one removes the question "which logger does this module use?" and the taxonomy makes the answer to "which namespace?" mechanical. Routing app→test surfacing through a single opt-in forwarder means a dev-mode E2E run is quiet unless explicitly asked for a given area, instead of being shaped by whatever `console.on` handlers happen to be installed. Adopting the `debug` library rather than maintaining a wrapper deletes code we owned that did the same job less well. + +## Alternatives considered + +- **Keep the homegrown wrapper, add per-feature flags:** re-grows the exact selection and wildcard machinery `debug` ships, as maintained code, for no gain. +- **Structured logger (pino/winston) on the app side:** built for server log pipelines and shipping; overweight for browser diagnostics whose only consumer is a developer with devtools open. `debug` is the browser-native idiom. + +## Consequences + +- `debug` is a runtime dependency, not a dev-only one. +- New modules pick a namespace from the taxonomy above; a genuinely new domain extends the taxonomy in the same PR. +- Reviewers reject new bare `console.log` for app narration in `src/`, and new ungated success-path `console.log` in `tests/`. +- Enabling diagnostics is `localStorage.debug=` (browser) or `DEBUG=` (node); nothing is enabled by default. +- The console-error guard is unchanged and complementary: every `console.error` must still toast or throw (ADR 0019/0013 — errors are persisted and visible). This ADR gates narration, not error signalling, and does not supersede that rule. diff --git a/docs/adr/0032-prefer-libraries-over-homegrown.md b/docs/adr/0032-prefer-libraries-over-homegrown.md new file mode 100644 index 000000000..c06c1443c --- /dev/null +++ b/docs/adr/0032-prefer-libraries-over-homegrown.md @@ -0,0 +1,36 @@ +# 0032 — Less code is better: prefer libraries over homegrown implementations + +For any capability a well-maintained library provides, we install the library. Writing our own implementation instead is a deliberate, documented exception — never the default. + +## Status + +Accepted + +## Context + +The code we own is the code we pay for — forever. Every homegrown utility is a line we test, secure, document, and carry through every future change; a dependency that does the same job is code someone else maintains on our behalf. A homegrown implementation also tends to be a worse version of a library that already exists: it reimplements a subset, misses edge cases, has no docs, and drifts as the person who wrote it moves on. Left unchecked, these accrete into a private standard library that duplicates, badly, things the ecosystem already solved. + +The principle is not "add every dependency." A dependency is also a liability — supply-chain surface, transitive weight, an upstream that can break or vanish. The rule is about *ownership*, not byte count: for a real capability, not-owning the code beats owning it; for something trivial, neither write much nor pull a heavyweight dependency to avoid a few lines. + +## Decision + +Reach for a well-maintained library first. Writing custom code for something a library provides requires an **explicit, deliberate, recorded** justification for rejecting the library — a line in the PR description for ordinary cases, a new ADR for a significant or repeated surface. "We wrote our own" carries the burden of proof; "we added a dependency" is the default. + +Legitimate, stated reasons to go custom: + +- No library covers the need, or the closest ones are unmaintained / red-flagged (abandoned, insecure, incompatible license). +- The need is small enough that a dependency's cost (supply chain, transitive deps, bundle) outweighs the handful of lines it would save. +- The library would demand more glue and adaptation than it removes. + +Absent such a reason, replacing owned code with a library — or deleting owned code a library makes redundant — is always a welcome change, done atomically with every call site migrated in the same PR (ADR 0017). + +## Why + +Minimising the code we own is the highest-leverage way to keep the system maintainable: unwritten code has no bugs, needs no tests, and never rots. Preferring libraries makes that concrete for the large class of problems the ecosystem has already solved well. Forcing the *justification* to be explicit stops homegrown reimplementation from happening by default — the usual path isn't a decision to reinvent, it's the absence of a decision to check for a library first. + +## Consequences + +- Reviewers challenge any new homegrown implementation of a solved problem and ask which library was considered and why it was rejected; an unrecorded reinvention is a review finding. +- Deleting a homegrown utility in favour of a library needs no special justification — it is the direction of travel. +- New dependencies are still weighed (maintenance, license, transitive cost); this ADR raises the bar for writing code, it does not lower the bar for adding deps. +- ADR 0031 (replacing the homegrown `debugLog` wrapper with the `debug` library) is the first application of this principle; expect more as owned utilities are retired. diff --git a/docs/adr/README.md b/docs/adr/README.md index 936b02040..927f4cb71 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,7 +18,7 @@ See [`_template.md`](_template.md). Copy, renumber, fill in. | N | Title | | ---- | -------------------------------------------------------------- | -| 0001 | Exception deduplication via AlreadyLoggedException | +| 0001 | Idempotent error persistence | | 0002 | Auth gate: single global gate with explicit allowlist | | 0003 | ETag-based optimistic concurrency for Job and PO edits | | 0004 | Job mutations require a self-contained delta envelope | @@ -40,3 +40,5 @@ See [`_template.md`](_template.md). Copy, renumber, fill in. | 0028 | Type annotations are data contracts | | 0029 | Servers run the production branch | | 0030 | First-class People and Company links | +| 0031 | One logging gate: the debug library with namespaces | +| 0032 | Less code is better: prefer libraries over homegrown implementations | diff --git a/docs/client_onboarding.md b/docs/client_onboarding.md index 5ed2eee3c..b5717da76 100644 --- a/docs/client_onboarding.md +++ b/docs/client_onboarding.md @@ -90,12 +90,15 @@ The client needs a Xero subscription. DocketWorks handles jobs and delegates inv - Used for leave, admin time, training, etc. **Sales Branding Theme** (Settings > Invoice settings): -- Ensure one branding theme contains the terms and conditions required on both - quotes and invoices -- Prefer making that theme first in Xero's branding-theme order; DocketWorks - imports the first theme during `xero --setup` -- If another theme must remain the Xero default, select the terms-bearing theme - later in DocketWorks Company Settings +- Configure the client's required quote and invoice presentation. +- Enter the approved quote wording in Xero's **Terms (Quotes)** field. +- Select the theme in DocketWorks Company Settings before production + finalisation. Demo seeding may select the first available theme. +- Review the DocketWorks **Xero quote terms** initially generated from the + company website's `/terms-of-trade` page, and replace it if the approved + wording differs. DocketWorks sends this copy on API-created quotes; Xero's + copy is required for emergency quotes created directly in Xero. Keep the two + fields manually in sync whenever the wording changes. ### 2b. You create the Xero Developer App @@ -188,7 +191,7 @@ For each provider the client wants to use: - [ ] **Provider name** (friendly label) - [ ] **Provider type** (Gemini / Claude / OpenAI / Mistral) -- [ ] **Model name** (e.g. `gemini-2.5-flash-lite-preview-06-17`) +- [ ] **Model name** (e.g. `gemini-flash-latest` for automatic Gemini upgrades) - [ ] **API key** - [ ] Whether it should be the **default** provider @@ -223,10 +226,11 @@ For production, set in the instance `.env`: Follow `uat_setup.md` (Part C) or the production deployment process. ```bash -# UAT -sudo scripts/server/instance.sh prepare-config -sudoedit /opt/docketworks/config/-.credentials.env -sudo scripts/server/instance.sh create +# Production (add --seed to both commands for a demo instance) +sudo scripts/server/instance.sh prepare-config prod +sudoedit /opt/docketworks/config/-prod.credentials.env +sudoedit /opt/docketworks/config/-prod.company-defaults.json +sudo scripts/server/instance.sh create prod --no-start ``` --- @@ -240,15 +244,15 @@ Once the instance is running: 1. Log into the app as admin 2. Admin > Xero > "Login with Xero" 3. Authorize the client's Xero organisation -4. Run: +4. In production, select the required live sales branding theme in Admin > Settings. +5. Run: ```bash - python manage.py xero --setup - python manage.py start_xero_sync + python manage.py finalize_instance_onboarding ``` -`xero --setup` imports the first sales branding theme returned in the connected -organisation's Xero order. It preserves a previously selected theme when that -theme still exists in the connected organisation. +Finalisation discovers the connected tenant, validates production Xero +configuration without creating remote objects, and enables automated sync only +after every onboarding check succeeds. Demo onboarding uses `--seed-xero`. ### 7b. Company Settings @@ -261,8 +265,9 @@ In Admin > Settings, configure: - Starting job/PO numbers and PO prefix - Google Drive folder IDs (Shared Drive, How We Work, SOPs, Reference Library) - Quote template ID and quotes folder ID (if applicable) -- Xero sales branding theme (confirm the selected theme contains the required - quote and invoice terms) +- Xero sales branding theme (controls quote and invoice presentation) +- Xero quote terms (copy the approved wording exactly to both DocketWorks and + Xero **Terms (Quotes)**; keep both fields in sync) - KPI thresholds (optional, can be tuned later) ### 7c. Create Shop Jobs diff --git a/docs/development_session.md b/docs/development_session.md index e52bde84d..55148e586 100644 --- a/docs/development_session.md +++ b/docs/development_session.md @@ -4,7 +4,7 @@ Steps to start a development session. For first-time setup, see [initial_install ## Start -1. **Terminal → Run Task → Start Hotfix Environment** — fans out to Vite, ngrok, Celery worker, Celery Beat in dedicated panels. +1. **Terminal → Run Task → Start Dev Environment** — fans out to Vite, ngrok, Celery worker, Celery Beat in dedicated panels. 2. **Run → Start Debugging** (F5) — launches Django under the debugger. Then visit your ngrok URL (e.g. `https://docketworks--dev.ngrok-free.app`). If the Xero token has expired, hit `/xero` and click "Login with Xero". diff --git a/docs/instance-setup-demo.md b/docs/instance-setup-demo.md index 4affd4dc6..e8bebe942 100644 --- a/docs/instance-setup-demo.md +++ b/docs/instance-setup-demo.md @@ -1,142 +1,78 @@ # Instance Setup: Demo -Onboard a prospect for a paid trial of DocketWorks. Uses dummy staff but the prospect's real rates, markups, and configuration. Connects to Xero Demo Company. +Create a non-production demonstration installation with 11 dummy staff and a +dedicated Xero Demo Company connection. -**Prerequisites:** Collect from the prospect before starting: -- Company name and acronym -- Charge-out rate, wage rate, time/materials markups, leave loading -- Working hours pattern -- Financial year start month, starting job/PO numbers, PO prefix - -**Assumes:** Base server setup is complete (`scripts/server/server-setup.sh`). - ---- - -## Step 1: Prepare Credentials - -```bash -sudo scripts/server/instance.sh prepare-config uat -``` - -Edit the root-owned credentials file: +## 1. Prepare persistent instance configuration ```bash +sudo scripts/server/instance.sh prepare-config uat --seed sudoedit /opt/docketworks/config/-uat.credentials.env +sudoedit /opt/docketworks/config/-uat.company-defaults.json ``` -Fill in: -- XERO_DEFAULT_USER_ID — the existing Xero Demo Company login/user ID that will own time entries -- GCP_CREDENTIALS — shared dev service account key -- EMAIL credentials - -XERO_DEFAULT_USER_ID must be present before `instance.sh create` runs. +Complete the credentials and keep `enable_xero_sync` false. Leave the demo +template's placeholder `xero_tenant_id` as-is; step 4 rebinds it (see +[README](../scripts/server/README.md#xero_tenant_id-in-the-company-defaults-json)). +This is offline configuration; no DocketWorks services or OAuth flow are involved. -Also fill in the Xero Client ID, Client Secret, Webhook Key, and Redirect URI -for the **Xero Demo Company** app. `instance.sh create` uses these values to -render and load the initial XeroApp fixture. - -## Step 2: Create Instance +## 2. Create the instance ```bash -sudo scripts/server/instance.sh create uat +sudo scripts/server/instance.sh create uat --seed --no-start ``` -**Check:** `https://-uat.docketworks.site` shows login page. +The command loads the configured demo Company/CompanyDefaults and 11 dummy +staff without starting gunicorn or Celery. Dummy staff initially have no Xero +employee IDs because those IDs belong to a particular Xero tenant. -## Step 3: Load Demo Data +Verify the bootstrap data: -```bash -# Company settings (starting point — will be customised in Step 5) -scripts/server/dw-run.sh -uat python manage.py loaddata apps/workflow/fixtures/company_defaults.json - -# Demo staff (11 dummy employees + admin) -scripts/server/dw-run.sh -uat python manage.py loaddata apps/workflow/fixtures/initial_data.json -``` - -**Check:** ```bash scripts/server/dw-run.sh -uat python scripts/restore_checks/check_company_defaults.py -``` - -## Step 3.5: Check Xero App Credentials - -```bash scripts/server/dw-run.sh -uat python scripts/restore_checks/check_xero_app.py ``` -## Step 4: Configure Company Settings +## 3. Authorise Xero Demo Company -In Admin > Settings, set the prospect's real values: -- Company name, acronym, address, email, website -- Charge-out rate, wage rate, markups, leave loading -- Working hours (Mon-Fri pattern) -- Financial year start month -- Starting job/PO numbers and PO prefix -- Shop client name +Log in as `defaultadmin@example.com` / `Default-admin-password`, open Admin > +Xero, and complete the existing OAuth flow. -Upload logos: Admin > Settings > Company > Logo and Logo Wide. +In Admin > Settings, enter demo wording in **Xero quote terms** that includes +the exact text `Terms of trade can be found`. Copy the same wording to Xero +**Terms (Quotes)**. DocketWorks sends its copy on API-created quotes; Xero's +copy covers quotes created directly in Xero. -## Step 5: Connect to Xero Demo Company - -1. Log in as admin (`defaultadmin@example.com` / `Default-admin-password`) -2. Admin > Xero > "Login with Xero" -3. Authorize "Demo Company" +## 4. Finalise onboarding ```bash -scripts/server/dw-run.sh -uat python manage.py xero --setup +scripts/server/dw-run.sh -uat python manage.py finalize_instance_onboarding --seed-xero ``` -`xero --setup` stores the first sales branding theme in the Demo Company's Xero -order. In Admin > Settings, change it to a terms-bearing theme if the selected -theme is not the one the prospect should see. - -**Note:** `xero --setup` creates the "Weekly Testing" payroll calendar in the Demo Company if it's missing (a weekly calendar anchored to a **Monday** — Docketworks payroll posting requires Mon→Sun periods). If you ever create one by hand instead (Payroll > Settings > Payroll Calendars), its period **must start on a Monday**; `xero --setup` fails loudly if the calendar it just created came back on any other day. +The explicit flag may create missing demo-only payroll objects, including the +configured weekly calendar and required pay items. It then selects a live +branding theme if none is configured, syncs accounts and pay items, links or +creates Xero Payroll employees for all dummy staff, creates the nine canonical +shop jobs, validates the result, and enables automated sync last. -## Step 6: Sync Xero Data +Failures exit non-zero and leave sync disabled. The command is safe to rerun +after correcting the cause. -```bash -# Chart of accounts -scripts/server/dw-run.sh -uat python manage.py start_xero_sync --entity accounts --force - -# Pay items -scripts/server/dw-run.sh -uat python manage.py xero --configure-payroll -``` - -## Step 7: Create Shop Jobs - -```bash -scripts/server/dw-run.sh -uat python manage.py create_shop_jobs -``` - -Creates: Annual Leave, Sick Leave, Bereavement Leave, Travel, Training, Business Development, Office Admin, Worker Admin, Bench. - -Edit the leave jobs in Admin to set their Xero Pay Item. - -## Step 8: Verify AI Providers - -AI providers are configured during instance creation (`instance.sh create`). Verify: - -```bash -scripts/server/dw-run.sh -uat python scripts/restore_checks/check_ai_providers.py -``` - -## Step 9: Final Sync - -```bash -scripts/server/dw-run.sh -uat python manage.py start_xero_sync -``` +After a monthly Xero Demo Company reset, run `xero --setup --seed-xero`; setup +discovers the replacement tenant and updates CompanyDefaults and the cache. +Restore the matching Xero **Terms (Quotes)** wording after the reset. -## Step 10: Verify +## 5. Verify -- [ ] Log in as admin — dashboard loads -- [ ] Staff list shows 11 demo employees -- [ ] Shop jobs visible on Kanban board -- [ ] Admin > Xero shows "Connected" -- [ ] Can create a new job and add time/materials -- [ ] A DocketWorks quote and invoice use the selected Xero branding theme -- [ ] Create a test timesheet entry +- Staff list shows 11 demo employees, all linked to Xero Payroll. +- Exactly nine shop jobs are visible. +- Admin > Xero reports connected. +- A normal Xero sync completes without errors. +- A test job, timesheet, quote, and invoice work as expected. +- The native Xero PDF for a DocketWorks-created quote contains + `Terms of trade can be found`. -## Login Credentials +Logins: - Admin: `defaultadmin@example.com` / `Default-admin-password` -- All staff: their email / `Default-staff-password` +- Staff: their fixture email / `Default-staff-password` diff --git a/docs/instance-setup-production.md b/docs/instance-setup-production.md index 12f1f7e64..5fc4ff2f9 100644 --- a/docs/instance-setup-production.md +++ b/docs/instance-setup-production.md @@ -1,164 +1,87 @@ # Instance Setup: Production -Set up a production instance for a client connecting to their real Xero organisation. +Set up one client installation against that client's real Xero organisation. +Complete the client-onboarding prerequisites first and ensure the required +payroll calendar, pay items, and invoice branding theme already exist in Xero. +Production onboarding validates those objects; it never creates them. -**Prerequisites:** Complete Phase 1-5 of [client_onboarding.md](client_onboarding.md) first (collect company details, configure Xero, create GCP service account, set up AI providers, configure email). - -**Assumes:** Base server setup is complete (`scripts/server/server-setup.sh`). - ---- - -## Step 1: Prepare Credentials +## 1. Prepare persistent instance configuration ```bash sudo scripts/server/instance.sh prepare-config prod -``` - -Edit the root-owned credentials file: - -```bash sudoedit /opt/docketworks/config/-prod.credentials.env +sudoedit /opt/docketworks/config/-prod.company-defaults.json ``` -Fill in: -- XERO_DEFAULT_USER_ID — the existing Xero login/user ID that will own time entries -- GCP_CREDENTIALS path (from Phase 3a of client_onboarding.md) -- EMAIL_HOST_USER + EMAIL_HOST_PASSWORD +Complete every required secret and replace every placeholder in the +company-defaults file, including the exact name of the existing Xero payroll +calendar. Set `xero_tenant_id` to any valid placeholder UUID; onboarding rebinds +it (see +[README](../scripts/server/README.md#xero_tenant_id-in-the-company-defaults-json)). +Keep `enable_xero_sync` false. -XERO_DEFAULT_USER_ID must be present before `instance.sh create` runs. +These root-owned files are the durable source for rebuilding and reconfiguring +the instance. `prepare-config` refuses to overwrite either file. -Also fill in the Xero Client ID, Client Secret, Webhook Key, and Redirect URI -from the client's Xero app. `instance.sh create` uses these values to render -and load the initial XeroApp fixture. - -## Step 2: Create Instance +## 2. Create the instance ```bash -sudo scripts/server/instance.sh create prod +sudo scripts/server/instance.sh create prod --no-start ``` -Creates: OS user, database, .env, code clone, frontend build, migrations, admin user, systemd services (gunicorn + celery), nightly backup timer, and nginx config. - -**Check:** `https://-prod.docketworks.site` shows login page. +Creation refuses existing or partial state. It creates the infrastructure, +runs migrations, and loads the configured Company and CompanyDefaults before +creating the admin account. It does not create dummy staff or start application +services. -## Step 2.5: Check Xero App Credentials +Check the app and private Xero app configuration: ```bash +scripts/server/dw-run.sh -prod python scripts/restore_checks/check_company_defaults.py scripts/server/dw-run.sh -prod python scripts/restore_checks/check_xero_app.py ``` -Expected: `XeroApp configured: -prod xero`. - -## Step 3: Connect to Xero - -Log into the app as admin (`defaultadmin@example.com` / `Default-admin-password`). - -Admin > Xero > "Login with Xero" > Authorize the client's Xero organisation. - -**Check:** in **Admin > Xero Apps** the row shows `Authorised: ✓`. -(There's no CLI check for this — `check_xero_app.py` is a pre-OAuth -existence check and doesn't read tokens.) - -## Step 4: Configure Xero - -```bash -scripts/server/dw-run.sh -prod python manage.py xero --setup -``` - -Sets `xero_tenant_id`, `xero_shortcode`, `xero_payroll_calendar_id`, and the -first sales branding theme in the connected organisation's Xero order. A valid -existing custom theme selection is preserved. - -**Requires:** The payroll calendar must already exist in Xero (created during client onboarding Phase 2a). - -## Step 5: Configure Company Settings - -In Admin > Settings, set all values collected in Phase 1 of client_onboarding.md: -- Company name, acronym, address, email, website, phone -- Charge-out rate, wage rate, markups, leave loading -- Working hours (Mon-Fri pattern) -- Financial year start month -- Starting job/PO numbers and PO prefix -- Shop client name (must match the Xero contact from Phase 2a) -- Google Drive folder IDs (Shared Drive, How We Work, SOPs, Reference Library) -- Quote template ID and quotes folder ID (if applicable) -- Xero sales branding theme — select the terms-bearing theme if it is not the - first theme imported by `xero --setup` - -Upload logos: Admin > Settings > Company > Logo and Logo Wide. - -## Step 6: Sync Xero Data - -```bash -# Chart of accounts -scripts/server/dw-run.sh -prod python manage.py start_xero_sync --entity accounts --force - -# Pay items -scripts/server/dw-run.sh -prod python manage.py xero --configure-payroll -``` -**Check:** -```bash -scripts/server/dw-run.sh -prod python scripts/restore_checks/check_xero_accounts.py -``` -Expected: `Total accounts synced: ~60+` +## 3. Start services and authorise Xero -## Step 7: Import Staff from Xero +Log in as `defaultadmin@example.com` / `Default-admin-password`, open Admin > +Xero, and complete the existing OAuth flow. -```bash -# Preview first -scripts/server/dw-run.sh -prod python manage.py xero --import-staff-dry-run - -# Import -scripts/server/dw-run.sh -prod python manage.py xero --import-staff -``` +In Admin > Settings, explicitly select the live Xero sales branding theme that +controls the client's required quote and invoice presentation. Enter the +approved quote wording in DocketWorks **Xero quote terms** (review the initial +wording generated from the company website's `/terms-of-trade` page), then copy +it exactly to Xero **Terms (Quotes)** for emergency quotes created directly in Xero. +Production finalisation does not select the first theme automatically. -This creates Staff records from Xero Payroll employees with wage rates and working hours. All imported staff get `password_needs_reset=True`. - -## Step 8: Create Shop Jobs +## 4. Finalise onboarding ```bash -scripts/server/dw-run.sh -prod python manage.py create_shop_jobs +scripts/server/dw-run.sh -prod python manage.py finalize_instance_onboarding ``` -Creates: Annual Leave, Sick Leave, Bereavement Leave, Travel, Training, Business Development, Office Admin, Worker Admin, Bench. - -Edit the leave jobs in Admin to set their Xero Pay Item (Annual Leave → Annual Leave type, etc.). - -## Step 9: Configure AI Providers - -In Admin > AI Providers, add each provider: -- Provider type, model name, API key -- Mark one as default - -## Step 10: Import Documents (if applicable) - -If SOPs were uploaded to Google Drive (Phase 3d of client_onboarding.md): - -```bash -scripts/server/dw-run.sh -prod python manage.py import_dropbox_hs_documents -``` - -## Step 11: Start Xero Sync - -```bash -scripts/server/dw-run.sh -prod python manage.py start_xero_sync -``` - -**Check:** No errors in output. Xero data appears in the app. - -## Step 12: Verify - -- [ ] Log in as admin — dashboard loads -- [ ] Staff list shows imported employees -- [ ] Shop jobs visible on Kanban board -- [ ] Create a test timesheet entry -- [ ] Admin > Xero shows "Connected" status -- [ ] A quote created from DocketWorks shows the required terms in its Xero PDF -- [ ] An invoice created from DocketWorks shows the required terms in its Xero PDF -- [ ] Password reset email works (test with a staff member) - -## Post-Setup - -- Change admin password from the default -- Have each staff member log in and set their password -- Monitor Xero sync for the first few days +The command is rerunnable. It discovers and stores the connected tenant, then validates the payroll calendar, +pay items, and selected branding theme; stores the tenant, shortcode, theme and +calendar IDs; syncs pay items and accounts; imports active staff from Xero; +creates or updates the nine canonical shop jobs; runs completion checks; and +sets `enable_xero_sync=true` only after every step succeeds. + +Any failure exits non-zero, persists the error, and leaves automated Xero sync +disabled. Fix the source configuration and rerun the same command. + +## 5. Verify and hand over + +- Staff list contains the expected Xero Payroll employees. +- Exactly nine shop jobs are present. +- Admin > Xero reports connected. +- A normal Xero sync completes without errors. +- Test quote and invoice PDFs use the selected branding theme. +- A DocketWorks-created quote PDF contains the configured quote terms. +- DocketWorks **Xero quote terms** and Xero **Terms (Quotes)** contain the same + approved wording. +- Password reset email works. +- Change the default admin password and have imported staff reset theirs. + +Use `instance.sh reconfigure prod` only after editing persistent +credentials for an already complete instance. The CompanyDefaults JSON is the +rebuild source; live business settings are subsequently managed in the app. +Reconfigure is not a repair command for partial creation. diff --git a/docs/restore-prod-to-nonprod.md b/docs/restore-prod-to-nonprod.md index bb287b033..6c0a3a761 100644 --- a/docs/restore-prod-to-nonprod.md +++ b/docs/restore-prod-to-nonprod.md @@ -24,20 +24,13 @@ Sections must run in the order written. The Connect to Xero OAuth section is a h - The fetched archive must pass the credential and migration-ledger verifier: ```bash python scripts/verify_scrubbed_backup.py \ - --allow-legacy-client-baseline \ restore/scrubbed__.dump ``` This fails if the archive is unreadable, predates the July migration squash, or contains DB-backed external-system credentials. Do not restore a failing archive. -- **TEMPORARY KAN-278:** `--allow-legacy-client-baseline` exists only while - production still uses the pre-cutover `client` app label. Remove this flag and - the pre-migration cutover sections below after every production instance has - migrated and produced a verified company-schema backup. - The dump must be from a prod release at or after the July 2026 migration squash (baseline `*_baseline` migrations). Older dumps carry a `django_migrations` ledger the current graph cannot migrate — restore those under a matching pre-squash checkout instead (see `docs/updating.md`). -- Celery Beat stopped. Beat ticks against the DB and Xero on a timer; if it fires during the reset/restore it will block `DROP SCHEMA` or race `seed_xero_from_database`. Stop it before Reset Database; the Celery Beat section restarts it. The worker can stay running — it has nothing to do without Beat dispatches. - - Dev: kill the `Celery Beat` task in its VS Code terminal. - - Server: `sudo systemctl stop celery-beat-` +- All application services stopped until the restored Xero sync gate is verified. --- @@ -97,39 +90,6 @@ PGPASSWORD="$DB_PASSWORD" pg_restore --no-owner --no-privileges --exit-on-error ./restore/scrubbed__.dump ``` -#### Capture Pre-Migration State - -**TEMPORARY KAN-278 CUTOVER STEP:** Remove this section after every production -instance has completed the client-to-company migration and produced a verified -company-schema backup. - -The restored dump may use the schema of the production release while the -checkout contains newer models. Do not run current ORM code against that old -schema. Capture count-only evidence through raw SQL instead: - -```bash -python scripts/restore_checks/capture_pre_migration_state.py -``` - -This requires the pre-KAN-278 `client_*` schema, verifies the squashed migration -ledger, and writes only aggregate counts to -`restore/pre_migration_state.json`. A missing table, unexpected new-schema -table, or empty production dataset is a hard stop. - -#### Relabel the Legacy Client App - -**TEMPORARY KAN-278 CUTOVER STEP:** The restored ledger and tables still use the -legacy `client` app label. Apply the same one-time, idempotent surgery that the -deployment workflow runs before Django migrations: - -```bash -python manage.py relabel_client_app -``` - -This must complete successfully before `migrate`. It keeps the squashed -baseline, removes obsolete pre-squash ledger rows, and changes the app label and -table prefixes without creating a second baseline. - #### Apply Django Migrations `django_migrations` rode along in the dump, so this only runs migrations the dev branch has beyond prod's state. On a fresh prod-aligned checkout it's a no-op. @@ -141,17 +101,11 @@ python manage.py migrate **Check:** ```bash -python scripts/restore_checks/check_post_migration_state.py python scripts/restore_checks/check_django_orm.py python manage.py showmigrations | grep '\[ \]' # Expect no output. ``` -The post-migration check compares against the captured counts and verifies the -client→company table cutover, Person/link ownership, job and call references, -merge structure, and persisted terminology. Any mismatch is a migration -failure; do not continue. - The branding-theme migration deliberately skips a scrubbed restore because its Xero OAuth tokens have been removed. The destination theme is populated later by the required `xero --setup` step after destination OAuth is connected. @@ -159,12 +113,14 @@ by the required `xero --setup` step after destination OAuth is connected. #### Load Company Defaults Fixture For demo restores only, this replaces your real company name and logos with the -shipped DocketWorks demo values. Tenant installs should load their -instance-owned `/opt/docketworks/instances//company_defaults.json` copy -instead of the shared repo fixture. +shipped DocketWorks demo values. The durable source for a tenant-specific +server rebuild remains the root-owned +`/opt/docketworks/config/.company-defaults.json`; do not maintain a +second long-lived instance-directory copy. ```bash python manage.py loaddata apps/workflow/fixtures/company_defaults.json +python manage.py shell -c "from apps.workflow.models import CompanyDefaults; CompanyDefaults.set_xero_sync_enabled(enabled=False); assert not CompanyDefaults.get_solo().enable_xero_sync" ``` #### Reload Private Configuration @@ -226,7 +182,10 @@ python scripts/fix_test_company.py #### Connect to Xero OAuth -**Dev only:** Before this step, **the user** must start ngrok, the backend, and the frontend in separate terminals. The agent must NEVER start these services on the user's behalf. See [development_session.md](development_session.md). +**Dev only:** Before this step, **the user** must start the complete normal +development environment using **Start Dev Environment**, then start Django with +F5. The agent must NEVER start these services on the user's behalf. See +[development_session.md](development_session.md). ```bash (cd frontend && npx tsx tests/scripts/xero-login.ts) @@ -238,12 +197,12 @@ This script automates the Xero OAuth login flow using Playwright. It navigates t #### Configure Xero Connection ```bash -python manage.py xero --setup +python manage.py xero --setup --seed-xero ``` **What this does:** Configures all required Xero settings in CompanyDefaults: -1. Sets `xero_tenant_id` from connected organisation +1. Discovers the first connected organisation and stores its tenant ID 2. Sets `xero_shortcode` for deep linking 3. Preserves a live sales branding theme selection or replaces a restored, cross-tenant ID with the first theme in the destination organisation's Xero @@ -263,7 +222,12 @@ Xero setup complete. **Note:** Requires `xero_payroll_calendar_name` to be set in CompanyDefaults (loaded from fixture in Load Company Defaults). -`--setup` provisions any payroll calendar, earnings rates, or leave types that are present in the restored DB but missing from this Xero org (e.g. a fresh demo org), so the seed step below can match every backup pay item by name. The payroll calendar it creates is a weekly calendar anchored to a **Monday** (payroll posting requires Mon→Sun periods); `--setup` aborts if Xero hands back a calendar starting on any other day. +`--setup --seed-xero` provisions any payroll calendar, earnings rates, or leave types that are present in the restored DB but missing from this Xero org (e.g. a fresh demo org), so the seed step below can match every backup pay item by name. The payroll calendar it creates is a weekly calendar anchored to a **Monday** (payroll posting requires Mon→Sun periods); setup aborts if Xero hands back a calendar starting on any other day. + +Do not run `finalize_instance_onboarding` for a restored production dataset. +Fresh-instance finalisation imports or creates staff before enabling sync; +restore instead uses the lower-level setup, pay-item sync, and +`seed_xero_from_database` sequence below to remap the restored dataset. #### Sync Pay Items from Xero @@ -309,15 +273,6 @@ python manage.py start_xero_sync **Expected output:** Error and warning free sync between local and Xero data. -#### Start Celery Beat (Dev) - -Beat is the periodic-task dispatcher that keeps Xero tokens refreshed, runs hourly syncs, weekly scraping, and nightly housekeeping. The worker also needs to be running so dispatched tasks actually execute. In separate terminals (each blocks forever): - -```bash -poetry run celery -A docketworks worker --concurrency=4 --loglevel=info -poetry run celery -A docketworks beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler -``` - #### Verify Celery Beat (Server) On server instances Beat is already running as a systemd service (`celery-beat-`), installed by `instance.sh create`. Verify: @@ -359,8 +314,11 @@ API working: 174 active jobs, 23 archived ``` Open one recreated quote and one recreated invoice in Xero and confirm their -PDFs use the selected destination branding theme and contain the required -terms. A successful API seed alone does not verify document presentation. +PDFs use the selected destination branding theme. Confirm the DocketWorks-created +quote PDF contains the quote terms configured in DocketWorks, and copy the same +wording to the destination Xero organisation's **Terms (Quotes)** field for +direct-Xero fallback quotes. A successful API seed alone does not verify +document content or presentation. #### Snapshot Verified Database @@ -391,10 +349,6 @@ ls -lh backups/post_restore_*.sql.gz | tail -1 #### Run Playwright Tests -Before E2E, restart the backend, Celery worker, and Celery Beat with -`XERO_READONLY=True`. The user starts these long-running services; the agent -does not. All three processes must use the flag because it is process-scoped. - ```bash cd frontend PATH="$PWD/../.venv/bin:$PATH" npm run test:e2e @@ -406,8 +360,8 @@ the lock, and run integrity checks. **Expected:** all tests and teardown pass. ## Cleanup -Retain the source dump, `restore/pre_migration_state.json`, and the verified -post-restore snapshot through E2E and release verification. After explicit +Retain the source dump and verified post-restore snapshot through E2E and +release verification. After explicit operator approval, remove only the named source dump. Never recursively remove `restore/`; it also contains E2E recovery artifacts. @@ -440,7 +394,6 @@ gunzip -c "$LATEST" | PGPASSWORD="$DB_PASSWORD" psql \ - **Scrubbed dump (consumer-side):** `restore/scrubbed__.dump` - **Scrubbed dump (producer-side, on prod):** `/restore/scrubbed__.dump` -- **Pre-migration count artifact:** `restore/pre_migration_state.json` - **Baseline snapshot:** `backups/post_restore_.sql.gz` ## First-time setup (existing instances only) diff --git a/docs/server_setup.md b/docs/server_setup.md index fe4b13fef..5fe79409d 100644 --- a/docs/server_setup.md +++ b/docs/server_setup.md @@ -114,20 +114,21 @@ Certs auto-renew via `certbot renew` using the same Dreamhost DNS hooks. ### Automated (recommended) ```bash -# Step 1: scaffold credentials file -sudo scripts/server/instance.sh prepare-config +# Step 1: scaffold credentials and CompanyDefaults config +sudo scripts/server/instance.sh prepare-config [--seed] -# Step 2: fill in the root-owned credentials +# Step 2: fill in both root-owned configuration files sudoedit /opt/docketworks/config/-.credentials.env +sudoedit /opt/docketworks/config/-.company-defaults.json # Step 3: create the instance -sudo scripts/server/instance.sh create +sudo scripts/server/instance.sh create [--seed] --no-start -# Re-run after root-owned credential/config edits +# Re-run after root-owned credential edits sudo scripts/server/instance.sh reconfigure -# Or with demo fixtures: -sudo scripts/server/instance.sh create --seed +# After deliberately starting services and completing OAuth: +scripts/server/dw-run.sh - python manage.py finalize_instance_onboarding [--seed-xero] ``` ### What instance.sh creates @@ -167,77 +168,35 @@ DROP ROLE dw_test; SQL ``` +### Migrating an instance created before durable CompanyDefaults config + +Before its next `instance.sh reconfigure`, create +`/opt/docketworks/config/.company-defaults.json` from the matching +production or demo template. Replace every placeholder, copy the current +`CompanyDefaults.xero_tenant_id` into it, and keep `enable_xero_sync` false in +that rebuild source. + +This is configuration rollout only. `reconfigure` does not reload the rebuild +source into a running database or run fresh-instance finalisation. + --- ## Part C.1: Post-Create Setup -After `instance.sh create` completes, the instance has infrastructure but no data. -Choose the path that matches your scenario: +After `instance.sh create`, the instance has infrastructure plus its +configured Company and CompanyDefaults. Choose the next data workflow: ### Path A: Backup Restore (e.g. MSM demo) For instances that need production data, follow [restore-prod-to-nonprod.md](restore-prod-to-nonprod.md). -### Path B: Fresh Prospect (new Xero org) - -For a prospect trying DocketWorks with their own Xero: - -1. **Instance created** — admin user auto-created (`defaultadmin@example.com` / `Default-admin-password`) - -2. **Load tenant CompanyDefaults fixture** - - ```bash - # Copy the shared template to tenant-owned config and edit it there. - cp apps/workflow/fixtures/company_defaults_prospect.json \ - /opt/docketworks/instances//company_defaults.json - - # Edit: replace all __PLACEHOLDER__ values with prospect's info - # Key fields: company_name, acronym, address, email, po_prefix, - # xero_payroll_calendar_name (must match their Xero calendar) - - # Load it - scripts/server/dw-run.sh python manage.py loaddata /opt/docketworks/instances//company_defaults.json - ``` - - Do not treat `apps/workflow/fixtures/company_defaults*.json` as tenant - state. They are shared starting templates; the instance-owned copy is the - value that survives reset/rebuild work. +### Path B: Fresh instance -3. **Xero OAuth** — log into `https://.docketworks.site` as admin, go to Admin > Xero Settings, click "Login with Xero" and authorize - -4. **Xero configuration** - - ```bash - scripts/server/dw-run.sh python manage.py xero --setup - scripts/server/dw-run.sh python manage.py xero --configure-payroll - scripts/server/dw-run.sh python manage.py start_xero_sync --entity accounts - ``` - - `xero --setup` stores the first sales branding theme in the connected - organisation's Xero order. In Admin > Settings, select the terms-bearing - theme if the organisation uses a different theme for customer documents. - - Existing connected installations do not need to rerun setup when they - receive the branding-theme migration: the migration selects and stores the - first live Xero theme before services restart. If Xero is unavailable, the - migration fails and the deployment must be retried. - -5. **Import staff from Xero** - - ```bash - # Preview first - scripts/server/dw-run.sh python manage.py xero --import-staff-dry-run - - # Then import - scripts/server/dw-run.sh python manage.py xero --import-staff - ``` - - This pulls employees from Xero Payroll and creates Staff records with their - wage rates and working hours. All imported staff get `password_needs_reset=True`. - -6. **Verify** — log in as admin, check Staff list, mark office staff via admin - UI, then create a quote and invoice and confirm their Xero PDFs contain the - required terms +Complete OAuth and run `finalize_instance_onboarding`. See +[instance-setup-production.md](instance-setup-production.md) or +[instance-setup-demo.md](instance-setup-demo.md). The root-owned +`/opt/docketworks/config/.company-defaults.json` is the durable tenant +configuration; repo fixtures are only templates. --- @@ -334,17 +293,17 @@ curl -s https://.docketworks.site/api/health ```bash # Create test instance -sudo scripts/server/instance.sh prepare-config test uat +sudo scripts/server/instance.sh prepare-config test uat --seed # Fill in credentials... -sudo scripts/server/instance.sh create test uat +sudo scripts/server/instance.sh create test uat --seed # Verify systemctl status gunicorn-test-uat curl https://test-uat.docketworks.site/api/health # Create second instance with seed data -sudo scripts/server/instance.sh prepare-config test2 uat -# Fill in credentials... +sudo scripts/server/instance.sh prepare-config test2 uat --seed +# Fill in both config files... sudo scripts/server/instance.sh create test2 uat --seed # Verify both work independently diff --git a/docs/superpowers/specs/2026-07-21-deploy-ref-control-design.md b/docs/superpowers/specs/2026-07-21-deploy-ref-control-design.md new file mode 100644 index 000000000..42a27a7a1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-21-deploy-ref-control-design.md @@ -0,0 +1,107 @@ +# Consistent transient ref control for deploys + +**Date:** 2026-07-21 +**Status:** Design approved, pending implementation plan + +## Context + +UAT feedback: there is no clean, consistent way to say *what to deploy* to an +instance. The capability is split and half-hidden: + +- `deploy.sh` accepts `--ref ` (default `origin/production`) and + logs the resolved ref→SHA — this is the sanctioned way to put a candidate on + UAT (ADR 0029: "no server is ever pointed at any other ref except transiently + via `deploy.sh --ref` for candidate verification on UAT"). +- `instance.sh create` **hardcodes** `origin/production` (no `--ref`), so a new + UAT box cannot be born on a candidate — you must create-on-production then + redeploy. +- Nothing reports what ref an instance is currently running. +- The docs actively mislead: `CLAUDE.md` says "`main` … is never deployed", + which is false — `main` is deployed to UAT for candidate verification; it is + never deployed *to production*. ADR 0029's own body already says this; only its + summary line and the `CLAUDE.md` paraphrase overstate it. + +Chosen direction (over durable per-instance pinning): keep ADR 0029's model — +prod = `production`, UAT candidates are transient, reboot catch-up still deploys +`production` — and make the *transient* story clean and consistent across the +create/deploy/status surfaces. + +## Non-goals + +- **No persisted per-instance ref.** ADR 0029 explicitly rejected per-instance + pinning as institutionalising fleet drift. `status` derives the ref by + comparing SHAs; nothing is stored. A UAT box on a candidate still reverts to + `production` on reboot — accepted. +- No change to the promotion (`main`→`production`) or hotfix workflow. + +## Design + +### 1. `instance.sh create --ref ` +- Add `--ref ` (default `origin/production`) to `do_configure`'s flag parser + (alongside `--seed`/`--fqdn`/`--no-start`, ~line 410), tracking an explicit + `REF_SET` flag to distinguish "passed" from "defaulted". +- At the create-only release block (`if [[ ! -L "$INSTANCE_DIR/app" ]]`, ~line + 597) resolve `$REF` instead of the hardcoded `origin/production`, and log + `Resolved to ` in the same words `deploy.sh` uses (line 274). +- **`reconfigure` rejects `--ref`.** `do_configure` backs both commands, but the + release block is skipped on reconfigure (app symlink already exists), so `--ref` + there would silently no-op. If `REF_SET` and command is `reconfigure`, error: + `reconfigure does not accept --ref; use deploy.sh --ref to re-point an existing instance.` + +### 2. `instance.sh status ` +- New subcommand in the dispatch `case` and usage. +- `fetch_local_repo`, read the running SHA from `$INSTANCE_DIR/app/.release-sha` + (and `deploy-state.env` for context), then report the SHA and **which ref it + matches**, derived by comparison: + - `== origin/production` (plus ahead/behind if not exactly at the tip), + - `== origin/main` → "main candidate", + - otherwise `candidate (matches no tracked ref)`. +- Read-only; persists nothing. + +### 3. Prod `--ref` guard (confirm + override) +- Factor a shared predicate into `common.sh`, e.g. + `require_production_ref_or_ack `, so `deploy.sh` + and `instance.sh create` enforce it identically (and it is unit-testable). +- Fires only when the target instance is prod (`-prod`) **and** the ref + is not `origin/production`: + - interactive TTY → prompt `refusing: non-production ref on a PROD instance. confirm? [y/N]`, proceed only on `y`; + - non-interactive, or to skip the prompt → require an explicit `--allow-prod-ref` flag; otherwise error. +- A merged hotfix deploys from the default `origin/production` and never trips + this. UAT/demo instances are never affected. +- Add `--allow-prod-ref` to both `deploy.sh` and `instance.sh create`. + +### 4. Docs +- `CLAUDE.md` (Environment Configuration): "`main` is never deployed" → + "`main` is never deployed *to production*; UAT verifies release candidates + (including `origin/main`) transiently via `deploy.sh --ref` / `create --ref`." +- `docs/adr/0029-servers-run-the-production-branch.md`: tighten the summary line + to match the body; add the prod `--ref` guard to Consequences. +- Usage/help: `instance.sh` header + main usage, and `scripts/server/README.md` + ("Creating an Instance", "Deploying Updates") document `create --ref`, + `status`, and `--allow-prod-ref`. + +## Testing + +Extend `apps/workflow/tests/test_xero_instance_templates.py` (the existing +grep-based script-contract suite): + +- `create` parses `--ref` and feeds `$REF` (not a literal `origin/production`) to + `resolve_release_ref`. +- `reconfigure` rejects `--ref` with the pointed-at-`deploy.sh` error. +- Both `deploy.sh` and `instance.sh` call the shared guard and expose + `--allow-prod-ref`. +- `status` is wired into dispatch and usage. + +Plus one **executable** test (like the existing `node_major_from_nvmrc` test that +sources the script): call `require_production_ref_or_ack` from `common.sh` with +stub inputs and assert it passes for `msm-uat`/any ref, passes for +`msm-prod`/`origin/production`, and refuses `msm-prod`/`origin/main` without the +allow flag. + +## Files + +- `scripts/server/instance.sh` — `create --ref`, `status`, reconfigure rejection, guard call +- `scripts/server/deploy.sh` — `--allow-prod-ref`, guard call +- `scripts/server/common.sh` — shared `require_production_ref_or_ack` +- `CLAUDE.md`, `docs/adr/0029-servers-run-the-production-branch.md`, `scripts/server/README.md` +- `apps/workflow/tests/test_xero_instance_templates.py` diff --git a/docs/urls/accounts.md b/docs/urls/accounts.md index 41e94134c..c14279202 100644 --- a/docs/urls/accounts.md +++ b/docs/urls/accounts.md @@ -21,7 +21,8 @@ | URL Pattern | View | Name | Description | |-------------|------|------|-------------| | `/staff/` | `staff_api.StaffListCreateAPIView` | `accounts:api_staff_list_create` | API endpoint for listing and creating staff members. | -| `/staff//` | `staff_api.StaffRetrieveUpdateDestroyAPIView` | `accounts:api_staff_detail` | API endpoint for retrieving, updating, and deleting individual staff members. | +| `/staff//` | `staff_api.StaffRetrieveUpdateAPIView` | `accounts:api_staff_detail` | API endpoint for retrieving and updating individual staff members. | +| `/staff//icon/` | `staff_icon_api.StaffIconAPIView` | `accounts:api_staff_icon` | Upload or remove the profile picture for a single staff member. | | `/staff/all/` | `staff_views.StaffListAPIView` | `accounts:api_staff_all_list` | API endpoint for retrieving list of staff members for Kanban board. | | `/staff/rates//` | `staff_views.get_staff_rates` | `accounts:get_staff_rates` | Retrieve wage rates for a specific staff member. | diff --git a/docs/urls/workflow.md b/docs/urls/workflow.md index 723c652a0..fb4e5185d 100644 --- a/docs/urls/workflow.md +++ b/docs/urls/workflow.md @@ -58,7 +58,6 @@ | URL Pattern | View | Name | Description | |-------------|------|------|-------------| | `/company-defaults/` | `company_defaults_api.CompanyDefaultsAPIView` | `api_company_defaults` | API view for managing company default settings. | -| `/enums//` | `get_enum_choices` | `get_enum_choices` | API endpoint to get enum choices. | | `/xero-errors/` | `xero_view.XeroErrorListAPIView` | `xero-error-list` | API view for listing Xero synchronization errors. | | `/xero-errors//` | `xero_view.XeroErrorDetailAPIView` | `xero-error-detail` | API view for retrieving a single Xero synchronization error. | diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 9efe79acb..732e5fec9 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -63,6 +63,7 @@ The backend (Django) is in the repo root. The frontend and backend are in the sa ### Error Handling 30. Every `console.error` must toast or throw — never silent failures +31. Debug logging goes through `debug` only: `import debug from 'debug'; const log = debug(':')` — no bare `console.log` for app narration, no bespoke log wrappers. Enable with `localStorage.debug='job:*'` (ADR 0031) --- diff --git a/frontend/README.md b/frontend/README.md index 0bbd284c0..bed4b2d7a 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -40,23 +40,6 @@ Source files live in the `src/` directory: - **types/** – TypeScript interfaces - **views/** – Page‑level Vue components -## Training Manual - -The staff training manual is built with VitePress and served at `/manual/` in production. - -```bash -npm run manual:dev # Dev server on port 5174 (hot-reload) -npm run manual:build # Production build to dist-manual/ -npm run manual:screenshots # Capture screenshots (needs running app + .env credentials) -``` - -Markdown source lives in `manual/`. Edit pages there and preview with `manual:dev`. - -PDF export is not wired up: the only working exporter (`vitepress-export-pdf`) is -unmaintained and its stale `vitepress` peer range blocks upgrading vitepress past -1.x, which is required to clear transitive `vite`/`esbuild` CVEs. The web build -is the supported delivery; revisit if an actively-maintained exporter appears. - ## Additional Documentation See `docs/overview.md` for a newcomer‑oriented explanation of the codebase. diff --git a/frontend/docs/e2e_testing_strategy.md b/frontend/docs/e2e_testing_strategy.md index 327fee172..01b0f4351 100644 --- a/frontend/docs/e2e_testing_strategy.md +++ b/frontend/docs/e2e_testing_strategy.md @@ -13,7 +13,14 @@ SMTP traffic) is accepted because mocked integrations have repeatedly hidden real-world breakage. - **Xero** — real Xero **demo company** in dev/UAT. Tests - create/delete invoices, quotes, POs against the demo org. + create/delete invoices, quotes, POs against the demo org. DocketWorks + sends the configured Xero quote terms in the quote API payload. In the + demo company only, those terms must contain the exact text + `Terms of trade can be found`; the quote E2E requires the native Xero + PDF to contain it. This marker is a demo-only fixture contract, not a + validation rule for production wording. Xero's Terms (Quotes) setting + is manually kept in sync only as a fallback for quotes created directly + in Xero. - **AI providers** (Claude / Gemini / Mistral) — real API calls. Tests consume credits. - **Email** — real SMTP to a test recipient. Catches template/auth diff --git a/frontend/eslint.config.ts b/frontend/eslint.config.ts index 7d218211b..4cff816ce 100644 --- a/frontend/eslint.config.ts +++ b/frontend/eslint.config.ts @@ -4,6 +4,12 @@ import pluginVue from 'eslint-plugin-vue' import skipFormatting from '@vue/eslint-config-prettier/skip-formatting' export default defineConfigWithVueTs( + { + linterOptions: { + reportUnusedDisableDirectives: 'error', + }, + }, + { name: 'app/files-to-lint', files: ['**/*.{ts,mts,tsx,vue}'], @@ -20,8 +26,6 @@ export default defineConfigWithVueTs( globalIgnores([ '**/dist/**', '**/dist-ssr/**', - '**/dist-manual/**', - 'manual/.vitepress/cache/**', '**/coverage/**', '**/scripts/**', '**/playwright-report/**', @@ -34,6 +38,21 @@ export default defineConfigWithVueTs( vueTsConfigs.recommended, skipFormatting, + { + name: 'app/rules', + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + ignoreRestSiblings: true, + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + }, + ], + }, + }, + { name: 'app/pages-routing', files: ['src/pages/**/*.vue'], @@ -41,4 +60,12 @@ export default defineConfigWithVueTs( 'vue/multi-word-component-names': 'off', }, }, + + { + name: 'app/vendored-ui', + files: ['src/components/ui/**/*.vue'], + rules: { + 'vue/multi-word-component-names': 'off', + }, + }, ) diff --git a/frontend/manual/.vitepress/config.ts b/frontend/manual/.vitepress/config.ts deleted file mode 100644 index efd1c865c..000000000 --- a/frontend/manual/.vitepress/config.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { defineConfig } from 'vitepress' - -export default defineConfig({ - title: 'DocketWorks Training Manual', - description: 'How to do your job - a cookbook for staff', - - // Served at /manual/ alongside the main app - base: '/manual/', - - // Output directory (relative to manual/ folder) - outDir: '../dist-manual', - - themeConfig: { - nav: [{ text: 'Home', link: '/' }], - - sidebar: [ - { - text: 'Customer Contact', - items: [{ text: 'New Customer Call', link: '/enquiries/new-customer-call' }], - }, - { - text: 'Jobs', - items: [ - { text: 'Understanding Job Finances', link: '/jobs/understanding-job-finances' }, - { text: 'Attach Files to a Job', link: '/jobs/attach-files' }, - ], - }, - { - text: 'Quoting', - items: [ - { text: 'Assess & Price a Job', link: '/quoting/assess-and-price' }, - { text: 'Send a Quote', link: '/quoting/send-quote' }, - ], - }, - { - text: 'Scheduling', - items: [{ text: 'Schedule a Job', link: '/scheduling/schedule-a-job' }], - }, - { - text: 'Fieldwork', - items: [{ text: 'Complete a Job On-Site', link: '/fieldwork/complete-a-job' }], - }, - { - text: 'Timesheets', - items: [{ text: 'End of Day Entry', link: '/timesheets/end-of-day-entry' }], - }, - { - text: 'Purchasing', - items: [{ text: 'Create a Purchase Order', link: '/purchasing/create-purchase-order' }], - }, - { - text: 'Invoicing', - items: [{ text: 'Invoice a Job', link: '/invoicing/invoice-a-job' }], - }, - { - text: 'Weekly & Monthly Procedures', - items: [{ text: 'Weekly Checklist', link: '/end-of-week/weekly-checklist' }], - }, - { - text: 'Management & Admin', - items: [ - { text: 'Run Reports', link: '/management/run-reports' }, - { text: 'Run Payroll', link: '/admin/run-payroll' }, - { text: 'Manage Staff', link: '/admin/manage-staff' }, - ], - }, - ], - - search: { - provider: 'local', - }, - - outline: { - level: [2, 3], - }, - }, - - lastUpdated: true, -}) diff --git a/frontend/manual/admin/manage-staff.md b/frontend/manual/admin/manage-staff.md deleted file mode 100644 index e42ed69f3..000000000 --- a/frontend/manual/admin/manage-staff.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: Manage Staff ---- - -# Manage Staff - -> **When to use:** You need to add a new team member, update someone's details, or change their permissions. - -## What You'll Need - -- [ ] Superuser access (only superusers can manage staff) -- [ ] The new person's details (name, email, wage rate) - -## Steps - -### 1. Open Staff Management - -Navigate to **Admin > Staff** (you need to be a superuser to see this). - - - -You'll see a table listing all staff members with their name, role (Office Staff / SuperUser), last login, and date joined. - -### 2. Add a new staff member - -Click **New Staff** to open the staff form. It has three tabs: - -#### Personal Info - - - -- **First Name** and **Last Name** -- as you'd expect. -- **Preferred Name** -- what they go by day-to-day (optional). -- **Email** -- their login email. Must be unique. -- **Password** -- at least 8 characters. They can change it later. -- **Base Wage Rate** -- their hourly wage in NZD. This is used for job costing calculations. -- **Xero User ID** -- links them to their Xero profile for payroll (optional). -- **Profile Icon** -- click the avatar to upload a photo. - -#### Working Hours - - - -Set the scheduled hours for each day of the week (Monday through Sunday). Enter in quarter-hour increments (0.25, 0.5, etc.). These are used in the timesheet summary to show whether a full day has been entered. - -#### Permissions - -- **Is Office Staff** -- tick this for office-based staff. It controls which navbar items they see (office staff see everything; non-office staff see a simplified view focused on their own time entry). -- **Is SuperUser** -- tick this for people who need admin access (staff management, system settings, etc.). - -### 3. Edit an existing staff member - -Click the **Edit** button next to any staff member in the table. The same form opens, pre-filled with their current details. You can change anything except their email. - -### 4. Remove a staff member - -Click the **Delete** button next to their name. You'll be asked to confirm before anything happens. - -## What Happens Next - -- New staff members can log in immediately with their email and password -- Their wage rate feeds into job costing whenever they enter time -- Their scheduled hours appear in the timesheet summary for checking daily totals -- If linked to Xero, their payroll data can be reconciled using the [Payroll report](/admin/run-payroll) - -## Tips - -::: tip -Set up working hours accurately -- they're used to check whether a full day of time has been entered. If someone works 7.5 hours and the system expects 8, it'll flag every day as incomplete. -::: - -::: warning -Be careful with the SuperUser permission. SuperUsers can see everything and manage all settings. Only give this to people who genuinely need it. -::: diff --git a/frontend/manual/admin/run-payroll.md b/frontend/manual/admin/run-payroll.md deleted file mode 100644 index 23cefb3cd..000000000 --- a/frontend/manual/admin/run-payroll.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: Run Payroll ---- - -# Run Payroll - -> **When to use:** You need to check that what Xero paid in payroll matches what DocketWorks calculated from timesheets. - -## What You'll Need - -- [ ] Access to the Payroll Reconciliation report -- [ ] Payroll already run in Xero for the period you're checking - -## Steps - -### 1. Open the Payroll Reconciliation report - -Navigate to **Reports > Reconciliation > Payroll (Xero)**. - - - -### 2. Set the date range - -Pick your start and end dates, or use one of the quick presets: - -- **This FY** -- current financial year -- **Last FY** -- previous financial year -- **Last 30 Weeks** -- a rolling window - -The dates snap to week boundaries automatically -- you don't need to worry about picking exact Monday-to-Sunday ranges. - -### 3. Read the summary cards - -At the top you'll see three cards: - -- **Xero Total** -- What Xero says was paid in gross payroll -- **DW Total** -- What DocketWorks calculated from timesheets and wage rates -- **Difference** -- The gap between the two, shown in dollars and as a percentage - -If the difference is small (a few dollars), everything's fine. If it's significant, something needs investigating. - -### 4. Use the heatmap to find problems - - - -The heatmap grid shows every week (rows) by every staff member (columns). Each cell is colour-coded: - -- **Green** -- Difference is less than $1. Xero and DWagree. -- **Blue shades** -- Xero paid more than DWcalculated (overpayment or a Xero adjustment). -- **Red shades** -- Xero paid less than DWcalculated (underpayment or missing hours in Xero). Darker red means a bigger gap. - -### 5. Drill into the details - -Hover over any cell to see the full breakdown: - -- **Xero**: Hours worked and gross amount paid -- **DW**: Hours entered and calculated cost -- **Gap**: The dollar difference -- **Hours impact**: How much of the gap comes from different hours -- **Rate impact**: How much comes from different wage rates - -This tells you whether the problem is missing hours (someone didn't enter all their time) or a rate mismatch (the wage rate in DWdoesn't match Xero). - -### 6. Export if needed - -Click **Export CSV** to download the data for further analysis in Excel, or to share with your accountant. - -## What Happens Next - -- If everything matches, you're done -- payroll is reconciled for that period -- If there are discrepancies, investigate the red/blue cells: - - **Missing hours**: Check timesheets for that staff member in that week - - **Rate mismatch**: Compare the wage rate in [Staff Management](/admin/manage-staff) with Xero - - **Xero adjustments**: Check if Xero has manual adjustments (leave, bonuses, etc.) that DWwouldn't know about - -## Tips - -::: tip -Run this report weekly right after payroll. Catching a discrepancy the same week is easy to fix. Finding it three months later is a headache. -::: - -::: warning -The report compares gross figures. If Xero has manual adjustments (sick leave, bonuses, deductions), they'll show as differences here. That's expected -- just make sure you can account for them. -::: diff --git a/frontend/manual/end-of-week/weekly-checklist.md b/frontend/manual/end-of-week/weekly-checklist.md deleted file mode 100644 index 6aeaf15e8..000000000 --- a/frontend/manual/end-of-week/weekly-checklist.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Weekly Checklist ---- - -# Weekly Checklist - -> **When to use:** End of the week admin procedures -- making sure nothing's fallen through the cracks. - -## What You'll Need - -- [ ] Access to reports and the Kanban board -- [ ] 30-60 minutes of uninterrupted time - -## Weekly Tasks - -### Review Outstanding Quotes - -Check for any quotes that have been sent but not responded to. Open the Kanban board and look at jobs sitting in the "Quoted" column. If anything's been there more than a week, follow up with the customer. - - - -### Check Unbilled Work - -Look for jobs that are complete but haven't been invoiced yet. These are jobs sitting in the "Complete" column on the Kanban board, or you can use the Job Aging report under Reports > Management to find them. - -Every week a completed job sits uninvoiced is a week you're not getting paid for work you've already done. - -### Timesheet Review - -Go through each staff member's timesheets for the week using the Daily Overview. Check that: - -- Everyone has entered a full day's hours for each day they worked -- The summary shows hours that match their scheduled hours -- There aren't unusual patterns (lots of non-billable time, missing days, etc.) - - - -If someone's hours don't add up, check with them before the week is out. It's much easier to fix on Friday than to reconstruct the following week. - -### Month End (Monthly) - -At the end of each month, run the month-end process under Admin. This is mostly for shop jobs -- it resets the hours each month so you start fresh. - - - -### Follow-up Actions - -- Chase any overdue customer payments (check in Xero) -- Review the [KPI Report](/management/run-reports) for the week -- are we tracking to target? -- Update job statuses on the Kanban board for anything that's moved along during the week -- Flag any jobs that are going over budget so they can be discussed - -## Tips - -::: tip -Make this a recurring calendar event every Friday afternoon. It takes less time than you think and prevents small problems from becoming big ones. -::: - -::: warning -Don't skip the timesheet review. Missing or inaccurate timesheets mean inaccurate job costing, which means you don't know if you're making or losing money. -::: diff --git a/frontend/manual/enquiries/new-customer-call.md b/frontend/manual/enquiries/new-customer-call.md deleted file mode 100644 index 596ca5cbd..000000000 --- a/frontend/manual/enquiries/new-customer-call.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: New Customer Call ---- - -# New Customer Call - -> **When to use:** A new or existing customer calls asking about work they need done. - -## What You'll Need - -- [ ] Customer name and contact details -- [ ] Site address (if different from customer address) -- [ ] Brief description of what they need - -## Steps - -### 1. Check if they're an existing customer - - - -Search for the customer by name or phone number. If they exist, you'll see their history. - -If they're new, you'll create a record as part of the next step. - -### 2. Create a new job - - - -Click **Create Job** in the navbar. Fill in the basic details: - -- **Job Number** -- This is auto-generated, just like our old paper job numbers. -- **Job Name** -- An internal nickname for the job. This is handy for searching later, so make it something memorable (e.g. "Smith fence repair" rather than "Job for Mr Smith"). -- **Client** -- This must match the entry in Xero. Start typing and it'll autocomplete. If the client doesn't exist, you can create one from here. -- **Job Description** -- This gets printed on invoices and quotes, so write it for the customer's eyes. Keep it professional and clear. - -### 3. Record the key details - - - -The other fields on the job form: - -- **Contact Name** and **Phone Number** -- Who to call about this job. -- **Order Number** -- The client's reference number, if they have one. -- **Job Notes** -- Internal notes that only we see. Use this for anything relevant -- site access codes, special instructions, who you spoke to on the phone. - -Don't worry about filling in everything right now. The critical fields are the client, job name, and a description. Everything else can be added later as you learn more. - -### 4. Set next steps - -Once you've saved the job, think about what happens next. Does someone need to go out for a site visit? Can you quote over the phone? Is it a straightforward T&M job that just needs scheduling? - -If you know, update the job status on the Kanban board to reflect where it's at. If not, leave it in the first column and it'll get picked up in the normal workflow. - -## What Happens Next - -- Job appears on the Kanban board where everyone can see it -- Someone picks it up to assess and price -- see [Assess & Price a Job](/quoting/assess-and-price) -- The job number is the reference for everything from here on out - -## Tips - -::: tip -Always confirm the site address -- it's often different from the customer's postal or billing address. Getting this wrong wastes everyone's time. -::: - -::: warning -Don't promise a quote timeframe without checking the schedule first. Better to say "we'll get back to you" than to commit to something you can't deliver. -::: diff --git a/frontend/manual/fieldwork/complete-a-job.md b/frontend/manual/fieldwork/complete-a-job.md deleted file mode 100644 index a271654cd..000000000 --- a/frontend/manual/fieldwork/complete-a-job.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: Complete a Job On-Site ---- - -# Complete a Job On-Site - -> **When to use:** You're on-site, the work is done, and you need to mark the job as complete. - -## What You'll Need - -- [ ] Access to DocketWorks (phone or tablet is fine) -- [ ] Any required sign-off or photos from the customer - -## Steps - -### 1. Record your time - -Make sure all your hours for this job are entered in the timesheets. If you haven't done it during the day, do it now while the details are fresh -- see [End of Day Entry](/timesheets/end-of-day-entry). - -### 2. Note any materials used - -If you used materials on-site that haven't been recorded yet, make a note. These need to be entered into the Reality section of the job so the costs are accurate. - -### 3. Take photos (if needed) - -If there's anything worth documenting -- the finished work, any issues found, before/after shots -- attach them to the job. Files added to a job automatically sync to Dropbox as well. See [Attach Files to a Job](/jobs/attach-files). - -### 4. Update the job status - -Move the job to "Complete" on the Kanban board (or ask office staff to do it if you don't have access). This signals to the office that the work is done and the job is ready for invoicing. - -## What Happens Next - -- Office staff see the job in the "Complete" column on the Kanban board -- They'll review the timesheets and materials, then [invoice the job](/invoicing/invoice-a-job) -- The job's actual costs feed into the KPI reports automatically - -## Tips - -::: tip -Take photos of finished work, especially for larger jobs. They're useful for resolving disputes and for showing prospective customers what you've done. -::: - -::: warning -Don't mark a job as complete until all your time is entered. Once it moves to invoicing, missing hours mean you either don't get paid for them or someone has to go back and fix it. -::: diff --git a/frontend/manual/index.md b/frontend/manual/index.md deleted file mode 100644 index 17e6015d6..000000000 --- a/frontend/manual/index.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: Home ---- - -# DocketWorks Training Manual - -Welcome to the training manual. This is a cookbook for doing your job - not a software manual. - -Each page covers a **business task** you need to complete, with step-by-step instructions. - -## Where to Start - -Pick the task you need to do: - -### Customer & Sales - -- [New Customer Call](/enquiries/new-customer-call) - When a customer contacts you -- [Assess & Price a Job](/quoting/assess-and-price) - Working out what to charge -- [Send a Quote](/quoting/send-quote) - Getting the quote to the customer - -### Understanding Jobs - -- [Understanding Job Finances](/jobs/understanding-job-finances) - How estimates, quotes, and reality fit together -- [Attach Files to a Job](/jobs/attach-files) - Drawings, photos, and documents - -### Getting Work Done - -- [Schedule a Job](/scheduling/schedule-a-job) - Assigning work to staff -- [Complete a Job On-Site](/fieldwork/complete-a-job) - What field staff do -- [End of Day Entry](/timesheets/end-of-day-entry) - Recording your time - -### Purchasing - -- [Create a Purchase Order](/purchasing/create-purchase-order) - Ordering materials from suppliers - -### Billing & Admin - -- [Invoice a Job](/invoicing/invoice-a-job) - Billing the customer -- [Weekly Checklist](/end-of-week/weekly-checklist) - End of week procedures -- [Run Reports](/management/run-reports) - Getting insights -- [Run Payroll](/admin/run-payroll) - Reconciling timesheets with Xero payroll -- [Manage Staff](/admin/manage-staff) - Adding and editing team members - ---- - -::: tip For Authors -To add or edit recipes, see the [Writing Guide](#writing-guide) below. -::: - -## Writing Guide - -Each recipe follows this structure: - -1. **When to use** - One line describing the situation -2. **What you'll need** - Prerequisites checklist -3. **Steps** - Numbered actions with explanations -4. **What happens next** - Outcomes and follow-ups -5. **Tips** - Business knowledge and warnings - -Screenshots are auto-generated. Mark where you need one with: - -```markdown - -``` diff --git a/frontend/manual/invoicing/invoice-a-job.md b/frontend/manual/invoicing/invoice-a-job.md deleted file mode 100644 index 9f2491f84..000000000 --- a/frontend/manual/invoicing/invoice-a-job.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: Invoice a Job ---- - -# Invoice a Job - -> **When to use:** The work is done, time and materials are recorded, and it's time to bill the customer. - -## What You'll Need - -- [ ] A completed job with all timesheet entries recorded -- [ ] Materials entered in the Reality section (if applicable) -- [ ] Customer billing details in the system - -## Steps - -### 1. Check the Reality section - - - -Before you invoice, open the job and look at the Reality section. This shows the real cost and revenue of the job -- what actually happened versus what you estimated. - -The timesheets section of the app automatically populates the time entries here. If staff have been entering their time correctly, the labour lines should already be filled in. Materials you'll need to enter directly into the Reality section for now. - -### 2. Compare estimate to reality - - - -The system keeps track of the totals for you. Check the Revenue vs Costs summary at the bottom of the job. You'll see columns for Estimate, Quote, and Reality side by side. - -Key things to look for: - -- **Revenue higher than cost** = the job made a profit -- **Cost higher than revenue** = the job lost money -- investigate before invoicing -- **Big gap between estimate and reality** = something went differently than planned - -### 3. Create the invoice - - - -Once you're satisfied that the numbers are right, create the invoice. This sends the billing information through to Xero where the actual invoice is generated and sent to the customer. - -### 4. Verify in Xero - -The invoice should appear in Xero shortly after creation. Check that the amounts match and that the customer details are correct. - -## What Happens Next - -- The invoice is created in Xero and sent to the customer -- The job status updates to reflect that it's been invoiced -- Payment tracking happens in Xero from this point -- The job's profitability is now final and will show in the KPI reports - -## Tips - -::: tip -Always check the Reality section before invoicing. If timesheets are missing or materials haven't been recorded, your invoice won't reflect the actual work done -- and you'll either undercharge the customer or have inaccurate job costing. -::: - -::: warning -If the Reality section shows the job lost money, don't just invoice and move on. Flag it so the team can learn from it. Was the estimate too low? Did the scope change? Understanding why helps prevent the same thing happening next time. -::: diff --git a/frontend/manual/jobs/attach-files.md b/frontend/manual/jobs/attach-files.md deleted file mode 100644 index 8dbab8ce8..000000000 --- a/frontend/manual/jobs/attach-files.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: Attach Files to a Job ---- - -# Attach Files to a Job - -> **When to use:** You need to add drawings, photos, documents, or any other files to a job. - -## What You'll Need - -- [ ] The file(s) you want to attach (drawings, photos, PDFs, etc.) -- [ ] The job open in DocketWorks - -## Steps - -### 1. Open the job and go to the Files section - -Navigate to the job and find the Attached Files area. - - - -### 2. Upload your files - -Drag files into the upload area, or click to browse and select them. You can upload multiple files at once. - -### 3. Verify the upload - -Your files will appear in the attachments list. They're available immediately to anyone viewing the job. - -## Dropbox Sync - -Any files you add to a job will immediately turn up in Dropbox as well. This is particularly helpful for the laser cutter -- upload a drawing to the job and it's available on the workshop machine straight away. - -The same applies in reverse. Any files you add to the job's Dropbox folder will appear in DocketWorks. So if it's easier to drop something into Dropbox from your phone or desktop, that works too. - -## What Happens Next - -- Files are attached to the job and visible to all staff -- Files sync to Dropbox automatically (both directions) -- They stay with the job as a permanent record -- useful for reference on repeat customers or warranty queries - -## Tips - -::: tip -Always attach drawings to the job rather than emailing them around. That way they're archived and anyone can find them later without hunting through their inbox. -::: - -::: tip -Take before/after photos on bigger jobs and attach them. They're invaluable for resolving disputes and for showing prospective customers your work. -::: diff --git a/frontend/manual/jobs/understanding-job-finances.md b/frontend/manual/jobs/understanding-job-finances.md deleted file mode 100644 index 058d0447b..000000000 --- a/frontend/manual/jobs/understanding-job-finances.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -title: Understanding Job Finances ---- - -# Understanding Job Finances - -> **When to use:** You want to understand how the money works on a job -- what the three financial stages mean and how they fit together. - -This isn't a step-by-step task. It's background knowledge that makes everything else make sense. - -## The Three-Part Structure - -From a commercial perspective, every job is divided into three parts: - -- **Time** -- Labour hours and wage/charge rates -- **Materials** -- Physical goods, supplies, stock items -- **Adjustments** -- Everything else: call-out fees, travel, discounts, write-offs - -You'll see this three-part breakdown everywhere in the system -- estimates, quotes, reality, and reports. - -## The Three Financial Stages - -Each job goes through three financial stages. Think of them as three different views of the same job: - -### Estimate (What we think it'll cost) - - - -The estimate is your internal "finger in the air" -- what you think the job will cost us and what we should charge. It has time, materials, and adjustments sections, each with as many lines as you need. - -Every job should have an estimate. Even a rough one is better than none. - -### Quote (What we tell the customer) - - - -The quote is the customer-facing version. Not every job needs a quote -- T&M (time and materials) jobs where you bill as you go don't need one. But if the customer wants a price upfront, this is where it lives. - -The **Copy Estimate to Quote** button gives you a starting point. From there you can adjust -- add contingency, simplify the breakdown, reduce the price to win the work. - -Remember: all jobs have an estimate, while only some have a quote. - -### Reality (What actually happened) - - - -The Reality section shows the real cost and revenue of the job. If revenue is higher than cost, the job made a profit. - -- **Time entries** are populated automatically from timesheets. When staff enter time against a job, it shows up here. -- **Materials** need to be entered directly into the Reality section for now. -- **Adjustments** are added manually as needed. - -## The Revenue vs Costs Summary - - - -At the bottom of the job you'll see summary tables that put it all together: - -**Revenue** shows what the customer is being charged: - -| Category | Estimate | Quote | Reality | -| ---------------------- | -------- | ------ | ------- | -| Total Time | $X | $X | $X | -| Total Materials | $X | $X | $X | -| Total Adjustments | $X | $X | $X | -| **Total Project Cost** | **$X** | **$X** | **$X** | - -**Costs** shows what it costs us: - -| Category | Estimate | Quote | Reality | -| ---------------------- | -------- | ------ | ------- | -| Total Time | $X | $X | $X | -| Total Materials | $X | $X | $X | -| Total Adjustments | $X | $X | $X | -| **Total Project Cost** | **$X** | **$X** | **$X** | - -The difference between Revenue and Costs is your profit. You can see at a glance whether the job is on track, over budget, or better than expected. - -## How It All Connects - -1. You [create a job](/enquiries/new-customer-call) and fill in the **Estimate** -2. If the customer needs a price, you populate the **Quote** and [send it](/quoting/send-quote) -3. Staff do the work and enter time -- the **Reality** section fills up automatically -4. When the job is done, you compare Reality to Estimate to see how you went -5. You [invoice the job](/invoicing/invoice-a-job) and it feeds into the [KPI reports](/management/run-reports) - -## Tips - -::: tip -Get in the habit of checking the Revenue vs Costs summary before invoicing. A quick glance tells you if the job made money or not -- and if not, why not. -::: - -::: warning -Don't confuse the Estimate with the Quote. The Estimate is for us (what we think it costs). The Quote is for the customer (what we're charging them). They can be different -- and often should be. -::: diff --git a/frontend/manual/management/run-reports.md b/frontend/manual/management/run-reports.md deleted file mode 100644 index 6e355fe43..000000000 --- a/frontend/manual/management/run-reports.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -title: Run Reports ---- - -# Run Reports - -> **When to use:** You want to know how the business is tracking -- profitability, job progress, staff output, or anything else that needs a number behind it. - -## What You'll Need - -- [ ] Access to DocketWorks (you need to be logged in) -- [ ] A rough idea of the date range you're interested in - -## Available Reports - -All reports live under the **Reports** menu in the navbar. They're grouped into categories: - -| Category | Reports | What they tell you | -| ------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------- | -| **CRM** | Clients | Customer list and contact details | -| **Management** | Job Aging, Job Movement, Job Profitability, KPI Reports, Sales Forecast, Staff Performance | How the business is performing day-to-day | -| **Reconciliation** | Payroll (Xero), Profit & Loss (Xero) | Whether our books line up with Xero | -| **Data Quality** | Archived Jobs Validation | Finds problems in archived job data | - -Most of the time you'll be living in the **Management** reports, especially KPI Reports and Job Profitability. - -## Steps - -### 1. Open a report - -Click **Reports** in the top navbar, pick a category, then pick the report you want. - - - -### 2. Set your date range - -Most reports let you pick a month and year. Some have a **Today** button to jump back to the current period. - - - -### 3. Read the data - -Each report is laid out differently, but they all load automatically once you pick your dates. No need to click a "Run" button -- just change the month and the data refreshes. - -### 4. Export (if needed) - -Reports that support it have an **Export** button in the top-right corner. This gives you a download you can open in Excel or share with your accountant. - -## The KPI Report (In Depth) - -The KPI report is the one you'll probably use the most. It tells you approximately how much profit the business made in a day or a month. - -This is based on **when the time or parts are added to the job**, not on when the job is invoiced. If someone adds four hours on a job today, it counts for today -- not whatever day the job happens to be invoiced. - -### Summary cards - -At the top of the page you'll see four cards: - - - -- **Labour** -- Billable hours billed to clients, total wages paid, and the average daily billable hours as a percentage. This is the big one. If the team billed 38 hours against a 45-hour target, it'll show as amber. Hit the target and it goes green. -- **Materials** -- Material profit, revenue, cost, and margin percentage. -- **Adjustments** -- Same breakdown as materials but for adjustments (discounts, write-offs, extras). -- **Profit** -- The bottom line: net profit, total revenue, gross profit, and net margin percentage. - -You can click any card to see a more detailed breakdown. - -### Calendar heatmap - -Below the cards is a calendar view of the month. Each day is colour-coded: - -- **Green** = good day (hit targets) -- **Amber** = okay, but below target -- **Red** = bad day (lost money or well below target) - - - -Each day shows the hours worked and the profit or loss for that day. You can see at a glance which days went well and which didn't. - -### Daily detail (click a day) - -Click on any day in the calendar and a detail panel pops up showing: - - - -- **Revenue table** -- Labour Revenue, Material Revenue, Adjustment Revenue, Total Revenue -- **Cost table** -- Labour Cost, Material Cost, Adjustment Cost, Total Cost -- **Gross Profit Breakdown** -- Labour Profit, Material Profit, Adjustment Profit, Total Gross Profit -- **Profit by Job** -- A table listing every job that had activity that day (Job #, Labour, Materials, Adjustments, Total). Profitable jobs show in green. Jobs that lost money show in red/pink. - -The negative numbers you'll sometimes see for time on jobs are internal jobs -- things like shop maintenance, training, or admin. Those are expected. - -## What Happens Next - -- Reports update in real time as staff enter time, materials, and adjustments throughout the day -- Use the data to spot problems early -- a run of red days means something needs attention -- Share exports with your accountant or use them in team meetings -- The Reconciliation reports (Payroll and P&L) are for checking that DocketWorks lines up with what's in Xero - -## Tips - -::: tip -The KPI report is most useful at the end of each day. Check it before you leave to see how the team went. A quick glance at the calendar heatmap tells you everything you need to know. -::: - -::: tip -If a day is showing red, click into it and look at the Profit by Job table. The red rows will tell you exactly which jobs lost money and why. -::: - -::: warning -The KPI numbers are based on when work is recorded, not when it's invoiced. Don't compare KPI figures directly to your Xero invoicing for the same month -- they measure different things. Use the Reconciliation reports for that. -::: diff --git a/frontend/manual/purchasing/create-purchase-order.md b/frontend/manual/purchasing/create-purchase-order.md deleted file mode 100644 index 2cfe0f969..000000000 --- a/frontend/manual/purchasing/create-purchase-order.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Create a Purchase Order ---- - -# Create a Purchase Order - -> **When to use:** You need to order materials or supplies from a supplier for a job. - -## What You'll Need - -- [ ] Supplier details (must be set up in the system) -- [ ] List of items to order (descriptions, quantities, costs) -- [ ] The job number to allocate the purchase against - -## Steps - -### 1. Create a new PO - -Navigate to **Purchases > Purchase Orders** in the navbar, then click **New Purchase Order**. - - - -Fill in the basic details: - -- **Supplier** -- Select from the dropdown. Must match an existing supplier. -- **Reference** -- Your internal reference or the supplier's quote number. -- **Expected Delivery Date** -- When you expect the goods to arrive. -- **Pickup Address** -- If applicable, where the goods will be picked up from. - -Click **Save** to create the PO and open it for editing. - -### 2. Add line items - - - -In the line items table, add each item you're ordering: - -| Field | What it means | -| --------------- | ---------------------------------------- | -| **Item Code** | The supplier's product code | -| **Description** | What you're ordering | -| **Quantity** | How many | -| **Unit Cost** | Price per unit | -| **Price TBC** | Tick this if you don't know the cost yet | -| **Job** | Which job this item is for | - -You can assign different line items to different jobs on the same PO. - -### 3. Review and submit - -Check the totals and make sure everything looks right. When you're ready, change the status from **Draft** to **Submitted**. - - - -Once submitted, the supplier and line items are locked -- you can still update delivery dates and pickup addresses, but the core order details are fixed. - -### 4. Send to the supplier - -Use the **Email** button to send the PO to the supplier via email, or **Print** to generate a PDF you can send manually. - -If the supplier is set up in Xero, you can use the **Sync with Xero** button to push the PO through to Xero as well. - -### 5. Record deliveries - -As goods arrive, update the received quantities on each line item. The PO status will automatically move from "Submitted" to "Partially Received" and finally "Fully Received" as you record deliveries. - -## What Happens Next - -- The PO is tracked in the system with a clear status (Draft → Submitted → Partially Received → Fully Received) -- Costs are allocated against the job(s) specified on each line -- If synced to Xero, the PO appears there for accounting purposes -- Material costs flow through to the job's Reality section and into the KPI reports - -## Tips - -::: tip -Use the **Price TBC** checkbox when you know you need to order something but are waiting on a price. This lets you get the order started without holding things up. -::: - -::: tip -Add comments on the PO using the comments section at the bottom. This is handy for tracking conversations with the supplier about delivery changes, back-orders, etc. -::: - -::: warning -Once a PO is submitted, you can't change the supplier or line items. If you need to make changes, you'll need to delete the PO and create a new one. Double-check before submitting. -::: diff --git a/frontend/manual/quoting/assess-and-price.md b/frontend/manual/quoting/assess-and-price.md deleted file mode 100644 index 0c7de47c7..000000000 --- a/frontend/manual/quoting/assess-and-price.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -title: Assess & Price a Job ---- - -# Assess & Price a Job - -> **When to use:** You've got a job on the board and you need to work out what to charge before sending a quote. - -## What You'll Need - -- [ ] A job already created in the system (from the enquiry) -- [ ] A reasonable idea of what the work involves (site visit notes, photos, customer conversation) -- [ ] Access to your rate cards (wage rates, charge-out rates, material costs) - -## Steps - -### 1. Open the job and go to the Estimate section - -Navigate to the job and find the Estimate area. This is where you figure out whether the job is actually worth doing. - - - -From a commercial perspective, jobs are divided into three parts: **time**, **materials**, and **adjustments**. The estimate mirrors this structure -- you'll see a section for each. - -Please don't skip this step. Are we in a financial mess because we've been taking on jobs without knowing if they're profitable, or because we've been taking on jobs despite knowing upfront that we wouldn't make money on them? The estimate is your "finger in the air" -- it doesn't need to be perfect, but it does need to exist. - -### 2. Estimate the time - -The first grid covers labour. For each line you can enter: - -| Column | What it means | -| ----------------- | ---------------------------------------------------- | -| **Description** | What the work is (e.g. "Install signage", "4 folds") | -| **Items** | How many of this task | -| **Mins/Hours** | Time per item | -| **Total Minutes** | Calculated from Items x Mins/Hours | -| **Wage Rate** | What we pay the staff member per hour | -| **Charge Rate** | What we charge the customer per hour | - - - -You can have just one line for the whole job, or break it into multiple lines -- e.g. "4 folds" on one line and "final installation" on another. Pragmatically, decide based on how big the job is. Small jobs just get a single total time; bigger jobs benefit from a breakdown so you can see where the hours go. - -### 3. Estimate the materials - -The second grid covers materials. For each line: - -| Column | What it means | -| --------------- | ------------------------------------------------- | -| **Item Code** | Product or material code | -| **Description** | What it is | -| **Quantity** | How many / how much | -| **Cost Rate** | What we pay for it | -| **Retail Rate** | What we charge the customer | -| **Revenue** | Calculated from Quantity x Retail Rate | -| **Comments** | Anything worth noting (supplier, lead time, etc.) | - - - -Same principle as time -- one line for "materials" on a small job is fine. On a bigger job, break it out so you can see the margins on each item. - -### 4. Add any adjustments - -The third grid is for adjustments -- anything that doesn't fit neatly into time or materials. Think call-out fees, travel, discounts, or allowances. - -| Column | What it means | -| -------------------- | --------------------------------------- | -| **Description** | What the adjustment is for | -| **Cost Adjustment** | What it costs us | -| **Price Adjustment** | What we charge (or credit) the customer | -| **Revenue** | The net effect on revenue | -| **Comments** | Context for anyone reviewing later | - - - -### 5. Review the totals and check the margin - -Before you move on, look at the overall picture. Does the margin make sense? Does the total price feel right for the scope of work? If the numbers say we'll lose money, that's a conversation to have _now_ -- not after we've done the work. - - - -### 6. Copy the estimate to a quote - -Once you're happy with the estimate, hit the **Copy Estimate to Quote** button. This takes your workings and creates the customer-facing quote from them. - - - -## What Happens Next - -- The estimate is saved on the job as your internal cost/price workings. -- The quote is generated from the estimate, ready to review and send. -- Next step: [Send the Quote](/quoting/send-quote) to the customer. - -## Tips - -::: tip -The estimate is for _us_ -- it's our internal view of cost vs. price. The quote is what the customer sees. You can be as detailed as you like in the estimate without worrying about what the customer will think. -::: - -::: tip -For small jobs, don't overthink it. One line for time, one line for materials, done. Save the detailed breakdowns for jobs where the complexity warrants it. -::: - -::: warning -Don't skip the estimate just because the job seems straightforward. A two-minute estimate that shows you'll make 15% margin is infinitely better than no estimate and a nasty surprise at invoicing time. -::: - -::: warning -If the estimate shows we'll lose money or barely break even, flag it before sending the quote. It's much easier to adjust pricing now than to explain a loss after the job is done. -::: diff --git a/frontend/manual/quoting/send-quote.md b/frontend/manual/quoting/send-quote.md deleted file mode 100644 index 6a73c7ea7..000000000 --- a/frontend/manual/quoting/send-quote.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: Send a Quote ---- - -# Send a Quote - -> **When to use:** The estimate is done, the quote section is filled in, and you need to get it in front of the customer. - -## What You'll Need - -- [ ] A completed estimate on the job (see [Assess & Price a Job](/quoting/assess-and-price)) -- [ ] The quote section populated (use the **Copy Estimate to Quote** button if you haven't already) -- [ ] Customer email address or other contact method - -## Steps - -### 1. Review the quote - - - -Open the job and go to the Quote section. The quote is what the customer will see, so check that it makes sense from their perspective. The structure is the same as the estimate -- time, materials, and adjustments -- but the numbers might differ. - -Remember: all jobs have an estimate, but only some have a quote. If it's a T&M (time and materials) job where you're billing as you go, you can skip the quote entirely and leave this section blank. - -### 2. Adjust if needed - -The **Copy Estimate to Quote** button gives you a starting point that matches your estimate exactly. From there you might want to: - -- Add some contingency (round up) -- Reduce the price to win the job -- Simplify the line items so the customer sees a cleaner breakdown - -The estimate stays unchanged as your internal record. The quote is the customer-facing version. - -### 3. Send the quote - - - -Use the **Quote Job** button to generate and send the quote to the customer. This creates a formatted quote document that goes out via email. - -## What Happens Next - -- The customer receives the quote by email -- The job status updates to reflect that a quote has been sent -- When the customer accepts, update the job status and move on to [scheduling](/scheduling/schedule-a-job) -- If they decline or want changes, adjust the quote and resend - -## Tips - -::: tip -If you used "Copy Estimate to Quote", double-check the numbers before sending. The estimate might have internal notes or line items that don't make sense to the customer. -::: - -::: warning -Don't send a quote without an estimate behind it. The estimate is how you know whether the job is profitable. Without it, you're guessing -- and guessing is how you lose money. -::: diff --git a/frontend/manual/scheduling/schedule-a-job.md b/frontend/manual/scheduling/schedule-a-job.md deleted file mode 100644 index a65fccf29..000000000 --- a/frontend/manual/scheduling/schedule-a-job.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Schedule a Job ---- - -# Schedule a Job - -> **When to use:** A quoted job has been accepted and needs to be assigned to staff for work. - -## What You'll Need - -- [ ] An accepted quote or approved job -- [ ] Knowledge of staff availability -- [ ] Customer's preferred timing (if any) - -## Steps - -### 1. Find the job on the Kanban board - - - -The Kanban board is your scheduling hub. Jobs that are ready to be scheduled will be sitting in the "Approved" column (or similar, depending on your workflow). - -### 2. Assign staff - - - -Use the staff panel at the top of the Kanban board to assign team members to the job. You can see who's already assigned to other jobs to avoid overloading anyone. - -### 3. Move the job to the right column - -Drag the job card to the appropriate status column (e.g. "In Progress" or "Scheduled") to reflect that it's been assigned and is ready to go. - -### 4. Confirm with the customer - -Let the customer know when to expect the team on-site. If there are specific access requirements or timing constraints, add them to the job notes so field staff can see them. - -## What Happens Next - -- Staff can see the job assigned to them on the Kanban board -- The job status is visible to everyone in the office -- Field staff do the work and record progress -- see [Complete a Job On-Site](/fieldwork/complete-a-job) - -## Tips - -::: tip -Check the job estimate before scheduling. If the estimate shows 2 days of work, make sure you've allowed for that in the schedule rather than squeezing it into a half-day gap. -::: - -::: warning -Don't move a job to "In Progress" until it's actually been assigned to someone. An unassigned job in the wrong column just creates confusion. -::: diff --git a/frontend/manual/timesheets/end-of-day-entry.md b/frontend/manual/timesheets/end-of-day-entry.md deleted file mode 100644 index a65dc1e1f..000000000 --- a/frontend/manual/timesheets/end-of-day-entry.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: End of Day Entry ---- - -# End of Day Entry - -> **When to use:** At the end of the day, you're entering time from staff time cards into the system. - -## What You'll Need - -- [ ] Time cards from staff (with job numbers, descriptions, and hours) -- [ ] Access to the Timesheets section - -## Steps - -### 1. Select the staff member and date - - - -Use the staff selector arrows at the top to pick whose time card you're entering. Set the date using the date picker, or hit **Today** to jump to the current date. - -The hours display in the header shows you at a glance how many hours have been entered versus the scheduled hours for that day. If the numbers don't match up, something's been missed. - -### 2. Choose the job number - - - -Start by entering the job number. This will populate the job name and client automatically -- you don't need to type those in. - -### 3. Enter the description and hours - - - -Write the description and the time exactly as the person has written it on their time card. The grid columns are: Job Number, Job Name, Client Name, Description, Hours, Rate, Wage, Bill, Billable, Date, and Approval Status. - -### 4. Handle non-billable time - -If it's not suitable to bill the client for the time -- say, fixing our own mistake -- untick the **Billable** checkbox. This way staff still get paid without overcharging the client or having to dump time onto shop jobs. - -You can add notes if you want, for example explaining why you unticked billable, but normally the description is enough. - -### 5. Check the daily summary - - - -The summary at the bottom tells you how many hours the staff member has entered for this day. Use it to quickly check whether a full day has been entered. If you see 4.0 out of 8.0, someone's time card is incomplete. - -It also shows you the split between billable and non-billable entries so you can spot anything unusual. - -### 6. Review from the overview - - - -From the overview you can see all open jobs with the time spent versus the estimate. This is the bar chart view -- blue bars are estimated hours, orange bars are actual hours. If the orange is creeping past the blue, that job is going over budget. - -### 7. Check a staff member's jobs - - - -From any staff member's view you can see the jobs they've put time on that day. Each job card shows the job number, name, client, status, estimated hours, and hours spent. - -You can click on the job number (shown in blue) to jump straight to that job. - -## What Happens Next - -- Time is recorded against the job and counts toward actual hours -- The hours feed into job costing -- estimated vs actual is tracked automatically -- Billable time will appear when it's time to invoice the client -- Non-billable time is still tracked for payroll but won't be charged to the client - -## Tips - -::: tip -Enter time at the end of each day while it's fresh. Trying to reconstruct a week's worth of time cards on Friday afternoon never goes well. -::: - -::: tip -If you see a staff member consistently under their scheduled hours in the summary, check with them before assuming the time card is wrong -- they may have had a shorter day. -::: - -::: warning -Don't just put non-billable time on a shop job to make it disappear. Use the billable checkbox on the actual job instead. That way you still have an accurate picture of how long the real job took -- you just aren't charging the client for it. -::: diff --git a/frontend/package-lock.json b/frontend/package-lock.json index eae158733..33ab94c38 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -29,9 +29,10 @@ "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", "dayjs": "^1.11.20", + "debug": "^4.4.3", "dompurify": "^3.4.11", "event-source-polyfill": "^1.0.31", - "js-cookie": "^3.0.7", + "js-cookie": "^3.0.8", "lodash-es": "^4.18.1", "lucide-vue-next": "^1.0.0", "openapi-zod-client": "^1.18.3", @@ -40,7 +41,7 @@ "qs": "^6.15.2", "quill": "^2.0.3", "reka-ui": "^2.8.2", - "rrweb-player": "^1.0.0-alpha.4", + "rrweb-player": "^2.1.0", "sortablejs": "^1.15.6", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.0", @@ -58,17 +59,18 @@ "@playwright/test": "^1.60.0", "@tsconfig/node22": "^22.0.1", "@types/adm-zip": "^0.5.7", + "@types/debug": "^4.1.13", "@types/dompurify": "^3.2.0", "@types/js-cookie": "^3.0.6", "@types/node": "^25.9.3", "@typescript-eslint/eslint-plugin": "^8.59.4", "@typescript-eslint/parser": "^8.64.0", - "@vitejs/plugin-vue": "^6.0.7", + "@vitejs/plugin-vue": "^6.0.8", "@vue/eslint-config-prettier": "^10.2.0", - "@vue/eslint-config-typescript": "^14.7.0", + "@vue/eslint-config-typescript": "^14.9.0", "@vue/test-utils": "^2.4.6", "@vue/tsconfig": "^0.9.1", - "adm-zip": "^0.5.17", + "adm-zip": "^0.6.0", "codesight": "^1.18.0", "dotenv": "^17.4.2", "eslint": "^10.7.0", @@ -2512,6 +2514,16 @@ "win32" ] }, + "node_modules/@rrweb/packer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rrweb/packer/-/packer-2.1.0.tgz", + "integrity": "sha512-M3Di2TuUN2xqgwK6OiLgikB0ebU1Wxx6iXMoSiDVhDZRgyc6euL5JQs3GclJOuYstDx+8jjbZ12hTcQe+Ai8WQ==", + "license": "MIT", + "dependencies": { + "@rrweb/types": "^2.1.0", + "fflate": "^0.4.4" + } + }, "node_modules/@rrweb/record": { "version": "2.0.0-alpha.20", "resolved": "https://registry.npmjs.org/@rrweb/record/-/record-2.0.0-alpha.20.tgz", @@ -2523,16 +2535,26 @@ "rrweb": "^2.0.0-alpha.20" } }, + "node_modules/@rrweb/replay": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rrweb/replay/-/replay-2.1.0.tgz", + "integrity": "sha512-a6s3Lzf4iIEL22o6+DHZrbjH/q6OIM9CFh/KzU6pDZ2mOYFP3u3Yd3IL57Oc9usnqFt/kNCvOjUf5JbBrQ9o7A==", + "license": "MIT", + "dependencies": { + "@rrweb/types": "^2.1.0", + "rrweb": "^2.1.0" + } + }, "node_modules/@rrweb/types": { - "version": "2.0.0-alpha.20", - "resolved": "https://registry.npmjs.org/@rrweb/types/-/types-2.0.0-alpha.20.tgz", - "integrity": "sha512-RbnDgKxA/odwB1R4gF7eUUj+rdSrq6ROQJsnMw7MIsGzlbSYvJeZN8YY4XqU0G6sKJvXI6bSzk7w/G94jNwzhw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rrweb/types/-/types-2.1.0.tgz", + "integrity": "sha512-svOHkcuukP6rIjz2umwVsS7uYct+2h0N97+bg+8IJYKIUj5+YjwIvqx5y9NILFKPIu3yk+Mb+KDFfJ8RV+4JlA==", "license": "MIT" }, "node_modules/@rrweb/utils": { - "version": "2.0.0-alpha.20", - "resolved": "https://registry.npmjs.org/@rrweb/utils/-/utils-2.0.0-alpha.20.tgz", - "integrity": "sha512-MTQOmhPRe39C0fYaCnnVYOufQsyGzwNXpUStKiyFSfGLUJrzuwhbRoUAKR5w6W2j5XuA0bIz3ZDIBztkquOhLw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rrweb/utils/-/utils-2.1.0.tgz", + "integrity": "sha512-hU7MT3TSdD+Do7FmMoHrBfPevFCheN5vo2mn97Y4vbXuwemopAmo8dqo5KJeMaA6oQQt7FinHJ0Xqb7k68Ci6Q==", "license": "MIT" }, "node_modules/@shikijs/core": { @@ -3927,7 +3949,9 @@ "peer": true }, "node_modules/@types/debug": { - "version": "4.1.12", + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", "license": "MIT", "dependencies": { "@types/ms": "*" @@ -4319,6 +4343,31 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/@typescript-eslint/type-utils": { + "version": "8.59.4", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.4.tgz", + "integrity": "sha512-uonTuPAAKr9XaBGqJ3LjYTh72zy5DyGesljO9gtmk/eFW0W1fRHjnwVYKB35Lm8d5Q5CluEW3gPHjTvZTmgrfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.59.4", + "@typescript-eslint/typescript-estree": "8.59.4", + "@typescript-eslint/utils": "8.59.4", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/@typescript-eslint/types": { "version": "8.59.4", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.4.tgz", @@ -4659,16 +4708,15 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/project-service": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.1.tgz", - "integrity": "sha512-+MuHQlHiEr00Of/IQbE/MmEoi44znZHbR/Pz7Opq4HryUOlRi+/44dro9Ycy8Fyo+/024IWtw8m4JUMCGTYxDg==", + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.65.0.tgz", + "integrity": "sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.59.1", - "@typescript-eslint/types": "^8.59.1", - "debug": "^4.4.3" + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/visitor-keys": "8.65.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4676,21 +4724,14 @@ "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.1.tgz", - "integrity": "sha512-LwuHQI4pDOYVKvmH2dkaJo6YZCSgouVgnS/z7yBPKBMvgtBvyLqiLy9Z6b7+m/TRcX1NFYUqZetI5Y+aT4GEfg==", + "node_modules/@typescript-eslint/types": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.65.0.tgz", + "integrity": "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg==", "dev": true, "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1" - }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -4699,545 +4740,205 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.1.tgz", - "integrity": "sha512-/0nEyPbX7gRsk0Uwfe4ALwwgxuA66d/l2mhRDNlAvaj4U3juhUtJNq0DsY8M2AYwwb9rEq2hrC3IcIcEt++iJA==", + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.65.0.tgz", + "integrity": "sha512-8C71BQkGjiMmXtop7pHVJu1l2NNShFdkCyD6a2ezzs5vU/L3LRtb69EtcteFwz0mYMPzIgOw0n6OV4VBUWZd7A==", "dev": true, "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.65.0", + "eslint-visitor-keys": "^5.0.0" + }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/type-utils": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.4.tgz", - "integrity": "sha512-uonTuPAAKr9XaBGqJ3LjYTh72zy5DyGesljO9gtmk/eFW0W1fRHjnwVYKB35Lm8d5Q5CluEW3gPHjTvZTmgrfA==", + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.59.4", - "@typescript-eslint/typescript-estree": "8.59.4", - "@typescript-eslint/utils": "8.59.4", - "debug": "^4.4.3", - "ts-api-utils": "^2.5.0" - }, + "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true, + "license": "ISC" + }, + "node_modules/@unovis/dagre-layout": { + "version": "0.8.8-2", + "resolved": "https://registry.npmjs.org/@unovis/dagre-layout/-/dagre-layout-0.8.8-2.tgz", + "integrity": "sha512-ZfDvfcYtzzhZhgKZty8XDi+zQIotfRqfNVF5M3dFQ9d9C5MTaRdbeBnPUkNrmlLJGgQ42HMOE2ajZLfm2VlRhg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@unovis/graphlibrary": "^2.2.0-2", + "lodash-es": "^4.17.21" + } + }, + "node_modules/@unovis/graphlibrary": { + "version": "2.2.0-2", + "resolved": "https://registry.npmjs.org/@unovis/graphlibrary/-/graphlibrary-2.2.0-2.tgz", + "integrity": "sha512-HeEzpd/vDyWiIJt0rnh+2ICXUIuF2N0+Z9OJJiKg0DB+eFUcD+bk+9QPhYHwkFwfxdjDA9fHi1DZ/O/bbV58Nw==", + "license": "MIT", + "peer": true, + "dependencies": { + "lodash-es": "^4.17.21" + } + }, + "node_modules/@unovis/ts": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@unovis/ts/-/ts-1.6.2.tgz", + "integrity": "sha512-kSOWRiZsU4Kk1yuJSSaJs+zJCKNg3+2A0/6/7/y1Cu5uHggwGPeE3CzKaNOal/Hpc46bnZw2VLI23mY+gDFbeQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@emotion/css": "^11.7.1", + "@juggle/resize-observer": "^3.3.1", + "@types/d3": "^7.4.0", + "@types/d3-collection": "^1.0.10", + "@types/d3-sankey": "^0.11.2", + "@types/dagre": "^0.7.50", + "@types/geojson": "^7946.0.8", + "@types/leaflet": "1.7.6", + "@types/supercluster": "^5.0.2", + "@types/three": "^0.135.0", + "@types/throttle-debounce": "^5.0.0", + "@types/topojson": "^3.2.3", + "@types/topojson-client": "^3.0.0", + "@types/topojson-specification": "^1.0.2", + "@unovis/dagre-layout": "0.8.8-2", + "@unovis/graphlibrary": "2.2.0-2", + "d3": "^7.2.1", + "d3-collection": "^1.0.7", + "d3-geo-projection": "^4.0.0", + "d3-interpolate-path": "^2.2.3", + "d3-sankey": "^0.12.3", + "elkjs": "^0.10.0", + "geojson": "^0.5.0", + "leaflet": "1.7.1", + "maplibre-gl": "^2.1.9", + "striptags": "^3.2.0", + "supercluster": "^7.1.5", + "three": "^0.135.0", + "throttle-debounce": "^5.0.0", + "to-px": "^1.1.0", + "topojson-client": "^3.1.0", + "tslib": "^2.3.1" + } + }, + "node_modules/@unovis/vue": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@unovis/vue/-/vue-1.6.2.tgz", + "integrity": "sha512-vFglLeiTnNk8VaCEjctAMQXi8Vi+dc217JLkbiIkOLNLXzOClv1ZLv7QjKplRHG+eq/DKC17WSBKINsjXuYbtQ==", + "license": "Apache-2.0", + "peerDependencies": { + "@unovis/ts": "1.6.2", + "vue": "^3" + } + }, + "node_modules/@vee-validate/zod": { + "version": "4.15.1", + "resolved": "https://registry.npmjs.org/@vee-validate/zod/-/zod-4.15.1.tgz", + "integrity": "sha512-329Z4TDBE5Vx0FdbA8S4eR9iGCFFUNGbxjpQ20ff5b5wGueScjocUIx9JHPa79LTG06RnlUR4XogQsjN4tecKA==", + "license": "MIT", + "dependencies": { + "type-fest": "^4.8.3", + "vee-validate": "4.15.1" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" + "zod": "^3.24.0" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/project-service": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.4.tgz", - "integrity": "sha512-Ly00Vu4oAacfDeHp2Zg85ioNG6l8HG+tN1D7J+xTHSxu9y0awYKJ2zH1rFBn8ZSfuGK+7FxK3Cgl3uAz0aZZLg==", + "node_modules/@vitejs/plugin-vue": { + "version": "6.0.8", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.8.tgz", + "integrity": "sha512-0ZjgOg7oO6farnNGup7yvoM/YXZV84OZxHAwtflItNa/6zzQyVb5LNxyea3FEKEX2XlagIKzrlH7wwxkKgtiew==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.59.4", - "@typescript-eslint/types": "^8.59.4", - "debug": "^4.4.3" + "@rolldown/pluginutils": "^1.0.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" + "node": "^20.19.0 || >=22.12.0" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", + "vue": "^3.2.25" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/scope-manager": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.4.tgz", - "integrity": "sha512-mUeR/3H1WrTAddJrwut8OoPjfauaztMQmRwV5fQTUyNVJCLiUXXe4lGEyYIL2oFDpP7UtgbGJXCt72wT0z2S3Q==", + "node_modules/@vitest/expect": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.7.tgz", + "integrity": "sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.59.4", - "@typescript-eslint/visitor-keys": "8.59.4" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.7", + "@vitest/utils": "4.1.7", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" + "url": "https://opencollective.com/vitest" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.4.tgz", - "integrity": "sha512-DLCpnKgD4alVxTBSKulK+gU1KCqOgUXfDRDXh2mZgzokQKa/70ax93I2uVO3m/LLvIAtWZIFoiifudmIqAxpMA==", + "node_modules/@vitest/mocker": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.7.tgz", + "integrity": "sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==", "dev": true, "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "dependencies": { + "@vitest/spy": "4.1.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" }, "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" + "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } } }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/types": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.4.tgz", - "integrity": "sha512-F1o7WJcCq+bc8dwcO/YsSEOudAH8RDtaOhM6wcAQhcUsFhnWQl81JKy48q1hoxAU0qrzM89+31GYh1515Zde3Q==", + "node_modules/@vitest/mocker/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", "dev": true, "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" + "dependencies": { + "@types/estree": "^1.0.0" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/typescript-estree": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.4.tgz", - "integrity": "sha512-F+RuOmcDXo4+TPdfd/TCLS3m2nw8gE9XXyZLrA3JBfaA5tz9TtdkyD3YJFmPxulyc2cKbEok/CvFE3MgSLWnag==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/project-service": "8.59.4", - "@typescript-eslint/tsconfig-utils": "8.59.4", - "@typescript-eslint/types": "8.59.4", - "@typescript-eslint/visitor-keys": "8.59.4", - "debug": "^4.4.3", - "minimatch": "^10.2.2", - "semver": "^7.7.3", - "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/utils": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.4.tgz", - "integrity": "sha512-cYXeNAUsG4lJo5dbc1FcKm+JwIWrj1/UpTORsC6tGMjEZ81DYcvIr9/ueikhMa/Y/gDQYGp+YX9/xQrXje5BJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.59.4", - "@typescript-eslint/types": "8.59.4", - "@typescript-eslint/typescript-estree": "8.59.4" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/@typescript-eslint/visitor-keys": { - "version": "8.59.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.4.tgz", - "integrity": "sha512-U3gxVaDVnuZKhSspW/MzMxE1kq7zOdc072FcSNoqA1I9p8HyKbBFfEHoWckBAMgNMph4MamwS5iTVzFmrnt8TQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.59.4", - "eslint-visitor-keys": "^5.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/types": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.1.tgz", - "integrity": "sha512-ZDCjgccSdYPw5Bxh+my4Z0lJU96ZDN7jbBzvmEn0FZx3RtU1C7VWl6NbDx94bwY3V5YsgwRzJPOgeY2Q/nLG8A==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.1.tgz", - "integrity": "sha512-OUd+vJS05sSkOip+BkZ/2NS8RMxrAAJemsC6vU3kmfLyeaJT0TftHkV9mcx2107MmsBVXXexhVu4F0TZXyMl4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/project-service": "8.59.1", - "@typescript-eslint/tsconfig-utils": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", - "debug": "^4.4.3", - "minimatch": "^10.2.2", - "semver": "^7.7.3", - "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.1.tgz", - "integrity": "sha512-3pIeoXhCeYH9FSCBI8P3iNwJlGuzPlYKkTlen2O9T1DSeeg8UG8jstq6BLk+Mda0qup7mgk4z4XL4OzRaxZ8LA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.1.tgz", - "integrity": "sha512-LdDNl6C5iJExcM0Yh0PwAIBb9PrSiCsWamF/JyEZawm3kFDnRoaq3LGE4bpyRao/fWeGKKyw7icx0YxrLFC5Cg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.59.1", - "eslint-visitor-keys": "^5.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@ungap/structured-clone": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", - "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", - "dev": true, - "license": "ISC" - }, - "node_modules/@unovis/dagre-layout": { - "version": "0.8.8-2", - "resolved": "https://registry.npmjs.org/@unovis/dagre-layout/-/dagre-layout-0.8.8-2.tgz", - "integrity": "sha512-ZfDvfcYtzzhZhgKZty8XDi+zQIotfRqfNVF5M3dFQ9d9C5MTaRdbeBnPUkNrmlLJGgQ42HMOE2ajZLfm2VlRhg==", - "license": "MIT", - "peer": true, - "dependencies": { - "@unovis/graphlibrary": "^2.2.0-2", - "lodash-es": "^4.17.21" - } - }, - "node_modules/@unovis/graphlibrary": { - "version": "2.2.0-2", - "resolved": "https://registry.npmjs.org/@unovis/graphlibrary/-/graphlibrary-2.2.0-2.tgz", - "integrity": "sha512-HeEzpd/vDyWiIJt0rnh+2ICXUIuF2N0+Z9OJJiKg0DB+eFUcD+bk+9QPhYHwkFwfxdjDA9fHi1DZ/O/bbV58Nw==", - "license": "MIT", - "peer": true, - "dependencies": { - "lodash-es": "^4.17.21" - } - }, - "node_modules/@unovis/ts": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/@unovis/ts/-/ts-1.6.2.tgz", - "integrity": "sha512-kSOWRiZsU4Kk1yuJSSaJs+zJCKNg3+2A0/6/7/y1Cu5uHggwGPeE3CzKaNOal/Hpc46bnZw2VLI23mY+gDFbeQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@emotion/css": "^11.7.1", - "@juggle/resize-observer": "^3.3.1", - "@types/d3": "^7.4.0", - "@types/d3-collection": "^1.0.10", - "@types/d3-sankey": "^0.11.2", - "@types/dagre": "^0.7.50", - "@types/geojson": "^7946.0.8", - "@types/leaflet": "1.7.6", - "@types/supercluster": "^5.0.2", - "@types/three": "^0.135.0", - "@types/throttle-debounce": "^5.0.0", - "@types/topojson": "^3.2.3", - "@types/topojson-client": "^3.0.0", - "@types/topojson-specification": "^1.0.2", - "@unovis/dagre-layout": "0.8.8-2", - "@unovis/graphlibrary": "2.2.0-2", - "d3": "^7.2.1", - "d3-collection": "^1.0.7", - "d3-geo-projection": "^4.0.0", - "d3-interpolate-path": "^2.2.3", - "d3-sankey": "^0.12.3", - "elkjs": "^0.10.0", - "geojson": "^0.5.0", - "leaflet": "1.7.1", - "maplibre-gl": "^2.1.9", - "striptags": "^3.2.0", - "supercluster": "^7.1.5", - "three": "^0.135.0", - "throttle-debounce": "^5.0.0", - "to-px": "^1.1.0", - "topojson-client": "^3.1.0", - "tslib": "^2.3.1" - } - }, - "node_modules/@unovis/vue": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/@unovis/vue/-/vue-1.6.2.tgz", - "integrity": "sha512-vFglLeiTnNk8VaCEjctAMQXi8Vi+dc217JLkbiIkOLNLXzOClv1ZLv7QjKplRHG+eq/DKC17WSBKINsjXuYbtQ==", - "license": "Apache-2.0", - "peerDependencies": { - "@unovis/ts": "1.6.2", - "vue": "^3" - } - }, - "node_modules/@vee-validate/zod": { - "version": "4.15.1", - "resolved": "https://registry.npmjs.org/@vee-validate/zod/-/zod-4.15.1.tgz", - "integrity": "sha512-329Z4TDBE5Vx0FdbA8S4eR9iGCFFUNGbxjpQ20ff5b5wGueScjocUIx9JHPa79LTG06RnlUR4XogQsjN4tecKA==", - "license": "MIT", - "dependencies": { - "type-fest": "^4.8.3", - "vee-validate": "4.15.1" - }, - "peerDependencies": { - "zod": "^3.24.0" - } - }, - "node_modules/@vitejs/plugin-vue": { - "version": "6.0.7", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.7.tgz", - "integrity": "sha512-km+p+XdSz9Sxm5rqUbqcSfZYaAniKxWBj1KURl+Jr7UaPvvX7BmaWMdP69I5rrFDeQGyxAG7NXdc57vz+snhWg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rolldown/pluginutils": "^1.0.1" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "peerDependencies": { - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vue": "^3.2.25" - } - }, - "node_modules/@vitest/expect": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.7.tgz", - "integrity": "sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.7", - "@vitest/utils": "4.1.7", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.7.tgz", - "integrity": "sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.7", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/mocker/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.7.tgz", - "integrity": "sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==", + "node_modules/@vitest/pretty-format": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.7.tgz", + "integrity": "sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==", "dev": true, "license": "MIT", "dependencies": { @@ -5465,113 +5166,426 @@ "version": "7.7.7", "license": "MIT", "dependencies": { - "@vue/devtools-kit": "^7.7.7" + "@vue/devtools-kit": "^7.7.7" + } + }, + "node_modules/@vue/devtools-core": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/@vue/devtools-core/-/devtools-core-8.1.5.tgz", + "integrity": "sha512-5e5jQOEssCdZA1wlFEUkIDtb+cAOWuLNWJ52fm4PBWbF7e3oTnM2fneaL42E5lJoolAaUQ678tv/XEb3h4e86Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vue/devtools-kit": "^8.1.5", + "@vue/devtools-shared": "^8.1.5" + }, + "peerDependencies": { + "vue": "^3.0.0" + } + }, + "node_modules/@vue/devtools-core/node_modules/@vue/devtools-kit": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-8.1.5.tgz", + "integrity": "sha512-FcSAxsi4eWuXLCB7Rv9lj0aIVHHPNVQ2BazGf4RJTc2JCqb4BQg0hk87ZFhminCfl+mD5OUI0rX2cgyu4kJOGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vue/devtools-shared": "^8.1.5", + "birpc": "^2.6.1", + "hookable": "^5.5.3", + "perfect-debounce": "^2.0.0" + } + }, + "node_modules/@vue/devtools-core/node_modules/@vue/devtools-shared": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-8.1.5.tgz", + "integrity": "sha512-mhT4zcPFhF+Xk1O4BfhhrbXzpmfqY03fS6xGpcllbQG7lDjhQf8pQHcTIhqQIYx1hfwtHmk/6jM96ele0UxPqQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vue/devtools-core/node_modules/perfect-debounce": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-2.1.0.tgz", + "integrity": "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vue/devtools-kit": { + "version": "7.7.9", + "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-7.7.9.tgz", + "integrity": "sha512-PyQ6odHSgiDVd4hnTP+aDk2X4gl2HmLDfiyEnn3/oV+ckFDuswRs4IbBT7vacMuGdwY/XemxBoh302ctbsptuA==", + "license": "MIT", + "dependencies": { + "@vue/devtools-shared": "^7.7.9", + "birpc": "^2.3.0", + "hookable": "^5.5.3", + "mitt": "^3.0.1", + "perfect-debounce": "^1.0.0", + "speakingurl": "^14.0.1", + "superjson": "^2.2.2" + } + }, + "node_modules/@vue/devtools-shared": { + "version": "7.7.9", + "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-7.7.9.tgz", + "integrity": "sha512-iWAb0v2WYf0QWmxCGy0seZNDPdO3Sp5+u78ORnyeonS6MT4PC7VPrryX2BpMJrwlDeaZ6BD4vP4XKjK0SZqaeA==", + "license": "MIT", + "dependencies": { + "rfdc": "^1.4.1" + } + }, + "node_modules/@vue/eslint-config-prettier": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/@vue/eslint-config-prettier/-/eslint-config-prettier-10.2.0.tgz", + "integrity": "sha512-GL3YBLwv/+b86yHcNNfPJxOTtVFJ4Mbc9UU3zR+KVoG7SwGTjPT+32fXamscNumElhcpXW3mT0DgzS9w32S7Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-config-prettier": "^10.0.1", + "eslint-plugin-prettier": "^5.2.2" + }, + "peerDependencies": { + "eslint": ">= 8.21.0", + "prettier": ">= 3.0.0" + } + }, + "node_modules/@vue/eslint-config-typescript": { + "version": "14.9.0", + "resolved": "https://registry.npmjs.org/@vue/eslint-config-typescript/-/eslint-config-typescript-14.9.0.tgz", + "integrity": "sha512-E3j9hDlfVf10F30MRcLTPY2IIhWIx1nsvkVukk14kTcuA+oBVot9zsP1hzsO+PAMDxV3Fd9FimBJtUBNBL5KFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/utils": "^8.60.0", + "fast-glob": "^3.3.3", + "typescript-eslint": "^8.60.0", + "vue-eslint-parser": "^10.4.0" + }, + "bin": { + "vue-eslint-config-typescript": "dist/bin.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "peerDependencies": { + "eslint": "^9.10.0 || ^10.0.0", + "eslint-plugin-vue": "^9.28.0 || ^10.0.0", + "typescript": ">=4.8.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/@typescript-eslint/utils": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.65.0.tgz", + "integrity": "sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.65.0", + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/typescript-estree": "8.65.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/@typescript-eslint/utils/node_modules/@typescript-eslint/typescript-estree": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.65.0.tgz", + "integrity": "sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.65.0", + "@typescript-eslint/tsconfig-utils": "8.65.0", + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/visitor-keys": "8.65.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/@typescript-eslint/utils/node_modules/@typescript-eslint/typescript-estree/node_modules/@typescript-eslint/project-service": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.65.0.tgz", + "integrity": "sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.65.0", + "@typescript-eslint/types": "^8.65.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/@typescript-eslint/utils/node_modules/@typescript-eslint/typescript-estree/node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.65.0.tgz", + "integrity": "sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/brace-expansion": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", + "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/ignore": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@vue/devtools-core": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-core/-/devtools-core-8.1.5.tgz", - "integrity": "sha512-5e5jQOEssCdZA1wlFEUkIDtb+cAOWuLNWJ52fm4PBWbF7e3oTnM2fneaL42E5lJoolAaUQ678tv/XEb3h4e86Q==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.65.0.tgz", + "integrity": "sha512-/ggrHAwyjENDusvyxbuqxAC2dTnZg/Z8F+fgQtYIz+L6n/9HfSlEZcFGV/NsMNa6CkGk0xUjUAFwC0vHOflvIA==", "dev": true, "license": "MIT", "dependencies": { - "@vue/devtools-kit": "^8.1.5", - "@vue/devtools-shared": "^8.1.5" + "@typescript-eslint/eslint-plugin": "8.65.0", + "@typescript-eslint/parser": "8.65.0", + "@typescript-eslint/typescript-estree": "8.65.0", + "@typescript-eslint/utils": "8.65.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "vue": "^3.0.0" + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@vue/devtools-core/node_modules/@vue/devtools-kit": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-8.1.5.tgz", - "integrity": "sha512-FcSAxsi4eWuXLCB7Rv9lj0aIVHHPNVQ2BazGf4RJTc2JCqb4BQg0hk87ZFhminCfl+mD5OUI0rX2cgyu4kJOGA==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.65.0.tgz", + "integrity": "sha512-IEgob78X12rHpUmtcwFsXhZdVGJtwTVP8FiCLZkR6GlYVrl2PcuB+KhCE5BlVC/eQpQnu8WXRtkHZuPar+gCRA==", "dev": true, "license": "MIT", "dependencies": { - "@vue/devtools-shared": "^8.1.5", - "birpc": "^2.6.1", - "hookable": "^5.5.3", - "perfect-debounce": "^2.0.0" + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.65.0", + "@typescript-eslint/type-utils": "8.65.0", + "@typescript-eslint/utils": "8.65.0", + "@typescript-eslint/visitor-keys": "8.65.0", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.65.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@vue/devtools-core/node_modules/@vue/devtools-shared": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-8.1.5.tgz", - "integrity": "sha512-mhT4zcPFhF+Xk1O4BfhhrbXzpmfqY03fS6xGpcllbQG7lDjhQf8pQHcTIhqQIYx1hfwtHmk/6jM96ele0UxPqQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@vue/devtools-core/node_modules/perfect-debounce": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-2.1.0.tgz", - "integrity": "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/eslint-plugin/node_modules/@typescript-eslint/type-utils": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.65.0.tgz", + "integrity": "sha512-YjaZ7PRI5qY7ax2L3PbvX0rRyGtipAReCWs0mhhDBHjH/vl0g0BonaGXrKdKpMbIIsMIwDgbk/xzkBTyAltS5g==", "dev": true, - "license": "MIT" - }, - "node_modules/@vue/devtools-kit": { - "version": "7.7.9", - "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-7.7.9.tgz", - "integrity": "sha512-PyQ6odHSgiDVd4hnTP+aDk2X4gl2HmLDfiyEnn3/oV+ckFDuswRs4IbBT7vacMuGdwY/XemxBoh302ctbsptuA==", "license": "MIT", "dependencies": { - "@vue/devtools-shared": "^7.7.9", - "birpc": "^2.3.0", - "hookable": "^5.5.3", - "mitt": "^3.0.1", - "perfect-debounce": "^1.0.0", - "speakingurl": "^14.0.1", - "superjson": "^2.2.2" + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/typescript-estree": "8.65.0", + "@typescript-eslint/utils": "8.65.0", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@vue/devtools-shared": { - "version": "7.7.9", - "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-7.7.9.tgz", - "integrity": "sha512-iWAb0v2WYf0QWmxCGy0seZNDPdO3Sp5+u78ORnyeonS6MT4PC7VPrryX2BpMJrwlDeaZ6BD4vP4XKjK0SZqaeA==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/parser": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.65.0.tgz", + "integrity": "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA==", + "dev": true, "license": "MIT", "dependencies": { - "rfdc": "^1.4.1" + "@typescript-eslint/scope-manager": "8.65.0", + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/typescript-estree": "8.65.0", + "@typescript-eslint/visitor-keys": "8.65.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@vue/eslint-config-prettier": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/@vue/eslint-config-prettier/-/eslint-config-prettier-10.2.0.tgz", - "integrity": "sha512-GL3YBLwv/+b86yHcNNfPJxOTtVFJ4Mbc9UU3zR+KVoG7SwGTjPT+32fXamscNumElhcpXW3mT0DgzS9w32S7Bw==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/typescript-estree": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.65.0.tgz", + "integrity": "sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg==", "dev": true, "license": "MIT", "dependencies": { - "eslint-config-prettier": "^10.0.1", - "eslint-plugin-prettier": "^5.2.2" + "@typescript-eslint/project-service": "8.65.0", + "@typescript-eslint/tsconfig-utils": "8.65.0", + "@typescript-eslint/types": "8.65.0", + "@typescript-eslint/visitor-keys": "8.65.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": ">= 8.21.0", - "prettier": ">= 3.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@vue/eslint-config-typescript": { - "version": "14.7.0", - "resolved": "https://registry.npmjs.org/@vue/eslint-config-typescript/-/eslint-config-typescript-14.7.0.tgz", - "integrity": "sha512-iegbMINVc+seZ/QxtzWiOBozctrHiF2WvGedruu2EbLujg9VuU0FQiNcN2z1ycuaoKKpF4m2qzB5HDEMKbxtIg==", + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/typescript-estree/node_modules/@typescript-eslint/project-service": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.65.0.tgz", + "integrity": "sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/utils": "^8.56.0", - "fast-glob": "^3.3.3", - "typescript-eslint": "^8.56.0", - "vue-eslint-parser": "^10.4.0" + "@typescript-eslint/tsconfig-utils": "^8.65.0", + "@typescript-eslint/types": "^8.65.0", + "debug": "^4.4.3" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, "peerDependencies": { - "eslint": "^9.10.0 || ^10.0.0", - "eslint-plugin-vue": "^9.28.0 || ^10.0.0", - "typescript": ">=4.8.4" + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@vue/eslint-config-typescript/node_modules/typescript-eslint/node_modules/@typescript-eslint/typescript-estree/node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.65.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.65.0.tgz", + "integrity": "sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@vue/language-core": { @@ -5873,13 +5887,13 @@ } }, "node_modules/adm-zip": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", - "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", + "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", "dev": true, "license": "MIT", "engines": { - "node": ">=12.0" + "node": ">=14.0" } }, "node_modules/agent-base": { @@ -7280,6 +7294,8 @@ }, "node_modules/debug": { "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -8102,6 +8118,12 @@ "reusify": "^1.0.4" } }, + "node_modules/fflate": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.9.tgz", + "integrity": "sha512-zdxgIEddhfsyCaWpJ2SdXEP8ZMrKJ6+5jl4OupODcywU0IhRk6gdXuVGcPICyfx2H97hVK7xmJtRLPjkxAX8Vw==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "dev": true, @@ -8823,13 +8845,10 @@ } }, "node_modules/js-cookie": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.7.tgz", - "integrity": "sha512-z/wZZgDrkNV1eA0ULjM/F9/50Ya8fbzgKneSpoPsXSGd0KnpdtHfOZWK+GcwLk+EZbS4F9RBhU+K2RgzuDaItw==", - "license": "MIT", - "engines": { - "node": ">=20" - } + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.8.tgz", + "integrity": "sha512-yeJd4aNAdYZQjaon2bpD/Gb0B/omw7HQOsynXXcOiWVCacbBcPlgn8S/d1X6blFSaHao7ozqtW7NZW19xpCtIw==", + "license": "MIT" }, "node_modules/js-tokens": { "version": "4.0.0", @@ -11506,44 +11525,45 @@ } }, "node_modules/rrdom": { - "version": "2.0.0-alpha.20", - "resolved": "https://registry.npmjs.org/rrdom/-/rrdom-2.0.0-alpha.20.tgz", - "integrity": "sha512-hoqjS4662LtBp82qEz9GrqU36UpEmCvTA2Hns3qdF7cklLFFy3G+0Th8hLytJENleHHWxsB5nWJ3eXz5mSRxdQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/rrdom/-/rrdom-2.1.0.tgz", + "integrity": "sha512-d7oaIHzwffxI74UJMZDGq/f97/Yfm3QX4CcTIlQelt2HsLavItLjp8x8nQ0g9Pet5+5wG1RGr9yvsxb1gJlo8A==", "license": "MIT", "dependencies": { - "rrweb-snapshot": "^2.0.0-alpha.20" + "rrweb-snapshot": "^2.1.0" } }, "node_modules/rrweb": { - "version": "2.0.0-alpha.20", - "resolved": "https://registry.npmjs.org/rrweb/-/rrweb-2.0.0-alpha.20.tgz", - "integrity": "sha512-CZKDlm+j1VA50Ko3gnMbpvguCAleljsTNXPnVk9aeNP8o6T6kolRbISHyDZpqZ4G+bdDLlQOignPP3jEsXs8Gg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/rrweb/-/rrweb-2.1.0.tgz", + "integrity": "sha512-gGGvd1eXWnOuJpQumH7+gPmBwTXzNrjmQyedWVePcEx0S5fe3VkQW4vw5f5ItY+vuigaaktte9cZKOg0OEvv+w==", "license": "MIT", "dependencies": { - "@rrweb/types": "^2.0.0-alpha.20", - "@rrweb/utils": "^2.0.0-alpha.20", + "@rrweb/types": "^2.1.0", + "@rrweb/utils": "^2.1.0", "@types/css-font-loading-module": "0.0.7", "@xstate/fsm": "^1.4.0", "base64-arraybuffer": "^1.0.1", "mitt": "^3.0.0", - "rrdom": "^2.0.0-alpha.20", - "rrweb-snapshot": "^2.0.0-alpha.20" + "rrdom": "^2.1.0", + "rrweb-snapshot": "^2.1.0" } }, "node_modules/rrweb-player": { - "version": "1.0.0-alpha.4", - "resolved": "https://registry.npmjs.org/rrweb-player/-/rrweb-player-1.0.0-alpha.4.tgz", - "integrity": "sha512-Wlmn9GZ5Fdqa37vd3TzsYdLl/JWEvXNUrLCrYpnOwEgmY409HwVIvvA5aIo7k582LoKgdRCsB87N+f0oWAR0Kg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/rrweb-player/-/rrweb-player-2.1.0.tgz", + "integrity": "sha512-EU0dCgrN66o2E8GAT67wDBRMgcJ1QJuu2lGa8PKO3xSosSS2eCaHlJWmX9fFCMvKkLsahWe15GnuqUjprCGMoA==", "license": "MIT", "dependencies": { - "@tsconfig/svelte": "^1.0.0", - "rrweb": "^2.0.0-alpha.4" + "@rrweb/packer": "^2.1.0", + "@rrweb/replay": "^2.1.0", + "@tsconfig/svelte": "^1.0.0" } }, "node_modules/rrweb-snapshot": { - "version": "2.0.0-alpha.20", - "resolved": "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-2.0.0-alpha.20.tgz", - "integrity": "sha512-YTNf9YVeaGRo/jxY3FKBge2c/Ojd/KTHmuWloUSB+oyPXuY73ZeeG873qMMmhIpqEn7hn7aBF1eWEQmP7wjf8A==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-2.1.0.tgz", + "integrity": "sha512-CFjUKWlO+Dl72gk8qwcDcNE/Pj9yr6IvGRAiRAx12pmJaSWaOKFoFgpQQ1j/mW0WKwEZXbHnMJL8M92gIsdwUQ==", "license": "MIT", "dependencies": { "postcss": "^8.4.38" @@ -12415,119 +12435,6 @@ "node": ">=14.17" } }, - "node_modules/typescript-eslint": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.59.1.tgz", - "integrity": "sha512-xqDcFVBmlrltH64lklOVp1wYxgJr6LVdg3NamBgH2OOQDLFdTKfIZXF5PfghrnXQKXZGTQs8tr1vL7fJvq8CTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/eslint-plugin": "8.59.1", - "@typescript-eslint/parser": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/utils": "8.59.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/typescript-eslint/node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.1.tgz", - "integrity": "sha512-BOziFIfE+6osHO9FoJG4zjoHUcvI7fTNBSpdAwrNH0/TLvzjsk2oo8XSSOT2HhqUyhZPfHv4UOffoJ9oEEQ7Ag==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/type-utils": "8.59.1", - "@typescript-eslint/utils": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", - "ignore": "^7.0.5", - "natural-compare": "^1.4.0", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^8.59.1", - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/typescript-eslint/node_modules/@typescript-eslint/parser": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.1.tgz", - "integrity": "sha512-HDQH9O/47Dxi1ceDhBXdaldtf/WV9yRYMjbjCuNk3qnaTD564qwv61Y7+gTxwxRKzSrgO5uhtw584igXVuuZkA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/typescript-eslint/node_modules/@typescript-eslint/type-utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.1.tgz", - "integrity": "sha512-klWPBR2ciQHS3f++ug/mVnWKPjBUo7icEL3FAO1lhAR1Z1i5NQYZ1EannMSRYcq5qCv5wNALlXr6fksRHyYl7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/utils": "8.59.1", - "debug": "^4.4.3", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/typescript-eslint/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/ufo": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index d8cc34c5a..906dd9086 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,6 +7,7 @@ "dev": "vite", "build": "run-p type-check \"build-only {@}\" --", "preview": "vite preview", + "preview:e2e": "vite build && vite preview", "build-only": "vite build", "type-check": "vue-tsc --build", "lint": "eslint . --fix", @@ -25,9 +26,6 @@ "test:e2e:ui": "playwright test --ui", "test:e2e:headed": "playwright test --headed", "test:e2e:reset": "npx tsx tests/scripts/e2e-reset.ts", - "manual:dev": "vitepress dev manual --port 5174", - "manual:build": "vitepress build manual", - "manual:preview": "vitepress preview manual", "manual:screenshots": "npx tsx scripts/capture-screenshots.ts" }, "dependencies": { @@ -52,9 +50,10 @@ "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", "dayjs": "^1.11.20", + "debug": "^4.4.3", "dompurify": "^3.4.11", "event-source-polyfill": "^1.0.31", - "js-cookie": "^3.0.7", + "js-cookie": "^3.0.8", "lodash-es": "^4.18.1", "lucide-vue-next": "^1.0.0", "openapi-zod-client": "^1.18.3", @@ -63,7 +62,7 @@ "qs": "^6.15.2", "quill": "^2.0.3", "reka-ui": "^2.8.2", - "rrweb-player": "^1.0.0-alpha.4", + "rrweb-player": "^2.1.0", "sortablejs": "^1.15.6", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.0", @@ -81,17 +80,18 @@ "@playwright/test": "^1.60.0", "@tsconfig/node22": "^22.0.1", "@types/adm-zip": "^0.5.7", + "@types/debug": "^4.1.13", "@types/dompurify": "^3.2.0", "@types/js-cookie": "^3.0.6", "@types/node": "^25.9.3", "@typescript-eslint/eslint-plugin": "^8.59.4", "@typescript-eslint/parser": "^8.64.0", - "@vitejs/plugin-vue": "^6.0.7", + "@vitejs/plugin-vue": "^6.0.8", "@vue/eslint-config-prettier": "^10.2.0", - "@vue/eslint-config-typescript": "^14.7.0", + "@vue/eslint-config-typescript": "^14.9.0", "@vue/test-utils": "^2.4.6", "@vue/tsconfig": "^0.9.1", - "adm-zip": "^0.5.17", + "adm-zip": "^0.6.0", "codesight": "^1.18.0", "dotenv": "^17.4.2", "eslint": "^10.7.0", diff --git a/frontend/schema.yml b/frontend/schema.yml index e76be9589..6bf59f665 100644 --- a/frontend/schema.yml +++ b/frontend/schema.yml @@ -621,7 +621,7 @@ paths: get: operationId: accounts_staff_list description: API endpoint for listing all staff members and creating new staff - members. Supports multipart/form data for file uploads (e.g., profile pictures). + members. Profile pictures are uploaded separately via the staff icon endpoint. summary: List and create staff members tags: - Staff Management @@ -638,19 +638,34 @@ paths: description: '' post: operationId: accounts_staff_create - description: Create a new staff member with the provided details. Supports multipart/form - data for file uploads (e.g., profile pictures). + description: Create a new staff member with the provided details. Profile pictures + are uploaded separately via the staff icon endpoint. summary: Create a new staff member tags: - Staff Management requestBody: content: - multipart/form-data: - schema: - $ref: '#/components/schemas/StaffCreateRequest' - application/x-www-form-urlencoded: + application/json: schema: $ref: '#/components/schemas/StaffCreateRequest' + examples: + CreateStaffMember: + value: + email: john.doe@example.com + first_name: John + last_name: Doe + preferred_name: Johnny + password: securepassword123 + wage_rate: '25.50' + is_office_staff: true + hours_mon: '8.00' + hours_tue: '8.00' + hours_wed: '8.00' + hours_thu: '8.00' + hours_fri: '8.00' + hours_sat: '0.00' + hours_sun: '0.00' + summary: Create Staff Member required: true security: - cookieAuth: [] @@ -664,11 +679,11 @@ paths: /api/accounts/staff/{id}/: get: operationId: accounts_staff_retrieve - description: API endpoint for retrieving, updating, and deleting individual - staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. - Includes comprehensive logging for update operations and handles multipart/form - data for file uploads. - summary: Retrieve, update, or delete staff member + description: API endpoint for retrieving and updating individual staff members. + Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging + for update operations. Profile pictures are uploaded separately via the staff + icon endpoint. Staff are not deleted; offboarding is done by setting date_left. + summary: Retrieve or update staff member parameters: - in: path name: id @@ -689,11 +704,11 @@ paths: description: '' put: operationId: accounts_staff_update - description: API endpoint for retrieving, updating, and deleting individual - staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. - Includes comprehensive logging for update operations and handles multipart/form - data for file uploads. - summary: Retrieve, update, or delete staff member + description: API endpoint for retrieving and updating individual staff members. + Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging + for update operations. Profile pictures are uploaded separately via the staff + icon endpoint. Staff are not deleted; offboarding is done by setting date_left. + summary: Retrieve or update staff member parameters: - in: path name: id @@ -705,12 +720,22 @@ paths: - Staff Management requestBody: content: - multipart/form-data: - schema: - $ref: '#/components/schemas/StaffRequest' - application/x-www-form-urlencoded: + application/json: schema: $ref: '#/components/schemas/StaffRequest' + examples: + UpdateStaffMember: + value: + first_name: Jane + last_name: Smith + preferred_name: Janie + wage_rate: '28.00' + hours_mon: '7.50' + hours_tue: '7.50' + hours_wed: '7.50' + hours_thu: '7.50' + hours_fri: '7.50' + summary: Update Staff Member required: true security: - cookieAuth: [] @@ -723,11 +748,11 @@ paths: description: '' patch: operationId: accounts_staff_partial_update - description: API endpoint for retrieving, updating, and deleting individual - staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. - Includes comprehensive logging for update operations and handles multipart/form - data for file uploads. - summary: Retrieve, update, or delete staff member + description: API endpoint for retrieving and updating individual staff members. + Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging + for update operations. Profile pictures are uploaded separately via the staff + icon endpoint. Staff are not deleted; offboarding is done by setting date_left. + summary: Retrieve or update staff member parameters: - in: path name: id @@ -739,12 +764,58 @@ paths: - Staff Management requestBody: content: - multipart/form-data: + application/json: schema: $ref: '#/components/schemas/PatchedStaffRequest' - application/x-www-form-urlencoded: + examples: + UpdateStaffMember: + value: + first_name: Jane + last_name: Smith + preferred_name: Janie + wage_rate: '28.00' + hours_mon: '7.50' + hours_tue: '7.50' + hours_wed: '7.50' + hours_thu: '7.50' + hours_fri: '7.50' + summary: Update Staff Member + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Staff' + description: '' + /api/accounts/staff/{id}/icon/: + post: + operationId: accounts_staff_icon_create + description: Replace a staff member's profile picture. This is a separate endpoint + because the staff resource itself is JSON-only — a file cannot ride inside + a JSON body. + summary: Upload a staff profile picture + parameters: + - in: path + name: id + schema: + type: string + format: uuid + required: true + tags: + - Staff Management + requestBody: + content: + multipart/form-data: schema: - $ref: '#/components/schemas/PatchedStaffRequest' + type: object + properties: + file: + type: string + format: binary + required: + - file security: - cookieAuth: [] responses: @@ -755,12 +826,11 @@ paths: $ref: '#/components/schemas/Staff' description: '' delete: - operationId: accounts_staff_destroy - description: API endpoint for retrieving, updating, and deleting individual - staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. - Includes comprehensive logging for update operations and handles multipart/form - data for file uploads. - summary: Retrieve, update, or delete staff member + operationId: accounts_staff_icon_destroy + description: "Clear a staff member's profile picture and delete the image from + disk. Idempotent: removing an absent picture succeeds, because the requested + end state already holds." + summary: Remove a staff profile picture parameters: - in: path name: id @@ -773,8 +843,12 @@ paths: security: - cookieAuth: [] responses: - '204': - description: No response body + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Staff' + description: '' /api/accounts/staff/all/: get: operationId: accounts_staff_all_list @@ -8988,6 +9062,212 @@ paths: schema: $ref: '#/components/schemas/AppError' description: '' + /api/workflow/notebook-lm-links/: + get: + operationId: workflow_notebook_lm_links_list + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + tags: + - workflow + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/NotebookLmLink' + description: '' + post: + operationId: workflow_notebook_lm_links_create + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + tags: + - workflow + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + required: true + security: + - cookieAuth: [] + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLink' + description: '' + /api/workflow/notebook-lm-links/{id}/: + get: + operationId: workflow_notebook_lm_links_retrieve + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + parameters: + - in: path + name: id + schema: + type: integer + description: A unique integer value identifying this NotebookLM Link. + required: true + tags: + - workflow + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLink' + description: '' + put: + operationId: workflow_notebook_lm_links_update + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + parameters: + - in: path + name: id + schema: + type: integer + description: A unique integer value identifying this NotebookLM Link. + required: true + tags: + - workflow + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/NotebookLmLinkRequest' + required: true + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLink' + description: '' + patch: + operationId: workflow_notebook_lm_links_partial_update + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + parameters: + - in: path + name: id + schema: + type: integer + description: A unique integer value identifying this NotebookLM Link. + required: true + tags: + - workflow + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PatchedNotebookLmLinkRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedNotebookLmLinkRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedNotebookLmLinkRequest' + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/NotebookLmLink' + description: '' + delete: + operationId: workflow_notebook_lm_links_destroy + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + parameters: + - in: path + name: id + schema: + type: integer + description: A unique integer value identifying this NotebookLM Link. + required: true + tags: + - workflow + security: + - cookieAuth: [] + responses: + '204': + description: No response body + /api/workflow/notebook-lm-links/menu/: + get: + operationId: workflow_notebook_lm_links_menu_list + description: |- + CRUD for NotebookLM training-menu links. + + Full CRUD is office-staff gated (the admin management surface). The extra + `menu` action is readable by any authenticated staff member and returns only + the enabled links they are allowed to see — the navbar reads that, so the + restriction filtering happens server-side. + tags: + - workflow + security: + - cookieAuth: [] + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/NotebookLmLink' + description: '' /api/workflow/xero-apps/: get: operationId: workflow_xero_apps_list @@ -9777,7 +10057,7 @@ components: * `OpenAI` - Openai model_name: type: string - description: Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17) + description: Model name (e.g., gemini-flash-latest) maxLength: 100 default: type: boolean @@ -9808,7 +10088,7 @@ components: * `OpenAI` - Openai model_name: type: string - description: Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17) + description: Model name (e.g., gemini-flash-latest) maxLength: 100 default: type: boolean @@ -9839,7 +10119,7 @@ components: * `OpenAI` - Openai model_name: type: string - description: Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17) + description: Model name (e.g., gemini-flash-latest) maxLength: 100 default: type: boolean @@ -9875,7 +10155,7 @@ components: * `OpenAI` - Openai model_name: type: string - description: Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17) + description: Model name (e.g., gemini-flash-latest) maxLength: 100 default: type: boolean @@ -10516,6 +10796,9 @@ components: type: string nullable: true readOnly: true + xero_quote_terms: + type: string + maxLength: 4000 company_name: type: string readOnly: true @@ -10648,10 +10931,9 @@ components: format: uuid nullable: true title: Xero sales branding theme - description: Branding theme applied to every quote and sales invoice created - in Xero. Select a theme containing the required terms and conditions; - it is configured during Xero setup and required before sales documents - can be created. + description: Controls the layout and presentation of every quote and sales + invoice created in Xero. It is configured during Xero setup and required + before sales documents can be created. enable_xero_sync: type: boolean description: Gate for Xero sync. Defaults True (prod). Dev fixture sets @@ -10902,6 +11184,10 @@ components: format: binary writeOnly: true nullable: true + xero_quote_terms: + type: string + minLength: 1 + maxLength: 4000 company_acronym: type: string nullable: true @@ -11033,10 +11319,9 @@ components: format: uuid nullable: true title: Xero sales branding theme - description: Branding theme applied to every quote and sales invoice created - in Xero. Select a theme containing the required terms and conditions; - it is configured during Xero setup and required before sales documents - can be created. + description: Controls the layout and presentation of every quote and sales + invoice created in Xero. It is configured during Xero setup and required + before sales documents can be created. enable_xero_sync: type: boolean description: Gate for Xero sync. Defaults True (prod). Dev fixture sets @@ -15863,7 +16148,6 @@ components: type: string icon_url: type: string - format: uri nullable: true required: - display_name @@ -16466,6 +16750,76 @@ components: - job_id - job_name - job_number + NotebookLmLink: + type: object + description: Serializer for NotebookLM training-menu links (read + write). + properties: + id: + type: integer + readOnly: true + name: + type: string + description: Menu item name + maxLength: 100 + url: + type: string + format: uri + description: NotebookLM notebook URL + maxLength: 200 + enabled: + type: boolean + description: Show this link in the training menu + restriction: + allOf: + - $ref: '#/components/schemas/RestrictionEnum' + description: |- + Who may see this link in the menu + + * `none` - All staff + * `superuser` - Superusers only + order: + type: integer + maximum: 2147483647 + minimum: -2147483648 + description: Menu display order + required: + - id + - name + - url + NotebookLmLinkRequest: + type: object + description: Serializer for NotebookLM training-menu links (read + write). + properties: + name: + type: string + minLength: 1 + description: Menu item name + maxLength: 100 + url: + type: string + format: uri + minLength: 1 + description: NotebookLM notebook URL + maxLength: 200 + enabled: + type: boolean + description: Show this link in the training menu + restriction: + allOf: + - $ref: '#/components/schemas/RestrictionEnum' + description: |- + Who may see this link in the menu + + * `none` - All staff + * `superuser` - Superusers only + order: + type: integer + maximum: 2147483647 + minimum: -2147483648 + description: Menu display order + required: + - name + - url NullEnum: enum: - null @@ -16682,7 +17036,7 @@ components: * `OpenAI` - Openai model_name: type: string - description: Specific model name (e.g., gemini-2.5-flash-lite-preview-06-17) + description: Model name (e.g., gemini-flash-latest) maxLength: 100 default: type: boolean @@ -16705,6 +17059,10 @@ components: format: binary writeOnly: true nullable: true + xero_quote_terms: + type: string + minLength: 1 + maxLength: 4000 company_acronym: type: string nullable: true @@ -16836,10 +17194,9 @@ components: format: uuid nullable: true title: Xero sales branding theme - description: Branding theme applied to every quote and sales invoice created - in Xero. Select a theme containing the required terms and conditions; - it is configured during Xero setup and required before sales documents - can be created. + description: Controls the layout and presentation of every quote and sales + invoice created in Xero. It is configured during Xero setup and required + before sales documents can be created. enable_xero_sync: type: boolean description: Gate for Xero sync. Defaults True (prod). Dev fixture sets @@ -17270,6 +17627,37 @@ components: minimum: 0 exclusiveMaximum: true description: Company-level rate used to seed JobLabourRate on new jobs + PatchedNotebookLmLinkRequest: + type: object + description: Serializer for NotebookLM training-menu links (read + write). + properties: + name: + type: string + minLength: 1 + description: Menu item name + maxLength: 100 + url: + type: string + format: uri + minLength: 1 + description: NotebookLM notebook URL + maxLength: 200 + enabled: + type: boolean + description: Show this link in the training menu + restriction: + allOf: + - $ref: '#/components/schemas/RestrictionEnum' + description: |- + Who may see this link in the menu + + * `none` - All staff + * `superuser` - Superusers only + order: + type: integer + maximum: 2147483647 + minimum: -2147483648 + description: Menu display order PatchedPersonContactMethodWriteRequest: type: object properties: @@ -17396,8 +17784,6 @@ components: $ref: '#/components/schemas/PurchaseOrderLineUpdateRequest' PatchedStaffRequest: type: object - description: Base serializer for Staff model with shared logic for create and - update operations. properties: email: type: string @@ -17517,11 +17903,6 @@ components: writeOnly: true minLength: 1 maxLength: 128 - icon: - type: string - format: binary - writeOnly: true - nullable: true PatchedStockItemRequest: type: object description: Serializer for individual stock items. @@ -20122,6 +20503,14 @@ components: description: |- * `merge` - merge * `review` - review + RestrictionEnum: + enum: + - none + - superuser + type: string + description: |- + * `none` - All staff + * `superuser` - Superusers only RoleEnum: enum: - user @@ -21083,8 +21472,6 @@ components: * `quality` - Quality - Prioritize Quality Staff: type: object - description: Base serializer for Staff model with shared logic for create and - update operations. properties: id: type: string @@ -21242,8 +21629,6 @@ components: - wage_rate StaffCreateRequest: type: object - description: Base serializer for Staff model with shared logic for create and - update operations. properties: email: type: string @@ -21373,11 +21758,6 @@ components: writeOnly: true minLength: 1 maxLength: 128 - icon: - type: string - format: binary - writeOnly: true - nullable: true required: - email - first_name @@ -21659,8 +22039,6 @@ components: - wage_rate StaffRequest: type: object - description: Base serializer for Staff model with shared logic for create and - update operations. properties: email: type: string @@ -21780,11 +22158,6 @@ components: writeOnly: true minLength: 1 maxLength: 128 - icon: - type: string - format: binary - writeOnly: true - nullable: true required: - email - first_name diff --git a/frontend/scripts/capture-screenshots.ts b/frontend/scripts/capture-screenshots.ts index 2a9d69b94..a8ab698e6 100644 --- a/frontend/scripts/capture-screenshots.ts +++ b/frontend/scripts/capture-screenshots.ts @@ -347,6 +347,9 @@ async function captureSingleScreenshot(options: CliOptions): Promise { const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, baseURL: baseUrl, + // Bypass the ngrok-free browser-warning interstitial when the app is + // served through an ngrok tunnel (harmless on non-ngrok hosts). + extraHTTPHeaders: { 'ngrok-skip-browser-warning': 'true' }, }) const failedResponses: PageDiagnostics['failedResponses'] = [] context.on('response', (response) => { @@ -418,6 +421,9 @@ async function captureScreenshots(): Promise { const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, baseURL: baseUrl, + // Bypass the ngrok-free browser-warning interstitial when the app is + // served through an ngrok tunnel (harmless on non-ngrok hosts). + extraHTTPHeaders: { 'ngrok-skip-browser-warning': 'true' }, }) const page = await context.newPage() diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 7a2e7c066..d7c089387 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -6,7 +6,7 @@ diff --git a/frontend/src/__tests__/App.test.ts b/frontend/src/__tests__/App.test.ts index 6d76e42b1..6c8a6fc2b 100644 --- a/frontend/src/__tests__/App.test.ts +++ b/frontend/src/__tests__/App.test.ts @@ -20,10 +20,6 @@ vi.mock('@/services/sessionReplayService', () => ({ stopSessionReplay: vi.fn().mockResolvedValue(undefined), })) -vi.mock('@/utils/debug', () => ({ - debugLog: vi.fn(), -})) - const user = { id: '11111111-1111-4111-8111-111111111111', username: 'cindy@example.com', diff --git a/frontend/src/__tests__/debug-forwarder.test.ts b/frontend/src/__tests__/debug-forwarder.test.ts new file mode 100644 index 000000000..c81dea5cf --- /dev/null +++ b/frontend/src/__tests__/debug-forwarder.test.ts @@ -0,0 +1,51 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +// The forwarder lives under tests/ (Playwright fixtures) which vitest excludes +// as a test-file root, but its pure bridge functions are imported here so the +// area->app-namespace mapping the whole forwarder depends on is guarded by unit +// tests. Relative import because `@/` only maps into src/. +import { browserDebugGlob, enabledAreas } from '../../tests/fixtures/debug-forwarder' + +describe('debug-forwarder bridge', () => { + const originalDebug = process.env.DEBUG + + beforeEach(() => { + delete process.env.DEBUG + }) + + afterEach(() => { + if (originalDebug === undefined) { + delete process.env.DEBUG + } else { + process.env.DEBUG = originalDebug + } + }) + + it('enables nothing when DEBUG is unset', () => { + expect(enabledAreas()).toEqual([]) + expect(browserDebugGlob()).toBeNull() + }) + + it('ignores non-e2e debug namespaces', () => { + process.env.DEBUG = 'job:autosave' + expect(enabledAreas()).toEqual([]) + expect(browserDebugGlob()).toBeNull() + }) + + it('maps a single area to its app glob', () => { + process.env.DEBUG = 'e2e:autosave' + expect(enabledAreas()).toEqual(['e2e:autosave']) + expect(browserDebugGlob()).toBe('job:autosave') + }) + + it('joins multiple areas into a comma-separated glob', () => { + process.env.DEBUG = 'e2e:kanban,e2e:job' + expect(browserDebugGlob()).toBe('kanban:*,job:*') + }) + + it('drops an e2e area that has no bridge entry', () => { + process.env.DEBUG = 'e2e:autosave,e2e:unknown' + expect(enabledAreas()).toEqual(['e2e:autosave', 'e2e:unknown']) + expect(browserDebugGlob()).toBe('job:autosave') + }) +}) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 1f2555be2..6d7ff16e7 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,7 +1,7 @@ import { Zodios } from '@zodios/core' import axios from 'axios' import { endpoints } from './generated/api' -import { debugLog } from '../utils/debug' +import debug from 'debug' import { trimStringsDeep } from '../utils/sanitize' import { isJobEndpoint, @@ -17,6 +17,8 @@ import { emitConcurrencyRetry } from '../composables/useConcurrencyEvents' import { emitPoConcurrencyRetry } from '../composables/usePoConcurrencyEvents' import { getSessionReplayId } from '@/services/sessionReplayState' +const log = debug('api:client') + // Global registry for ETag management to avoid circular imports let etagManager: { getETag: (jobId: string) => string | null @@ -100,7 +102,7 @@ axios.interceptors.request.use( const etag = etagManager.getETag(jobId) if (etag) { config.headers['If-Match'] = etag - debugLog(`[ETags] Added If-Match header for ${url}:`, etag) + log(`Added If-Match header for ${url}:`, etag) } } } @@ -114,9 +116,9 @@ axios.interceptors.request.use( try { const body = typeof config.data === 'string' ? JSON.parse(config.data) : config.data poId = body?.purchase_order_id || null - debugLog(`[PO ETags] Extracted PO ID from delivery receipt body:`, poId) + log(`Extracted PO ID from delivery receipt body:`, poId) } catch (e) { - debugLog(`[PO ETags] Failed to parse delivery receipt body:`, e) + log(`Failed to parse delivery receipt body:`, e) } } else { // For other PO endpoints, extract from URL @@ -127,7 +129,7 @@ axios.interceptors.request.use( const etag = poEtagManager.getETag(poId) if (etag) { config.headers['If-Match'] = etag - debugLog(`[PO ETags] Added If-Match header for ${url}:`, etag) + log(`Added If-Match header for ${url}:`, etag) } } } @@ -143,7 +145,7 @@ axios.interceptors.request.use( const jobId = extractJobId(url) if (jobId && isJobMutationEndpoint(url)) { - debugLog(`[ETags] Validation error for job ${jobId} - letting JobDelta handle it`) + log(`Validation error for job ${jobId} - letting JobDelta handle it`) // JobDelta service will surface the error to user, no silent reload } } @@ -162,7 +164,7 @@ axios.interceptors.response.use( const jobId = extractJobId(url) if (jobId && etagManager) { etagManager.setETag(jobId, etag) - debugLog(`[ETags] Captured ETag for ${url}:`, etag) + log(`Captured ETag for ${url}:`, etag) } } @@ -171,7 +173,7 @@ axios.interceptors.response.use( const poId = extractPoId(url) if (poId && poEtagManager) { poEtagManager.setETag(poId, etag) - debugLog(`[PO ETags] Captured ETag for ${url}:`, etag) + log(`Captured ETag for ${url}:`, etag) } } @@ -185,14 +187,14 @@ axios.interceptors.response.use( const poId = extractPoId(url) if (jobId && jobReloadManager) { - debugLog(`[ETags] Concurrency conflict detected for job ${jobId}, reloading data`) + log(`Concurrency conflict detected for job ${jobId}, reloading data`) // Reload job data to get fresh ETag try { await jobReloadManager.reloadJobOnConflict(jobId) - debugLog(`[ETags] Successfully reloaded job ${jobId} after concurrency conflict`) + log(`Successfully reloaded job ${jobId} after concurrency conflict`) } catch (reloadError) { - debugLog(`[ETags] Failed to reload job ${jobId}:`, reloadError) + log(`Failed to reload job ${jobId}:`, reloadError) } // Show persistent user notification with retry option @@ -216,14 +218,14 @@ axios.interceptors.response.use( } if (poId && poReloadManager) { - debugLog(`[PO ETags] Concurrency conflict detected for PO ${poId}, reloading data`) + log(`Concurrency conflict detected for PO ${poId}, reloading data`) // Reload PO data to get fresh ETag try { await poReloadManager.reloadPoOnConflict(poId) - debugLog(`[PO ETags] Successfully reloaded PO ${poId} after concurrency conflict`) + log(`Successfully reloaded PO ${poId} after concurrency conflict`) } catch (reloadError) { - debugLog(`[PO ETags] Failed to reload PO ${poId}:`, reloadError) + log(`Failed to reload PO ${poId}:`, reloadError) } // Show persistent user notification with retry option @@ -254,14 +256,14 @@ axios.interceptors.response.use( const poId = extractPoId(url) if (jobId && jobReloadManager) { - debugLog(`[ETags] Missing ETag for job ${jobId}, reloading data`) + log(`Missing ETag for job ${jobId}, reloading data`) // Reload job data to get ETag try { await jobReloadManager.reloadJobOnConflict(jobId) - debugLog(`[ETags] Successfully reloaded job ${jobId} to get ETag`) + log(`Successfully reloaded job ${jobId} to get ETag`) } catch (reloadError) { - debugLog(`[ETags] Failed to reload job ${jobId}:`, reloadError) + log(`Failed to reload job ${jobId}:`, reloadError) } // Show persistent user notification with retry option @@ -285,14 +287,14 @@ axios.interceptors.response.use( } if (poId && poReloadManager) { - debugLog(`[PO ETags] Missing ETag for PO ${poId}, reloading data`) + log(`Missing ETag for PO ${poId}, reloading data`) // Reload PO data to get ETag try { await poReloadManager.reloadPoOnConflict(poId) - debugLog(`[PO ETags] Successfully reloaded PO ${poId} to get ETag`) + log(`Successfully reloaded PO ${poId} to get ETag`) } catch (reloadError) { - debugLog(`[PO ETags] Failed to reload PO ${poId}:`, reloadError) + log(`Failed to reload PO ${poId}:`, reloadError) } // Show persistent user notification with retry option diff --git a/frontend/src/api/generated/api.ts b/frontend/src/api/generated/api.ts index 3980d8521..3a4c76662 100644 --- a/frontend/src/api/generated/api.ts +++ b/frontend/src/api/generated/api.ts @@ -495,7 +495,6 @@ const StaffCreateRequest = z.object({ groups: z.array(z.number().int()).optional(), user_permissions: z.array(z.number().int()).optional(), password: z.string().min(1).max(128), - icon: z.instanceof(File).nullish(), }) const StaffRequest = z.object({ email: z.string().min(1).max(254).email(), @@ -519,7 +518,6 @@ const StaffRequest = z.object({ groups: z.array(z.number().int()).optional(), user_permissions: z.array(z.number().int()).optional(), password: z.string().min(1).max(128).optional(), - icon: z.instanceof(File).nullish(), }) const PatchedStaffRequest = z .object({ @@ -544,7 +542,6 @@ const PatchedStaffRequest = z groups: z.array(z.number().int()), user_permissions: z.array(z.number().int()), password: z.string().min(1).max(128), - icon: z.instanceof(File).nullable(), }) .partial() const KanbanStaff = z.object({ @@ -905,6 +902,7 @@ const CompanyDefaults = z.object({ id: z.number().int(), logo_url: z.string().nullable(), logo_wide_url: z.string().nullable(), + xero_quote_terms: z.string().max(4000).optional(), company_name: z.string(), company_acronym: z.string().max(10).nullish(), time_markup: z.number().gt(-1000).lt(1000).optional(), @@ -971,6 +969,7 @@ const CompanyDefaults = z.object({ const CompanyDefaultsRequest = z.object({ logo: z.instanceof(File).nullish(), logo_wide: z.instanceof(File).nullish(), + xero_quote_terms: z.string().min(1).max(4000).optional(), company_acronym: z.string().max(10).nullish(), time_markup: z.number().gt(-1000).lt(1000).optional(), materials_markup: z.number().gt(-1000).lt(1000).optional(), @@ -1035,6 +1034,7 @@ const PatchedCompanyDefaultsRequest = z .object({ logo: z.instanceof(File).nullable(), logo_wide: z.instanceof(File).nullable(), + xero_quote_terms: z.string().min(1).max(4000), company_acronym: z.string().max(10).nullable(), time_markup: z.number().gt(-1000).lt(1000), materials_markup: z.number().gt(-1000).lt(1000), @@ -2066,7 +2066,7 @@ const PreviewQuoteResponse = z const KanbanJobPerson = z.object({ id: z.string().uuid(), display_name: z.string(), - icon_url: z.string().url().nullable(), + icon_url: z.string().nullable(), }) const KanbanJob = z.object({ id: z.string().uuid(), @@ -3547,6 +3547,31 @@ const AppErrorRequest = z.object({ session_replay: z.string().uuid().nullish(), resolved_by: z.string().uuid().nullish(), }) +const RestrictionEnum = z.enum(['none', 'superuser']) +const NotebookLmLink = z.object({ + id: z.number().int(), + name: z.string().max(100), + url: z.string().max(200).url(), + enabled: z.boolean().optional(), + restriction: RestrictionEnum.optional(), + order: z.number().int().gte(-2147483648).lte(2147483647).optional(), +}) +const NotebookLmLinkRequest = z.object({ + name: z.string().min(1).max(100), + url: z.string().min(1).max(200).url(), + enabled: z.boolean().optional(), + restriction: RestrictionEnum.optional(), + order: z.number().int().gte(-2147483648).lte(2147483647).optional(), +}) +const PatchedNotebookLmLinkRequest = z + .object({ + name: z.string().min(1).max(100), + url: z.string().min(1).max(200).url(), + enabled: z.boolean(), + restriction: RestrictionEnum, + order: z.number().int().gte(-2147483648).lte(2147483647), + }) + .partial() const XeroApp = z.object({ id: z.string().uuid(), label: z.string().max(64), @@ -4131,6 +4156,10 @@ export const schemas = { PatchedAIProviderCreateUpdateRequest, AIProviderRequest, AppErrorRequest, + RestrictionEnum, + NotebookLmLink, + NotebookLmLinkRequest, + PatchedNotebookLmLinkRequest, XeroApp, XeroAppCreateRequest, XeroAppCreate, @@ -4620,7 +4649,7 @@ Returns: method: 'get', path: '/api/accounts/staff/', alias: 'accounts_staff_list', - description: `API endpoint for listing all staff members and creating new staff members. Supports multipart/form data for file uploads (e.g., profile pictures).`, + description: `API endpoint for listing all staff members and creating new staff members. Profile pictures are uploaded separately via the staff icon endpoint.`, requestFormat: 'json', response: z.array(Staff), }, @@ -4628,8 +4657,8 @@ Returns: method: 'post', path: '/api/accounts/staff/', alias: 'accounts_staff_create', - description: `Create a new staff member with the provided details. Supports multipart/form data for file uploads (e.g., profile pictures).`, - requestFormat: 'form-data', + description: `Create a new staff member with the provided details. Profile pictures are uploaded separately via the staff icon endpoint.`, + requestFormat: 'json', parameters: [ { name: 'body', @@ -4643,7 +4672,7 @@ Returns: method: 'get', path: '/api/accounts/staff/:id/', alias: 'accounts_staff_retrieve', - description: `API endpoint for retrieving, updating, and deleting individual staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. Includes comprehensive logging for update operations and handles multipart/form data for file uploads.`, + description: `API endpoint for retrieving and updating individual staff members. Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging for update operations. Profile pictures are uploaded separately via the staff icon endpoint. Staff are not deleted; offboarding is done by setting date_left.`, requestFormat: 'json', parameters: [ { @@ -4658,8 +4687,8 @@ Returns: method: 'put', path: '/api/accounts/staff/:id/', alias: 'accounts_staff_update', - description: `API endpoint for retrieving, updating, and deleting individual staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. Includes comprehensive logging for update operations and handles multipart/form data for file uploads.`, - requestFormat: 'form-data', + description: `API endpoint for retrieving and updating individual staff members. Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging for update operations. Profile pictures are uploaded separately via the staff icon endpoint. Staff are not deleted; offboarding is done by setting date_left.`, + requestFormat: 'json', parameters: [ { name: 'body', @@ -4678,8 +4707,8 @@ Returns: method: 'patch', path: '/api/accounts/staff/:id/', alias: 'accounts_staff_partial_update', - description: `API endpoint for retrieving, updating, and deleting individual staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. Includes comprehensive logging for update operations and handles multipart/form data for file uploads.`, - requestFormat: 'form-data', + description: `API endpoint for retrieving and updating individual staff members. Supports GET (retrieve) and PUT/PATCH (update). Includes comprehensive logging for update operations. Profile pictures are uploaded separately via the staff icon endpoint. Staff are not deleted; offboarding is done by setting date_left.`, + requestFormat: 'json', parameters: [ { name: 'body', @@ -4694,11 +4723,31 @@ Returns: ], response: Staff, }, + { + method: 'post', + path: '/api/accounts/staff/:id/icon/', + alias: 'accounts_staff_icon_create', + description: `Replace a staff member's profile picture. This is a separate endpoint because the staff resource itself is JSON-only — a file cannot ride inside a JSON body.`, + requestFormat: 'form-data', + parameters: [ + { + name: 'body', + type: 'Body', + schema: z.object({ file: z.instanceof(File) }), + }, + { + name: 'id', + type: 'Path', + schema: z.string().uuid(), + }, + ], + response: Staff, + }, { method: 'delete', - path: '/api/accounts/staff/:id/', - alias: 'accounts_staff_destroy', - description: `API endpoint for retrieving, updating, and deleting individual staff members. Supports GET (retrieve), PUT/PATCH (update), and DELETE operations. Includes comprehensive logging for update operations and handles multipart/form data for file uploads.`, + path: '/api/accounts/staff/:id/icon/', + alias: 'accounts_staff_icon_destroy', + description: `Clear a staff member's profile picture and delete the image from disk. Idempotent: removing an absent picture succeeds, because the requested end state already holds.`, requestFormat: 'json', parameters: [ { @@ -4707,7 +4756,7 @@ Returns: schema: z.string().uuid(), }, ], - response: z.void(), + response: Staff, }, { method: 'get', @@ -10366,6 +10415,142 @@ Endpoints: ], response: AppError, }, + { + method: 'get', + path: '/api/workflow/notebook-lm-links/', + alias: 'workflow_notebook_lm_links_list', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + response: z.array(NotebookLmLink), + }, + { + method: 'post', + path: '/api/workflow/notebook-lm-links/', + alias: 'workflow_notebook_lm_links_create', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + parameters: [ + { + name: 'body', + type: 'Body', + schema: NotebookLmLinkRequest, + }, + ], + response: NotebookLmLink, + }, + { + method: 'get', + path: '/api/workflow/notebook-lm-links/:id/', + alias: 'workflow_notebook_lm_links_retrieve', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + parameters: [ + { + name: 'id', + type: 'Path', + schema: z.number().int(), + }, + ], + response: NotebookLmLink, + }, + { + method: 'put', + path: '/api/workflow/notebook-lm-links/:id/', + alias: 'workflow_notebook_lm_links_update', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + parameters: [ + { + name: 'body', + type: 'Body', + schema: NotebookLmLinkRequest, + }, + { + name: 'id', + type: 'Path', + schema: z.number().int(), + }, + ], + response: NotebookLmLink, + }, + { + method: 'patch', + path: '/api/workflow/notebook-lm-links/:id/', + alias: 'workflow_notebook_lm_links_partial_update', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + parameters: [ + { + name: 'body', + type: 'Body', + schema: PatchedNotebookLmLinkRequest, + }, + { + name: 'id', + type: 'Path', + schema: z.number().int(), + }, + ], + response: NotebookLmLink, + }, + { + method: 'delete', + path: '/api/workflow/notebook-lm-links/:id/', + alias: 'workflow_notebook_lm_links_destroy', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + parameters: [ + { + name: 'id', + type: 'Path', + schema: z.number().int(), + }, + ], + response: z.void(), + }, + { + method: 'get', + path: '/api/workflow/notebook-lm-links/menu/', + alias: 'workflow_notebook_lm_links_menu_list', + description: `CRUD for NotebookLM training-menu links. + +Full CRUD is office-staff gated (the admin management surface). The extra +`menu` action is readable by any authenticated staff member and returns only +the enabled links they are allowed to see — the navbar reads that, so the +restriction filtering happens server-side.`, + requestFormat: 'json', + response: z.array(NotebookLmLink), + }, { method: 'get', path: '/api/workflow/xero-apps/', diff --git a/frontend/src/components/AIProvidersDialog.vue b/frontend/src/components/AIProvidersDialog.vue index b519fbd45..d27d37513 100644 --- a/frontend/src/components/AIProvidersDialog.vue +++ b/frontend/src/components/AIProvidersDialog.vue @@ -99,7 +99,7 @@ diff --git a/frontend/src/components/DataTable.vue b/frontend/src/components/DataTable.vue index 4e4da09f5..997cb6984 100644 --- a/frontend/src/components/DataTable.vue +++ b/frontend/src/components/DataTable.vue @@ -1,4 +1,4 @@ - diff --git a/frontend/src/components/board/WorkshopModeView.vue b/frontend/src/components/board/WorkshopModeView.vue index a74235101..384d7d10a 100644 --- a/frontend/src/components/board/WorkshopModeView.vue +++ b/frontend/src/components/board/WorkshopModeView.vue @@ -20,9 +20,11 @@ import { DrawerTitle, } from '@/components/ui/drawer' import { toast } from 'vue-sonner' -import { debugLog } from '@/utils/debug' +import debug from 'debug' import type { z } from 'zod' +const log = debug('workshop:board') + type WorkshopJob = z.infer const router = useRouter() @@ -64,7 +66,7 @@ const loadJobs = async () => { } } } catch (error) { - debugLog('Error loading workshop jobs:', error) + log('Error loading workshop jobs:', error) toast.error('Failed to load jobs. Please try again.') } finally { loading.value = false diff --git a/frontend/src/components/chat/README.md b/frontend/src/components/chat/README.md index 7ec3a709b..fa95e06e8 100644 --- a/frontend/src/components/chat/README.md +++ b/frontend/src/components/chat/README.md @@ -76,7 +76,7 @@ interface ToolCall { interface McpMetadata { tool_calls?: ToolCall[] // Array of executed tool calls tool_definitions?: ToolDefinition[] // Available tools for session - model?: string // AI model used (e.g., "gemini-1.5-pro") + model?: string // AI model used (e.g., "gemini-flash-latest") system_prompt?: string // System prompt used user_message?: string // Original user message chat_history?: any[] // Conversation history diff --git a/frontend/src/components/crm/PhoneCallTable.vue b/frontend/src/components/crm/PhoneCallTable.vue index c338a327f..729780465 100644 --- a/frontend/src/components/crm/PhoneCallTable.vue +++ b/frontend/src/components/crm/PhoneCallTable.vue @@ -85,11 +85,16 @@ Assign company first +