diff --git a/CHANGELOG.md b/CHANGELOG.md index 089899a..fb0b5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- API endpoint for sending single letters without a shipment. +- Admin page for viewing api letters and deactivating api users. +- Added get_pdf param to get_letter api endpoint to allow not downloading letter files. + +### Changed + +- Added created_by and post_type to letters db table to support single letters. +- Made shipment_id nullable in letters db table. +- Api users can not be deleted if any letters are attached to them. +- Added indexes to the letters, nemsms messages and registration tasks db tables. + ## [0.3.0] ### Added diff --git a/SCHEMAS.md b/SCHEMAS.md deleted file mode 100644 index db12742..0000000 --- a/SCHEMAS.md +++ /dev/null @@ -1,42 +0,0 @@ -# Database Schemas - -## Shipments - -Shipments that has been created in the ui. - -| Column | Type | Note | -| ----------- | -------- | ------------- | -| id | int | PK | -| name | str | | -| description | str | | -| template_id | int | FK(templates) | -| created_at | datetime | | -| created_by | str | | -| status | str | | -| type | str | NemSMS/DP | -| | | | - -## Letters - -Descriptions of each letter inside shipments. - -| Column | Type | Note | -| ------------ | -------- | ------------- | -| id | int | PK | -| shipment_id | int | FK(shipments) | -| recipient_id | int | | -| updated_at | datetime | | -| status | str | | -| field_data | str | json | -| | | | - -## Templates - -Docx templates used to generate letters. - -| Column | Type | Note | -| ----------- | ---- | ---- | -| id | int | PK | -| file_name | str | | -| file_data | blob | | -| field_names | str | | diff --git a/src/OpenPostbud/__main__.py b/src/OpenPostbud/__main__.py index e03dfb4..87f3b18 100644 --- a/src/OpenPostbud/__main__.py +++ b/src/OpenPostbud/__main__.py @@ -3,7 +3,7 @@ import argparse from OpenPostbud.database.check_registration import registration_job -from OpenPostbud.database.digital_post import shipments +from OpenPostbud.database.digital_post import shipments, letters from OpenPostbud.database.nemsms import nemsms_shipments from OpenPostbud.middleware import authentication from OpenPostbud.database import connection @@ -20,6 +20,7 @@ def database_cleanup(*_): shipments.delete_old_shipments() registration_job.delete_old_registration_jobs() nemsms_shipments.delete_old_shipments() + letters.delete_old_single_letters() def create_database(*_): diff --git a/src/OpenPostbud/database/api_users.py b/src/OpenPostbud/database/api_users.py index 0908383..14f2390 100644 --- a/src/OpenPostbud/database/api_users.py +++ b/src/OpenPostbud/database/api_users.py @@ -41,6 +41,12 @@ def get_api_users() -> tuple[ApiUser]: return tuple(result) +def get_api_user(id: str) -> ApiUser | None: + """Get a single api user from the database.""" + with connection.get_session() as session: + return session.get(ApiUser, id) + + def create_api_user(name: str) -> str: """Add a new api user to the database with the given name. A random api key is generated for the new user. @@ -78,6 +84,15 @@ def delete_api_user(user_id: str): return False +def deactivate_api_user(user_id: str): + """Deactivate the api user with the given id.""" + with connection.get_session() as session: + user = session.get(ApiUser, user_id) + if user: + user.active = False + session.commit() + + def verify_api_key(api_key: str) -> ApiUser | None: """Verify an api key against the database. @@ -89,7 +104,7 @@ def verify_api_key(api_key: str) -> ApiUser | None: """ # The api key is assumed to be of the form "id.key" if not re.fullmatch(r"[\w-]+\.[\w-]+", api_key): - return False + return None id, key = api_key.split(".") diff --git a/src/OpenPostbud/database/digital_post/letters.py b/src/OpenPostbud/database/digital_post/letters.py index b5f74f7..d693482 100644 --- a/src/OpenPostbud/database/digital_post/letters.py +++ b/src/OpenPostbud/database/digital_post/letters.py @@ -2,20 +2,23 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta import json from enum import Enum +import logging import re from sqlalchemy import ForeignKey, insert, select, String, update from sqlalchemy.orm import Mapped, mapped_column +from OpenPostbud import config from OpenPostbud.database.base import Base from OpenPostbud.database import connection from OpenPostbud.database.data_types.encrypted_string import EncryptedString from OpenPostbud.database.data_types.id_generator import create_id from OpenPostbud.database.common import ShipmentStatus, PostType from OpenPostbud.database.digital_post import templates +from OpenPostbud.database.digital_post import shipments from OpenPostbud.database import document_storage from OpenPostbud.utils import docx_util @@ -51,18 +54,41 @@ def is_mandatory_for(self, post_type: PostType) -> bool: return self.mandatory_digital or self.mandatory_physical +create_letter_id = create_id("L-", 10) + +# Maximum size of a payload before base 64 encoding accepted by the receiving APIs. +DIGITAL_MAX_PAYLOAD_BYTES = 74 * 1024 * 1024 +PHYSICAL_MAX_PAYLOAD_BYTES = 7 * 1024 * 1024 + +# The magic bytes at the start of every pdf file. +PDF_MAGIC_BYTES = b"%PDF-" + + +def max_payload_bytes(post_type: PostType) -> int: + """The maximum payload size allowed for the given post type. + + AUTO uses the smaller of the two limits, since the letter must be + sendable whichever route the recipient takes. + """ + if post_type == PostType.DIGITAL: + return DIGITAL_MAX_PAYLOAD_BYTES + return PHYSICAL_MAX_PAYLOAD_BYTES + + class Letter(Base): """An ORM class representing a letter.""" __tablename__ = "Letters" - id: Mapped[str] = mapped_column(String(12), primary_key=True, default=create_id("L-", 10)) - shipment_id: Mapped[str] = mapped_column(ForeignKey("Shipments.id", ondelete="CASCADE")) + id: Mapped[str] = mapped_column(String(12), primary_key=True, default=create_letter_id) + shipment_id: Mapped[str] = mapped_column(ForeignKey("Shipments.id", ondelete="CASCADE"), nullable=True) recipient_id: Mapped[str] = mapped_column(EncryptedString()) updated_at: Mapped[datetime] = mapped_column(default=datetime.now) status: Mapped[ShipmentStatus] = mapped_column(default=ShipmentStatus.WAITING) message: Mapped[str] = mapped_column(String(100), nullable=True) field_data: Mapped[str] = mapped_column(EncryptedString()) transaction_id: Mapped[str] = mapped_column(nullable=True) + post_type: Mapped[PostType] = mapped_column(default=PostType.DIGITAL) + created_by: Mapped[str] = mapped_column(String(50)) sent_as: Mapped[PostType] = mapped_column(nullable=True) def to_row_dict(self) -> dict[str, str]: @@ -76,16 +102,32 @@ def to_row_dict(self) -> dict[str, str]: "sent_as": self.sent_as.value if self.sent_as else "" } - def merge_letter(self) -> bytes: - """Merge the letter's merge field data with its template - and convert to pdf. + def get_document(self) -> bytes: + """Get the letter's final document as a pdf. + + A single letter has its document supplied when it's created, so the + document is read from the document storage as is. + + A letter belonging to a shipment is merged from the shipment's + template and the letter's merge field data. The result is cached in + the document storage so the merge only happens once. Returns: - The merged pdf letter as bytes. + The letter's pdf as bytes. + + Raises: + FileNotFoundError: If a single letter's document is missing from + the document storage. """ - stored_file = document_storage.get_letter_doc(self.shipment_id, self.id) - if stored_file: - return stored_file + if self.shipment_id is None: + document = document_storage.get_single_letter_doc(self.id) + if document is None: + raise FileNotFoundError(f"The document of single letter {self.id} is missing from the document storage.") + return document + + cached_file = document_storage.get_letter_doc(self.shipment_id, self.id) + if cached_file: + return cached_file template = templates.get_template_by_shipment(self.shipment_id) @@ -131,20 +173,31 @@ def add_letters(shipment_id: str, csv_data: list[dict[str, str]]): """Add multiple new letters to the database based on a csv file containing letter merge data. + The letters inherit their post type and creator from the shipment. + Args: shipment_id: The id of the shipment the letters belong to. csv_data: A list of dictionaries containing merge data. + + Raises: + ValueError: If no shipment with the given id exists. """ + shipment = shipments.get_shipment(shipment_id) + if not shipment: + raise ValueError("No shipment found with the given id.") + letter_dicts = [] for line in csv_data: - recipient = line[MemoFields.MEMO_MODTAGER.key] - del line[MemoFields.MEMO_MODTAGER.key] + field_data = dict(line) + recipient = field_data.pop(MemoFields.MEMO_MODTAGER.key) letter_dicts.append( { "shipment_id": shipment_id, "recipient_id": recipient, - "field_data": json.dumps(line) + "field_data": json.dumps(field_data), + "post_type": shipment.post_type, + "created_by": shipment.created_by } ) @@ -153,6 +206,47 @@ def add_letters(shipment_id: str, csv_data: list[dict[str, str]]): session.commit() +def add_single_letter(recipient_id: str, memo_label: str | None, post_type: PostType, created_by: str, document: bytes) -> str: + """Add a single letter without a shipment to the database. + + The letter's document is supplied instead of being merged from a + template. It's written to the document storage before the letter row is + committed, so the shipment worker never picks up a letter whose document + is missing. If the row can't be committed the document is deleted again + to avoid orphan files. + + Args: + recipient_id: The cpr or cvr number of the recipient. + memo_label: The label shown to the recipient in Digital Post. + post_type: How the letter should be sent. + created_by: The id of the api user creating the letter. + document: The letter's document as a pdf. + + Returns: + The id of the new letter. + """ + letter = Letter( + id=create_letter_id(), + shipment_id=None, + recipient_id=recipient_id, + field_data=json.dumps({MemoFields.MEMO_LABEL.key: memo_label}), + post_type=post_type, + created_by=created_by + ) + + document_storage.save_single_letter_doc(letter.id, document) + + try: + with connection.get_session() as session: + session.add(letter) + session.commit() + except Exception: + document_storage.delete_single_letter_doc(letter.id) + raise + + return letter.id + + def get_letters(shipment_id: str) -> tuple[Letter]: """Get all letters belonging to a shipment.""" with connection.get_session() as session: @@ -161,6 +255,14 @@ def get_letters(shipment_id: str) -> tuple[Letter]: return tuple(result) +def get_letters_by_user(user_id: str) -> tuple[Letter]: + """Get all letters created by the given user.""" + with connection.get_session() as session: + query = select(Letter).where(Letter.created_by == user_id) + result = session.execute(query).scalars() + return tuple(result) + + def abort_letters(shipment_id: str, user: str): """Set all waiting letters in the given shipment to aborted. Also add a message about who aborted. @@ -183,3 +285,23 @@ def abort_letters(shipment_id: str, user: str): ) session.execute(query) session.commit() + + +def delete_old_single_letters(): + """Delete single letters that are older than SHIPMENT_LIFETIME_DAYS.""" + logging.info("Cleaning up old single letters.") + + with connection.get_session() as session: + query = select(Letter).where( + (datetime.today() - timedelta(days=config.SHIPMENT_LIFETIME_DAYS)) > Letter.updated_at, + Letter.shipment_id.is_(None) + ) + letters = list(session.execute(query).scalars()) + + for letter in letters: + document_storage.delete_single_letter_doc(letter.id) + session.delete(letter) + + session.commit() + + logging.info(f"Deleted {len(letters)} old single letters.") diff --git a/src/OpenPostbud/database/digital_post/shipments.py b/src/OpenPostbud/database/digital_post/shipments.py index d70277c..d7944be 100644 --- a/src/OpenPostbud/database/digital_post/shipments.py +++ b/src/OpenPostbud/database/digital_post/shipments.py @@ -41,7 +41,7 @@ def get_deletion_date(self) -> datetime: return self.created_at + timedelta(days=config.SHIPMENT_LIFETIME_DAYS) -def add_shipment(name: str, description: str, created_by: str, template_id: int, post_type: PostType = PostType.DIGITAL) -> int: +def add_shipment(name: str, description: str, created_by: str, template_id: int, post_type: PostType = PostType.DIGITAL) -> str: """Add a new Shipment to the database. Args: diff --git a/src/OpenPostbud/database/document_storage.py b/src/OpenPostbud/database/document_storage.py index b97aa06..4770162 100644 --- a/src/OpenPostbud/database/document_storage.py +++ b/src/OpenPostbud/database/document_storage.py @@ -10,6 +10,7 @@ STORAGE_FOLDER = Path("OpenPostbud_document_storage") SHIPMENTS_FOLDER = STORAGE_FOLDER / "Shipments" +SINGLE_LETTERS_FOLDER = STORAGE_FOLDER / "Single_letters" LETTER_SUFFIX = ".pdf" # Supported file types per the SF1601 documentation @@ -45,6 +46,30 @@ def _get_shipment_folder(shipment_id: str) -> Path: return SHIPMENTS_FOLDER / shipment_id +def _get_letter_path(shipment_id: str, letter_id: str) -> Path: + """Get the path to the doc file of a letter belonging to a shipment.""" + return (_get_shipment_folder(shipment_id) / letter_id).with_suffix(LETTER_SUFFIX) + + +def _get_single_letter_path(letter_id: str) -> Path: + """Get the path to the doc file of a single letter without a shipment.""" + return (SINGLE_LETTERS_FOLDER / letter_id).with_suffix(LETTER_SUFFIX) + + +def _write_doc(doc_path: Path, doc_bytes: bytes): + """Write a document to the given path, creating the folder if needed.""" + doc_path.parent.mkdir(parents=True, exist_ok=True) + doc_path.write_bytes(doc_bytes) + + +def _read_doc(doc_path: Path) -> bytes | None: + """Read the document at the given path if it exists.""" + try: + return doc_path.read_bytes() + except FileNotFoundError: + return None + + def delete_shipment_docs(shipment_id: str): """Delete all stored documents associated with the given shipment.""" folder_path = _get_shipment_folder(shipment_id) @@ -52,31 +77,35 @@ def delete_shipment_docs(shipment_id: str): shutil.rmtree(folder_path) -def _get_letter_path(shipment_id: str, letter_id: str) -> Path: - """Get the path to the letter's doc file.""" - folder_path = _get_shipment_folder(shipment_id) - return (folder_path / letter_id).with_suffix(LETTER_SUFFIX) - - def save_letter_doc(shipment_id: str, letter_id: str, doc_bytes: bytes): - """Save a letter's document to the document storage. + """Cache the merged document of a letter belonging to a shipment. It's assumed the document is a pdf file. """ - letter_path = _get_letter_path(shipment_id, letter_id) - letter_path.parent.mkdir(parents=True, exist_ok=True) - letter_path.write_bytes(doc_bytes) + _write_doc(_get_letter_path(shipment_id, letter_id), doc_bytes) def get_letter_doc(shipment_id: str, letter_id: str) -> bytes | None: - """Get a letter's document from the document storage - if it exists. + """Get the cached merged document of a letter belonging to a shipment, + if it has been merged before. """ - letter_path = _get_letter_path(shipment_id, letter_id) + return _read_doc(_get_letter_path(shipment_id, letter_id)) - try: - return letter_path.read_bytes() - except FileNotFoundError: - return None + +def save_single_letter_doc(letter_id: str, doc_bytes: bytes): + """Save the document of a single letter without a shipment. + It's assumed the document is a pdf file. + """ + _write_doc(_get_single_letter_path(letter_id), doc_bytes) + + +def get_single_letter_doc(letter_id: str) -> bytes | None: + """Get the document of a single letter without a shipment, if it exists.""" + return _read_doc(_get_single_letter_path(letter_id)) + + +def delete_single_letter_doc(letter_id: str): + """Delete the stored document for a single letter without a shipment.""" + _get_single_letter_path(letter_id).unlink(missing_ok=True) def _get_attachments_folder(shipment_id: str) -> Path: diff --git a/src/OpenPostbud/database/migrations/sql/003_letter_post_type_nullable_shipment.sql b/src/OpenPostbud/database/migrations/sql/003_letter_post_type_nullable_shipment.sql new file mode 100644 index 0000000..eca2804 --- /dev/null +++ b/src/OpenPostbud/database/migrations/sql/003_letter_post_type_nullable_shipment.sql @@ -0,0 +1,28 @@ +CREATE TABLE "Letters_new" ( + id VARCHAR(12) NOT NULL, + shipment_id VARCHAR(12), + recipient_id BINARY NOT NULL, + updated_at DATETIME NOT NULL, + status VARCHAR(9) NOT NULL, + message VARCHAR(100), + field_data BINARY NOT NULL, + transaction_id VARCHAR, + sent_as VARCHAR(8), + post_type VARCHAR(8) NOT NULL, + created_by VARCHAR(50) NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY(shipment_id) REFERENCES "Shipments" (id) ON DELETE CASCADE +) + + +INSERT INTO "Letters_new" (id, shipment_id, recipient_id, updated_at, status, message, field_data, transaction_id, sent_as, post_type, created_by) +SELECT id, shipment_id, recipient_id, updated_at, status, message, field_data, transaction_id, sent_as, + COALESCE((SELECT post_type FROM "Shipments" WHERE "Shipments".id = "Letters".shipment_id), 'DIGITAL'), + COALESCE((SELECT created_by FROM "Shipments" WHERE "Shipments".id = "Letters".shipment_id), 'unknown') +FROM "Letters" + + +DROP TABLE "Letters" + + +ALTER TABLE "Letters_new" RENAME TO "Letters" diff --git a/src/OpenPostbud/database/migrations/sql/004_add_indexes.sql b/src/OpenPostbud/database/migrations/sql/004_add_indexes.sql new file mode 100644 index 0000000..5a1af54 --- /dev/null +++ b/src/OpenPostbud/database/migrations/sql/004_add_indexes.sql @@ -0,0 +1,22 @@ +CREATE INDEX letters_shipment_id ON "Letters"(shipment_id) + + +CREATE INDEX letters_transaction_id ON "Letters"(transaction_id) + + +CREATE INDEX letters_status ON "Letters"(status) + + +CREATE INDEX nemsms_messages_shipment_id ON "NemSMS_Messages"(shipment_id) + + +CREATE INDEX nemsms_messages_transaction_id ON "NemSMS_Messages"(transaction_id) + + +CREATE INDEX nemsms_messages_status ON "NemSMS_Messages"(status) + + +CREATE INDEX registration_tasks_job_id ON "RegistrationTasks"(job_id) + + +CREATE INDEX registration_tasks_status ON "RegistrationTasks"(status) \ No newline at end of file diff --git a/src/OpenPostbud/routes/admin/api_users.py b/src/OpenPostbud/routes/admin/api_users.py index a8f7e27..e9f4f1a 100644 --- a/src/OpenPostbud/routes/admin/api_users.py +++ b/src/OpenPostbud/routes/admin/api_users.py @@ -1,10 +1,12 @@ """This module is responsible for the admin page for api users.""" -from nicegui import ui, APIRouter +from nicegui import app, ui, APIRouter from nicegui.events import ClickEventArguments +from fastapi import HTTPException from OpenPostbud import ui_components from OpenPostbud.database import api_users +from OpenPostbud.database.digital_post import letters router = APIRouter() @@ -15,6 +17,15 @@ {'name': "created_at", 'label': "Oprettet", 'field': "created_at"} ] +LETTERS_COLUMNS = [ + {'name': "id", 'label': "ID", 'field': "id"}, + {'name': "recipient", 'label': "Modtager", 'field': "recipient"}, + {'name': "status", 'label': "Status", 'field': "status"}, + {'name': "sent_as", 'label': "Sendt som", 'field': "sent_as"}, + {'name': "updated_at", 'label': "Status Opdateret", 'field': "updated_at"}, + {'name': "Message", 'label': "Besked", 'field': "message"} +] + COLUMN_DEFAULTS = {'align': 'left', 'sortable': True, 'style': 'padding-right: 5rem'} @@ -22,11 +33,11 @@ def api_users_page(): """Show the api users page.""" ui_components.header() - ApiUserPage() + ApiUsersPage() -class ApiUserPage: - """A class representing the api user page.""" +class ApiUsersPage: + """A class representing the api users page.""" def __init__(self): ui.label("Velkommen til Api brugere!").classes("text-4xl") ui.button("Ny api bruger", on_click=self._add_api_user) @@ -42,12 +53,7 @@ def _update_table(self): def _row_click(self, event): """Open a dialog for the clicked api user row.""" - with ui.dialog(value=True) as dialog, ui.card(): - row = event.args[1] - ui.label(f"{row['name']} - {row['id']}").classes("text-xl") - with ui.row(): - ui.button("Slet", on_click=lambda e: self._delete_user(row['id'], dialog)) - ui.button("Luk", on_click=dialog.close) + ui.navigate.to(app.url_path_for("API User Detail", api_user_id=event.args[1]['id'])) async def _add_api_user(self): """Show a popup prompt for a new api user name and create the user with the given name.""" @@ -68,14 +74,67 @@ def copy_button_click(event: ClickEventArguments): ui.button("Kopier", on_click=copy_button_click) ui.button("Luk", on_click=dialog.close).props("flat") - async def _delete_user(self, user_id: str, dialog: ui.dialog): - """Show a confirmation popup and delete the user with the given id.""" - if not await ui_components.question_popup(f"Vil du slette api bruger {user_id}?", "Ja", "Nej"): + +@router.page("/api-users/{api_user_id}", name="API User Detail") +def api_user_detail_page(api_user_id: str): + """Show the api user detail page.""" + ui_components.header() + ApiUserDetailPage(api_user_id) + + +class ApiUserDetailPage: + """A class representing the api user detail page.""" + def __init__(self, api_user_id: str): + self.user = api_users.get_api_user(api_user_id) + if not self.user: + raise HTTPException(404, f"Ingen api bruger med id {api_user_id} fundet.") + + self._show_header() + + with ui.row(): + self.disable_button = ui_components.DisableButton("Deaktiver", on_click=self._deactivate_user) + if not self.user.active: + self.disable_button.disable() + ui.button("Slet", on_click=self._delete_user) + + self._show_letters_table() + + def _show_letters_table(self): + """Show the letters table.""" + letter_rows = [letter.to_row_dict() for letter in letters.get_letters_by_user(self.user.id)] + self.letter_table = ui_components.SearchTable(title="Breve", rows=letter_rows, columns=LETTERS_COLUMNS, column_defaults=COLUMN_DEFAULTS, pagination=50, download_button=True, search_field=True) + ui_components.obscure_id_column(self.letter_table, "recipient") + + async def _deactivate_user(self): + """Show a confirmation popup and deactivate the user with the given id.""" + if not await ui_components.question_popup(f"Vil du deaktivere api bruger {self.user.id}?", "Ja", "Nej"): + return + + api_users.deactivate_api_user(self.user.id) + ui.notify(f"Api bruger deaktiveret: {self.user.id}", type='positive') + self.disable_button.disable() + self._show_header.refresh() + + async def _delete_user(self): + """Show a confirmation popup and delete the user with the given id. + An api user can only be deleted if it doesn't have any letters attached. + """ + if letters.get_letters_by_user(self.user.id): + await ui_components.info_popup("Du kan ikke slette en api bruger med eksisterende breve tilknyttet.") + return + + if not await ui_components.question_popup(f"Vil du slette api bruger {self.user.id}?", "Ja", "Nej"): return - if api_users.delete_api_user(user_id): - ui.notify(f"Api bruger slettet: {user_id}", type='positive') - self._update_table() - dialog.close() - else: - ui.notify(f"Bruger ikke fundet: {user_id}", type='negative') + api_users.delete_api_user(self.user.id) + ui.navigate.to(app.url_path_for("API Users")) + + @ui.refreshable + def _show_header(self): + self.user = api_users.get_api_user(self.user.id) + with ui.row(): + ui.label(f"Api bruger {self.user.name} - {self.user.id}").classes("text-4xl") + if self.user.active: + ui.chip("Aktiv", color="positive", text_color="white").classes("text-lg") + else: + ui.chip("Inaktiv", text_color="white").classes("text-lg") diff --git a/src/OpenPostbud/routes/api/dependencies.py b/src/OpenPostbud/routes/api/dependencies.py new file mode 100644 index 0000000..8255daf --- /dev/null +++ b/src/OpenPostbud/routes/api/dependencies.py @@ -0,0 +1,23 @@ +"""Shared FastAPI dependencies for the api routes.""" + +from fastapi import Depends +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi.exceptions import HTTPException + +import jwt + +from OpenPostbud import config + + +security_scheme = HTTPBearer() + + +def check_bearer_token(credentials: HTTPAuthorizationCredentials = Depends(security_scheme)) -> dict[str, str]: + """Check the validity of an incoming bearer JWT token.""" + try: + payload = jwt.decode(credentials.credentials, config.API_JWT_SECRET, algorithms=["HS256"]) + return payload + except jwt.ExpiredSignatureError: + raise HTTPException(401, "Token expired") # pylint: disable=raise-missing-from + except jwt.InvalidTokenError: + raise HTTPException(401, "Invalid token") # pylint: disable=raise-missing-from diff --git a/src/OpenPostbud/routes/api/letters.py b/src/OpenPostbud/routes/api/letters.py new file mode 100644 index 0000000..5b1fef2 --- /dev/null +++ b/src/OpenPostbud/routes/api/letters.py @@ -0,0 +1,113 @@ +"""This module defines routes for the letters api.""" + +from datetime import datetime +from typing import Annotated +import base64 +import binascii + +from fastapi import APIRouter, Depends, status +from fastapi.exceptions import HTTPException +from pydantic import BaseModel, Field + +from OpenPostbud.database import connection +from OpenPostbud.database.common import PostType +from OpenPostbud.database.digital_post import letters +from OpenPostbud.routes.api.dependencies import check_bearer_token + + +router = APIRouter() + + +class LetterDetail(BaseModel): + """A model representing a letter response.""" + id: str + shipment_id: str | None + recipient_id: str + status: str + status_time: datetime + sent_as: PostType | None + letter_pdf: str | None = Field(description="Base64-encoded file contents.") + + +class SendLetterModel(BaseModel): + """A model representing a single letter to be sent.""" + recipient_id: str + memo_label: str | None + post_type: PostType + letter_document: str = Field(description="Base64-encoded pdf file contents.") + + +class SendLetterResponse(BaseModel): + """A model describing a response from the send_letter endpoint.""" + id: str + + +@router.post("/send_letter", tags=["Letters"]) +def send_letter(letter: SendLetterModel, token: Annotated[dict[str, str], Depends(check_bearer_token)]) -> SendLetterResponse: + """Send a single letter without a shipment.""" + + if letters.MemoFields.MEMO_LABEL.is_mandatory_for(letter.post_type) and letter.memo_label is None: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Memo label should be set on digital letters.") + + if letter.memo_label is not None and not letters.MemoFields.MEMO_LABEL.pattern.fullmatch(letter.memo_label): + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Memo label doesn't match the required pattern: {letters.MemoFields.MEMO_LABEL.pattern.pattern}") + + if not letters.MemoFields.MEMO_MODTAGER.pattern.fullmatch(letter.recipient_id): + raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Recipient doesn't match the required pattern: {letters.MemoFields.MEMO_MODTAGER.pattern.pattern}") + + try: + letter_document = base64.b64decode(letter.letter_document, validate=True) + except binascii.Error as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Letter document isn't valid base64.") from exc + + if not letter_document.startswith(letters.PDF_MAGIC_BYTES): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Letter document isn't a pdf file.") + + # Reject oversized documents here rather than letting the letter fail in + # the shipment worker after it has been accepted. + max_bytes = letters.max_payload_bytes(letter.post_type) + if len(letter_document) > max_bytes: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + f"Letter document exceeds the maximum size of {max_bytes} bytes for post type {letter.post_type.name}." + ) + + letter_id = letters.add_single_letter( + recipient_id=letter.recipient_id, + memo_label=letter.memo_label, + post_type=letter.post_type, + created_by=token["sub"], + document=letter_document + ) + + return SendLetterResponse(id=letter_id) + + +@router.get("/letter/{letter_id}", tags=["Letters"]) +def get_letter(letter_id: str, get_pdf: bool = True) -> LetterDetail: + """Get a letter by id. Merges and returns the final letter as a pdf + in base 64.""" + with connection.get_session() as session: + letter = session.get(letters.Letter, letter_id) + + if not letter: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "No letter exists with the given id") + + if get_pdf: + try: + pdf = letter.get_document() + except FileNotFoundError as exc: + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "The letter's document is no longer available.") from exc + pdf_64 = base64.b64encode(pdf).decode() + else: + pdf_64 = None + + return LetterDetail( + id=letter.id, + shipment_id=letter.shipment_id, + recipient_id=letter.recipient_id, + status=letter.status, + status_time=letter.updated_at, + letter_pdf=pdf_64, + sent_as=letter.sent_as + ) diff --git a/src/OpenPostbud/routes/api/router.py b/src/OpenPostbud/routes/api/router.py index a0bc59f..9032160 100644 --- a/src/OpenPostbud/routes/api/router.py +++ b/src/OpenPostbud/routes/api/router.py @@ -3,32 +3,15 @@ from typing import Annotated from fastapi import APIRouter, Depends, Security -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from fastapi.exceptions import HTTPException -import jwt - -from OpenPostbud.routes.api import shipments -from OpenPostbud import config - - -security_scheme = HTTPBearer() - - -def check_bearer_token(credentials: HTTPAuthorizationCredentials = Depends(security_scheme)) -> dict[str, str]: - """Check the validity of an incoming bearer JWT token.""" - try: - payload = jwt.decode(credentials.credentials, config.API_JWT_SECRET, algorithms=["HS256"]) - return payload - except jwt.ExpiredSignatureError: - raise HTTPException(401, "Token expired") # pylint: disable=raise-missing-from - except jwt.InvalidTokenError: - raise HTTPException(401, "Invalid token") # pylint: disable=raise-missing-from +from OpenPostbud.routes.api import shipments, letters +from OpenPostbud.routes.api.dependencies import check_bearer_token # Router object for all api routes router = APIRouter(prefix="/api", dependencies=[Security(check_bearer_token)]) router.include_router(shipments.router) +router.include_router(letters.router) @router.get("/hello", tags=["Test"], description="Used to test correct connection to the api.") diff --git a/src/OpenPostbud/routes/api/shipments.py b/src/OpenPostbud/routes/api/shipments.py index 7b522c8..6af51be 100644 --- a/src/OpenPostbud/routes/api/shipments.py +++ b/src/OpenPostbud/routes/api/shipments.py @@ -7,7 +7,7 @@ from fastapi.exceptions import HTTPException from pydantic import BaseModel, Field -from OpenPostbud.database import connection, document_storage +from OpenPostbud.database import document_storage from OpenPostbud.database.digital_post import shipments as shipments_db from OpenPostbud.database.digital_post import letters as letters_db @@ -32,15 +32,6 @@ class ShipmentDetail(ShipmentModel): has_attachments: bool -class LetterDetail(BaseModel): - """A pydantic model representing a letter response.""" - id: str - shipment_id: str - recipient_id: str - status: str - letter_pdf: str = Field(description="Base64-encoded file contents.") - - class AttachmentModel(BaseModel): """A pydantic model representing an attachment response.""" file_name: str @@ -96,25 +87,3 @@ def get_attachments(shipment_id: str) -> list[AttachmentModel]: attachments = document_storage.get_attachments(shipment_id) return [AttachmentModel(file_name=a.name, file_data=base64.b64encode(a.data).decode()) for a in attachments] - - -@router.get("/letter/{letter_id}", tags=["Letters"]) -def get_letter(letter_id: str) -> LetterDetail: - """Get a letter by id. Merges and returns the final letter as a pdf - in base 64.""" - with connection.get_session() as session: - letter = session.get(letters_db.Letter, letter_id) - - if not letter: - raise HTTPException(status.HTTP_400_BAD_REQUEST, "No letter exists with the given id") - - pdf = letter.merge_letter() - pdf_64 = base64.b64encode(pdf).decode() - - return LetterDetail( - id=letter.id, - shipment_id=letter.shipment_id, - recipient_id=letter.recipient_id, - status=letter.status, - letter_pdf=pdf_64 - ) diff --git a/src/OpenPostbud/ui_components.py b/src/OpenPostbud/ui_components.py index 7839fc2..7ad22d8 100644 --- a/src/OpenPostbud/ui_components.py +++ b/src/OpenPostbud/ui_components.py @@ -118,6 +118,15 @@ async def text_input_popup(prompt: str, input_label: str) -> str: return await dialog +async def info_popup(information: str): + """Show an awaitable popup that shows information.""" + with ui.dialog(value=True).props('persistent') as dialog, ui.card(): + ui.label(information).classes("text-lg") + ui.button("Luk", on_click=lambda e: dialog.submit("")) + + await dialog + + class DisableButton(ui.button): """An extension of ui.button that turns grey when disabled.""" def _handle_enabled_change(self, enabled: bool) -> None: diff --git a/src/OpenPostbud/workers/nemsms_worker.py b/src/OpenPostbud/workers/nemsms_worker.py index 72c1c98..e8bbe37 100644 --- a/src/OpenPostbud/workers/nemsms_worker.py +++ b/src/OpenPostbud/workers/nemsms_worker.py @@ -16,6 +16,7 @@ from OpenPostbud import config from OpenPostbud.database import connection from OpenPostbud.database.nemsms.nemsms_messages import NemSMSMessage +from OpenPostbud.database.nemsms.nemsms_shipments import NemSMSShipment from OpenPostbud.database.nemsms import nemsms_shipments from OpenPostbud.database.common import ShipmentStatus @@ -38,7 +39,8 @@ def start_process(): send_message(message, kombit_access) except Timeout: message.set_status(ShipmentStatus.WAITING, message="Timeout. Prøver igen.") - logging.error(f"Sending message {message.id} timed out.") + logging.error(f"Sending message {message.id} timed out. Sleeping for {config.SHIPMENT_WORKER_SLEEP_TIME} seconds.") + time.sleep(config.SHIPMENT_WORKER_SLEEP_TIME) except Exception as e: # pylint: disable=broad-exception-caught message.set_status(ShipmentStatus.FAILED, message="Systemfejl: e.__class__.__name__") logging.error(f"Sending message {message.id} failed: {e}") @@ -57,10 +59,12 @@ def get_waiting_message() -> NemSMSMessage | None: with connection.get_session() as session: sub_q = ( select(NemSMSMessage.id) + .join(NemSMSShipment, NemSMSMessage.shipment_id == NemSMSShipment.id) .where( NemSMSMessage.status == ShipmentStatus.WAITING, - datetime.now() - timedelta(seconds=config.SHIPMENT_WORKER_DELAY) > NemSMSMessage.updated_at + datetime.now() - timedelta(seconds=config.SHIPMENT_WORKER_DELAY) > NemSMSShipment.created_at ) + .order_by(NemSMSShipment.created_at, NemSMSMessage.shipment_id, NemSMSMessage.updated_at) .limit(1) .scalar_subquery() ) diff --git a/src/OpenPostbud/workers/shipment_worker.py b/src/OpenPostbud/workers/shipment_worker.py index b241924..98d94de 100644 --- a/src/OpenPostbud/workers/shipment_worker.py +++ b/src/OpenPostbud/workers/shipment_worker.py @@ -10,7 +10,7 @@ import uuid import json -from sqlalchemy import select, update +from sqlalchemy import or_, select, update from python_serviceplatformen.authentication import KombitAccess from python_serviceplatformen import digital_post from python_serviceplatformen.models.message import Message, MessageHeader, MessageBody, MainDocument, Sender, Recipient, File, AdditionalDocument @@ -19,18 +19,12 @@ from OpenPostbud import config from OpenPostbud.database import connection -from OpenPostbud.database.digital_post.letters import Letter, MemoFields -from OpenPostbud.database.digital_post import shipments +from OpenPostbud.database.digital_post.letters import Letter, MemoFields, DIGITAL_MAX_PAYLOAD_BYTES, PHYSICAL_MAX_PAYLOAD_BYTES from OpenPostbud.database.digital_post.shipments import Shipment from OpenPostbud.database.common import ShipmentStatus, PostType from OpenPostbud.database import document_storage -# Maximum size of files before base 64 encoding accepted by the receiving APIs. -DIGITAL_MAX_PAYLOAD_BYTES = 74 * 1024 * 1024 -PHYSICAL_MAX_PAYLOAD_BYTES = 7 * 1024 * 1024 - - def start_process(): """The entry point of the worker process. @@ -49,7 +43,8 @@ def start_process(): send_letter(letter, kombit_access) except Timeout: letter.set_status(ShipmentStatus.WAITING, message="Timeout. Prøver igen.") - logging.error(f"Sending letter {letter.id} timed out.") + logging.error(f"Sending letter {letter.id} timed out. Sleeping for {config.SHIPMENT_WORKER_SLEEP_TIME} seconds.") + time.sleep(config.SHIPMENT_WORKER_SLEEP_TIME) except HTTPError as e: response_body = e.response.text if e.response is not None else "" letter.set_status(ShipmentStatus.FAILED, message=f"Systemfejl: {e.__class__.__name__}") @@ -72,12 +67,15 @@ def get_waiting_letter() -> Letter | None: with connection.get_session() as session: sub_q = ( select(Letter.id) - .join(Shipment, Letter.shipment_id == Shipment.id) + .outerjoin(Shipment, Letter.shipment_id == Shipment.id) .where( Letter.status == ShipmentStatus.WAITING, - datetime.now() - timedelta(seconds=config.SHIPMENT_WORKER_DELAY) > Letter.updated_at + or_( + datetime.now() - timedelta(seconds=config.SHIPMENT_WORKER_DELAY) > Shipment.created_at, + Letter.shipment_id.is_(None) + ) ) - .order_by(Shipment.created_at, Letter.shipment_id) + .order_by(Shipment.created_at.nulls_first(), Letter.shipment_id, Letter.updated_at) .limit(1) .scalar_subquery() ) @@ -101,22 +99,20 @@ def get_waiting_letter() -> Letter | None: def send_letter(letter: Letter, kombit_access: KombitAccess): - """Send a letter according to its shipment's post type. + """Send a letter according to its post type. - Digital Post and Auto shipments first check if the recipient is registered - for Digital Post. Digital shipments fail if the recipient isn't registered, - while Auto shipments fall back to physical mail. Physical shipments are + Digital Post and Auto letters first check if the recipient is registered + for Digital Post. Digital letters fail if the recipient isn't registered, + while Auto letters fall back to physical mail. Physical letters are always sent as physical mail. """ - shipment = shipments.get_shipment(letter.shipment_id) - - if shipment.post_type == PostType.PHYSICAL: + if letter.post_type == PostType.PHYSICAL: send_physical(letter, kombit_access) return is_registered = digital_post.is_registered(letter.recipient_id, 'digitalpost', kombit_access) - if shipment.post_type == PostType.DIGITAL: + if letter.post_type == PostType.DIGITAL: if not is_registered: letter.set_status(ShipmentStatus.FAILED, message="Ikke tilmeldt Digital Post") logging.info(f"Letter not sent. The recipient is not registered for Digital Post. {letter.id}") @@ -131,7 +127,7 @@ def send_letter(letter: Letter, kombit_access: KombitAccess): def send_digital(letter: Letter, kombit_access: KombitAccess): """Send a letter as Digital Post.""" - document = letter.merge_letter() + document = letter.get_document() b64_doc = base64.b64encode(document).decode() label = json.loads(letter.field_data)[MemoFields.MEMO_LABEL.key] @@ -140,7 +136,7 @@ def send_digital(letter: Letter, kombit_access: KombitAccess): id_type = "CPR" if len(letter.recipient_id) == 10 else "CVR" - attachments = _get_attachments(letter.shipment_id) + attachments = _get_attachments(letter.shipment_id) if letter.shipment_id else [] payload_size = len(document) + sum(len(attachment.data) for attachment in attachments) if payload_size > DIGITAL_MAX_PAYLOAD_BYTES: @@ -207,7 +203,7 @@ def send_physical(letter: Letter, kombit_access: KombitAccess): The recipient address must be present in the letter itself so it shows through the window of the envelope. The recipient id is therefore not sent. """ - document = letter.merge_letter() + document = letter.get_document() if len(document) > PHYSICAL_MAX_PAYLOAD_BYTES: letter.set_status(ShipmentStatus.FAILED, message=f"Fjernpost størrelse oversteg {PHYSICAL_MAX_PAYLOAD_BYTES/(1024*1024)}MB") diff --git a/uv.lock b/uv.lock index 9b6a38b..5fc68ee 100644 --- a/uv.lock +++ b/uv.lock @@ -1028,7 +1028,7 @@ wheels = [ [[package]] name = "openpostbud" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "cryptography" },