Feature/single letters

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- API endpoint for sending single letters without a shipment.
+- Admin page for viewing api letters and deactivating api users.
+- Added get_pdf param to get_letter api endpoint to allow not downloading letter files.
+
+### Changed
+
+- Added created_by and post_type to letters db table to support single letters.
+- Made shipment_id nullable in letters db table.
+- Api users can not be deleted if any letters are attached to them.
+- Added indexes to the letters, nemsms messages and registration tasks db tables.
+
 ## [0.3.0]
 
 ### Added

src/OpenPostbud/__main__.py

@@ -3,7 +3,7 @@
 import argparse
 
 from OpenPostbud.database.check_registration import registration_job
-from OpenPostbud.database.digital_post import shipments
+from OpenPostbud.database.digital_post import shipments, letters
 from OpenPostbud.database.nemsms import nemsms_shipments
 from OpenPostbud.middleware import authentication
 from OpenPostbud.database import connection
@@ -20,6 +20,7 @@ def database_cleanup(*_):
     shipments.delete_old_shipments()
     registration_job.delete_old_registration_jobs()
     nemsms_shipments.delete_old_shipments()
+    letters.delete_old_single_letters()
 
 
 def create_database(*_):

src/OpenPostbud/database/api_users.py

@@ -41,6 +41,12 @@ def get_api_users() -> tuple[ApiUser]:
         return tuple(result)
 
 
+def get_api_user(id: str) -> ApiUser | None:
+    """Get a single api user from the database."""
+    with connection.get_session() as session:
+        return session.get(ApiUser, id)
+
+
 def create_api_user(name: str) -> str:
     """Add a new api user to the database with the given name.
     A random api key is generated for the new user.
@@ -78,6 +84,15 @@ def delete_api_user(user_id: str):
         return False
 
 
+def deactivate_api_user(user_id: str):
+    """Deactivate the api user with the given id."""
+    with connection.get_session() as session:
+        user = session.get(ApiUser, user_id)
+        if user:
+            user.active = False
+            session.commit()
+
+
 def verify_api_key(api_key: str) -> ApiUser | None:
     """Verify an api key against the database.
 
@@ -89,7 +104,7 @@ def verify_api_key(api_key: str) -> ApiUser | None:
     """
     # The api key is assumed to be of the form "id.key"
     if not re.fullmatch(r"[\w-]+\.[\w-]+", api_key):
-        return False
+        return None
 
     id, key = api_key.split(".")
 

src/OpenPostbud/database/digital_post/letters.py

@@ -2,20 +2,23 @@
 
 from __future__ import annotations
 
-from datetime import datetime
+from datetime import datetime, timedelta
 import json
 from enum import Enum
+import logging
 import re
 
 from sqlalchemy import ForeignKey, insert, select, String, update
 from sqlalchemy.orm import Mapped, mapped_column
 
+from OpenPostbud import config
 from OpenPostbud.database.base import Base
 from OpenPostbud.database import connection
 from OpenPostbud.database.data_types.encrypted_string import EncryptedString
 from OpenPostbud.database.data_types.id_generator import create_id
 from OpenPostbud.database.common import ShipmentStatus, PostType
 from OpenPostbud.database.digital_post import templates
+from OpenPostbud.database.digital_post import shipments
 from OpenPostbud.database import document_storage
 from OpenPostbud.utils import docx_util
 
@@ -51,18 +54,41 @@ def is_mandatory_for(self, post_type: PostType) -> bool:
         return self.mandatory_digital or self.mandatory_physical
 
 
+create_letter_id = create_id("L-", 10)
+
+# Maximum size of a payload before base 64 encoding accepted by the receiving APIs.
+DIGITAL_MAX_PAYLOAD_BYTES = 74 * 1024 * 1024
+PHYSICAL_MAX_PAYLOAD_BYTES = 7 * 1024 * 1024
+
+# The magic bytes at the start of every pdf file.
+PDF_MAGIC_BYTES = b"%PDF-"
+
+
+def max_payload_bytes(post_type: PostType) -> int:
+    """The maximum payload size allowed for the given post type.
+
+    AUTO uses the smaller of the two limits, since the letter must be
+    sendable whichever route the recipient takes.
+    """
+    if post_type == PostType.DIGITAL:
+        return DIGITAL_MAX_PAYLOAD_BYTES
+    return PHYSICAL_MAX_PAYLOAD_BYTES
+
+
 class Letter(Base):
     """An ORM class representing a letter."""
     __tablename__ = "Letters"
 
-    id: Mapped[str] = mapped_column(String(12), primary_key=True, default=create_id("L-", 10))
-    shipment_id: Mapped[str] = mapped_column(ForeignKey("Shipments.id", ondelete="CASCADE"))
+    id: Mapped[str] = mapped_column(String(12), primary_key=True, default=create_letter_id)
+    shipment_id: Mapped[str] = mapped_column(ForeignKey("Shipments.id", ondelete="CASCADE"), nullable=True)
     recipient_id: Mapped[str] = mapped_column(EncryptedString())
     updated_at: Mapped[datetime] = mapped_column(default=datetime.now)
     status: Mapped[ShipmentStatus] = mapped_column(default=ShipmentStatus.WAITING)
     message: Mapped[str] = mapped_column(String(100), nullable=True)
     field_data: Mapped[str] = mapped_column(EncryptedString())
     transaction_id: Mapped[str] = mapped_column(nullable=True)
+    post_type: Mapped[PostType] = mapped_column(default=PostType.DIGITAL)
+    created_by: Mapped[str] = mapped_column(String(50))
     sent_as: Mapped[PostType] = mapped_column(nullable=True)
 
     def to_row_dict(self) -> dict[str, str]:
@@ -76,16 +102,32 @@ def to_row_dict(self) -> dict[str, str]:
             "sent_as": self.sent_as.value if self.sent_as else ""
         }
 
-    def merge_letter(self) -> bytes:
-        """Merge the letter's merge field data with its template
-        and convert to pdf.
+    def get_document(self) -> bytes:
+        """Get the letter's final document as a pdf.
+
+        A single letter has its document supplied when it's created, so the
+        document is read from the document storage as is.
+
+        A letter belonging to a shipment is merged from the shipment's
+        template and the letter's merge field data. The result is cached in
+        the document storage so the merge only happens once.
 
         Returns:
-            The merged pdf letter as bytes.
+            The letter's pdf as bytes.
+
+        Raises:
+            FileNotFoundError: If a single letter's document is missing from
+                the document storage.
         """
-        stored_file = document_storage.get_letter_doc(self.shipment_id, self.id)
-        if stored_file:
-            return stored_file
+        if self.shipment_id is None:
+            document = document_storage.get_single_letter_doc(self.id)
+            if document is None:
+                raise FileNotFoundError(f"The document of single letter {self.id} is missing from the document storage.")
+            return document
+
+        cached_file = document_storage.get_letter_doc(self.shipment_id, self.id)
+        if cached_file:
+            return cached_file
 
         template = templates.get_template_by_shipment(self.shipment_id)
 
@@ -131,20 +173,31 @@ def add_letters(shipment_id: str, csv_data: list[dict[str, str]]):
     """Add multiple new letters to the database based
     on a csv file containing letter merge data.
 
+    The letters inherit their post type and creator from the shipment.
+
     Args:
         shipment_id: The id of the shipment the letters belong to.
         csv_data: A list of dictionaries containing merge data.
+
+    Raises:
+        ValueError: If no shipment with the given id exists.
     """
+    shipment = shipments.get_shipment(shipment_id)
+    if not shipment:
+        raise ValueError("No shipment found with the given id.")
+
     letter_dicts = []
 
     for line in csv_data:
-        recipient = line[MemoFields.MEMO_MODTAGER.key]
-        del line[MemoFields.MEMO_MODTAGER.key]
+        field_data = dict(line)
+        recipient = field_data.pop(MemoFields.MEMO_MODTAGER.key)
         letter_dicts.append(
             {
                 "shipment_id": shipment_id,
                 "recipient_id": recipient,
-                "field_data": json.dumps(line)
+                "field_data": json.dumps(field_data),
+                "post_type": shipment.post_type,
+                "created_by": shipment.created_by
             }
         )
 
@@ -153,6 +206,47 @@ def add_letters(shipment_id: str, csv_data: list[dict[str, str]]):
         session.commit()
 
 
+def add_single_letter(recipient_id: str, memo_label: str | None, post_type: PostType, created_by: str, document: bytes) -> str:
+    """Add a single letter without a shipment to the database.
+
+    The letter's document is supplied instead of being merged from a
+    template. It's written to the document storage before the letter row is
+    committed, so the shipment worker never picks up a letter whose document
+    is missing. If the row can't be committed the document is deleted again
+    to avoid orphan files.
+
+    Args:
+        recipient_id: The cpr or cvr number of the recipient.
+        memo_label: The label shown to the recipient in Digital Post.
+        post_type: How the letter should be sent.
+        created_by: The id of the api user creating the letter.
+        document: The letter's document as a pdf.
+
+    Returns:
+        The id of the new letter.
+    """
+    letter = Letter(
+        id=create_letter_id(),
+        shipment_id=None,
+        recipient_id=recipient_id,
+        field_data=json.dumps({MemoFields.MEMO_LABEL.key: memo_label}),
+        post_type=post_type,
+        created_by=created_by
+    )
+
+    document_storage.save_single_letter_doc(letter.id, document)
+
+    try:
+        with connection.get_session() as session:
+            session.add(letter)
+            session.commit()
+    except Exception:
+        document_storage.delete_single_letter_doc(letter.id)
+        raise
+
+    return letter.id
+
+
 def get_letters(shipment_id: str) -> tuple[Letter]:
     """Get all letters belonging to a shipment."""
     with connection.get_session() as session:
@@ -161,6 +255,14 @@ def get_letters(shipment_id: str) -> tuple[Letter]:
         return tuple(result)
 
 
+def get_letters_by_user(user_id: str) -> tuple[Letter]:
+    """Get all letters created by the given user."""
+    with connection.get_session() as session:
+        query = select(Letter).where(Letter.created_by == user_id)
+        result = session.execute(query).scalars()
+        return tuple(result)
+
+
 def abort_letters(shipment_id: str, user: str):
     """Set all waiting letters in the given shipment to
     aborted. Also add a message about who aborted.
@@ -183,3 +285,23 @@ def abort_letters(shipment_id: str, user: str):
         )
         session.execute(query)
         session.commit()
+
+
+def delete_old_single_letters():
+    """Delete single letters that are older than SHIPMENT_LIFETIME_DAYS."""
+    logging.info("Cleaning up old single letters.")
+
+    with connection.get_session() as session:
+        query = select(Letter).where(
+            (datetime.today() - timedelta(days=config.SHIPMENT_LIFETIME_DAYS)) > Letter.updated_at,
+            Letter.shipment_id.is_(None)
+        )
+        letters = list(session.execute(query).scalars())
+
+        for letter in letters:
+            document_storage.delete_single_letter_doc(letter.id)
+            session.delete(letter)
+
+        session.commit()
+
+    logging.info(f"Deleted {len(letters)} old single letters.")

src/OpenPostbud/database/digital_post/shipments.py

@@ -41,7 +41,7 @@ def get_deletion_date(self) -> datetime:
         return self.created_at + timedelta(days=config.SHIPMENT_LIFETIME_DAYS)
 
 
-def add_shipment(name: str, description: str, created_by: str, template_id: int, post_type: PostType = PostType.DIGITAL) -> int:
+def add_shipment(name: str, description: str, created_by: str, template_id: int, post_type: PostType = PostType.DIGITAL) -> str:
     """Add a new Shipment to the database.
 
     Args: