From 74eb581bfe1a894a0e4f911eb4db7132d6ead08d Mon Sep 17 00:00:00 2001 From: "david.noel@withpigment.com" Date: Sat, 11 Jul 2026 18:25:26 +0200 Subject: [PATCH 1/2] Add Amendement model and preserve analysis tables on rebuild Introduce the raw `amendements` table (fields sourced directly from the Tricoteuses /amendements endpoint, grouped into identity, text, author, attachment, status and dates). Column names mirror the JSON keys so the existing schema-driven ETL loads them without changes. Scope the rebuild to ETL-managed tables (ETL_TABLES) so future analysis tables are neither dropped by `create_db` nor treated as source files by the ETL loop. This lets analysis results survive a re-download without introducing Alembic. Add `.env.example` wiring the ETL to the local docker-compose Postgres. --- .env.example | 11 +++++++++ etl/database.py | 26 ++++++++++++++++--- models/__init__.py | 1 + models/amendement.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 .env.example create mode 100644 models/amendement.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f731d9c --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# Connexion PostgreSQL utilisée par l'ETL (etl/database.py). +# Ces valeurs correspondent à l'instance locale définie dans docker-compose.yml. +# Copier ce fichier en `.env` : `cp .env.example .env` +PG_USER=postgres +PG_PWD=postgres +PG_DB=ipolitics +PG_HOST=localhost +PG_PORT=5432 + +# Mettre à True pour logguer toutes les requêtes SQL émises par SQLAlchemy. +PG_ECHO=False diff --git a/etl/database.py b/etl/database.py index aa7bc90..bd13010 100644 --- a/etl/database.py +++ b/etl/database.py @@ -5,6 +5,13 @@ import models # noqa: F401 # pyright: ignore[reportUnusedImport] # registers all ORM models with Base.metadata from models.base import Base +# Tables alimentées par l'ETL depuis les fichiers JSON de ./data. +# Seules ces tables sont détruites lors d'un rebuild et parcourues par l'ETL. +# Les tables d'analyse (ajoutées plus tard) en sont volontairement exclues afin +# que leurs résultats survivent à un rebuild et ne soient pas traitées comme des +# fichiers source à charger. +ETL_TABLES = {"dossiers", "amendements"} + def _get_db_url(): PG_USER = getenv("PG_USER") @@ -29,16 +36,27 @@ def get_engine(): return create_engine(pg_url, poolclass=pool.NullPool, echo=bool(PG_ECHO)) +def _get_etl_tables(): + """Return the schema definitions of the ETL-managed tables only.""" + return [table for table in Base.metadata.sorted_tables if table.name in ETL_TABLES] + + def create_db(): - """Drop the current DB and recreate from the schema.""" + """Rebuild the ETL-managed tables from the schema. + + Only the tables listed in ETL_TABLES are dropped and recreated. Analysis + tables are left untouched so their results survive a rebuild; create_all is + idempotent and (re)creates any missing table without altering existing ones. + """ print("Creating DB") engine = get_engine() - print(Base.metadata.tables) - Base.metadata.drop_all(engine) + etl_tables = _get_etl_tables() + print(etl_tables) + Base.metadata.drop_all(engine, tables=etl_tables) Base.metadata.create_all(engine) print("Db was created") return Base.metadata.tables def get_tables_definition(): - return Base.metadata.sorted_tables + return _get_etl_tables() diff --git a/models/__init__.py b/models/__init__.py index ddca4b6..d82f96b 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1 +1,2 @@ +from models.amendement import Amendement # noqa: F401 from models.dossier import User # noqa: F401 diff --git a/models/amendement.py b/models/amendement.py new file mode 100644 index 0000000..723183d --- /dev/null +++ b/models/amendement.py @@ -0,0 +1,59 @@ +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column + +from models.base import Base + + +class Amendement(Base): + """Amendement brut tel que renvoyé par l'API des tricoteuses (endpoint /amendements). + + Les noms d'attributs sont en camelCase afin de correspondre exactement aux clés + du JSON source : l'ETL (etl/extraction.py) s'appuie sur cette correspondance + 1:1 entre nom de colonne et nom de champ JSON. + + Seul `uid` est non nullable (clé primaire) ; les autres champs restent nullable + car les amendements sont hétérogènes (budgétaires, sous-amendements...) et + n'exposent pas toujours l'ensemble des champs. + """ + + __tablename__ = "amendements" + + # --- Identité --- + uid: Mapped[str] = mapped_column(primary_key=True) + numeroLong: Mapped[str | None] + numeroOrdreDepot: Mapped[int | None] + legislature: Mapped[int | None] + chambre: Mapped[str | None] + dataset: Mapped[int | None] + + # --- Texte (contenu analysé) --- + exposeSommaire: Mapped[str | None] = mapped_column(Text) + dispositif: Mapped[str | None] = mapped_column(Text) + + # --- Auteur / signataires --- + acteurRefUid: Mapped[str | None] + groupePolitiqueRefUid: Mapped[str | None] + organeRefUid: Mapped[str | None] + typeAuteur: Mapped[str | None] + nomRepresentation: Mapped[str | None] + signatairesLibelle: Mapped[str | None] + nombreCoSignataires: Mapped[int | None] + + # --- Rattachement (références molles vers d'autres entités de l'API) --- + dossierRefUid: Mapped[str | None] + documentRefUid: Mapped[str | None] + etapeLegislativeRefUid: Mapped[str | None] + codeEtape: Mapped[str | None] + + # --- Statut / sort --- + sortAmendement: Mapped[str | None] + etatCode: Mapped[str | None] + etatLibelle: Mapped[str | None] + triAmendement: Mapped[str | None] + + # --- Dates (conservées en texte : l'ETL insère les valeurs JSON brutes) --- + dateDepot: Mapped[str | None] + dateSort: Mapped[str | None] + dateMaj: Mapped[str | None] + datePublication: Mapped[str | None] + datePremierAjout: Mapped[str | None] From 85f3026bfb674088f052b47441af264ced35d907 Mon Sep 17 00:00:00 2001 From: "david.noel@withpigment.com" Date: Sat, 11 Jul 2026 19:44:26 +0200 Subject: [PATCH 2/2] Fix PG_ECHO always enabling SQL echo getenv returns a string, so bool(getenv("PG_ECHO", False)) was truthy for any non-empty value including "False", leaving SQL echo permanently on. Parse the flag explicitly instead. --- etl/database.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etl/database.py b/etl/database.py index bd13010..47904c3 100644 --- a/etl/database.py +++ b/etl/database.py @@ -31,9 +31,10 @@ def _get_db_url(): def get_engine(): """Return a configured SQLAlchemy engine""" - PG_ECHO = getenv("PG_ECHO", False) + # getenv renvoie une chaîne : bool("False") vaudrait True, d'où la comparaison explicite. + pg_echo = getenv("PG_ECHO", "").strip().lower() == "true" pg_url = _get_db_url() - return create_engine(pg_url, poolclass=pool.NullPool, echo=bool(PG_ECHO)) + return create_engine(pg_url, poolclass=pool.NullPool, echo=pg_echo) def _get_etl_tables():