Merge pull request #191 from SANDAG/yml-to-toml

[PULL REQUEST] POC transition from yml to toml

Author: KPHSANDAG

.gitattributes

@@ -1 +1 @@
-*.yml diff
+*.toml diff

.gitignore

@@ -7,7 +7,7 @@ __pycache__/
 *.so
 
 # Secrets file
-secrets.yml
+secrets.toml
 
 # Log file
 log.txt

README.md

@@ -1,108 +1,110 @@
 ## Setup
 
-Clone the repository and ensure an installation of [uv](https://docs.astral.sh/uv/getting-started/installation/) exists. Create a local virtual environment by running `uv venv` then `uv sync` in the command line. Ensure that a `secrets.yml` file exists
-
-### Configuration of Private Data in secrets.yml
-In order to avoid exposing certain data to the public this repository uses a secrets file to store sensitive configurations in addition to a standard configuration file. This file is stored in the root directory of the repository as `secrets.yml` and is included in the `.gitignore` intentionally to avoid it ever being committed to the repository.
-
-The `secrets.yml` should mirror the following structure.
-
-```yaml
-sql:
-  estimates:
-    server: <SqlInstanceName>  # SQL instance containing estimates database
-    database: <SqlDatabaseName>  # database within SQL instance containing SQL build objects
-  gis:
-    server: <SqlInstanceName>  # SQL instance containing GIS database
-    database: <SqlDatabaseName>  # database within instance containing GIS datasets (GQ/LUDU)
-  staging: <FolderPath>  # unconditional network folder path visible to SQL instance for BULK INSERT
+Clone the repository and ensure an installation of [uv](https://docs.astral.sh/uv/getting-started/installation/) exists. Create a local virtual environment by running `uv venv` then `uv sync` in the command line. Ensure that a `secrets.toml` file exists
+
+### Configuration of Private Data in secrets.toml
+In order to avoid exposing certain data to the public this repository uses a secrets file to store sensitive configurations in addition to a standard configuration file. This file is stored in the root directory of the repository as `secrets.toml` and is included in the `.gitignore` intentionally to avoid it ever being committed to the repository.
+
+The `secrets.toml` should mirror the following structure.
+
+```toml
+[sql.estimates]
+server = "<SqlInstanceName>"  # SQL instance containing estimates database
+database = "<SqlDatabaseName>"  # database within SQL instance containing SQL build objects
+
+[sql.gis]
+server = "<SqlInstanceName>"  # SQL instance containing GIS database
+database = "<SqlDatabaseName>"  # database within instance containing GIS datasets (GQ/LUDU)
+
+[sql]
+staging = "<FolderPath>"  # unconditional network folder path visible to SQL instance for BULK INSERT
 ```
 
 ## Running
 
-Set the configuration file `config.yml` parameters specific to the run in the project root directory. Finally, simply execute `uv run main.py` in the main project directory
+Set the configuration file `config.toml` parameters specific to the run in the project root directory. Finally, simply execute `uv run main.py` in the main project directory
 
 ### Configuration File Settings     
 
 The default version of the runtime configuration file is copied here, with comments explaining each and every key/value pair
 
-```yaml
+```toml
 # Configuration for what parts of the Estimates Program to run. Since this file may be
 # modified from the default settings, you can always restore to default using the copy
 # stored in README.md. For brevity, detailed comments have been removed from this file
 
 # The 'run' section contains configuration for running every module of the Estimates
 # Program for a specified set of years
-run:
-  
-  # Whether to use the 'run' section. Mutually exclusive with 'debug' mode
-  enabled: False
-  
-  # The MGRA series to use for this run. Currently only 'mgra15' is valid
-  mgra: mgra15
-  
-  # The first year inclusive to start running from
-  start_year: 2020
-  
-  # The last year inclusive to end running with
-  end_year: 2023
-  
-  # The code version
-  version: 0.0.0-dev
-  
-  # Additional notes on this run
-  comments: Example comment
+[run]
+
+# Whether to use the 'run' section. Mutually exclusive with 'debug' mode
+enabled = false
+
+# The MGRA series to use for this run. Currently only 'mgra15' is valid
+mgra = "mgra15"
+
+# The first year inclusive to start running from
+start_year = 2020
+
+# The last year inclusive to end running with
+end_year = 2023
+
+# The code version
+version = "0.0.0-dev"
+
+# Additional notes on this run
+comments = "Example comment"
 
 # The 'debug' section contains configuration for running a subset of modules of the
 # Estimates Program for a given set of years. All parameters must be provided except for
-# 'run_id', 'version', and 'comments'. If 'run_id' is 'null', then a new 'run_id' will 
+# 'run_id', 'version', and 'comments'. If 'run_id' is -1, then a new 'run_id' will 
 # be automatically created, similar to 'run' mode
-debug:
-  
-  # Whether to use the 'debug' section. Mutually exclusive with 'run' mode
-  enabled: False
-  
-  # (Optional) If provided, then most parameters in the 'debug' section will be pulled 
-  # from '[run].[metadata]'. If not provided, then a new 'run_id' will be automatically 
-  # created
-  run_id: null
-  
-  # The first year inclusive and last year inclusive to run. In the case that...
-  # * The value of 'run_id' is 'null', the values will be loaded into [metadata].[run]
-  #   and will be used as is
-  # * The value of 'run_id' is not 'null', the values will be checked against the values
-  #   already in '[run].[metadata]'
-  start_year: 2020
-  end_year: 2023
-  
-  # (Optional) The code version. If provided, then 'run_id' must be 'null'
-  version: 0.0.0-dev
-  
-  # (Optional) Additional notes on this run. If provided, then 'run_id' must be 'null'
-  comments: null
-  
-  # Whether to run the 'startup' module
-  startup: False
-  
-  # Whether to run the 'housing_and_households' module. If enabled, then any above 
-  # modules must all be enabled due to module dependencies
-  housing_and_households: False
-  
-  # Whether to run the 'population' module. If enabled, then any above modules must all
-  # be enabled due to module dependencies
-  population: False
-  
-  # Whether to run the 'population_by_ase' module. If enabled, then any above modules 
-  # must all be enabled due to module dependencies
-  population_by_ase: False
-  
-  # Whether to run the 'household_characteristics' module. If enabled, then any above 
-  # modules must all be enabled due to module dependencies
-  household_characteristics: False
-  
-  # Whether to run the 'staging' module. If enabled, then any above modules must all be
-  # enabled due to module dependencies
-  staging: False
+[debug]
+
+# Whether to use the 'debug' section. Mutually exclusive with 'run' mode
+enabled = false
+
+# (Optional) If provided, then most parameters in the 'debug' section will be pulled 
+# from '[run].[metadata]'. If not provided, then a new 'run_id' will be automatically 
+# created. Use -1 to indicate no run_id (TOML doesn't support null)
+run_id = -1
+
+# The first year inclusive and last year inclusive to run. In the case that...
+# * The value of 'run_id' is -1, the values will be loaded into [metadata].[run]
+#   and will be used as is
+# * The value of 'run_id' is not -1, the values will be checked against the values
+#   already in '[run].[metadata]'
+start_year = 2020
+end_year = 2023
+
+# (Optional) The code version. If provided, then 'run_id' must be -1
+version = "0.0.0-dev"
+
+# (Optional) Additional notes on this run. If provided, then 'run_id' must be -1
+comments = ""
+
+# Whether to run the 'startup' module
+startup = false
+
+# Whether to run the 'housing_and_households' module. If enabled, then any above 
+# modules must all be enabled due to module dependencies
+housing_and_households = false
+
+# Whether to run the 'population' module. If enabled, then any above modules must all
+# be enabled due to module dependencies
+population = false
+
+# Whether to run the 'population_by_ase' module. If enabled, then any above modules 
+# must all be enabled due to module dependencies
+population_by_ase = false
+
+# Whether to run the 'household_characteristics' module. If enabled, then any above 
+# modules must all be enabled due to module dependencies
+household_characteristics = false
+
+# Whether to run the 'staging' module. If enabled, then any above modules must all be
+# enabled due to module dependencies
+staging = false
 ```
 
 ### Production Database Schema

config.yml config.tomlconfig.yml renamed to config.toml

@@ -4,28 +4,28 @@
 
 # The `run` section contains configuration for running every module of the Estimates
 # Program for a specified set of years
-run:
-  enabled: True
-  mgra: mgra15
-  start_year: 2020
-  end_year: 2024
-  version: 1.1.1-dev
-  comments: Example comment
+[run]
+enabled = true
+mgra = "mgra15"
+start_year = 2020
+end_year = 2024
+version = "1.1.1-dev"
+comments = "Example comment"
 
 # The `debug` section contains configuration for running a subset of modules of the
 # Estimates Program for a given set of years. All parameters must be provided except for
 # `run_id` and `comments`. If `run_id` is `null`, then a new `run_id` will be
 # automatically created, similar to `run` mode
-debug:
-  enabled: False
-  run_id: null
-  start_year: 2022
-  end_year: 2023
-  version: 1.1.1-dev
-  comments: null
-  startup: False
-  housing_and_households: False
-  population: False
-  population_by_ase: False
-  household_characteristics: False
-  staging: False
+[debug]
+enabled = false
+run_id = -1 # -1 is interpreted as None
+start_year = 2022
+end_year = 2023
+version = "1.1.1-dev"
+comments = ""
+startup = false
+housing_and_households = false
+population = false
+population_by_ase = false
+household_characteristics = false
+staging = false

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "Estimates-Program"
-version = "0.0.0" # Not used, refer to config.yml instead
+version = "0.0.0" # Not used, refer to config.toml instead
 requires-python = ">=3.11"
 dependencies = [
     "black>=25.12.0,<26.0.0",
@@ -9,6 +9,5 @@ dependencies = [
     "numpy>=2.4.0,<3.0.0",
     "pandas>=2.3.3,<3.0.0",
     "pyodbc>=5.3.0,<6.0.0",
-    "pyyaml>=6.0.3,<7.0.0",
     "sqlalchemy>=2.0.45,<3.0.0",
 ]

python/parsers.py

@@ -22,7 +22,7 @@ class InputParser:
         _end_year (int): Used to store the end_year, whether of a new run or an
             existing run
         run_instructions (dict): Explicit instructions on which modules/years to run.
-            The YAML config file is parsed and this dictionary is filled no matter if
+            The toml config file is parsed and this dictionary is filled no matter if
             run or debug mode is enabled
         run_id (int): The run identifier parsed from the configuration.
         mgra_version (int): The MGRA version we are running on. Depending on run mode,
@@ -60,6 +60,10 @@ def parse_config(self) -> None:
         Returns:
             None
         """
+        # Convert -1 to None for run_id (TOML doesn't support null/None)
+        if self._config.get("debug", {}).get("run_id") == -1:
+            self._config["debug"]["run_id"] = None
+
         self._validate_config()
         self.run_id = self._parse_run_id()
         self.mgra_version = self._parse_mgra_version()

python/utils.py

@@ -1,7 +1,7 @@
 import logging
 import math
 import pathlib
-import yaml
+import tomllib
 
 import numpy as np
 import pandas as pd
@@ -50,12 +50,12 @@
 # SQL CONFIGURATION #
 #####################
 
-# Load secrets YAML file
+# Load secrets TOML file
 try:
-    with open(ROOT_FOLDER / "secrets.yml", "r") as file:
-        _secrets = yaml.safe_load(file)
+    with open(ROOT_FOLDER / "secrets.toml", "rb") as file:
+        _secrets = tomllib.load(file)
 except IOError:
-    raise IOError("secrets.yml does not exist, see README.md")
+    raise IOError("secrets.toml does not exist, see README.md")
 
 # Create SQLAlchemy engine(s)
 ESTIMATES_ENGINE = sql.create_engine(
@@ -89,15 +89,15 @@
 # RUNTIME CONFIGURATION #
 #########################
 
-# Load configuration YAML file
+# Load configuration TOML file
 try:
-    with open(ROOT_FOLDER / "config.yml", "r") as file:
-        config = yaml.safe_load(file)
+    with open(ROOT_FOLDER / "config.toml", "rb") as file:
+        config = tomllib.load(file)
 except IOError:
-    raise IOError("config.yml does not exist, see README.md")
+    raise IOError("config.toml does not exist, see README.md")
 
 # Initialize input parser
-# Parse the configuration YAML file and validate its contents
+# Parse the configuration TOML file and validate its contents
 input_parser = parsers.InputParser(config=config, engine=ESTIMATES_ENGINE)
 input_parser.parse_config()
 

reporting/reporting.py

@@ -8,7 +8,7 @@
 
 # We cannot import python.utils, as just importing will cause a new [run_id]` value and
 # new log file to be created. Instead, copy what we need for now :(
-import yaml
+import tomllib
 import pathlib
 import textwrap
 import sqlalchemy as sql
@@ -17,10 +17,10 @@
 
 ROOT_FOLDER = pathlib.Path(__file__).parent.resolve().parent
 try:
-    with open(ROOT_FOLDER / "secrets.yml", "r") as file:
-        _secrets = yaml.safe_load(file)
+    with open(ROOT_FOLDER / "secrets.toml", "rb") as file:
+        _secrets = tomllib.load(file)
 except IOError:
-    raise IOError("secrets.yml does not exist, see README.md")
+    raise IOError("secrets.toml does not exist, see README.md")
 
 # Create SQLAlchemy engine(s)
 ESTIMATES_ENGINE = sql.create_engine(