Deprecation of old logs/warnings

All logs and warnings are now handled by the logger

Author: jacob-a-brown

backend/config.py

@@ -18,9 +18,10 @@
 import time
 from datetime import datetime, timedelta
 
-import click
 import shapely.wkt
 
+from backend.logging import Loggable
+
 from .bounding_polygons import get_county_polygon
 from .connectors.nmbgmr.source import (
     NMBGMRSiteSource,
@@ -91,7 +92,7 @@ def get_source(source):
     return None
 
 
-class Config(object):
+class Config(Loggable):
     site_limit: int = 0
     dry: bool = False
 
@@ -135,10 +136,10 @@ class Config(object):
     use_csv: bool = True
     use_geojson: bool = False
 
-    logs: list = []
-    warnings: list = []
-
     def __init__(self, model=None, payload=None):
+        # need to initialize logger
+        super().__init__()
+
         self.bbox = {}
         if model:
             if model.wkt:
@@ -316,32 +317,22 @@ def now_ms(self, days=0):
         # return current time in milliseconds
         return int((datetime.now() - td).timestamp() * 1000)
 
-    def report(self, log_report: bool = False):
+    def report(self):
         def _report_attributes(title, attrs):
             s = f"---- {title} --------------------------------------------------"
-            click.secho(s, fg="yellow")
-            if log_report:
-                self.logs.append(s)
+            self.log(s)
 
             for k in attrs:
                 v = getattr(self, k)
                 s = f"{k}: {v}"
-                click.secho(s, fg="yellow")
-
-                if log_report:
-                    self.logs.append(s)
+                self.log(s)
 
             s = ""
-            click.secho(s, fg="yellow")
+            self.log(s)
 
-            if log_report:
-                self.logs.append(s)
 
         s = "---- Begin configuration -------------------------------------\n"
-        click.secho(s, fg="yellow")
-
-        if log_report:
-            self.logs.append(s)
+        self.log(s)
 
         sources = [f"use_source_{s}" for s in SOURCE_KEYS]
         attrs = [
@@ -373,26 +364,23 @@ def _report_attributes(title, attrs):
         )
 
         s = "---- End configuration -------------------------------------\n"
-        click.secho(s, fg="yellow")
-
-        if log_report:
-            self.logs.append(s)
+        self.log(s)
 
     def validate(self):
         if not self._validate_bbox():
-            click.secho("Invalid bounding box", fg="red")
+            self.warn("Invalid bounding box")
             sys.exit(2)
 
         if not self._validate_county():
-            click.secho("Invalid county", fg="red")
+            self.warn("Invalid county")
             sys.exit(2)
 
         if not self._validate_date(self.start_date):
-            click.secho(f"Invalid start date {self.start_date}", fg="red")
+            self.warn(f"Invalid start date {self.start_date}")
             sys.exit(2)
 
         if not self._validate_date(self.end_date):
-            click.secho("Invalid end date", fg="red")
+            self.warn(f"Invalid end date {self.end_date}")
             sys.exit(2)
 
     def _extract_date(self, d):

backend/persister.py

@@ -18,12 +18,10 @@
 import os
 import shutil
 
-import click
 import pandas as pd
 import geopandas as gpd
 
 from backend.logging import Loggable
-from backend.record import SiteRecord
 
 try:
     from google.cloud import storage
@@ -40,8 +38,6 @@ def __init__(self):
         self.combined = []
         self.timeseries = []
         self.sites = []
-        self.logs = []
-        self.warnings = []
 
         super().__init__()
         # self.keys = record_klass.keys
@@ -98,24 +94,6 @@ def dump_sites(self, path: str):
         else:
             self.log("no sites to dump", fg="red")
 
-    def dump_logs(self, path: str):
-        if self.logs:
-            self.log(f"dumping logs to {os.path.abspath(path)}")
-            with open(path, "w") as f:
-                for l in self.logs:
-                    f.write(f"{l}\n")
-        else:
-            self.log("no logs to dump")
-
-    def dump_warnings(self, path: str):
-        if self.warnings:
-            self.log(f"dumping warnings to {os.path.abspath(path)}")
-            with open(path, "w") as f:
-                for w in self.warnings:
-                    f.write(f"{w}\n")
-        else:
-            self.log("no warnings to dump")
-
     def save(self, path: str):
         if self.records:
             path = self.add_extension(path)

backend/unifier.py

@@ -41,7 +41,7 @@ def health_check(source: BaseSiteSource) -> bool:
 
 
 def unify_sites(config):
-    print("Unifying sites")
+    print("Unifying sites\n")
 
     # def func(config, persister):
     #     for source in config.site_sources():
@@ -52,8 +52,8 @@ def unify_sites(config):
 
 
 def unify_analytes(config):
-    print("Unifying analytes")
-    config.report(log_report=True)
+    print("Unifying analytes\n")
+    # config.report() -- report is done in cli.py, no need to do it twice
     config.validate()
 
     if not config.dry:
@@ -63,9 +63,9 @@ def unify_analytes(config):
 
 
 def unify_waterlevels(config):
-    print("Unifying waterlevels")
+    print("Unifying waterlevels\n")
 
-    config.report(log_report=True)
+    # config.report() -- report is done in cli.py, no need to do it twice
     config.validate()
 
     if not config.dry:
@@ -182,14 +182,9 @@ def _site_wrapper(site_source, parameter_source, persister, config):
         import traceback
 
         exc = traceback.format_exc()
-        print(exc)
-        print(f"Failed to unify {site_source}")
+        config.warn(exc)
+        config.warn(f"Failed to unify {site_source}")
 
-        config.logs.append(exc)
-        config.logs.append(f"Failed to unify {site_source}")
-
-        config.warnings.append(exc)
-        config.warnings.append(f"Failed to unify {site_source}")
 
 
 def _unify_parameter(
@@ -209,10 +204,6 @@ def _unify_parameter(
         persister.dump_combined(f"{config.output_path}.combined")
         persister.dump_timeseries(f"{config.output_path}_timeseries")
 
-    persister.logs.extend(config.logs)
-    persister.warnings.extend(config.warnings)
-    persister.dump_logs(f"{config.output_path}.logs.txt")
-    persister.dump_warnings(f"{config.output_path}.warnings.txt")
     persister.finalize(config.output_name)