From d1b46a140528b773b000303b68653779c00797e1 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Thu, 13 Oct 2022 15:54:05 +0000
Subject: [PATCH 01/38] quite stuck, df.to_sql isn't working

---
 python/geocode.py | 65 +++++++++++++++++++----------------------------
 1 file changed, 26 insertions(+), 39 deletions(-)

diff --git a/python/geocode.py b/python/geocode.py
index ddfce207..bd6d6e62 100644
--- a/python/geocode.py
+++ b/python/geocode.py
@@ -11,6 +11,7 @@
 
 g = Geosupport()
 
+OUTPUT_TABLE_NAME = "_INIT_geo"
 
 def geocode(input):
     # collect inputs
@@ -84,74 +85,60 @@ def parse_output(geo):
     )
 
 
-def load_applications(engine):
+def load_init_devdb(engine):
     df = pd.read_sql(
         """
         SELECT 
             uid, 
-            regexp_replace(
-                trim(house_number), 
-                '(^|)0*', '', ''
-            ) as house_number,
-            REGEXP_REPLACE(street_name, '[\s]{2,}' ,' ' , 'g') as street_name, 
-            borough,
-            source
-        FROM (
-            SELECT 
-                distinct ogc_fid as uid, 
-                housenumber as house_number,
-                streetname as street_name, 
-                borough,
-                'bis' as source
-            FROM dob_jobapplications UNION
-            SELECT 
-                distinct ogc_fid as uid, 
-                house_no as house_number,
-                street_name as street_name, 
-                borough,
-                'now' as source
-            FROM dob_now_applications
-        ) a LIMIT 400000
+            address_numbr as house_number,
+            REGEXP_REPLACE(address_street, '[\s]{2,}' ,' ' , 'g') as street_name, 
+            boro as borough
+        FROM _INIT_devdb LIMIT 10000
         """,
         engine,
     )
-    
+    print('loaded df from database')
     return df
 
-def geocode_insert_sql(df):
-    records = df.to_dict("records")
+def geocode_insert_sql(records, engine):
 
     # Multiprocess
     with Pool(processes=cpu_count()) as pool:
-        it = tqdm(pool.map(geocode, records, 1000))
-    # it = tqdm(list(map(geocode, records)))
-
+        it = pool.map(geocode, records, len(records)//4)
+    # it = list(map(geocode, records))
     df = pd.DataFrame(it)
+
     df.to_sql(
-        "dob_geocode_results",
+        OUTPUT_TABLE_NAME,
         con=engine,
         if_exists="append",
         index=False,
-        method=psql_insert_copy,
     )
 
+    print(f'records in _INIT_geocoded:')
+    print(engine.execute(f"SELECT count(*) from {OUTPUT_TABLE_NAME}").fetchall()[0][0])
+
+
 def clear_dob_geocode_results(engine):
-    engine.execute("DROP TABLE IF EXISTS dob_geocode_results")
+    engine.execute(f"DROP TABLE IF EXISTS {OUTPUT_TABLE_NAME}" )
 
 if __name__ == "__main__":
     # connect to BUILD_ENGINE
     engine = create_engine(os.environ["BUILD_ENGINE"])
 
-    df = load_applications(engine)
     clear_dob_geocode_results(engine)
+
+    df = load_init_devdb(engine)
+    records = df.to_dict("records")
+    del df
     # df = df.iloc[:2000,:]
     start =0
-    chunk_size = 50000
-    end = chunk_size
-    while end <= df.shape[0]:
+    chunk_size = 1000
+    end = min(chunk_size, len(records))
+    while start < len(records):
         print(f"geocoding records {start} through {end}")
-        geocode_insert_sql(df.iloc[start:end,:])
+        geocode_insert_sql(records[start:end], engine)
         start = end 
-        end = min(end+chunk_size, df.shape[0])
+        end = min(end+chunk_size, len(records))
         
 

From fe461884e56a97137272d02ba906b0dac120071b Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Fri, 14 Oct 2022 20:59:59 +0000
Subject: [PATCH 02/38] geocoding during build seems to work. Next implement
 similar process for geocode_hny and check work against final_devDB built with
 the old process

---
 bash/02_build_devdb.sh |  5 ++++-
 python/geocode.py      | 35 +++++++++++++++++++----------------
 sql/_geo.sql           |  9 +++------
 3 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/bash/02_build_devdb.sh b/bash/02_build_devdb.sh
index de535559..87981777 100755
--- a/bash/02_build_devdb.sh
+++ b/bash/02_build_devdb.sh
@@ -9,7 +9,10 @@ psql $BUILD_ENGINE -f sql/now/_init.sql
 psql $BUILD_ENGINE -f sql/_init.sql
 count _INIT_devdb
 
-display "Assign geoms to _GEO_devdb and create GEO_devdb"
+display "Geocoding DOB records"
+poetry run python3 -m python.geocode
+
+# display "Assign geoms to _GEO_devdb and create GEO_devdb"
 psql $BUILD_ENGINE -f sql/_geo.sql
 psql $BUILD_ENGINE -f sql/_geo_corrections.sql
 count GEO_devdb
diff --git a/python/geocode.py b/python/geocode.py
index bd6d6e62..ab464c4f 100644
--- a/python/geocode.py
+++ b/python/geocode.py
@@ -1,17 +1,19 @@
 from multiprocessing import Pool, cpu_count
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, types
 from geosupport import Geosupport, GeosupportError
 from python.utils import psql_insert_copy
 import pandas as pd
 import os
 from tqdm import tqdm
 from dotenv import main
+import json
 
 main.load_dotenv()
 
 g = Geosupport()
 
-OUTPUT_TABLE_NAME = "_INIT_geo"
+OUTPUT_TABLE_NAME = "_init_geocoded"
+
 
 def geocode(input):
     # collect inputs
@@ -90,24 +92,26 @@ def load_init_devdb(engine):
         """
         SELECT 
             uid, 
+            job_number, 
             address_numbr as house_number,
             REGEXP_REPLACE(address_street, '[\s]{2,}' ,' ' , 'g') as street_name, 
             boro as borough
-        FROM _INIT_devdb LIMIT 10000
+        FROM _INIT_devdb 
         """,
         engine,
     )
-    print('loaded df from database')
+    print("loaded df from database")
     return df
 
+
 def geocode_insert_sql(records, engine):
 
     # Multiprocess
     with Pool(processes=cpu_count()) as pool:
-        it = pool.map(geocode, records, len(records)//4)
+        it = tqdm(pool.map(geocode, records, len(records) // 4))
     # it = list(map(geocode, records))
     df = pd.DataFrame(it)
-
+    df.replace({"latitude":{"":None}, "longitude":{"":None}}, inplace=True)
     df.to_sql(
         OUTPUT_TABLE_NAME,
         con=engine,
@@ -115,12 +119,13 @@ def geocode_insert_sql(records, engine):
         index=False,
     )
 
-    print(f'records in _INIT_geocoded:')
-    print(engine.execute(f"SELECT count(*) from {OUTPUT_TABLE_NAME}").fetchall()[0][0])
+    # print(f'records in _INIT_geocoded:')
+    # print(engine.execute(f"SELECT count(*) from {OUTPUT_TABLE_NAME}").fetchall()[0][0])
 
 
 def clear_dob_geocode_results(engine):
-    engine.execute(f"DROP TABLE IF EXISTS {OUTPUT_TABLE_NAME}" )
+    engine.execute(f"DROP TABLE IF EXISTS {OUTPUT_TABLE_NAME}")
+
 
 if __name__ == "__main__":
     # connect to BUILD_ENGINE
@@ -130,15 +135,13 @@ def clear_dob_geocode_results(engine):
 
     df = load_init_devdb(engine)
     records = df.to_dict("records")
+
     del df
-    # df = df.iloc[:2000,:]
-    start =0
-    chunk_size = 1000
+    start = 0
+    chunk_size = 10**4
     end = min(chunk_size, len(records))
     while start < len(records):
         print(f"geocoding records {start} through {end}")
         geocode_insert_sql(records[start:end], engine)
-        start = end 
-        end = min(end+chunk_size, len(records))
-        
-
+        start = end
+        end = min(end + chunk_size, len(records))
diff --git a/sql/_geo.sql b/sql/_geo.sql
index dd0c2a25..0855bdae 100644
--- a/sql/_geo.sql
+++ b/sql/_geo.sql
@@ -114,11 +114,8 @@ DRAFT as (
         b.longitude::double precision as geo_longitude,
         b.mode
 	FROM _INIT_devdb a
-	LEFT JOIN _GEO_devdb b
-	ON (CASE 
-            WHEN source = 'bis' THEN b.uid::text
-            ELSE (b.uid::integer + (SELECT MAX(_INIT_BIS_devdb.uid::integer) FROM _INIT_BIS_devdb))::text
-        END)::text = a.uid::text
+	LEFT JOIN _init_geocoded b
+	ON a.uid = b.uid
 ),
 GEOM_dob_bin_bldgfootprints as (
     SELECT distinct
@@ -253,7 +250,7 @@ SELECT
 INTO GEO_devdb
 FROM DRAFT a
 LEFT JOIN GEOM_dob_latlon b
-ON a.uid = b.uid;
+ON a.uid::text = b.uid::text;
 
 -- Create index
 CREATE INDEX GEO_devdb_geom_idx ON GEO_devdb 

From 70d1e920eca7c30d4e0f7b09588ff216ab784c06 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Mon, 17 Oct 2022 19:27:49 +0000
Subject: [PATCH 03/38] HNY_geo populates, but HNY_devdb does not. I suspect
 some sort of type error, will ask Te for help

---
 bash/02_build_devdb.sh |  3 +++
 python/geocode_hny.py  | 32 ++++++++++++++++++--------------
 sql/_hny.sql           | 25 +++++++++----------------
 3 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/bash/02_build_devdb.sh b/bash/02_build_devdb.sh
index 87981777..7f2e55d9 100755
--- a/bash/02_build_devdb.sh
+++ b/bash/02_build_devdb.sh
@@ -88,6 +88,9 @@ display "Combining _MID_devdb with STATUS_devdb to create MID_devdb,
 psql $BUILD_ENGINE -f sql/mid.sql
 count MID_devdb
 
+display "Geocoding HNY records"
+poetry run python3 -m python.geocode_hny
+
 display "Creating HNY fields: 
       hny_id,
       classa_hnyaff,
diff --git a/python/geocode_hny.py b/python/geocode_hny.py
index 917e8fba..13909163 100644
--- a/python/geocode_hny.py
+++ b/python/geocode_hny.py
@@ -12,18 +12,18 @@
 g = Geosupport()
 
 
-def geocode(input):
+def geocode(hny):
     # collect inputs
-    uid = str(input.pop("ogc_fid"))
-    hnum = input.pop("number")
-    sname = input.pop("street")
-    borough = input.pop("borough")
+    uid = str(hny.get("ogc_fid"))
+    hnum = hny.get("number")
+    sname = hny.get("street")
+    borough = hny.get("borough")
 
     try:
         geo = g["1B"](
             street_name=sname, house_number=hnum, borough=borough, mode="regular"
         )
-        geo = parse_output(geo)
+        geo = add_geocode(hny, geo)
         geo.update(dict(uid=uid, mode="regular", func="1B", status="success"))
         return geo
     except GeosupportError:
@@ -31,17 +31,18 @@ def geocode(input):
             geo = g["1B"](
                 street_name=sname, house_number=hnum, borough=borough, mode="tpad"
             )
-            geo = parse_output(geo)
+            geo = add_geocode(hny, geo)
             geo.update(dict(uid=uid, mode="tpad", func="1B", status="success"))
             return geo
         except GeosupportError as e:
-            geo = parse_output(e.result)
+            geo = add_geocode(hny, e.result)
             geo.update(uid=uid, mode="tpad", func="1B", status="failure")
             return geo
 
 
-def parse_output(geo):
-    return dict(
+def add_geocode(hny, geo):
+
+    new_fields = dict(
         # Normalized address:
         geo_sname=geo.get("First Street Name Normalized", ""),
         geo_hnum=geo.get("House Number - Display Format", ""),
@@ -53,10 +54,12 @@ def parse_output(geo):
             "Building Identification Number (BIN) of Input Address or NAP", ""
         ),
         geo_bbl=geo.get("BOROUGH BLOCK LOT (BBL)", {}).get(
-            "BOROUGH BLOCK LOT (BBL)", "",
+            "BOROUGH BLOCK LOT (BBL)",
+            "",
         ),
     )
 
+    return  hny | new_fields
 
 if __name__ == "__main__":
     # connect to postgres db
@@ -87,13 +90,14 @@ def parse_output(geo):
     # Multiprocess
     with Pool(processes=5) as pool:
         it = pool.map(geocode, records, 1000)
+        # it = map(geocode, records)
 
     print("Geocoding finished, dumping to postgres ...")
-    df=pd.DataFrame(it)
+    df = pd.DataFrame(it)
     df.to_sql(
-        'hny_geocode_results',
+        "hny_geocode_results",
         con=engine,
         if_exists="replace",
         index=False,
         method=psql_insert_copy,
-    )
\ No newline at end of file
+    )
diff --git a/sql/_hny.sql b/sql/_hny.sql
index 8cb26fdb..1cd918e3 100644
--- a/sql/_hny.sql
+++ b/sql/_hny.sql
@@ -121,24 +121,17 @@ CREATE TABLE IF NOT EXISTS CORR_hny_matches (
 DROP TABLE IF EXISTS HNY_geo;
 -- 1) Merge with geocoding results and create a unique ID
 WITH hny AS (
-        SELECT a.project_id||'/'||COALESCE(LPAD(a.building_id, 6, '0'), '') as hny_id,
-                a.project_id as hny_project_id,
-                a.*, 
-                b.geo_bbl, 
-                b.geo_bin, 
-                b.geo_latitude, 
-                b.geo_longitude,
-                (CASE WHEN b.geo_longitude IS NOT NULL 
-                        AND b.geo_latitude IS NOT NULL
-                    THEN ST_SetSRID(ST_MakePoint(b.geo_longitude::NUMERIC, 
-                                    b.geo_latitude::NUMERIC),4326)
+        SELECT project_id||'/'||COALESCE(LPAD(building_id, 6, '0'), '') as hny_id,
+                project_id as hny_project_id,
+                *, 
+               
+                (CASE WHEN geo_longitude IS NOT NULL 
+                        AND geo_latitude IS NOT NULL
+                    THEN ST_SetSRID(ST_MakePoint(geo_longitude::NUMERIC, 
+                                    geo_latitude::NUMERIC),4326)
                     ELSE NULL
                 END) AS geom
-        FROM hpd_hny_units_by_building a
-        JOIN hny_geocode_results b
-        ON a.ogc_fid::text = b.uid
-        WHERE a.reporting_construction_type = 'New Construction'
-        AND a.project_name <> 'CONFIDENTIAL')
+        FROM hny_geocode_results)
 
 SELECT * 
 INTO HNY_geo

From e5f07f8c132c5fcc5dce2676f133c3da8ad4f9b9 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Mon, 17 Oct 2022 20:58:03 +0000
Subject: [PATCH 04/38] format with black

---
 python/geocode.py     | 2 +-
 python/geocode_hny.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/geocode.py b/python/geocode.py
index ab464c4f..e1644b8f 100644
--- a/python/geocode.py
+++ b/python/geocode.py
@@ -111,7 +111,7 @@ def geocode_insert_sql(records, engine):
         it = tqdm(pool.map(geocode, records, len(records) // 4))
     # it = list(map(geocode, records))
     df = pd.DataFrame(it)
-    df.replace({"latitude":{"":None}, "longitude":{"":None}}, inplace=True)
+    df.replace({"latitude": {"": None}, "longitude": {"": None}}, inplace=True)
     df.to_sql(
         OUTPUT_TABLE_NAME,
         con=engine,
diff --git a/python/geocode_hny.py b/python/geocode_hny.py
index 13909163..d24c5b30 100644
--- a/python/geocode_hny.py
+++ b/python/geocode_hny.py
@@ -59,7 +59,8 @@ def add_geocode(hny, geo):
         ),
     )
 
-    return  hny | new_fields
+    return hny | new_fields
+
 
 if __name__ == "__main__":
     # connect to postgres db

From e0bdd287d0a106d76f73e43eaba4521151113a30 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Tue, 18 Oct 2022 15:14:03 +0000
Subject: [PATCH 05/38] create HNY_devdb with inner join

---
 sql/_hny.sql | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/sql/_hny.sql b/sql/_hny.sql
index 1cd918e3..8a74026d 100644
--- a/sql/_hny.sql
+++ b/sql/_hny.sql
@@ -413,17 +413,17 @@ WITH
             one_dev_to_many_hny,
             one_hny_to_many_dev
         FROM RELATEFLAGS_hny_matches a
-        WHERE one_hny_to_many_dev = 1),
+        WHERE one_hny_to_many_dev = 1)
 
     -- Combine into a single look-up table					
-	HNY_lookup AS(					
-			SELECT * FROM one_to_one
-			UNION
-			SELECT * FROM one_to_many
-                -- Many-to-many cases are further resolved in many_to_one table, so don't include
-				WHERE job_number||hny_id NOT IN (SELECT job_number||hny_id FROM many_to_one)
-			UNION
-			SELECT * FROM many_to_one)
+SELECT * INTO HNY_lookup FROM (					
+        SELECT * FROM one_to_one
+        UNION
+        SELECT * FROM one_to_many
+            -- Many-to-many cases are further resolved in many_to_one table, so don't include
+            WHERE job_number||hny_id NOT IN (SELECT job_number||hny_id FROM many_to_one)
+        UNION
+        SELECT * FROM many_to_one) as tmp;
 
 
 -- 7) MERGE WITH devdb  
@@ -440,7 +440,7 @@ SELECT a.job_number,
         END) AS hny_jobrelate
 INTO HNY_devdb
 FROM MID_devdb a
-LEFT JOIN HNY_lookup b
+INNER JOIN HNY_lookup b
 ON a.job_number = b.job_number;
 
 

From 36319169d1e7150288b2b6aa18152086d4cbe1c9 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Tue, 18 Oct 2022 15:14:14 +0000
Subject: [PATCH 06/38] clean print statements and format

---
 python/geocode.py     | 11 +++--------
 python/geocode_hny.py |  1 -
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/python/geocode.py b/python/geocode.py
index e1644b8f..49703132 100644
--- a/python/geocode.py
+++ b/python/geocode.py
@@ -1,12 +1,10 @@
 from multiprocessing import Pool, cpu_count
-from sqlalchemy import create_engine, types
+from sqlalchemy import create_engine
 from geosupport import Geosupport, GeosupportError
 from python.utils import psql_insert_copy
 import pandas as pd
 import os
-from tqdm import tqdm
 from dotenv import main
-import json
 
 main.load_dotenv()
 
@@ -108,8 +106,8 @@ def geocode_insert_sql(records, engine):
 
     # Multiprocess
     with Pool(processes=cpu_count()) as pool:
-        it = tqdm(pool.map(geocode, records, len(records) // 4))
-    # it = list(map(geocode, records))
+        it = pool.map(geocode, records, len(records) // 4)
+
     df = pd.DataFrame(it)
     df.replace({"latitude": {"": None}, "longitude": {"": None}}, inplace=True)
     df.to_sql(
@@ -119,9 +117,6 @@ def geocode_insert_sql(records, engine):
         index=False,
     )
 
-    # print(f'records in _INIT_geocoded:')
-    # print(engine.execute(f"SELECT count(*) from {OUTPUT_TABLE_NAME}").fetchall()[0][0])
-
 
 def clear_dob_geocode_results(engine):
     engine.execute(f"DROP TABLE IF EXISTS {OUTPUT_TABLE_NAME}")
diff --git a/python/geocode_hny.py b/python/geocode_hny.py
index d24c5b30..2987c8b0 100644
--- a/python/geocode_hny.py
+++ b/python/geocode_hny.py
@@ -91,7 +91,6 @@ def add_geocode(hny, geo):
     # Multiprocess
     with Pool(processes=5) as pool:
         it = pool.map(geocode, records, 1000)
-        # it = map(geocode, records)
 
     print("Geocoding finished, dumping to postgres ...")
     df = pd.DataFrame(it)

From 9cb2a5843e6c1a2e7b6dc9b72f7cde1ec20f25c6 Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Fri, 21 Oct 2022 11:14:34 -0400
Subject: [PATCH 07/38] install minio

---
 .devcontainer/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index c6b32507..49a97e0e 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -12,7 +12,9 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/
 RUN apt-get update 
 # && export DEBIAN_FRONTEND=noninteractive \
 RUN apt-get -y install --no-install-recommends postgresql-client 
-
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
+    && chmod +x mc\
+    && sudo mv ./mc /usr/bin
 
 # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
 # COPY requirements.txt /tmp/pip-tmp/

From a568df7dc9f683292f9f9b6ac86449c9de12d48a Mon Sep 17 00:00:00 2001
From: Sasha Weinstein <70385853+SashaWeinstein@users.noreply.github.com>
Date: Fri, 21 Oct 2022 11:14:58 -0400
Subject: [PATCH 08/38] this docker compose doesn't work yet but it's a good
 start

---
 Dockerfile         | 36 ++++++++++++++++++++++++++++++++++++
 docker-compose.yml | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 Dockerfile
 create mode 100644 docker-compose.yml

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..8a4ce9c0
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,36 @@
+# See here for image contents: https://github.com/NYCPlanning/docker-geosupport
+
+# [Choice] Geosupport version
+ARG VERSION_GEO="22.2.2"
+FROM nycplanning/docker-geosupport:${VERSION_GEO}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+## Install postgres
+RUN apt-get update 
+# && export DEBIAN_FRONTEND=noninteractive \
+RUN apt-get -y install --no-install-recommends postgresql-client 
+RUN apt-get install -y jq
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
+    && chmod +x mc\
+    && mv ./mc /usr/bin
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+# RUN apt-get install jq
+
+# Install poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="~/.local/bin:$PATH"
+
+# RUN poetry install
+RUN /usr/local/bin/python3 -m pip install -U bandit
+RUN /usr/local/bin/python3 -m pip install -U black
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..a6ed5963
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+  devdb:
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+    volumes:
+      - .:/workspace
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+    network_mode: service:db
+    # Uncomment the next line to use a non-root user for all processes.
+    # user: vscode
+
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  db:
+    image: postgis/postgis:11-3.0-alpine
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: postgres
+    # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+volumes:
+  postgres-data:

From c7a46900c93831108d061593b78b05d83ba0bd9c Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Wed, 9 Nov 2022 18:51:36 +0000
Subject: [PATCH 09/38] fix minio installation and add poetry to path

---
 .devcontainer/Dockerfile        | 13 +++++++------
 .devcontainer/devcontainer.json |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 49a97e0e..6a161331 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -5,16 +5,17 @@ ARG VERSION_GEO="22.2.2"
 FROM nycplanning/docker-geosupport:${VERSION_GEO}
 
 # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+# ARG NODE_VERSION="none"
+# RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
 
 ## Install postgres
 RUN apt-get update 
 # && export DEBIAN_FRONTEND=noninteractive \
-RUN apt-get -y install --no-install-recommends postgresql-client 
-RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
+RUN apt-get -y install --no-install-recommends postgresql-client wget
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc\
     && chmod +x mc\
-    && sudo mv ./mc /usr/bin
+    && mv ./mc /usr/bin
+
 
 # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
 # COPY requirements.txt /tmp/pip-tmp/
@@ -28,7 +29,7 @@ RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
 
 # Install poetry
 RUN curl -sSL https://install.python-poetry.org | python3 -
-ENV PATH="~/.local/bin:$PATH"
+ENV PATH=$HOME/.local/bin:$PATH
 
 RUN /usr/local/bin/python3 -m pip install -U bandit
 RUN /usr/local/bin/python3 -m pip install -U black
\ No newline at end of file
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index d0aba0f0..7b75d381 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -37,7 +37,7 @@
         5432
     ],
     // Use 'postCreateCommand' to run commands after the container is created.
-    "postCreateCommand": "apt-get install -y jq && poetry install",
+    "postCreateCommand": "export PATH=$HOME/.local/bin:$PATH && apt-get install -y jq && poetry install",
     // Adding id_rsa so that we can push to github from the dev container
     "initializeCommand": "ssh-add $HOME/.ssh/id_rsa"
     // export PATH=$PATH:$HOME/.local/bin"

From 229fe15308a9d141c700c9a6129f8669be99893a Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Wed, 9 Nov 2022 22:09:31 +0000
Subject: [PATCH 10/38] add poetry path to make sure poetry command works

---
 bash/config.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bash/config.sh b/bash/config.sh
index 63801305..cf476246 100755
--- a/bash/config.sh
+++ b/bash/config.sh
@@ -11,6 +11,9 @@ function set_env {
   done
 }
 
+# set path
+export PATH=$HOME/.local/bin:$PATH 
+
 # Setting Environmental Variables
 set_env .env version.env
 DATE=$(date "+%Y-%m-%d")

From 133cb38a24ddfc7ab14a42d3342a9f2d3b7833a4 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 15:28:38 +0000
Subject: [PATCH 11/38] remove import geocoding files and add wait geocoding
 process

---
 bash/01_dataloading.sh | 12 +-----------
 bash/02_build_devdb.sh |  2 ++
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/bash/01_dataloading.sh b/bash/01_dataloading.sh
index f1ef99fd..516c1fba 100755
--- a/bash/01_dataloading.sh
+++ b/bash/01_dataloading.sh
@@ -28,29 +28,19 @@ import_public dcp_firecompanies &
 import_public dcp_policeprecincts &
 import_public dob_cofos &
 import_public dof_shoreline &
-import_public hny_geocode_results &
 
-## Geocode results shares index with _geo_devdb
-psql $BUILD_ENGINE -c "DROP TABLE IF EXISTS _geo_devdb;" 
 case $MODE in
     weekly) 
         import_public dob_permitissuance &
         import_public dob_jobapplications &
-        import_public dob_geocode_results &
     ;;
     *) 
         import_public dob_permitissuance $DOB_DATA_DATE &
         import_public dob_jobapplications $DOB_DATA_DATE &
-        import_public dob_geocode_results $DOB_DATA_DATE &
     ;;
 esac
 
 psql $BUILD_ENGINE -f sql/_create.sql 
 
 wait 
-display "data loading is complete"
-
-psql $BUILD_ENGINE -c "
-    ALTER TABLE dob_geocode_results
-    RENAME TO _GEO_devdb;
-"
\ No newline at end of file
+display "data loading is complete"
\ No newline at end of file
diff --git a/bash/02_build_devdb.sh b/bash/02_build_devdb.sh
index 7f2e55d9..1d1f231e 100755
--- a/bash/02_build_devdb.sh
+++ b/bash/02_build_devdb.sh
@@ -11,6 +11,7 @@ count _INIT_devdb
 
 display "Geocoding DOB records"
 poetry run python3 -m python.geocode
+wait 
 
 # display "Assign geoms to _GEO_devdb and create GEO_devdb"
 psql $BUILD_ENGINE -f sql/_geo.sql
@@ -90,6 +91,7 @@ count MID_devdb
 
 display "Geocoding HNY records"
 poetry run python3 -m python.geocode_hny
+wait
 
 display "Creating HNY fields: 
       hny_id,

From 64155230d928a9f3fd08272d1d29e1820bbd8644 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 16:02:03 +0000
Subject: [PATCH 12/38] dockerfile for actions for poetry path

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 8a4ce9c0..1250db7c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,7 +29,7 @@ RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
 
 # Install poetry
 RUN curl -sSL https://install.python-poetry.org | python3 -
-ENV PATH="~/.local/bin:$PATH"
+ENV PATH=$HOME/.local/bin:$PATH
 
 # RUN poetry install
 RUN /usr/local/bin/python3 -m pip install -U bandit

From a5d1587a75093ed219932823980f917043212343 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 16:18:52 +0000
Subject: [PATCH 13/38] add different poetry path

---
 bash/config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bash/config.sh b/bash/config.sh
index cf476246..4eeabc6c 100755
--- a/bash/config.sh
+++ b/bash/config.sh
@@ -12,7 +12,7 @@ function set_env {
 }
 
 # set path
-export PATH=$HOME/.local/bin:$PATH 
+export PATH="$HOME/.local/share/pypoetry:$PATH"
 
 # Setting Environmental Variables
 set_env .env version.env

From 5a9590216397b8080b5583520c528bd36e252208 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 16:29:28 +0000
Subject: [PATCH 14/38] use path that works locally

---
 bash/config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bash/config.sh b/bash/config.sh
index 4eeabc6c..10e15b86 100755
--- a/bash/config.sh
+++ b/bash/config.sh
@@ -12,7 +12,7 @@ function set_env {
 }
 
 # set path
-export PATH="$HOME/.local/share/pypoetry:$PATH"
+export PATH=$PATH:$HOME/.local/bin
 
 # Setting Environmental Variables
 set_env .env version.env

From f656df5e6e8e4e73546ed0fe17d6f6bfcc810230 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 16:37:42 +0000
Subject: [PATCH 15/38] specify poetry install location

---
 Dockerfile | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1250db7c..86807663 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,18 +5,18 @@ ARG VERSION_GEO="22.2.2"
 FROM nycplanning/docker-geosupport:${VERSION_GEO}
 
 # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+# ARG NODE_VERSION="none"
+# RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
 
 ## Install postgres
 RUN apt-get update 
 # && export DEBIAN_FRONTEND=noninteractive \
-RUN apt-get -y install --no-install-recommends postgresql-client 
-RUN apt-get install -y jq
-RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
+RUN apt-get -y install --no-install-recommends postgresql-client wget
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc\
     && chmod +x mc\
     && mv ./mc /usr/bin
 
+
 # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
 # COPY requirements.txt /tmp/pip-tmp/
 # RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
@@ -28,9 +28,8 @@ RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc\
 # RUN apt-get install jq
 
 # Install poetry
-RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$HOME/.local/bin python3 -
 ENV PATH=$HOME/.local/bin:$PATH
 
-# RUN poetry install
 RUN /usr/local/bin/python3 -m pip install -U bandit
 RUN /usr/local/bin/python3 -m pip install -U black
\ No newline at end of file

From bcc575ddd46798528e174f6fece338515cf4ad21 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 17:29:31 +0000
Subject: [PATCH 16/38] print path

---
 Dockerfile     | 4 ++--
 bash/config.sh | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 86807663..3aa258dc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,8 +28,8 @@ RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc\
 # RUN apt-get install jq
 
 # Install poetry
-RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$HOME/.local/bin python3 -
-ENV PATH=$HOME/.local/bin:$PATH
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="$HOME/.local/bin:$PATH"
 
 RUN /usr/local/bin/python3 -m pip install -U bandit
 RUN /usr/local/bin/python3 -m pip install -U black
\ No newline at end of file
diff --git a/bash/config.sh b/bash/config.sh
index 10e15b86..8a88f437 100755
--- a/bash/config.sh
+++ b/bash/config.sh
@@ -13,6 +13,7 @@ function set_env {
 
 # set path
 export PATH=$PATH:$HOME/.local/bin
+echo $PATH
 
 # Setting Environmental Variables
 set_env .env version.env

From 32ded07c3f815149c0fd3e18d611acdf26661701 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 17:45:26 +0000
Subject: [PATCH 17/38] add path to binary

---
 bash/config.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bash/config.sh b/bash/config.sh
index 8a88f437..a0368a16 100755
--- a/bash/config.sh
+++ b/bash/config.sh
@@ -13,6 +13,7 @@ function set_env {
 
 # set path
 export PATH=$PATH:$HOME/.local/bin
+export PATH=$PATH:$HOME/.local/share/pypoetry/venv/bin/poetry
 echo $PATH
 
 # Setting Environmental Variables

From b591cda2175837088bdbcb3e288825b0a3b1ce97 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 19:35:04 +0000
Subject: [PATCH 18/38] add poetry installation in test.yml

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 82a995bb..c652fb20 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -84,6 +84,7 @@ jobs:
           chmod +x mc
           sudo mv ./mc /usr/bin
           mc alias set spaces $AWS_S3_ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY --api S3v4
+          curl -sSL https://install.python-poetry.org | python3 -
 
       - name: 1. dataloading for HED weekly builds
         if: >-

From df8def1cedf74eb091d60584f3ccc8eb00324b90 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 19:59:13 +0000
Subject: [PATCH 19/38] poetry install

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c652fb20..aee0d7e1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -85,6 +85,8 @@ jobs:
           sudo mv ./mc /usr/bin
           mc alias set spaces $AWS_S3_ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY --api S3v4
           curl -sSL https://install.python-poetry.org | python3 -
+          export PATH=$PATH:$HOME/.local/bin
+          poetry install
 
       - name: 1. dataloading for HED weekly builds
         if: >-

From b12c85e0ef78e7d60f0b7d604208a8bd518cbcb5 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:08:48 +0000
Subject: [PATCH 20/38] update python

---
 .github/workflows/test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aee0d7e1..eee1e68b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -55,7 +55,10 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' 
 
       - name: config workflows
         id: config

From 00b24386cd86565c3b7f9252bf7aac3f3f92a570 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:20:06 +0000
Subject: [PATCH 21/38] docker compose status

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index eee1e68b..1a45f6bc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -109,6 +109,7 @@ jobs:
       - name: 2. build dev_db ...
         if: steps.config.outputs.rebuild == 'yes'
         run: |
+          docker compose ps
           ./devdb.sh build
           ./devdb.sh aggregate
 

From 7d5ab844a3e4996b2a8be029c5e207804485b926 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:33:34 +0000
Subject: [PATCH 22/38] add geosupoort container to actions

---
 .github/workflows/test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1a45f6bc..ba48e96d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -47,6 +47,7 @@ jobs:
           --health-retries 5
         ports:
           - 25060:5432
+
     env:
       BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
@@ -54,6 +55,8 @@ jobs:
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    container:
+      image: nycplanning/docker-geosupport:latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

From 6eaedad22f6882bdcb1abc87e7834cdf2b95ea8b Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:42:32 +0000
Subject: [PATCH 23/38] remove sudo and break up steps

---
 .github/workflows/test.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ba48e96d..ba0eeac2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -47,7 +47,6 @@ jobs:
           --health-retries 5
         ports:
           - 25060:5432
-
     env:
       BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
@@ -62,7 +61,6 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10' 
-
       - name: config workflows
         id: config
         run: |
@@ -84,16 +82,22 @@ jobs:
 
       - name: install dependencies ...
         run: |
-          sudo apt update
-          sudo apt install -y gdal-bin
+          apt update
+          apt install -y gdal-bin
+
+      - name: Install and configure minio client ...
+        run: |
           curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
           chmod +x mc
-          sudo mv ./mc /usr/bin
+          mv ./mc /usr/bin/ 
           mc alias set spaces $AWS_S3_ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY --api S3v4
+
+      - name: Poetry Install
+        run: |
           curl -sSL https://install.python-poetry.org | python3 -
           export PATH=$PATH:$HOME/.local/bin
           poetry install
-
+        
       - name: 1. dataloading for HED weekly builds
         if: >-
           steps.config.outputs.weekly == 'yes' && 

From bf526bb854f36d59b8c6f095df6cf880d3116d96 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:46:44 +0000
Subject: [PATCH 24/38] remove docker command

---
 .github/workflows/test.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ba0eeac2..39859804 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,6 +33,15 @@ jobs:
         ! contains(github.event.head_commit.message, '[skip]')
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
+    env:
+      BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb
+      HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
+      EDM_DATA: ${{ secrets.EDM_DATA }}
+      AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    container:
+      image: nycplanning/docker-geosupport:latest
     services:
       db:
         image: postgis/postgis:12-3.0-alpine
@@ -47,15 +56,7 @@ jobs:
           --health-retries 5
         ports:
           - 25060:5432
-    env:
-      BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb
-      HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
-      EDM_DATA: ${{ secrets.EDM_DATA }}
-      AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-    container:
-      image: nycplanning/docker-geosupport:latest
+
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -116,7 +117,6 @@ jobs:
       - name: 2. build dev_db ...
         if: steps.config.outputs.rebuild == 'yes'
         run: |
-          docker compose ps
           ./devdb.sh build
           ./devdb.sh aggregate
 

From ef8c361e062f2b47e14c8b0dbf73a48ffb320e97 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:52:47 +0000
Subject: [PATCH 25/38] add postsql install

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 39859804..90286c31 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -84,7 +84,7 @@ jobs:
       - name: install dependencies ...
         run: |
           apt update
-          apt install -y gdal-bin
+          apt install -y gdal-bin postgresql-client
 
       - name: Install and configure minio client ...
         run: |

From 316356f41345f1acd6a2dbd626a8f7bcc4a93132 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 20:57:33 +0000
Subject: [PATCH 26/38] change to port 5432

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 90286c31..47228abd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -55,7 +55,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
         ports:
-          - 25060:5432
+          - 5432:5432
 
     steps:
       - uses: actions/checkout@v3

From d263ba0bd66c3d691218e5734905b80b2f17fe44 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Thu, 10 Nov 2022 21:01:20 +0000
Subject: [PATCH 27/38] remove space and change it back to 25060

---
 .github/workflows/test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 47228abd..57bb80f9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -55,8 +55,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
         ports:
-          - 5432:5432
-
+          - 25060:5432
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

From e3ae085cf83a74de36d3d93f62cbb3d25c2b11a0 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 09:53:06 -0500
Subject: [PATCH 28/38] give build engine a different port

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 57bb80f9..0a112a0f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb
+      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/devdb
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
@@ -55,7 +55,7 @@ jobs:
           --health-timeout 5s
           --health-retries 5
         ports:
-          - 25060:5432
+          - 5432:5432
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

From fb1316eb211b01d3674362f17e04cd4a3573f88e Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 09:59:11 -0500
Subject: [PATCH 29/38] use postgres not local host

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0a112a0f..d7fe8ba4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/devdb
+      BUILD_ENGINE: postgresql://postgres:postgres@postgres:5432/devdb
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}

From 9b19b5b44d743ee6663080e423dc598ba7aa8a9d Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 10:25:20 -0500
Subject: [PATCH 30/38] remove apt update

---
 .github/workflows/test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d7fe8ba4..5d051c0b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@postgres:5432/devdb
+      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/devdb
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
@@ -82,7 +82,6 @@ jobs:
 
       - name: install dependencies ...
         run: |
-          apt update
           apt install -y gdal-bin postgresql-client
 
       - name: Install and configure minio client ...

From 2017132ef4ca771107d35ec887a05be6134ea192 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 10:53:02 -0500
Subject: [PATCH 31/38] add jq

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5d051c0b..a6acb87a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -82,7 +82,7 @@ jobs:
 
       - name: install dependencies ...
         run: |
-          apt install -y gdal-bin postgresql-client
+          apt install -y gdal-bin postgresql-client jq
 
       - name: Install and configure minio client ...
         run: |

From 27bcf8d6d1d2020a79449ef7bc5f7beadaa08479 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 11:13:37 -0500
Subject: [PATCH 32/38] add localhost to arg for postgis image

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a6acb87a..01b1931d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -46,6 +46,7 @@ jobs:
       db:
         image: postgis/postgis:12-3.0-alpine
         env:
+          POSTGRES_HOST: localhost
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: devdb
         options: >-
@@ -83,6 +84,7 @@ jobs:
       - name: install dependencies ...
         run: |
           apt install -y gdal-bin postgresql-client jq
+          docker compose 
 
       - name: Install and configure minio client ...
         run: |

From 4a5ee463ceef273c582f599d44d7f9fbd754323e Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Mon, 14 Nov 2022 11:17:20 -0500
Subject: [PATCH 33/38] remove docker command

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 01b1931d..0ba7368f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -84,7 +84,6 @@ jobs:
       - name: install dependencies ...
         run: |
           apt install -y gdal-bin postgresql-client jq
-          docker compose 
 
       - name: Install and configure minio client ...
         run: |

From 64e94a4b03950164061da47f1b701ac17ea03ebc Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Fri, 2 Dec 2022 13:56:09 -0500
Subject: [PATCH 34/38] specify different build engine

---
 .github/workflows/test.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0ba7368f..81a01b46 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/devdb
+      BUILD_ENGINE: postgresql://postgres:postgres@postgres:5432/postgres
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
@@ -46,9 +46,7 @@ jobs:
       db:
         image: postgis/postgis:12-3.0-alpine
         env:
-          POSTGRES_HOST: localhost
           POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: devdb
         options: >-
           --shm-size=1g
           --health-cmd pg_isready

From 569f84379903982e0827c2e5b8e0e53a4889a937 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Fri, 2 Dec 2022 14:08:53 -0500
Subject: [PATCH 35/38] use localhost

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 81a01b46..252a03aa 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-20.04
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@postgres:5432/postgres
+      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/postgres
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}

From 62b8c4cd1c28230c9affa590496d831e860bf441 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Fri, 2 Dec 2022 14:20:23 -0500
Subject: [PATCH 36/38] use ubuntu latest

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 252a03aa..f1acc518 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
         github.event_name == 'push' &&
         ! contains(github.event.head_commit.message, '[skip]')
       ) || github.event_name != 'push'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/postgres
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}

From 78026ec234c91eb363accd03c6fb962a71ca3aa4 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Fri, 2 Dec 2022 14:24:19 -0500
Subject: [PATCH 37/38] specify user and db

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f1acc518..998d7e83 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -46,7 +46,9 @@ jobs:
       db:
         image: postgis/postgis:12-3.0-alpine
         env:
+          POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
         options: >-
           --shm-size=1g
           --health-cmd pg_isready

From 0574380d753037665f312a3d408c53ab59f20d92 Mon Sep 17 00:00:00 2001
From: Te Du <tedudevelop@gmail.com>
Date: Fri, 2 Dec 2022 14:51:26 -0500
Subject: [PATCH 38/38] replace localhost

---
 .github/workflows/test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 998d7e83..8ac35fec 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -34,7 +34,7 @@ jobs:
       ) || github.event_name != 'push'
     runs-on: ubuntu-latest
     env:
-      BUILD_ENGINE: postgresql://postgres:postgres@localhost:5432/postgres
+      BUILD_ENGINE: postgresql://postgres:postgres@127.0.0.1:5432/postgres
       HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }}
       EDM_DATA: ${{ secrets.EDM_DATA }}
       AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
@@ -109,6 +109,9 @@ jobs:
           steps.config.outputs.rebuild == 'yes' &&
           steps.config.outputs.weekly == 'no'
         run: ./devdb.sh dataloading edm && ls -l
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
 
       - name: Clear cache
         run: rm -rf .library