diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c6b32507..6a161331 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,13 +5,16 @@ ARG VERSION_GEO="22.2.2" FROM nycplanning/docker-geosupport:${VERSION_GEO} # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 -ARG NODE_VERSION="none" -RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi +# ARG NODE_VERSION="none" +# RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi ## Install postgres RUN apt-get update # && export DEBIAN_FRONTEND=noninteractive \ -RUN apt-get -y install --no-install-recommends postgresql-client +RUN apt-get -y install --no-install-recommends postgresql-client wget +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc\ + && chmod +x mc\ + && mv ./mc /usr/bin # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. @@ -26,7 +29,7 @@ RUN apt-get -y install --no-install-recommends postgresql-client # Install poetry RUN curl -sSL https://install.python-poetry.org | python3 - -ENV PATH="~/.local/bin:$PATH" +ENV PATH=$HOME/.local/bin:$PATH RUN /usr/local/bin/python3 -m pip install -U bandit RUN /usr/local/bin/python3 -m pip install -U black \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d0aba0f0..7b75d381 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -37,7 +37,7 @@ 5432 ], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "apt-get install -y jq && poetry install", + "postCreateCommand": "export PATH=$HOME/.local/bin:$PATH && apt-get install -y jq && poetry install", // Adding id_rsa so that we can push to github from the dev container "initializeCommand": "ssh-add $HOME/.ssh/id_rsa" // export PATH=$PATH:$HOME/.local/bin" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82a995bb..8ac35fec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,13 +32,23 @@ jobs: github.event_name == 'push' && ! contains(github.event.head_commit.message, '[skip]') ) || github.event_name != 'push' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + env: + BUILD_ENGINE: postgresql://postgres:postgres@127.0.0.1:5432/postgres + HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }} + EDM_DATA: ${{ secrets.EDM_DATA }} + AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + container: + image: nycplanning/docker-geosupport:latest services: db: image: postgis/postgis:12-3.0-alpine env: + POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: devdb + POSTGRES_DB: postgres options: >- --shm-size=1g --health-cmd pg_isready @@ -46,17 +56,12 @@ jobs: --health-timeout 5s --health-retries 5 ports: - - 25060:5432 - env: - BUILD_ENGINE: postgresql://postgres:postgres@localhost:25060/devdb - HED_BUILD_ENGINE: ${{ secrets.HED_BUILD_ENGINE }} - EDM_DATA: ${{ secrets.EDM_DATA }} - AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - 5432:5432 steps: - - uses: actions/checkout@v2 - + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: config workflows id: config run: | @@ -78,13 +83,21 @@ jobs: - name: install dependencies ... run: | - sudo apt update - sudo apt install -y gdal-bin + apt install -y gdal-bin postgresql-client jq + + - name: Install and configure minio client ... + run: | curl -O https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc - sudo mv ./mc /usr/bin + mv ./mc /usr/bin/ mc alias set spaces $AWS_S3_ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY --api S3v4 + - name: Poetry Install + run: | + curl -sSL https://install.python-poetry.org | python3 - + export PATH=$PATH:$HOME/.local/bin + poetry install + - name: 1. dataloading for HED weekly builds if: >- steps.config.outputs.weekly == 'yes' && @@ -96,6 +109,9 @@ jobs: steps.config.outputs.rebuild == 'yes' && steps.config.outputs.weekly == 'no' run: ./devdb.sh dataloading edm && ls -l + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} - name: Clear cache run: rm -rf .library diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3aa258dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# See here for image contents: https://github.com/NYCPlanning/docker-geosupport + +# [Choice] Geosupport version +ARG VERSION_GEO="22.2.2" +FROM nycplanning/docker-geosupport:${VERSION_GEO} + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +# ARG NODE_VERSION="none" +# RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +## Install postgres +RUN apt-get update +# && export DEBIAN_FRONTEND=noninteractive \ +RUN apt-get -y install --no-install-recommends postgresql-client wget +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc\ + && chmod +x mc\ + && mv ./mc /usr/bin + + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends +# RUN apt-get install jq + +# Install poetry +RUN curl -sSL https://install.python-poetry.org | python3 - +ENV PATH="$HOME/.local/bin:$PATH" + +RUN /usr/local/bin/python3 -m pip install -U bandit +RUN /usr/local/bin/python3 -m pip install -U black \ No newline at end of file diff --git a/bash/01_dataloading.sh b/bash/01_dataloading.sh index f1ef99fd..516c1fba 100755 --- a/bash/01_dataloading.sh +++ b/bash/01_dataloading.sh @@ -28,29 +28,19 @@ import_public dcp_firecompanies & import_public dcp_policeprecincts & import_public dob_cofos & import_public dof_shoreline & -import_public hny_geocode_results & -## Geocode results shares index with _geo_devdb -psql $BUILD_ENGINE -c "DROP TABLE IF EXISTS _geo_devdb;" case $MODE in weekly) import_public dob_permitissuance & import_public dob_jobapplications & - import_public dob_geocode_results & ;; *) import_public dob_permitissuance $DOB_DATA_DATE & import_public dob_jobapplications $DOB_DATA_DATE & - import_public dob_geocode_results $DOB_DATA_DATE & ;; esac psql $BUILD_ENGINE -f sql/_create.sql wait -display "data loading is complete" - -psql $BUILD_ENGINE -c " - ALTER TABLE dob_geocode_results - RENAME TO _GEO_devdb; -" \ No newline at end of file +display "data loading is complete" \ No newline at end of file diff --git a/bash/02_build_devdb.sh b/bash/02_build_devdb.sh index de535559..1d1f231e 100755 --- a/bash/02_build_devdb.sh +++ b/bash/02_build_devdb.sh @@ -9,7 +9,11 @@ psql $BUILD_ENGINE -f sql/now/_init.sql psql $BUILD_ENGINE -f sql/_init.sql count _INIT_devdb -display "Assign geoms to _GEO_devdb and create GEO_devdb" +display "Geocoding DOB records" +poetry run python3 -m python.geocode +wait + +# display "Assign geoms to _GEO_devdb and create GEO_devdb" psql $BUILD_ENGINE -f sql/_geo.sql psql $BUILD_ENGINE -f sql/_geo_corrections.sql count GEO_devdb @@ -85,6 +89,10 @@ display "Combining _MID_devdb with STATUS_devdb to create MID_devdb, psql $BUILD_ENGINE -f sql/mid.sql count MID_devdb +display "Geocoding HNY records" +poetry run python3 -m python.geocode_hny +wait + display "Creating HNY fields: hny_id, classa_hnyaff, diff --git a/bash/config.sh b/bash/config.sh index 63801305..a0368a16 100755 --- a/bash/config.sh +++ b/bash/config.sh @@ -11,6 +11,11 @@ function set_env { done } +# set path +export PATH=$PATH:$HOME/.local/bin +export PATH=$PATH:$HOME/.local/share/pypoetry/venv/bin/poetry +echo $PATH + # Setting Environmental Variables set_env .env version.env DATE=$(date "+%Y-%m-%d") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..a6ed5963 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: "3" + +services: + devdb: + build: + context: . + dockerfile: Dockerfile + + volumes: + - .:/workspace + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: postgis/postgis:11-3.0-alpine + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + +volumes: + postgres-data: diff --git a/python/geocode.py b/python/geocode.py index ddfce207..49703132 100644 --- a/python/geocode.py +++ b/python/geocode.py @@ -4,13 +4,14 @@ from python.utils import psql_insert_copy import pandas as pd import os -from tqdm import tqdm from dotenv import main main.load_dotenv() g = Geosupport() +OUTPUT_TABLE_NAME = "_init_geocoded" + def geocode(input): # collect inputs @@ -84,74 +85,58 @@ def parse_output(geo): ) -def load_applications(engine): +def load_init_devdb(engine): df = pd.read_sql( """ SELECT uid, - regexp_replace( - trim(house_number), - '(^|)0*', '', '' - ) as house_number, - REGEXP_REPLACE(street_name, '[\s]{2,}' ,' ' , 'g') as street_name, - borough, - source - FROM ( - SELECT - distinct ogc_fid as uid, - housenumber as house_number, - streetname as street_name, - borough, - 'bis' as source - FROM dob_jobapplications UNION - SELECT - distinct ogc_fid as uid, - house_no as house_number, - street_name as street_name, - borough, - 'now' as source - FROM dob_now_applications - ) a LIMIT 400000 + job_number, + address_numbr as house_number, + REGEXP_REPLACE(address_street, '[\s]{2,}' ,' ' , 'g') as street_name, + boro as borough + FROM _INIT_devdb """, engine, ) - + print("loaded df from database") return df -def geocode_insert_sql(df): - records = df.to_dict("records") + +def geocode_insert_sql(records, engine): # Multiprocess with Pool(processes=cpu_count()) as pool: - it = tqdm(pool.map(geocode, records, 1000)) - # it = tqdm(list(map(geocode, records))) + it = pool.map(geocode, records, len(records) // 4) df = pd.DataFrame(it) + df.replace({"latitude": {"": None}, "longitude": {"": None}}, inplace=True) df.to_sql( - "dob_geocode_results", + OUTPUT_TABLE_NAME, con=engine, if_exists="append", index=False, - method=psql_insert_copy, ) + def clear_dob_geocode_results(engine): - engine.execute("DROP TABLE IF EXISTS dob_geocode_results") + engine.execute(f"DROP TABLE IF EXISTS {OUTPUT_TABLE_NAME}") + if __name__ == "__main__": # connect to BUILD_ENGINE engine = create_engine(os.environ["BUILD_ENGINE"]) - df = load_applications(engine) clear_dob_geocode_results(engine) - # df = df.iloc[:2000,:] - start =0 - chunk_size = 50000 - end = chunk_size - while end <= df.shape[0]: - print(f"geocoding records {start} through {end}") - geocode_insert_sql(df.iloc[start:end,:]) - start = end - end = min(end+chunk_size, df.shape[0]) - + df = load_init_devdb(engine) + records = df.to_dict("records") + + del df + start = 0 + chunk_size = 10**4 + end = min(chunk_size, len(records)) + while start < len(records): + print(f"geocoding records {start} through {end}") + geocode_insert_sql(records[start:end], engine) + start = end + end = min(end + chunk_size, len(records)) diff --git a/python/geocode_hny.py b/python/geocode_hny.py index 917e8fba..2987c8b0 100644 --- a/python/geocode_hny.py +++ b/python/geocode_hny.py @@ -12,18 +12,18 @@ g = Geosupport() -def geocode(input): +def geocode(hny): # collect inputs - uid = str(input.pop("ogc_fid")) - hnum = input.pop("number") - sname = input.pop("street") - borough = input.pop("borough") + uid = str(hny.get("ogc_fid")) + hnum = hny.get("number") + sname = hny.get("street") + borough = hny.get("borough") try: geo = g["1B"]( street_name=sname, house_number=hnum, borough=borough, mode="regular" ) - geo = parse_output(geo) + geo = add_geocode(hny, geo) geo.update(dict(uid=uid, mode="regular", func="1B", status="success")) return geo except GeosupportError: @@ -31,17 +31,18 @@ def geocode(input): geo = g["1B"]( street_name=sname, house_number=hnum, borough=borough, mode="tpad" ) - geo = parse_output(geo) + geo = add_geocode(hny, geo) geo.update(dict(uid=uid, mode="tpad", func="1B", status="success")) return geo except GeosupportError as e: - geo = parse_output(e.result) + geo = add_geocode(hny, e.result) geo.update(uid=uid, mode="tpad", func="1B", status="failure") return geo -def parse_output(geo): - return dict( +def add_geocode(hny, geo): + + new_fields = dict( # Normalized address: geo_sname=geo.get("First Street Name Normalized", ""), geo_hnum=geo.get("House Number - Display Format", ""), @@ -53,10 +54,13 @@ def parse_output(geo): "Building Identification Number (BIN) of Input Address or NAP", "" ), geo_bbl=geo.get("BOROUGH BLOCK LOT (BBL)", {}).get( - "BOROUGH BLOCK LOT (BBL)", "", + "BOROUGH BLOCK LOT (BBL)", + "", ), ) + return hny | new_fields + if __name__ == "__main__": # connect to postgres db @@ -89,11 +93,11 @@ def parse_output(geo): it = pool.map(geocode, records, 1000) print("Geocoding finished, dumping to postgres ...") - df=pd.DataFrame(it) + df = pd.DataFrame(it) df.to_sql( - 'hny_geocode_results', + "hny_geocode_results", con=engine, if_exists="replace", index=False, method=psql_insert_copy, - ) \ No newline at end of file + ) diff --git a/sql/_geo.sql b/sql/_geo.sql index dd0c2a25..0855bdae 100644 --- a/sql/_geo.sql +++ b/sql/_geo.sql @@ -114,11 +114,8 @@ DRAFT as ( b.longitude::double precision as geo_longitude, b.mode FROM _INIT_devdb a - LEFT JOIN _GEO_devdb b - ON (CASE - WHEN source = 'bis' THEN b.uid::text - ELSE (b.uid::integer + (SELECT MAX(_INIT_BIS_devdb.uid::integer) FROM _INIT_BIS_devdb))::text - END)::text = a.uid::text + LEFT JOIN _init_geocoded b + ON a.uid = b.uid ), GEOM_dob_bin_bldgfootprints as ( SELECT distinct @@ -253,7 +250,7 @@ SELECT INTO GEO_devdb FROM DRAFT a LEFT JOIN GEOM_dob_latlon b -ON a.uid = b.uid; +ON a.uid::text = b.uid::text; -- Create index CREATE INDEX GEO_devdb_geom_idx ON GEO_devdb diff --git a/sql/_hny.sql b/sql/_hny.sql index 8cb26fdb..8a74026d 100644 --- a/sql/_hny.sql +++ b/sql/_hny.sql @@ -121,24 +121,17 @@ CREATE TABLE IF NOT EXISTS CORR_hny_matches ( DROP TABLE IF EXISTS HNY_geo; -- 1) Merge with geocoding results and create a unique ID WITH hny AS ( - SELECT a.project_id||'/'||COALESCE(LPAD(a.building_id, 6, '0'), '') as hny_id, - a.project_id as hny_project_id, - a.*, - b.geo_bbl, - b.geo_bin, - b.geo_latitude, - b.geo_longitude, - (CASE WHEN b.geo_longitude IS NOT NULL - AND b.geo_latitude IS NOT NULL - THEN ST_SetSRID(ST_MakePoint(b.geo_longitude::NUMERIC, - b.geo_latitude::NUMERIC),4326) + SELECT project_id||'/'||COALESCE(LPAD(building_id, 6, '0'), '') as hny_id, + project_id as hny_project_id, + *, + + (CASE WHEN geo_longitude IS NOT NULL + AND geo_latitude IS NOT NULL + THEN ST_SetSRID(ST_MakePoint(geo_longitude::NUMERIC, + geo_latitude::NUMERIC),4326) ELSE NULL END) AS geom - FROM hpd_hny_units_by_building a - JOIN hny_geocode_results b - ON a.ogc_fid::text = b.uid - WHERE a.reporting_construction_type = 'New Construction' - AND a.project_name <> 'CONFIDENTIAL') + FROM hny_geocode_results) SELECT * INTO HNY_geo @@ -420,17 +413,17 @@ WITH one_dev_to_many_hny, one_hny_to_many_dev FROM RELATEFLAGS_hny_matches a - WHERE one_hny_to_many_dev = 1), + WHERE one_hny_to_many_dev = 1) -- Combine into a single look-up table - HNY_lookup AS( - SELECT * FROM one_to_one - UNION - SELECT * FROM one_to_many - -- Many-to-many cases are further resolved in many_to_one table, so don't include - WHERE job_number||hny_id NOT IN (SELECT job_number||hny_id FROM many_to_one) - UNION - SELECT * FROM many_to_one) +SELECT * INTO HNY_lookup FROM ( + SELECT * FROM one_to_one + UNION + SELECT * FROM one_to_many + -- Many-to-many cases are further resolved in many_to_one table, so don't include + WHERE job_number||hny_id NOT IN (SELECT job_number||hny_id FROM many_to_one) + UNION + SELECT * FROM many_to_one) as tmp; -- 7) MERGE WITH devdb @@ -447,7 +440,7 @@ SELECT a.job_number, END) AS hny_jobrelate INTO HNY_devdb FROM MID_devdb a -LEFT JOIN HNY_lookup b +INNER JOIN HNY_lookup b ON a.job_number = b.job_number;