From 4bbcbc188830077d597442e9029bac4193bd1766 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 14:43:55 -0500 Subject: [PATCH 01/11] add dcp_housing to dataloading --- pluto_build/01_dataloading.sh | 3 ++- pluto_build/01_dataloading_minor.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pluto_build/01_dataloading.sh b/pluto_build/01_dataloading.sh index 2b4dce29..7cf3bd27 100755 --- a/pluto_build/01_dataloading.sh +++ b/pluto_build/01_dataloading.sh @@ -55,7 +55,8 @@ import_public lpc_historic_districts & import_public lpc_landmarks & import_public dcp_colp & import_public dof_dtm & -import_public dof_condo +import_public dof_condo & +import_public dcp_housing wait diff --git a/pluto_build/01_dataloading_minor.sh b/pluto_build/01_dataloading_minor.sh index 5ae39ff7..8ee08334 100755 --- a/pluto_build/01_dataloading_minor.sh +++ b/pluto_build/01_dataloading_minor.sh @@ -58,6 +58,7 @@ import_public dpr_greenthumb $DPR_GREENTHUMB_VERSION & import_public dsny_frequencies $DSNY_FREQUENCIES_VERSION & import_public lpc_historic_districts $LPC_HISTORIC_DISTRICTS_VERSION & import_public lpc_landmarks $LPC_LANDMARKS_VESRSION +import_public dcp_housing $DCP_HOUSING # import pluto corrections import_public pluto_corrections $PLUTO_CORRECTIONS_VERSION & From a827d79bc0307fbd2592e6eaa57cd749a6403623 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 14:45:24 -0500 Subject: [PATCH 02/11] add updated source data versions --- pluto_build/sql/source_data_versions.sql | 2 ++ pluto_build/version.env | 1 + 2 files changed, 3 insertions(+) diff --git a/pluto_build/sql/source_data_versions.sql b/pluto_build/sql/source_data_versions.sql index 7d3c2925..61f30e85 100644 --- a/pluto_build/sql/source_data_versions.sql +++ b/pluto_build/sql/source_data_versions.sql @@ -70,4 +70,6 @@ CREATE TABLE source_data_versions as( (SELECT 'fema_pfirms2015_100yr' as schema_name, v from fema_pfirms2015_100yr limit 1) union (SELECT 'doitt_buildingcentroids' as schema_name, v from pluto_input_numbldgs limit 1) + union + (SELECT 'dcp_housing' as schema_name, v from dcp_housing limit 1) ) a order by schema_name); \ No newline at end of file diff --git a/pluto_build/version.env b/pluto_build/version.env index 184add99..115793a7 100644 --- a/pluto_build/version.env +++ b/pluto_build/version.env @@ -16,5 +16,6 @@ DPR_GREENTHUMB_VERSION=20221101 DSNY_FREQUENCIES_VERSION=20221105 LPC_HISTORIC_DISTRICTS_VERSION=20220526 LPC_LANDMARKS_VESRSION=20220613 +DCP_HOUSING=22Q2 PLUTO_CORRECTIONS_VERSION=21v4 \ No newline at end of file From 7c71fe8ab4aeede100bfab18199b3933b6aafc46 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 14:47:56 -0500 Subject: [PATCH 03/11] add logic and qaqc table for pts condo --- pluto_build/sql/qaqc_pts_condo.sql | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 pluto_build/sql/qaqc_pts_condo.sql diff --git a/pluto_build/sql/qaqc_pts_condo.sql b/pluto_build/sql/qaqc_pts_condo.sql new file mode 100644 index 00000000..ff1d23b4 --- /dev/null +++ b/pluto_build/sql/qaqc_pts_condo.sql @@ -0,0 +1,60 @@ +-- Record PTS records that meet the criteria of +-- The units and coop_apts values are greater than one for the billing bbl AND +-- Two or more unit bbl records have units and coop_apts values greater than 1 +CREATE TABLE IF NOT EXISTS qaqc_pts_condo( + primebbl text, + bbl text, + units numeric, + coop_apts numeric, + job_number text, + units_co text, + date_lastupdt varchar, + new_value text, + old_value text +); + +INSERT INTO qaqc_pts_condo +WITH pts_subset as ( +SELECT primebbl, bbl, units, coop_apts +FROM pluto_rpad_geo + WHERE primebbl IN ( + SELECT primebbl FROM ( + SELECT primebbl, COUNT(*) + FROM pluto_rpad_geo + WHERE tl NOT LIKE '75%' + AND RIGHT(primebbl,4) LIKE '75%' + AND units::integer > 1 + AND coop_apts::integer > 1 + GROUP BY primebbl, units, coop_apts) as badbases + WHERE count>1) + AND primebbl IN ( + SELECT primebbl FROM ( + SELECT primebbl + FROM pluto_rpad_geo + WHERE tl LIKE '75%' + AND units::integer > 1 + AND coop_apts::integer > 1) as badbillings)), +-- get the most recent DOB record for a BBL based on date of last update field +dob_subset as ( +SELECT * FROM dcp_housing b +WHERE b.bbl||b.date_lastupdt IN ( + SELECT bbl||max(date_lastupdt::date) maxDate + FROM dcp_housing + GROUP BY bbl)), +-- select only corrections to unitsres field +corrections_subset as ( +SELECT * +FROM pluto_corrections +WHERE field='unitsres'), +-- Join PTS and DOB subsets, preserving all PTS records +pts_dob as ( +SELECT a.*, b.job_number, round(b.units_co::numeric,0)::text as units_co, b.date_lastupdt +FROM pts_subset a +LEFT JOIN dob_subset b +ON a.bbl=b.bbl +AND a.coop_apts::text<>round(b.units_co::numeric,0)::text) +-- Join on corrections to produce final output +SELECT a.*, c.new_value, c.old_value +FROM pts_dob a +LEFT JOIN corrections_subset c +ON a.bbl=c.bbl; \ No newline at end of file From e7b0949e0262bc10fc6fb2ecea83da86bbbc402f Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 15:02:52 -0500 Subject: [PATCH 04/11] add qaqc_pts_condo to qaqc and export .sh --- pluto_build/05_qaqc.sh | 2 +- pluto_build/06_export.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pluto_build/05_qaqc.sh b/pluto_build/05_qaqc.sh index da753f94..9d1583f8 100755 --- a/pluto_build/05_qaqc.sh +++ b/pluto_build/05_qaqc.sh @@ -44,7 +44,7 @@ function QAQC { } # QAQC MISMATCH ANALYSIS -for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql +for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql sql/qaqc_pts_condo.sql do for mapped in true false do diff --git a/pluto_build/06_export.sh b/pluto_build/06_export.sh index 180d6519..b1108ad8 100755 --- a/pluto_build/06_export.sh +++ b/pluto_build/06_export.sh @@ -58,7 +58,7 @@ mkdir -p output/dof && mkdir -p output/qaqc && (cd output/qaqc - for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier + for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier qaqc_pts_condo do psql $BUILD_ENGINE -c "\COPY ( SELECT * FROM ${table} From ec9b5413b7d88b336efe7060b12602317fe5e134 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 15:09:56 -0500 Subject: [PATCH 05/11] add sql script for qaqc_housing_units --- pluto_build/sql/qaqc_housing_units.sql | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 pluto_build/sql/qaqc_housing_units.sql diff --git a/pluto_build/sql/qaqc_housing_units.sql b/pluto_build/sql/qaqc_housing_units.sql new file mode 100644 index 00000000..cd885856 --- /dev/null +++ b/pluto_build/sql/qaqc_housing_units.sql @@ -0,0 +1,34 @@ +-- Create qaqc table All PLUTO records where there is a match in Housing Database +-- and the PLUTO residential units value does not match the housing database certificates +-- of occupancy value. Have flag indicating if bbl has residential unit correction in manual corrections table. +-- select PLUTO records that have a match in the HousingDB subset where unitsres does not equal units co +CREATE TABLE IF NOT EXISTS qaqc_housing_units( + bbl text, + job_number text, + unitsres numeric, + units_co text, + new_value text, + old_value text +); + +INSERT INTO qaqc_housing_units +WITH base as( +SELECT DISTINCT round(a.bbl::numeric,0)::text as bbl,b.job_number,a.unitsres,round(b.units_co::numeric,0)::text as units_co +FROM pluto a, dcp_housing b +WHERE b.bbl||b.date_lastupdt IN ( +-- get the most recent DOB record for a BBL based on date of last update field + SELECT bbl||max(date_lastupdt::date) maxDate + FROM dcp_housing + GROUP BY bbl) +AND round(a.bbl::numeric,0)::text=b.bbl +AND a.unitsres<>round(b.units_co::numeric,0)::text), +-- select only corrections to unitsres field +corrections_subset as ( +SELECT * +FROM pluto_corrections +WHERE field='unitsres') +-- combine PLUTO, DOB, and corrections data for final output +SELECT a.*, c.new_value, c.old_value +FROM base a +LEFT JOIN corrections_subset c +ON a.bbl=c.bbl; From 814b2b24d581adb9bb7965fc8b64130f72d198f5 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Thu, 9 Feb 2023 15:12:56 -0500 Subject: [PATCH 06/11] add qaqc housing units to qaqc and export shell script --- pluto_build/05_qaqc.sh | 2 +- pluto_build/06_export.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pluto_build/05_qaqc.sh b/pluto_build/05_qaqc.sh index 9d1583f8..ad1839c7 100755 --- a/pluto_build/05_qaqc.sh +++ b/pluto_build/05_qaqc.sh @@ -44,7 +44,7 @@ function QAQC { } # QAQC MISMATCH ANALYSIS -for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql sql/qaqc_pts_condo.sql +for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql sql/qaqc_pts_condo.sql sql/qaqc_housing_units.sql do for mapped in true false do diff --git a/pluto_build/06_export.sh b/pluto_build/06_export.sh index b1108ad8..917fdeb6 100755 --- a/pluto_build/06_export.sh +++ b/pluto_build/06_export.sh @@ -58,7 +58,7 @@ mkdir -p output/dof && mkdir -p output/qaqc && (cd output/qaqc - for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier qaqc_pts_condo + for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier qaqc_pts_condo qaqc_housing_units do psql $BUILD_ENGINE -c "\COPY ( SELECT * FROM ${table} From 51f87f4e724fbfee7ca25d3deb52c5355608ed95 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 10 Feb 2023 09:25:09 -0500 Subject: [PATCH 07/11] concert column to text --- pluto_build/sql/qaqc_housing_units.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluto_build/sql/qaqc_housing_units.sql b/pluto_build/sql/qaqc_housing_units.sql index cd885856..e76429b7 100644 --- a/pluto_build/sql/qaqc_housing_units.sql +++ b/pluto_build/sql/qaqc_housing_units.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS qaqc_housing_units( bbl text, job_number text, - unitsres numeric, + unitsres text, units_co text, new_value text, old_value text From 7ca3a4a9a923aba93409ce6adb8e6d61d2991347 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 10 Feb 2023 11:33:19 -0500 Subject: [PATCH 08/11] add description of cols --- pluto_build/sql/qaqc_pts_condo.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pluto_build/sql/qaqc_pts_condo.sql b/pluto_build/sql/qaqc_pts_condo.sql index ff1d23b4..2cfd8620 100644 --- a/pluto_build/sql/qaqc_pts_condo.sql +++ b/pluto_build/sql/qaqc_pts_condo.sql @@ -1,6 +1,20 @@ -- Record PTS records that meet the criteria of -- The units and coop_apts values are greater than one for the billing bbl AND -- Two or more unit bbl records have units and coop_apts values greater than 1 + +-- Data Dictionary + +-- primebbl - Billing BBL +-- bbl - Unit level BBl +-- units - The number of units listed by Department of Finance for the property (compare to units_co in dcp_housing) +-- coop_apts - The number of coop apartments listed for the property +-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). +-- units_co - The number of units listed on the DOB issued Certificate of Occupancy +-- date_lastupdt - The date of the last update to the DOB record for the job filing. +-- new_value +-- old_value + + CREATE TABLE IF NOT EXISTS qaqc_pts_condo( primebbl text, bbl text, From 5d57405454be5446743edc4fd98625cd6a0ebc4d Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 10 Feb 2023 11:44:43 -0500 Subject: [PATCH 09/11] add data dict --- pluto_build/sql/qaqc_pts_condo.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pluto_build/sql/qaqc_pts_condo.sql b/pluto_build/sql/qaqc_pts_condo.sql index 2cfd8620..1e19f1b8 100644 --- a/pluto_build/sql/qaqc_pts_condo.sql +++ b/pluto_build/sql/qaqc_pts_condo.sql @@ -11,8 +11,8 @@ -- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). -- units_co - The number of units listed on the DOB issued Certificate of Occupancy -- date_lastupdt - The date of the last update to the DOB record for the job filing. --- new_value --- old_value +-- new_value - The new number of units as reported by the pluto_corrections file +-- old_value - The previous number of units as reported by the pluto_correction file CREATE TABLE IF NOT EXISTS qaqc_pts_condo( From d0f4c128c110c42656eaf78b0bee13b1cc7bd475 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Fri, 10 Feb 2023 11:49:07 -0500 Subject: [PATCH 10/11] add data dict --- pluto_build/sql/qaqc_housing_units.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pluto_build/sql/qaqc_housing_units.sql b/pluto_build/sql/qaqc_housing_units.sql index e76429b7..913c3a97 100644 --- a/pluto_build/sql/qaqc_housing_units.sql +++ b/pluto_build/sql/qaqc_housing_units.sql @@ -2,6 +2,17 @@ -- and the PLUTO residential units value does not match the housing database certificates -- of occupancy value. Have flag indicating if bbl has residential unit correction in manual corrections table. -- select PLUTO records that have a match in the HousingDB subset where unitsres does not equal units co + + +-- Data Dictionary +-- bbl - billing BBL +-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). +-- units_res - The number of residential units as reported by dcp_housing database +-- units_co - The number of units listed on the DOB issued Certificate of Occupancy +-- new_value +-- old_value + + CREATE TABLE IF NOT EXISTS qaqc_housing_units( bbl text, job_number text, From 8b972452173d8b3bf639bfe5ecfa17ba81cf88c6 Mon Sep 17 00:00:00 2001 From: mbh329 Date: Tue, 14 Feb 2023 11:37:44 -0500 Subject: [PATCH 11/11] clean up comments --- pluto_build/sql/qaqc_housing_units.sql | 9 ++++----- pluto_build/sql/qaqc_pts_condo.sql | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pluto_build/sql/qaqc_housing_units.sql b/pluto_build/sql/qaqc_housing_units.sql index 913c3a97..54adddf2 100644 --- a/pluto_build/sql/qaqc_housing_units.sql +++ b/pluto_build/sql/qaqc_housing_units.sql @@ -1,16 +1,15 @@ --- Create qaqc table All PLUTO records where there is a match in Housing Database +-- Create qaqc table of all PLUTO records where there is a match in Housing Database -- and the PLUTO residential units value does not match the housing database certificates -- of occupancy value. Have flag indicating if bbl has residential unit correction in manual corrections table. --- select PLUTO records that have a match in the HousingDB subset where unitsres does not equal units co -- Data Dictionary -- bbl - billing BBL -- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). --- units_res - The number of residential units as reported by dcp_housing database +-- unitsres - The number of residential units as reported by PLUTO database -- units_co - The number of units listed on the DOB issued Certificate of Occupancy --- new_value --- old_value +-- new_value - The new number of residential units as reported by the current version of PLUTO +-- old_value - The old number of residential units as reported by the previous version of PLUTO CREATE TABLE IF NOT EXISTS qaqc_housing_units( diff --git a/pluto_build/sql/qaqc_pts_condo.sql b/pluto_build/sql/qaqc_pts_condo.sql index 1e19f1b8..0dae97ad 100644 --- a/pluto_build/sql/qaqc_pts_condo.sql +++ b/pluto_build/sql/qaqc_pts_condo.sql @@ -1,4 +1,3 @@ --- Record PTS records that meet the criteria of -- The units and coop_apts values are greater than one for the billing bbl AND -- Two or more unit bbl records have units and coop_apts values greater than 1 @@ -7,7 +6,7 @@ -- primebbl - Billing BBL -- bbl - Unit level BBl -- units - The number of units listed by Department of Finance for the property (compare to units_co in dcp_housing) --- coop_apts - The number of coop apartments listed for the property +-- coop_apts - The number of residential units listed for the property -- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). -- units_co - The number of units listed on the DOB issued Certificate of Occupancy -- date_lastupdt - The date of the last update to the DOB record for the job filing.