diff --git a/pluto_build/01_dataloading.sh b/pluto_build/01_dataloading.sh index 2b4dce29..7cf3bd27 100755 --- a/pluto_build/01_dataloading.sh +++ b/pluto_build/01_dataloading.sh @@ -55,7 +55,8 @@ import_public lpc_historic_districts & import_public lpc_landmarks & import_public dcp_colp & import_public dof_dtm & -import_public dof_condo +import_public dof_condo & +import_public dcp_housing wait diff --git a/pluto_build/01_dataloading_minor.sh b/pluto_build/01_dataloading_minor.sh index 5ae39ff7..8ee08334 100755 --- a/pluto_build/01_dataloading_minor.sh +++ b/pluto_build/01_dataloading_minor.sh @@ -58,6 +58,7 @@ import_public dpr_greenthumb $DPR_GREENTHUMB_VERSION & import_public dsny_frequencies $DSNY_FREQUENCIES_VERSION & import_public lpc_historic_districts $LPC_HISTORIC_DISTRICTS_VERSION & import_public lpc_landmarks $LPC_LANDMARKS_VESRSION +import_public dcp_housing $DCP_HOUSING # import pluto corrections import_public pluto_corrections $PLUTO_CORRECTIONS_VERSION & diff --git a/pluto_build/05_qaqc.sh b/pluto_build/05_qaqc.sh index da753f94..ad1839c7 100755 --- a/pluto_build/05_qaqc.sh +++ b/pluto_build/05_qaqc.sh @@ -44,7 +44,7 @@ function QAQC { } # QAQC MISMATCH ANALYSIS -for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql +for file in sql/qaqc_aggregate.sql sql/qaqc_mismatch.sql sql/qaqc_null.sql sql/qaqc_outlier.sql sql/qaqc_pts_condo.sql sql/qaqc_housing_units.sql do for mapped in true false do diff --git a/pluto_build/06_export.sh b/pluto_build/06_export.sh index 180d6519..917fdeb6 100755 --- a/pluto_build/06_export.sh +++ b/pluto_build/06_export.sh @@ -58,7 +58,7 @@ mkdir -p output/dof && mkdir -p output/qaqc && (cd output/qaqc - for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier + for table in qaqc_aggregate qaqc_expected qaqc_mismatch qaqc_null qaqc_outlier qaqc_pts_condo qaqc_housing_units do psql $BUILD_ENGINE -c "\COPY ( SELECT * FROM ${table} diff --git a/pluto_build/sql/qaqc_housing_units.sql b/pluto_build/sql/qaqc_housing_units.sql new file mode 100644 index 00000000..54adddf2 --- /dev/null +++ b/pluto_build/sql/qaqc_housing_units.sql @@ -0,0 +1,44 @@ +-- Create qaqc table of all PLUTO records where there is a match in Housing Database +-- and the PLUTO residential units value does not match the housing database certificates +-- of occupancy value. Have flag indicating if bbl has residential unit correction in manual corrections table. + + +-- Data Dictionary +-- bbl - billing BBL +-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). +-- unitsres - The number of residential units as reported by PLUTO database +-- units_co - The number of units listed on the DOB issued Certificate of Occupancy +-- new_value - The new number of residential units as reported by the current version of PLUTO +-- old_value - The old number of residential units as reported by the previous version of PLUTO + + +CREATE TABLE IF NOT EXISTS qaqc_housing_units( + bbl text, + job_number text, + unitsres text, + units_co text, + new_value text, + old_value text +); + +INSERT INTO qaqc_housing_units +WITH base as( +SELECT DISTINCT round(a.bbl::numeric,0)::text as bbl,b.job_number,a.unitsres,round(b.units_co::numeric,0)::text as units_co +FROM pluto a, dcp_housing b +WHERE b.bbl||b.date_lastupdt IN ( +-- get the most recent DOB record for a BBL based on date of last update field + SELECT bbl||max(date_lastupdt::date) maxDate + FROM dcp_housing + GROUP BY bbl) +AND round(a.bbl::numeric,0)::text=b.bbl +AND a.unitsres<>round(b.units_co::numeric,0)::text), +-- select only corrections to unitsres field +corrections_subset as ( +SELECT * +FROM pluto_corrections +WHERE field='unitsres') +-- combine PLUTO, DOB, and corrections data for final output +SELECT a.*, c.new_value, c.old_value +FROM base a +LEFT JOIN corrections_subset c +ON a.bbl=c.bbl; diff --git a/pluto_build/sql/qaqc_pts_condo.sql b/pluto_build/sql/qaqc_pts_condo.sql new file mode 100644 index 00000000..0dae97ad --- /dev/null +++ b/pluto_build/sql/qaqc_pts_condo.sql @@ -0,0 +1,73 @@ +-- The units and coop_apts values are greater than one for the billing bbl AND +-- Two or more unit bbl records have units and coop_apts values greater than 1 + +-- Data Dictionary + +-- primebbl - Billing BBL +-- bbl - Unit level BBl +-- units - The number of units listed by Department of Finance for the property (compare to units_co in dcp_housing) +-- coop_apts - The number of residential units listed for the property +-- job_number - The DOB job application number assigned when the applicant begins the application. This is the unique identifier for the application submitted to the Department of Buildings (DOB). +-- units_co - The number of units listed on the DOB issued Certificate of Occupancy +-- date_lastupdt - The date of the last update to the DOB record for the job filing. +-- new_value - The new number of units as reported by the pluto_corrections file +-- old_value - The previous number of units as reported by the pluto_correction file + + +CREATE TABLE IF NOT EXISTS qaqc_pts_condo( + primebbl text, + bbl text, + units numeric, + coop_apts numeric, + job_number text, + units_co text, + date_lastupdt varchar, + new_value text, + old_value text +); + +INSERT INTO qaqc_pts_condo +WITH pts_subset as ( +SELECT primebbl, bbl, units, coop_apts +FROM pluto_rpad_geo + WHERE primebbl IN ( + SELECT primebbl FROM ( + SELECT primebbl, COUNT(*) + FROM pluto_rpad_geo + WHERE tl NOT LIKE '75%' + AND RIGHT(primebbl,4) LIKE '75%' + AND units::integer > 1 + AND coop_apts::integer > 1 + GROUP BY primebbl, units, coop_apts) as badbases + WHERE count>1) + AND primebbl IN ( + SELECT primebbl FROM ( + SELECT primebbl + FROM pluto_rpad_geo + WHERE tl LIKE '75%' + AND units::integer > 1 + AND coop_apts::integer > 1) as badbillings)), +-- get the most recent DOB record for a BBL based on date of last update field +dob_subset as ( +SELECT * FROM dcp_housing b +WHERE b.bbl||b.date_lastupdt IN ( + SELECT bbl||max(date_lastupdt::date) maxDate + FROM dcp_housing + GROUP BY bbl)), +-- select only corrections to unitsres field +corrections_subset as ( +SELECT * +FROM pluto_corrections +WHERE field='unitsres'), +-- Join PTS and DOB subsets, preserving all PTS records +pts_dob as ( +SELECT a.*, b.job_number, round(b.units_co::numeric,0)::text as units_co, b.date_lastupdt +FROM pts_subset a +LEFT JOIN dob_subset b +ON a.bbl=b.bbl +AND a.coop_apts::text<>round(b.units_co::numeric,0)::text) +-- Join on corrections to produce final output +SELECT a.*, c.new_value, c.old_value +FROM pts_dob a +LEFT JOIN corrections_subset c +ON a.bbl=c.bbl; \ No newline at end of file diff --git a/pluto_build/sql/source_data_versions.sql b/pluto_build/sql/source_data_versions.sql index 7d3c2925..61f30e85 100644 --- a/pluto_build/sql/source_data_versions.sql +++ b/pluto_build/sql/source_data_versions.sql @@ -70,4 +70,6 @@ CREATE TABLE source_data_versions as( (SELECT 'fema_pfirms2015_100yr' as schema_name, v from fema_pfirms2015_100yr limit 1) union (SELECT 'doitt_buildingcentroids' as schema_name, v from pluto_input_numbldgs limit 1) + union + (SELECT 'dcp_housing' as schema_name, v from dcp_housing limit 1) ) a order by schema_name); \ No newline at end of file diff --git a/pluto_build/version.env b/pluto_build/version.env index 184add99..115793a7 100644 --- a/pluto_build/version.env +++ b/pluto_build/version.env @@ -16,5 +16,6 @@ DPR_GREENTHUMB_VERSION=20221101 DSNY_FREQUENCIES_VERSION=20221105 LPC_HISTORIC_DISTRICTS_VERSION=20220526 LPC_LANDMARKS_VESRSION=20220613 +DCP_HOUSING=22Q2 PLUTO_CORRECTIONS_VERSION=21v4 \ No newline at end of file