Database merge

.gitignore

@@ -26,3 +26,4 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 *.env
+/database/TEST

Database2.0/AUTOMATION/automation_insert.py

@@ -0,0 +1,60 @@
+import openpyxl
+import random
+
+# Initialize random seed for reproducibility
+random.seed(42)
+
+# Define sites and their basic information
+sites_data = [
+    (1, "Downtown Health Center", "Metropolis", "ON", "Fixed"),
+    (2, "Uptown Clinic", "Gotham", "BC", "Mobile"),
+    (3, "Greenfield Medical Center", "Greenfield", "QC", "Fixed"),
+    (4, "Valleyview Community Clinic", "Valleyview", "BC", "Fixed"),
+    (5, "Sunset Wellness Hub", "Sunset City", "SK", "Fixed"),
+    (6, "Rosewood Mobile Health Unit", "Rosewood", "MB", "Mobile")
+]
+months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
+def distribute_value(total, categories):
+    parts = [random.randint(0, total) for _ in range(categories - 1)] + [0, total]
+    parts.sort()
+    return [parts[i+1] - parts[i] for i in range(categories)]
+
+def generate_data(site_id, site_name, city, province, site_type, base_total_consumption):
+    total_consumption_variation = random.randint(-500, 500)  # Allow small variation
+    total_consumption = max(500, base_total_consumption + total_consumption_variation)
+    new_clients = random.randint(1, total_consumption // 2)
+    unique_clients = total_consumption - new_clients
+    age_distribution = distribute_value(total_consumption, 6)
+    gender_distribution = distribute_value(total_consumption, 3)
+    total_referrals = random.randint(10, 100)
+    total_visits = total_consumption + round(total_referrals * random.uniform(0.1, 0.5))
+    substance_use = [random.randint(0, total_visits) for _ in range(14)]
+    overdoses = random.randint(15, 50)
+    naloxone_administered = random.randint(int(overdoses * 0.1), int(overdoses * 0.7))
+
+    return [
+        site_id, site_name, city, province, site_type, 2022, total_visits, total_consumption,
+        *age_distribution, *gender_distribution, new_clients, unique_clients,
+        *substance_use, overdoses, random.randint(15, 50), random.randint(15, 50),
+        random.randint(0, 10), naloxone_administered, random.randint(0, 20), total_referrals
+    ]
+
+# Create a new Excel workbook and select the active worksheet
+wb = openpyxl.Workbook()
+ws = wb.active
+
+# Define column names
+columns = ["Site ID", "Site Name", "City", "Province", "Site Type", "Report Year", "Total Visits", "Total Consumption", "Under 20", "Age 20-29", "Age 30-39", "Age 40-49", "Age 50-59", "Over 60", "Age Unknown", "Male Clients", "Female Clients", "Other Clients", "New Clients", "Unique Clients", "Cocaine Usage", "Crack Usage", "Methamphetamine Usage", "Amphetamine Usage", "Heroin Usage", "Fentanyl Usage", "Oxycontin/Oxycodone Usage", "Morphine Usage", "Hydromorphone/Dilaudid Usage", "Unspecified Opioid Usage", "Speedball Usage", "Other Substances Usage", "Unknown/Not Specified Substance Usage", "Non-Fatal Overdoses", "Fatal Overdoses", "Other Medical Emergencies", "Law Enforcement Calls", "Naloxone Administered", "EMS Called", "Total Referrals"]
+
+
+# Generate and write data for each site for each month
+base_total_consumption = 1000  # Starting point for total consumption
+for month in months:
+    for site in sites_data:
+        row =[month] + generate_data(*site, base_total_consumption)
+        ws.append(row)
+    base_total_consumption += random.randint(-100, 100)  # Adjust base for the next month to simulate trends
+
+
+# Save the workbook to a file
+wb.save("yearly_populated_site_data2.xlsx")

Database2.0/Database setup.md

@@ -0,0 +1,22 @@
+1. Create Database
+2. Create Schema
+3. Create Enums
+4. Create Tables
+5. Insert into Roles tables
+6. Apply RBAC using DO in RLS
+Implement the RBAC model by defining roles and permissions.
+DO $$
+BEGIN
+    -- RBAC setup queries
+END
+$$;
+7. Define and Implement RLS
+8. Add all Functions
+9. Insert Initial Site just (just site_id one) and run the create functions
+10. Add all Triggers and Triggers
+11. Create all Views
+12. Create all Materialized Views
+13. Add any Additional Views
+14. Insert into Sites Staffs forms etc
+15. Test
+

Database2.0/FUNCTIONS/create_site_relevant_analysis.psql

@@ -0,0 +1,79 @@
+CREATE OR REPLACE FUNCTION site_schema.create_site_analysis_view(site_id_arg INTEGER)
+    RETURNS void
+    LANGUAGE 'plpgsql'  
+AS $BODY$
+DECLARE
+    dynamic_sql TEXT;
+BEGIN
+    dynamic_sql := format(' CREATE OR REPLACE VIEW site_schema.site_%s_relevant_analysis AS
+		WITH incident_drug_usage_stats AS (
+			SELECT
+				sub.site_id,
+				qo.determined_answer AS drug,
+				COUNT(DISTINCT sub.submission_id) AS incident_drug_usage
+			FROM
+				site_schema.submissions sub
+			INNER JOIN
+				site_schema.answers ans ON sub.submission_id = ans.submission_id
+			INNER JOIN
+				site_schema.question_options qo ON ans.question_option_id = qo.question_option_id
+			WHERE
+				sub.form_id = 2 AND ans.question_id = 14  -- Assumes question_id 14 relates to "What drugs were consumed?" in form 2
+			GROUP BY
+				sub.site_id, qo.determined_answer
+		),
+		null_responses AS (
+			SELECT
+				sub.site_id,
+				COUNT(*) FILTER (WHERE ans.question_option_id IS NULL AND ans.answer_text IS NULL AND ans.answer_date IS NULL AND ans.answer_boolean IS NULL) AS null_count
+			FROM
+				site_schema.submissions sub
+			JOIN
+				site_schema.answers ans ON sub.submission_id = ans.submission_id
+			GROUP BY
+				sub.site_id
+		),
+		referral_services_needed AS (
+			SELECT
+				sub.site_id,
+				qo.determined_answer AS service,
+				COUNT(DISTINCT sub.submission_id) AS referral_count
+			FROM
+				site_schema.submissions sub
+			JOIN
+				site_schema.answers ans ON sub.submission_id = ans.submission_id
+			JOIN
+				site_schema.question_options qo ON ans.question_option_id = qo.question_option_id
+			WHERE
+				sub.form_id = 3 AND ans.question_id = 19  -- Assumes question_id 19 relates to "Type of service referred to" in form 3
+			GROUP BY
+				sub.site_id, qo.determined_answer
+		)
+		SELECT
+			inc.site_id,
+			inc.drug,
+			inc.incident_drug_usage,
+			nr.null_count,
+			AVG(nr.null_count) OVER (PARTITION BY inc.site_id) AS avg_null_responses,
+			rs.service AS referral_service,
+			rs.referral_count,
+			CASE --Ensure no divisible zero
+				WHEN SUM(rs.referral_count) OVER (PARTITION BY inc.site_id) = 0 THEN 0
+				ELSE rs.referral_count::DECIMAL / SUM(rs.referral_count) OVER (PARTITION BY inc.site_id) * 100 
+			END AS percentage_of_total_referrals
+		FROM
+			incident_drug_usage_stats inc
+		LEFT JOIN
+			null_responses nr ON inc.site_id = nr.site_id
+		LEFT JOIN
+			referral_services_needed rs ON inc.site_id = rs.site_id
+        WHERE
+			inc.site_id = %L 
+		ORDER BY
+			inc.site_id, inc.incident_drug_usage DESC, nr.null_count DESC, rs.referral_count DESC;
+        ', site_id_arg, site_id_arg); -- Use the site_id_arg for dynamic view name and filtering
+
+    -- Execute the dynamic SQL to create the view
+    EXECUTE dynamic_sql;
+END;
+$BODY$;

Database2.0/FUNCTIONS/create_site_view.psql

@@ -0,0 +1,79 @@
+--select site_schema.create_site_view(..);
+CREATE OR REPLACE FUNCTION site_schema.create_site_view(site_id_arg INTEGER)
+    RETURNS void
+    LANGUAGE 'plpgsql'  
+AS $BODY$
+DECLARE
+    dynamic_sql TEXT;
+BEGIN
+    -- Define the dynamic SQL for creating the site-specific view
+    dynamic_sql := format('CREATE OR REPLACE VIEW site_schema.site_%s_summary_metrics AS
+		SELECT
+			s.site_id,
+			s.site_name,
+            DATE_TRUNC(''week'', sub.created_date) AS report_week, 
+            DATE_TRUNC(''month'', sub.created_date) AS report_month,
+            DATE_TRUNC(''day'', sub.created_date) AS report_day, 
+    		EXTRACT(YEAR FROM sub.created_date) AS report_year,
+			 -- Add count of submissions for form ID 3 where question_id 18 is 36
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 THEN sub.submission_id END) +
+			COUNT(DISTINCT CASE WHEN sub.form_id = 3 AND ans.question_id = 18 AND ans.question_option_id = 36 THEN sub.submission_id END) AS total_visits,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 THEN sub.submission_id END) AS total_consumption,
+			-- Age demographics
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) < 20 THEN 1 END) AS under_20,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) BETWEEN 20 AND 29 THEN 1 END) AS age_20_29,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) BETWEEN 30 AND 39 THEN 1 END) AS age_30_39,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) BETWEEN 40 AND 49 THEN 1 END) AS age_40_49,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) BETWEEN 50 AND 59 THEN 1  END) AS age_50_59,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND EXTRACT(YEAR FROM AGE(CAST(ans.answer_date AS DATE))) >= 60 THEN 1  END) AS over_60,
+			COUNT(CASE  WHEN sub.form_id = 1 AND  ans.question_id = 2 AND (ans.answer_date IS NULL OR ans.answer_date IS NULL) THEN 1 END) AS age_unknown,
+			-- Client demographics based on question_option_id
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 3 AND ans.question_option_id = 1 THEN sub.submission_id END) AS Male_clients,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 3 AND ans.question_option_id = 2 THEN sub.submission_id END) AS Female_clients,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 3 AND ans.question_option_id = 3 THEN sub.submission_id END) AS Other_clients,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND ans.question_id = 4 AND ans.answer_boolean = TRUE  THEN sub.submission_id  END) AS new_clients,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND ans.question_id = 4 AND ans.answer_boolean = FALSE THEN sub.submission_id  END) AS unique_clients,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 4 THEN sub.submission_id END) AS cocaine_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 5 THEN sub.submission_id END) AS crack_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 6 THEN sub.submission_id END) AS methamphetamine_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 7 THEN sub.submission_id END) AS amphetamine_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 8 THEN sub.submission_id END) AS heroin_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 9 THEN sub.submission_id END) AS fentanyl_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 10 THEN sub.submission_id END) AS oxycontin_oxycodone_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 11 THEN sub.submission_id END) AS morphine_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 12 THEN sub.submission_id END) AS hydromorphone_dilaudid_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 13 THEN sub.submission_id END) AS unspecified_opioid_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 14 THEN sub.submission_id END) AS speedball_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 15 THEN sub.submission_id END) AS other_substances_usage,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 1 AND  ans.question_id = 6 AND ans.question_option_id = 16 THEN sub.submission_id END) AS unknown_not_specified_substance_usage,	
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 10 AND ans.question_option_id = 17 THEN sub.submission_id END) AS non_fatal_overdoses, -- Count of non-fatal overdoses
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 10 AND ans.question_option_id = 18 THEN sub.submission_id END) AS fatal_overdoses,    -- Count of fatal overdoses
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 10 AND ans.question_option_id = 19 THEN sub.submission_id END) AS other_medical_emergencies,     -- Count of other medical emergencies
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 10 AND ans.question_option_id =20 THEN sub.submission_id END) AS law_enforcement_calls,    -- Count of law enforcement calls
+			-- Count of incidents where naloxone was administered
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 11 AND ans.answer_boolean = TRUE THEN sub.submission_id END) AS naloxone_administered,
+			-- Count of incidents where EMS was called
+			COUNT(DISTINCT CASE WHEN sub.form_id = 2 AND  ans.question_id = 12 AND ans.answer_boolean = TRUE THEN sub.submission_id END) AS ems_called,
+			COUNT(DISTINCT CASE WHEN sub.form_id = 3 THEN sub.submission_id END) AS total_referrals
+		FROM
+			site_schema.submissions sub
+		JOIN
+			site_schema.sc_sites s ON sub.site_id = s.site_id
+		JOIN
+			site_schema.answers ans ON sub.submission_id = ans.submission_id
+		WHERE
+			sub.site_id = %L AND
+            sub.form_id IN (1, 2, 3)
+		GROUP BY
+			s.site_id,
+			s.site_name,
+            report_week, report_month, report_day, sub.created_date,
+			report_year
+		ORDER BY
+    		sub.created_date DESC;',
+        site_id_arg, site_id_arg); -- Use the site_id_arg for dynamic view name and filtering
+
+    -- Execute the dynamic SQL to create the view
+    EXECUTE dynamic_sql;
+END;
+$BODY$

Database2.0/FUNCTIONS/fetch_submissions_and_answers.psql

@@ -0,0 +1,50 @@
+--Verify the staff Row Level Acess to get data, then gets the submissions bewtween date range for the site_id
+CREATE OR REPLACE FUNCTION fetch_submissions_with_answers(
+    site_id_arg INT,
+	staff_id_arg INT,
+	start_date timestamp with time zone DEFAULT NULL,
+    end_date timestamp with time zone DEFAULT NULL
+) 
+RETURNS TABLE (
+    submission_id INT,
+    staff_id INT,
+    form_id INT,
+    site_id INT,
+    created_date timestamp with time zone,
+    question_id INT,
+    question_text TEXT,
+    question_option_id INT,
+   /* determined_answer text,*/
+    answer_text TEXT,
+    answer_date DATE,
+    answer_boolean BOOLEAN
+) LANGUAGE 'plpgsql'
+AS $BODY$
+DECLARE
+    has_access BOOLEAN := FALSE;
+BEGIN
+    -- Verify if the staff has access to the site's data
+    SELECT EXISTS (
+        SELECT 1
+        FROM site_schema.staffs s
+        JOIN site_schema.roles r ON s.role_id = r.role_id
+        WHERE s.staff_id = staff_id_arg AND s.site_id = site_id_arg AND r.role_id IN (1, 4, 5) -- =roles 1, 4, 5 are admin and site_admin, site_collector
+    ) INTO has_access;
+
+    IF has_access THEN
+        RETURN QUERY
+        SELECT s.submission_id, s.staff_id, s.form_id, s.site_id, s.created_date,
+               q.question_id, q.question_text, 
+               /*qo.determined_answer, */
+               a.question_option_id, a.answer_text, a.answer_date, a.answer_boolean
+        FROM site_schema.submissions s
+        JOIN site_schema.answers a ON s.submission_id = a.submission_id
+        JOIN site_schema.questions q ON a.question_id = q.question_id 
+        LEFT JOIN site_schema.question_options qo ON a.question_option_id = qo.question_option_id
+        WHERE (start_date IS NULL OR s.created_date >= start_date) AND (end_date IS NULL OR s.created_date <= end_date)
+        AND s.site_id = site_id_arg;
+    ELSE
+        RAISE EXCEPTION 'Staff member % does not have access to site %', staff_id_arg, site_id_arg;
+    END IF;
+END;
+$BODY$ 

Database2.0/FUNCTIONS/generate_staff_email_passwords.psql

@@ -0,0 +1,39 @@
+--Generate random email and password for clint using role_id and site_id
+CREATE OR REPLACE FUNCTION site_schema."site_schema.generate_staff_email_passwords"(role_id integer, site_id integer DEFAULT NULL)
+RETURNS void
+LANGUAGE 'plpgsql'  
+AS $BODY$
+DECLARE
+	s_site_id integer := site_id;
+    s_role_id integer := role_id;
+    staff_record RECORD;
+    new_email TEXT;
+    new_password TEXT;
+    email_domain TEXT := '@example.com';
+    password_characters TEXT := 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+    password_length INT := 5; -- length of the password
+BEGIN
+    FOR staff_record IN
+        SELECT s.site_id, s.staff_id, r.role_name
+        FROM site_schema.staffs s
+        JOIN site_schema.roles r ON s.role_id = r.role_id
+        WHERE (s.site_id = s_site_id OR s_site_id IS NULL)
+          AND s.role_id = s_role_id
+    LOOP
+        -- Generate new email incorporating the role
+        new_email := 'staff_' || staff_record.role_name || '_' || staff_record.staff_id || email_domain;
+
+        -- Generate new password
+        new_password := '';
+        FOR i IN 1..password_length LOOP
+            new_password := new_password || substr(password_characters, (random() * (length(password_characters) - 1) + 1)::INTEGER, 1);
+        END LOOP;
+        new_password := md5(new_password); -- Hash the password,
+        -- Update staff record with new email and password hash
+        UPDATE site_schema.staffs
+        SET email = new_email, password_hash = new_password
+        WHERE staff_id = staff_record.staff_id;
+    END LOOP;
+END;
+$BODY$
+

Database2.0/FUNCTIONS/update_master_site_view.psql

@@ -0,0 +1,54 @@
+CREATE OR REPLACE FUNCTION site_schema.update_master_site_view()
+RETURNS TRIGGER 
+LANGUAGE 'plpgsql'
+AS $BODY$
+DECLARE
+    view_record RECORD;
+    metrics TEXT := 'CREATE OR REPLACE VIEW site_schema.master_site_view AS ';
+    analysis TEXT := 'CREATE OR REPLACE VIEW site_schema.master_site_relevant_analysis_view AS ';
+    metrics_view_count INTEGER := 0;
+    analysis_view_count INTEGER := 0;
+BEGIN
+    -- Updating the master_site_relevant_analysis_view showing all current only metric views
+    FOR view_record IN
+        SELECT table_name
+        FROM information_schema.views
+        WHERE table_schema = 'site_schema' AND table_name LIKE 'site_%_summary_metrics'
+    LOOP
+        IF metrics_view_count > 0 THEN
+            metrics := metrics || ' UNION ALL ';
+        END IF;
+        metrics := metrics || format('SELECT * FROM site_schema.%I', view_record.table_name);
+        metrics_view_count := metrics_view_count + 1;
+    END LOOP;
+
+    -- Execute for master_site_view
+    IF metrics_view_count > 0 THEN
+        EXECUTE metrics;
+    ELSE
+        RAISE NOTICE 'No site-specific summary metrics views found to include in master_site_view.';
+    END IF;
+
+    -- Updating the master_site_relevant_analysis_view showing current only analysis views
+    FOR view_record IN
+        SELECT table_name
+        FROM information_schema.views
+        WHERE table_schema = 'site_schema' AND table_name LIKE 'site_%_relevant_analysis'
+    LOOP
+        IF analysis_view_count > 0 THEN
+            analysis := analysis || ' UNION ALL ';
+        END IF;
+        analysis := analysis || format('SELECT * FROM site_schema.%I', view_record.table_name);
+        analysis_view_count := analysis_view_count + 1;
+    END LOOP;
+
+    -- Execute for master_site_relevant_analysis_view
+    IF analysis_view_count > 0 THEN
+        EXECUTE analysis;
+    ELSE
+        RAISE NOTICE 'No site-specific relevant analysis views found to include in master_site_relevant_analysis_view.';
+    END IF;
+END;
+$BODY$ 
+
+