Merge pull request #45 from DataKitchen/release/4.16.3

Release/4.16.3

Author: datakitchen-devops

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "4.12.6"
+version = "4.16.3"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },

testgen/commands/queries/generate_tests_query.py

@@ -43,6 +43,7 @@ def ReplaceParms(self, strInputString):
         strInputString = strInputString.replace("{GENERATION_SET}", self.generation_set)
         strInputString = strInputString.replace("{AS_OF_DATE}", self.as_of_date)
         strInputString = strInputString.replace("{DATA_SCHEMA}", self.data_schema)
+        strInputString = strInputString.replace("{ID_SEPARATOR}", "`" if self.sql_flavor == "databricks" else '"')
 
         return strInputString
 

testgen/commands/run_execute_cat_tests.py

@@ -63,7 +63,7 @@ def ParseCATResults(clsCATExecute):
     RunActionQueryList("DKTG", [strQuery])
 
 
-def FinalizeTestRun(clsCATExecute: CCATExecutionSQL):
+def FinalizeTestRun(clsCATExecute: CCATExecutionSQL, username: str | None = None):
     _, row_counts = RunActionQueryList(("DKTG"), [
         clsCATExecute.FinalizeTestResultsSQL(),
         clsCATExecute.PushTestRunStatusUpdateSQL(),
@@ -89,6 +89,7 @@ def FinalizeTestRun(clsCATExecute: CCATExecutionSQL):
     MixpanelService().send_event(
         "run-tests",
         source=settings.ANALYTICS_JOB_SOURCE,
+        username=username,
         sql_flavor=clsCATExecute.flavor,
         test_count=row_counts[0],
         run_duration=(end_time - date_service.parse_now(clsCATExecute.run_date)).total_seconds(),
@@ -97,7 +98,7 @@ def FinalizeTestRun(clsCATExecute: CCATExecutionSQL):
 
 
 def run_cat_test_queries(
-    dctParms, strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, minutes_offset=0, spinner=None
+    dctParms, strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, username=None, minutes_offset=0, spinner=None
 ):
     booErrors = False
 
@@ -167,4 +168,4 @@ def run_cat_test_queries(
 
     finally:
         LOG.info("Finalizing test run")
-        FinalizeTestRun(clsCATExecute)
+        FinalizeTestRun(clsCATExecute, username)

testgen/commands/run_execute_tests.py

@@ -18,6 +18,7 @@
     date_service,
 )
 from testgen.common.database.database_service import ExecuteDBQuery, empty_cache
+from testgen.ui.session import session
 
 from .run_execute_cat_tests import run_cat_test_queries
 from .run_refresh_data_chars import run_refresh_data_chars_queries
@@ -109,7 +110,7 @@ def run_execution_steps_in_background(project_code, test_suite):
         empty_cache()
         background_thread = threading.Thread(
             target=run_execution_steps,
-            args=(project_code, test_suite),
+            args=(project_code, test_suite, session.username),
         )
         background_thread.start()
     else:
@@ -121,6 +122,7 @@ def run_execution_steps_in_background(project_code, test_suite):
 def run_execution_steps(
     project_code: str,
     test_suite: str,
+    username: str | None = None,
     minutes_offset: int=0,
     spinner: Spinner=None,
 ) -> str:
@@ -178,7 +180,7 @@ def run_execution_steps(
 
     LOG.info("CurrentStep: Execute Step - CAT Test Execution")
     if run_cat_test_queries(
-        test_exec_params, test_run_id, test_time, project_code, test_suite, error_msg, minutes_offset, spinner
+        test_exec_params, test_run_id, test_time, project_code, test_suite, error_msg, username, minutes_offset, spinner
     ):
         has_errors = True
 

testgen/commands/run_get_entities.py

@@ -1,7 +1,6 @@
 import logging
 
 from testgen.common import RetrieveDBResultsToList, read_template_sql_file
-from testgen.common.encrypt import DecryptText
 
 LOG = logging.getLogger("testgen")
 
@@ -29,17 +28,6 @@ def run_list_connections():
     return RetrieveDBResultsToList("DKTG", sql_template)
 
 
-def run_get_connection(connection_id):
-    sql_template = read_template_sql_file("get_connection.sql", "get_entities")
-    sql_template = sql_template.replace("{CONNECTION_ID}", str(connection_id))
-    rows, _ = RetrieveDBResultsToList("DKTG", sql_template)
-    connection = rows.pop()._asdict()
-    connection["password"] = DecryptText(connection["project_pw_encrypted"]) if connection["project_pw_encrypted"] else None
-    connection["private_key"] = DecryptText(connection["private_key"]) if connection["private_key"] else None
-    connection["private_key_passphrase"] = DecryptText(connection["private_key_passphrase"]) if connection["private_key_passphrase"] else ""
-    return connection
-
-
 def run_table_group_list(project_code):
     sql_template = read_template_sql_file("get_table_group_list.sql", "get_entities")
     sql_template = sql_template.replace("{PROJECT_CODE}", project_code)

testgen/commands/run_profiling_bridge.py

@@ -22,6 +22,7 @@
 )
 from testgen.common.database.database_service import empty_cache
 from testgen.common.mixpanel_service import MixpanelService
+from testgen.ui.session import session
 
 booClean = True
 LOG = logging.getLogger("testgen")
@@ -238,7 +239,7 @@ def run_profiling_in_background(table_group_id):
         empty_cache()
         background_thread = threading.Thread(
             target=run_profiling_queries,
-            args=(table_group_id,),
+            args=(table_group_id, session.username),
         )
         background_thread.start()
     else:
@@ -247,7 +248,7 @@ def run_profiling_in_background(table_group_id):
         subprocess.Popen(script)  # NOQA S603
 
 
-def run_profiling_queries(strTableGroupsID, spinner=None):
+def run_profiling_queries(strTableGroupsID, username=None, spinner=None):
     if strTableGroupsID is None:
         raise ValueError("Table Group ID was not specified")
 
@@ -518,6 +519,7 @@ def run_profiling_queries(strTableGroupsID, spinner=None):
         MixpanelService().send_event(
             "run-profiling",
             source=settings.ANALYTICS_JOB_SOURCE,
+            username=username,
             sql_flavor=clsProfiling.flavor,
             sampling=clsProfiling.profile_use_sampling == "Y",
             table_count=table_count,

testgen/template/dbsetup/030_initialize_new_schema_structure.sql

@@ -101,10 +101,11 @@ CREATE TABLE table_groups
     profile_use_sampling     VARCHAR(3) DEFAULT 'N',
     profile_sample_percent   VARCHAR(3) DEFAULT '30',
     profile_sample_min_count BIGINT DEFAULT 100000,
-    profiling_delay_days     VARCHAR(3) DEFAULT '0' ,
+    profiling_delay_days     VARCHAR(3) DEFAULT '0',
     profile_flag_cdes        BOOLEAN DEFAULT TRUE,
     profile_do_pair_rules    VARCHAR(3) DEFAULT 'N',
     profile_pair_rule_pct    INTEGER DEFAULT 95,
+    include_in_dashboard     BOOLEAN DEFAULT TRUE,
     description              VARCHAR(1000),
     data_source              VARCHAR(40),
     source_system            VARCHAR(40),