From 7c02f96e0de40125c92567e246622c3f1aa22e80 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 22 Feb 2021 15:45:07 -0500 Subject: [PATCH 01/19] uploading new notebook --- .../purviewScan/databricksPurviewScan.py | 328 ++++++++++++++++++ 1 file changed, 328 insertions(+) create mode 100644 code/databricks/purviewScan/databricksPurviewScan.py diff --git a/code/databricks/purviewScan/databricksPurviewScan.py b/code/databricks/purviewScan/databricksPurviewScan.py new file mode 100644 index 0000000..2173fa2 --- /dev/null +++ b/code/databricks/purviewScan/databricksPurviewScan.py @@ -0,0 +1,328 @@ +# Databricks notebook source +# Install the required libraries +dbutils.library.installPyPI("pyapacheatlas") +dbutils.library.restartPython() + +# COMMAND ---------- + +# Leveraging parameters from the ADF pipeline +dbutils.widgets.text("tenantID","") +dbutils.widgets.text("clientID","") +dbutils.widgets.text("purviewAccountName","") +dbutils.widgets.text("dataLandingZoneName","") + +tenant_id = dbutils.widgets.get("tenantID") +client_id = dbutils.widgets.get("clientID") +purview_account_name = dbutils.widgets.get("purviewAccountName") +data_landing_zone_name = dbutils.widgets.get("dataLandingZoneName") + +# Fetch the client_secret from the Azure Key Vault +# specified in spark configuration +client_secret = spark.conf.get("spark.clientsecret") + +# COMMAND ---------- + +# Connect to Purview +import json +import os +from pyapacheatlas.auth import ServicePrincipalAuthentication +from pyapacheatlas.core import PurviewClient, AtlasEntity, AtlasProcess, TypeCategory +from pyapacheatlas.core.util import GuidTracker +from pyapacheatlas.core.typedef import AtlasAttributeDef, EntityTypeDef, RelationshipTypeDef + +oauth = ServicePrincipalAuthentication( + tenant_id=os.environ.get("TENANT_ID", tenant_id), + client_id=os.environ.get("CLIENT_ID", client_id), + client_secret=os.environ.get("CLIENT_SECRET", client_secret) + ) +client = PurviewClient( + account_name = os.environ.get("PURVIEW_NAME", purview_account_name), + authentication=oauth +) +guid = GuidTracker() + +# COMMAND ---------- + +# Set up type definitions + +# Databricks database type definition +type_databricks_database = EntityTypeDef( + name="databricks_database", + description="databricks_database", + superTypes = ["DataSet"], + relationshipAttributeDefs=[ + { + "name": "tables", + "typeName": "databricks_table", + "isOptional": True, + "cardinality": "SET", + "relationshipTypeName": "databricks_table_to_database", + "isLegacyAttribute": False + } + ] + ) + +#Databricks table type definition +type_databricks_table = EntityTypeDef( + name="databricks_table", + description="databricks_table", + attributeDefs=[ + AtlasAttributeDef(name="format") + ], + superTypes = ["DataSet"], + options = {"schemaElementAttribute":"columns"}, + relationshipAttributeDefs=[ + { + "name": "columns", + "typeName": "databricks_table_column", + "isOptional": True, + "cardinality": "SET", + "relationshipTypeName": "databricks_table_to_columns", + "isLegacyAttribute": False + }, + { + "name": "database", + "typeName": "databricks_database", + "isOptional": False, + "cardinality": "SINGLE", + "relationshipTypeName": "databricks_table_to_database", + "isLegacyAttribute": False + } + + ] + ) + +# Databricks table column type definition +type_databricks_columns = EntityTypeDef( + name="databricks_table_column", + description="databricks_table_column", + attributeDefs=[ + AtlasAttributeDef(name="data_type") + ], + superTypes = ["DataSet"], + relationshipAttributeDefs=[ + { + "name": "table", + "typeName": "databricks_table", + "isOptional": True, + "cardinality": "SET", + "relationshipTypeName": "databricks_table_to_columns", + "isLegacyAttribute": False + } + ] +) + +# Column to table relationship +databricks_column_to_table_relationship = RelationshipTypeDef( + name="databricks_table_to_columns", + relationshipCategory="COMPOSITION", + endDef1={ + "type": "databricks_table", + "name": "columns", + "isContainer": True, + "cardinality": "SET", + "isLegacyAttribute": False + }, + endDef2={ + "type": "databricks_table_column", + "name": "table", + "isContainer": False, + "cardinality": "SINGLE", + "isLegacyAttribute": False + } +) + +# Table to database relationship +databricks_table_to_database_relationship = RelationshipTypeDef( + name="databricks_table_to_database", + relationshipCategory="COMPOSITION", + endDef1={ + "type": "databricks_database", + "name": "tables", + "isContainer": True, + "cardinality": "SET", + "isLegacyAttribute": False + }, + endDef2={ + "type": "databricks_table", + "name": "database", + "isContainer": False, + "cardinality": "SINGLE", + "isLegacyAttribute": False + } +) + +# Upload the type definitions +# Note: This is a one-time upload +typedef_results = client.upload_typedefs( + entityDefs = [type_databricks_database, type_databricks_table, type_databricks_columns], + relationshipDefs = [databricks_table_to_database_relationship, databricks_column_to_table_relationship], + force_update=True) + +# COMMAND ---------- + +# Scan the databases in Databricks + +df_databases = spark.sql("SHOW DATABASES") +incoming_databases = df_databases.select("namespace").rdd.flatMap(lambda x: x).collect() +dict_tables = [] + +for database in incoming_databases: + spark.sql("USE {}".format(database)) + df_tables = spark.sql("SHOW TABLES") + dict_tables.append([row.asDict() for row in df_tables.collect()]) + +# Flatten the list of lists of dictionaries +dict_tables_flat = [val for sublist in dict_tables for val in sublist] + +# Create databases, tables and columns in Purview + +for dictionary in dict_tables_flat: + + # Filter out temporary tables + + if dictionary["isTemporary"] is False: + database_name = dictionary["database"] + + # Create an asset for the databricks table + + atlas_input_database = AtlasEntity( + name = database_name, + qualified_name = data_landing_zone_name+"://"+database_name, + typeName="databricks_database", + guid=guid.get_guid() + ) + table_name = dictionary["tableName"] + + # Create an asset for the databricks table + + atlas_input_table = AtlasEntity( + name = table_name, + qualified_name = data_landing_zone_name+"://"+database_name+"/"+table_name, + typeName="databricks_table", + relationshipAttributes = {"database":atlas_input_database.to_json(minimum=True)}, + guid=guid.get_guid() + ) + print("Table: "+table_name+" Database: "+database_name) + + # Create columns + + spark.sql("USE {}".format(database_name)) + df_columns = spark.sql("SHOW COLUMNS IN {}".format(table_name)) + df_columns.show() + df_description = spark.sql("DESCRIBE TABLE {}".format(table_name)) + + # Iterate over the input data frame's columns and create them + + table_columns = df_columns.select("col_name").rdd.flatMap(lambda x: x).collect() + atlas_input_table_columns = [] + + for each_column in table_columns: + + # Get the data type for this column + + column_data_type = df_description.filter("col_name == '{}'".format(each_column)).select("data_type").rdd.flatMap(lambda x: x).collect() + + # Create an asset for the table column + + temp_column = AtlasEntity( + name = each_column, + typeName = "databricks_table_column", + qualified_name = data_landing_zone_name+"://"+database_name+"/"+table_name+"#"+each_column, + guid=guid.get_guid(), + attributes = {"data_type": column_data_type[0]}, + relationshipAttributes = {"table":atlas_input_table.to_json(minimum=True)} + ) + atlas_input_table_columns.append(temp_column) + + batch = [atlas_input_database, atlas_input_table] + atlas_input_table_columns + + # Upload all newly created entities! + + client.upload_entities(batch=batch) + +# COMMAND ---------- + +# Clean up purview for any deleted or renamed assets + +# Fetch existing databricks databases in Purview using search and filter. + +existing_databases = [] +filter_setup = {"typeName": "databricks_database", "includeSubTypes": True} +search = client.search_entities("*", search_filter=filter_setup) + +for database_result in search: + existing_databases.append(database_result["name"]) + +# Clean up databases, tables & columns in Purview. + +for db in existing_databases: + + if db not in incoming_databases: + print("Deleted database in Purview: ", db) + db_guid = client.get_entity( + qualifiedName = data_landing_zone_name+"://"+db, + typeName="databricks_database" + )["entities"][0] + #print(json.dumps(table_guid["guid"], indent=2)) + client.delete_entity(guid=table_guid["guid"]) + else: + existing_tables = [] + filter_again = {"typeName": "databricks_table", "includeSubTypes": True} + search_again = client.search_entities("qualifiedName:"+db+"*", search_filter=filter_again) + + for table_result in search_again: + existing_tables.append(table_result["name"]) + + # Fetch incoming tables within this database + + spark.sql("USE {}".format(db)) + df_db_tables = spark.sql("SHOW TABLES") + incoming_tables = df_db_tables.select("tableName").rdd.flatMap(lambda x: x).collect() + print("database: ", db) + print("existing tables: ", existing_tables) + print("incoming_tables:", incoming_tables) + + # Removed deleted or renamed tables from Purview + + for tbl in existing_tables: + if tbl not in incoming_tables: + print("Deleted table in Purview: ", tbl) + table_guid = client.get_entity( + qualifiedName = data_landing_zone_name+"://"+db+"/"+tbl, + typeName="databricks_table" + )["entities"][0] + #print(json.dumps(table_guid["guid"], indent=2)) + client.delete_entity(guid=table_guid["guid"]) + else: # Let's look at the columns! + df_tbl_columns = spark.sql("SHOW COLUMNS IN {}".format(tbl)) + df_tbl_description = spark.sql("DESCRIBE TABLE {}".format(tbl)) # do we need this?? + incoming_columns = df_tbl_columns.select("col_name").rdd.flatMap(lambda x: x).collect() + + # Fetch existing columns + + existing_columns = [] + deleted_columns = [] # ?? + purview_columns = client.get_entity( + qualifiedName = data_landing_zone_name+"://"+db+"/"+tbl, + typeName="databricks_table" + )["entities"][0]["relationshipAttributes"] + + # Get the names of all the existing columns + + for col_name in purview_columns["columns"]: + + existing_columns.append(col_name["displayText"]) + + # Clean up deleted columns in Purview + + for purview_col in existing_columns: + if purview_col not in incoming_columns: + print("Deleted column in "+db+"/"+tbl+": "+purview_col) + column_guid = client.get_entity( + qualifiedName = data_landing_zone_name+"://"+db+"/"+tbl+"#"+purview_col, + typeName="databricks_table_column" + )["entities"][0] + client.delete_entity(guid=column_guid["guid"]) + + print("-----------------") From 5e8c6f03d5c47a4474a3bcb5d5b20858ab60bfe8 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 22 Feb 2021 16:39:58 -0500 Subject: [PATCH 02/19] removing library in favour of alternative method --- code/databricks/purviewScan/databricksPurviewScan.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/code/databricks/purviewScan/databricksPurviewScan.py b/code/databricks/purviewScan/databricksPurviewScan.py index 2173fa2..54d0c6b 100644 --- a/code/databricks/purviewScan/databricksPurviewScan.py +++ b/code/databricks/purviewScan/databricksPurviewScan.py @@ -1,9 +1,4 @@ # Databricks notebook source -# Install the required libraries -dbutils.library.installPyPI("pyapacheatlas") -dbutils.library.restartPython() - -# COMMAND ---------- # Leveraging parameters from the ADF pipeline dbutils.widgets.text("tenantID","") From b429e4aeed46dded9cbb59148f59ba5b824f3821 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Tue, 23 Feb 2021 14:40:03 -0500 Subject: [PATCH 03/19] deploy arm template --- infra/DataFactory/deploy.dataFactory.json | 159 ++++++++++++++++++- infra/DataFactory/params.dataFactory003.json | 45 ++++++ 2 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 infra/DataFactory/params.dataFactory003.json diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 363b037..1e97737 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -84,6 +84,27 @@ "metadata": { "description": "Specifies the ID of the private dns zone for data factory portal." } + }, + // Parameters from Databricks Table Scan for Purview + "databricksLinkedService_accessToken": { + "type": "secureString", + "metadata": "Secure string for 'accessToken' of 'databricksLinkedService'" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { + "type": "string", + "defaultValue": "{tenand-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { + "type": "string", + "defaultValue": "{client-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { + "type": "string", + "defaultValue": "{purview-account}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { + "type": "string", + "defaultValue": "{data-landing-zone}" } }, "functions": [], @@ -107,7 +128,8 @@ "privateDnsZoneIdDataFactory": "[parameters('privateDnsZoneIdDataFactory')]", "privateDnsZoneIdPortal": "[parameters('privateDnsZoneIdPortal')]", "privateEndpointNameDataFactory": "[concat(variables('dataFactoryName'), '-datafactory-private-endpoint')]", - "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]" + "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]", + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]" }, "resources": [ { @@ -415,6 +437,141 @@ }, "subscriptionId": "[variables('keyVaultSubscriptionId')]", "resourceGroup": "[variables('keyVaultResourceGroupName')]" + }, + // Resources for Databricks Table Scan for Purview + { + "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "notebookPurviewScan", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebookPath": "/databricksPurviewScan", + "baseParameters": { + "tenantID": { + "value": "@pipeline().parameters.tenantID", + "type": "Expression" + }, + "clientID": { + "value": "@pipeline().parameters.clientID", + "type": "Expression" + }, + "purviewAccountName": { + "value": "@pipeline().parameters.purviewAccountName", + "type": "Expression" + }, + "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" + }, + "libraries": [ + { + "pypi": { + "package": "pyapacheatlas" + } + } + ] + }, + "linkedServiceName": { + "referenceName": "databricksLinkedService", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "tenantID": { + "type": "string" + }, + "clientID": { + "type": "string" + }, + "purviewAccountName": { + "type": "string" + }, + "dataLandingZoneName": { + "type": "string" + } + }, + "annotations": [], + "lastPublishTime": "2021-02-10T22:35:54Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]" + ] + }, + { + "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "description": "Runs every ten minutes, not activated by default", + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "databricksTableScanPipeline", + "type": "PipelineReference" + }, + "parameters": { + "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", + "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", + "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", + "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" + } + } + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 10, + "startTime": "2021-02-10T19:37:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" + ] + }, + { + "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureDatabricks", + "typeProperties": { + // todo: modify path to point to databricks workspace url + "domain": "https://adb-947245053288056.16.azuredatabricks.net", + "accessToken": { + "type": "SecureString", + "value": "[parameters('databricksLinkedService_accessToken')]" + }, + "newClusterNodeType": "Standard_DS3_v2", + "newClusterNumOfWorker": "2", + "newClusterSparkConf": { + // todo: modify path to purview client secret in key vault + "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + }, + "newClusterSparkEnvVars": { + "PYSPARK_PYTHON": "/databricks/python3/bin/python3" + }, + "newClusterVersion": "7.3.x-scala2.12" + } + }, + "dependsOn": [] } ], "outputs": { diff --git a/infra/DataFactory/params.dataFactory003.json b/infra/DataFactory/params.dataFactory003.json new file mode 100644 index 0000000..0a1ef52 --- /dev/null +++ b/infra/DataFactory/params.dataFactory003.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "location": { + "value": "northeurope" + }, + "dataFactoryName": { + "value": "dn001-datafactory001" + }, + "dataFactoryGitAccount": { + "value": "" + }, + "dataFactoryGitRepo": { + "value": "" + }, + "dataFactoryGitCollaborationBranch": { + "value": "" + }, + "dataFactoryGitRootFolder": { + "value": "" + }, + "dataFactoryGitType": { + "value": "FactoryGitHubConfiguration" + }, + "keyVaultId": { + "value": "/subscriptions/2f68ca09-59d9-4ab5-ad11-c54872bfa28d/resourceGroups/dn001-metadata/providers/Microsoft.KeyVault/vaults/dn001-keyvault001" + }, + "sqlServerId": { + "value": "/subscriptions/2f68ca09-59d9-4ab5-ad11-c54872bfa28d/resourceGroups/dn001-metadata/providers/Microsoft.Sql/servers/dn001-sqlserver001" + }, + "sqlDatabaseId": { + "value": "/subscriptions/2f68ca09-59d9-4ab5-ad11-c54872bfa28d/resourceGroups/dn001-metadata/providers/Microsoft.Sql/servers/dn001-sqlserver001/databases/AdfMetastoreDB" + }, + "subnetId": { + "value": "/subscriptions/2f68ca09-59d9-4ab5-ad11-c54872bfa28d/resourceGroups/dn001-network/providers/Microsoft.Network/virtualNetworks/dn001-vnet/subnets/dn001-privatelink-subnet" + }, + "privateDnsZoneIdDataFactory": { + "value": "/subscriptions/4060c03e-0d2e-44b7-82a3-da9376fe50b2/resourceGroups/dh-global-dns/providers/Microsoft.Network/privateDnsZones/privatelink.datafactory.azure.net" + }, + "privateDnsZoneIdPortal": { + "value": "/subscriptions/4060c03e-0d2e-44b7-82a3-da9376fe50b2/resourceGroups/dh-global-dns/providers/Microsoft.Network/privateDnsZones/privatelink.adf.azure.com" + } + } +} \ No newline at end of file From d75e895a20575a330e04a4b1db625b9ab0c2094f Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Tue, 23 Feb 2021 14:42:43 -0500 Subject: [PATCH 04/19] arm parameters file --- infra/DataFactory/params.dataFactory003.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/infra/DataFactory/params.dataFactory003.json b/infra/DataFactory/params.dataFactory003.json index 0a1ef52..0d72db6 100644 --- a/infra/DataFactory/params.dataFactory003.json +++ b/infra/DataFactory/params.dataFactory003.json @@ -6,7 +6,7 @@ "value": "northeurope" }, "dataFactoryName": { - "value": "dn001-datafactory001" + "value": "dn001-datafactory003" }, "dataFactoryGitAccount": { "value": "" @@ -40,6 +40,21 @@ }, "privateDnsZoneIdPortal": { "value": "/subscriptions/4060c03e-0d2e-44b7-82a3-da9376fe50b2/resourceGroups/dh-global-dns/providers/Microsoft.Network/privateDnsZones/privatelink.adf.azure.com" + }, + "databricksLinkedService_accessToken": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { + "value": "{tenand-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { + "value": "{client-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { + "value": "{purview-account}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { + "value": "{data-landing-zone}" } } } \ No newline at end of file From fb293e453896d50defefd0374d1185c4af0b7545 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Wed, 24 Feb 2021 16:10:17 -0500 Subject: [PATCH 05/19] minor comment typo --- code/databricks/purviewScan/databricksPurviewScan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/databricks/purviewScan/databricksPurviewScan.py b/code/databricks/purviewScan/databricksPurviewScan.py index 54d0c6b..cd25eb7 100644 --- a/code/databricks/purviewScan/databricksPurviewScan.py +++ b/code/databricks/purviewScan/databricksPurviewScan.py @@ -179,7 +179,7 @@ if dictionary["isTemporary"] is False: database_name = dictionary["database"] - # Create an asset for the databricks table + # Create an asset for the databricks databricks atlas_input_database = AtlasEntity( name = database_name, From 3ca3f3012c2442377660170effe8a5ce8e22f6dc Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Fri, 26 Feb 2021 11:39:39 -0500 Subject: [PATCH 06/19] removed comments --- infra/DataFactory/deploy.dataFactory.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 1e97737..c154671 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -85,7 +85,6 @@ "description": "Specifies the ID of the private dns zone for data factory portal." } }, - // Parameters from Databricks Table Scan for Purview "databricksLinkedService_accessToken": { "type": "secureString", "metadata": "Secure string for 'accessToken' of 'databricksLinkedService'" @@ -438,7 +437,6 @@ "subscriptionId": "[variables('keyVaultSubscriptionId')]", "resourceGroup": "[variables('keyVaultResourceGroupName')]" }, - // Resources for Databricks Table Scan for Purview { "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", "type": "Microsoft.DataFactory/factories/pipelines", @@ -553,8 +551,7 @@ "annotations": [], "type": "AzureDatabricks", "typeProperties": { - // todo: modify path to point to databricks workspace url - "domain": "https://adb-947245053288056.16.azuredatabricks.net", + "domain": "{databricks-uri}", "accessToken": { "type": "SecureString", "value": "[parameters('databricksLinkedService_accessToken')]" @@ -562,8 +559,7 @@ "newClusterNodeType": "Standard_DS3_v2", "newClusterNumOfWorker": "2", "newClusterSparkConf": { - // todo: modify path to purview client secret in key vault - "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + "spark.clientsecret": "{secret}" }, "newClusterSparkEnvVars": { "PYSPARK_PYTHON": "/databricks/python3/bin/python3" From 50d586fb654ad6c2ff715b33126ad84fb83baba2 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 15:25:46 -0500 Subject: [PATCH 07/19] changed schema version --- infra/DataFactory/deploy.dataFactory.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index c154671..9e26ad9 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -1,5 +1,6 @@ { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + //"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "location": { @@ -413,7 +414,7 @@ "resources": [ { "type": "Microsoft.KeyVault/vaults/accessPolicies", - "apiVersion": "2019-09-01", + "apiVersion": "2016-10-01", "name": "[concat(variables('keyVaultName'), '/add')]", "properties": { "accessPolicies": [ From 3bb61c7416468bc2b5128f538ad2e86d8218c48b Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 15:46:37 -0500 Subject: [PATCH 08/19] schema changes --- infra/DataFactory/deploy.dataFactory.json | 49 ++--------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 9e26ad9..4b55d40 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -1,6 +1,5 @@ { - //"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "location": { @@ -16,40 +15,6 @@ "description": "Specifies the name of the data factory." } }, - "dataFactoryGitAccount": { - "type": "string", - "metadata": { - "description": "Specifies the account name for the repository connection." - } - }, - "dataFactoryGitRepo": { - "type": "string", - "metadata": { - "description": "Specifies the repo name for the repository connection." - } - }, - "dataFactoryGitCollaborationBranch": { - "type": "string", - "metadata": { - "description": "Specifies the collaboration branch name for the repository connection." - } - }, - "dataFactoryGitRootFolder": { - "type": "string", - "metadata": { - "description": "Specifies the root folder in the branch for the repository connection." - } - }, - "dataFactoryGitType": { - "type": "string", - "allowedValues": [ - "FactoryGitHubConfiguration", - "FactoryVSTSConfiguration" - ], - "metadata": { - "description": "Specifies the type of git connection." - } - }, "keyVaultId": { "type": "string", "metadata": { @@ -87,8 +52,7 @@ } }, "databricksLinkedService_accessToken": { - "type": "secureString", - "metadata": "Secure string for 'accessToken' of 'databricksLinkedService'" + "type": "secureString" }, "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { "type": "string", @@ -112,11 +76,6 @@ "location": "[parameters('location')]", "dataFactoryName": "[parameters('dataFactoryName')]", "azureManagedVnetIntegrationRuntimeName": "azureManagedVnetIntegrationRuntime", - "dataFactoryGitAccount": "[parameters('dataFactoryGitAccount')]", - "dataFactoryGitRepo": "[parameters('dataFactoryGitRepo')]", - "dataFactoryGitCollaborationBranch": "[parameters('dataFactoryGitCollaborationBranch')]", - "dataFactoryGitRootFolder": "[parameters('dataFactoryGitRootFolder')]", - "dataFactoryGitType": "[parameters('dataFactoryGitType')]", "keyVaultId": "[parameters('keyVaultId')]", "keyVaultName": "[if(empty(variables('keyVaultId')), 'keyVaultNotSet', last(split(variables('keyVaultId'), '/')))]", "keyVaultSubscriptionId": "[if(empty(variables('keyVaultId')), subscription().subscriptionId, split(variables('keyVaultId'), '/')[2])]", @@ -211,7 +170,7 @@ }, { "condition": "[not(empty(variables('keyVaultId')))]", - "type": "managedVirtualNetworks/managedPrivateEndpoints", + "type": "managedPrivateEndpoints", "apiVersion": "2018-06-01", "name": "[concat('default/', replace(variables('keyVaultName'), '-', ''))]", "dependsOn": [ @@ -252,7 +211,7 @@ }, { "condition": "[not(empty(variables('sqlDatabaseId')))]", - "type": "managedVirtualNetworks/managedPrivateEndpoints", + "type": "managedPrivateEndpoints", "apiVersion": "2018-06-01", "name": "[concat('default/', replace(variables('sqlDatabaseName'), '-', ''))]", "dependsOn": [ From bea2258e990be96d512304803152040aaa76f67b Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 15:59:18 -0500 Subject: [PATCH 09/19] undoing template adds --- infra/DataFactory/deploy.dataFactory.json | 199 +++++----------------- 1 file changed, 43 insertions(+), 156 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 4b55d40..363b037 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -15,6 +15,40 @@ "description": "Specifies the name of the data factory." } }, + "dataFactoryGitAccount": { + "type": "string", + "metadata": { + "description": "Specifies the account name for the repository connection." + } + }, + "dataFactoryGitRepo": { + "type": "string", + "metadata": { + "description": "Specifies the repo name for the repository connection." + } + }, + "dataFactoryGitCollaborationBranch": { + "type": "string", + "metadata": { + "description": "Specifies the collaboration branch name for the repository connection." + } + }, + "dataFactoryGitRootFolder": { + "type": "string", + "metadata": { + "description": "Specifies the root folder in the branch for the repository connection." + } + }, + "dataFactoryGitType": { + "type": "string", + "allowedValues": [ + "FactoryGitHubConfiguration", + "FactoryVSTSConfiguration" + ], + "metadata": { + "description": "Specifies the type of git connection." + } + }, "keyVaultId": { "type": "string", "metadata": { @@ -50,25 +84,6 @@ "metadata": { "description": "Specifies the ID of the private dns zone for data factory portal." } - }, - "databricksLinkedService_accessToken": { - "type": "secureString" - }, - "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { - "type": "string", - "defaultValue": "{tenand-id}" - }, - "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { - "type": "string", - "defaultValue": "{client-id}" - }, - "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { - "type": "string", - "defaultValue": "{purview-account}" - }, - "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { - "type": "string", - "defaultValue": "{data-landing-zone}" } }, "functions": [], @@ -76,6 +91,11 @@ "location": "[parameters('location')]", "dataFactoryName": "[parameters('dataFactoryName')]", "azureManagedVnetIntegrationRuntimeName": "azureManagedVnetIntegrationRuntime", + "dataFactoryGitAccount": "[parameters('dataFactoryGitAccount')]", + "dataFactoryGitRepo": "[parameters('dataFactoryGitRepo')]", + "dataFactoryGitCollaborationBranch": "[parameters('dataFactoryGitCollaborationBranch')]", + "dataFactoryGitRootFolder": "[parameters('dataFactoryGitRootFolder')]", + "dataFactoryGitType": "[parameters('dataFactoryGitType')]", "keyVaultId": "[parameters('keyVaultId')]", "keyVaultName": "[if(empty(variables('keyVaultId')), 'keyVaultNotSet', last(split(variables('keyVaultId'), '/')))]", "keyVaultSubscriptionId": "[if(empty(variables('keyVaultId')), subscription().subscriptionId, split(variables('keyVaultId'), '/')[2])]", @@ -87,8 +107,7 @@ "privateDnsZoneIdDataFactory": "[parameters('privateDnsZoneIdDataFactory')]", "privateDnsZoneIdPortal": "[parameters('privateDnsZoneIdPortal')]", "privateEndpointNameDataFactory": "[concat(variables('dataFactoryName'), '-datafactory-private-endpoint')]", - "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]", - "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]" + "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]" }, "resources": [ { @@ -170,7 +189,7 @@ }, { "condition": "[not(empty(variables('keyVaultId')))]", - "type": "managedPrivateEndpoints", + "type": "managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "name": "[concat('default/', replace(variables('keyVaultName'), '-', ''))]", "dependsOn": [ @@ -211,7 +230,7 @@ }, { "condition": "[not(empty(variables('sqlDatabaseId')))]", - "type": "managedPrivateEndpoints", + "type": "managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "name": "[concat('default/', replace(variables('sqlDatabaseName'), '-', ''))]", "dependsOn": [ @@ -373,7 +392,7 @@ "resources": [ { "type": "Microsoft.KeyVault/vaults/accessPolicies", - "apiVersion": "2016-10-01", + "apiVersion": "2019-09-01", "name": "[concat(variables('keyVaultName'), '/add')]", "properties": { "accessPolicies": [ @@ -396,138 +415,6 @@ }, "subscriptionId": "[variables('keyVaultSubscriptionId')]", "resourceGroup": "[variables('keyVaultResourceGroupName')]" - }, - { - "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "notebookPurviewScan", - "type": "DatabricksNotebook", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 2, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "notebookPath": "/databricksPurviewScan", - "baseParameters": { - "tenantID": { - "value": "@pipeline().parameters.tenantID", - "type": "Expression" - }, - "clientID": { - "value": "@pipeline().parameters.clientID", - "type": "Expression" - }, - "purviewAccountName": { - "value": "@pipeline().parameters.purviewAccountName", - "type": "Expression" - }, - "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" - }, - "libraries": [ - { - "pypi": { - "package": "pyapacheatlas" - } - } - ] - }, - "linkedServiceName": { - "referenceName": "databricksLinkedService", - "type": "LinkedServiceReference" - } - } - ], - "parameters": { - "tenantID": { - "type": "string" - }, - "clientID": { - "type": "string" - }, - "purviewAccountName": { - "type": "string" - }, - "dataLandingZoneName": { - "type": "string" - } - }, - "annotations": [], - "lastPublishTime": "2021-02-10T22:35:54Z" - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]" - ] - }, - { - "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", - "type": "Microsoft.DataFactory/factories/triggers", - "apiVersion": "2018-06-01", - "properties": { - "description": "Runs every ten minutes, not activated by default", - "annotations": [], - "runtimeState": "Stopped", - "pipelines": [ - { - "pipelineReference": { - "referenceName": "databricksTableScanPipeline", - "type": "PipelineReference" - }, - "parameters": { - "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", - "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", - "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", - "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" - } - } - ], - "type": "ScheduleTrigger", - "typeProperties": { - "recurrence": { - "frequency": "Minute", - "interval": 10, - "startTime": "2021-02-10T19:37:00Z", - "timeZone": "UTC" - } - } - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" - ] - }, - { - "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", - "type": "Microsoft.DataFactory/factories/linkedServices", - "apiVersion": "2018-06-01", - "properties": { - "annotations": [], - "type": "AzureDatabricks", - "typeProperties": { - "domain": "{databricks-uri}", - "accessToken": { - "type": "SecureString", - "value": "[parameters('databricksLinkedService_accessToken')]" - }, - "newClusterNodeType": "Standard_DS3_v2", - "newClusterNumOfWorker": "2", - "newClusterSparkConf": { - "spark.clientsecret": "{secret}" - }, - "newClusterSparkEnvVars": { - "PYSPARK_PYTHON": "/databricks/python3/bin/python3" - }, - "newClusterVersion": "7.3.x-scala2.12" - } - }, - "dependsOn": [] } ], "outputs": { From 401182b4499421a209fc593f2c4a0e6c48d5f456 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 16:17:10 -0500 Subject: [PATCH 10/19] removing missing variables --- infra/DataFactory/deploy.dataFactory.json | 48 +---------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 363b037..333c2d3 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -14,41 +14,7 @@ "metadata": { "description": "Specifies the name of the data factory." } - }, - "dataFactoryGitAccount": { - "type": "string", - "metadata": { - "description": "Specifies the account name for the repository connection." - } - }, - "dataFactoryGitRepo": { - "type": "string", - "metadata": { - "description": "Specifies the repo name for the repository connection." - } - }, - "dataFactoryGitCollaborationBranch": { - "type": "string", - "metadata": { - "description": "Specifies the collaboration branch name for the repository connection." - } - }, - "dataFactoryGitRootFolder": { - "type": "string", - "metadata": { - "description": "Specifies the root folder in the branch for the repository connection." - } - }, - "dataFactoryGitType": { - "type": "string", - "allowedValues": [ - "FactoryGitHubConfiguration", - "FactoryVSTSConfiguration" - ], - "metadata": { - "description": "Specifies the type of git connection." - } - }, + }, "keyVaultId": { "type": "string", "metadata": { @@ -91,11 +57,6 @@ "location": "[parameters('location')]", "dataFactoryName": "[parameters('dataFactoryName')]", "azureManagedVnetIntegrationRuntimeName": "azureManagedVnetIntegrationRuntime", - "dataFactoryGitAccount": "[parameters('dataFactoryGitAccount')]", - "dataFactoryGitRepo": "[parameters('dataFactoryGitRepo')]", - "dataFactoryGitCollaborationBranch": "[parameters('dataFactoryGitCollaborationBranch')]", - "dataFactoryGitRootFolder": "[parameters('dataFactoryGitRootFolder')]", - "dataFactoryGitType": "[parameters('dataFactoryGitType')]", "keyVaultId": "[parameters('keyVaultId')]", "keyVaultName": "[if(empty(variables('keyVaultId')), 'keyVaultNotSet', last(split(variables('keyVaultId'), '/')))]", "keyVaultSubscriptionId": "[if(empty(variables('keyVaultId')), subscription().subscriptionId, split(variables('keyVaultId'), '/')[2])]", @@ -119,13 +80,6 @@ "type": "SystemAssigned" }, "properties": { - // "repoConfiguration": { - // "accountName": "[variables('dataFactoryGitAccount')]", - // "repositoryName": "[variables('dataFactoryGitRepo')]", - // "collaborationBranch": "[variables('dataFactoryGitCollaborationBranch')]", - // "rootFolder": "[variables('dataFactoryGitRootFolder')]", - // "type": "[variables('dataFactoryGitType')]" - // }, "globalParameters": { }, "publicNetworkAccess": "Disabled" From eae650bf503cc010ba81850b3f375cd9ef051704 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 17:07:58 -0500 Subject: [PATCH 11/19] step1 --- infra/DataFactory/deploy.dataFactory.json | 175 ++++++++++++++++++++-- 1 file changed, 161 insertions(+), 14 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 333c2d3..4f09b8a 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -2,6 +2,10 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { + "databricksLinkedService_accessToken": { + "type": "secureString", + "metadata": "Secure string for accessToken of databricksLinkedService" + }, "location": { "type": "string", "defaultValue": "[resourceGroup().location]", @@ -14,7 +18,41 @@ "metadata": { "description": "Specifies the name of the data factory." } - }, + }, + "dataFactoryGitAccount": { + "type": "string", + "metadata": { + "description": "Specifies the account name for the repository connection." + } + }, + "dataFactoryGitRepo": { + "type": "string", + "metadata": { + "description": "Specifies the repo name for the repository connection." + } + }, + "dataFactoryGitCollaborationBranch": { + "type": "string", + "metadata": { + "description": "Specifies the collaboration branch name for the repository connection." + } + }, + "dataFactoryGitRootFolder": { + "type": "string", + "metadata": { + "description": "Specifies the root folder in the branch for the repository connection." + } + }, + "dataFactoryGitType": { + "type": "string", + "allowedValues": [ + "FactoryGitHubConfiguration", + "FactoryVSTSConfiguration" + ], + "metadata": { + "description": "Specifies the type of git connection." + } + }, "keyVaultId": { "type": "string", "metadata": { @@ -54,9 +92,15 @@ }, "functions": [], "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]", "location": "[parameters('location')]", "dataFactoryName": "[parameters('dataFactoryName')]", "azureManagedVnetIntegrationRuntimeName": "azureManagedVnetIntegrationRuntime", + "dataFactoryGitAccount": "[parameters('dataFactoryGitAccount')]", + "dataFactoryGitRepo": "[parameters('dataFactoryGitRepo')]", + "dataFactoryGitCollaborationBranch": "[parameters('dataFactoryGitCollaborationBranch')]", + "dataFactoryGitRootFolder": "[parameters('dataFactoryGitRootFolder')]", + "dataFactoryGitType": "[parameters('dataFactoryGitType')]", "keyVaultId": "[parameters('keyVaultId')]", "keyVaultName": "[if(empty(variables('keyVaultId')), 'keyVaultNotSet', last(split(variables('keyVaultId'), '/')))]", "keyVaultSubscriptionId": "[if(empty(variables('keyVaultId')), subscription().subscriptionId, split(variables('keyVaultId'), '/')[2])]", @@ -71,20 +115,123 @@ "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]" }, "resources": [ - { - "type": "Microsoft.DataFactory/factories", - "apiVersion": "2018-06-01", - "name": "[variables('dataFactoryName')]", - "location": "[variables('location')]", - "identity": { - "type": "SystemAssigned" - }, - "properties": { - "globalParameters": { + { + "type": "Microsoft.DataFactory/factories", + "apiVersion": "2018-06-01", + "name": "[variables('dataFactoryName')]", + "location": "[variables('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + // "repoConfiguration": { + // "accountName": "[variables('dataFactoryGitAccount')]", + // "repositoryName": "[variables('dataFactoryGitRepo')]", + // "collaborationBranch": "[variables('dataFactoryGitCollaborationBranch')]", + // "rootFolder": "[variables('dataFactoryGitRootFolder')]", + // "type": "[variables('dataFactoryGitType')]" + // }, + "globalParameters": { + }, + "publicNetworkAccess": "Disabled" + }, + "resources": [ + { + "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureDatabricks", + "typeProperties": { + "domain": "https://adb-947245053288056.16.azuredatabricks.net", + "accessToken": { + "type": "SecureString", + "value": "[parameters('databricksLinkedService_accessToken')]" + }, + "newClusterNodeType": "Standard_DS3_v2", + "newClusterNumOfWorker": "2", + "newClusterSparkConf": { + "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + }, + "newClusterSparkEnvVars": { + "PYSPARK_PYTHON": "/databricks/python3/bin/python3" + }, + "newClusterVersion": "7.3.x-scala2.12" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "notebookPurviewScan", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebookPath": "/databricksPurviewScan", + "baseParameters": { + "tenantID": { + "value": "@pipeline().parameters.tenantID", + "type": "Expression" + }, + "clientID": { + "value": "@pipeline().parameters.clientID", + "type": "Expression" + }, + "purviewAccountName": { + "value": "@pipeline().parameters.purviewAccountName", + "type": "Expression" + }, + "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" + }, + "libraries": [ + { + "pypi": { + "package": "pyapacheatlas" + } + } + ] + }, + "linkedServiceName": { + "referenceName": "databricksLinkedService", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "tenantID": { + "type": "string" + }, + "clientID": { + "type": "string" + }, + "purviewAccountName": { + "type": "string" + }, + "dataLandingZoneName": { + "type": "string" + } + }, + "annotations": [], + "lastPublishTime": "2021-02-10T22:35:54Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]" + ] }, - "publicNetworkAccess": "Disabled" - }, - "resources": [ { "type": "managedVirtualNetworks", "apiVersion": "2018-06-01", From 402a86e84f1f93f68e115adf982d96b203212913 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 17:44:54 -0500 Subject: [PATCH 12/19] obj warning --- infra/DataFactory/deploy.dataFactory.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 4f09b8a..37a90fc 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -4,7 +4,9 @@ "parameters": { "databricksLinkedService_accessToken": { "type": "secureString", - "metadata": "Secure string for accessToken of databricksLinkedService" + "metadata": { + "description": "Secure string for accessToken of databricksLinkedService" + } }, "location": { "type": "string", From 53614ae4af752b7cd350cfd0e11d1e15fc953df3 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 18:03:28 -0500 Subject: [PATCH 13/19] db access token --- infra/DataFactory/params.dataFactory001.json | 3 +++ infra/DataFactory/params.dataFactory002.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/infra/DataFactory/params.dataFactory001.json b/infra/DataFactory/params.dataFactory001.json index 0a1ef52..347158c 100644 --- a/infra/DataFactory/params.dataFactory001.json +++ b/infra/DataFactory/params.dataFactory001.json @@ -40,6 +40,9 @@ }, "privateDnsZoneIdPortal": { "value": "/subscriptions/4060c03e-0d2e-44b7-82a3-da9376fe50b2/resourceGroups/dh-global-dns/providers/Microsoft.Network/privateDnsZones/privatelink.adf.azure.com" + }, + "databricksLinkedService_accessToken": { + "value": "" } } } \ No newline at end of file diff --git a/infra/DataFactory/params.dataFactory002.json b/infra/DataFactory/params.dataFactory002.json index bb34a10..45bd481 100644 --- a/infra/DataFactory/params.dataFactory002.json +++ b/infra/DataFactory/params.dataFactory002.json @@ -40,6 +40,9 @@ }, "privateDnsZoneIdPortal": { "value": "/subscriptions/4060c03e-0d2e-44b7-82a3-da9376fe50b2/resourceGroups/dh-global-dns/providers/Microsoft.Network/privateDnsZones/privatelink.adf.azure.com" + }, + "databricksLinkedService_accessToken": { + "value": "" } } } \ No newline at end of file From 97da7873cfd8b2a0e46a0a35a4399becf2c1c23e Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 18:09:13 -0500 Subject: [PATCH 14/19] dependency error fix --- infra/DataFactory/deploy.dataFactory.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 37a90fc..eeb83af 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -162,7 +162,9 @@ "newClusterVersion": "7.3.x-scala2.12" } }, - "dependsOn": [] + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" + ] }, { "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", From 738a4fe5cb25370ed99a9721515422a4d168d40b Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 18:14:16 -0500 Subject: [PATCH 15/19] yet another dependency fix --- infra/DataFactory/deploy.dataFactory.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index eeb83af..33bffd4 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -146,7 +146,7 @@ "annotations": [], "type": "AzureDatabricks", "typeProperties": { - "domain": "https://adb-947245053288056.16.azuredatabricks.net", + "domain": "{databricksWorkspaceUrl}", "accessToken": { "type": "SecureString", "value": "[parameters('databricksLinkedService_accessToken')]" @@ -233,7 +233,8 @@ "lastPublishTime": "2021-02-10T22:35:54Z" }, "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]" + "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]", + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" ] }, { From a1d0195796465401d3afcc7ec95086e0dcacc690 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 19:04:23 -0500 Subject: [PATCH 16/19] pipeline trigger resource --- infra/DataFactory/deploy.dataFactory.json | 152 +++++++++++++++---- infra/DataFactory/params.dataFactory001.json | 12 ++ infra/DataFactory/params.dataFactory002.json | 12 ++ 3 files changed, 144 insertions(+), 32 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 33bffd4..b15b335 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -2,6 +2,22 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { + "type": "string", + "defaultValue": "{tenand-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { + "type": "string", + "defaultValue": "{client-id}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { + "type": "string", + "defaultValue": "{purview-account}" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { + "type": "string", + "defaultValue": "{data-landing-zone}" + }, "databricksLinkedService_accessToken": { "type": "secureString", "metadata": { @@ -118,6 +134,42 @@ }, "resources": [ { + "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "description": "Runs every ten minutes, not activated by default", + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "databricksTableScanPipeline", + "type": "PipelineReference" + }, + "parameters": { + "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", + "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", + "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", + "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" + } + } + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 10, + "startTime": "2021-02-10T19:37:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" + ] + }, + { "type": "Microsoft.DataFactory/factories", "apiVersion": "2018-06-01", "name": "[variables('dataFactoryName')]", @@ -138,39 +190,75 @@ "publicNetworkAccess": "Disabled" }, "resources": [ - { - "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", - "type": "Microsoft.DataFactory/factories/linkedServices", - "apiVersion": "2018-06-01", - "properties": { - "annotations": [], - "type": "AzureDatabricks", - "typeProperties": { - "domain": "{databricksWorkspaceUrl}", - "accessToken": { - "type": "SecureString", - "value": "[parameters('databricksLinkedService_accessToken')]" - }, - "newClusterNodeType": "Standard_DS3_v2", - "newClusterNumOfWorker": "2", - "newClusterSparkConf": { - "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" - }, - "newClusterSparkEnvVars": { - "PYSPARK_PYTHON": "/databricks/python3/bin/python3" - }, - "newClusterVersion": "7.3.x-scala2.12" - } + { + "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "description": "Runs every ten minutes, not activated by default", + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "databricksTableScanPipeline", + "type": "PipelineReference" + }, + "parameters": { + "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", + "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", + "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", + "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" + } + } + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 10, + "startTime": "2021-02-10T19:37:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" + ] }, - "dependsOn": [ - "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" - ] - }, - { - "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { + { + "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureDatabricks", + "typeProperties": { + "domain": "{databricksWorkspaceUrl}", + "accessToken": { + "type": "SecureString", + "value": "[parameters('databricksLinkedService_accessToken')]" + }, + "newClusterNodeType": "Standard_DS3_v2", + "newClusterNumOfWorker": "2", + "newClusterSparkConf": { + "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + }, + "newClusterSparkEnvVars": { + "PYSPARK_PYTHON": "/databricks/python3/bin/python3" + }, + "newClusterVersion": "7.3.x-scala2.12" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" + ] + }, + { + "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { "activities": [ { "name": "notebookPurviewScan", diff --git a/infra/DataFactory/params.dataFactory001.json b/infra/DataFactory/params.dataFactory001.json index 347158c..5de6c49 100644 --- a/infra/DataFactory/params.dataFactory001.json +++ b/infra/DataFactory/params.dataFactory001.json @@ -43,6 +43,18 @@ }, "databricksLinkedService_accessToken": { "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { + "value": "" } } } \ No newline at end of file diff --git a/infra/DataFactory/params.dataFactory002.json b/infra/DataFactory/params.dataFactory002.json index 45bd481..4667100 100644 --- a/infra/DataFactory/params.dataFactory002.json +++ b/infra/DataFactory/params.dataFactory002.json @@ -43,6 +43,18 @@ }, "databricksLinkedService_accessToken": { "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName": { + "value": "" + }, + "runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName": { + "value": "" } } } \ No newline at end of file From f2bec0b724abf484363b4662bb30e48a9298c75b Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 19:22:03 -0500 Subject: [PATCH 17/19] removing dups --- infra/DataFactory/deploy.dataFactory.json | 314 ++++++++++------------ 1 file changed, 139 insertions(+), 175 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index b15b335..383fa04 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -133,197 +133,161 @@ "privateEndpointNamePortal": "[concat(variables('dataFactoryName'), '-portal-private-endpoint')]" }, "resources": [ + { + "type": "Microsoft.DataFactory/factories", + "apiVersion": "2018-06-01", + "name": "[variables('dataFactoryName')]", + "location": "[variables('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + // "repoConfiguration": { + // "accountName": "[variables('dataFactoryGitAccount')]", + // "repositoryName": "[variables('dataFactoryGitRepo')]", + // "collaborationBranch": "[variables('dataFactoryGitCollaborationBranch')]", + // "rootFolder": "[variables('dataFactoryGitRootFolder')]", + // "type": "[variables('dataFactoryGitType')]" + // }, + "globalParameters": { + }, + "publicNetworkAccess": "Disabled" + }, + "resources": [ + { + "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "description": "Runs every ten minutes, not activated by default", + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "databricksTableScanPipeline", + "type": "PipelineReference" + }, + "parameters": { + "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", + "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", + "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", + "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" + } + } + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 10, + "startTime": "2021-02-10T19:37:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" + ] + }, { - "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", - "type": "Microsoft.DataFactory/factories/triggers", + "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", + "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { - "description": "Runs every ten minutes, not activated by default", "annotations": [], - "runtimeState": "Stopped", - "pipelines": [ - { - "pipelineReference": { - "referenceName": "databricksTableScanPipeline", - "type": "PipelineReference" - }, - "parameters": { - "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", - "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", - "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", - "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" - } - } - ], - "type": "ScheduleTrigger", + "type": "AzureDatabricks", "typeProperties": { - "recurrence": { - "frequency": "Minute", - "interval": 10, - "startTime": "2021-02-10T19:37:00Z", - "timeZone": "UTC" - } + "domain": "{databricksWorkspaceUrl}", + "accessToken": { + "type": "SecureString", + "value": "[parameters('databricksLinkedService_accessToken')]" + }, + "newClusterNodeType": "Standard_DS3_v2", + "newClusterNumOfWorker": "2", + "newClusterSparkConf": { + "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + }, + "newClusterSparkEnvVars": { + "PYSPARK_PYTHON": "/databricks/python3/bin/python3" + }, + "newClusterVersion": "7.3.x-scala2.12" } }, "dependsOn": [ - "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" - ] - }, - { - "type": "Microsoft.DataFactory/factories", - "apiVersion": "2018-06-01", - "name": "[variables('dataFactoryName')]", - "location": "[variables('location')]", - "identity": { - "type": "SystemAssigned" - }, - "properties": { - // "repoConfiguration": { - // "accountName": "[variables('dataFactoryGitAccount')]", - // "repositoryName": "[variables('dataFactoryGitRepo')]", - // "collaborationBranch": "[variables('dataFactoryGitCollaborationBranch')]", - // "rootFolder": "[variables('dataFactoryGitRootFolder')]", - // "type": "[variables('dataFactoryGitType')]" - // }, - "globalParameters": { - }, - "publicNetworkAccess": "Disabled" - }, - "resources": [ + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" + ] + }, + { + "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ { - "name": "[concat(parameters('dataFactoryName'), '/runEveryTenMinutes')]", - "type": "Microsoft.DataFactory/factories/triggers", - "apiVersion": "2018-06-01", - "properties": { - "description": "Runs every ten minutes, not activated by default", - "annotations": [], - "runtimeState": "Stopped", - "pipelines": [ - { - "pipelineReference": { - "referenceName": "databricksTableScanPipeline", - "type": "PipelineReference" - }, - "parameters": { - "tenantID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_tenantID')]", - "clientID": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_clientID')]", - "purviewAccountName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_purviewAccountName')]", - "dataLandingZoneName": "[parameters('runEveryTenMinutes_properties_databricksTableScanPipeline_parameters_dataLandingZoneName')]" - } - } - ], - "type": "ScheduleTrigger", - "typeProperties": { - "recurrence": { - "frequency": "Minute", - "interval": 10, - "startTime": "2021-02-10T19:37:00Z", - "timeZone": "UTC" - } - } + "name": "notebookPurviewScan", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false }, - "dependsOn": [ - "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" - ] - }, - { - "name": "[concat(parameters('dataFactoryName'), '/databricksLinkedService')]", - "type": "Microsoft.DataFactory/factories/linkedServices", - "apiVersion": "2018-06-01", - "properties": { - "annotations": [], - "type": "AzureDatabricks", - "typeProperties": { - "domain": "{databricksWorkspaceUrl}", - "accessToken": { - "type": "SecureString", - "value": "[parameters('databricksLinkedService_accessToken')]" + "userProperties": [], + "typeProperties": { + "notebookPath": "/databricksPurviewScan", + "baseParameters": { + "tenantID": { + "value": "@pipeline().parameters.tenantID", + "type": "Expression" }, - "newClusterNodeType": "Standard_DS3_v2", - "newClusterNumOfWorker": "2", - "newClusterSparkConf": { - "spark.clientsecret": "{{secrets/purviewClient/clientSecretKV}}" + "clientID": { + "value": "@pipeline().parameters.clientID", + "type": "Expression" }, - "newClusterSparkEnvVars": { - "PYSPARK_PYTHON": "/databricks/python3/bin/python3" + "purviewAccountName": { + "value": "@pipeline().parameters.purviewAccountName", + "type": "Expression" }, - "newClusterVersion": "7.3.x-scala2.12" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" - ] - }, - { - "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "notebookPurviewScan", - "type": "DatabricksNotebook", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 2, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "notebookPath": "/databricksPurviewScan", - "baseParameters": { - "tenantID": { - "value": "@pipeline().parameters.tenantID", - "type": "Expression" - }, - "clientID": { - "value": "@pipeline().parameters.clientID", - "type": "Expression" - }, - "purviewAccountName": { - "value": "@pipeline().parameters.purviewAccountName", - "type": "Expression" - }, - "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" - }, - "libraries": [ - { - "pypi": { - "package": "pyapacheatlas" - } - } - ] - }, - "linkedServiceName": { - "referenceName": "databricksLinkedService", - "type": "LinkedServiceReference" - } - } - ], - "parameters": { - "tenantID": { - "type": "string" - }, - "clientID": { - "type": "string" + "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" }, - "purviewAccountName": { - "type": "string" - }, - "dataLandingZoneName": { - "type": "string" - } + "libraries": [ + { + "pypi": { + "package": "pyapacheatlas" + } + } + ] }, - "annotations": [], - "lastPublishTime": "2021-02-10T22:35:54Z" + "linkedServiceName": { + "referenceName": "databricksLinkedService", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "tenantID": { + "type": "string" }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]", - "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" - ] + "clientID": { + "type": "string" + }, + "purviewAccountName": { + "type": "string" + }, + "dataLandingZoneName": { + "type": "string" + } + }, + "annotations": [], + "lastPublishTime": "2021-02-10T22:35:54Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]", + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" + ] }, { "type": "managedVirtualNetworks", From ade964f329a1644c7401ef355d59dfa0489e022d Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Mon, 1 Mar 2021 19:27:20 -0500 Subject: [PATCH 18/19] dependsOn --- infra/DataFactory/deploy.dataFactory.json | 127 +++++++++++----------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index 383fa04..46a80cc 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -187,7 +187,8 @@ } }, "dependsOn": [ - "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]" + "[concat(variables('factoryId'), '/pipelines/databricksTableScanPipeline')]", + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" ] }, { @@ -219,75 +220,75 @@ ] }, { - "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "notebookPurviewScan", - "type": "DatabricksNotebook", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 2, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "notebookPath": "/databricksPurviewScan", - "baseParameters": { - "tenantID": { - "value": "@pipeline().parameters.tenantID", - "type": "Expression" - }, - "clientID": { - "value": "@pipeline().parameters.clientID", - "type": "Expression" - }, - "purviewAccountName": { - "value": "@pipeline().parameters.purviewAccountName", - "type": "Expression" - }, - "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" + "name": "[concat(parameters('dataFactoryName'), '/databricksTableScanPipeline')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "notebookPurviewScan", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 2, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false }, - "libraries": [ - { - "pypi": { - "package": "pyapacheatlas" + "userProperties": [], + "typeProperties": { + "notebookPath": "/databricksPurviewScan", + "baseParameters": { + "tenantID": { + "value": "@pipeline().parameters.tenantID", + "type": "Expression" + }, + "clientID": { + "value": "@pipeline().parameters.clientID", + "type": "Expression" + }, + "purviewAccountName": { + "value": "@pipeline().parameters.purviewAccountName", + "type": "Expression" + }, + "dataLandingZoneName": "@pipeline().parameters.dataLandingZoneName" + }, + "libraries": [ + { + "pypi": { + "package": "pyapacheatlas" + } } - } - ] + ] + }, + "linkedServiceName": { + "referenceName": "databricksLinkedService", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "tenantID": { + "type": "string" }, - "linkedServiceName": { - "referenceName": "databricksLinkedService", - "type": "LinkedServiceReference" + "clientID": { + "type": "string" + }, + "purviewAccountName": { + "type": "string" + }, + "dataLandingZoneName": { + "type": "string" } - } - ], - "parameters": { - "tenantID": { - "type": "string" }, - "clientID": { - "type": "string" - }, - "purviewAccountName": { - "type": "string" + "annotations": [], + "lastPublishTime": "2021-02-10T22:35:54Z" }, - "dataLandingZoneName": { - "type": "string" - } - }, - "annotations": [], - "lastPublishTime": "2021-02-10T22:35:54Z" - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]", + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]", "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]" - ] + ] }, { "type": "managedVirtualNetworks", From e38c9a76c2b27bea5f2f51e5474c40fea041ef62 Mon Sep 17 00:00:00 2001 From: Hamood Aleem Date: Tue, 2 Mar 2021 18:29:42 -0500 Subject: [PATCH 19/19] typo --- infra/DataFactory/deploy.dataFactory.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/DataFactory/deploy.dataFactory.json b/infra/DataFactory/deploy.dataFactory.json index c6a6248..46241cd 100644 --- a/infra/DataFactory/deploy.dataFactory.json +++ b/infra/DataFactory/deploy.dataFactory.json @@ -32,7 +32,7 @@ "description": "Secure string for accessToken of databricksLinkedService" } }, - "databricksWorkSpaceUrl": { + "databricksWorkspaceUrl": { "type": "string", "metadata": { "description": "Specifies databricks workspace url." @@ -141,7 +141,7 @@ "purviewAccountName": "[parameters('purviewAccountName')]", "databricksAccessToken": "[parameters('databricksAccessToken')]", "dataLandingZoneName": "[parameters('dataLandingZoneName')]", - "databricksWorkspaceUrl": "[parameters('databricksWorkSpaceUrl')]", + "databricksWorkspaceUrl": "[parameters('databricksWorkspaceUrl')]", "purviewSecretPath": "[parameters('purviewSecretPath')]", "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]", "location": "[parameters('location')]",