From 194d78c5fd7bf21af6cccb0ab88fffdd0af67a57 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:02:27 -0500 Subject: [PATCH 01/10] Fixes Hub deployment Disk Encryption Set fails if Key Vault public access is disabled Fixes #205 --- research-hub/main.bicep | 1 + 1 file changed, 1 insertion(+) diff --git a/research-hub/main.bicep b/research-hub/main.bicep index af7b25e..9eb42cc 100644 --- a/research-hub/main.bicep +++ b/research-hub/main.bicep @@ -379,6 +379,7 @@ module diskEncryptionSetModule '../shared-modules/security/diskEncryptionSet.bic name: replace(resourceNamingStructureNoSub, '{rtype}', 'des') kvRoleDefinitionId: rolesModule.outputs.roles.KeyVaultCryptoServiceEncryptionUser } + dependsOn: [uamiKvRbacModule] } /* From 7fc6d1dd52e5e0ed2c613583d2415be68d87150e Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:47:32 -0500 Subject: [PATCH 02/10] Fix hub deploy.ps1 not showing output values --- research-hub/deploy.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research-hub/deploy.ps1 b/research-hub/deploy.ps1 index a37d9b4..d3940c5 100644 --- a/research-hub/deploy.ps1 +++ b/research-hub/deploy.ps1 @@ -87,7 +87,7 @@ $DeploymentResults = New-AzDeployment @CmdLetParameters if ($DeploymentResults.ProvisioningState -eq 'Succeeded') { Write-Host "🔥 Deployment successful!" - $DeploymentResult.Outputs | Format-Table -Property Key, @{Name = 'Value'; Expression = { $_.Value.Value } } + $DeploymentResults.Outputs | Format-Table -Property Key, @{Name = 'Value'; Expression = { $_.Value.Value } } } else { $DeploymentResults From 1c640547a7748cc671f9160d9bfee6ba1e9328c7 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:23:25 -0500 Subject: [PATCH 03/10] Add words to custom spell check dictionary --- .vscode/settings.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 13d1f32..bccc1a3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,12 @@ { "cSpell.words": [ "aadcdn", + "Adls", "ADWS", "autologon", "azkms", + "azureblob", + "azuredatafactory", "azureedge", "azureportal", "azuresynapse", @@ -21,6 +24,7 @@ "privatelink", "rdweb", "reactblade", + "resourcegroups", "sovereignprodimedatahotfix", "sovereignprodimedatapri", "sovereignprodimedatasec", From 347071af66dc35afa3c73a2c64245a5f093756a9 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:33:22 -0500 Subject: [PATCH 04/10] Add Logic App action to move export rejected file back to private storage account Add additional ADF pipeline to move from File Share to File Share Add `airlockProcessNotificationEmail` parameter for future use Modify Logic App parameter names for increased readability --- research-spoke/main.bicep | 7 +- .../spoke-modules/airlock/adf.bicep | 56 +- .../content/adfPipeline-Files2Files.json | 91 +++ .../airlock/content/logicAppWorkflow.json | 629 +++++++++--------- .../spoke-modules/airlock/logicApp.bicep | 49 +- .../spoke-modules/airlock/main.bicep | 22 +- shared-modules/types/pipelineNamesType.bicep | 6 + 7 files changed, 539 insertions(+), 321 deletions(-) create mode 100644 research-spoke/spoke-modules/airlock/content/adfPipeline-Files2Files.json create mode 100644 shared-modules/types/pipelineNamesType.bicep diff --git a/research-spoke/main.bicep b/research-spoke/main.bicep index 26545fa..03096e9 100644 --- a/research-spoke/main.bicep +++ b/research-spoke/main.bicep @@ -114,6 +114,8 @@ param adminEntraIdObjectId string param isAirlockReviewCentralized bool = false @description('The email address of the reviewer for this project.') param airlockApproverEmail string +@description('The email address where process notifications (designed to be the principal investigator) will be sent.') +param airlockProcessNotificationEmail string @description('The allowed file extensions for ingest.') param allowedIngestFileExtensions array = [] @@ -485,7 +487,6 @@ var storageAccountReaderRoleAssignmentForResearcherGroup = { description: 'Read access to the storage account is required to use Azure Storage Explorer.' } - // Deploy the project's private storage account module storageModule './spoke-modules/storage/main.bicep' = { name: take(replace(deploymentNameStructure, '{rtype}', 'storage'), 64) @@ -670,7 +671,7 @@ module airlockModule './spoke-modules/airlock/main.bicep' = { airlockFileShareName: isAirlockReviewCentralized ? centralAirlockFileShareName : fileShareNames.exportReview approverEmail: airlockApproverEmail - + // TODO: Refactor honestBrokerEntraObjectId: honestBrokerEntraIdObjectId honestBrokerRoleDefinitionId: rolesModule.outputs.roles.StorageFileDataSMBShareReader @@ -731,6 +732,8 @@ module airlockModule './spoke-modules/airlock/main.bicep' = { usePrivateEndpoints: usePrivateEndpoints allowedIngestFileExtensions: allowedIngestFileExtensions + + processNotificationEmail: airlockProcessNotificationEmail } } diff --git a/research-spoke/spoke-modules/airlock/adf.bicep b/research-spoke/spoke-modules/airlock/adf.bicep index 382b6e4..5b715b8 100644 --- a/research-spoke/spoke-modules/airlock/adf.bicep +++ b/research-spoke/spoke-modules/airlock/adf.bicep @@ -22,10 +22,15 @@ param encryptionKeyVaultUri string param adfEncryptionKeyName string +// TODO: Use pipelineNamesType param pipelineNameAdls2Files string = 'pipe-data_move-adls_to_files' param pipelineNameFilesToAdls string = 'pipe-data_move-files_to_adls' +param pipelineNameFilesToFiles string = 'pipe-data_move-files_to_files' + param debugMode bool +import { pipelineNamesType } from '../../../shared-modules/types/pipelineNamesType.bicep' + var baseName = !empty(subWorkloadName) ? replace(namingStructure, '{subWorkloadName}', subWorkloadName) : replace(namingStructure, '-{subWorkloadName}', '') @@ -380,7 +385,7 @@ resource dfsDataset 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { } } -resource pipeline 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { +resource pipelineAdls2Files 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { name: pipelineNameAdls2Files parent: adf properties: { @@ -451,6 +456,46 @@ resource pipelineFilesToAdls 'Microsoft.DataFactory/factories/pipelines@2018-06- ] } +resource pipelineFilesToFiles 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: pipelineNameFilesToFiles + parent: adf + properties: { + activities: loadJsonContent('./content/adfPipeline-Files2Files.json') + parameters: { + sinkConnStringKvBaseUrl: { + type: 'String' + } + sinkFileShareName: { + type: 'String' + } + sourceStorageAccountName: { + type: 'String' + } + sinkStorageAccountName: { + type: 'String' + } + sourceFolderPath: { + type: 'String' + } + sinkFolderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + sourceFileShareName: { + type: 'String' + } + sourceConnStringKvBaseUrl: { + type: 'String' + } + } + } + dependsOn: [ + AzFilesDataset + ] +} + // LATER: Abstract to storage-RoleAssignment module var storageAccountRoleDefinitionId = roles.StorageBlobDataContributor @@ -468,6 +513,9 @@ resource adfPrivateStgRole 'Microsoft.Authorization/roleAssignments@2022-04-01' output principalId string = adf.identity.principalId output name string = adf.name -// LATER: Rename output pipeline name -output pipelineName string = pipelineNameAdls2Files -output pipelineNameFilesToAdls string = pipelineNameFilesToAdls + +output pipelineNames pipelineNamesType = { + blobToFileShare: pipelineNameAdls2Files + fileShareToBlob: pipelineNameFilesToAdls + fileShareToFileShare: pipelineNameFilesToFiles +} diff --git a/research-spoke/spoke-modules/airlock/content/adfPipeline-Files2Files.json b/research-spoke/spoke-modules/airlock/content/adfPipeline-Files2Files.json new file mode 100644 index 0000000..004c444 --- /dev/null +++ b/research-spoke/spoke-modules/airlock/content/adfPipeline-Files2Files.json @@ -0,0 +1,91 @@ +[ + { + "name": "Move data", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "AzureFileStorageReadSettings", + "recursive": true, + "deleteFilesAfterCompletion": true + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureFileStorageWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "AzFilesDataset", + "type": "DatasetReference", + "parameters": { + "fileName": { + "value": "@pipeline().parameters.fileName", + "type": "Expression" + }, + "storageAccountName": { + "value": "@pipeline().parameters.sourceStorageAccountName", + "type": "Expression" + }, + "folderPath": { + "value": "@pipeline().parameters.sourceFolderPath", + "type": "Expression" + }, + "fileShareName": { + "value": "@pipeline().parameters.sourceFileShareName", + "type": "Expression" + }, + "connStringKvBaseUrl": { + "value": "@pipeline().parameters.sourceConnStringKvBaseUrl", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzFilesDataset", + "type": "DatasetReference", + "parameters": { + "fileName": { + "value": "@pipeline().parameters.fileName", + "type": "Expression" + }, + "storageAccountName": { + "value": "@pipeline().parameters.sinkStorageAccountName", + "type": "Expression" + }, + "folderPath": { + "value": "@pipeline().parameters.sinkFolderPath", + "type": "Expression" + }, + "fileShareName": { + "value": "@pipeline().parameters.sinkFileShareName", + "type": "Expression" + }, + "connStringKvBaseUrl": { + "value": "@pipeline().parameters.sinkConnStringKvBaseUrl", + "type": "Expression" + } + } + } + ] + } +] diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 7503517..49e15c7 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -1,300 +1,331 @@ { - "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#", - "actions": { - "Initialize_pipelineStatus_variable": { - "inputs": { - "variables": [ - { - "name": "pipelineStatus", - "type": "String", - "value": "Queued" - } - ] - }, - "runAfter": { - "Move_file_from_private_ADLS_to_Airlock_File_Share": [ - "Succeeded" - ] - }, - "type": "InitializeVariable" - }, - "Move_file_from_private_ADLS_to_Airlock_File_Share": { - "inputs": { - "body": { - "fileName": "@triggerBody()?['Name']", - "sinkConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", - "sinkFileShareName": "@parameters('sinkFileShareName')", - "sinkFolderPath": "@parameters('sinkFolderPath')", - "sinkStorageAccountName": "@parameters('sinkStorageAccountName')", - "sourceFolderPath": "@parameters('sourceFolderPath')", - "sourceStorageAccountName": "@parameters('sourceStorageAccountName')" - }, - "host": { - "connection": { - "name": "@parameters('$connections')['azuredatafactory']['connectionId']" - } - }, - "method": "post", - "path": "/subscriptions/@{parameters('subscriptionId')}/resourcegroups/@{parameters('dataFactoryRG')}/providers/Microsoft.DataFactory/factories/@{parameters('dataFactoryName')}/pipelines/@{encodeURIComponent('pipe-data_move-adls_to_files')}/CreateRun", - "queries": { - "x-ms-api-version": "2017-09-01-preview" - } - }, - "runAfter": {}, - "type": "ApiConnection" - }, - "Pipeline_successful": { - "actions": { - "Export_approved": { - "actions": { - "Move_file_from_Airlock_File_Share_to_public_ADLS": { - "inputs": { - "body": { - "fileName": "@triggerBody()?['Name']", - "sinkFolderPath": "@parameters('exportApprovedContainerName')", - "sinkStorageAccountName": "@parameters('finalSinkStorageAccountName')", - "sourceConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", - "sourceFileShareName": "@parameters('sinkFileShareName')", - "sourceFolderPath": "@concat(parameters('sinkFolderPath'), '/', parameters('sourceFolderPath'))", - "sourceStorageAccountName": "@parameters('sinkStorageAccountName')" - }, - "host": { - "connection": { - "name": "@parameters('$connections')['azuredatafactory']['connectionId']" - } - }, - "method": "post", - "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent('pipe-data_move-files_to_adls')}/CreateRun", - "queries": { - "x-ms-api-version": "2017-09-01-preview" - } - }, - "runAfter": {}, - "type": "ApiConnection" - } - }, - "expression": { - "and": [ - { - "equals": [ - "@body('Send_approval_email')?['SelectedOption']", - "Approve" - ] - } - ] - }, - "runAfter": { - "Send_approval_email": [ - "Succeeded" - ] - }, - "type": "If" - }, - "Send_approval_email": { - "inputs": { - "body": { - "Message": { - "Body": "A new request for export has been received.\n\nFile: @{parameters('sinkFolderPath')}/@{triggerBody()?['Name']}", - "HideHTMLMessage": false, - "Importance": "Normal", - "Options": "Approve, Reject", - "ShowHTMLConfirmationDialog": false, - "Subject": "Data Export Request", - "To": "@parameters('notificationEmail')" - }, - "NotificationUrl": "@{listCallbackUrl()}" - }, - "host": { - "connection": { - "name": "@parameters('$connections')['office365']['connectionId']" - } - }, - "path": "/approvalmail/$subscriptions" - }, - "runAfter": {}, - "type": "ApiConnectionWebhook" - } - }, - "else": { - "actions": { - "Send_an_email_(V2)": { - "inputs": { - "body": { - "Body": "
Pipeline run failed
", - "Importance": "Normal", - "Subject": "Pipeline Failed", - "To": "@parameters('notificationEmail')" - }, - "host": { - "connection": { - "name": "@parameters('$connections')['office365']['connectionId']" - } - }, - "method": "post", - "path": "/v2/Mail" - }, - "runAfter": {}, - "type": "ApiConnection" - } - } - }, - "expression": { - "and": [ - { - "equals": [ - "@body('Get_a_pipeline_run')?['status']", - "Succeeded" - ] - } - ] - }, - "runAfter": { - "Until_pipeline_is_finished": [ - "Succeeded" - ] - }, - "type": "If" - }, - "Until_pipeline_is_finished": { - "actions": { - "Delay": { - "inputs": { - "interval": { - "count": 1, - "unit": "Minute" - } - }, - "runAfter": {}, - "type": "Wait" - }, - "Get_a_pipeline_run": { - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['azuredatafactory']['connectionId']" - } - }, - "method": "get", - "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelineRuns/@{encodeURIComponent(body('Move_file_from_private_ADLS_to_Airlock_File_Share')?['runId'])}", - "queries": { - "x-ms-api-version": "2017-09-01-preview" - } - }, - "runAfter": { - "Delay": [ - "Succeeded" - ] - }, - "type": "ApiConnection" - }, - "Set_variable": { - "inputs": { - "name": "pipelineStatus", - "value": "@body('Get_a_pipeline_run')?['status']" - }, - "runAfter": { - "Get_a_pipeline_run": [ - "Succeeded" - ] - }, - "type": "SetVariable" - } - }, - "expression": "@and(not(equals(variables('pipelineStatus'), 'Queued')),not(equals(variables('pipelineStatus'), 'InProgress')))", - "limit": { - "count": 20, - "timeout": "PT1H" - }, - "runAfter": { - "Initialize_pipelineStatus_variable": [ - "Succeeded" - ] - }, - "type": "Until" - } - }, - "contentVersion": "1.0.0.0", - "outputs": {}, - "parameters": { - "$connections": { - "defaultValue": {}, - "type": "Object" - }, - "dataFactoryName": { - "defaultValue": "", - "type": "String" - }, - "dataFactoryRG": { - "defaultValue": "", - "type": "String" - }, - "exportApprovedContainerName": { - "defaultValue": "", - "type": "String" - }, - "finalSinkStorageAccountName": { - "defaultValue": "", - "type": "String" - }, - "notificationEmail": { - "defaultValue": "", - "type": "String" - }, - "sinkFileShareName": { - "defaultValue": "", - "type": "String" - }, - "sinkFolderPath": { - "defaultValue": "", - "type": "String" - }, - "sinkStorageAccountName": { - "defaultValue": "", - "type": "String" - }, - "sourceFolderPath": { - "defaultValue": "", - "type": "String" - }, - "sourceStorageAccountName": { - "defaultValue": "", - "type": "String" - }, - "subscriptionId": { - "defaultValue": "", - "type": "String" - }, - "airlockConnStringKvBaseUrl": { - "defaultValue": "", - "type": "String" - } - }, - "triggers": { - "When_a_blob_is_added_or_modified_(properties_only)_(V2)": { - "evaluatedRecurrence": { - "frequency": "Minute", - "interval": 1 - }, - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['azureblob']['connectionId']" - } - }, - "method": "get", - "path": "/v2/datasets/@{encodeURIComponent(encodeURIComponent(parameters('sourceStorageAccountName')))}/triggers/batch/onupdatedfile", - "queries": { - "checkBothCreatedAndModifiedDateTime": false, - "folderId": "/export-request", - "maxFileCount": 1 - } - }, - "recurrence": { - "frequency": "Minute", - "interval": 1 - }, - "splitOn": "@triggerBody()", - "type": "ApiConnection" - } - } -} \ No newline at end of file + "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#", + "contentVersion": "1.0.0.0", + "actions": { + "Initialize_pipelineStatus_variable": { + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "pipelineStatus", + "type": "String", + "value": "Queued" + } + ] + }, + "runAfter": { + "Move_file_from_private_ADLS_to_Airlock_File_Share": ["Succeeded"] + } + }, + "Move_file_from_private_ADLS_to_Airlock_File_Share": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azuredatafactory']['connectionId']" + } + }, + "method": "post", + "body": { + "fileName": "@triggerBody()?['Name']", + "sinkConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", + "sinkFileShareName": "@parameters('airlockFileShareName')", + "sinkFolderPath": "@parameters('airlockFolderPath')", + "sinkStorageAccountName": "@parameters('airlockStorageAccountName')", + "sourceFolderPath": "@parameters('privateFolderPath')", + "sourceStorageAccountName": "@parameters('privateStorageAccountName')" + }, + "path": "/subscriptions/@{parameters('subscriptionId')}/resourcegroups/@{parameters('dataFactoryRG')}/providers/Microsoft.DataFactory/factories/@{parameters('dataFactoryName')}/pipelines/@{encodeURIComponent('pipe-data_move-adls_to_files')}/CreateRun", + "queries": { + "x-ms-api-version": "2017-09-01-preview" + } + }, + "runAfter": {} + }, + "Pipeline_successful": { + "type": "If", + "expression": { + "and": [ + { + "equals": ["@body('Get_a_pipeline_run')?['status']", "Succeeded"] + } + ] + }, + "actions": { + "Export_approved": { + "type": "If", + "expression": { + "and": [ + { + "equals": [ + "@body('Send_approval_email')?['SelectedOption']", + "Approve" + ] + } + ] + }, + "actions": { + "Move_file_from_Airlock_File_Share_to_public_ADLS": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azuredatafactory']['connectionId']" + } + }, + "method": "post", + "body": { + "fileName": "@triggerBody()?['Name']", + "sinkFolderPath": "@parameters('exportApprovedContainerName')", + "sinkStorageAccountName": "@parameters('publicStorageAccountName')", + "sourceConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", + "sourceFileShareName": "@parameters('airlockFileShareName')", + "sourceFolderPath": "@concat(parameters('airlockFolderPath'), '/', parameters('privateFolderPath'))", + "sourceStorageAccountName": "@parameters('airlockStorageAccountName')" + }, + "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent('pipe-data_move-files_to_adls')}/CreateRun", + "queries": { + "x-ms-api-version": "2017-09-01-preview" + } + } + } + }, + "else": { + "actions": { + "Move_file_from_Airlock_File_Share_to_private_File_Share_rejected_folder": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azuredatafactory']['connectionId']" + } + }, + "method": "post", + "body": { + "fileName": "@triggerBody()?['Name']", + "sinkFolderPath": "export-rejected", + "sinkFileShareName": "@parameters('privateFileShareName')", + "sinkStorageAccountName": "@parameters('privateStorageAccountName')", + "sourceConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", + "sourceFileShareName": "@parameters('airlockFileShareName')", + "sourceFolderPath": "@concat(parameters('airlockFolderPath'), '/', parameters('privateFolderPath'))", + "sourceStorageAccountName": "@parameters('airlockStorageAccountName')", + "sinkConnStringKvBaseUrl": "@parameters('privateConnStringKvBaseUrl')" + }, + "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent('pipe-data_move-files_to_files')}/CreateRun", + "queries": { + "x-ms-api-version": "2017-09-01-preview" + } + } + } + } + }, + "runAfter": { + "Send_approval_email": ["Succeeded"] + } + }, + "Send_approval_email": { + "type": "ApiConnectionWebhook", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['office365']['connectionId']" + } + }, + "body": { + "NotificationUrl": "@{listCallbackUrl()}", + "Message": { + "To": "@parameters('approverEmail')", + "Body": "A new request for export has been received.\n\nFile: @{parameters('airlockFolderPath')}/@{triggerBody()?['Name']}", + "Importance": "Normal", + "HideHTMLMessage": false, + "ShowHTMLConfirmationDialog": false, + "Subject": "Data Export Request", + "Options": "Approve, Reject" + } + }, + "path": "/approvalmail/$subscriptions" + } + } + }, + "else": { + "actions": { + "Send_an_email_(V2)": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['office365']['connectionId']" + } + }, + "method": "post", + "body": { + "To": "@parameters('approverEmail')", + "Subject": "Pipeline Failed", + "Body": "Pipeline run failed
", + "Importance": "Normal" + }, + "path": "/v2/Mail" + } + } + } + }, + "runAfter": { + "Until_pipeline_is_finished": ["Succeeded"] + } + }, + "Until_pipeline_is_finished": { + "type": "Until", + "expression": "@and(not(equals(variables('pipelineStatus'), 'Queued')),not(equals(variables('pipelineStatus'), 'InProgress')))", + "limit": { + "count": 20, + "timeout": "PT1H" + }, + "actions": { + "Delay": { + "type": "Wait", + "inputs": { + "interval": { + "count": 1, + "unit": "Minute" + } + } + }, + "Get_a_pipeline_run": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azuredatafactory']['connectionId']" + } + }, + "method": "get", + "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelineRuns/@{encodeURIComponent(body('Move_file_from_private_ADLS_to_Airlock_File_Share')?['runId'])}", + "queries": { + "x-ms-api-version": "2017-09-01-preview" + } + }, + "runAfter": { + "Delay": ["Succeeded"] + } + }, + "Set_variable": { + "type": "SetVariable", + "inputs": { + "name": "pipelineStatus", + "value": "@body('Get_a_pipeline_run')?['status']" + }, + "runAfter": { + "Get_a_pipeline_run": ["Succeeded"] + } + } + }, + "runAfter": { + "Initialize_pipelineStatus_variable": ["Succeeded"] + } + } + }, + "outputs": {}, + "parameters": { + "dataFactoryName": { + "defaultValue": "", + "type": "String" + }, + "dataFactoryRG": { + "defaultValue": "", + "type": "String" + }, + "exportApprovedContainerName": { + "defaultValue": "", + "type": "String" + }, + "publicStorageAccountName": { + "defaultValue": "", + "type": "String" + }, + "approverEmail": { + "defaultValue": "", + "type": "String" + }, + "airlockFileShareName": { + "defaultValue": "", + "type": "String" + }, + "airlockFolderPath": { + "defaultValue": "", + "type": "String" + }, + "airlockStorageAccountName": { + "defaultValue": "", + "type": "String" + }, + "privateFolderPath": { + "defaultValue": "", + "type": "String" + }, + "privateStorageAccountName": { + "defaultValue": "", + "type": "String" + }, + "subscriptionId": { + "defaultValue": "", + "type": "String" + }, + "airlockConnStringKvBaseUrl": { + "defaultValue": "", + "type": "String" + }, + "processNotificationEmail": { + "defaultValue": "", + "type": "String" + }, + "privateFileShareName": { + "defaultValue": "", + "type": "String" + }, + "privateConnStringKvBaseUrl": { + "defaultValue": "", + "type": "String" + }, + "privateContainerName": { + "defaultValue": "", + "type": "String" + }, + "pipelineNameBlobToFileShare": { + "defaultValue": "", + "type": "String" + }, + "pipelineNameFileShareToBlob": { + "defaultValue": "", + "type": "String" + }, + "$connections": { + "type": "Object", + "defaultValue": {} + } + }, + "triggers": { + "When_a_blob_is_added_or_modified_(properties_only)_(V2)": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "get", + "path": "/v2/datasets/@{encodeURIComponent(encodeURIComponent(parameters('privateStorageAccountName')))}/triggers/batch/onupdatedfile", + "queries": { + "folderId": "@concat('/', parameters('privateContainerName'))", + "maxFileCount": 1, + "checkBothCreatedAndModifiedDateTime": false + } + }, + "recurrence": { + "frequency": "Minute", + "interval": 1 + }, + "splitOn": "@triggerBody()" + } + } +} diff --git a/research-spoke/spoke-modules/airlock/logicApp.bicep b/research-spoke/spoke-modules/airlock/logicApp.bicep index 8c6879a..ce8fdbe 100644 --- a/research-spoke/spoke-modules/airlock/logicApp.bicep +++ b/research-spoke/spoke-modules/airlock/logicApp.bicep @@ -11,19 +11,30 @@ param prjPublicStorageAcctName string @maxLength(24) param airlockStorageAcctName string param airlockFileShareName string +param privateFileShareName string param approverEmail string +param processNotificationEmail string param sourceFolderPath string -param sinkFolderPath string +param sinkFolderPath string // TODO: Rename to airlockFolderPath +param exportApprovedContainerName string +param privateContainerName string + +param pipelineNames pipelineNamesType @description('The URI of the Key Vault that contains the connection string airlock review storage account.') param keyVaultUri string +@description('The URI of the Key Vault that contains the connection string for the project private storage account\'s file share.') +param privateConnStringKvBaseUrl string + param roles object param deploymentNameStructure string param subWorkloadName string param tags object = {} +import { pipelineNamesType } from '../../../shared-modules/types/pipelineNamesType.bicep' + var baseName = !empty(subWorkloadName) ? replace(namingStructure, '{subWorkloadName}', subWorkloadName) : replace(namingStructure, '-{subWorkloadName}', '') @@ -145,35 +156,51 @@ resource logicApp 'Microsoft.Logic/workflows@2019-05-01' = { dataFactoryName: { value: adf.name } - sourceStorageAccountName: { + privateStorageAccountName: { value: prjStorageAcctName } - sourceFolderPath: { + privateFolderPath: { value: sourceFolderPath } - sinkStorageAccountName: { + airlockStorageAccountName: { value: airlockStorageAcctName } - notificationEmail: { + approverEmail: { value: approverEmail } - sinkFileShareName: { + airlockFileShareName: { value: airlockFileShareName } - sinkFolderPath: { + airlockFolderPath: { value: sinkFolderPath } - finalSinkStorageAccountName: { + publicStorageAccountName: { value: prjPublicStorageAcctName } // LATER: Add parameters for pipeline names airlockConnStringKvBaseUrl: { value: keyVaultUri } - // TODO: Add parameter for source container name (for trigger value) + privateContainerName: { + value: privateContainerName + } exportApprovedContainerName: { - // TODO: Do not hardcode container name - value: 'export-approved' + value: exportApprovedContainerName + } + processNotificationEmail: { + value: processNotificationEmail + } + privateFileShareName: { + value: privateFileShareName + } + privateConnStringKvBaseUrl: { + value: privateConnStringKvBaseUrl + } + pipelineNameBlobToFileShare: { + value: pipelineNames.blobToFileShare + } + pipelineNameFileShareToBlob: { + value: pipelineNames.fileShareToBlob } } } diff --git a/research-spoke/spoke-modules/airlock/main.bicep b/research-spoke/spoke-modules/airlock/main.bicep index 3016ebb..166ce78 100644 --- a/research-spoke/spoke-modules/airlock/main.bicep +++ b/research-spoke/spoke-modules/airlock/main.bicep @@ -29,6 +29,8 @@ param containerNames object = { @description('The email address where export approval requests will be sent.') param approverEmail string +@description('The email address where process notifications (designed to be the principal investigator) will be sent.') +param processNotificationEmail string param roles object @@ -111,8 +113,7 @@ module uamiModule '../../../shared-modules/security/uami.bicep' = { } } - -module spokeAirlockStFileShareRbacModule '../../../module-library/roleAssignments/roleAssignment-st-fileShare.bicep' ={ +module spokeAirlockStFileShareRbacModule '../../../module-library/roleAssignments/roleAssignment-st-fileShare.bicep' = { #disable-next-line BCP334 name: take(replace(deploymentNameStructure, '{rtype}', 'st-airlock-fs-${airlockFileShareName}-rbac'), 64) params: { @@ -166,7 +167,7 @@ module spokeAirlockStorageAccountModule '../storage/main.bicep' = if (!useCentra uamiPrincipalId: hubManagementVmUamiPrincipalId uamiClientId: hubManagementVmUamiClientId roles: roles - + // The airlock storage uses file shares via ADF, so access keys are used allowSharedKeyAccess: true } @@ -256,6 +257,7 @@ var airlockStorageAccountName = useCentralizedReview var airlocKeyVaultUri = useCentralizedReview ? centralizedModule.outputs.centralKeyVaultUri : keyVault.properties.vaultUri +var spokeKeyVaultUri = keyVault.properties.vaultUri // Logic app for export review (moves file to airlock review storage account and sends approval email) module logicAppModule 'logicApp.bicep' = { @@ -276,6 +278,16 @@ module logicAppModule 'logicApp.bicep' = { deploymentNameStructure: deploymentNameStructure roles: roles tags: tags + exportApprovedContainerName: containerNames.exportApproved + pipelineNames: { + blobToFileShare: adfModule.outputs.pipelineNames.blobToFileShare + fileShareToBlob: adfModule.outputs.pipelineNames.fileShareToBlob + fileShareToFileShare: adfModule.outputs.pipelineNames.fileShareToFileShare + } + privateConnStringKvBaseUrl: spokeKeyVaultUri + privateContainerName: containerNames.exportRequest + privateFileShareName: spokePrivateFileShareName + processNotificationEmail: processNotificationEmail } } @@ -399,7 +411,7 @@ module ingestTriggerModule 'adfTrigger.bicep' = if (length(allowedIngestFileExte workspaceName: workspaceName storageAccountId: publicStorageAccountModule.outputs.id storageAccountType: 'Public' - ingestPipelineName: adfModule.outputs.pipelineName + ingestPipelineName: adfModule.outputs.pipelineNames.blobToFileShare #disable-next-line BCP334 BCP335 sourceStorageAccountName: publicStorageAccountModule.outputs.name sinkStorageAccountName: spokePrivateStorageAccountName @@ -421,7 +433,7 @@ module extensionIngestTriggersModule 'adfTrigger.bicep' = [ workspaceName: workspaceName storageAccountId: publicStorageAccountModule.outputs.id storageAccountType: 'Public' - ingestPipelineName: adfModule.outputs.pipelineName + ingestPipelineName: adfModule.outputs.pipelineNames.blobToFileShare #disable-next-line BCP334 BCP335 sourceStorageAccountName: publicStorageAccountModule.outputs.name sinkStorageAccountName: spokePrivateStorageAccountName diff --git a/shared-modules/types/pipelineNamesType.bicep b/shared-modules/types/pipelineNamesType.bicep new file mode 100644 index 0000000..64cd5dd --- /dev/null +++ b/shared-modules/types/pipelineNamesType.bicep @@ -0,0 +1,6 @@ +@export() +type pipelineNamesType = { + blobToFileShare: string + fileShareToBlob: string + fileShareToFileShare: string +} From c2736fa8e2316b4f40f222b6dc6f3f0077b3f5ef Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:53:38 -0500 Subject: [PATCH 05/10] Send emails to process notification address after approval or rejection of airlock export request --- .../airlock/content/logicAppWorkflow.json | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 49e15c7..5686dd4 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -88,6 +88,29 @@ "x-ms-api-version": "2017-09-01-preview" } } + }, + "Send_approval_email_to_PI": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['office365']['connectionId']" + } + }, + "method": "post", + "body": { + "To": "@parameters('processNotificationEmail')", + "Subject": "Airlock Export Request Approved", + "Body": "@{parameters('approverEmail')} approved the export request of file \"@{parameters('airlockFolderPath')}/@{triggerBody()?['Name']}\".
The file is being moved to the project's external storage account in the \"@{parameters('exportApprovedContainerName')}\" blob container. This usually takes 1-2 minutes.
", + "Importance": "Normal" + }, + "path": "/v2/Mail" + }, + "runAfter": { + "Move_file_from_Airlock_File_Share_to_public_ADLS": [ + "Succeeded" + ] + } } }, "else": { @@ -117,6 +140,29 @@ "x-ms-api-version": "2017-09-01-preview" } } + }, + "Send_rejection_email_to_PI": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['office365']['connectionId']" + } + }, + "method": "post", + "body": { + "To": "@parameters('processNotificationEmail')", + "Subject": "Airlock Export Request Rejected by Approver", + "Body": "@{parameters('approverEmail')} rejected the export request of file \"@{parameters('airlockFolderPath')}/@{triggerBody()?['Name']}\".
The file is being moved back to the project's file share in the \"export-rejected\" folder. This usually takes 1-2 minutes.
", + "Importance": "Normal" + }, + "path": "/v2/Mail" + }, + "runAfter": { + "Move_file_from_Airlock_File_Share_to_private_File_Share_rejected_folder": [ + "Succeeded" + ] + } } } }, @@ -150,7 +196,7 @@ }, "else": { "actions": { - "Send_an_email_(V2)": { + "Send_failure_email_to_approver_and_PI": { "type": "ApiConnection", "inputs": { "host": { @@ -160,9 +206,9 @@ }, "method": "post", "body": { - "To": "@parameters('approverEmail')", - "Subject": "Pipeline Failed", - "Body": "Pipeline run failed
", + "To": "@{parameters('approverEmail')},@{parameters('processNotificationEmail')}", + "Subject": "Airlock Pipeline Failed", + "Body": "Pipeline run failed moving file \"@{parameters('airlockFolderPath')}/@{triggerBody()?['Name']}\" from the spoke's export-requested container to the airlock file share.
For troubleshooting purposes:
Data Factory pipeline run ID: @{body('Move_file_from_private_ADLS_to_Airlock_File_Share')?['runId']}
Data Factory pipeline status: @{variables('pipelineStatus')}
", "Importance": "Normal" }, "path": "/v2/Mail" From 1edea156a4830028fd87a703ff87b0e91d8da7da Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:56:47 -0500 Subject: [PATCH 06/10] Add Logic App parameter for "export rejected" folder name --- .../spoke-modules/airlock/content/logicAppWorkflow.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 5686dd4..75f871f 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -126,7 +126,7 @@ "method": "post", "body": { "fileName": "@triggerBody()?['Name']", - "sinkFolderPath": "export-rejected", + "sinkFolderPath": "@parameters('exportRejectedFolderName')", "sinkFileShareName": "@parameters('privateFileShareName')", "sinkStorageAccountName": "@parameters('privateStorageAccountName')", "sourceConnStringKvBaseUrl": "@parameters('airlockConnStringKvBaseUrl')", @@ -348,6 +348,10 @@ "$connections": { "type": "Object", "defaultValue": {} + }, + "exportRejectedFolderName": { + "type": "String", + "defaultValue": "export-rejected" } }, "triggers": { From 3abeca76338483908737bfe2d9647bffb4e9224b Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 12:13:19 -0500 Subject: [PATCH 07/10] Update Bicep parameter name for clarity --- research-spoke/spoke-modules/airlock/logicApp.bicep | 4 ++-- research-spoke/spoke-modules/airlock/main.bicep | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/research-spoke/spoke-modules/airlock/logicApp.bicep b/research-spoke/spoke-modules/airlock/logicApp.bicep index ce8fdbe..626f9f7 100644 --- a/research-spoke/spoke-modules/airlock/logicApp.bicep +++ b/research-spoke/spoke-modules/airlock/logicApp.bicep @@ -15,7 +15,7 @@ param privateFileShareName string param approverEmail string param processNotificationEmail string param sourceFolderPath string -param sinkFolderPath string // TODO: Rename to airlockFolderPath +param airlockFolderPath string param exportApprovedContainerName string param privateContainerName string @@ -172,7 +172,7 @@ resource logicApp 'Microsoft.Logic/workflows@2019-05-01' = { value: airlockFileShareName } airlockFolderPath: { - value: sinkFolderPath + value: airlockFolderPath } publicStorageAccountName: { value: prjPublicStorageAcctName diff --git a/research-spoke/spoke-modules/airlock/main.bicep b/research-spoke/spoke-modules/airlock/main.bicep index 166ce78..5e1c711 100644 --- a/research-spoke/spoke-modules/airlock/main.bicep +++ b/research-spoke/spoke-modules/airlock/main.bicep @@ -271,7 +271,9 @@ module logicAppModule 'logicApp.bicep' = { airlockStorageAcctName: airlockStorageAccountName adfName: adfModule.outputs.name approverEmail: approverEmail - sinkFolderPath: spokePrivateStorageAccountName + // Create a folder in the Airlock file share with the name of the private storage account, + // which ensures uniqueness if multiple spokes use the same airlock review storage account + airlockFolderPath: spokePrivateStorageAccountName sourceFolderPath: containerNames.exportRequest prjPublicStorageAcctName: publicStorageAccountModule.outputs.name keyVaultUri: airlocKeyVaultUri From 1365e4db31b6d17dad0ca53893949aee8c2fc0b9 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 12:14:24 -0500 Subject: [PATCH 08/10] Change hardcoded pipeline name to parameter value Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../spoke-modules/airlock/content/logicAppWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 75f871f..3954d32 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -35,7 +35,7 @@ "sourceFolderPath": "@parameters('privateFolderPath')", "sourceStorageAccountName": "@parameters('privateStorageAccountName')" }, - "path": "/subscriptions/@{parameters('subscriptionId')}/resourcegroups/@{parameters('dataFactoryRG')}/providers/Microsoft.DataFactory/factories/@{parameters('dataFactoryName')}/pipelines/@{encodeURIComponent('pipe-data_move-adls_to_files')}/CreateRun", + "path": "/subscriptions/@{parameters('subscriptionId')}/resourcegroups/@{parameters('dataFactoryRG')}/providers/Microsoft.DataFactory/factories/@{parameters('dataFactoryName')}/pipelines/@{encodeURIComponent(parameters('pipelineNameBlobToFileShare'))}/CreateRun", "queries": { "x-ms-api-version": "2017-09-01-preview" } From 105a4bdf390eea5afd03e2daa7b141f0b844efc4 Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 12:16:50 -0500 Subject: [PATCH 09/10] Change hardcoded pipeline name to parameter value Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../spoke-modules/airlock/content/logicAppWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 3954d32..5c9244a 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -83,7 +83,7 @@ "sourceFolderPath": "@concat(parameters('airlockFolderPath'), '/', parameters('privateFolderPath'))", "sourceStorageAccountName": "@parameters('airlockStorageAccountName')" }, - "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent('pipe-data_move-files_to_adls')}/CreateRun", + "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent(parameters('pipelineNameFileShareToBlob'))}/CreateRun", "queries": { "x-ms-api-version": "2017-09-01-preview" } From ceab0912135cbd56ae5e39b688d85c99b85c2bba Mon Sep 17 00:00:00 2001 From: Sven Aelterman <17446043+SvenAelterman@users.noreply.github.com> Date: Sun, 7 Sep 2025 12:21:50 -0500 Subject: [PATCH 10/10] Change hardcoded pipeline name to parameter value (again) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../spoke-modules/airlock/content/logicAppWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json index 5c9244a..1da5994 100644 --- a/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json +++ b/research-spoke/spoke-modules/airlock/content/logicAppWorkflow.json @@ -135,7 +135,7 @@ "sourceStorageAccountName": "@parameters('airlockStorageAccountName')", "sinkConnStringKvBaseUrl": "@parameters('privateConnStringKvBaseUrl')" }, - "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent('pipe-data_move-files_to_files')}/CreateRun", + "path": "/subscriptions/@{encodeURIComponent(parameters('subscriptionId'))}/resourcegroups/@{encodeURIComponent(parameters('dataFactoryRG'))}/providers/Microsoft.DataFactory/factories/@{encodeURIComponent(parameters('dataFactoryName'))}/pipelines/@{encodeURIComponent(parameters('pipelineNameFileShareToFileShare'))}/CreateRun", "queries": { "x-ms-api-version": "2017-09-01-preview" }