Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions private/Get-RequestHeader.ps1
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
function Get-RequestHeader {
$Version = (Get-Command -Name Get-AzAccessToken).Version
$SynapseToken = Get-AzAccessToken -ResourceUrl 'https://dev.azuresynapse.net'
if ($Version -ge '5.0.0') {
$BSTR = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($SynapseToken.Token)
$token = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto($BSTR)
[System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($BSTR)
if ($SynapseToken.Token -is [System.Security.SecureString]) {
$ptr = [System.Runtime.InteropServices.Marshal]::SecureStringToCoTaskMemUnicode($SynapseToken.Token)
$token = [System.Runtime.InteropServices.Marshal]::PtrToStringUni($ptr)
[System.Runtime.InteropServices.Marshal]::ZeroFreeCoTaskMemUnicode($ptr)
}
else {
$token = $SynapseToken.token
$token = $SynapseToken.Token ?? $SynapseToken.token
}
#$token = Get-AzAccessToken -ResourceUrl 'https://management.azure.com' #audience
$Header = @{
Expand Down
18 changes: 18 additions & 0 deletions private/Save-SynapseObjectAsFile.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@ function Save-SynapseObjectAsFile {
$newFileName = Join-Path $obj.Synapse.Location "$($obj.Type)\~$($obj.Name).json"
Write-Debug "Writing file: $newFileName"

# Fix PowerShell ConvertFrom-Json/ConvertTo-Json bug where single-element arrays become objects
# For pipeline objects, ensure activities is always an array
if ($obj.Type -eq 'pipeline' -and $obj.Body.properties.PSObject.Properties['activities']) {
if ($obj.Body.properties.activities -isnot [Array]) {
Write-Verbose "Converting single pipeline activity to array for: $($obj.Name)"
$obj.Body.properties.activities = @($obj.Body.properties.activities)
}
# Also fix nested activities inside ForEach/IfCondition/Switch/Until activities
foreach ($activity in $obj.Body.properties.activities) {
if ($null -ne $activity.PSObject.Properties['typeProperties'] -and
$null -ne $activity.typeProperties.PSObject.Properties['activities'] -and
$activity.typeProperties.activities -isnot [Array]) {
Write-Verbose "Converting nested activities to array in $($activity.name)"
$activity.typeProperties.activities = @($activity.typeProperties.activities)
}
}
}

$output = ($obj.Body | ConvertTo-Json -Compress:$true -Depth 100)
$Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False
[IO.File]::WriteAllLines($newFileName, $output, $Utf8NoBomEncoding)
Expand Down
19 changes: 18 additions & 1 deletion public/Get-SynapseFromService.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ function Get-SynapseFromService {
)
Write-Debug "BEGIN: Get-SynapseromService(WorkspaceName=$WorkspaceName"

# Minimal wrapper class used as REST-API fallback for Get-AzSynapsePipeline.
# Named 'PipelineResource' so Get-SimplifiedType strips 'Resource' -> 'Pipeline'.
if (-not ('PipelineResource' -as [type])) {
Add-Type -TypeDefinition 'public class PipelineResource { public string Name { get; set; } }'
}

$synapse = New-Object -TypeName Synapse
$synapse.Name = $WorkspaceName

Expand All @@ -39,7 +45,18 @@ function Get-SynapseFromService {
Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $synapse.IntegrationRuntimes.Count)
$synapse.LinkedServices = Get-AzSynapseLinkedService -WorkspaceName $WorkspaceName | ToArray
Write-Host ("LinkedServices: {0} object(s) loaded." -f $synapse.LinkedServices.Count)
$synapse.Pipelines = Get-AzSynapsePipeline -WorkspaceName $WorkspaceName | ToArray
try {
$synapse.Pipelines = Get-AzSynapsePipeline -WorkspaceName $WorkspaceName | ToArray
} catch {
Write-Warning "Get-AzSynapsePipeline failed (likely a single-activity pipeline stored as object in Synapse): falling back to REST API. Error: $_"
$h = Get-RequestHeader
$response = Invoke-RestMethod -Method GET -Uri "https://$WorkspaceName.dev.azuresynapse.net/pipelines?api-version=2020-12-01" -Headers $h
$synapse.Pipelines = @($response.value | ForEach-Object {
$p = New-Object PipelineResource
$p.Name = $_.name
$p
})
}
Write-Host ("Pipelines: {0} object(s) loaded." -f $synapse.Pipelines.Count)
$synapse.DataFlows = Get-AzSynapseDataFlow -WorkspaceName $WorkspaceName | ToArray
Write-Host ("DataFlows: {0} object(s) loaded." -f $synapse.DataFlows.Count)
Expand Down
101 changes: 101 additions & 0 deletions test/Get-RequestHeader.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
BeforeDiscovery {
$ModuleRootPath = $PSScriptRoot | Split-Path -Parent
$moduleManifestName = 'azure.synapse.tools.psd1'
$moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName

Import-Module -Name $moduleManifestPath -Force -Verbose:$false
}

InModuleScope azure.synapse.tools {
$testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper'
Import-Module -Name $testHelperPath -Force

Describe 'Get-RequestHeader' -Tag 'Unit' {

It 'Should exist' {
{ Get-Command -Name Get-RequestHeader -ErrorAction Stop } | Should -Not -Throw
}

Context 'Az.Accounts >= 3.0.0 (SecureString token - cross-platform unicode unwrap)' {
BeforeAll {
$plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig'
$secureToken = ConvertTo-SecureString $plainToken -AsPlainText -Force

Mock Get-Module {
[PSCustomObject]@{ Version = [version]'3.0.0' }
} -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools

Mock Get-AzAccessToken {
[PSCustomObject]@{ Token = $secureToken }
} -ModuleName azure.synapse.tools

$script:header = Get-RequestHeader
}

It 'Should return a Bearer token header' {
$script:header['Authorization'] | Should -Match '^Bearer '
}

It 'Should correctly unwrap the SecureString to a plain-text token' {
$script:header['Authorization'] | Should -Be "Bearer $plainToken"
}

It 'Should set Content-Type to application/json' {
$script:header['Content-Type'] | Should -Be 'application/json'
}
}

Context 'Az.Accounts < 3.0.0 (plain string token)' {
BeforeAll {
$plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig'

Mock Get-Module {
[PSCustomObject]@{ Version = [version]'2.9.0' }
} -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools

Mock Get-AzAccessToken {
[PSCustomObject]@{ token = $plainToken }
} -ModuleName azure.synapse.tools

$script:header = Get-RequestHeader
}

It 'Should return a Bearer token header' {
$script:header['Authorization'] | Should -Match '^Bearer '
}

It 'Should use the plain-text token directly' {
$script:header['Authorization'] | Should -Be "Bearer $plainToken"
}

It 'Should set Content-Type to application/json' {
$script:header['Content-Type'] | Should -Be 'application/json'
}
}

Context 'Az.Accounts module not loaded (null version fallback)' {
BeforeAll {
$plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig'

Mock Get-Module {
$null
} -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools

# Plain string token (old Az.Accounts < 3.0.0) uses .token property
Mock Get-AzAccessToken {
[PSCustomObject]@{ token = $plainToken }
} -ModuleName azure.synapse.tools

$script:header = Get-RequestHeader
}

It 'Should not throw' {
{ Get-RequestHeader } | Should -Not -Throw
}

It 'Should still return a valid Authorization header' {
$script:header['Authorization'] | Should -Be "Bearer $plainToken"
}
}
}
}
119 changes: 119 additions & 0 deletions test/Get-SynapseFromService.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
BeforeDiscovery {
$ModuleRootPath = $PSScriptRoot | Split-Path -Parent
$moduleManifestName = 'azure.synapse.tools.psd1'
$moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName

Import-Module -Name $moduleManifestPath -Force -Verbose:$false
}

# Stub out Az.Synapse cmdlets so Pester can mock them without Az.Synapse being installed
foreach ($cmd in @(
'Get-AzSynapseWorkspace', 'Get-AzSynapseNotebook', 'Get-AzSynapseDataset',
'Get-AzSynapseIntegrationRuntime', 'Get-AzSynapseLinkedService',
'Get-AzSynapsePipeline', 'Get-AzSynapseDataFlow', 'Get-AzSynapseTrigger'
)) {
if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) {
$null = New-Item -Path "Function:Global:$cmd" -Value { }
}
}

InModuleScope azure.synapse.tools {
$testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper'
Import-Module -Name $testHelperPath -Force

Describe 'Get-SynapseFromService' -Tag 'Unit' {

It 'Should exist' {
{ Get-Command -Name Get-SynapseFromService -ErrorAction Stop } | Should -Not -Throw
}

BeforeAll {
$workspaceName = 'synws-test-01'

Mock Get-AzSynapseWorkspace {
[PSCustomObject]@{ Id = '/subscriptions/00000000/resourceGroups/rg/providers/Microsoft.Synapse/workspaces/synws-test-01'; Location = 'uksouth' }
} -ModuleName azure.synapse.tools

Mock Get-AzSynapseNotebook { @() } -ModuleName azure.synapse.tools
Mock Get-AzSynapseDataset { @() } -ModuleName azure.synapse.tools
Mock Get-AzSynapseIntegrationRuntime { @() } -ModuleName azure.synapse.tools
Mock Get-AzSynapseLinkedService { @() } -ModuleName azure.synapse.tools
Mock Get-AzSynapseDataFlow { @() } -ModuleName azure.synapse.tools
Mock Get-AzSynapseTrigger { @() } -ModuleName azure.synapse.tools
}

Context 'Get-AzSynapsePipeline succeeds' {
BeforeAll {
Mock Get-AzSynapsePipeline {
@(
[PSCustomObject]@{ Name = 'pl_pipeline_a' },
[PSCustomObject]@{ Name = 'pl_pipeline_b' }
)
} -ModuleName azure.synapse.tools

$script:result = Get-SynapseFromService -WorkspaceName $workspaceName
}

It 'Should return a Synapse object' {
$script:result.GetType().Name | Should -Be 'Synapse'
}

It 'Should populate Pipelines from Az cmdlet' {
@($script:result.Pipelines).Count | Should -Be 2
}

It 'Should contain the expected pipeline names' {
$script:result.Pipelines.Name | Should -Contain 'pl_pipeline_a'
$script:result.Pipelines.Name | Should -Contain 'pl_pipeline_b'
}
}

Context 'Get-AzSynapsePipeline fails (single-activity object deserialization bug)' {
BeforeAll {
Mock Get-AzSynapsePipeline {
throw "The requested operation requires an element of type 'Array', but the target element has type 'Object'."
} -ModuleName azure.synapse.tools

Mock Get-RequestHeader {
@{ 'Authorization' = 'Bearer dummy'; 'Content-Type' = 'application/json' }
} -ModuleName azure.synapse.tools

Mock Invoke-RestMethod {
[PSCustomObject]@{
value = @(
[PSCustomObject]@{ name = 'pl_single_activity_pipeline' },
[PSCustomObject]@{ name = 'pl_another_pipeline' }
)
}
} -ModuleName azure.synapse.tools

$script:result = Get-SynapseFromService -WorkspaceName $workspaceName
}

It 'Should return a Synapse object despite the Az cmdlet failure' {
$script:result.GetType().Name | Should -Be 'Synapse'
}

It 'Should fall back to REST API and populate Pipelines' {
@($script:result.Pipelines).Count | Should -Be 2
}

It 'Should produce PipelineResource objects so Get-SimplifiedType resolves to Pipeline' {
$script:result.Pipelines[0].GetType().Name | Should -Be 'PipelineResource'
}

It 'Should contain the expected pipeline names from REST response' {
$script:result.Pipelines.Name | Should -Contain 'pl_single_activity_pipeline'
$script:result.Pipelines.Name | Should -Contain 'pl_another_pipeline'
}

It 'Should call Invoke-RestMethod with the correct Synapse pipelines endpoint' {
# Verify via side-effect: if wrong URI or no call, pipelines would be empty
# The mock is URI-specific so if the wrong URI was used, Invoke-RestMethod
# would call through to the real implementation and fail
@($script:result.Pipelines).Count | Should -BeGreaterThan 0
$script:result.Pipelines[0].Name | Should -Be 'pl_single_activity_pipeline'
}
}
}
}
100 changes: 100 additions & 0 deletions test/Save-SynapseObjectAsFile.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
BeforeDiscovery {
$ModuleRootPath = $PSScriptRoot | Split-Path -Parent
$moduleManifestName = 'azure.synapse.tools.psd1'
$moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName

Import-Module -Name $moduleManifestPath -Force -Verbose:$false
}

InModuleScope azure.synapse.tools {
$testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper'
Import-Module -Name $testHelperPath -Force

Describe 'Save-SynapseObjectAsFile' -Tag 'Unit' {

BeforeAll {
$script:tmpDir = New-TemporaryDirectory
New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'pipeline') -Force | Out-Null
New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'dataset') -Force | Out-Null
}

AfterAll {
Remove-Item $script:tmpDir.FullName -Recurse -Force -ErrorAction SilentlyContinue
}

It 'Should exist' {
{ Get-Command -Name Save-SynapseObjectAsFile -ErrorAction Stop } | Should -Not -Throw
}

Context 'Single top-level activity coerced to object by ConvertFrom-Json' {
BeforeAll {
$synapse = [Synapse]::new()
$synapse.Location = $script:tmpDir.FullName
$obj = [SynapseObject]::new()
$obj.Name = 'pl_single_activity'
$obj.Type = 'pipeline'
$obj.Synapse = $synapse
$obj.Body = '{"name":"pl_single_activity","properties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}' | ConvertFrom-Json
$script:result = Save-SynapseObjectAsFile -obj $obj
}
It 'Should serialise activities as a JSON array' {
$raw = Get-Content $script:result -Raw
$raw | Should -Match '"activities":\['
$raw | Should -Not -Match '"activities":\{'
}
}

Context 'Multiple top-level activities already an array' {
BeforeAll {
$synapse = [Synapse]::new()
$synapse.Location = $script:tmpDir.FullName
$obj = [SynapseObject]::new()
$obj.Name = 'pl_multi_activity'
$obj.Type = 'pipeline'
$obj.Synapse = $synapse
$obj.Body = '{"name":"pl_multi_activity","properties":{"activities":[{"name":"act1","type":"Copy","typeProperties":{}},{"name":"act2","type":"Copy","typeProperties":{}}]}}' | ConvertFrom-Json
$script:result = Save-SynapseObjectAsFile -obj $obj
}
It 'Should preserve all activities and write them as a JSON array' {
$json = Get-Content $script:result -Raw | ConvertFrom-Json
@($json.properties.activities).Count | Should -Be 2
}
}

Context 'ForEach nested single activity coerced to object by ConvertFrom-Json' {
BeforeAll {
$synapse = [Synapse]::new()
$synapse.Location = $script:tmpDir.FullName
$obj = [SynapseObject]::new()
$obj.Name = 'pl_foreach_single_nested_activity'
$obj.Type = 'pipeline'
$obj.Synapse = $synapse
$obj.Body = '{"name":"pl_foreach_single_nested_activity","properties":{"activities":[{"name":"forEach1","type":"ForEach","typeProperties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}]}}' | ConvertFrom-Json
$script:result = Save-SynapseObjectAsFile -obj $obj
}
It 'Should serialise nested typeProperties.activities as a JSON array' {
$raw = Get-Content $script:result -Raw
$raw | Should -Match '"typeProperties":\{"activities":\['
$raw | Should -Not -Match '"typeProperties":\{"activities":\{'
}
}

Context 'Non-pipeline object' {
BeforeAll {
$synapse = [Synapse]::new()
$synapse.Location = $script:tmpDir.FullName
$obj = [SynapseObject]::new()
$obj.Name = 'ds_test'
$obj.Type = 'dataset'
$obj.Synapse = $synapse
$obj.Body = '{"name":"ds_test","properties":{"type":"AzureBlob"}}' | ConvertFrom-Json
$script:result = Save-SynapseObjectAsFile -obj $obj
}
It 'Should write the object to file without modification' {
$json = Get-Content $script:result -Raw | ConvertFrom-Json
$json.name | Should -Be 'ds_test'
$json.properties.type | Should -Be 'AzureBlob'
}
}
}
}