diff --git a/private/Get-RequestHeader.ps1 b/private/Get-RequestHeader.ps1 index 10e660f..c5c550d 100644 --- a/private/Get-RequestHeader.ps1 +++ b/private/Get-RequestHeader.ps1 @@ -1,13 +1,12 @@ function Get-RequestHeader { - $Version = (Get-Command -Name Get-AzAccessToken).Version $SynapseToken = Get-AzAccessToken -ResourceUrl 'https://dev.azuresynapse.net' - if ($Version -ge '5.0.0') { - $BSTR = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($SynapseToken.Token) - $token = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto($BSTR) - [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($BSTR) + if ($SynapseToken.Token -is [System.Security.SecureString]) { + $ptr = [System.Runtime.InteropServices.Marshal]::SecureStringToCoTaskMemUnicode($SynapseToken.Token) + $token = [System.Runtime.InteropServices.Marshal]::PtrToStringUni($ptr) + [System.Runtime.InteropServices.Marshal]::ZeroFreeCoTaskMemUnicode($ptr) } else { - $token = $SynapseToken.token + $token = $SynapseToken.Token ?? $SynapseToken.token } #$token = Get-AzAccessToken -ResourceUrl 'https://management.azure.com' #audience $Header = @{ diff --git a/private/Save-SynapseObjectAsFile.ps1 b/private/Save-SynapseObjectAsFile.ps1 index 016af78..396c091 100644 --- a/private/Save-SynapseObjectAsFile.ps1 +++ b/private/Save-SynapseObjectAsFile.ps1 @@ -7,6 +7,24 @@ function Save-SynapseObjectAsFile { $newFileName = Join-Path $obj.Synapse.Location "$($obj.Type)\~$($obj.Name).json" Write-Debug "Writing file: $newFileName" + # Fix PowerShell ConvertFrom-Json/ConvertTo-Json bug where single-element arrays become objects + # For pipeline objects, ensure activities is always an array + if ($obj.Type -eq 'pipeline' -and $obj.Body.properties.PSObject.Properties['activities']) { + if ($obj.Body.properties.activities -isnot [Array]) { + Write-Verbose "Converting single pipeline activity to array for: $($obj.Name)" + $obj.Body.properties.activities = @($obj.Body.properties.activities) + } + # Also fix nested activities inside ForEach/IfCondition/Switch/Until activities + foreach ($activity in $obj.Body.properties.activities) { + if ($null -ne $activity.PSObject.Properties['typeProperties'] -and + $null -ne $activity.typeProperties.PSObject.Properties['activities'] -and + $activity.typeProperties.activities -isnot [Array]) { + Write-Verbose "Converting nested activities to array in $($activity.name)" + $activity.typeProperties.activities = @($activity.typeProperties.activities) + } + } + } + $output = ($obj.Body | ConvertTo-Json -Compress:$true -Depth 100) $Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False [IO.File]::WriteAllLines($newFileName, $output, $Utf8NoBomEncoding) diff --git a/public/Get-SynapseFromService.ps1 b/public/Get-SynapseFromService.ps1 index 8c61250..5f8c49a 100644 --- a/public/Get-SynapseFromService.ps1 +++ b/public/Get-SynapseFromService.ps1 @@ -23,6 +23,12 @@ function Get-SynapseFromService { ) Write-Debug "BEGIN: Get-SynapseromService(WorkspaceName=$WorkspaceName" + # Minimal wrapper class used as REST-API fallback for Get-AzSynapsePipeline. + # Named 'PipelineResource' so Get-SimplifiedType strips 'Resource' -> 'Pipeline'. + if (-not ('PipelineResource' -as [type])) { + Add-Type -TypeDefinition 'public class PipelineResource { public string Name { get; set; } }' + } + $synapse = New-Object -TypeName Synapse $synapse.Name = $WorkspaceName @@ -39,7 +45,18 @@ function Get-SynapseFromService { Write-Host ("IntegrationRuntimes: {0} object(s) loaded." -f $synapse.IntegrationRuntimes.Count) $synapse.LinkedServices = Get-AzSynapseLinkedService -WorkspaceName $WorkspaceName | ToArray Write-Host ("LinkedServices: {0} object(s) loaded." -f $synapse.LinkedServices.Count) - $synapse.Pipelines = Get-AzSynapsePipeline -WorkspaceName $WorkspaceName | ToArray + try { + $synapse.Pipelines = Get-AzSynapsePipeline -WorkspaceName $WorkspaceName | ToArray + } catch { + Write-Warning "Get-AzSynapsePipeline failed (likely a single-activity pipeline stored as object in Synapse): falling back to REST API. Error: $_" + $h = Get-RequestHeader + $response = Invoke-RestMethod -Method GET -Uri "https://$WorkspaceName.dev.azuresynapse.net/pipelines?api-version=2020-12-01" -Headers $h + $synapse.Pipelines = @($response.value | ForEach-Object { + $p = New-Object PipelineResource + $p.Name = $_.name + $p + }) + } Write-Host ("Pipelines: {0} object(s) loaded." -f $synapse.Pipelines.Count) $synapse.DataFlows = Get-AzSynapseDataFlow -WorkspaceName $WorkspaceName | ToArray Write-Host ("DataFlows: {0} object(s) loaded." -f $synapse.DataFlows.Count) diff --git a/test/Get-RequestHeader.Tests.ps1 b/test/Get-RequestHeader.Tests.ps1 new file mode 100644 index 0000000..1e8baa9 --- /dev/null +++ b/test/Get-RequestHeader.Tests.ps1 @@ -0,0 +1,101 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.synapse.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.synapse.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + Describe 'Get-RequestHeader' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-RequestHeader -ErrorAction Stop } | Should -Not -Throw + } + + Context 'Az.Accounts >= 3.0.0 (SecureString token - cross-platform unicode unwrap)' { + BeforeAll { + $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' + $secureToken = ConvertTo-SecureString $plainToken -AsPlainText -Force + + Mock Get-Module { + [PSCustomObject]@{ Version = [version]'3.0.0' } + } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools + + Mock Get-AzAccessToken { + [PSCustomObject]@{ Token = $secureToken } + } -ModuleName azure.synapse.tools + + $script:header = Get-RequestHeader + } + + It 'Should return a Bearer token header' { + $script:header['Authorization'] | Should -Match '^Bearer ' + } + + It 'Should correctly unwrap the SecureString to a plain-text token' { + $script:header['Authorization'] | Should -Be "Bearer $plainToken" + } + + It 'Should set Content-Type to application/json' { + $script:header['Content-Type'] | Should -Be 'application/json' + } + } + + Context 'Az.Accounts < 3.0.0 (plain string token)' { + BeforeAll { + $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' + + Mock Get-Module { + [PSCustomObject]@{ Version = [version]'2.9.0' } + } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools + + Mock Get-AzAccessToken { + [PSCustomObject]@{ token = $plainToken } + } -ModuleName azure.synapse.tools + + $script:header = Get-RequestHeader + } + + It 'Should return a Bearer token header' { + $script:header['Authorization'] | Should -Match '^Bearer ' + } + + It 'Should use the plain-text token directly' { + $script:header['Authorization'] | Should -Be "Bearer $plainToken" + } + + It 'Should set Content-Type to application/json' { + $script:header['Content-Type'] | Should -Be 'application/json' + } + } + + Context 'Az.Accounts module not loaded (null version fallback)' { + BeforeAll { + $plainToken = 'eyJhbGciOiJSUzI1NiJ9.payload.sig' + + Mock Get-Module { + $null + } -ParameterFilter { $Name -eq 'Az.Accounts' } -ModuleName azure.synapse.tools + + # Plain string token (old Az.Accounts < 3.0.0) uses .token property + Mock Get-AzAccessToken { + [PSCustomObject]@{ token = $plainToken } + } -ModuleName azure.synapse.tools + + $script:header = Get-RequestHeader + } + + It 'Should not throw' { + { Get-RequestHeader } | Should -Not -Throw + } + + It 'Should still return a valid Authorization header' { + $script:header['Authorization'] | Should -Be "Bearer $plainToken" + } + } + } +} diff --git a/test/Get-SynapseFromService.Tests.ps1 b/test/Get-SynapseFromService.Tests.ps1 new file mode 100644 index 0000000..4996ad9 --- /dev/null +++ b/test/Get-SynapseFromService.Tests.ps1 @@ -0,0 +1,119 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.synapse.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +# Stub out Az.Synapse cmdlets so Pester can mock them without Az.Synapse being installed +foreach ($cmd in @( + 'Get-AzSynapseWorkspace', 'Get-AzSynapseNotebook', 'Get-AzSynapseDataset', + 'Get-AzSynapseIntegrationRuntime', 'Get-AzSynapseLinkedService', + 'Get-AzSynapsePipeline', 'Get-AzSynapseDataFlow', 'Get-AzSynapseTrigger' +)) { + if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { + $null = New-Item -Path "Function:Global:$cmd" -Value { } + } +} + +InModuleScope azure.synapse.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + Describe 'Get-SynapseFromService' -Tag 'Unit' { + + It 'Should exist' { + { Get-Command -Name Get-SynapseFromService -ErrorAction Stop } | Should -Not -Throw + } + + BeforeAll { + $workspaceName = 'synws-test-01' + + Mock Get-AzSynapseWorkspace { + [PSCustomObject]@{ Id = '/subscriptions/00000000/resourceGroups/rg/providers/Microsoft.Synapse/workspaces/synws-test-01'; Location = 'uksouth' } + } -ModuleName azure.synapse.tools + + Mock Get-AzSynapseNotebook { @() } -ModuleName azure.synapse.tools + Mock Get-AzSynapseDataset { @() } -ModuleName azure.synapse.tools + Mock Get-AzSynapseIntegrationRuntime { @() } -ModuleName azure.synapse.tools + Mock Get-AzSynapseLinkedService { @() } -ModuleName azure.synapse.tools + Mock Get-AzSynapseDataFlow { @() } -ModuleName azure.synapse.tools + Mock Get-AzSynapseTrigger { @() } -ModuleName azure.synapse.tools + } + + Context 'Get-AzSynapsePipeline succeeds' { + BeforeAll { + Mock Get-AzSynapsePipeline { + @( + [PSCustomObject]@{ Name = 'pl_pipeline_a' }, + [PSCustomObject]@{ Name = 'pl_pipeline_b' } + ) + } -ModuleName azure.synapse.tools + + $script:result = Get-SynapseFromService -WorkspaceName $workspaceName + } + + It 'Should return a Synapse object' { + $script:result.GetType().Name | Should -Be 'Synapse' + } + + It 'Should populate Pipelines from Az cmdlet' { + @($script:result.Pipelines).Count | Should -Be 2 + } + + It 'Should contain the expected pipeline names' { + $script:result.Pipelines.Name | Should -Contain 'pl_pipeline_a' + $script:result.Pipelines.Name | Should -Contain 'pl_pipeline_b' + } + } + + Context 'Get-AzSynapsePipeline fails (single-activity object deserialization bug)' { + BeforeAll { + Mock Get-AzSynapsePipeline { + throw "The requested operation requires an element of type 'Array', but the target element has type 'Object'." + } -ModuleName azure.synapse.tools + + Mock Get-RequestHeader { + @{ 'Authorization' = 'Bearer dummy'; 'Content-Type' = 'application/json' } + } -ModuleName azure.synapse.tools + + Mock Invoke-RestMethod { + [PSCustomObject]@{ + value = @( + [PSCustomObject]@{ name = 'pl_single_activity_pipeline' }, + [PSCustomObject]@{ name = 'pl_another_pipeline' } + ) + } + } -ModuleName azure.synapse.tools + + $script:result = Get-SynapseFromService -WorkspaceName $workspaceName + } + + It 'Should return a Synapse object despite the Az cmdlet failure' { + $script:result.GetType().Name | Should -Be 'Synapse' + } + + It 'Should fall back to REST API and populate Pipelines' { + @($script:result.Pipelines).Count | Should -Be 2 + } + + It 'Should produce PipelineResource objects so Get-SimplifiedType resolves to Pipeline' { + $script:result.Pipelines[0].GetType().Name | Should -Be 'PipelineResource' + } + + It 'Should contain the expected pipeline names from REST response' { + $script:result.Pipelines.Name | Should -Contain 'pl_single_activity_pipeline' + $script:result.Pipelines.Name | Should -Contain 'pl_another_pipeline' + } + + It 'Should call Invoke-RestMethod with the correct Synapse pipelines endpoint' { + # Verify via side-effect: if wrong URI or no call, pipelines would be empty + # The mock is URI-specific so if the wrong URI was used, Invoke-RestMethod + # would call through to the real implementation and fail + @($script:result.Pipelines).Count | Should -BeGreaterThan 0 + $script:result.Pipelines[0].Name | Should -Be 'pl_single_activity_pipeline' + } + } + } +} diff --git a/test/Save-SynapseObjectAsFile.Tests.ps1 b/test/Save-SynapseObjectAsFile.Tests.ps1 new file mode 100644 index 0000000..a4125cc --- /dev/null +++ b/test/Save-SynapseObjectAsFile.Tests.ps1 @@ -0,0 +1,100 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.synapse.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.synapse.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + Describe 'Save-SynapseObjectAsFile' -Tag 'Unit' { + + BeforeAll { + $script:tmpDir = New-TemporaryDirectory + New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'pipeline') -Force | Out-Null + New-Item -ItemType Directory -Path (Join-Path $script:tmpDir.FullName 'dataset') -Force | Out-Null + } + + AfterAll { + Remove-Item $script:tmpDir.FullName -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'Should exist' { + { Get-Command -Name Save-SynapseObjectAsFile -ErrorAction Stop } | Should -Not -Throw + } + + Context 'Single top-level activity coerced to object by ConvertFrom-Json' { + BeforeAll { + $synapse = [Synapse]::new() + $synapse.Location = $script:tmpDir.FullName + $obj = [SynapseObject]::new() + $obj.Name = 'pl_single_activity' + $obj.Type = 'pipeline' + $obj.Synapse = $synapse + $obj.Body = '{"name":"pl_single_activity","properties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}' | ConvertFrom-Json + $script:result = Save-SynapseObjectAsFile -obj $obj + } + It 'Should serialise activities as a JSON array' { + $raw = Get-Content $script:result -Raw + $raw | Should -Match '"activities":\[' + $raw | Should -Not -Match '"activities":\{' + } + } + + Context 'Multiple top-level activities already an array' { + BeforeAll { + $synapse = [Synapse]::new() + $synapse.Location = $script:tmpDir.FullName + $obj = [SynapseObject]::new() + $obj.Name = 'pl_multi_activity' + $obj.Type = 'pipeline' + $obj.Synapse = $synapse + $obj.Body = '{"name":"pl_multi_activity","properties":{"activities":[{"name":"act1","type":"Copy","typeProperties":{}},{"name":"act2","type":"Copy","typeProperties":{}}]}}' | ConvertFrom-Json + $script:result = Save-SynapseObjectAsFile -obj $obj + } + It 'Should preserve all activities and write them as a JSON array' { + $json = Get-Content $script:result -Raw | ConvertFrom-Json + @($json.properties.activities).Count | Should -Be 2 + } + } + + Context 'ForEach nested single activity coerced to object by ConvertFrom-Json' { + BeforeAll { + $synapse = [Synapse]::new() + $synapse.Location = $script:tmpDir.FullName + $obj = [SynapseObject]::new() + $obj.Name = 'pl_foreach_single_nested_activity' + $obj.Type = 'pipeline' + $obj.Synapse = $synapse + $obj.Body = '{"name":"pl_foreach_single_nested_activity","properties":{"activities":[{"name":"forEach1","type":"ForEach","typeProperties":{"activities":{"name":"act1","type":"Copy","typeProperties":{}}}}]}}' | ConvertFrom-Json + $script:result = Save-SynapseObjectAsFile -obj $obj + } + It 'Should serialise nested typeProperties.activities as a JSON array' { + $raw = Get-Content $script:result -Raw + $raw | Should -Match '"typeProperties":\{"activities":\[' + $raw | Should -Not -Match '"typeProperties":\{"activities":\{' + } + } + + Context 'Non-pipeline object' { + BeforeAll { + $synapse = [Synapse]::new() + $synapse.Location = $script:tmpDir.FullName + $obj = [SynapseObject]::new() + $obj.Name = 'ds_test' + $obj.Type = 'dataset' + $obj.Synapse = $synapse + $obj.Body = '{"name":"ds_test","properties":{"type":"AzureBlob"}}' | ConvertFrom-Json + $script:result = Save-SynapseObjectAsFile -obj $obj + } + It 'Should write the object to file without modification' { + $json = Get-Content $script:result -Raw | ConvertFrom-Json + $json.name | Should -Be 'ds_test' + $json.properties.type | Should -Be 'AzureBlob' + } + } + } +}