From 0e8b08cbaa194303e59b77832fb613c55a1da39d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:13:37 +0000 Subject: [PATCH 01/19] Initial plan From ff81f990757eb1852322d8e5e588bb13d13e1794 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:15:54 +0000 Subject: [PATCH 02/19] Initial plan From f5011dbc39e12b05e9b17473b18b8359ba1a1fad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:26:21 +0000 Subject: [PATCH 03/19] Implement improved dependency management for AzureDataLakeManagement module Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- .../AzureDataLakeManagement.psd1 | 7 +- .../AzureDataLakeManagement.psm1 | 264 ++++++++++++++++-- README.md | 51 ++++ Tests/DependencyManagement.Tests.ps1 | 96 +++++++ example-dependency-management.ps1 | 29 ++ 5 files changed, 427 insertions(+), 20 deletions(-) create mode 100644 Tests/DependencyManagement.Tests.ps1 create mode 100644 example-dependency-management.ps1 diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index f5a77fe..6692573 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -51,7 +51,7 @@ PowerShellVersion = '5.1' # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module -RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') +# RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() @@ -72,7 +72,8 @@ RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') FunctionsToExport = 'Get-AADObjectId', 'Get-AzureSubscriptionInfo', 'Add-DataLakeFolder', 'Remove-DataLakeFolder', 'Set-DataLakeFolderACL', 'Get-DataLakeFolderACL', 'Move-DataLakeFolder', - 'Remove-DataLakeFolderACL' + 'Remove-DataLakeFolderACL', 'Test-ModuleDependencies', + 'Install-ModuleDependencies', 'Import-ModuleDependencies' # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. CmdletsToExport = @() @@ -119,7 +120,7 @@ PrivateData = @{ # RequireLicenseAcceptance = $false # External dependent modules of this module - # ExternalModuleDependencies = @() + ExternalModuleDependencies = @('Az.Storage', 'AzureAD', 'Az.Accounts') } # End of PSData hashtable diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index 4af5223..769187e 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -1,3 +1,213 @@ +#region Dependency Management Functions + +<# +.SYNOPSIS + Tests if required modules are available and optionally installs them. + +.DESCRIPTION + The Test-ModuleDependencies function checks if the required modules for AzureDataLakeManagement are available. + It can optionally install missing modules and provides user feedback about the dependency status. + +.PARAMETER AutoInstall + If specified, automatically installs missing required modules from PowerShell Gallery. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Test-ModuleDependencies + Checks for required modules and displays status information. + +.EXAMPLE + PS C:\> Test-ModuleDependencies -AutoInstall + Checks for required modules and automatically installs any that are missing. + +.NOTES + Required modules: Az.Storage, AzureAD, Az.Accounts + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Test-ModuleDependencies { + [CmdletBinding()] + param( + [switch]$AutoInstall, + [switch]$Quiet + ) + + $requiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') + $missingModules = @() + $availableModules = @() + + if (-not $Quiet) { + Write-Host "Checking AzureDataLakeManagement module dependencies..." -ForegroundColor Yellow + } + + foreach ($moduleName in $requiredModules) { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $missingModules += $moduleName + if (-not $Quiet) { + Write-Warning "Missing required module: $moduleName" + } + } else { + $availableModules += $moduleName + if (-not $Quiet) { + Write-Host "✓ Found module: $moduleName (Version: $($module[0].Version))" -ForegroundColor Green + } + } + } + + if ($missingModules.Count -eq 0) { + if (-not $Quiet) { + Write-Host "✓ All required modules are available." -ForegroundColor Green + } + return $true + } + + if ($AutoInstall) { + if (-not $Quiet) { + Write-Host "Installing missing modules..." -ForegroundColor Yellow + } + return Install-ModuleDependencies -Modules $missingModules -Quiet:$Quiet + } else { + if (-not $Quiet) { + Write-Host "`nTo install missing modules, run:" -ForegroundColor Cyan + Write-Host "Test-ModuleDependencies -AutoInstall" -ForegroundColor White + Write-Host "`nOr install manually:" -ForegroundColor Cyan + foreach ($module in $missingModules) { + Write-Host "Install-Module -Name $module -Force" -ForegroundColor White + } + } + return $false + } +} + +<# +.SYNOPSIS + Installs required modules for AzureDataLakeManagement. + +.DESCRIPTION + The Install-ModuleDependencies function installs the specified required modules from PowerShell Gallery. + +.PARAMETER Modules + Array of module names to install. If not specified, installs all required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Install-ModuleDependencies + Installs all required modules for AzureDataLakeManagement. + +.EXAMPLE + PS C:\> Install-ModuleDependencies -Modules @('Az.Storage') + Installs only the Az.Storage module. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Install-ModuleDependencies { + [CmdletBinding()] + param( + [string[]]$Modules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [switch]$Quiet + ) + + $successCount = 0 + $failureCount = 0 + + foreach ($moduleName in $Modules) { + try { + if (-not $Quiet) { + Write-Host "Installing module: $moduleName..." -ForegroundColor Yellow + } + + Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber -ErrorAction Stop + + if (-not $Quiet) { + Write-Host "✓ Successfully installed: $moduleName" -ForegroundColor Green + } + $successCount++ + } + catch { + Write-Error "Failed to install module $moduleName`: $($_.Exception.Message)" + $failureCount++ + } + } + + if (-not $Quiet) { + if ($failureCount -eq 0) { + Write-Host "✓ All modules installed successfully." -ForegroundColor Green + } else { + Write-Warning "Installed $successCount modules, failed to install $failureCount modules." + } + } + + return ($failureCount -eq 0) +} + +<# +.SYNOPSIS + Imports required modules with proper error handling. + +.DESCRIPTION + The Import-ModuleDependencies function imports the required modules for AzureDataLakeManagement functions. + It provides better error handling and user feedback compared to individual Import-Module calls. + +.PARAMETER RequiredModules + Array of module names to import. Defaults to the core required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Import-ModuleDependencies + Imports all required modules for AzureDataLakeManagement. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Import-ModuleDependencies { + [CmdletBinding()] + param( + [string[]]$RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [switch]$Quiet + ) + + $importFailures = @() + + foreach ($moduleName in $RequiredModules) { + try { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $importFailures += $moduleName + Write-Error "Module $moduleName is not available. Please install it first." + continue + } + + Import-Module -Name $moduleName -ErrorAction Stop -Force + if (-not $Quiet) { + Write-Verbose "Successfully imported module: $moduleName" + } + } + catch { + $importFailures += $moduleName + Write-Error "Failed to import module $moduleName`: $($_.Exception.Message)" + } + } + + if ($importFailures.Count -gt 0) { + Write-Error "Failed to import modules: $($importFailures -join ', '). Some functions may not work correctly." + return $false + } + + return $true +} + +#endregion + <# .SYNOPSIS This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. @@ -253,10 +463,10 @@ function Add-DataLakeFolder return } - # Check if the Az.Storage module is installed - if (-not (Get-Module -Name Az.Storage -ListAvailable)) - { - Import-Module -Name Az.Storage # Install the Az.Storage module if it's not installed + # Check if the Az.Storage module is available and import it + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { + Write-Error 'Required module Az.Storage is not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return } # Get the Data Lake Storage account @@ -503,10 +713,10 @@ function Set-DataLakeFolderACL [switch]$DoNotApplyACLRecursively ) - if (-not (Get-Module -Name Az.Storage -ListAvailable)) - { - Write-Verbose 'Installing Az.Storage module.' - Import-Module -Name Az.Storage + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return } $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName @@ -754,9 +964,11 @@ function Get-DataLakeFolderACL [string]$FolderPath = '/' # Path to the folder in the Data Lake ) - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } # Remove leading slash or backslash from the folder path if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) @@ -867,9 +1079,11 @@ function Move-DataLakeFolder [string]$DestinationFolderPath # Destination folder path ) - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } try { @@ -965,9 +1179,11 @@ function Remove-DataLakeFolderACL ) - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } # Remove leading slash or backslash from the folder path if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) @@ -1023,4 +1239,18 @@ function Remove-DataLakeFolderACL } } +#region Module Initialization +# This code runs when the module is imported +# Check dependencies on module import +$dependencyCheckResult = Test-ModuleDependencies -Quiet + +if (-not $dependencyCheckResult) { + Write-Warning @" +AzureDataLakeManagement module loaded with missing dependencies. +Some functions may not work correctly until required modules are installed. +Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +"@ +} +#endregion + Export-ModuleMember -Function * diff --git a/README.md b/README.md index 99ddf8a..e02b31a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,57 @@ My goal, is to make a straight forward set of functions that will assist a user To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement +## Dependency Management + +Starting with version 2025.1.1, the module includes improved dependency management features: + +### Required Dependencies +The module requires the following PowerShell modules: +- `Az.Storage` - For Azure Storage operations +- `AzureAD` - For Azure Active Directory operations +- `Az.Accounts` - For Azure authentication + +### Automatic Dependency Checking +When you import the module, it automatically checks for missing dependencies and provides helpful guidance: + +```powershell +Import-Module AzureDataLakeManagement +# Output: WARNING: AzureDataLakeManagement module loaded with missing dependencies. +# Some functions may not work correctly until required modules are installed. +# Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +``` + +### Dependency Management Functions + +#### Test-ModuleDependencies +Check which dependencies are available: +```powershell +Test-ModuleDependencies +``` + +Automatically install missing dependencies: +```powershell +Test-ModuleDependencies -AutoInstall +``` + +#### Install-ModuleDependencies +Install all or specific dependencies: +```powershell +Install-ModuleDependencies +Install-ModuleDependencies -Modules @('Az.Storage') +``` + +#### Manual Installation +You can also install dependencies manually: +```powershell +Install-Module -Name Az.Storage -Force +Install-Module -Name AzureAD -Force +Install-Module -Name Az.Accounts -Force +``` + +### Improved Error Handling +Functions now provide clearer error messages when dependencies are missing, guiding users to install the required modules. + *** ## Version History: diff --git a/Tests/DependencyManagement.Tests.ps1 b/Tests/DependencyManagement.Tests.ps1 new file mode 100644 index 0000000..3b9f86b --- /dev/null +++ b/Tests/DependencyManagement.Tests.ps1 @@ -0,0 +1,96 @@ +#Requires -Modules Pester + +BeforeAll { + # Import the module + $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' + Import-Module $ModulePath -Force +} + +Describe 'AzureDataLakeManagement Dependency Management' { + + Context 'Test-ModuleDependencies Function' { + It 'Should export Test-ModuleDependencies function' { + Get-Command -Name 'Test-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Test-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'AutoInstall' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Storage' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'AzureAD' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Accounts' } + + $result = Test-ModuleDependencies -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Install-ModuleDependencies Function' { + It 'Should export Install-ModuleDependencies function' { + Get-Command -Name 'Install-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Install-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'Modules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + } + + Context 'Import-ModuleDependencies Function' { + It 'Should export Import-ModuleDependencies function' { + Get-Command -Name 'Import-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Import-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'RequiredModules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $ListAvailable -eq $true } + $result = Import-ModuleDependencies -RequiredModules @('NonExistentModule') -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Module Manifest' { + It 'Should declare external module dependencies in manifest' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'AzureAD' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Accounts' + } + + It 'Should export dependency management functions' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.ExportedFunctions.Keys | Should -Contain 'Test-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Install-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Import-ModuleDependencies' + } + } + + Context 'Integration with Existing Functions' { + It 'Should have updated Add-DataLakeFolder to use centralized dependency checking' { + $functionContent = (Get-Command Add-DataLakeFolder).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + + It 'Should have updated Set-DataLakeFolderACL to use centralized dependency checking' { + $functionContent = (Get-Command Set-DataLakeFolderACL).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + } +} + +AfterAll { + # Clean up + Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue +} \ No newline at end of file diff --git a/example-dependency-management.ps1 b/example-dependency-management.ps1 new file mode 100644 index 0000000..1bcfed1 --- /dev/null +++ b/example-dependency-management.ps1 @@ -0,0 +1,29 @@ +# Example script demonstrating improved dependency management in AzureDataLakeManagement + +# Import the module - it will now provide helpful feedback about missing dependencies +Import-Module .\AzureDataLakeManagement\AzureDataLakeManagement.psd1 + +# Check what dependencies are missing +Test-ModuleDependencies + +# To install missing dependencies automatically, you can run: +# Test-ModuleDependencies -AutoInstall + +# Or install them manually: +# Install-Module -Name Az.Storage -Force +# Install-Module -Name AzureAD -Force +# Install-Module -Name Az.Accounts -Force + +# The module will now provide better error messages when functions are called without required dependencies +# For example: +# Add-DataLakeFolder -SubscriptionName 'test' -ResourceGroupName 'test' -StorageAccountName 'test' -ContainerName 'test' -FolderPath 'test' + +Write-Host " +Dependency Management Features: +- Test-ModuleDependencies: Check which dependencies are available +- Test-ModuleDependencies -AutoInstall: Automatically install missing dependencies +- Install-ModuleDependencies: Install specific dependencies +- Import-ModuleDependencies: Import dependencies with better error handling + +The module will automatically check dependencies when imported and provide helpful guidance. +" -ForegroundColor Green \ No newline at end of file From 2acf7e68ee377a9c90f965f62fc7f83bbdcd59cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:28:08 +0000 Subject: [PATCH 04/19] Add comprehensive GitHub Copilot instructions for Azure Data Lake Management module Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- .github/copilot-instructions.md | 322 ++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..1ca88d4 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,322 @@ +# Azure Data Lake Management PowerShell Module + +Always follow these instructions first and only search or use bash commands when you encounter information that contradicts what is documented here or when these instructions are incomplete. + +This repository contains a PowerShell module for managing Azure Data Lake Storage Gen 2 folders and Access Control Lists (ACLs). The module simplifies ACL management by using object names rather than IDs and provides functions to create, delete, move folders and manage permissions recursively. + +## Working Effectively + +### Prerequisites and Environment Setup +- Install PowerShell 7+ (PowerShell Core): Download from https://github.com/PowerShell/PowerShell/releases +- Install required Azure PowerShell modules: + ```powershell + Install-Module -Name Az.Storage -Scope CurrentUser -Force + Install-Module -Name AzureAD -Scope CurrentUser -Force + Install-Module -Name Az.Accounts -Scope CurrentUser -Force + ``` +- Authenticate to Azure before testing: + ```powershell + Connect-AzAccount + Connect-AzureAD + ``` + +### Code Quality and Validation +- Run PSScriptAnalyzer for code quality checks (takes ~3 seconds): + ```powershell + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + ``` +- Test module manifest (takes ~1 second): + ```powershell + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + **Note**: Will show warnings about missing Az.Storage, AzureAD, and Az.Accounts modules if they're not installed. This is expected in offline environments. +- ALWAYS run PSScriptAnalyzer before committing changes or the code quality will deteriorate. + +### Offline Development and Testing +When Azure modules or connectivity is not available: +- Module import will work but functions will fail at runtime +- PSScriptAnalyzer and manifest testing work completely offline +- Function syntax and help documentation can be validated offline +- Use these commands for offline validation: + ```powershell + # These work without Azure connectivity + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + Get-Help Add-DataLakeFolder -Examples + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + +### Module Development and Testing +- Import the module for testing (~1 second): + ```powershell + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + ``` +- Get available functions: + ```powershell + Get-Command -Module AzureDataLakeManagement + ``` +- Access function help and examples (~1 second per function): + ```powershell + Get-Help Add-DataLakeFolder -Examples + Get-Help Set-DataLakeFolderACL -Full + ``` +- **CRITICAL**: Test functions only with test/development Azure resources. Never test against production data. + +### Publishing Process +- **Prerequisites for Publishing**: + - PowerShell Gallery API Key (set as environment variable `PSGalleryKey`) + - Module version updated in `.psd1` file + - All PSScriptAnalyzer warnings addressed + - Manual validation completed + +- **Manual publish to PowerShell Gallery** (~30 seconds): + ```powershell + # Set your API key first + $env:PSGalleryKey = "your-api-key-here" + .\publish.ps1 + ``` + +- **GitHub Actions publish**: + - Workflow: `.github/workflows/manual_publish.yml` + - Requires `PSGalleryKey` secret configured in repository + - Manually triggered via GitHub Actions UI + - Uses `workflow_dispatch` trigger (not automatic) + +- **Pre-publish validation checklist**: + ```powershell + # 1. Code quality check + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + + # 2. Module manifest validation + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + + # 3. Module import test + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + + # 4. Verify version number is updated in .psd1 + # 5. Complete manual validation scenarios with test Azure resources + ``` + +## Key Module Components + +### Primary Functions (8 total): +1. **Get-AADObjectId** - Retrieve Azure AD object details by name/UPN +2. **Get-AzureSubscriptionInfo** - Get subscription information +3. **Add-DataLakeFolder** - Create folder structures in Data Lake Storage +4. **Remove-DataLakeFolder** - Delete folders from Data Lake Storage +5. **Set-DataLakeFolderACL** - Apply ACL permissions to folders (recursively) +6. **Get-DataLakeFolderACL** - Retrieve current ACL permissions +7. **Move-DataLakeFolder** - Move/rename folders between containers +8. **Remove-DataLakeFolderACL** - Remove ACL permissions from folders + +### Core Files: +- `AzureDataLakeManagement/AzureDataLakeManagement.psm1` - Main module (1026 lines, 8 functions) +- `AzureDataLakeManagement/AzureDataLakeManagement.psd1` - Module manifest and metadata +- `example.ps1` - Complete usage examples showing folder creation and ACL management +- `publish.ps1` - PowerShell Gallery publishing script + +## Validation and Testing + +### Code Quality Validation +Run these before every commit: +```powershell +# Static analysis (3 seconds) - NEVER CANCEL +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Module manifest validation (1 second) +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Manual Validation Scenarios +**CRITICAL**: Always test against development/test Azure resources only. Complete these scenarios after making changes: + +1. **Authentication and Module Import Test** (1-2 minutes): + ```powershell + Connect-AzAccount + Connect-AzureAD + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + # Should show all 8 functions + ``` + +2. **Basic Folder Operations Test** (5-10 minutes): + ```powershell + # Use test subscription and storage account + $subName = 'your-test-subscription' + $rgName = 'test-resource-group' + $storageAccountName = 'teststorageaccount' + $containerName = 'test-container' + + # Create test folder structure + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset\sample-folder' + + # Verify folder exists in Azure Storage Explorer or portal + + # Test folder move operation + Move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'test-dataset\sample-folder' -DestinationContainerName $containerName -destinationFolderPath 'test-dataset\moved-folder' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset' + ``` + +3. **ACL Management Test** (5-10 minutes): + ```powershell + # Create test folder + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + + # Apply test ACL (use test user/group) + Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' -accessControlType Read + + # Verify ACL was applied + Get-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' + + # Test ACL removal + Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + ``` + +4. **Azure AD Object Resolution Test** (2-3 minutes): + ```powershell + # Test user lookup + Get-AADObjectId -Identity 'test-user@domain.com' + + # Test group lookup + Get-AADObjectId -Identity 'Test Group Name' + + # Test service principal lookup + Get-AADObjectId -Identity 'Test Service Principal' + + # Should return ObjectId, ObjectType, and DisplayName for each + ``` + +5. **Complete example.ps1 Workflow Test** (10-15 minutes): + ```powershell + # Modify variables in example.ps1 first, then run: + .\example.ps1 + + # Verify in Azure portal: + # - Multiple folder structures created + # - ACL permissions applied correctly + # - Test folders cleaned up properly + ``` + +### Development Workflow Timing +- PSScriptAnalyzer execution: ~3 seconds - NEVER CANCEL +- Module manifest testing: ~1 second +- Module import: ~1 second (without Azure dependencies) +- Module import with Azure modules: ~2-5 seconds (depends on Azure module loading) +- Single folder operation: ~3-10 seconds (depends on Azure latency) +- ACL operations: ~5-15 seconds (depends on Azure AD latency and hierarchy depth) +- Full validation scenario: ~10-20 minutes total +- Complete example.ps1 workflow: ~5-15 minutes (creates multiple folders and ACLs) + +### Using example.ps1 for Learning +The `example.ps1` file demonstrates a complete workflow: +1. Authentication to Azure and Azure AD +2. Creating hierarchical folder structures +3. Setting various ACL types (user, group, service principal) +4. Error handling scenarios +5. Cleanup operations + +**CRITICAL**: Always modify the variables in example.ps1 before running: +```powershell +$subName = 'your-test-subscription' # Change this +$rgName = 'your-test-resource-group' # Change this +$storageAccountName = 'your-test-storage' # Change this +$containerName = 'test-container' # Change this +``` + +## Common Development Tasks + +### Adding New Functions +1. Add function to `AzureDataLakeManagement.psm1` +2. Update `FunctionsToExport` in `AzureDataLakeManagement.psd1` +3. Add usage example to `example.ps1` +4. Run PSScriptAnalyzer validation +5. Test with manual validation scenarios + +### Debugging Issues +- Use VS Code with PowerShell extension for debugging +- Import module with `-Force` to reload changes +- Use `-Verbose` parameter on functions for detailed output +- Check Azure portal/Storage Explorer to verify actual changes + +### Common Error Scenarios +1. **Missing Azure Authentication**: Functions fail with authentication errors + - Solution: Run `Connect-AzAccount` and `Connect-AzureAD` + +2. **Module Dependencies Missing**: Import fails with module not found errors + - Solution: Install Az.Storage, AzureAD, and Az.Accounts modules + +3. **Path Format Issues**: Functions expect backslash separators in folderPath + - Correct: `'dataset1\folder1\subfolder'` + - Incorrect: `'dataset1/folder1/subfolder'` + +4. **Permissions Issues**: ACL operations fail due to insufficient permissions + - Ensure Azure AD permissions and Storage Account permissions are configured + +5. **Storage Account Access**: Operations fail if storage account keys are not accessible + - Module requires either storage account key access OR proper Azure AD permissions + +### Known Code Quality Issues +PSScriptAnalyzer currently identifies 17 warnings that should be addressed in new code: +- 5 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) +- 6 unused parameter warnings (remove unused parameters) +- 3 unused variable warnings (remove unused variables) +- 3 missing `ShouldProcess` support warnings for state-changing functions (Add-DataLakeFolder, Set-DataLakeFolderACL, Remove-DataLakeFolderACL) + +Run `Invoke-ScriptAnalyzer` to see the complete list with line numbers and detailed guidance. + +## Repository Structure Reference +``` +. +├── .github/ +│ └── workflows/ +│ └── manual_publish.yml # GitHub Actions publishing workflow +├── .vscode/ +│ ├── launch.json # VS Code debugging configuration +│ └── settings.json # VS Code settings +├── AzureDataLakeManagement/ +│ ├── AzureDataLakeManagement.psd1 # Module manifest +│ └── AzureDataLakeManagement.psm1 # Main module (8 functions) +├── .gitignore +├── LICENSE +├── README.md # Project overview and version history +├── example.ps1 # Complete usage examples +└── publish.ps1 # PowerShell Gallery publishing script +``` + +## Quick Reference Commands + +### Daily Development +```powershell +# Load and test module +Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + +# Code quality check (run before commit) +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Test manifest +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Azure Authentication +```powershell +Connect-AzAccount +Connect-AzureAD +Get-AzSubscription # Verify connection +``` + +### Function Usage Pattern +```powershell +# All functions follow this parameter pattern: +-SubscriptionName # Azure subscription name +-ResourceGroupName # Resource group containing storage account +-StorageAccountName # Storage account name +-ContainerName # Container/filesystem name +-folderPath # Path within container (use backslash separators) +``` \ No newline at end of file From 8139773076e34785e6129bacfa101daa679d79f7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:19:11 +0000 Subject: [PATCH 05/19] Initial plan From 19b73aca5de321dab9574cad32328edef25b2b4a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:25:15 +0000 Subject: [PATCH 06/19] Add dev container support with PowerShell and required VS Code extensions Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- .devcontainer/devcontainer.json | 46 +++++++++++++++++++++++++++++++++ README.md | 43 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..42de1c0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,46 @@ +{ + "name": "Azure Data Lake Management PowerShell", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + + // Features to install - PowerShell is a predefined feature + "features": { + "ghcr.io/devcontainers/features/powershell:1": { + "version": "latest" + } + }, + + // Configure tool-specific properties + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create + "settings": { + "terminal.integrated.defaultProfile.linux": "pwsh", + "powershell.powerShellDefaultVersion": "PowerShell (latest)", + "githubPullRequests.remotes": [ + "https://github.com/SteveCInVA/AzureDataLakeManagement.git" + ] + }, + + // Add the IDs of extensions you want installed when the container is created + "extensions": [ + "ms-vscode.powershell", + "pspester.pester-test", + "github.copilot", + "github.vscode-github-actions", + "jgclark.vscode-todo-highlight" + ] + } + }, + + // Use 'postCreateCommand' to run commands after the container is created + "postCreateCommand": "pwsh -Command 'Install-Module -Name PSScriptAnalyzer, Pester -Force -Scope CurrentUser -SkipPublisherCheck'" + + // Use 'forwardPorts' to make a list of ports inside the container available locally + // "forwardPorts": [], + + // Use 'postStartCommand' to run commands each time the container starts + // "postStartCommand": "" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root + // "remoteUser": "root" +} diff --git a/README.md b/README.md index e02b31a..fde89c8 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,49 @@ My goal, is to make a straight forward set of functions that will assist a user To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement +## Development Environment + +### Using Dev Containers (Recommended) + +This repository includes support for Visual Studio Code dev containers, providing a consistent development environment with all required tools pre-installed. + +#### Prerequisites +- [Visual Studio Code](https://code.visualstudio.com/) +- [Docker Desktop](https://www.docker.com/products/docker-desktop) +- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VS Code + +#### Getting Started with Dev Containers +1. Clone the repository +2. Open the repository folder in Visual Studio Code +3. When prompted, click "Reopen in Container" (or use Command Palette: `Dev Containers: Reopen in Container`) +4. VS Code will build the container and install all dependencies automatically + +#### What's Included +The dev container includes: +- **PowerShell 7+** - Latest version installed automatically +- **Pre-installed VS Code Extensions:** + - PowerShell - Language support and debugging + - Pester Test - Testing framework support + - GitHub Copilot - AI-powered code assistance + - GitHub Actions - Workflow file support + - TODO Highlight v2 - Highlight TODO comments +- **PowerShell Modules:** + - PSScriptAnalyzer - For code quality checks + - Pester - For testing + +#### Working in the Dev Container +Once the container is running, you can: +- Import the module: `Import-Module ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 -Force` +- Run code quality checks: `Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1` +- Test the module: `Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1` +- Run Pester tests: `Invoke-Pester -Path ./Tests` + +### Local Development (Without Dev Containers) + +If you prefer to develop locally without containers, ensure you have: +- PowerShell 7+ installed +- Required PowerShell modules (see Dependency Management section below) + ## Dependency Management Starting with version 2025.1.1, the module includes improved dependency management features: From 4b4209afdf1583138942b0bc618b3bd6aed8e237 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:28:06 +0000 Subject: [PATCH 07/19] Add comprehensive tests for dev container configuration Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- Tests/DevContainer.Tests.ps1 | 89 ++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 Tests/DevContainer.Tests.ps1 diff --git a/Tests/DevContainer.Tests.ps1 b/Tests/DevContainer.Tests.ps1 new file mode 100644 index 0000000..d2564a2 --- /dev/null +++ b/Tests/DevContainer.Tests.ps1 @@ -0,0 +1,89 @@ +#Requires -Modules Pester + +BeforeAll { + $DevContainerPath = Join-Path $PSScriptRoot '..' '.devcontainer' 'devcontainer.json' +} + +Describe 'DevContainer Configuration' { + + Context 'File Existence' { + It 'Should have a devcontainer.json file' { + Test-Path $DevContainerPath | Should -BeTrue + } + } + + Context 'Configuration Content' { + BeforeAll { + # Read and parse the devcontainer.json file (JSONC - JSON with Comments) + $content = Get-Content $DevContainerPath -Raw + # Remove single-line comments more carefully (avoid URLs with //) + $lines = $content -split "`n" + $cleanedLines = @() + foreach ($line in $lines) { + # Skip comment-only lines + if ($line -match '^\s*//') { + continue + } + # Remove trailing comments but preserve URLs and quoted strings + # Match everything before comment outside of quotes + elseif ($line -match '^([^"]*"[^"]*"[^"]*)\s*//' -or $line -match '^(.*[^:])\s*//') { + # Keep content before comment, but be careful with URLs + if ($line -notmatch '"https?://') { + $cleanedLines += $matches[1] + } + else { + # Line contains URL, keep it as-is + $cleanedLines += $line + } + } + else { + # Keep line as-is + $cleanedLines += $line + } + } + $jsonContent = $cleanedLines -join "`n" + $config = $jsonContent | ConvertFrom-Json + } + + It 'Should have a name property' { + $config.name | Should -Not -BeNullOrEmpty + } + + It 'Should specify PowerShell feature' { + $config.features.PSObject.Properties.Name | Should -Contain 'ghcr.io/devcontainers/features/powershell:1' + } + + It 'Should have PowerShell extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'ms-vscode.powershell' + } + + It 'Should have Pester Test extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'pspester.pester-test' + } + + It 'Should have GitHub Copilot extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.copilot' + } + + It 'Should have GitHub Actions extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.vscode-github-actions' + } + + It 'Should have TODO Highlight extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'jgclark.vscode-todo-highlight' + } + + It 'Should install PSScriptAnalyzer and Pester in postCreateCommand' { + $config.postCreateCommand | Should -Match 'PSScriptAnalyzer' + $config.postCreateCommand | Should -Match 'Pester' + } + + It 'Should set PowerShell as default terminal' { + $config.customizations.vscode.settings.'terminal.integrated.defaultProfile.linux' | Should -Be 'pwsh' + } + } +} + +AfterAll { + # Clean up +} From 11a711920c70add241e7943be233d6f9b8b3542f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:31:13 +0000 Subject: [PATCH 08/19] Clean up test file - remove empty AfterAll block Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- Tests/DevContainer.Tests.ps1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tests/DevContainer.Tests.ps1 b/Tests/DevContainer.Tests.ps1 index d2564a2..175abe7 100644 --- a/Tests/DevContainer.Tests.ps1 +++ b/Tests/DevContainer.Tests.ps1 @@ -83,7 +83,3 @@ Describe 'DevContainer Configuration' { } } } - -AfterAll { - # Clean up -} From 6cb9e14afbcd46075fe70aaa29280bd0e7c133b1 Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Tue, 4 Nov 2025 16:39:57 +0000 Subject: [PATCH 09/19] README with module usage and dependency management details, improve test scripts for dependency management functions, update example scripts for clarity, and ensure consistent formatting in publish script. --- .devcontainer/devcontainer.json | 92 +- .github/copilot-instructions.md | 642 ++--- .github/workflows/manual_publish.yml | 36 +- .gitignore | 4 +- .vscode/launch.json | 28 +- .vscode/settings.json | 11 +- .../AzureDataLakeManagement.psd1 | 272 +- .../AzureDataLakeManagement.psm1 | 2512 ++++++++--------- LICENSE | 42 +- README.md | 277 +- Tests/DependencyManagement.Tests.ps1 | 190 +- Tests/DevContainer.Tests.ps1 | 170 +- example-dependency-management.ps1 | 56 +- example.ps1 | 92 +- publish.ps1 | 14 +- 15 files changed, 2234 insertions(+), 2204 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 42de1c0..44d2603 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,46 +1,46 @@ -{ - "name": "Azure Data Lake Management PowerShell", - "image": "mcr.microsoft.com/devcontainers/base:ubuntu", - - // Features to install - PowerShell is a predefined feature - "features": { - "ghcr.io/devcontainers/features/powershell:1": { - "version": "latest" - } - }, - - // Configure tool-specific properties - "customizations": { - "vscode": { - // Set *default* container specific settings.json values on container create - "settings": { - "terminal.integrated.defaultProfile.linux": "pwsh", - "powershell.powerShellDefaultVersion": "PowerShell (latest)", - "githubPullRequests.remotes": [ - "https://github.com/SteveCInVA/AzureDataLakeManagement.git" - ] - }, - - // Add the IDs of extensions you want installed when the container is created - "extensions": [ - "ms-vscode.powershell", - "pspester.pester-test", - "github.copilot", - "github.vscode-github-actions", - "jgclark.vscode-todo-highlight" - ] - } - }, - - // Use 'postCreateCommand' to run commands after the container is created - "postCreateCommand": "pwsh -Command 'Install-Module -Name PSScriptAnalyzer, Pester -Force -Scope CurrentUser -SkipPublisherCheck'" - - // Use 'forwardPorts' to make a list of ports inside the container available locally - // "forwardPorts": [], - - // Use 'postStartCommand' to run commands each time the container starts - // "postStartCommand": "" - - // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root - // "remoteUser": "root" -} +{ + "name": "Azure Data Lake Management PowerShell", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + + // Features to install - PowerShell is a predefined feature + "features": { + "ghcr.io/devcontainers/features/powershell:1": { + "version": "latest" + } + }, + + // Configure tool-specific properties + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create + "settings": { + "terminal.integrated.defaultProfile.linux": "pwsh", + "powershell.powerShellDefaultVersion": "PowerShell (latest)", + "githubPullRequests.remotes": [ + "https://github.com/SteveCInVA/AzureDataLakeManagement.git" + ] + }, + + // Add the IDs of extensions you want installed when the container is created + "extensions": [ + "ms-vscode.powershell", + "pspester.pester-test", + "github.copilot", + "github.vscode-github-actions", + "jgclark.vscode-todo-highlight" + ] + } + }, + + // Use 'postCreateCommand' to run commands after the container is created + "postCreateCommand": "pwsh -Command 'Install-Module -Name PSScriptAnalyzer, Pester -Force -Scope CurrentUser -SkipPublisherCheck'" + + // Use 'forwardPorts' to make a list of ports inside the container available locally + // "forwardPorts": [], + + // Use 'postStartCommand' to run commands each time the container starts + // "postStartCommand": "" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root + // "remoteUser": "root" +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1ca88d4..ac64c75 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,322 +1,322 @@ -# Azure Data Lake Management PowerShell Module - -Always follow these instructions first and only search or use bash commands when you encounter information that contradicts what is documented here or when these instructions are incomplete. - -This repository contains a PowerShell module for managing Azure Data Lake Storage Gen 2 folders and Access Control Lists (ACLs). The module simplifies ACL management by using object names rather than IDs and provides functions to create, delete, move folders and manage permissions recursively. - -## Working Effectively - -### Prerequisites and Environment Setup -- Install PowerShell 7+ (PowerShell Core): Download from https://github.com/PowerShell/PowerShell/releases -- Install required Azure PowerShell modules: - ```powershell - Install-Module -Name Az.Storage -Scope CurrentUser -Force - Install-Module -Name AzureAD -Scope CurrentUser -Force - Install-Module -Name Az.Accounts -Scope CurrentUser -Force - ``` -- Authenticate to Azure before testing: - ```powershell - Connect-AzAccount - Connect-AzureAD - ``` - -### Code Quality and Validation -- Run PSScriptAnalyzer for code quality checks (takes ~3 seconds): - ```powershell - Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 - ``` -- Test module manifest (takes ~1 second): - ```powershell - Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 - ``` - **Note**: Will show warnings about missing Az.Storage, AzureAD, and Az.Accounts modules if they're not installed. This is expected in offline environments. -- ALWAYS run PSScriptAnalyzer before committing changes or the code quality will deteriorate. - -### Offline Development and Testing -When Azure modules or connectivity is not available: -- Module import will work but functions will fail at runtime -- PSScriptAnalyzer and manifest testing work completely offline -- Function syntax and help documentation can be validated offline -- Use these commands for offline validation: - ```powershell - # These work without Azure connectivity - Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' - Get-Command -Module AzureDataLakeManagement - Get-Help Add-DataLakeFolder -Examples - Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 - Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 - ``` - -### Module Development and Testing -- Import the module for testing (~1 second): - ```powershell - Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' - ``` -- Get available functions: - ```powershell - Get-Command -Module AzureDataLakeManagement - ``` -- Access function help and examples (~1 second per function): - ```powershell - Get-Help Add-DataLakeFolder -Examples - Get-Help Set-DataLakeFolderACL -Full - ``` -- **CRITICAL**: Test functions only with test/development Azure resources. Never test against production data. - -### Publishing Process -- **Prerequisites for Publishing**: - - PowerShell Gallery API Key (set as environment variable `PSGalleryKey`) - - Module version updated in `.psd1` file - - All PSScriptAnalyzer warnings addressed - - Manual validation completed - -- **Manual publish to PowerShell Gallery** (~30 seconds): - ```powershell - # Set your API key first - $env:PSGalleryKey = "your-api-key-here" - .\publish.ps1 - ``` - -- **GitHub Actions publish**: - - Workflow: `.github/workflows/manual_publish.yml` - - Requires `PSGalleryKey` secret configured in repository - - Manually triggered via GitHub Actions UI - - Uses `workflow_dispatch` trigger (not automatic) - -- **Pre-publish validation checklist**: - ```powershell - # 1. Code quality check - Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 - - # 2. Module manifest validation - Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 - - # 3. Module import test - Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' - Get-Command -Module AzureDataLakeManagement - - # 4. Verify version number is updated in .psd1 - # 5. Complete manual validation scenarios with test Azure resources - ``` - -## Key Module Components - -### Primary Functions (8 total): -1. **Get-AADObjectId** - Retrieve Azure AD object details by name/UPN -2. **Get-AzureSubscriptionInfo** - Get subscription information -3. **Add-DataLakeFolder** - Create folder structures in Data Lake Storage -4. **Remove-DataLakeFolder** - Delete folders from Data Lake Storage -5. **Set-DataLakeFolderACL** - Apply ACL permissions to folders (recursively) -6. **Get-DataLakeFolderACL** - Retrieve current ACL permissions -7. **Move-DataLakeFolder** - Move/rename folders between containers -8. **Remove-DataLakeFolderACL** - Remove ACL permissions from folders - -### Core Files: -- `AzureDataLakeManagement/AzureDataLakeManagement.psm1` - Main module (1026 lines, 8 functions) -- `AzureDataLakeManagement/AzureDataLakeManagement.psd1` - Module manifest and metadata -- `example.ps1` - Complete usage examples showing folder creation and ACL management -- `publish.ps1` - PowerShell Gallery publishing script - -## Validation and Testing - -### Code Quality Validation -Run these before every commit: -```powershell -# Static analysis (3 seconds) - NEVER CANCEL -Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 - -# Module manifest validation (1 second) -Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 -``` - -### Manual Validation Scenarios -**CRITICAL**: Always test against development/test Azure resources only. Complete these scenarios after making changes: - -1. **Authentication and Module Import Test** (1-2 minutes): - ```powershell - Connect-AzAccount - Connect-AzureAD - Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' - Get-Command -Module AzureDataLakeManagement - # Should show all 8 functions - ``` - -2. **Basic Folder Operations Test** (5-10 minutes): - ```powershell - # Use test subscription and storage account - $subName = 'your-test-subscription' - $rgName = 'test-resource-group' - $storageAccountName = 'teststorageaccount' - $containerName = 'test-container' - - # Create test folder structure - Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset\sample-folder' - - # Verify folder exists in Azure Storage Explorer or portal - - # Test folder move operation - Move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'test-dataset\sample-folder' -DestinationContainerName $containerName -destinationFolderPath 'test-dataset\moved-folder' - - # Clean up - Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset' - ``` - -3. **ACL Management Test** (5-10 minutes): - ```powershell - # Create test folder - Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' - - # Apply test ACL (use test user/group) - Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' -accessControlType Read - - # Verify ACL was applied - Get-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' - - # Test ACL removal - Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' - - # Clean up - Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' - ``` - -4. **Azure AD Object Resolution Test** (2-3 minutes): - ```powershell - # Test user lookup - Get-AADObjectId -Identity 'test-user@domain.com' - - # Test group lookup - Get-AADObjectId -Identity 'Test Group Name' - - # Test service principal lookup - Get-AADObjectId -Identity 'Test Service Principal' - - # Should return ObjectId, ObjectType, and DisplayName for each - ``` - -5. **Complete example.ps1 Workflow Test** (10-15 minutes): - ```powershell - # Modify variables in example.ps1 first, then run: - .\example.ps1 - - # Verify in Azure portal: - # - Multiple folder structures created - # - ACL permissions applied correctly - # - Test folders cleaned up properly - ``` - -### Development Workflow Timing -- PSScriptAnalyzer execution: ~3 seconds - NEVER CANCEL -- Module manifest testing: ~1 second -- Module import: ~1 second (without Azure dependencies) -- Module import with Azure modules: ~2-5 seconds (depends on Azure module loading) -- Single folder operation: ~3-10 seconds (depends on Azure latency) -- ACL operations: ~5-15 seconds (depends on Azure AD latency and hierarchy depth) -- Full validation scenario: ~10-20 minutes total -- Complete example.ps1 workflow: ~5-15 minutes (creates multiple folders and ACLs) - -### Using example.ps1 for Learning -The `example.ps1` file demonstrates a complete workflow: -1. Authentication to Azure and Azure AD -2. Creating hierarchical folder structures -3. Setting various ACL types (user, group, service principal) -4. Error handling scenarios -5. Cleanup operations - -**CRITICAL**: Always modify the variables in example.ps1 before running: -```powershell -$subName = 'your-test-subscription' # Change this -$rgName = 'your-test-resource-group' # Change this -$storageAccountName = 'your-test-storage' # Change this -$containerName = 'test-container' # Change this -``` - -## Common Development Tasks - -### Adding New Functions -1. Add function to `AzureDataLakeManagement.psm1` -2. Update `FunctionsToExport` in `AzureDataLakeManagement.psd1` -3. Add usage example to `example.ps1` -4. Run PSScriptAnalyzer validation -5. Test with manual validation scenarios - -### Debugging Issues -- Use VS Code with PowerShell extension for debugging -- Import module with `-Force` to reload changes -- Use `-Verbose` parameter on functions for detailed output -- Check Azure portal/Storage Explorer to verify actual changes - -### Common Error Scenarios -1. **Missing Azure Authentication**: Functions fail with authentication errors - - Solution: Run `Connect-AzAccount` and `Connect-AzureAD` - -2. **Module Dependencies Missing**: Import fails with module not found errors - - Solution: Install Az.Storage, AzureAD, and Az.Accounts modules - -3. **Path Format Issues**: Functions expect backslash separators in folderPath - - Correct: `'dataset1\folder1\subfolder'` - - Incorrect: `'dataset1/folder1/subfolder'` - -4. **Permissions Issues**: ACL operations fail due to insufficient permissions - - Ensure Azure AD permissions and Storage Account permissions are configured - -5. **Storage Account Access**: Operations fail if storage account keys are not accessible - - Module requires either storage account key access OR proper Azure AD permissions - -### Known Code Quality Issues -PSScriptAnalyzer currently identifies 17 warnings that should be addressed in new code: -- 5 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) -- 6 unused parameter warnings (remove unused parameters) -- 3 unused variable warnings (remove unused variables) -- 3 missing `ShouldProcess` support warnings for state-changing functions (Add-DataLakeFolder, Set-DataLakeFolderACL, Remove-DataLakeFolderACL) - -Run `Invoke-ScriptAnalyzer` to see the complete list with line numbers and detailed guidance. - -## Repository Structure Reference -``` -. -├── .github/ -│ └── workflows/ -│ └── manual_publish.yml # GitHub Actions publishing workflow -├── .vscode/ -│ ├── launch.json # VS Code debugging configuration -│ └── settings.json # VS Code settings -├── AzureDataLakeManagement/ -│ ├── AzureDataLakeManagement.psd1 # Module manifest -│ └── AzureDataLakeManagement.psm1 # Main module (8 functions) -├── .gitignore -├── LICENSE -├── README.md # Project overview and version history -├── example.ps1 # Complete usage examples -└── publish.ps1 # PowerShell Gallery publishing script -``` - -## Quick Reference Commands - -### Daily Development -```powershell -# Load and test module -Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' - -# Code quality check (run before commit) -Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 - -# Test manifest -Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 -``` - -### Azure Authentication -```powershell -Connect-AzAccount -Connect-AzureAD -Get-AzSubscription # Verify connection -``` - -### Function Usage Pattern -```powershell -# All functions follow this parameter pattern: --SubscriptionName # Azure subscription name --ResourceGroupName # Resource group containing storage account --StorageAccountName # Storage account name --ContainerName # Container/filesystem name --folderPath # Path within container (use backslash separators) +# Azure Data Lake Management PowerShell Module + +Always follow these instructions first and only search or use bash commands when you encounter information that contradicts what is documented here or when these instructions are incomplete. + +This repository contains a PowerShell module for managing Azure Data Lake Storage Gen 2 folders and Access Control Lists (ACLs). The module simplifies ACL management by using object names rather than IDs and provides functions to create, delete, move folders and manage permissions recursively. + +## Working Effectively + +### Prerequisites and Environment Setup +- Install PowerShell 7+ (PowerShell Core): Download from https://github.com/PowerShell/PowerShell/releases +- Install required Azure PowerShell modules: + ```powershell + Install-Module -Name Az.Storage -Scope CurrentUser -Force + Install-Module -Name AzureAD -Scope CurrentUser -Force + Install-Module -Name Az.Accounts -Scope CurrentUser -Force + ``` +- Authenticate to Azure before testing: + ```powershell + Connect-AzAccount + Connect-AzureAD + ``` + +### Code Quality and Validation +- Run PSScriptAnalyzer for code quality checks (takes ~3 seconds): + ```powershell + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + ``` +- Test module manifest (takes ~1 second): + ```powershell + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + **Note**: Will show warnings about missing Az.Storage, AzureAD, and Az.Accounts modules if they're not installed. This is expected in offline environments. +- ALWAYS run PSScriptAnalyzer before committing changes or the code quality will deteriorate. + +### Offline Development and Testing +When Azure modules or connectivity is not available: +- Module import will work but functions will fail at runtime +- PSScriptAnalyzer and manifest testing work completely offline +- Function syntax and help documentation can be validated offline +- Use these commands for offline validation: + ```powershell + # These work without Azure connectivity + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + Get-Help Add-DataLakeFolder -Examples + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + +### Module Development and Testing +- Import the module for testing (~1 second): + ```powershell + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + ``` +- Get available functions: + ```powershell + Get-Command -Module AzureDataLakeManagement + ``` +- Access function help and examples (~1 second per function): + ```powershell + Get-Help Add-DataLakeFolder -Examples + Get-Help Set-DataLakeFolderACL -Full + ``` +- **CRITICAL**: Test functions only with test/development Azure resources. Never test against production data. + +### Publishing Process +- **Prerequisites for Publishing**: + - PowerShell Gallery API Key (set as environment variable `PSGalleryKey`) + - Module version updated in `.psd1` file + - All PSScriptAnalyzer warnings addressed + - Manual validation completed + +- **Manual publish to PowerShell Gallery** (~30 seconds): + ```powershell + # Set your API key first + $env:PSGalleryKey = "your-api-key-here" + .\publish.ps1 + ``` + +- **GitHub Actions publish**: + - Workflow: `.github/workflows/manual_publish.yml` + - Requires `PSGalleryKey` secret configured in repository + - Manually triggered via GitHub Actions UI + - Uses `workflow_dispatch` trigger (not automatic) + +- **Pre-publish validation checklist**: + ```powershell + # 1. Code quality check + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + + # 2. Module manifest validation + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + + # 3. Module import test + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + + # 4. Verify version number is updated in .psd1 + # 5. Complete manual validation scenarios with test Azure resources + ``` + +## Key Module Components + +### Primary Functions (8 total): +1. **Get-AADObjectId** - Retrieve Azure AD object details by name/UPN +2. **Get-AzureSubscriptionInfo** - Get subscription information +3. **Add-DataLakeFolder** - Create folder structures in Data Lake Storage +4. **Remove-DataLakeFolder** - Delete folders from Data Lake Storage +5. **Set-DataLakeFolderACL** - Apply ACL permissions to folders (recursively) +6. **Get-DataLakeFolderACL** - Retrieve current ACL permissions +7. **Move-DataLakeFolder** - Move/rename folders between containers +8. **Remove-DataLakeFolderACL** - Remove ACL permissions from folders + +### Core Files: +- `AzureDataLakeManagement/AzureDataLakeManagement.psm1` - Main module (1026 lines, 8 functions) +- `AzureDataLakeManagement/AzureDataLakeManagement.psd1` - Module manifest and metadata +- `example.ps1` - Complete usage examples showing folder creation and ACL management +- `publish.ps1` - PowerShell Gallery publishing script + +## Validation and Testing + +### Code Quality Validation +Run these before every commit: +```powershell +# Static analysis (3 seconds) - NEVER CANCEL +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Module manifest validation (1 second) +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Manual Validation Scenarios +**CRITICAL**: Always test against development/test Azure resources only. Complete these scenarios after making changes: + +1. **Authentication and Module Import Test** (1-2 minutes): + ```powershell + Connect-AzAccount + Connect-AzureAD + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + # Should show all 8 functions + ``` + +2. **Basic Folder Operations Test** (5-10 minutes): + ```powershell + # Use test subscription and storage account + $subName = 'your-test-subscription' + $rgName = 'test-resource-group' + $storageAccountName = 'teststorageaccount' + $containerName = 'test-container' + + # Create test folder structure + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset\sample-folder' + + # Verify folder exists in Azure Storage Explorer or portal + + # Test folder move operation + Move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'test-dataset\sample-folder' -DestinationContainerName $containerName -destinationFolderPath 'test-dataset\moved-folder' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset' + ``` + +3. **ACL Management Test** (5-10 minutes): + ```powershell + # Create test folder + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + + # Apply test ACL (use test user/group) + Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' -accessControlType Read + + # Verify ACL was applied + Get-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' + + # Test ACL removal + Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + ``` + +4. **Azure AD Object Resolution Test** (2-3 minutes): + ```powershell + # Test user lookup + Get-AADObjectId -Identity 'test-user@domain.com' + + # Test group lookup + Get-AADObjectId -Identity 'Test Group Name' + + # Test service principal lookup + Get-AADObjectId -Identity 'Test Service Principal' + + # Should return ObjectId, ObjectType, and DisplayName for each + ``` + +5. **Complete example.ps1 Workflow Test** (10-15 minutes): + ```powershell + # Modify variables in example.ps1 first, then run: + .\example.ps1 + + # Verify in Azure portal: + # - Multiple folder structures created + # - ACL permissions applied correctly + # - Test folders cleaned up properly + ``` + +### Development Workflow Timing +- PSScriptAnalyzer execution: ~3 seconds - NEVER CANCEL +- Module manifest testing: ~1 second +- Module import: ~1 second (without Azure dependencies) +- Module import with Azure modules: ~2-5 seconds (depends on Azure module loading) +- Single folder operation: ~3-10 seconds (depends on Azure latency) +- ACL operations: ~5-15 seconds (depends on Azure AD latency and hierarchy depth) +- Full validation scenario: ~10-20 minutes total +- Complete example.ps1 workflow: ~5-15 minutes (creates multiple folders and ACLs) + +### Using example.ps1 for Learning +The `example.ps1` file demonstrates a complete workflow: +1. Authentication to Azure and Azure AD +2. Creating hierarchical folder structures +3. Setting various ACL types (user, group, service principal) +4. Error handling scenarios +5. Cleanup operations + +**CRITICAL**: Always modify the variables in example.ps1 before running: +```powershell +$subName = 'your-test-subscription' # Change this +$rgName = 'your-test-resource-group' # Change this +$storageAccountName = 'your-test-storage' # Change this +$containerName = 'test-container' # Change this +``` + +## Common Development Tasks + +### Adding New Functions +1. Add function to `AzureDataLakeManagement.psm1` +2. Update `FunctionsToExport` in `AzureDataLakeManagement.psd1` +3. Add usage example to `example.ps1` +4. Run PSScriptAnalyzer validation +5. Test with manual validation scenarios + +### Debugging Issues +- Use VS Code with PowerShell extension for debugging +- Import module with `-Force` to reload changes +- Use `-Verbose` parameter on functions for detailed output +- Check Azure portal/Storage Explorer to verify actual changes + +### Common Error Scenarios +1. **Missing Azure Authentication**: Functions fail with authentication errors + - Solution: Run `Connect-AzAccount` and `Connect-AzureAD` + +2. **Module Dependencies Missing**: Import fails with module not found errors + - Solution: Install Az.Storage, AzureAD, and Az.Accounts modules + +3. **Path Format Issues**: Functions expect backslash separators in folderPath + - Correct: `'dataset1\folder1\subfolder'` + - Incorrect: `'dataset1/folder1/subfolder'` + +4. **Permissions Issues**: ACL operations fail due to insufficient permissions + - Ensure Azure AD permissions and Storage Account permissions are configured + +5. **Storage Account Access**: Operations fail if storage account keys are not accessible + - Module requires either storage account key access OR proper Azure AD permissions + +### Known Code Quality Issues +PSScriptAnalyzer currently identifies 17 warnings that should be addressed in new code: +- 5 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) +- 6 unused parameter warnings (remove unused parameters) +- 3 unused variable warnings (remove unused variables) +- 3 missing `ShouldProcess` support warnings for state-changing functions (Add-DataLakeFolder, Set-DataLakeFolderACL, Remove-DataLakeFolderACL) + +Run `Invoke-ScriptAnalyzer` to see the complete list with line numbers and detailed guidance. + +## Repository Structure Reference +``` +. +├── .github/ +│ └── workflows/ +│ └── manual_publish.yml # GitHub Actions publishing workflow +├── .vscode/ +│ ├── launch.json # VS Code debugging configuration +│ └── settings.json # VS Code settings +├── AzureDataLakeManagement/ +│ ├── AzureDataLakeManagement.psd1 # Module manifest +│ └── AzureDataLakeManagement.psm1 # Main module (8 functions) +├── .gitignore +├── LICENSE +├── README.md # Project overview and version history +├── example.ps1 # Complete usage examples +└── publish.ps1 # PowerShell Gallery publishing script +``` + +## Quick Reference Commands + +### Daily Development +```powershell +# Load and test module +Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + +# Code quality check (run before commit) +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Test manifest +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Azure Authentication +```powershell +Connect-AzAccount +Connect-AzureAD +Get-AzSubscription # Verify connection +``` + +### Function Usage Pattern +```powershell +# All functions follow this parameter pattern: +-SubscriptionName # Azure subscription name +-ResourceGroupName # Resource group containing storage account +-StorageAccountName # Storage account name +-ContainerName # Container/filesystem name +-folderPath # Path within container (use backslash separators) ``` \ No newline at end of file diff --git a/.github/workflows/manual_publish.yml b/.github/workflows/manual_publish.yml index 7ad4c2f..b02a35d 100644 --- a/.github/workflows/manual_publish.yml +++ b/.github/workflows/manual_publish.yml @@ -1,18 +1,18 @@ -name: Publish -on: [workflow_dispatch] - -permissions: - contents: read - pull-requests: write - -jobs: - build: - name: Publish - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Publish - env: - PSGalleryKey: ${{ secrets.PSGalleryKey }} - run: .\publish.ps1 - shell: pwsh +name: Publish +on: [workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + build: + name: Publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Publish + env: + PSGalleryKey: ${{ secrets.PSGalleryKey }} + run: .\publish.ps1 + shell: pwsh diff --git a/.gitignore b/.gitignore index 3546e64..c0cd43d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -AzureDataLakeManagement.zip - +AzureDataLakeManagement.zip + diff --git a/.vscode/launch.json b/.vscode/launch.json index 2fdae60..64eb2e7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,14 +1,14 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "PowerShell: Module Interactive Session", - "type": "PowerShell", - "request": "launch", - "script": "Import-Module -Force '${workspaceFolder}/AzureDataLakeManagement/AzureDataLakeManagement.psm1'" - } - ] -} +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "PowerShell: Module Interactive Session", + "type": "PowerShell", + "request": "launch", + "script": "Import-Module -Force '${workspaceFolder}/AzureDataLakeManagement/AzureDataLakeManagement.psm1'" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index a42f3ed..028065f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ -{ - "githubPullRequests.remotes": [ - "https://github.com/SteveCInVA/AzureDataLakeManagement.git" - ] -} +{ + "githubPullRequests.remotes": [ + "https://github.com/SteveCInVA/AzureDataLakeManagement.git" + ], + "powershell.pester.codeLens": false +} diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index 6692573..d7ef980 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -1,136 +1,136 @@ -# -# Module manifest for module 'AzureDataLakeManagement' -# -# Generated by: Steve Carroll (Microsoft) -# -# Generated on: 12/1/2023 -# - -@{ - -# Script module or binary module file associated with this manifest. -RootModule = 'AzureDataLakeManagement.psm1' - -# Version number of this module. -ModuleVersion = '2025.1.1' - -# Supported PSEditions -CompatiblePSEditions = @('Desktop', 'Core') - -# ID used to uniquely identify this module -GUID = 'b0b0b0b0-b0b0-b0b0-b0b0-b0b0b0b0b0b0' - -# Author of this module -Author = 'Steve Carroll' - -# Company or vendor of this module -CompanyName = 'Microsoft' - -# Copyright statement for this module -Copyright = '(c) 2023 Microsoft Corporation. All rights reserved.' - -# Description of the functionality provided by this module -Description = 'Azure Data Lake Management Module' - -# Minimum version of the Windows PowerShell engine required by this module -PowerShellVersion = '5.1' - -# Name of the Windows PowerShell host required by this module -# PowerShellHostName = '' - -# Minimum version of the Windows PowerShell host required by this module -# PowerShellHostVersion = '' - -# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# DotNetFrameworkVersion = '' - -# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# CLRVersion = '' - -# Processor architecture (None, X86, Amd64) required by this module -# ProcessorArchitecture = '' - -# Modules that must be imported into the global environment prior to importing this module -# RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') - -# Assemblies that must be loaded prior to importing this module -# RequiredAssemblies = @() - -# Script files (.ps1) that are run in the caller's environment prior to importing this module. -# ScriptsToProcess = @() - -# Type files (.ps1xml) to be loaded when importing this module -# TypesToProcess = @() - -# Format files (.ps1xml) to be loaded when importing this module -# FormatsToProcess = @() - -# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess -# NestedModules = @() - -# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Get-AADObjectId', 'Get-AzureSubscriptionInfo', 'Add-DataLakeFolder', - 'Remove-DataLakeFolder', 'Set-DataLakeFolderACL', - 'Get-DataLakeFolderACL', 'Move-DataLakeFolder', - 'Remove-DataLakeFolderACL', 'Test-ModuleDependencies', - 'Install-ModuleDependencies', 'Import-ModuleDependencies' - -# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. -CmdletsToExport = @() - -# Variables to export from this module -# VariablesToExport = @() - -# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. -AliasesToExport = @() - -# DSC resources to export from this module -# DscResourcesToExport = @() - -# List of all modules packaged with this module -# ModuleList = @() - -# List of all files packaged with this module -# FileList = @() - -# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. -PrivateData = @{ - - PSData = @{ - - # Tags applied to this module. These help with module discovery in online galleries. - Tags = 'Azure','DataLake','Security' - - # A URL to the license for this module. - # LicenseUri = '' - - # A URL to the main website for this project. - ProjectUri = 'https://github.com/SteveCInVA/AzureDataLakeManagement' - - # A URL to an icon representing this module. - # IconUri = '' - - # ReleaseNotes of this module - # ReleaseNotes = '' - - # Prerelease string of this module - # Prerelease = '' - - # Flag to indicate whether the module requires explicit user acceptance for install/update/save - # RequireLicenseAcceptance = $false - - # External dependent modules of this module - ExternalModuleDependencies = @('Az.Storage', 'AzureAD', 'Az.Accounts') - - } # End of PSData hashtable - - } # End of PrivateData hashtable - -# HelpInfo URI of this module -HelpInfoURI = 'https://github.com/SteveCInVA/AzureDataLakeManagement/blob/main/README.md' - -# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. -# DefaultCommandPrefix = '' - -} - +# +# Module manifest for module 'AzureDataLakeManagement' +# +# Generated by: Steve Carroll (Microsoft) +# +# Generated on: 12/1/2023 +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = 'AzureDataLakeManagement.psm1' + +# Version number of this module. +ModuleVersion = '2025.1.1' + +# Supported PSEditions +CompatiblePSEditions = @('Desktop', 'Core') + +# ID used to uniquely identify this module +GUID = 'b0b0b0b0-b0b0-b0b0-b0b0-b0b0b0b0b0b0' + +# Author of this module +Author = 'Steve Carroll' + +# Company or vendor of this module +CompanyName = 'Microsoft' + +# Copyright statement for this module +Copyright = '(c) 2023 Microsoft Corporation. All rights reserved.' + +# Description of the functionality provided by this module +Description = 'Azure Data Lake Management Module' + +# Minimum version of the Windows PowerShell engine required by this module +PowerShellVersion = '5.1' + +# Name of the Windows PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the Windows PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# CLRVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') + +# Assemblies that must be loaded prior to importing this module +# RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +# ScriptsToProcess = @() + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +# FormatsToProcess = @() + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +# NestedModules = @() + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +FunctionsToExport = 'Get-AADObjectId', 'Get-AzureSubscriptionInfo', 'Add-DataLakeFolder', + 'Remove-DataLakeFolder', 'Set-DataLakeFolderACL', + 'Get-DataLakeFolderACL', 'Move-DataLakeFolder', + 'Remove-DataLakeFolderACL', 'Test-ModuleDependencies', + 'Install-ModuleDependencies', 'Import-ModuleDependencies' + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = @() + +# Variables to export from this module +# VariablesToExport = @() + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = @() + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + Tags = 'Azure','DataLake','Security' + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + ProjectUri = 'https://github.com/SteveCInVA/AzureDataLakeManagement' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + # Prerelease string of this module + # Prerelease = '' + + # Flag to indicate whether the module requires explicit user acceptance for install/update/save + # RequireLicenseAcceptance = $false + + # External dependent modules of this module + ExternalModuleDependencies = @('Az.Storage', 'AzureAD', 'Az.Accounts') + + } # End of PSData hashtable + + } # End of PrivateData hashtable + +# HelpInfo URI of this module +HelpInfoURI = 'https://github.com/SteveCInVA/AzureDataLakeManagement/blob/main/README.md' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} + diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index 769187e..23307f5 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -1,1256 +1,1256 @@ -#region Dependency Management Functions - -<# -.SYNOPSIS - Tests if required modules are available and optionally installs them. - -.DESCRIPTION - The Test-ModuleDependencies function checks if the required modules for AzureDataLakeManagement are available. - It can optionally install missing modules and provides user feedback about the dependency status. - -.PARAMETER AutoInstall - If specified, automatically installs missing required modules from PowerShell Gallery. - -.PARAMETER Quiet - If specified, suppresses informational output and only shows errors. - -.EXAMPLE - PS C:\> Test-ModuleDependencies - Checks for required modules and displays status information. - -.EXAMPLE - PS C:\> Test-ModuleDependencies -AutoInstall - Checks for required modules and automatically installs any that are missing. - -.NOTES - Required modules: Az.Storage, AzureAD, Az.Accounts - Author: Stephen Carroll - Microsoft - Date: 2025-01-09 -#> -function Test-ModuleDependencies { - [CmdletBinding()] - param( - [switch]$AutoInstall, - [switch]$Quiet - ) - - $requiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') - $missingModules = @() - $availableModules = @() - - if (-not $Quiet) { - Write-Host "Checking AzureDataLakeManagement module dependencies..." -ForegroundColor Yellow - } - - foreach ($moduleName in $requiredModules) { - $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue - if ($null -eq $module) { - $missingModules += $moduleName - if (-not $Quiet) { - Write-Warning "Missing required module: $moduleName" - } - } else { - $availableModules += $moduleName - if (-not $Quiet) { - Write-Host "✓ Found module: $moduleName (Version: $($module[0].Version))" -ForegroundColor Green - } - } - } - - if ($missingModules.Count -eq 0) { - if (-not $Quiet) { - Write-Host "✓ All required modules are available." -ForegroundColor Green - } - return $true - } - - if ($AutoInstall) { - if (-not $Quiet) { - Write-Host "Installing missing modules..." -ForegroundColor Yellow - } - return Install-ModuleDependencies -Modules $missingModules -Quiet:$Quiet - } else { - if (-not $Quiet) { - Write-Host "`nTo install missing modules, run:" -ForegroundColor Cyan - Write-Host "Test-ModuleDependencies -AutoInstall" -ForegroundColor White - Write-Host "`nOr install manually:" -ForegroundColor Cyan - foreach ($module in $missingModules) { - Write-Host "Install-Module -Name $module -Force" -ForegroundColor White - } - } - return $false - } -} - -<# -.SYNOPSIS - Installs required modules for AzureDataLakeManagement. - -.DESCRIPTION - The Install-ModuleDependencies function installs the specified required modules from PowerShell Gallery. - -.PARAMETER Modules - Array of module names to install. If not specified, installs all required modules. - -.PARAMETER Quiet - If specified, suppresses informational output and only shows errors. - -.EXAMPLE - PS C:\> Install-ModuleDependencies - Installs all required modules for AzureDataLakeManagement. - -.EXAMPLE - PS C:\> Install-ModuleDependencies -Modules @('Az.Storage') - Installs only the Az.Storage module. - -.NOTES - Author: Stephen Carroll - Microsoft - Date: 2025-01-09 -#> -function Install-ModuleDependencies { - [CmdletBinding()] - param( - [string[]]$Modules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), - [switch]$Quiet - ) - - $successCount = 0 - $failureCount = 0 - - foreach ($moduleName in $Modules) { - try { - if (-not $Quiet) { - Write-Host "Installing module: $moduleName..." -ForegroundColor Yellow - } - - Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber -ErrorAction Stop - - if (-not $Quiet) { - Write-Host "✓ Successfully installed: $moduleName" -ForegroundColor Green - } - $successCount++ - } - catch { - Write-Error "Failed to install module $moduleName`: $($_.Exception.Message)" - $failureCount++ - } - } - - if (-not $Quiet) { - if ($failureCount -eq 0) { - Write-Host "✓ All modules installed successfully." -ForegroundColor Green - } else { - Write-Warning "Installed $successCount modules, failed to install $failureCount modules." - } - } - - return ($failureCount -eq 0) -} - -<# -.SYNOPSIS - Imports required modules with proper error handling. - -.DESCRIPTION - The Import-ModuleDependencies function imports the required modules for AzureDataLakeManagement functions. - It provides better error handling and user feedback compared to individual Import-Module calls. - -.PARAMETER RequiredModules - Array of module names to import. Defaults to the core required modules. - -.PARAMETER Quiet - If specified, suppresses informational output and only shows errors. - -.EXAMPLE - PS C:\> Import-ModuleDependencies - Imports all required modules for AzureDataLakeManagement. - -.NOTES - Author: Stephen Carroll - Microsoft - Date: 2025-01-09 -#> -function Import-ModuleDependencies { - [CmdletBinding()] - param( - [string[]]$RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), - [switch]$Quiet - ) - - $importFailures = @() - - foreach ($moduleName in $RequiredModules) { - try { - $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue - if ($null -eq $module) { - $importFailures += $moduleName - Write-Error "Module $moduleName is not available. Please install it first." - continue - } - - Import-Module -Name $moduleName -ErrorAction Stop -Force - if (-not $Quiet) { - Write-Verbose "Successfully imported module: $moduleName" - } - } - catch { - $importFailures += $moduleName - Write-Error "Failed to import module $moduleName`: $($_.Exception.Message)" - } - } - - if ($importFailures.Count -gt 0) { - Write-Error "Failed to import modules: $($importFailures -join ', '). Some functions may not work correctly." - return $false - } - - return $true -} - -#endregion - -<# -.SYNOPSIS - This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. - -.DESCRIPTION - Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Azure AD. - -.PARAMETER Identity - The Identity parameter specifies the user principal name, group display name, or service principal display name of the object to retrieve. This parameter is mandatory. - -.EXAMPLE - PS C:\> Get-AADObjectId -Identity "johndoe@contoso.com" - ObjectId ObjectType DisplayName - -------- ---------- ----------- - 12345678-1234-1234-1234-1234567890ab User John Doe - - This example retrieves the object ID, object type, and display name for the Azure AD user with the user principal name "johndoe@contoso.com". - -.EXAMPLE - PS C:\> Get-AADObjectId -Identity "HR Group" - ObjectId ObjectType DisplayName - -------- ---------- ----------- - 87654321-4321-4321-4321-ba0987654321 Group HR Group - - This example retrieves the object ID, object type, and display name for the Azure AD group with the display name "HR Group". - -.NOTES - This function requires an active connection to Azure AD using Connect-AzureAD. If the specified identity does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-AADObjectId -{ - param ( - # The identity for which the Azure AD Object ID is to be fetched - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string]$Identity - ) - - # Replacing single quotes in the identity with double single quotes - $Identity = $Identity.Replace("'", "''") - - try - { - # Initializing user, group, and service principal to null - $user = $null - $group = $null - $sp = $null - - # Try to get the user, group, and service principal in one go - $user = Get-AzureADUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $user) - { - $group = Get-AzureADGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $group) - { - $sp = Get-AzureADServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue - } - } - - # Check which object is not null and assign the corresponding values - if ($null -ne $user) - { - $objectType = 'User' - $objectId = $user.ObjectId - $displayName = $user.DisplayName - } - elseif ($null -ne $group) - { - $objectType = 'Group' - $objectId = $group.ObjectId - $displayName = $group.DisplayName - } - elseif ($null -ne $sp) - { - $objectType = 'ServicePrincipal' - $objectId = $sp.ObjectId - $displayName = $sp.DisplayName - } - else - { - Write-Error ('Object not found. Unable to find object "{0}" in Azure AD.' -f $Identity) - return - } - } - catch [Microsoft.Open.Azure.AD.CommonLibrary.AadNeedAuthenticationException] - { - Write-Error 'You must be authenticated to Azure AD to run this command. Run Connect-AzureAD to authenticate.' - return - } - catch - { - Write-Error $_.Exception.Message - return - } - - # Output the object details - Write-Verbose "Object ID: $objectId" - Write-Verbose "Object Type: $objectType" - Write-Verbose "Object Name: $displayName" - - # Create a custom object to return - $object = [PSCustomObject]@{ - ObjectId = $objectId - ObjectType = $objectType - DisplayName = $displayName - } - return $object -} - -<# -.SYNOPSIS - Retrieves the subscription ID and tenant ID for a specified Azure subscription. - -.DESCRIPTION - The Get-AzureSubscriptionInfo function takes a subscription name as a parameter and returns a custom object containing the subscription ID and tenant ID for the specified Azure subscription. It requires an active connection to Azure. - -.PARAMETER SubscriptionName - The SubscriptionName parameter specifies the name of the Azure subscription for which to retrieve the subscription ID and tenant ID. This parameter is mandatory. - -.EXAMPLE - PS C:\> Get-AzureSubscriptionInfo -SubscriptionName 'MySubscription' - SubscriptionId TenantId - -------------- -------- - 12345678-1234-1234-1234-1234567890ab 87654321-4321-4321-4321-ba0987654321 - - This example retrieves the subscription ID and tenant ID for the Azure subscription named 'MySubscription'. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-AzureSubscriptionInfo -{ - param ( - # The name of the Azure subscription - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string]$SubscriptionName - ) - - try - { - # Get the subscription details - $subscription = Get-AzSubscription -SubscriptionName $SubscriptionName - - # Check if the subscription exists - if ($null -eq $subscription) - { - Write-Error('Subscription "{0}" not found.', $SubscriptionName) - return - } - else - { - # Write verbose messages for debugging - Write-Verbose 'Function: Get-AzureSubscriptionInfo: Subscription found.' - Write-Verbose "SubscriptionID: $subscription.id SubscriptionName: $subscription.Name" - } - } - catch - { - # Handle exceptions and write an error message - Write-Error 'Ensure you have run Connect-AzAccount and that the subscription exists.' - return - } - - # Get the subscription ID and tenant ID - $subscriptionId = $subscription.SubscriptionId - $tenantId = $subscription.TenantId - - # Create a custom object to return - $object = [PSCustomObject]@{ - SubscriptionId = $subscriptionId - TenantId = $tenantId - } - - return $object -} - -<# -.SYNOPSIS - Creates a folder in a Data Lake Storage account. - -.DESCRIPTION - The Add-DataLakeFolder function creates a folder (or folder hierarchy) in a Data Lake storage account container. It requires an active connection to Azure. - -.PARAMETER SubscriptionName - The name of the Azure subscription to use. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container in the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder to create. May be a single folder or a folder hierarchy (e.g. 'folder1/folder2/folder3'). This parameter is mandatory. - -.PARAMETER ErrorIfFolderExists - Optional switch to throw error if folder exists. If not specified, will return the existing folder. - -.EXAMPLE - PS C:\> Add-DataLakeFolder -SubscriptionName 'MySubscription' -ResourceGroupName 'MyResourceGroup' -StorageAccountName 'MyStorageAccount' -ContainerName 'MyContainer' -FolderPath 'folder1/folder2/folder3' - This example creates a folder hierarchy 'folder1/folder2/folder3' in the specified Data Lake storage account container. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Add-DataLakeFolder -{ - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $true)] - [string]$FolderPath, # Path to the folder - - [switch]$ErrorIfFolderExists # Flag to indicate if an error should be thrown if the folder exists - ) - - # Get the subscription ID - $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { - Write-Error 'Subscription not found.' - return - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - - # Check if the Az.Storage module is available and import it - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { - Write-Error 'Required module Az.Storage is not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' - return - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # Create the folder - try - { - $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop - } - catch - { - if ($ErrorIfFolderExists) - { - Write-Error "Folder $FolderPath already exists." - return - } - $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists - return - } - - if ($null -eq $ret) - { - Write-Error 'Failed to create the folder.' - return - } - else - { - return $ret # Return the created folder - } -} - -<# -.SYNOPSIS - Deletes a folder from an Azure Data Lake Storage Gen2 account. - -.DESCRIPTION - The Remove-DataLakeFolder function deletes a folder from an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, and folder path as input parameters. If the folder does not exist, it will return an error unless the -ErrorIfFolderDoesNotExist switch is used. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container containing the folder. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder to delete. This parameter is mandatory. - -.PARAMETER ErrorIfFolderDoesNotExist - If this switch is used, the function will not return an error if the folder does not exist. - -.EXAMPLE - PS C:\> Remove-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "MyFolder" - This example deletes the folder "MyFolder" from the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the "MySubscription" Azure subscription. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Remove-DataLakeFolder -{ - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $true)] - [string]$FolderPath, # Path to the folder - - [switch]$ErrorIfFolderDoesNotExist # Flag to indicate if an error should be thrown if the folder does not exist - ) - - # Get the subscription ID - $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { - Write-Error 'Subscription not found.' - return - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # Ensure the folder exists before deleting - try - { - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -ErrorAction Stop - } - catch - { - if ($ErrorIfFolderDoesNotExist) - { - Write-Error "Folder '$FolderPath' does not exist to delete." - return - } - return - } - - # Delete the folder - if ($null -ne $folderExists) - { - $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force - } - - if ($null -ne $ret) - { - Write-Error 'Failed to delete the folder.' - return - } - else - { - Write-Host "Folder $ContainerName\$FolderPath deleted successfully." - return - } -} - -<# -.SYNOPSIS - Sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. - -.DESCRIPTION - The Set-DataLakeFolderACL function sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, folder path, identity, and access control type as input parameters. Optionally, it can also set the ACL for the container and include the default scope in the ACL. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container containing the folder. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is mandatory. - -.PARAMETER Identity - The identity to use in the ACL. This parameter is mandatory. - -.PARAMETER AccessControlType - The type of access control to apply to the folder. Valid values are 'Read' and 'Write'. This parameter is mandatory. - -.PARAMETER SetContainerACL - A switch parameter that specifies whether to set the ACL for the container. This parameter is optional. - -.PARAMETER IncludeDefaultScope - A switch parameter that specifies whether to include the default scope in the ACL. This parameter is optional. - -.PARAMETER DoNotApplyACLRecursively - A switch parameter that specifies whether to set the ACL recursively. This parameter is optional. - -.EXAMPLE - PS C:\> Set-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" -Identity "MyIdentity" -AccessControlType "Read" -IncludeDefaultScope - This example sets the ACL for the folder "/MyFolder" in the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" for the identity "MyIdentity" with read access and includes the default scope in the ACL. - -.NOTES - This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Set-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, - - [Parameter(Mandatory = $true)] - [string]$ContainerName, - - [Parameter(Mandatory = $true)] - [string]$FolderPath, - - [Parameter(Mandatory = $true)] - [string]$Identity, - - [ValidateSet('Read', 'Write')] - [Parameter(Mandatory = $true)] - [string]$AccessControlType, - - [switch]$SetContainerACL, - - [switch]$IncludeDefaultScope, - - [switch]$DoNotApplyACLRecursively - ) - - # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { - Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' - return - } - - $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName - if ($null -eq $sub) - { - Write-Error 'Subscription not found. Ensure you have run Connect-AzAccount before execution.' - return - } - else - { - $subId = $sub.SubscriptionId - } - - # Get the object ID of the identity to use in the ACL - $identityObj = Get-AADObjectId -Identity $Identity - if ($null -eq $identityObj) - { - Write-Error 'Identity not found.' - return - } - else - { - Write-Verbose ('{0} ID: {1} Display Name: {2}' -f $identityObj.ObjectType, $identityObj.ObjectId, $identityObj.DisplayName) - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - else - { - Write-Verbose $subContext.Name - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - else - { - Write-Verbose $storageAccount.StorageAccountName - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # verify the folder exists before setting the ACL - try - { - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - if ($null -eq $folderExists) - { - Write-Error('Folder not found.') - return - } - } - catch - { - Write-Error('Folder not found.') - return - } - - # translate the access control type to applied permission - $permission = switch ( $AccessControlType ) - { 'Read' - { 'r-x' - } - 'Write' - { 'rwx' - } - default - { '' - } - } - - $identityType = switch ($identityObj.ObjectType) - { 'User' - { 'user' - } - 'Group' - { 'group' - } - 'ServicePrincipal' - { 'user' - } - 'ManagedIdentity' - { 'other' - } - default - { '' - } - } - - # set the ACL at the container level - if ($SetContainerACL) - { - Write-Verbose 'set container ACL' - $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the container.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'Container ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - - # get the ACL for the folder - $acl = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath).ACL - - try - { - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - - } - catch [Microsoft.PowerShell.Commands.WriteErrorException] - { - Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' - return - } - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - ###################### - # default scope - if ($IncludeDefaultScope) - { - Write-Verbose 'include default scope' - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the default scope.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'Default Scope ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - - -} - -<# -.SYNOPSIS - Gets the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. - -.DESCRIPTION - The Get-DataLakeFolderACL function retrieves the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, and container name as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is optional. If omitted, the function will revert to the root of the container. - -.EXAMPLE - PS C:\> Get-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" - This example gets the ACL for the folder "/MyFolder" in the container "MyContainer" of the storage account "MyStorageAccount" in the resource group "MyResourceGroup" of the Azure subscription "MySubscription". - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $false)] - [string]$FolderPath = '/' # Path to the folder in the Data Lake - ) - - # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { - Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' - return - } - - # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { - $FolderPath = $FolderPath.Substring(1) - } - - try - { - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - - # Get the ACLs for the folder - $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL - - # Process each ACL - $aclResults = foreach ($ace in $acls) - { - if ($ace.EntityId) - { - # Get the AD object for the entity - $adObject = Get-AzureADObjectByObjectId -ObjectIds $ace.EntityId - - # Create a custom object with the ACL info - [PSCustomObject]@{ - DisplayName = $adObject.DisplayName - ObjectId = $ace.EntityId - ObjectType = $adObject.ObjectType - Permissions = $ace.Permissions - DefaultScope = $ace.DefaultScope - } - } - } - - # Return the results - return $aclResults - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -<# -.SYNOPSIS - Moves a folder in Azure Data Lake Storage Gen2 to a new location. - -.DESCRIPTION - The Move-DataLakeFolder function moves a folder in Azure Data Lake Storage Gen2 to a new location. It requires the subscription name, resource group name, storage account name, source container name, source folder path, and destination folder path as input parameters. Optionally, it can also take a destination container name. If the destination container name is not provided, the function will use the source container name. - -.PARAMETER SubscriptionName - The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER SourceContainerName - The name of the source container for the move operation. This parameter is mandatory. - -.PARAMETER SourceFolderPath - The path of the folder to move. This parameter is mandatory. - -.PARAMETER DestinationContainerName - The name of the destination container for the move operation. This parameter is optional. If not specified, the function will use the source container name. - -.PARAMETER DestinationFolderPath - The path of the destination folder. This parameter is mandatory. - -.EXAMPLE - PS C:\> Move-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -SourceContainerName "MySourceContainer" -SourceFolderPath "/MySourceFolder" -DestinationFolderPath "/MyDestinationFolder" - This example moves the folder "/MySourceFolder" from the container "MySourceContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription" to the folder "/MyDestinationFolder" in the same container. - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Move-DataLakeFolder -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$SourceContainerName, # Source container name - - [Parameter(Mandatory = $true)] - [string]$SourceFolderPath, # Source folder path - - [Parameter(Mandatory = $false)] - [string]$DestinationContainerName, # Destination container name - - [Parameter(Mandatory = $true)] - [string]$DestinationFolderPath # Destination folder path - ) - - # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { - Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' - return - } - - try - { - - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath - - # If destination container name is not provided, use source container name - if (-not $DestinationContainerName) - { - $DestinationContainerName = $SourceContainerName - } - - # Move the folder - $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force - - # Write verbose output and return the result - Write-Verbose ('Function: Move-DataLakeFolder') - Write-Verbose "Folder moved: $DestinationFolderPath" - return $ret - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -<# -.SYNOPSIS - Removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. - -.DESCRIPTION - The Remove-DataLakeFolderACL function removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, container name, and identity as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. - -.PARAMETER SubscriptionName - The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is optional. If not specified, the function will revert to the root of the container. - -.PARAMETER Identity - The identity to remove from the ACL. This parameter is mandatory. - -.PARAMETER DoNotApplyACLRecursively - A switch parameter that specifies whether to remove the identity from the ACL recursively. This parameter is optional. - -.EXAMPLE - PS C:\> Remove-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -Identity "MyIdentity" - This example removes the identity "MyIdentity" from the ACL of the root of the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription". - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Remove-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $false)] - [string]$FolderPath = '/', # Path to the folder in the Data Lake - - [Parameter(Mandatory = $true)] - [string]$Identity, # Identity to remove from the ACL - - [Parameter(Mandatory = $false)] - [switch]$DoNotApplyACLRecursively # Flag to indicate if the ACL should not be applied recursively - - ) - - # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { - Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' - return - } - - # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { - $FolderPath = $FolderPath.Substring(1) - } - - try - { - - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Get the object ID of the identity to use in the ACL - $identityObj = Get-AADObjectId -Identity $Identity - $id = $identityObj.ObjectId - - # Get the folder - $folder = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - - # Get the ACLs for the folder - $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL - - # Remove the specified identity from the ACL - $newacl = $acls | Where-Object { -not ($_.AccessControlType -eq $identityObj.ObjectType -and $_.EntityId -eq $id) } - - # Update the ACL - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } - - # Check if the update was successful - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to update the ACL.' - Write-Error $result.FailedEntries - } - else - { - Write-Host 'ACL updated successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -#region Module Initialization -# This code runs when the module is imported -# Check dependencies on module import -$dependencyCheckResult = Test-ModuleDependencies -Quiet - -if (-not $dependencyCheckResult) { - Write-Warning @" -AzureDataLakeManagement module loaded with missing dependencies. -Some functions may not work correctly until required modules are installed. -Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. -"@ -} -#endregion - -Export-ModuleMember -Function * +#region Dependency Management Functions + +<# +.SYNOPSIS + Tests if required modules are available and optionally installs them. + +.DESCRIPTION + The Test-ModuleDependencies function checks if the required modules for AzureDataLakeManagement are available. + It can optionally install missing modules and provides user feedback about the dependency status. + +.PARAMETER AutoInstall + If specified, automatically installs missing required modules from PowerShell Gallery. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Test-ModuleDependencies + Checks for required modules and displays status information. + +.EXAMPLE + PS C:\> Test-ModuleDependencies -AutoInstall + Checks for required modules and automatically installs any that are missing. + +.NOTES + Required modules: Az.Storage, AzureAD, Az.Accounts + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Test-ModuleDependencies { + [CmdletBinding()] + param( + [switch]$AutoInstall, + [switch]$Quiet + ) + + $requiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') + $missingModules = @() + $availableModules = @() + + if (-not $Quiet) { + Write-Host "Checking AzureDataLakeManagement module dependencies..." -ForegroundColor Yellow + } + + foreach ($moduleName in $requiredModules) { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $missingModules += $moduleName + if (-not $Quiet) { + Write-Warning "Missing required module: $moduleName" + } + } else { + $availableModules += $moduleName + if (-not $Quiet) { + Write-Host "✓ Found module: $moduleName (Version: $($module[0].Version))" -ForegroundColor Green + } + } + } + + if ($missingModules.Count -eq 0) { + if (-not $Quiet) { + Write-Host "✓ All required modules are available." -ForegroundColor Green + } + return $true + } + + if ($AutoInstall) { + if (-not $Quiet) { + Write-Host "Installing missing modules..." -ForegroundColor Yellow + } + return Install-ModuleDependencies -Modules $missingModules -Quiet:$Quiet + } else { + if (-not $Quiet) { + Write-Host "`nTo install missing modules, run:" -ForegroundColor Cyan + Write-Host "Test-ModuleDependencies -AutoInstall" -ForegroundColor White + Write-Host "`nOr install manually:" -ForegroundColor Cyan + foreach ($module in $missingModules) { + Write-Host "Install-Module -Name $module -Force" -ForegroundColor White + } + } + return $false + } +} + +<# +.SYNOPSIS + Installs required modules for AzureDataLakeManagement. + +.DESCRIPTION + The Install-ModuleDependencies function installs the specified required modules from PowerShell Gallery. + +.PARAMETER Modules + Array of module names to install. If not specified, installs all required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Install-ModuleDependencies + Installs all required modules for AzureDataLakeManagement. + +.EXAMPLE + PS C:\> Install-ModuleDependencies -Modules @('Az.Storage') + Installs only the Az.Storage module. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Install-ModuleDependencies { + [CmdletBinding()] + param( + [string[]]$Modules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [switch]$Quiet + ) + + $successCount = 0 + $failureCount = 0 + + foreach ($moduleName in $Modules) { + try { + if (-not $Quiet) { + Write-Host "Installing module: $moduleName..." -ForegroundColor Yellow + } + + Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber -ErrorAction Stop + + if (-not $Quiet) { + Write-Host "✓ Successfully installed: $moduleName" -ForegroundColor Green + } + $successCount++ + } + catch { + Write-Error "Failed to install module $moduleName`: $($_.Exception.Message)" + $failureCount++ + } + } + + if (-not $Quiet) { + if ($failureCount -eq 0) { + Write-Host "✓ All modules installed successfully." -ForegroundColor Green + } else { + Write-Warning "Installed $successCount modules, failed to install $failureCount modules." + } + } + + return ($failureCount -eq 0) +} + +<# +.SYNOPSIS + Imports required modules with proper error handling. + +.DESCRIPTION + The Import-ModuleDependencies function imports the required modules for AzureDataLakeManagement functions. + It provides better error handling and user feedback compared to individual Import-Module calls. + +.PARAMETER RequiredModules + Array of module names to import. Defaults to the core required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Import-ModuleDependencies + Imports all required modules for AzureDataLakeManagement. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Import-ModuleDependencies { + [CmdletBinding()] + param( + [string[]]$RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [switch]$Quiet + ) + + $importFailures = @() + + foreach ($moduleName in $RequiredModules) { + try { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $importFailures += $moduleName + Write-Error "Module $moduleName is not available. Please install it first." + continue + } + + Import-Module -Name $moduleName -ErrorAction Stop -Force + if (-not $Quiet) { + Write-Verbose "Successfully imported module: $moduleName" + } + } + catch { + $importFailures += $moduleName + Write-Error "Failed to import module $moduleName`: $($_.Exception.Message)" + } + } + + if ($importFailures.Count -gt 0) { + Write-Error "Failed to import modules: $($importFailures -join ', '). Some functions may not work correctly." + return $false + } + + return $true +} + +#endregion + +<# +.SYNOPSIS + This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. + +.DESCRIPTION + Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Azure AD. + +.PARAMETER Identity + The Identity parameter specifies the user principal name, group display name, or service principal display name of the object to retrieve. This parameter is mandatory. + +.EXAMPLE + PS C:\> Get-AADObjectId -Identity "johndoe@contoso.com" + ObjectId ObjectType DisplayName + -------- ---------- ----------- + 12345678-1234-1234-1234-1234567890ab User John Doe + + This example retrieves the object ID, object type, and display name for the Azure AD user with the user principal name "johndoe@contoso.com". + +.EXAMPLE + PS C:\> Get-AADObjectId -Identity "HR Group" + ObjectId ObjectType DisplayName + -------- ---------- ----------- + 87654321-4321-4321-4321-ba0987654321 Group HR Group + + This example retrieves the object ID, object type, and display name for the Azure AD group with the display name "HR Group". + +.NOTES + This function requires an active connection to Azure AD using Connect-AzureAD. If the specified identity does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Get-AADObjectId +{ + param ( + # The identity for which the Azure AD Object ID is to be fetched + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$Identity + ) + + # Replacing single quotes in the identity with double single quotes + $Identity = $Identity.Replace("'", "''") + + try + { + # Initializing user, group, and service principal to null + $user = $null + $group = $null + $sp = $null + + # Try to get the user, group, and service principal in one go + $user = Get-AzureADUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue + if ($null -eq $user) + { + $group = Get-AzureADGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + if ($null -eq $group) + { + $sp = Get-AzureADServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + } + } + + # Check which object is not null and assign the corresponding values + if ($null -ne $user) + { + $objectType = 'User' + $objectId = $user.ObjectId + $displayName = $user.DisplayName + } + elseif ($null -ne $group) + { + $objectType = 'Group' + $objectId = $group.ObjectId + $displayName = $group.DisplayName + } + elseif ($null -ne $sp) + { + $objectType = 'ServicePrincipal' + $objectId = $sp.ObjectId + $displayName = $sp.DisplayName + } + else + { + Write-Error ('Object not found. Unable to find object "{0}" in Azure AD.' -f $Identity) + return + } + } + catch [Microsoft.Open.Azure.AD.CommonLibrary.AadNeedAuthenticationException] + { + Write-Error 'You must be authenticated to Azure AD to run this command. Run Connect-AzureAD to authenticate.' + return + } + catch + { + Write-Error $_.Exception.Message + return + } + + # Output the object details + Write-Verbose "Object ID: $objectId" + Write-Verbose "Object Type: $objectType" + Write-Verbose "Object Name: $displayName" + + # Create a custom object to return + $object = [PSCustomObject]@{ + ObjectId = $objectId + ObjectType = $objectType + DisplayName = $displayName + } + return $object +} + +<# +.SYNOPSIS + Retrieves the subscription ID and tenant ID for a specified Azure subscription. + +.DESCRIPTION + The Get-AzureSubscriptionInfo function takes a subscription name as a parameter and returns a custom object containing the subscription ID and tenant ID for the specified Azure subscription. It requires an active connection to Azure. + +.PARAMETER SubscriptionName + The SubscriptionName parameter specifies the name of the Azure subscription for which to retrieve the subscription ID and tenant ID. This parameter is mandatory. + +.EXAMPLE + PS C:\> Get-AzureSubscriptionInfo -SubscriptionName 'MySubscription' + SubscriptionId TenantId + -------------- -------- + 12345678-1234-1234-1234-1234567890ab 87654321-4321-4321-4321-ba0987654321 + + This example retrieves the subscription ID and tenant ID for the Azure subscription named 'MySubscription'. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Get-AzureSubscriptionInfo +{ + param ( + # The name of the Azure subscription + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$SubscriptionName + ) + + try + { + # Get the subscription details + $subscription = Get-AzSubscription -SubscriptionName $SubscriptionName + + # Check if the subscription exists + if ($null -eq $subscription) + { + Write-Error('Subscription "{0}" not found.', $SubscriptionName) + return + } + else + { + # Write verbose messages for debugging + Write-Verbose 'Function: Get-AzureSubscriptionInfo: Subscription found.' + Write-Verbose "SubscriptionID: $subscription.id SubscriptionName: $subscription.Name" + } + } + catch + { + # Handle exceptions and write an error message + Write-Error 'Ensure you have run Connect-AzAccount and that the subscription exists.' + return + } + + # Get the subscription ID and tenant ID + $subscriptionId = $subscription.SubscriptionId + $tenantId = $subscription.TenantId + + # Create a custom object to return + $object = [PSCustomObject]@{ + SubscriptionId = $subscriptionId + TenantId = $tenantId + } + + return $object +} + +<# +.SYNOPSIS + Creates a folder in a Data Lake Storage account. + +.DESCRIPTION + The Add-DataLakeFolder function creates a folder (or folder hierarchy) in a Data Lake storage account container. It requires an active connection to Azure. + +.PARAMETER SubscriptionName + The name of the Azure subscription to use. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container in the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder to create. May be a single folder or a folder hierarchy (e.g. 'folder1/folder2/folder3'). This parameter is mandatory. + +.PARAMETER ErrorIfFolderExists + Optional switch to throw error if folder exists. If not specified, will return the existing folder. + +.EXAMPLE + PS C:\> Add-DataLakeFolder -SubscriptionName 'MySubscription' -ResourceGroupName 'MyResourceGroup' -StorageAccountName 'MyStorageAccount' -ContainerName 'MyContainer' -FolderPath 'folder1/folder2/folder3' + This example creates a folder hierarchy 'folder1/folder2/folder3' in the specified Data Lake storage account container. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Add-DataLakeFolder +{ + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $true)] + [string]$FolderPath, # Path to the folder + + [switch]$ErrorIfFolderExists # Flag to indicate if an error should be thrown if the folder exists + ) + + # Get the subscription ID + $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId + if ($null -eq $subId) + { + Write-Error 'Subscription not found.' + return + } + + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) + { + Write-Error 'Failed to set the Azure context.' + return + } + + # Check if the Az.Storage module is available and import it + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { + Write-Error 'Required module Az.Storage is not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) + { + Write-Error 'Storage account not found.' + return + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) + { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # Create the folder + try + { + $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop + } + catch + { + if ($ErrorIfFolderExists) + { + Write-Error "Folder $FolderPath already exists." + return + } + $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists + return + } + + if ($null -eq $ret) + { + Write-Error 'Failed to create the folder.' + return + } + else + { + return $ret # Return the created folder + } +} + +<# +.SYNOPSIS + Deletes a folder from an Azure Data Lake Storage Gen2 account. + +.DESCRIPTION + The Remove-DataLakeFolder function deletes a folder from an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, and folder path as input parameters. If the folder does not exist, it will return an error unless the -ErrorIfFolderDoesNotExist switch is used. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container containing the folder. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder to delete. This parameter is mandatory. + +.PARAMETER ErrorIfFolderDoesNotExist + If this switch is used, the function will not return an error if the folder does not exist. + +.EXAMPLE + PS C:\> Remove-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "MyFolder" + This example deletes the folder "MyFolder" from the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the "MySubscription" Azure subscription. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Remove-DataLakeFolder +{ + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $true)] + [string]$FolderPath, # Path to the folder + + [switch]$ErrorIfFolderDoesNotExist # Flag to indicate if an error should be thrown if the folder does not exist + ) + + # Get the subscription ID + $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId + if ($null -eq $subId) + { + Write-Error 'Subscription not found.' + return + } + + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) + { + Write-Error 'Failed to set the Azure context.' + return + } + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) + { + Write-Error 'Storage account not found.' + return + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) + { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # Ensure the folder exists before deleting + try + { + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -ErrorAction Stop + } + catch + { + if ($ErrorIfFolderDoesNotExist) + { + Write-Error "Folder '$FolderPath' does not exist to delete." + return + } + return + } + + # Delete the folder + if ($null -ne $folderExists) + { + $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force + } + + if ($null -ne $ret) + { + Write-Error 'Failed to delete the folder.' + return + } + else + { + Write-Host "Folder $ContainerName\$FolderPath deleted successfully." + return + } +} + +<# +.SYNOPSIS + Sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. + +.DESCRIPTION + The Set-DataLakeFolderACL function sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, folder path, identity, and access control type as input parameters. Optionally, it can also set the ACL for the container and include the default scope in the ACL. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container containing the folder. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is mandatory. + +.PARAMETER Identity + The identity to use in the ACL. This parameter is mandatory. + +.PARAMETER AccessControlType + The type of access control to apply to the folder. Valid values are 'Read' and 'Write'. This parameter is mandatory. + +.PARAMETER SetContainerACL + A switch parameter that specifies whether to set the ACL for the container. This parameter is optional. + +.PARAMETER IncludeDefaultScope + A switch parameter that specifies whether to include the default scope in the ACL. This parameter is optional. + +.PARAMETER DoNotApplyACLRecursively + A switch parameter that specifies whether to set the ACL recursively. This parameter is optional. + +.EXAMPLE + PS C:\> Set-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" -Identity "MyIdentity" -AccessControlType "Read" -IncludeDefaultScope + This example sets the ACL for the folder "/MyFolder" in the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" for the identity "MyIdentity" with read access and includes the default scope in the ACL. + +.NOTES + This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Set-DataLakeFolderACL +{ + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, + + [Parameter(Mandatory = $true)] + [string]$ContainerName, + + [Parameter(Mandatory = $true)] + [string]$FolderPath, + + [Parameter(Mandatory = $true)] + [string]$Identity, + + [ValidateSet('Read', 'Write')] + [Parameter(Mandatory = $true)] + [string]$AccessControlType, + + [switch]$SetContainerACL, + + [switch]$IncludeDefaultScope, + + [switch]$DoNotApplyACLRecursively + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName + if ($null -eq $sub) + { + Write-Error 'Subscription not found. Ensure you have run Connect-AzAccount before execution.' + return + } + else + { + $subId = $sub.SubscriptionId + } + + # Get the object ID of the identity to use in the ACL + $identityObj = Get-AADObjectId -Identity $Identity + if ($null -eq $identityObj) + { + Write-Error 'Identity not found.' + return + } + else + { + Write-Verbose ('{0} ID: {1} Display Name: {2}' -f $identityObj.ObjectType, $identityObj.ObjectId, $identityObj.DisplayName) + } + + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) + { + Write-Error 'Failed to set the Azure context.' + return + } + else + { + Write-Verbose $subContext.Name + } + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) + { + Write-Error 'Storage account not found.' + return + } + else + { + Write-Verbose $storageAccount.StorageAccountName + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) + { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # verify the folder exists before setting the ACL + try + { + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + if ($null -eq $folderExists) + { + Write-Error('Folder not found.') + return + } + } + catch + { + Write-Error('Folder not found.') + return + } + + # translate the access control type to applied permission + $permission = switch ( $AccessControlType ) + { 'Read' + { 'r-x' + } + 'Write' + { 'rwx' + } + default + { '' + } + } + + $identityType = switch ($identityObj.ObjectType) + { 'User' + { 'user' + } + 'Group' + { 'group' + } + 'ServicePrincipal' + { 'user' + } + 'ManagedIdentity' + { 'other' + } + default + { '' + } + } + + # set the ACL at the container level + if ($SetContainerACL) + { + Write-Verbose 'set container ACL' + $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL + + if ($result.FailedEntries.Count -gt 0) + { + Write-Error 'Failed to set the ACL for the container.' + Write-Error $result.FailedEntries + return + } + else + { + Write-Host 'Container ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + + # get the ACL for the folder + $acl = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath).ACL + + try + { + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl + if(-not $DoNotApplyACLRecursively) + { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else + { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + + } + catch [Microsoft.PowerShell.Commands.WriteErrorException] + { + Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' + return + } + + if ($result.FailedEntries.Count -gt 0) + { + Write-Error 'Failed to set the ACL.' + Write-Error $result.FailedEntries + return + } + else + { + Write-Host 'ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + ###################### + # default scope + if ($IncludeDefaultScope) + { + Write-Verbose 'include default scope' + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope + if(-not $DoNotApplyACLRecursively) + { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else + { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + + if ($result.FailedEntries.Count -gt 0) + { + Write-Error 'Failed to set the ACL for the default scope.' + Write-Error $result.FailedEntries + return + } + else + { + Write-Host 'Default Scope ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + + +} + +<# +.SYNOPSIS + Gets the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. + +.DESCRIPTION + The Get-DataLakeFolderACL function retrieves the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, and container name as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is optional. If omitted, the function will revert to the root of the container. + +.EXAMPLE + PS C:\> Get-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" + This example gets the ACL for the folder "/MyFolder" in the container "MyContainer" of the storage account "MyStorageAccount" in the resource group "MyResourceGroup" of the Azure subscription "MySubscription". + +.NOTES + This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Get-DataLakeFolderACL +{ + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $false)] + [string]$FolderPath = '/' # Path to the folder in the Data Lake + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Remove leading slash or backslash from the folder path + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) + { + $FolderPath = $FolderPath.Substring(1) + } + + try + { + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Check if the folder exists + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + + # Get the ACLs for the folder + $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL + + # Process each ACL + $aclResults = foreach ($ace in $acls) + { + if ($ace.EntityId) + { + # Get the AD object for the entity + $adObject = Get-AzureADObjectByObjectId -ObjectIds $ace.EntityId + + # Create a custom object with the ACL info + [PSCustomObject]@{ + DisplayName = $adObject.DisplayName + ObjectId = $ace.EntityId + ObjectType = $adObject.ObjectType + Permissions = $ace.Permissions + DefaultScope = $ace.DefaultScope + } + } + } + + # Return the results + return $aclResults + } + catch + { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +<# +.SYNOPSIS + Moves a folder in Azure Data Lake Storage Gen2 to a new location. + +.DESCRIPTION + The Move-DataLakeFolder function moves a folder in Azure Data Lake Storage Gen2 to a new location. It requires the subscription name, resource group name, storage account name, source container name, source folder path, and destination folder path as input parameters. Optionally, it can also take a destination container name. If the destination container name is not provided, the function will use the source container name. + +.PARAMETER SubscriptionName + The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER SourceContainerName + The name of the source container for the move operation. This parameter is mandatory. + +.PARAMETER SourceFolderPath + The path of the folder to move. This parameter is mandatory. + +.PARAMETER DestinationContainerName + The name of the destination container for the move operation. This parameter is optional. If not specified, the function will use the source container name. + +.PARAMETER DestinationFolderPath + The path of the destination folder. This parameter is mandatory. + +.EXAMPLE + PS C:\> Move-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -SourceContainerName "MySourceContainer" -SourceFolderPath "/MySourceFolder" -DestinationFolderPath "/MyDestinationFolder" + This example moves the folder "/MySourceFolder" from the container "MySourceContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription" to the folder "/MyDestinationFolder" in the same container. + +.NOTES + This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Move-DataLakeFolder +{ + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$SourceContainerName, # Source container name + + [Parameter(Mandatory = $true)] + [string]$SourceFolderPath, # Source folder path + + [Parameter(Mandatory = $false)] + [string]$DestinationContainerName, # Destination container name + + [Parameter(Mandatory = $true)] + [string]$DestinationFolderPath # Destination folder path + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + try + { + + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Check if the folder exists + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath + + # If destination container name is not provided, use source container name + if (-not $DestinationContainerName) + { + $DestinationContainerName = $SourceContainerName + } + + # Move the folder + $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force + + # Write verbose output and return the result + Write-Verbose ('Function: Move-DataLakeFolder') + Write-Verbose "Folder moved: $DestinationFolderPath" + return $ret + } + catch + { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +<# +.SYNOPSIS + Removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. + +.DESCRIPTION + The Remove-DataLakeFolderACL function removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, container name, and identity as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. + +.PARAMETER SubscriptionName + The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is optional. If not specified, the function will revert to the root of the container. + +.PARAMETER Identity + The identity to remove from the ACL. This parameter is mandatory. + +.PARAMETER DoNotApplyACLRecursively + A switch parameter that specifies whether to remove the identity from the ACL recursively. This parameter is optional. + +.EXAMPLE + PS C:\> Remove-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -Identity "MyIdentity" + This example removes the identity "MyIdentity" from the ACL of the root of the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription". + +.NOTES + This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Remove-DataLakeFolderACL +{ + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $false)] + [string]$FolderPath = '/', # Path to the folder in the Data Lake + + [Parameter(Mandatory = $true)] + [string]$Identity, # Identity to remove from the ACL + + [Parameter(Mandatory = $false)] + [switch]$DoNotApplyACLRecursively # Flag to indicate if the ACL should not be applied recursively + + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Remove leading slash or backslash from the folder path + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) + { + $FolderPath = $FolderPath.Substring(1) + } + + try + { + + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Get the object ID of the identity to use in the ACL + $identityObj = Get-AADObjectId -Identity $Identity + $id = $identityObj.ObjectId + + # Get the folder + $folder = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + + # Get the ACLs for the folder + $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL + + # Remove the specified identity from the ACL + $newacl = $acls | Where-Object { -not ($_.AccessControlType -eq $identityObj.ObjectType -and $_.EntityId -eq $id) } + + # Update the ACL + if(-not $DoNotApplyACLRecursively) + { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } + else + { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } + + # Check if the update was successful + if ($result.FailedEntries.Count -gt 0) + { + Write-Error 'Failed to update the ACL.' + Write-Error $result.FailedEntries + } + else + { + Write-Host 'ACL updated successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + catch + { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +#region Module Initialization +# This code runs when the module is imported +# Check dependencies on module import +$dependencyCheckResult = Test-ModuleDependencies -Quiet + +if (-not $dependencyCheckResult) { + Write-Warning @" +AzureDataLakeManagement module loaded with missing dependencies. +Some functions may not work correctly until required modules are installed. +Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +"@ +} +#endregion + +Export-ModuleMember -Function * diff --git a/LICENSE b/LICENSE index 9c38f61..2c0bbe8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,21 @@ -MIT License - -Copyright (c) 2023 SteveCInVA - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License + +Copyright (c) 2023 SteveCInVA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index fde89c8..e5556fb 100644 --- a/README.md +++ b/README.md @@ -1,124 +1,153 @@ -# AzureDataLakeManagement -This project was created to help simplify the process of managing an Azure Datalake specifically around updating existing ACL's to child objects within the lake. -Yes, this can be accomplished with Azure Storage Explorer, however come customers don't like to install new software. - -My goal, is to make a straight forward set of functions that will assist a user in configuring folders and the associated ACL's in an ADLS Gen 2 storage container using the objects names rather than ID's. - -To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement - -## Development Environment - -### Using Dev Containers (Recommended) - -This repository includes support for Visual Studio Code dev containers, providing a consistent development environment with all required tools pre-installed. - -#### Prerequisites -- [Visual Studio Code](https://code.visualstudio.com/) -- [Docker Desktop](https://www.docker.com/products/docker-desktop) -- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VS Code - -#### Getting Started with Dev Containers -1. Clone the repository -2. Open the repository folder in Visual Studio Code -3. When prompted, click "Reopen in Container" (or use Command Palette: `Dev Containers: Reopen in Container`) -4. VS Code will build the container and install all dependencies automatically - -#### What's Included -The dev container includes: -- **PowerShell 7+** - Latest version installed automatically -- **Pre-installed VS Code Extensions:** - - PowerShell - Language support and debugging - - Pester Test - Testing framework support - - GitHub Copilot - AI-powered code assistance - - GitHub Actions - Workflow file support - - TODO Highlight v2 - Highlight TODO comments -- **PowerShell Modules:** - - PSScriptAnalyzer - For code quality checks - - Pester - For testing - -#### Working in the Dev Container -Once the container is running, you can: -- Import the module: `Import-Module ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 -Force` -- Run code quality checks: `Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1` -- Test the module: `Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1` -- Run Pester tests: `Invoke-Pester -Path ./Tests` - -### Local Development (Without Dev Containers) - -If you prefer to develop locally without containers, ensure you have: -- PowerShell 7+ installed -- Required PowerShell modules (see Dependency Management section below) - -## Dependency Management - -Starting with version 2025.1.1, the module includes improved dependency management features: - -### Required Dependencies -The module requires the following PowerShell modules: -- `Az.Storage` - For Azure Storage operations -- `AzureAD` - For Azure Active Directory operations -- `Az.Accounts` - For Azure authentication - -### Automatic Dependency Checking -When you import the module, it automatically checks for missing dependencies and provides helpful guidance: - -```powershell -Import-Module AzureDataLakeManagement -# Output: WARNING: AzureDataLakeManagement module loaded with missing dependencies. -# Some functions may not work correctly until required modules are installed. -# Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. -``` - -### Dependency Management Functions - -#### Test-ModuleDependencies -Check which dependencies are available: -```powershell -Test-ModuleDependencies -``` - -Automatically install missing dependencies: -```powershell -Test-ModuleDependencies -AutoInstall -``` - -#### Install-ModuleDependencies -Install all or specific dependencies: -```powershell -Install-ModuleDependencies -Install-ModuleDependencies -Modules @('Az.Storage') -``` - -#### Manual Installation -You can also install dependencies manually: -```powershell -Install-Module -Name Az.Storage -Force -Install-Module -Name AzureAD -Force -Install-Module -Name Az.Accounts -Force -``` - -### Improved Error Handling -Functions now provide clearer error messages when dependencies are missing, guiding users to install the required modules. - -*** - -## Version History: - -- 2025.1.1 - 01/09/2025 -Issue 27 - Added optional switch to set-DataLakeFolderACL and remove-DataLakeFolderACL functions to enable the user to not recursively apply permissions on children of the path specified. - -- 2024.1.1 - 01/09/2024 -Issue 22 - Fixed issue where a lack of Azure Permissions to Microsoft.Storage/storageAccounts/listKeys/action would cause failure to execute even with correct AzureAD permissions on objects. - -- 2023.12.3 - 12/01/2023 -Published via Github Actions to [PowershellGallery.com](https://www.powershellgallery.com/packages/AzureDataLakeManagement) - -- 2023.12.2 - 12/01/2023 -Removed - Github Action Publish testing - -- 2023.12.1 - 12/01/2023 -Function optimization and improved consistency of help content. - -- 2023.11.2 - 11/13/2023 -Added function to remove an ACL from a folder and all inherited children - +# AzureDataLakeManagement +This project was created to help simplify the process of managing an Azure Datalake specifically around updating existing ACL's to child objects within the lake. +Yes, this can be accomplished with Azure Storage Explorer, however come customers don't like to install new software. + +My goal, is to make a straight forward set of functions that will assist a user in configuring folders and the associated ACL's in an ADLS Gen 2 storage container using the objects names rather than ID's. + +To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement + +## Module Usage + +- For example usage see: [example.ps1](example.ps1) +- For module installation support see: [example-dependency-management.ps1](example-dependency-management.ps1) + +## Functions Supported: + +### Folder Management +- Add-DataLakeFolder +- Move-DataLakeFolder +- Remove-DataLakeFolder + +### ACL Management +- Get-DataLakeFolderACL +- Set-DataLakeFolderACL +- Remove-DataLakeFolderACL + +### Support Functions +- Get-AADObjectId +- Get-AzureSubscriptionInfo + +### Module Installation Support +- Import-ModuleDependencies +- Install-ModuleDependencies +- Test-ModuleDependencies + +## Development Environment + +### Using Dev Containers (Recommended) + +This repository includes support for Visual Studio Code dev containers, providing a consistent development environment with all required tools pre-installed. + +#### Prerequisites +- [Visual Studio Code](https://code.visualstudio.com/) +- [Docker Desktop](https://www.docker.com/products/docker-desktop) +- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VS Code + +#### Getting Started with Dev Containers +1. Clone the repository +2. Open the repository folder in Visual Studio Code +3. When prompted, click "Reopen in Container" (or use Command Palette: `Dev Containers: Reopen in Container`) +4. VS Code will build the container and install all dependencies automatically + +#### What's Included +The dev container includes: +- **PowerShell 7+** - Latest version installed automatically +- **Pre-installed VS Code Extensions:** + - PowerShell - Language support and debugging + - Pester Test - Testing framework support + - GitHub Copilot - AI-powered code assistance + - GitHub Actions - Workflow file support + - TODO Highlight v2 - Highlight TODO comments +- **PowerShell Modules:** + - PSScriptAnalyzer - For code quality checks + - Pester - For testing + +#### Working in the Dev Container +Once the container is running, you can: +- Import the module: `Import-Module ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 -Force` +- Run code quality checks: `Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1` +- Test the module: `Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1` +- Run Pester tests: `Invoke-Pester -Path ./Tests` + +### Local Development (Without Dev Containers) + +If you prefer to develop locally without containers, ensure you have: +- PowerShell 7+ installed +- Required PowerShell modules (see Dependency Management section below) + +## Dependency Management + +Starting with version 2025.1.1, the module includes improved dependency management features: + +### Required Dependencies +The module requires the following PowerShell modules: +- `Az.Storage` - For Azure Storage operations +- `AzureAD` - For Azure Active Directory operations +- `Az.Accounts` - For Azure authentication + +### Automatic Dependency Checking +When you import the module, it automatically checks for missing dependencies and provides helpful guidance: + +```powershell +Import-Module AzureDataLakeManagement +# Output: WARNING: AzureDataLakeManagement module loaded with missing dependencies. +# Some functions may not work correctly until required modules are installed. +# Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +``` + +### Dependency Management Functions + +#### Test-ModuleDependencies +Check which dependencies are available: +```powershell +Test-ModuleDependencies +``` + +Automatically install missing dependencies: +```powershell +Test-ModuleDependencies -AutoInstall +``` + +#### Install-ModuleDependencies +Install all or specific dependencies: +```powershell +Install-ModuleDependencies +Install-ModuleDependencies -Modules @('Az.Storage') +``` + +#### Manual Installation +You can also install dependencies manually: +```powershell +Install-Module -Name Az.Storage -Force +Install-Module -Name AzureAD -Force +Install-Module -Name Az.Accounts -Force +``` + +### Improved Error Handling +Functions now provide clearer error messages when dependencies are missing, guiding users to install the required modules. + +*** + +## Version History: + +- 2025.11.1 - 11/04/2025 +Issue 34 - Added support for Visual Studio Code - Dev Containers to improve development / testing for solution. Improved documentation. + +- 2025.1.1 - 01/09/2025 +Issue 27 - Added optional switch to set-DataLakeFolderACL and remove-DataLakeFolderACL functions to enable the user to not recursively apply permissions on children of the path specified. + +- 2024.1.1 - 01/09/2024 +Issue 22 - Fixed issue where a lack of Azure Permissions to Microsoft.Storage/storageAccounts/listKeys/action would cause failure to execute even with correct AzureAD permissions on objects. + +- 2023.12.3 - 12/01/2023 +Published via Github Actions to [PowershellGallery.com](https://www.powershellgallery.com/packages/AzureDataLakeManagement) + +- 2023.12.2 - 12/01/2023 +Removed - Github Action Publish testing + +- 2023.12.1 - 12/01/2023 +Function optimization and improved consistency of help content. + +- 2023.11.2 - 11/13/2023 +Added function to remove an ACL from a folder and all inherited children + diff --git a/Tests/DependencyManagement.Tests.ps1 b/Tests/DependencyManagement.Tests.ps1 index 3b9f86b..313173d 100644 --- a/Tests/DependencyManagement.Tests.ps1 +++ b/Tests/DependencyManagement.Tests.ps1 @@ -1,96 +1,96 @@ -#Requires -Modules Pester - -BeforeAll { - # Import the module - $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' - Import-Module $ModulePath -Force -} - -Describe 'AzureDataLakeManagement Dependency Management' { - - Context 'Test-ModuleDependencies Function' { - It 'Should export Test-ModuleDependencies function' { - Get-Command -Name 'Test-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty - } - - It 'Should have the correct parameters' { - $command = Get-Command -Name 'Test-ModuleDependencies' - $command.Parameters.Keys | Should -Contain 'AutoInstall' - $command.Parameters.Keys | Should -Contain 'Quiet' - } - - It 'Should return boolean value' { - Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Storage' } - Mock Get-Module { $null } -ParameterFilter { $Name -eq 'AzureAD' } - Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Accounts' } - - $result = Test-ModuleDependencies -Quiet - $result | Should -BeOfType [System.Boolean] - } - } - - Context 'Install-ModuleDependencies Function' { - It 'Should export Install-ModuleDependencies function' { - Get-Command -Name 'Install-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty - } - - It 'Should have the correct parameters' { - $command = Get-Command -Name 'Install-ModuleDependencies' - $command.Parameters.Keys | Should -Contain 'Modules' - $command.Parameters.Keys | Should -Contain 'Quiet' - } - } - - Context 'Import-ModuleDependencies Function' { - It 'Should export Import-ModuleDependencies function' { - Get-Command -Name 'Import-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty - } - - It 'Should have the correct parameters' { - $command = Get-Command -Name 'Import-ModuleDependencies' - $command.Parameters.Keys | Should -Contain 'RequiredModules' - $command.Parameters.Keys | Should -Contain 'Quiet' - } - - It 'Should return boolean value' { - Mock Get-Module { $null } -ParameterFilter { $ListAvailable -eq $true } - $result = Import-ModuleDependencies -RequiredModules @('NonExistentModule') -Quiet - $result | Should -BeOfType [System.Boolean] - } - } - - Context 'Module Manifest' { - It 'Should declare external module dependencies in manifest' { - $manifest = Test-ModuleManifest -Path $ModulePath - $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' - $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'AzureAD' - $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Accounts' - } - - It 'Should export dependency management functions' { - $manifest = Test-ModuleManifest -Path $ModulePath - $manifest.ExportedFunctions.Keys | Should -Contain 'Test-ModuleDependencies' - $manifest.ExportedFunctions.Keys | Should -Contain 'Install-ModuleDependencies' - $manifest.ExportedFunctions.Keys | Should -Contain 'Import-ModuleDependencies' - } - } - - Context 'Integration with Existing Functions' { - It 'Should have updated Add-DataLakeFolder to use centralized dependency checking' { - $functionContent = (Get-Command Add-DataLakeFolder).Definition - $functionContent | Should -Match 'Import-ModuleDependencies' - $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' - } - - It 'Should have updated Set-DataLakeFolderACL to use centralized dependency checking' { - $functionContent = (Get-Command Set-DataLakeFolderACL).Definition - $functionContent | Should -Match 'Import-ModuleDependencies' - $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' - } - } -} - -AfterAll { - # Clean up - Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue +#Requires -Modules Pester + +BeforeAll { + # Import the module + $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' + Import-Module $ModulePath -Force +} + +Describe 'AzureDataLakeManagement Dependency Management' { + + Context 'Test-ModuleDependencies Function' { + It 'Should export Test-ModuleDependencies function' { + Get-Command -Name 'Test-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Test-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'AutoInstall' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Storage' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'AzureAD' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Accounts' } + + $result = Test-ModuleDependencies -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Install-ModuleDependencies Function' { + It 'Should export Install-ModuleDependencies function' { + Get-Command -Name 'Install-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Install-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'Modules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + } + + Context 'Import-ModuleDependencies Function' { + It 'Should export Import-ModuleDependencies function' { + Get-Command -Name 'Import-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Import-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'RequiredModules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $ListAvailable -eq $true } + $result = Import-ModuleDependencies -RequiredModules @('NonExistentModule') -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Module Manifest' { + It 'Should declare external module dependencies in manifest' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'AzureAD' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Accounts' + } + + It 'Should export dependency management functions' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.ExportedFunctions.Keys | Should -Contain 'Test-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Install-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Import-ModuleDependencies' + } + } + + Context 'Integration with Existing Functions' { + It 'Should have updated Add-DataLakeFolder to use centralized dependency checking' { + $functionContent = (Get-Command Add-DataLakeFolder).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + + It 'Should have updated Set-DataLakeFolderACL to use centralized dependency checking' { + $functionContent = (Get-Command Set-DataLakeFolderACL).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + } +} + +AfterAll { + # Clean up + Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue } \ No newline at end of file diff --git a/Tests/DevContainer.Tests.ps1 b/Tests/DevContainer.Tests.ps1 index 175abe7..4e14d81 100644 --- a/Tests/DevContainer.Tests.ps1 +++ b/Tests/DevContainer.Tests.ps1 @@ -1,85 +1,85 @@ -#Requires -Modules Pester - -BeforeAll { - $DevContainerPath = Join-Path $PSScriptRoot '..' '.devcontainer' 'devcontainer.json' -} - -Describe 'DevContainer Configuration' { - - Context 'File Existence' { - It 'Should have a devcontainer.json file' { - Test-Path $DevContainerPath | Should -BeTrue - } - } - - Context 'Configuration Content' { - BeforeAll { - # Read and parse the devcontainer.json file (JSONC - JSON with Comments) - $content = Get-Content $DevContainerPath -Raw - # Remove single-line comments more carefully (avoid URLs with //) - $lines = $content -split "`n" - $cleanedLines = @() - foreach ($line in $lines) { - # Skip comment-only lines - if ($line -match '^\s*//') { - continue - } - # Remove trailing comments but preserve URLs and quoted strings - # Match everything before comment outside of quotes - elseif ($line -match '^([^"]*"[^"]*"[^"]*)\s*//' -or $line -match '^(.*[^:])\s*//') { - # Keep content before comment, but be careful with URLs - if ($line -notmatch '"https?://') { - $cleanedLines += $matches[1] - } - else { - # Line contains URL, keep it as-is - $cleanedLines += $line - } - } - else { - # Keep line as-is - $cleanedLines += $line - } - } - $jsonContent = $cleanedLines -join "`n" - $config = $jsonContent | ConvertFrom-Json - } - - It 'Should have a name property' { - $config.name | Should -Not -BeNullOrEmpty - } - - It 'Should specify PowerShell feature' { - $config.features.PSObject.Properties.Name | Should -Contain 'ghcr.io/devcontainers/features/powershell:1' - } - - It 'Should have PowerShell extension configured' { - $config.customizations.vscode.extensions | Should -Contain 'ms-vscode.powershell' - } - - It 'Should have Pester Test extension configured' { - $config.customizations.vscode.extensions | Should -Contain 'pspester.pester-test' - } - - It 'Should have GitHub Copilot extension configured' { - $config.customizations.vscode.extensions | Should -Contain 'github.copilot' - } - - It 'Should have GitHub Actions extension configured' { - $config.customizations.vscode.extensions | Should -Contain 'github.vscode-github-actions' - } - - It 'Should have TODO Highlight extension configured' { - $config.customizations.vscode.extensions | Should -Contain 'jgclark.vscode-todo-highlight' - } - - It 'Should install PSScriptAnalyzer and Pester in postCreateCommand' { - $config.postCreateCommand | Should -Match 'PSScriptAnalyzer' - $config.postCreateCommand | Should -Match 'Pester' - } - - It 'Should set PowerShell as default terminal' { - $config.customizations.vscode.settings.'terminal.integrated.defaultProfile.linux' | Should -Be 'pwsh' - } - } -} +#Requires -Modules Pester + +BeforeAll { + $DevContainerPath = Join-Path $PSScriptRoot '..' '.devcontainer' 'devcontainer.json' +} + +Describe 'DevContainer Configuration' { + + Context 'File Existence' { + It 'Should have a devcontainer.json file' { + Test-Path $DevContainerPath | Should -BeTrue + } + } + + Context 'Configuration Content' { + BeforeAll { + # Read and parse the devcontainer.json file (JSONC - JSON with Comments) + $content = Get-Content $DevContainerPath -Raw + # Remove single-line comments more carefully (avoid URLs with //) + $lines = $content -split "`n" + $cleanedLines = @() + foreach ($line in $lines) { + # Skip comment-only lines + if ($line -match '^\s*//') { + continue + } + # Remove trailing comments but preserve URLs and quoted strings + # Match everything before comment outside of quotes + elseif ($line -match '^([^"]*"[^"]*"[^"]*)\s*//' -or $line -match '^(.*[^:])\s*//') { + # Keep content before comment, but be careful with URLs + if ($line -notmatch '"https?://') { + $cleanedLines += $matches[1] + } + else { + # Line contains URL, keep it as-is + $cleanedLines += $line + } + } + else { + # Keep line as-is + $cleanedLines += $line + } + } + $jsonContent = $cleanedLines -join "`n" + $config = $jsonContent | ConvertFrom-Json + } + + It 'Should have a name property' { + $config.name | Should -Not -BeNullOrEmpty + } + + It 'Should specify PowerShell feature' { + $config.features.PSObject.Properties.Name | Should -Contain 'ghcr.io/devcontainers/features/powershell:1' + } + + It 'Should have PowerShell extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'ms-vscode.powershell' + } + + It 'Should have Pester Test extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'pspester.pester-test' + } + + It 'Should have GitHub Copilot extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.copilot' + } + + It 'Should have GitHub Actions extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.vscode-github-actions' + } + + It 'Should have TODO Highlight extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'jgclark.vscode-todo-highlight' + } + + It 'Should install PSScriptAnalyzer and Pester in postCreateCommand' { + $config.postCreateCommand | Should -Match 'PSScriptAnalyzer' + $config.postCreateCommand | Should -Match 'Pester' + } + + It 'Should set PowerShell as default terminal' { + $config.customizations.vscode.settings.'terminal.integrated.defaultProfile.linux' | Should -Be 'pwsh' + } + } +} diff --git a/example-dependency-management.ps1 b/example-dependency-management.ps1 index 1bcfed1..164641a 100644 --- a/example-dependency-management.ps1 +++ b/example-dependency-management.ps1 @@ -1,29 +1,29 @@ -# Example script demonstrating improved dependency management in AzureDataLakeManagement - -# Import the module - it will now provide helpful feedback about missing dependencies -Import-Module .\AzureDataLakeManagement\AzureDataLakeManagement.psd1 - -# Check what dependencies are missing -Test-ModuleDependencies - -# To install missing dependencies automatically, you can run: -# Test-ModuleDependencies -AutoInstall - -# Or install them manually: -# Install-Module -Name Az.Storage -Force -# Install-Module -Name AzureAD -Force -# Install-Module -Name Az.Accounts -Force - -# The module will now provide better error messages when functions are called without required dependencies -# For example: -# Add-DataLakeFolder -SubscriptionName 'test' -ResourceGroupName 'test' -StorageAccountName 'test' -ContainerName 'test' -FolderPath 'test' - -Write-Host " -Dependency Management Features: -- Test-ModuleDependencies: Check which dependencies are available -- Test-ModuleDependencies -AutoInstall: Automatically install missing dependencies -- Install-ModuleDependencies: Install specific dependencies -- Import-ModuleDependencies: Import dependencies with better error handling - -The module will automatically check dependencies when imported and provide helpful guidance. +# Example script demonstrating improved dependency management in AzureDataLakeManagement + +# Import the module - it will now provide helpful feedback about missing dependencies +Import-Module .\AzureDataLakeManagement\AzureDataLakeManagement.psd1 + +# Check what dependencies are missing +Test-ModuleDependencies + +# To install missing dependencies automatically, you can run: +# Test-ModuleDependencies -AutoInstall + +# Or install them manually: +# Install-Module -Name Az.Storage -Force +# Install-Module -Name AzureAD -Force +# Install-Module -Name Az.Accounts -Force + +# The module will now provide better error messages when functions are called without required dependencies +# For example: +# Add-DataLakeFolder -SubscriptionName 'test' -ResourceGroupName 'test' -StorageAccountName 'test' -ContainerName 'test' -FolderPath 'test' + +Write-Host " +Dependency Management Features: +- Test-ModuleDependencies: Check which dependencies are available +- Test-ModuleDependencies -AutoInstall: Automatically install missing dependencies +- Install-ModuleDependencies: Install specific dependencies +- Import-ModuleDependencies: Import dependencies with better error handling + +The module will automatically check dependencies when imported and provide helpful guidance. " -ForegroundColor Green \ No newline at end of file diff --git a/example.ps1 b/example.ps1 index f17ff80..822e6fd 100644 --- a/example.ps1 +++ b/example.ps1 @@ -1,46 +1,46 @@ -import-module AzureDatalakeManagement - -Connect-AzAccount -Connect-AzureAd - -$subName = '' -$rgName = 'resourceGroup01' -$storageAccountName = 'storage01' -$containerName = 'bronze' - -#create basic dataset folder structure -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleB\test1\subA\subB' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleC' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset2' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset3' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' - -#add duplicate folder in error -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' - -#add duplicate folder in error & show error -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -ErrorIfFolderExists - -#Set user acl at the root of the dataset but don't set default scope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity 'sam@contoso.com' -accessControlType Read - -#Set user acl at the root of the dataset and set default scope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity 'sam@contoso.com' -accessControlType Read -IncludeDefaultScope - -#Set user acl at the root of the dataset and configure the container for access by the user -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' -accessControlType Write -IncludeDefaultScope -SetContainerACL - -#set service acl at sub folder -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB\test1' -Identity '' -accessControlType Write -IncludeDefaultScope - -#set group acl at root of dataset -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity "" -accessControlType Read -IncludeDefaultScope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity "" -accessControlType Read -IncludeDefaultScope - -#remove folder from specified container -remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' - -#move sub folder from one dataset to another -move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'dataset1\sampleB' -DestinationContainerName $containerName -destinationFolderPath 'dataset2\sampleb' - +import-module AzureDatalakeManagement + +Connect-AzAccount -UseDeviceAuthentication +Connect-AzureAd + +$subName = '' +$rgName = 'resourceGroup01' +$storageAccountName = 'storage01' +$containerName = 'bronze' + +#create basic dataset folder structure +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleB\test1\subA\subB' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleC' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset2' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset3' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' + +#add duplicate folder in error +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' + +#add duplicate folder in error & show error +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -ErrorIfFolderExists + +#Set user acl at the root of the dataset but don't set default scope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity 'sam@contoso.com' -accessControlType Read + +#Set user acl at the root of the dataset and set default scope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity 'sam@contoso.com' -accessControlType Read -IncludeDefaultScope + +#Set user acl at the root of the dataset and configure the container for access by the user +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' -accessControlType Write -IncludeDefaultScope -SetContainerACL + +#set service acl at sub folder +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB\test1' -Identity '' -accessControlType Write -IncludeDefaultScope + +#set group acl at root of dataset +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity "" -accessControlType Read -IncludeDefaultScope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity "" -accessControlType Read -IncludeDefaultScope + +#remove folder from specified container +remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' + +#move sub folder from one dataset to another +move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'dataset1\sampleB' -DestinationContainerName $containerName -destinationFolderPath 'dataset2\sampleb' + diff --git a/publish.ps1 b/publish.ps1 index ef02346..8b8db33 100644 --- a/publish.ps1 +++ b/publish.ps1 @@ -1,7 +1,7 @@ - -$parameters = @{ - NuGetApiKey = $env:PSGalleryKey - Path = "$PSScriptRoot\AzureDataLakeManagement" -} -Publish-Module @parameters - + +$parameters = @{ + NuGetApiKey = $env:PSGalleryKey + Path = "$PSScriptRoot\AzureDataLakeManagement" +} +Publish-Module @parameters + From 71f4fd03300fed96f276e85a60f9c3eb22f3fdac Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Tue, 4 Nov 2025 16:43:18 +0000 Subject: [PATCH 10/19] Update module version and PowerShell version in module manifest --- AzureDataLakeManagement/AzureDataLakeManagement.psd1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index d7ef980..0e8b5bd 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -12,7 +12,7 @@ RootModule = 'AzureDataLakeManagement.psm1' # Version number of this module. -ModuleVersion = '2025.1.1' +ModuleVersion = '2025.11.1' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') @@ -33,7 +33,7 @@ Copyright = '(c) 2023 Microsoft Corporation. All rights reserved.' Description = 'Azure Data Lake Management Module' # Minimum version of the Windows PowerShell engine required by this module -PowerShellVersion = '5.1' +PowerShellVersion = '7.0' # Name of the Windows PowerShell host required by this module # PowerShellHostName = '' From ea7d2b2edc5a62b65bd973320161209195a7dfcb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 16:58:44 +0000 Subject: [PATCH 11/19] Initial plan From 692dead7ffa97a0b532de4f915df2d2b3294e559 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:08:09 +0000 Subject: [PATCH 12/19] Migrate from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility - Updated module dependencies from AzureAD to Microsoft.Graph.Users and Microsoft.Graph.Groups - Modified Get-AADObjectId to use Get-MgUser, Get-MgGroup, Get-MgServicePrincipal - Updated Get-DataLakeFolderACL to use Get-MgDirectoryObject - Updated all ACL functions to use new dependencies - Updated module manifest to version 2025.11.2 - Updated README.md with new dependency information and migration notes - Updated example.ps1 to use Connect-MgGraph instead of Connect-AzureAD - Updated tests to reflect new dependencies - All tests passing Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- .../AzureDataLakeManagement.psd1 | 4 +- .../AzureDataLakeManagement.psm1 | 69 ++++++++++--------- README.md | 29 +++++++- Tests/DependencyManagement.Tests.ps1 | 3 +- example.ps1 | 3 +- 5 files changed, 70 insertions(+), 38 deletions(-) diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index 0e8b5bd..5e6b9bb 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -12,7 +12,7 @@ RootModule = 'AzureDataLakeManagement.psm1' # Version number of this module. -ModuleVersion = '2025.11.1' +ModuleVersion = '2025.11.2' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') @@ -120,7 +120,7 @@ PrivateData = @{ # RequireLicenseAcceptance = $false # External dependent modules of this module - ExternalModuleDependencies = @('Az.Storage', 'AzureAD', 'Az.Accounts') + ExternalModuleDependencies = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts') } # End of PSData hashtable diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index 23307f5..d21029c 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -23,7 +23,7 @@ Checks for required modules and automatically installs any that are missing. .NOTES - Required modules: Az.Storage, AzureAD, Az.Accounts + Required modules: Az.Storage, Microsoft.Graph.Users, Microsoft.Graph.Groups, Az.Accounts Author: Stephen Carroll - Microsoft Date: 2025-01-09 #> @@ -34,7 +34,7 @@ function Test-ModuleDependencies { [switch]$Quiet ) - $requiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') + $requiredModules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts') $missingModules = @() $availableModules = @() @@ -110,7 +110,7 @@ function Test-ModuleDependencies { function Install-ModuleDependencies { [CmdletBinding()] param( - [string[]]$Modules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [string[]]$Modules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts'), [switch]$Quiet ) @@ -172,7 +172,7 @@ function Install-ModuleDependencies { function Import-ModuleDependencies { [CmdletBinding()] param( - [string[]]$RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts'), + [string[]]$RequiredModules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts'), [switch]$Quiet ) @@ -213,7 +213,7 @@ function Import-ModuleDependencies { This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. .DESCRIPTION - Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Azure AD. + Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Microsoft Graph. .PARAMETER Identity The Identity parameter specifies the user principal name, group display name, or service principal display name of the object to retrieve. This parameter is mandatory. @@ -235,10 +235,11 @@ function Import-ModuleDependencies { This example retrieves the object ID, object type, and display name for the Azure AD group with the display name "HR Group". .NOTES - This function requires an active connection to Azure AD using Connect-AzureAD. If the specified identity does not exist, the function will return an error message. + This function requires an active connection to Microsoft Graph using Connect-MgGraph. If the specified identity does not exist, the function will return an error message. Author: Stephen Carroll - Microsoft Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> function Get-AADObjectId { @@ -249,7 +250,7 @@ function Get-AADObjectId [string]$Identity ) - # Replacing single quotes in the identity with double single quotes + # Replacing single quotes in the identity with double single quotes for filter syntax $Identity = $Identity.Replace("'", "''") try @@ -259,14 +260,15 @@ function Get-AADObjectId $group = $null $sp = $null - # Try to get the user, group, and service principal in one go - $user = Get-AzureADUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue + # Try to get the user, group, and service principal + # Using Microsoft Graph cmdlets instead of AzureAD + $user = Get-MgUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue if ($null -eq $user) { - $group = Get-AzureADGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + $group = Get-MgGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue if ($null -eq $group) { - $sp = Get-AzureADServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + $sp = Get-MgServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue } } @@ -274,19 +276,19 @@ function Get-AADObjectId if ($null -ne $user) { $objectType = 'User' - $objectId = $user.ObjectId + $objectId = $user.Id $displayName = $user.DisplayName } elseif ($null -ne $group) { $objectType = 'Group' - $objectId = $group.ObjectId + $objectId = $group.Id $displayName = $group.DisplayName } elseif ($null -ne $sp) { $objectType = 'ServicePrincipal' - $objectId = $sp.ObjectId + $objectId = $sp.Id $displayName = $sp.DisplayName } else @@ -295,13 +297,14 @@ function Get-AADObjectId return } } - catch [Microsoft.Open.Azure.AD.CommonLibrary.AadNeedAuthenticationException] - { - Write-Error 'You must be authenticated to Azure AD to run this command. Run Connect-AzureAD to authenticate.' - return - } catch { + # Check if the error is due to missing authentication + if ($_.Exception.Message -match 'Authentication needed|not authenticated|Connect-MgGraph') + { + Write-Error 'You must be authenticated to Microsoft Graph to run this command. Run Connect-MgGraph to authenticate.' + return + } Write-Error $_.Exception.Message return } @@ -675,10 +678,11 @@ function Remove-DataLakeFolder This example sets the ACL for the folder "/MyFolder" in the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" for the identity "MyIdentity" with read access and includes the default scope in the ACL. .NOTES - This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. Author: Stephen Carroll - Microsoft Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> function Set-DataLakeFolderACL { @@ -714,7 +718,7 @@ function Set-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } @@ -939,10 +943,11 @@ function Set-DataLakeFolderACL This example gets the ACL for the folder "/MyFolder" in the container "MyContainer" of the storage account "MyStorageAccount" in the resource group "MyResourceGroup" of the Azure subscription "MySubscription". .NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. Author: Stephen Carroll - Microsoft Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> function Get-DataLakeFolderACL { @@ -965,7 +970,7 @@ function Get-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } @@ -991,14 +996,14 @@ function Get-DataLakeFolderACL { if ($ace.EntityId) { - # Get the AD object for the entity - $adObject = Get-AzureADObjectByObjectId -ObjectIds $ace.EntityId + # Get the AD object for the entity using Microsoft Graph + $adObject = Get-MgDirectoryObject -DirectoryObjectId $ace.EntityId -ErrorAction SilentlyContinue # Create a custom object with the ACL info [PSCustomObject]@{ - DisplayName = $adObject.DisplayName + DisplayName = if ($adObject) { $adObject.AdditionalProperties.displayName } else { $null } ObjectId = $ace.EntityId - ObjectType = $adObject.ObjectType + ObjectType = if ($adObject) { $adObject.AdditionalProperties.'@odata.type' -replace '#microsoft.graph.', '' } else { $null } Permissions = $ace.Permissions DefaultScope = $ace.DefaultScope } @@ -1048,10 +1053,11 @@ function Get-DataLakeFolderACL This example moves the folder "/MySourceFolder" from the container "MySourceContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription" to the folder "/MyDestinationFolder" in the same container. .NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. + This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. Author: Stephen Carroll - Microsoft Date: 2021-08-31 + Updated: 2025-01-09 - Removed unnecessary AzureAD dependency #> function Move-DataLakeFolder { @@ -1080,7 +1086,7 @@ function Move-DataLakeFolder ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } @@ -1147,10 +1153,11 @@ function Move-DataLakeFolder This example removes the identity "MyIdentity" from the ACL of the root of the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription". .NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. Author: Stephen Carroll - Microsoft Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> function Remove-DataLakeFolderACL { @@ -1180,7 +1187,7 @@ function Remove-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'AzureAD') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } diff --git a/README.md b/README.md index e5556fb..47ee459 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,17 @@ If you prefer to develop locally without containers, ensure you have: ## Dependency Management -Starting with version 2025.1.1, the module includes improved dependency management features: +Starting with version 2025.1.1, the module includes improved dependency management features. **Version 2025.11.2 migrates from the deprecated AzureAD module to Microsoft.Graph for PowerShell 7+ compatibility.** ### Required Dependencies The module requires the following PowerShell modules: - `Az.Storage` - For Azure Storage operations -- `AzureAD` - For Azure Active Directory operations +- `Microsoft.Graph.Users` - For Microsoft Graph user operations (replaces AzureAD) +- `Microsoft.Graph.Groups` - For Microsoft Graph group operations (replaces AzureAD) - `Az.Accounts` - For Azure authentication +**Important**: The legacy `AzureAD` module is no longer supported as it is incompatible with PowerShell 7+ and has been deprecated by Microsoft. This module now uses the Microsoft Graph PowerShell SDK. + ### Automatic Dependency Checking When you import the module, it automatically checks for missing dependencies and provides helpful guidance: @@ -119,10 +122,21 @@ Install-ModuleDependencies -Modules @('Az.Storage') You can also install dependencies manually: ```powershell Install-Module -Name Az.Storage -Force -Install-Module -Name AzureAD -Force +Install-Module -Name Microsoft.Graph.Users -Force +Install-Module -Name Microsoft.Graph.Groups -Force Install-Module -Name Az.Accounts -Force ``` +### Authentication +Connect to both Azure and Microsoft Graph: +```powershell +# Connect to Azure +Connect-AzAccount + +# Connect to Microsoft Graph (replaces Connect-AzureAD) +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + ### Improved Error Handling Functions now provide clearer error messages when dependencies are missing, guiding users to install the required modules. @@ -130,6 +144,15 @@ Functions now provide clearer error messages when dependencies are missing, guid ## Version History: +- 2025.11.2 - 01/04/2025 +**BREAKING CHANGE**: Migrated from deprecated AzureAD module to Microsoft.Graph PowerShell SDK for PowerShell 7+ compatibility. + - Replaced `AzureAD` dependency with `Microsoft.Graph.Users` and `Microsoft.Graph.Groups` + - Updated `Get-AADObjectId` to use Microsoft Graph cmdlets (`Get-MgUser`, `Get-MgGroup`, `Get-MgServicePrincipal`) + - Updated `Get-DataLakeFolderACL` to use `Get-MgDirectoryObject` + - Updated authentication from `Connect-AzureAD` to `Connect-MgGraph` + - All ACL-related functions now support PowerShell 7+ + - Updated documentation and examples + - 2025.11.1 - 11/04/2025 Issue 34 - Added support for Visual Studio Code - Dev Containers to improve development / testing for solution. Improved documentation. diff --git a/Tests/DependencyManagement.Tests.ps1 b/Tests/DependencyManagement.Tests.ps1 index 313173d..772d850 100644 --- a/Tests/DependencyManagement.Tests.ps1 +++ b/Tests/DependencyManagement.Tests.ps1 @@ -63,7 +63,8 @@ Describe 'AzureDataLakeManagement Dependency Management' { It 'Should declare external module dependencies in manifest' { $manifest = Test-ModuleManifest -Path $ModulePath $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' - $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'AzureAD' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Users' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Groups' $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Accounts' } diff --git a/example.ps1 b/example.ps1 index 822e6fd..5696532 100644 --- a/example.ps1 +++ b/example.ps1 @@ -1,7 +1,8 @@ import-module AzureDatalakeManagement Connect-AzAccount -UseDeviceAuthentication -Connect-AzureAd +# Connect to Microsoft Graph (replaces Connect-AzureAD for PowerShell 7+ compatibility) +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" $subName = '' $rgName = 'resourceGroup01' From 950cc0c0244078b3bae01692a8fbf74f0908c58e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:10:05 +0000 Subject: [PATCH 13/19] Add comprehensive Microsoft.Graph migration tests - Created MicrosoftGraph.Tests.ps1 with 19 new tests - Tests verify correct migration from AzureAD to Microsoft.Graph - Validates cmdlet replacements in all affected functions - Confirms documentation updates - Tests backward compatibility of function outputs - All 41 tests passing (12 + 10 + 19) Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- Tests/MicrosoftGraph.Tests.ps1 | 136 +++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 Tests/MicrosoftGraph.Tests.ps1 diff --git a/Tests/MicrosoftGraph.Tests.ps1 b/Tests/MicrosoftGraph.Tests.ps1 new file mode 100644 index 0000000..424fd3e --- /dev/null +++ b/Tests/MicrosoftGraph.Tests.ps1 @@ -0,0 +1,136 @@ +#Requires -Modules Pester + +BeforeAll { + # Import the module + $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' + Import-Module $ModulePath -Force +} + +Describe 'AzureDataLakeManagement Microsoft.Graph Migration' { + + Context 'Module Dependencies' { + It 'Should not reference AzureAD module in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + # Should not have AzureAD as a standalone module reference (but AzureDataLakeManagement is OK) + $moduleContent | Should -Not -Match "@\('Az\.Storage',\s*'AzureAD'" + } + + It 'Should reference Microsoft.Graph.Users in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + $moduleContent | Should -Match "Microsoft\.Graph\.Users" + } + + It 'Should reference Microsoft.Graph.Groups in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + $moduleContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'Get-AADObjectId Function Migration' { + It 'Should use Get-MgUser instead of Get-AzureADUser' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgUser" + $functionContent | Should -Not -Match "Get-AzureADUser" + } + + It 'Should use Get-MgGroup instead of Get-AzureADGroup' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgGroup" + $functionContent | Should -Not -Match "Get-AzureADGroup" + } + + It 'Should use Get-MgServicePrincipal instead of Get-AzureADServicePrincipal' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgServicePrincipal" + $functionContent | Should -Not -Match "Get-AzureADServicePrincipal" + } + + It 'Should use .Id property instead of .ObjectId' { + $functionContent = (Get-Command Get-AADObjectId).Definition + # Check that we assign to objectId from .Id property + $functionContent | Should -Match '\$objectId\s*=\s*\$\w+\.Id' + } + + It 'Should reference Connect-MgGraph in help/comments' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Connect-MgGraph" + } + } + + Context 'Get-DataLakeFolderACL Function Migration' { + It 'Should use Get-MgDirectoryObject instead of Get-AzureADObjectByObjectId' { + $functionContent = (Get-Command Get-DataLakeFolderACL).Definition + $functionContent | Should -Match "Get-MgDirectoryObject" + $functionContent | Should -Not -Match "Get-AzureADObjectByObjectId" + } + + It 'Should require Microsoft.Graph modules' { + $functionContent = (Get-Command Get-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'ACL Functions Dependencies' { + It 'Set-DataLakeFolderACL should require Microsoft.Graph modules' { + $functionContent = (Get-Command Set-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + + It 'Remove-DataLakeFolderACL should require Microsoft.Graph modules' { + $functionContent = (Get-Command Remove-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'Documentation Updates' { + It 'README should mention Microsoft.Graph modules' { + $readmeContent = Get-Content (Join-Path $PSScriptRoot '..' 'README.md') -Raw + $readmeContent | Should -Match "Microsoft\.Graph" + } + + It 'README should mention Connect-MgGraph' { + $readmeContent = Get-Content (Join-Path $PSScriptRoot '..' 'README.md') -Raw + $readmeContent | Should -Match "Connect-MgGraph" + } + + It 'example.ps1 should use Connect-MgGraph' { + $exampleContent = Get-Content (Join-Path $PSScriptRoot '..' 'example.ps1') -Raw + $exampleContent | Should -Match "Connect-MgGraph" + # The comment is OK, but we should not have an actual Connect-AzureAD command + $exampleContent | Should -Not -Match "^Connect-AzureAD" -Because "Connect-AzureAD should be commented out or replaced" + } + + It 'Module version should be updated to 2025.11.2 or higher' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.Version.Major | Should -BeGreaterOrEqual 2025 + $manifest.Version.Minor | Should -BeGreaterOrEqual 11 + $manifest.Version.Build | Should -BeGreaterOrEqual 2 + } + } + + Context 'Backward Compatibility' { + It 'Get-AADObjectId should still return ObjectId property' { + # This ensures backward compatibility - the property name should remain ObjectId + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "ObjectId\s*=" + } + + It 'Get-AADObjectId should still return ObjectType property' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "ObjectType\s*=" + } + + It 'Get-AADObjectId should still return DisplayName property' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "DisplayName\s*=" + } + } +} + +AfterAll { + # Clean up + Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue +} From 15eaf159801a04458eb6dd25eba59bd5b376e79c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:12:41 +0000 Subject: [PATCH 14/19] Address code review feedback: Improve property access and error handling - Improved Microsoft Graph object property access in Get-DataLakeFolderACL - Added robust property checking for DisplayName and ObjectType - Enhanced error detection for authentication issues to be more specific - Check for AuthenticationException and 401/Unauthorized errors - All tests still passing (41/41) Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- .../AzureDataLakeManagement.psm1 | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index d21029c..afb45cc 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -300,7 +300,10 @@ function Get-AADObjectId catch { # Check if the error is due to missing authentication - if ($_.Exception.Message -match 'Authentication needed|not authenticated|Connect-MgGraph') + # Microsoft Graph throws specific error codes for authentication issues + if ($_.Exception.GetType().Name -match 'AuthenticationException|UnauthorizedAccessException' -or + $_.Exception.Message -match '401|Unauthorized|authentication.*required' -or + $_.FullyQualifiedErrorId -match 'Authentication') { Write-Error 'You must be authenticated to Microsoft Graph to run this command. Run Connect-MgGraph to authenticate.' return @@ -999,11 +1002,29 @@ function Get-DataLakeFolderACL # Get the AD object for the entity using Microsoft Graph $adObject = Get-MgDirectoryObject -DirectoryObjectId $ace.EntityId -ErrorAction SilentlyContinue + # Extract display name and object type from the directory object + $displayName = $null + $objectType = $null + + if ($adObject) { + # Try to get DisplayName from the object properties first, then from AdditionalProperties + if ($adObject.PSObject.Properties.Name -contains 'DisplayName') { + $displayName = $adObject.DisplayName + } elseif ($adObject.AdditionalProperties.ContainsKey('displayName')) { + $displayName = $adObject.AdditionalProperties['displayName'] + } + + # Extract object type from odata.type + if ($adObject.AdditionalProperties.ContainsKey('@odata.type')) { + $objectType = $adObject.AdditionalProperties['@odata.type'] -replace '#microsoft.graph.', '' + } + } + # Create a custom object with the ACL info [PSCustomObject]@{ - DisplayName = if ($adObject) { $adObject.AdditionalProperties.displayName } else { $null } + DisplayName = $displayName ObjectId = $ace.EntityId - ObjectType = if ($adObject) { $adObject.AdditionalProperties.'@odata.type' -replace '#microsoft.graph.', '' } else { $null } + ObjectType = $objectType Permissions = $ace.Permissions DefaultScope = $ace.DefaultScope } From 71ac50326de354f1653202f33a5933eddff072c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:14:08 +0000 Subject: [PATCH 15/19] Add comprehensive migration guide for AzureAD to Microsoft.Graph - Created MIGRATION.md with detailed upgrade instructions - Includes breaking changes, step-by-step migration steps - Provides troubleshooting guide and common use cases - Documents authentication changes and scope requirements - Includes rollback instructions for emergency situations Co-authored-by: SteveCInVA <37545884+SteveCInVA@users.noreply.github.com> --- MIGRATION.md | 209 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 MIGRATION.md diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..7e2451e --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,209 @@ +# Migration Guide: AzureAD to Microsoft.Graph + +## Overview +Version 2025.11.2 of the AzureDataLakeManagement module migrates from the deprecated AzureAD PowerShell module to the Microsoft.Graph PowerShell SDK. This change is necessary for PowerShell 7+ compatibility and aligns with Microsoft's recommendations. + +## Why This Change? +- **AzureAD module is deprecated**: Microsoft announced the deprecation of the AzureAD and MSOnline modules with retirement scheduled for late 2025 +- **PowerShell 7+ incompatibility**: The AzureAD module only works with Windows PowerShell 5.1 and is not compatible with PowerShell Core (7+) +- **Modern authentication**: Microsoft Graph SDK uses MSAL (Microsoft Authentication Library) with better security and support for modern authentication methods +- **Future-proof**: Microsoft Graph is the unified endpoint for all Microsoft 365 services with regular updates + +## Breaking Changes + +### Module Dependencies +**Before (v2025.11.1 and earlier):** +```powershell +# Required modules +- Az.Storage +- AzureAD +- Az.Accounts +``` + +**After (v2025.11.2 and later):** +```powershell +# Required modules +- Az.Storage +- Microsoft.Graph.Users +- Microsoft.Graph.Groups +- Az.Accounts +``` + +### Authentication +**Before:** +```powershell +Connect-AzAccount +Connect-AzureAD +``` + +**After:** +```powershell +Connect-AzAccount +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +## Migration Steps + +### Step 1: Install New Dependencies +```powershell +# Uninstall old AzureAD module (optional) +Uninstall-Module -Name AzureAD + +# Install Microsoft Graph modules +Install-Module -Name Microsoft.Graph.Users -Force +Install-Module -Name Microsoft.Graph.Groups -Force + +# Or use the module's built-in dependency management +Import-Module AzureDataLakeManagement +Test-ModuleDependencies -AutoInstall +``` + +### Step 2: Update Authentication in Scripts +Replace all instances of `Connect-AzureAD` with: +```powershell +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +**Note on Scopes:** +- `User.Read.All`: Required for reading user information +- `Group.Read.All`: Required for reading group information +- `Application.Read.All`: Required for reading service principal information + +For production/automation scenarios, consider using certificate-based authentication: +```powershell +Connect-MgGraph -ClientId "YOUR_CLIENT_ID" -TenantId "YOUR_TENANT_ID" -CertificateThumbprint "YOUR_CERT_THUMBPRINT" +``` + +### Step 3: Update Module Version +```powershell +# Update to the latest version +Update-Module -Name AzureDataLakeManagement + +# Verify version +Get-Module -Name AzureDataLakeManagement -ListAvailable | Select-Object Name, Version +``` + +## Function Changes + +### No Changes to Function Signatures +All public functions maintain the same parameters and behavior: +- `Get-AADObjectId` +- `Get-AzureSubscriptionInfo` +- `Add-DataLakeFolder` +- `Remove-DataLakeFolder` +- `Set-DataLakeFolderACL` +- `Get-DataLakeFolderACL` +- `Move-DataLakeFolder` +- `Remove-DataLakeFolderACL` + +### Internal Changes +The following cmdlet replacements were made internally: +- `Get-AzureADUser` → `Get-MgUser` +- `Get-AzureADGroup` → `Get-MgGroup` +- `Get-AzureADServicePrincipal` → `Get-MgServicePrincipal` +- `Get-AzureADObjectByObjectId` → `Get-MgDirectoryObject` + +## Troubleshooting + +### Error: "Module not found" +```powershell +# Solution: Install the required modules +Test-ModuleDependencies -AutoInstall +``` + +### Error: "Authentication needed" +```powershell +# Solution: Connect to Microsoft Graph with appropriate scopes +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +### Error: "Insufficient privileges" +```powershell +# Solution: Ensure your account has the required permissions +# Or use delegated permissions with admin consent +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +### Compatibility Issues +If you experience issues: +1. Verify PowerShell version: `$PSVersionTable.PSVersion` +2. Check installed modules: `Get-Module -ListAvailable | Where-Object Name -match "Graph|Az"` +3. Update all Az modules: `Update-Module -Name Az.*` +4. Restart PowerShell session + +## Testing Your Migration + +### Test Basic Functionality +```powershell +# Import module +Import-Module AzureDataLakeManagement + +# Check dependencies +Test-ModuleDependencies + +# Authenticate +Connect-AzAccount +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +# Test object lookup +Get-AADObjectId -Identity "user@yourdomain.com" +``` + +### Verify ACL Operations +```powershell +# Test ACL retrieval (replace with your values) +Get-DataLakeFolderACL -SubscriptionName "YourSubscription" ` + -ResourceGroupName "YourResourceGroup" ` + -StorageAccountName "yourstorageaccount" ` + -ContainerName "yourcontainer" ` + -FolderPath "yourfolder" +``` + +## Support and Resources + +### Documentation +- [Microsoft Graph PowerShell SDK Documentation](https://learn.microsoft.com/en-us/powershell/microsoftgraph/) +- [Upgrade from Azure AD PowerShell to Microsoft Graph PowerShell](https://learn.microsoft.com/en-us/powershell/microsoftgraph/migration-steps) + +### Common Use Cases + +#### Finding a User +```powershell +# Old way (AzureAD) +Get-AzureADUser -Filter "UserPrincipalName eq 'user@domain.com'" + +# New way (Microsoft Graph) +Get-MgUser -Filter "UserPrincipalName eq 'user@domain.com'" + +# Using the module (unchanged) +Get-AADObjectId -Identity "user@domain.com" +``` + +#### Finding a Group +```powershell +# Old way (AzureAD) +Get-AzureADGroup -Filter "DisplayName eq 'GroupName'" + +# New way (Microsoft Graph) +Get-MgGroup -Filter "DisplayName eq 'GroupName'" + +# Using the module (unchanged) +Get-AADObjectId -Identity "GroupName" +``` + +## Feedback and Issues +If you encounter any issues with the migration, please: +1. Review this migration guide +2. Check the [GitHub Issues](https://github.com/SteveCInVA/AzureDataLakeManagement/issues) +3. Create a new issue with details about your environment and the error + +## Rollback (Not Recommended) +If you need to temporarily roll back to the old version: +```powershell +# Install specific old version +Install-Module -Name AzureDataLakeManagement -RequiredVersion 2025.11.1 -Force + +# Note: This is only a temporary solution as AzureAD module will be retired +``` + +**Important**: The rollback is only temporary. You should plan to migrate to Microsoft.Graph as the AzureAD module will stop working when Microsoft completes the retirement. From 89319c244084a42d4ba69c24ef49b325c57ccd1d Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Tue, 4 Nov 2025 20:54:07 +0000 Subject: [PATCH 16/19] Update module dependencies to replace AzureAD with Microsoft.Graph modules for improved compatibility --- .../AzureDataLakeManagement.psd1 | 2 +- .../AzureDataLakeManagement.psm1 | 19 +++++++++---------- MIGRATION.md | 7 ++++++- README.md | 11 +++++++---- Tests/DependencyManagement.Tests.ps1 | 11 +++++++---- example-dependency-management.ps1 | 6 ++++-- example.ps1 | 10 ++++++++-- 7 files changed, 42 insertions(+), 24 deletions(-) diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index 5e6b9bb..5ae6d99 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -120,7 +120,7 @@ PrivateData = @{ # RequireLicenseAcceptance = $false # External dependent modules of this module - ExternalModuleDependencies = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts') + ExternalModuleDependencies = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') } # End of PSData hashtable diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index afb45cc..505044c 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -23,7 +23,7 @@ Checks for required modules and automatically installs any that are missing. .NOTES - Required modules: Az.Storage, Microsoft.Graph.Users, Microsoft.Graph.Groups, Az.Accounts + Required modules: Az.Storage, Microsoft.Graph.Users, Microsoft.Graph.Groups, Microsoft.Graph.DirectoryObjects, Microsoft.Graph.Applications Author: Stephen Carroll - Microsoft Date: 2025-01-09 #> @@ -33,8 +33,8 @@ function Test-ModuleDependencies { [switch]$AutoInstall, [switch]$Quiet ) - - $requiredModules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts') + + $requiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') $missingModules = @() $availableModules = @() @@ -110,7 +110,7 @@ function Test-ModuleDependencies { function Install-ModuleDependencies { [CmdletBinding()] param( - [string[]]$Modules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts'), + [string[]]$Modules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), [switch]$Quiet ) @@ -172,7 +172,7 @@ function Install-ModuleDependencies { function Import-ModuleDependencies { [CmdletBinding()] param( - [string[]]$RequiredModules = @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Az.Accounts'), + [string[]]$RequiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), [switch]$Quiet ) @@ -721,7 +721,7 @@ function Set-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } @@ -973,7 +973,7 @@ function Get-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } @@ -1008,12 +1008,11 @@ function Get-DataLakeFolderACL if ($adObject) { # Try to get DisplayName from the object properties first, then from AdditionalProperties - if ($adObject.PSObject.Properties.Name -contains 'DisplayName') { + if ($adObject.PSObject.Properties.Name -contains 'displayName') { $displayName = $adObject.DisplayName } elseif ($adObject.AdditionalProperties.ContainsKey('displayName')) { $displayName = $adObject.AdditionalProperties['displayName'] } - # Extract object type from odata.type if ($adObject.AdditionalProperties.ContainsKey('@odata.type')) { $objectType = $adObject.AdditionalProperties['@odata.type'] -replace '#microsoft.graph.', '' @@ -1208,7 +1207,7 @@ function Remove-DataLakeFolderACL ) # Check if required modules are available and import them - if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups') -Quiet)) { + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' return } diff --git a/MIGRATION.md b/MIGRATION.md index 7e2451e..dd19445 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -18,15 +18,18 @@ Version 2025.11.2 of the AzureDataLakeManagement module migrates from the deprec - Az.Storage - AzureAD - Az.Accounts + ``` **After (v2025.11.2 and later):** ```powershell # Required modules - Az.Storage +- Microsoft.Graph.Applications - Microsoft.Graph.Users - Microsoft.Graph.Groups -- Az.Accounts +- Microsoft.Graph.DirectoryObjects + ``` ### Authentication @@ -50,8 +53,10 @@ Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All Uninstall-Module -Name AzureAD # Install Microsoft Graph modules +Install-Module -Name Microsoft.Graph.Applications -Force Install-Module -Name Microsoft.Graph.Users -Force Install-Module -Name Microsoft.Graph.Groups -Force +Install-Module -Name Microsoft.Graph.DirectoryObjects -Force # Or use the module's built-in dependency management Import-Module AzureDataLakeManagement diff --git a/README.md b/README.md index 47ee459..22309a1 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,11 @@ Starting with version 2025.1.1, the module includes improved dependency manageme ### Required Dependencies The module requires the following PowerShell modules: - `Az.Storage` - For Azure Storage operations +- `Microsoft.Graph.Applications` - For Microsoft Graph application operations (replaces AzureAD) - `Microsoft.Graph.Users` - For Microsoft Graph user operations (replaces AzureAD) - `Microsoft.Graph.Groups` - For Microsoft Graph group operations (replaces AzureAD) -- `Az.Accounts` - For Azure authentication +- `Microsoft.Graph.DirectoryObjects` - For Microsoft Graph directory objects (replaces AzureAD) + **Important**: The legacy `AzureAD` module is no longer supported as it is incompatible with PowerShell 7+ and has been deprecated by Microsoft. This module now uses the Microsoft Graph PowerShell SDK. @@ -122,16 +124,17 @@ Install-ModuleDependencies -Modules @('Az.Storage') You can also install dependencies manually: ```powershell Install-Module -Name Az.Storage -Force +Install-Module -Name Microsoft.Graph.Applications -Force Install-Module -Name Microsoft.Graph.Users -Force Install-Module -Name Microsoft.Graph.Groups -Force -Install-Module -Name Az.Accounts -Force +Install-Module -Name Microsoft.Graph.DirectoryObjects -Force ``` ### Authentication Connect to both Azure and Microsoft Graph: ```powershell # Connect to Azure -Connect-AzAccount +Connect-AzAccount -UseDeviceAuthentication # Connect to Microsoft Graph (replaces Connect-AzureAD) Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" @@ -146,7 +149,7 @@ Functions now provide clearer error messages when dependencies are missing, guid - 2025.11.2 - 01/04/2025 **BREAKING CHANGE**: Migrated from deprecated AzureAD module to Microsoft.Graph PowerShell SDK for PowerShell 7+ compatibility. - - Replaced `AzureAD` dependency with `Microsoft.Graph.Users` and `Microsoft.Graph.Groups` + - Replaced `AzureAD` dependency with `Microsoft.Graph.Applications`, `Microsoft.Graph.Users`, `Microsoft.Graph.Groups` and `Microsoft.Graph.DirectoryObjects` - Updated `Get-AADObjectId` to use Microsoft Graph cmdlets (`Get-MgUser`, `Get-MgGroup`, `Get-MgServicePrincipal`) - Updated `Get-DataLakeFolderACL` to use `Get-MgDirectoryObject` - Updated authentication from `Connect-AzureAD` to `Connect-MgGraph` diff --git a/Tests/DependencyManagement.Tests.ps1 b/Tests/DependencyManagement.Tests.ps1 index 772d850..331bf0a 100644 --- a/Tests/DependencyManagement.Tests.ps1 +++ b/Tests/DependencyManagement.Tests.ps1 @@ -21,9 +21,11 @@ Describe 'AzureDataLakeManagement Dependency Management' { It 'Should return boolean value' { Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Storage' } - Mock Get-Module { $null } -ParameterFilter { $Name -eq 'AzureAD' } - Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Accounts' } - + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Applications' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Users' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Groups' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.DirectoryObjects' } + $result = Test-ModuleDependencies -Quiet $result | Should -BeOfType [System.Boolean] } @@ -63,9 +65,10 @@ Describe 'AzureDataLakeManagement Dependency Management' { It 'Should declare external module dependencies in manifest' { $manifest = Test-ModuleManifest -Path $ModulePath $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Applications' $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Users' $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Groups' - $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Accounts' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.DirectoryObjects' } It 'Should export dependency management functions' { diff --git a/example-dependency-management.ps1 b/example-dependency-management.ps1 index 164641a..8b6bbb0 100644 --- a/example-dependency-management.ps1 +++ b/example-dependency-management.ps1 @@ -11,8 +11,10 @@ Test-ModuleDependencies # Or install them manually: # Install-Module -Name Az.Storage -Force -# Install-Module -Name AzureAD -Force -# Install-Module -Name Az.Accounts -Force +# Install-Module -Name Microsoft.Graph.Applications -Force +# Install-Module -Name Microsoft.Graph.Users -Force +# Install-Module -Name Microsoft.Graph.Groups -Force +# Install-Module -Name Microsoft.Graph.DirectoryObjects -Force # The module will now provide better error messages when functions are called without required dependencies # For example: diff --git a/example.ps1 b/example.ps1 index 5696532..13ed58f 100644 --- a/example.ps1 +++ b/example.ps1 @@ -36,8 +36,14 @@ set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -Sto set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB\test1' -Identity '' -accessControlType Write -IncludeDefaultScope #set group acl at root of dataset -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity "" -accessControlType Read -IncludeDefaultScope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity "" -accessControlType Read -IncludeDefaultScope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity '' -accessControlType Read -IncludeDefaultScope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity '' -accessControlType Read -IncludeDefaultScope + +#set acl with no recursion +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB' -Identity '' -accessControlType Write -IncludeDefaultScope -DoNotApplyACLRecursively + +#remove ACL from folder +remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' #remove folder from specified container remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' From 9338b6298768ba147f5e9cd750ae7712787d0ea9 Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Tue, 4 Nov 2025 21:04:35 +0000 Subject: [PATCH 17/19] Fix PowerShell default version setting in devcontainer configuration --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 44d2603..5ac527e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,7 @@ // Set *default* container specific settings.json values on container create "settings": { "terminal.integrated.defaultProfile.linux": "pwsh", - "powershell.powerShellDefaultVersion": "PowerShell (latest)", + "powershell.powerShellDefaultVersion": "PowerShell", "githubPullRequests.remotes": [ "https://github.com/SteveCInVA/AzureDataLakeManagement.git" ] From 693be52b05528e3a59706cebc16d7313c102c00b Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Tue, 4 Nov 2025 22:14:04 +0000 Subject: [PATCH 18/19] Fixed spelling error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 22309a1..c70d64f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AzureDataLakeManagement This project was created to help simplify the process of managing an Azure Datalake specifically around updating existing ACL's to child objects within the lake. -Yes, this can be accomplished with Azure Storage Explorer, however come customers don't like to install new software. +Yes, this can be accomplished with Azure Storage Explorer, however some customers don't like to install new software. My goal, is to make a straight forward set of functions that will assist a user in configuring folders and the associated ACL's in an ADLS Gen 2 storage container using the objects names rather than ID's. From ff08462cae3518e815e69e27872c69083d8523c0 Mon Sep 17 00:00:00 2001 From: SteveCInVA Date: Fri, 7 Nov 2025 21:41:56 +0000 Subject: [PATCH 19/19] Add support for -whatif and -confirm + remove warnings. --- .github/copilot-instructions.md | 52 +- .../AzureDataLakeManagement.psd1 | 2 +- .../AzureDataLakeManagement.psm1 | 533 ++++++++---------- README.md | 5 +- 4 files changed, 278 insertions(+), 314 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ac64c75..f11d92a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -11,13 +11,16 @@ This repository contains a PowerShell module for managing Azure Data Lake Storag - Install required Azure PowerShell modules: ```powershell Install-Module -Name Az.Storage -Scope CurrentUser -Force - Install-Module -Name AzureAD -Scope CurrentUser -Force - Install-Module -Name Az.Accounts -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Applications -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Users -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Groups -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.DirectoryObjects -Scope CurrentUser -Force + ``` - Authenticate to Azure before testing: ```powershell Connect-AzAccount - Connect-AzureAD + Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" ``` ### Code Quality and Validation @@ -29,16 +32,16 @@ This repository contains a PowerShell module for managing Azure Data Lake Storag ```powershell Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 ``` - **Note**: Will show warnings about missing Az.Storage, AzureAD, and Az.Accounts modules if they're not installed. This is expected in offline environments. + **Note**: Will show warnings about missing Az.Storage, Microsoft.Graph.Applications, Microsoft.Graph.Users, Microsoft.Graph.Groups, and Microsoft.Graph.DirectoryObjects modules if they're not installed. This is expected in offline environments. - ALWAYS run PSScriptAnalyzer before committing changes or the code quality will deteriorate. ### Offline Development and Testing When Azure modules or connectivity is not available: - Module import will work but functions will fail at runtime - PSScriptAnalyzer and manifest testing work completely offline -- Function syntax and help documentation can be validated offline -- Use these commands for offline validation: - ```powershell +- Function syntax and help documentation can be validated offline +- Use these commands for offline validation: + ```powershell # These work without Azure connectivity Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' Get-Command -Module AzureDataLakeManagement @@ -135,7 +138,7 @@ Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 1. **Authentication and Module Import Test** (1-2 minutes): ```powershell Connect-AzAccount - Connect-AzureAD + Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' Get-Command -Module AzureDataLakeManagement # Should show all 8 functions @@ -144,10 +147,10 @@ Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 2. **Basic Folder Operations Test** (5-10 minutes): ```powershell # Use test subscription and storage account - $subName = 'your-test-subscription' - $rgName = 'test-resource-group' - $storageAccountName = 'teststorageaccount' - $containerName = 'test-container' + $subName = '' + $rgName = 'resourceGroup01' + $storageAccountName = 'storage01' + $containerName = 'bronze' # Create test folder structure Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset\sample-folder' @@ -167,13 +170,13 @@ Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' # Apply test ACL (use test user/group) - Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' -accessControlType Read + Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' -accessControlType Read # Verify ACL was applied Get-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' # Test ACL removal - Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'test-user@domain.com' + Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' # Clean up Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' @@ -182,13 +185,13 @@ Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 4. **Azure AD Object Resolution Test** (2-3 minutes): ```powershell # Test user lookup - Get-AADObjectId -Identity 'test-user@domain.com' + Get-AADObjectId -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' # Test group lookup - Get-AADObjectId -Identity 'Test Group Name' + Get-AADObjectId -Identity 'allcompany' # Test service principal lookup - Get-AADObjectId -Identity 'Test Service Principal' + Get-AADObjectId -Identity 'CompliancePolicy' # Should return ObjectId, ObjectType, and DisplayName for each ``` @@ -224,10 +227,10 @@ The `example.ps1` file demonstrates a complete workflow: **CRITICAL**: Always modify the variables in example.ps1 before running: ```powershell -$subName = 'your-test-subscription' # Change this -$rgName = 'your-test-resource-group' # Change this -$storageAccountName = 'your-test-storage' # Change this -$containerName = 'test-container' # Change this +$subName = '' +$rgName = 'resourceGroup01' +$storageAccountName = 'storage01' +$containerName = 'bronze' ``` ## Common Development Tasks @@ -264,10 +267,9 @@ $containerName = 'test-container' # Change this ### Known Code Quality Issues PSScriptAnalyzer currently identifies 17 warnings that should be addressed in new code: -- 5 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) +- 8 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) - 6 unused parameter warnings (remove unused parameters) -- 3 unused variable warnings (remove unused variables) -- 3 missing `ShouldProcess` support warnings for state-changing functions (Add-DataLakeFolder, Set-DataLakeFolderACL, Remove-DataLakeFolderACL) +- 3 instances of Use Singular nouns in function names (rename functions to use singular nouns) Run `Invoke-ScriptAnalyzer` to see the complete list with line numbers and detailed guidance. @@ -307,7 +309,7 @@ Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 ### Azure Authentication ```powershell Connect-AzAccount -Connect-AzureAD +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" Get-AzSubscription # Verify connection ``` diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index 5ae6d99..9df6389 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -12,7 +12,7 @@ RootModule = 'AzureDataLakeManagement.psm1' # Version number of this module. -ModuleVersion = '2025.11.2' +ModuleVersion = '2025.11.3' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index 505044c..3b0d5f9 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -1,4 +1,4 @@ -#region Dependency Management Functions +#region Dependency Management Functions <# .SYNOPSIS @@ -29,6 +29,7 @@ #> function Test-ModuleDependencies { [CmdletBinding()] + [OutputType([System.Boolean])] param( [switch]$AutoInstall, [switch]$Quiet @@ -37,11 +38,11 @@ function Test-ModuleDependencies { $requiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') $missingModules = @() $availableModules = @() - + if (-not $Quiet) { Write-Host "Checking AzureDataLakeManagement module dependencies..." -ForegroundColor Yellow } - + foreach ($moduleName in $requiredModules) { $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue if ($null -eq $module) { @@ -49,27 +50,29 @@ function Test-ModuleDependencies { if (-not $Quiet) { Write-Warning "Missing required module: $moduleName" } - } else { + } + else { $availableModules += $moduleName if (-not $Quiet) { Write-Host "✓ Found module: $moduleName (Version: $($module[0].Version))" -ForegroundColor Green } } } - + if ($missingModules.Count -eq 0) { if (-not $Quiet) { Write-Host "✓ All required modules are available." -ForegroundColor Green } return $true } - + if ($AutoInstall) { if (-not $Quiet) { Write-Host "Installing missing modules..." -ForegroundColor Yellow } return Install-ModuleDependencies -Modules $missingModules -Quiet:$Quiet - } else { + } + else { if (-not $Quiet) { Write-Host "`nTo install missing modules, run:" -ForegroundColor Cyan Write-Host "Test-ModuleDependencies -AutoInstall" -ForegroundColor White @@ -113,18 +116,18 @@ function Install-ModuleDependencies { [string[]]$Modules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), [switch]$Quiet ) - + $successCount = 0 $failureCount = 0 - + foreach ($moduleName in $Modules) { try { if (-not $Quiet) { Write-Host "Installing module: $moduleName..." -ForegroundColor Yellow } - + Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber -ErrorAction Stop - + if (-not $Quiet) { Write-Host "✓ Successfully installed: $moduleName" -ForegroundColor Green } @@ -135,15 +138,16 @@ function Install-ModuleDependencies { $failureCount++ } } - + if (-not $Quiet) { if ($failureCount -eq 0) { Write-Host "✓ All modules installed successfully." -ForegroundColor Green - } else { + } + else { Write-Warning "Installed $successCount modules, failed to install $failureCount modules." } } - + return ($failureCount -eq 0) } @@ -171,13 +175,14 @@ function Install-ModuleDependencies { #> function Import-ModuleDependencies { [CmdletBinding()] + [OutputType([System.Boolean])] param( [string[]]$RequiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), [switch]$Quiet ) - + $importFailures = @() - + foreach ($moduleName in $RequiredModules) { try { $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue @@ -186,7 +191,7 @@ function Import-ModuleDependencies { Write-Error "Module $moduleName is not available. Please install it first." continue } - + Import-Module -Name $moduleName -ErrorAction Stop -Force if (-not $Quiet) { Write-Verbose "Successfully imported module: $moduleName" @@ -197,12 +202,12 @@ function Import-ModuleDependencies { Write-Error "Failed to import module $moduleName`: $($_.Exception.Message)" } } - + if ($importFailures.Count -gt 0) { Write-Error "Failed to import modules: $($importFailures -join ', '). Some functions may not work correctly." return $false } - + return $true } @@ -241,8 +246,7 @@ function Import-ModuleDependencies { Date: 2021-08-31 Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> -function Get-AADObjectId -{ +function Get-AADObjectId { param ( # The identity for which the Azure AD Object ID is to be fetched [Parameter(Mandatory = $true)] @@ -253,8 +257,7 @@ function Get-AADObjectId # Replacing single quotes in the identity with double single quotes for filter syntax $Identity = $Identity.Replace("'", "''") - try - { + try { # Initializing user, group, and service principal to null $user = $null $group = $null @@ -263,48 +266,40 @@ function Get-AADObjectId # Try to get the user, group, and service principal # Using Microsoft Graph cmdlets instead of AzureAD $user = Get-MgUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $user) - { + if ($null -eq $user) { $group = Get-MgGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $group) - { + if ($null -eq $group) { $sp = Get-MgServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue } } # Check which object is not null and assign the corresponding values - if ($null -ne $user) - { + if ($null -ne $user) { $objectType = 'User' $objectId = $user.Id $displayName = $user.DisplayName } - elseif ($null -ne $group) - { + elseif ($null -ne $group) { $objectType = 'Group' $objectId = $group.Id $displayName = $group.DisplayName } - elseif ($null -ne $sp) - { + elseif ($null -ne $sp) { $objectType = 'ServicePrincipal' $objectId = $sp.Id $displayName = $sp.DisplayName } - else - { + else { Write-Error ('Object not found. Unable to find object "{0}" in Azure AD.' -f $Identity) return } } - catch - { + catch { # Check if the error is due to missing authentication # Microsoft Graph throws specific error codes for authentication issues if ($_.Exception.GetType().Name -match 'AuthenticationException|UnauthorizedAccessException' -or $_.Exception.Message -match '401|Unauthorized|authentication.*required' -or - $_.FullyQualifiedErrorId -match 'Authentication') - { + $_.FullyQualifiedErrorId -match 'Authentication') { Write-Error 'You must be authenticated to Microsoft Graph to run this command. Run Connect-MgGraph to authenticate.' return } @@ -350,8 +345,7 @@ function Get-AADObjectId Author: Stephen Carroll - Microsoft Date: 2021-08-31 #> -function Get-AzureSubscriptionInfo -{ +function Get-AzureSubscriptionInfo { param ( # The name of the Azure subscription [Parameter(Mandatory = $true)] @@ -359,26 +353,22 @@ function Get-AzureSubscriptionInfo [string]$SubscriptionName ) - try - { + try { # Get the subscription details $subscription = Get-AzSubscription -SubscriptionName $SubscriptionName # Check if the subscription exists - if ($null -eq $subscription) - { + if ($null -eq $subscription) { Write-Error('Subscription "{0}" not found.', $SubscriptionName) return } - else - { + else { # Write verbose messages for debugging Write-Verbose 'Function: Get-AzureSubscriptionInfo: Subscription found.' Write-Verbose "SubscriptionID: $subscription.id SubscriptionName: $subscription.Name" } } - catch - { + catch { # Handle exceptions and write an error message Write-Error 'Ensure you have run Connect-AzAccount and that the subscription exists.' return @@ -432,8 +422,8 @@ function Get-AzureSubscriptionInfo Author: Stephen Carroll - Microsoft Date: 2021-08-31 #> -function Add-DataLakeFolder -{ +function Add-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory = $true)] [string]$SubscriptionName, # Azure subscription name @@ -455,18 +445,22 @@ function Add-DataLakeFolder # Get the subscription ID $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { + if ($null -eq $subId) { Write-Error 'Subscription not found.' return } # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } } # Check if the Az.Storage module is available and import it @@ -477,45 +471,42 @@ function Add-DataLakeFolder # Get the Data Lake Storage account $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { + if ($null -eq $storageAccount) { Write-Error 'Storage account not found.' return } # Set the context to the Data Lake Storage account $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { + if ($null -eq $ctx) { Write-Error 'Failed to set the Data Lake Storage account context.' return } # Create the folder - try - { - $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop - } - catch - { - if ($ErrorIfFolderExists) - { - Write-Error "Folder $FolderPath already exists." + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", 'Create Data Lake Folder')) { + try { + $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop + } + catch { + if ($ErrorIfFolderExists) { + Write-Error "Folder $FolderPath already exists." + return + } + $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists return } - $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists - return - } - if ($null -eq $ret) - { - Write-Error 'Failed to create the folder.' - return - } - else - { - return $ret # Return the created folder + if ($null -eq $ret) { + Write-Error 'Failed to create the folder.' + return + } + else { + return $ret # Return the created folder + } } + + } <# @@ -553,8 +544,8 @@ function Add-DataLakeFolder Author: Stephen Carroll - Microsoft Date: 2021-08-31 #> -function Remove-DataLakeFolder -{ +function Remove-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory = $true)] [string]$SubscriptionName, # Azure subscription name @@ -576,45 +567,44 @@ function Remove-DataLakeFolder # Get the subscription ID $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { + if ($null -eq $subId) { Write-Error 'Subscription not found.' return } # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } } # Get the Data Lake Storage account $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { + if ($null -eq $storageAccount) { Write-Error 'Storage account not found.' return } # Set the context to the Data Lake Storage account $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { + if ($null -eq $ctx) { Write-Error 'Failed to set the Data Lake Storage account context.' return } # Ensure the folder exists before deleting - try - { + try { $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -ErrorAction Stop } - catch - { - if ($ErrorIfFolderDoesNotExist) - { + catch { + if ($ErrorIfFolderDoesNotExist) { Write-Error "Folder '$FolderPath' does not exist to delete." return } @@ -622,18 +612,17 @@ function Remove-DataLakeFolder } # Delete the folder - if ($null -ne $folderExists) - { - $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", 'Remove Data Lake Folder')) { + if ($null -ne $folderExists) { + $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force + } } - if ($null -ne $ret) - { + if ($null -ne $ret) { Write-Error 'Failed to delete the folder.' return } - else - { + else { Write-Host "Folder $ContainerName\$FolderPath deleted successfully." return } @@ -687,9 +676,8 @@ function Remove-DataLakeFolder Date: 2021-08-31 Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> -function Set-DataLakeFolderACL -{ - [CmdletBinding()] +function Set-DataLakeFolderACL { + [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory = $true)] [string]$SubscriptionName, @@ -727,194 +715,178 @@ function Set-DataLakeFolderACL } $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName - if ($null -eq $sub) - { + if ($null -eq $sub) { Write-Error 'Subscription not found. Ensure you have run Connect-AzAccount before execution.' return } - else - { + else { $subId = $sub.SubscriptionId } # Get the object ID of the identity to use in the ACL $identityObj = Get-AADObjectId -Identity $Identity - if ($null -eq $identityObj) - { + if ($null -eq $identityObj) { Write-Error 'Identity not found.' return } - else - { + else { Write-Verbose ('{0} ID: {1} Display Name: {2}' -f $identityObj.ObjectType, $identityObj.ObjectId, $identityObj.DisplayName) } # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - else - { - Write-Verbose $subContext.Name + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } } + # Get the Data Lake Storage account $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { + if ($null -eq $storageAccount) { Write-Error 'Storage account not found.' return } - else - { + else { Write-Verbose $storageAccount.StorageAccountName } # Set the context to the Data Lake Storage account $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { + if ($null -eq $ctx) { Write-Error 'Failed to set the Data Lake Storage account context.' return } # verify the folder exists before setting the ACL - try - { + try { $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - if ($null -eq $folderExists) - { + if ($null -eq $folderExists) { Write-Error('Folder not found.') return } } - catch - { + catch { Write-Error('Folder not found.') return } # translate the access control type to applied permission - $permission = switch ( $AccessControlType ) - { 'Read' - { 'r-x' + $permission = switch ( $AccessControlType ) { + 'Read' { + 'r-x' } - 'Write' - { 'rwx' + 'Write' { + 'rwx' } - default - { '' + default { + '' } } - $identityType = switch ($identityObj.ObjectType) - { 'User' - { 'user' + $identityType = switch ($identityObj.ObjectType) { + 'User' { + 'user' } - 'Group' - { 'group' + 'Group' { + 'group' } - 'ServicePrincipal' - { 'user' + 'ServicePrincipal' { + 'user' } - 'ManagedIdentity' - { 'other' + 'ManagedIdentity' { + 'other' } - default - { '' + default { + '' } } # set the ACL at the container level - if ($SetContainerACL) - { - Write-Verbose 'set container ACL' - $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the container.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'Container ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + if ($SetContainerACL) { + if ($PSCmdlet.ShouldProcess($ContainerName, "Set container ACL for identity '$($identityObj.DisplayName)' with Read access")) { + Write-Verbose 'set container ACL' + $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL + + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL for the container.' + Write-Error $result.FailedEntries + return + } + else { + Write-Host 'Container ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } } } # get the ACL for the folder $acl = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath).ACL - try - { - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - - } - catch [Microsoft.PowerShell.Commands.WriteErrorException] - { - Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' - return - } + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Set ACL for identity '$($identityObj.DisplayName)' with $AccessControlType access")) { + try { + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - ###################### - # default scope - if ($IncludeDefaultScope) - { - Write-Verbose 'include default scope' - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + catch [Microsoft.PowerShell.Commands.WriteErrorException] { + Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' + return } - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the default scope.' + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL.' Write-Error $result.FailedEntries return } - else - { - Write-Host 'Default Scope ACL set successfully.' + else { + Write-Host 'ACL set successfully.' Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) } } + ###################### + # default scope + if ($IncludeDefaultScope) { + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Set default scope ACL for identity '$($identityObj.DisplayName)' with $AccessControlType access")) { + Write-Verbose 'include default scope' + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL for the default scope.' + Write-Error $result.FailedEntries + return + } + else { + Write-Host 'Default Scope ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + } } @@ -952,8 +924,7 @@ function Set-DataLakeFolderACL Date: 2021-08-31 Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> -function Get-DataLakeFolderACL -{ +function Get-DataLakeFolderACL { [CmdletBinding()] param( [Parameter(Mandatory = $true)] @@ -979,38 +950,35 @@ function Get-DataLakeFolderACL } # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) { $FolderPath = $FolderPath.Substring(1) } - try - { + try { $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + # Verify the folder exists (will throw error if not found) + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the ACLs for the folder $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL # Process each ACL - $aclResults = foreach ($ace in $acls) - { - if ($ace.EntityId) - { + $aclResults = foreach ($ace in $acls) { + if ($ace.EntityId) { # Get the AD object for the entity using Microsoft Graph $adObject = Get-MgDirectoryObject -DirectoryObjectId $ace.EntityId -ErrorAction SilentlyContinue # Extract display name and object type from the directory object $displayName = $null $objectType = $null - + if ($adObject) { # Try to get DisplayName from the object properties first, then from AdditionalProperties if ($adObject.PSObject.Properties.Name -contains 'displayName') { $displayName = $adObject.DisplayName - } elseif ($adObject.AdditionalProperties.ContainsKey('displayName')) { + } + elseif ($adObject.AdditionalProperties.ContainsKey('displayName')) { $displayName = $adObject.AdditionalProperties['displayName'] } # Extract object type from odata.type @@ -1033,8 +1001,7 @@ function Get-DataLakeFolderACL # Return the results return $aclResults } - catch - { + catch { # Write any errors to the console Write-Error $_.Exception.Message } @@ -1079,9 +1046,8 @@ function Get-DataLakeFolderACL Date: 2021-08-31 Updated: 2025-01-09 - Removed unnecessary AzureAD dependency #> -function Move-DataLakeFolder -{ - [CmdletBinding()] +function Move-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory = $true)] [string]$SubscriptionName, # Azure subscription name @@ -1111,30 +1077,29 @@ function Move-DataLakeFolder return } - try - { + try { $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath # If destination container name is not provided, use source container name - if (-not $DestinationContainerName) - { + if (-not $DestinationContainerName) { $DestinationContainerName = $SourceContainerName } # Move the folder - $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force + if ($PSCmdlet.ShouldProcess("$SourceContainerName\$SourceFolderPath", "Move folder to $DestinationContainerName\$DestinationFolderPath")) { + $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force - # Write verbose output and return the result - Write-Verbose ('Function: Move-DataLakeFolder') - Write-Verbose "Folder moved: $DestinationFolderPath" - return $ret + # Write verbose output and return the result + Write-Verbose ('Function: Move-DataLakeFolder') + Write-Verbose "Folder moved: $DestinationFolderPath" + return $ret + } } - catch - { + catch { # Write any errors to the console Write-Error $_.Exception.Message } @@ -1179,9 +1144,8 @@ function Move-DataLakeFolder Date: 2021-08-31 Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility #> -function Remove-DataLakeFolderACL -{ - [CmdletBinding()] +function Remove-DataLakeFolderACL { + [CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory = $true)] [string]$SubscriptionName, # Azure subscription name @@ -1213,13 +1177,11 @@ function Remove-DataLakeFolderACL } # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) { $FolderPath = $FolderPath.Substring(1) } - try - { + try { $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName @@ -1228,7 +1190,7 @@ function Remove-DataLakeFolderACL $id = $identityObj.ObjectId # Get the folder - $folder = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the ACLs for the folder $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL @@ -1237,30 +1199,27 @@ function Remove-DataLakeFolderACL $newacl = $acls | Where-Object { -not ($_.AccessControlType -eq $identityObj.ObjectType -and $_.EntityId -eq $id) } # Update the ACL - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Remove ACL for identity '$($identityObj.DisplayName)'")) { + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } - # Check if the update was successful - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to update the ACL.' - Write-Error $result.FailedEntries - } - else - { - Write-Host 'ACL updated successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + # Check if the update was successful + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to update the ACL.' + Write-Error $result.FailedEntries + } + else { + Write-Host 'ACL updated successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } } } - catch - { + catch { # Write any errors to the console Write-Error $_.Exception.Message } diff --git a/README.md b/README.md index c70d64f..d85ec43 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,10 @@ Functions now provide clearer error messages when dependencies are missing, guid ## Version History: -- 2025.11.2 - 01/04/2025 +- 2025.11.3 - 11/7/2025 +Code updates to support "ShouldProcess" functionality on all functions that cause changes. Functions now support the -whatif and -confirm flags in areas that would change data. Eliminated other build warnings identified from Invoke-ScriptAnalyzer. + +- 2025.11.2 - 11/04/2025 **BREAKING CHANGE**: Migrated from deprecated AzureAD module to Microsoft.Graph PowerShell SDK for PowerShell 7+ compatibility. - Replaced `AzureAD` dependency with `Microsoft.Graph.Applications`, `Microsoft.Graph.Users`, `Microsoft.Graph.Groups` and `Microsoft.Graph.DirectoryObjects` - Updated `Get-AADObjectId` to use Microsoft Graph cmdlets (`Get-MgUser`, `Get-MgGroup`, `Get-MgServicePrincipal`)