diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..5ac527e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,46 @@ +{ + "name": "Azure Data Lake Management PowerShell", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + + // Features to install - PowerShell is a predefined feature + "features": { + "ghcr.io/devcontainers/features/powershell:1": { + "version": "latest" + } + }, + + // Configure tool-specific properties + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create + "settings": { + "terminal.integrated.defaultProfile.linux": "pwsh", + "powershell.powerShellDefaultVersion": "PowerShell", + "githubPullRequests.remotes": [ + "https://github.com/SteveCInVA/AzureDataLakeManagement.git" + ] + }, + + // Add the IDs of extensions you want installed when the container is created + "extensions": [ + "ms-vscode.powershell", + "pspester.pester-test", + "github.copilot", + "github.vscode-github-actions", + "jgclark.vscode-todo-highlight" + ] + } + }, + + // Use 'postCreateCommand' to run commands after the container is created + "postCreateCommand": "pwsh -Command 'Install-Module -Name PSScriptAnalyzer, Pester -Force -Scope CurrentUser -SkipPublisherCheck'" + + // Use 'forwardPorts' to make a list of ports inside the container available locally + // "forwardPorts": [], + + // Use 'postStartCommand' to run commands each time the container starts + // "postStartCommand": "" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root + // "remoteUser": "root" +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..f11d92a --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,324 @@ +# Azure Data Lake Management PowerShell Module + +Always follow these instructions first and only search or use bash commands when you encounter information that contradicts what is documented here or when these instructions are incomplete. + +This repository contains a PowerShell module for managing Azure Data Lake Storage Gen 2 folders and Access Control Lists (ACLs). The module simplifies ACL management by using object names rather than IDs and provides functions to create, delete, move folders and manage permissions recursively. + +## Working Effectively + +### Prerequisites and Environment Setup +- Install PowerShell 7+ (PowerShell Core): Download from https://github.com/PowerShell/PowerShell/releases +- Install required Azure PowerShell modules: + ```powershell + Install-Module -Name Az.Storage -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Applications -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Users -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.Groups -Scope CurrentUser -Force + Install-Module -Name Microsoft.Graph.DirectoryObjects -Scope CurrentUser -Force + + ``` +- Authenticate to Azure before testing: + ```powershell + Connect-AzAccount + Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + ``` + +### Code Quality and Validation +- Run PSScriptAnalyzer for code quality checks (takes ~3 seconds): + ```powershell + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + ``` +- Test module manifest (takes ~1 second): + ```powershell + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + **Note**: Will show warnings about missing Az.Storage, Microsoft.Graph.Applications, Microsoft.Graph.Users, Microsoft.Graph.Groups, and Microsoft.Graph.DirectoryObjects modules if they're not installed. This is expected in offline environments. +- ALWAYS run PSScriptAnalyzer before committing changes or the code quality will deteriorate. + +### Offline Development and Testing +When Azure modules or connectivity is not available: +- Module import will work but functions will fail at runtime +- PSScriptAnalyzer and manifest testing work completely offline +- Function syntax and help documentation can be validated offline +- Use these commands for offline validation: + ```powershell + # These work without Azure connectivity + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + Get-Help Add-DataLakeFolder -Examples + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + ``` + +### Module Development and Testing +- Import the module for testing (~1 second): + ```powershell + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + ``` +- Get available functions: + ```powershell + Get-Command -Module AzureDataLakeManagement + ``` +- Access function help and examples (~1 second per function): + ```powershell + Get-Help Add-DataLakeFolder -Examples + Get-Help Set-DataLakeFolderACL -Full + ``` +- **CRITICAL**: Test functions only with test/development Azure resources. Never test against production data. + +### Publishing Process +- **Prerequisites for Publishing**: + - PowerShell Gallery API Key (set as environment variable `PSGalleryKey`) + - Module version updated in `.psd1` file + - All PSScriptAnalyzer warnings addressed + - Manual validation completed + +- **Manual publish to PowerShell Gallery** (~30 seconds): + ```powershell + # Set your API key first + $env:PSGalleryKey = "your-api-key-here" + .\publish.ps1 + ``` + +- **GitHub Actions publish**: + - Workflow: `.github/workflows/manual_publish.yml` + - Requires `PSGalleryKey` secret configured in repository + - Manually triggered via GitHub Actions UI + - Uses `workflow_dispatch` trigger (not automatic) + +- **Pre-publish validation checklist**: + ```powershell + # 1. Code quality check + Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + + # 2. Module manifest validation + Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 + + # 3. Module import test + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + + # 4. Verify version number is updated in .psd1 + # 5. Complete manual validation scenarios with test Azure resources + ``` + +## Key Module Components + +### Primary Functions (8 total): +1. **Get-AADObjectId** - Retrieve Azure AD object details by name/UPN +2. **Get-AzureSubscriptionInfo** - Get subscription information +3. **Add-DataLakeFolder** - Create folder structures in Data Lake Storage +4. **Remove-DataLakeFolder** - Delete folders from Data Lake Storage +5. **Set-DataLakeFolderACL** - Apply ACL permissions to folders (recursively) +6. **Get-DataLakeFolderACL** - Retrieve current ACL permissions +7. **Move-DataLakeFolder** - Move/rename folders between containers +8. **Remove-DataLakeFolderACL** - Remove ACL permissions from folders + +### Core Files: +- `AzureDataLakeManagement/AzureDataLakeManagement.psm1` - Main module (1026 lines, 8 functions) +- `AzureDataLakeManagement/AzureDataLakeManagement.psd1` - Module manifest and metadata +- `example.ps1` - Complete usage examples showing folder creation and ACL management +- `publish.ps1` - PowerShell Gallery publishing script + +## Validation and Testing + +### Code Quality Validation +Run these before every commit: +```powershell +# Static analysis (3 seconds) - NEVER CANCEL +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Module manifest validation (1 second) +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Manual Validation Scenarios +**CRITICAL**: Always test against development/test Azure resources only. Complete these scenarios after making changes: + +1. **Authentication and Module Import Test** (1-2 minutes): + ```powershell + Connect-AzAccount + Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + Get-Command -Module AzureDataLakeManagement + # Should show all 8 functions + ``` + +2. **Basic Folder Operations Test** (5-10 minutes): + ```powershell + # Use test subscription and storage account + $subName = '' + $rgName = 'resourceGroup01' + $storageAccountName = 'storage01' + $containerName = 'bronze' + + # Create test folder structure + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset\sample-folder' + + # Verify folder exists in Azure Storage Explorer or portal + + # Test folder move operation + Move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'test-dataset\sample-folder' -DestinationContainerName $containerName -destinationFolderPath 'test-dataset\moved-folder' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'test-dataset' + ``` + +3. **ACL Management Test** (5-10 minutes): + ```powershell + # Create test folder + Add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + + # Apply test ACL (use test user/group) + Set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' -accessControlType Read + + # Verify ACL was applied + Get-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' + + # Test ACL removal + Remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'acl-test' -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' + + # Clean up + Remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'acl-test' + ``` + +4. **Azure AD Object Resolution Test** (2-3 minutes): + ```powershell + # Test user lookup + Get-AADObjectId -Identity 'stecarr@MngEnvMCAP254199.onmicrosoft.com' + + # Test group lookup + Get-AADObjectId -Identity 'allcompany' + + # Test service principal lookup + Get-AADObjectId -Identity 'CompliancePolicy' + + # Should return ObjectId, ObjectType, and DisplayName for each + ``` + +5. **Complete example.ps1 Workflow Test** (10-15 minutes): + ```powershell + # Modify variables in example.ps1 first, then run: + .\example.ps1 + + # Verify in Azure portal: + # - Multiple folder structures created + # - ACL permissions applied correctly + # - Test folders cleaned up properly + ``` + +### Development Workflow Timing +- PSScriptAnalyzer execution: ~3 seconds - NEVER CANCEL +- Module manifest testing: ~1 second +- Module import: ~1 second (without Azure dependencies) +- Module import with Azure modules: ~2-5 seconds (depends on Azure module loading) +- Single folder operation: ~3-10 seconds (depends on Azure latency) +- ACL operations: ~5-15 seconds (depends on Azure AD latency and hierarchy depth) +- Full validation scenario: ~10-20 minutes total +- Complete example.ps1 workflow: ~5-15 minutes (creates multiple folders and ACLs) + +### Using example.ps1 for Learning +The `example.ps1` file demonstrates a complete workflow: +1. Authentication to Azure and Azure AD +2. Creating hierarchical folder structures +3. Setting various ACL types (user, group, service principal) +4. Error handling scenarios +5. Cleanup operations + +**CRITICAL**: Always modify the variables in example.ps1 before running: +```powershell +$subName = '' +$rgName = 'resourceGroup01' +$storageAccountName = 'storage01' +$containerName = 'bronze' +``` + +## Common Development Tasks + +### Adding New Functions +1. Add function to `AzureDataLakeManagement.psm1` +2. Update `FunctionsToExport` in `AzureDataLakeManagement.psd1` +3. Add usage example to `example.ps1` +4. Run PSScriptAnalyzer validation +5. Test with manual validation scenarios + +### Debugging Issues +- Use VS Code with PowerShell extension for debugging +- Import module with `-Force` to reload changes +- Use `-Verbose` parameter on functions for detailed output +- Check Azure portal/Storage Explorer to verify actual changes + +### Common Error Scenarios +1. **Missing Azure Authentication**: Functions fail with authentication errors + - Solution: Run `Connect-AzAccount` and `Connect-AzureAD` + +2. **Module Dependencies Missing**: Import fails with module not found errors + - Solution: Install Az.Storage, AzureAD, and Az.Accounts modules + +3. **Path Format Issues**: Functions expect backslash separators in folderPath + - Correct: `'dataset1\folder1\subfolder'` + - Incorrect: `'dataset1/folder1/subfolder'` + +4. **Permissions Issues**: ACL operations fail due to insufficient permissions + - Ensure Azure AD permissions and Storage Account permissions are configured + +5. **Storage Account Access**: Operations fail if storage account keys are not accessible + - Module requires either storage account key access OR proper Azure AD permissions + +### Known Code Quality Issues +PSScriptAnalyzer currently identifies 17 warnings that should be addressed in new code: +- 8 instances of `Write-Host` usage (use `Write-Output`, `Write-Verbose`, or `Write-Information`) +- 6 unused parameter warnings (remove unused parameters) +- 3 instances of Use Singular nouns in function names (rename functions to use singular nouns) + +Run `Invoke-ScriptAnalyzer` to see the complete list with line numbers and detailed guidance. + +## Repository Structure Reference +``` +. +├── .github/ +│ └── workflows/ +│ └── manual_publish.yml # GitHub Actions publishing workflow +├── .vscode/ +│ ├── launch.json # VS Code debugging configuration +│ └── settings.json # VS Code settings +├── AzureDataLakeManagement/ +│ ├── AzureDataLakeManagement.psd1 # Module manifest +│ └── AzureDataLakeManagement.psm1 # Main module (8 functions) +├── .gitignore +├── LICENSE +├── README.md # Project overview and version history +├── example.ps1 # Complete usage examples +└── publish.ps1 # PowerShell Gallery publishing script +``` + +## Quick Reference Commands + +### Daily Development +```powershell +# Load and test module +Import-Module -Force './AzureDataLakeManagement/AzureDataLakeManagement.psm1' + +# Code quality check (run before commit) +Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 + +# Test manifest +Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1 +``` + +### Azure Authentication +```powershell +Connect-AzAccount +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +Get-AzSubscription # Verify connection +``` + +### Function Usage Pattern +```powershell +# All functions follow this parameter pattern: +-SubscriptionName # Azure subscription name +-ResourceGroupName # Resource group containing storage account +-StorageAccountName # Storage account name +-ContainerName # Container/filesystem name +-folderPath # Path within container (use backslash separators) +``` \ No newline at end of file diff --git a/.github/workflows/manual_publish.yml b/.github/workflows/manual_publish.yml index 7ad4c2f..b02a35d 100644 --- a/.github/workflows/manual_publish.yml +++ b/.github/workflows/manual_publish.yml @@ -1,18 +1,18 @@ -name: Publish -on: [workflow_dispatch] - -permissions: - contents: read - pull-requests: write - -jobs: - build: - name: Publish - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Publish - env: - PSGalleryKey: ${{ secrets.PSGalleryKey }} - run: .\publish.ps1 - shell: pwsh +name: Publish +on: [workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + build: + name: Publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Publish + env: + PSGalleryKey: ${{ secrets.PSGalleryKey }} + run: .\publish.ps1 + shell: pwsh diff --git a/.gitignore b/.gitignore index 3546e64..c0cd43d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -AzureDataLakeManagement.zip - +AzureDataLakeManagement.zip + diff --git a/.vscode/launch.json b/.vscode/launch.json index 2fdae60..64eb2e7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,14 +1,14 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "PowerShell: Module Interactive Session", - "type": "PowerShell", - "request": "launch", - "script": "Import-Module -Force '${workspaceFolder}/AzureDataLakeManagement/AzureDataLakeManagement.psm1'" - } - ] -} +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "PowerShell: Module Interactive Session", + "type": "PowerShell", + "request": "launch", + "script": "Import-Module -Force '${workspaceFolder}/AzureDataLakeManagement/AzureDataLakeManagement.psm1'" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index a42f3ed..028065f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ -{ - "githubPullRequests.remotes": [ - "https://github.com/SteveCInVA/AzureDataLakeManagement.git" - ] -} +{ + "githubPullRequests.remotes": [ + "https://github.com/SteveCInVA/AzureDataLakeManagement.git" + ], + "powershell.pester.codeLens": false +} diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 index f5a77fe..9df6389 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psd1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psd1 @@ -1,135 +1,136 @@ -# -# Module manifest for module 'AzureDataLakeManagement' -# -# Generated by: Steve Carroll (Microsoft) -# -# Generated on: 12/1/2023 -# - -@{ - -# Script module or binary module file associated with this manifest. -RootModule = 'AzureDataLakeManagement.psm1' - -# Version number of this module. -ModuleVersion = '2025.1.1' - -# Supported PSEditions -CompatiblePSEditions = @('Desktop', 'Core') - -# ID used to uniquely identify this module -GUID = 'b0b0b0b0-b0b0-b0b0-b0b0-b0b0b0b0b0b0' - -# Author of this module -Author = 'Steve Carroll' - -# Company or vendor of this module -CompanyName = 'Microsoft' - -# Copyright statement for this module -Copyright = '(c) 2023 Microsoft Corporation. All rights reserved.' - -# Description of the functionality provided by this module -Description = 'Azure Data Lake Management Module' - -# Minimum version of the Windows PowerShell engine required by this module -PowerShellVersion = '5.1' - -# Name of the Windows PowerShell host required by this module -# PowerShellHostName = '' - -# Minimum version of the Windows PowerShell host required by this module -# PowerShellHostVersion = '' - -# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# DotNetFrameworkVersion = '' - -# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# CLRVersion = '' - -# Processor architecture (None, X86, Amd64) required by this module -# ProcessorArchitecture = '' - -# Modules that must be imported into the global environment prior to importing this module -RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') - -# Assemblies that must be loaded prior to importing this module -# RequiredAssemblies = @() - -# Script files (.ps1) that are run in the caller's environment prior to importing this module. -# ScriptsToProcess = @() - -# Type files (.ps1xml) to be loaded when importing this module -# TypesToProcess = @() - -# Format files (.ps1xml) to be loaded when importing this module -# FormatsToProcess = @() - -# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess -# NestedModules = @() - -# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Get-AADObjectId', 'Get-AzureSubscriptionInfo', 'Add-DataLakeFolder', - 'Remove-DataLakeFolder', 'Set-DataLakeFolderACL', - 'Get-DataLakeFolderACL', 'Move-DataLakeFolder', - 'Remove-DataLakeFolderACL' - -# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. -CmdletsToExport = @() - -# Variables to export from this module -# VariablesToExport = @() - -# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. -AliasesToExport = @() - -# DSC resources to export from this module -# DscResourcesToExport = @() - -# List of all modules packaged with this module -# ModuleList = @() - -# List of all files packaged with this module -# FileList = @() - -# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. -PrivateData = @{ - - PSData = @{ - - # Tags applied to this module. These help with module discovery in online galleries. - Tags = 'Azure','DataLake','Security' - - # A URL to the license for this module. - # LicenseUri = '' - - # A URL to the main website for this project. - ProjectUri = 'https://github.com/SteveCInVA/AzureDataLakeManagement' - - # A URL to an icon representing this module. - # IconUri = '' - - # ReleaseNotes of this module - # ReleaseNotes = '' - - # Prerelease string of this module - # Prerelease = '' - - # Flag to indicate whether the module requires explicit user acceptance for install/update/save - # RequireLicenseAcceptance = $false - - # External dependent modules of this module - # ExternalModuleDependencies = @() - - } # End of PSData hashtable - - } # End of PrivateData hashtable - -# HelpInfo URI of this module -HelpInfoURI = 'https://github.com/SteveCInVA/AzureDataLakeManagement/blob/main/README.md' - -# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. -# DefaultCommandPrefix = '' - -} - +# +# Module manifest for module 'AzureDataLakeManagement' +# +# Generated by: Steve Carroll (Microsoft) +# +# Generated on: 12/1/2023 +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = 'AzureDataLakeManagement.psm1' + +# Version number of this module. +ModuleVersion = '2025.11.3' + +# Supported PSEditions +CompatiblePSEditions = @('Desktop', 'Core') + +# ID used to uniquely identify this module +GUID = 'b0b0b0b0-b0b0-b0b0-b0b0-b0b0b0b0b0b0' + +# Author of this module +Author = 'Steve Carroll' + +# Company or vendor of this module +CompanyName = 'Microsoft' + +# Copyright statement for this module +Copyright = '(c) 2023 Microsoft Corporation. All rights reserved.' + +# Description of the functionality provided by this module +Description = 'Azure Data Lake Management Module' + +# Minimum version of the Windows PowerShell engine required by this module +PowerShellVersion = '7.0' + +# Name of the Windows PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the Windows PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# CLRVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @('Az.Storage', 'AzureAD', 'Az.Accounts') + +# Assemblies that must be loaded prior to importing this module +# RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +# ScriptsToProcess = @() + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +# FormatsToProcess = @() + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +# NestedModules = @() + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +FunctionsToExport = 'Get-AADObjectId', 'Get-AzureSubscriptionInfo', 'Add-DataLakeFolder', + 'Remove-DataLakeFolder', 'Set-DataLakeFolderACL', + 'Get-DataLakeFolderACL', 'Move-DataLakeFolder', + 'Remove-DataLakeFolderACL', 'Test-ModuleDependencies', + 'Install-ModuleDependencies', 'Import-ModuleDependencies' + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = @() + +# Variables to export from this module +# VariablesToExport = @() + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = @() + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + Tags = 'Azure','DataLake','Security' + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + ProjectUri = 'https://github.com/SteveCInVA/AzureDataLakeManagement' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + # Prerelease string of this module + # Prerelease = '' + + # Flag to indicate whether the module requires explicit user acceptance for install/update/save + # RequireLicenseAcceptance = $false + + # External dependent modules of this module + ExternalModuleDependencies = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') + + } # End of PSData hashtable + + } # End of PrivateData hashtable + +# HelpInfo URI of this module +HelpInfoURI = 'https://github.com/SteveCInVA/AzureDataLakeManagement/blob/main/README.md' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} + diff --git a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 index 4af5223..3b0d5f9 100644 --- a/AzureDataLakeManagement/AzureDataLakeManagement.psm1 +++ b/AzureDataLakeManagement/AzureDataLakeManagement.psm1 @@ -1,1026 +1,1242 @@ -<# -.SYNOPSIS - This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. - -.DESCRIPTION - Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Azure AD. - -.PARAMETER Identity - The Identity parameter specifies the user principal name, group display name, or service principal display name of the object to retrieve. This parameter is mandatory. - -.EXAMPLE - PS C:\> Get-AADObjectId -Identity "johndoe@contoso.com" - ObjectId ObjectType DisplayName - -------- ---------- ----------- - 12345678-1234-1234-1234-1234567890ab User John Doe - - This example retrieves the object ID, object type, and display name for the Azure AD user with the user principal name "johndoe@contoso.com". - -.EXAMPLE - PS C:\> Get-AADObjectId -Identity "HR Group" - ObjectId ObjectType DisplayName - -------- ---------- ----------- - 87654321-4321-4321-4321-ba0987654321 Group HR Group - - This example retrieves the object ID, object type, and display name for the Azure AD group with the display name "HR Group". - -.NOTES - This function requires an active connection to Azure AD using Connect-AzureAD. If the specified identity does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-AADObjectId -{ - param ( - # The identity for which the Azure AD Object ID is to be fetched - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string]$Identity - ) - - # Replacing single quotes in the identity with double single quotes - $Identity = $Identity.Replace("'", "''") - - try - { - # Initializing user, group, and service principal to null - $user = $null - $group = $null - $sp = $null - - # Try to get the user, group, and service principal in one go - $user = Get-AzureADUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $user) - { - $group = Get-AzureADGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue - if ($null -eq $group) - { - $sp = Get-AzureADServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue - } - } - - # Check which object is not null and assign the corresponding values - if ($null -ne $user) - { - $objectType = 'User' - $objectId = $user.ObjectId - $displayName = $user.DisplayName - } - elseif ($null -ne $group) - { - $objectType = 'Group' - $objectId = $group.ObjectId - $displayName = $group.DisplayName - } - elseif ($null -ne $sp) - { - $objectType = 'ServicePrincipal' - $objectId = $sp.ObjectId - $displayName = $sp.DisplayName - } - else - { - Write-Error ('Object not found. Unable to find object "{0}" in Azure AD.' -f $Identity) - return - } - } - catch [Microsoft.Open.Azure.AD.CommonLibrary.AadNeedAuthenticationException] - { - Write-Error 'You must be authenticated to Azure AD to run this command. Run Connect-AzureAD to authenticate.' - return - } - catch - { - Write-Error $_.Exception.Message - return - } - - # Output the object details - Write-Verbose "Object ID: $objectId" - Write-Verbose "Object Type: $objectType" - Write-Verbose "Object Name: $displayName" - - # Create a custom object to return - $object = [PSCustomObject]@{ - ObjectId = $objectId - ObjectType = $objectType - DisplayName = $displayName - } - return $object -} - -<# -.SYNOPSIS - Retrieves the subscription ID and tenant ID for a specified Azure subscription. - -.DESCRIPTION - The Get-AzureSubscriptionInfo function takes a subscription name as a parameter and returns a custom object containing the subscription ID and tenant ID for the specified Azure subscription. It requires an active connection to Azure. - -.PARAMETER SubscriptionName - The SubscriptionName parameter specifies the name of the Azure subscription for which to retrieve the subscription ID and tenant ID. This parameter is mandatory. - -.EXAMPLE - PS C:\> Get-AzureSubscriptionInfo -SubscriptionName 'MySubscription' - SubscriptionId TenantId - -------------- -------- - 12345678-1234-1234-1234-1234567890ab 87654321-4321-4321-4321-ba0987654321 - - This example retrieves the subscription ID and tenant ID for the Azure subscription named 'MySubscription'. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-AzureSubscriptionInfo -{ - param ( - # The name of the Azure subscription - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string]$SubscriptionName - ) - - try - { - # Get the subscription details - $subscription = Get-AzSubscription -SubscriptionName $SubscriptionName - - # Check if the subscription exists - if ($null -eq $subscription) - { - Write-Error('Subscription "{0}" not found.', $SubscriptionName) - return - } - else - { - # Write verbose messages for debugging - Write-Verbose 'Function: Get-AzureSubscriptionInfo: Subscription found.' - Write-Verbose "SubscriptionID: $subscription.id SubscriptionName: $subscription.Name" - } - } - catch - { - # Handle exceptions and write an error message - Write-Error 'Ensure you have run Connect-AzAccount and that the subscription exists.' - return - } - - # Get the subscription ID and tenant ID - $subscriptionId = $subscription.SubscriptionId - $tenantId = $subscription.TenantId - - # Create a custom object to return - $object = [PSCustomObject]@{ - SubscriptionId = $subscriptionId - TenantId = $tenantId - } - - return $object -} - -<# -.SYNOPSIS - Creates a folder in a Data Lake Storage account. - -.DESCRIPTION - The Add-DataLakeFolder function creates a folder (or folder hierarchy) in a Data Lake storage account container. It requires an active connection to Azure. - -.PARAMETER SubscriptionName - The name of the Azure subscription to use. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container in the Data Lake Storage account. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder to create. May be a single folder or a folder hierarchy (e.g. 'folder1/folder2/folder3'). This parameter is mandatory. - -.PARAMETER ErrorIfFolderExists - Optional switch to throw error if folder exists. If not specified, will return the existing folder. - -.EXAMPLE - PS C:\> Add-DataLakeFolder -SubscriptionName 'MySubscription' -ResourceGroupName 'MyResourceGroup' -StorageAccountName 'MyStorageAccount' -ContainerName 'MyContainer' -FolderPath 'folder1/folder2/folder3' - This example creates a folder hierarchy 'folder1/folder2/folder3' in the specified Data Lake storage account container. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Add-DataLakeFolder -{ - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $true)] - [string]$FolderPath, # Path to the folder - - [switch]$ErrorIfFolderExists # Flag to indicate if an error should be thrown if the folder exists - ) - - # Get the subscription ID - $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { - Write-Error 'Subscription not found.' - return - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - - # Check if the Az.Storage module is installed - if (-not (Get-Module -Name Az.Storage -ListAvailable)) - { - Import-Module -Name Az.Storage # Install the Az.Storage module if it's not installed - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # Create the folder - try - { - $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop - } - catch - { - if ($ErrorIfFolderExists) - { - Write-Error "Folder $FolderPath already exists." - return - } - $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists - return - } - - if ($null -eq $ret) - { - Write-Error 'Failed to create the folder.' - return - } - else - { - return $ret # Return the created folder - } -} - -<# -.SYNOPSIS - Deletes a folder from an Azure Data Lake Storage Gen2 account. - -.DESCRIPTION - The Remove-DataLakeFolder function deletes a folder from an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, and folder path as input parameters. If the folder does not exist, it will return an error unless the -ErrorIfFolderDoesNotExist switch is used. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container containing the folder. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder to delete. This parameter is mandatory. - -.PARAMETER ErrorIfFolderDoesNotExist - If this switch is used, the function will not return an error if the folder does not exist. - -.EXAMPLE - PS C:\> Remove-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "MyFolder" - This example deletes the folder "MyFolder" from the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the "MySubscription" Azure subscription. - -.NOTES - This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Remove-DataLakeFolder -{ - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $true)] - [string]$FolderPath, # Path to the folder - - [switch]$ErrorIfFolderDoesNotExist # Flag to indicate if an error should be thrown if the folder does not exist - ) - - # Get the subscription ID - $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId - if ($null -eq $subId) - { - Write-Error 'Subscription not found.' - return - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # Ensure the folder exists before deleting - try - { - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -ErrorAction Stop - } - catch - { - if ($ErrorIfFolderDoesNotExist) - { - Write-Error "Folder '$FolderPath' does not exist to delete." - return - } - return - } - - # Delete the folder - if ($null -ne $folderExists) - { - $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force - } - - if ($null -ne $ret) - { - Write-Error 'Failed to delete the folder.' - return - } - else - { - Write-Host "Folder $ContainerName\$FolderPath deleted successfully." - return - } -} - -<# -.SYNOPSIS - Sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. - -.DESCRIPTION - The Set-DataLakeFolderACL function sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, folder path, identity, and access control type as input parameters. Optionally, it can also set the ACL for the container and include the default scope in the ACL. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container containing the folder. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is mandatory. - -.PARAMETER Identity - The identity to use in the ACL. This parameter is mandatory. - -.PARAMETER AccessControlType - The type of access control to apply to the folder. Valid values are 'Read' and 'Write'. This parameter is mandatory. - -.PARAMETER SetContainerACL - A switch parameter that specifies whether to set the ACL for the container. This parameter is optional. - -.PARAMETER IncludeDefaultScope - A switch parameter that specifies whether to include the default scope in the ACL. This parameter is optional. - -.PARAMETER DoNotApplyACLRecursively - A switch parameter that specifies whether to set the ACL recursively. This parameter is optional. - -.EXAMPLE - PS C:\> Set-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" -Identity "MyIdentity" -AccessControlType "Read" -IncludeDefaultScope - This example sets the ACL for the folder "/MyFolder" in the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" for the identity "MyIdentity" with read access and includes the default scope in the ACL. - -.NOTES - This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Set-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, - - [Parameter(Mandatory = $true)] - [string]$ContainerName, - - [Parameter(Mandatory = $true)] - [string]$FolderPath, - - [Parameter(Mandatory = $true)] - [string]$Identity, - - [ValidateSet('Read', 'Write')] - [Parameter(Mandatory = $true)] - [string]$AccessControlType, - - [switch]$SetContainerACL, - - [switch]$IncludeDefaultScope, - - [switch]$DoNotApplyACLRecursively - ) - - if (-not (Get-Module -Name Az.Storage -ListAvailable)) - { - Write-Verbose 'Installing Az.Storage module.' - Import-Module -Name Az.Storage - } - - $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName - if ($null -eq $sub) - { - Write-Error 'Subscription not found. Ensure you have run Connect-AzAccount before execution.' - return - } - else - { - $subId = $sub.SubscriptionId - } - - # Get the object ID of the identity to use in the ACL - $identityObj = Get-AADObjectId -Identity $Identity - if ($null -eq $identityObj) - { - Write-Error 'Identity not found.' - return - } - else - { - Write-Verbose ('{0} ID: {1} Display Name: {2}' -f $identityObj.ObjectType, $identityObj.ObjectId, $identityObj.DisplayName) - } - - # Set the current Azure context - $subContext = Set-AzContext -Subscription $subId - if ($null -eq $subContext) - { - Write-Error 'Failed to set the Azure context.' - return - } - else - { - Write-Verbose $subContext.Name - } - - # Get the Data Lake Storage account - $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName - if ($null -eq $storageAccount) - { - Write-Error 'Storage account not found.' - return - } - else - { - Write-Verbose $storageAccount.StorageAccountName - } - - # Set the context to the Data Lake Storage account - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - if ($null -eq $ctx) - { - Write-Error 'Failed to set the Data Lake Storage account context.' - return - } - - # verify the folder exists before setting the ACL - try - { - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - if ($null -eq $folderExists) - { - Write-Error('Folder not found.') - return - } - } - catch - { - Write-Error('Folder not found.') - return - } - - # translate the access control type to applied permission - $permission = switch ( $AccessControlType ) - { 'Read' - { 'r-x' - } - 'Write' - { 'rwx' - } - default - { '' - } - } - - $identityType = switch ($identityObj.ObjectType) - { 'User' - { 'user' - } - 'Group' - { 'group' - } - 'ServicePrincipal' - { 'user' - } - 'ManagedIdentity' - { 'other' - } - default - { '' - } - } - - # set the ACL at the container level - if ($SetContainerACL) - { - Write-Verbose 'set container ACL' - $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL - $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the container.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'Container ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - - # get the ACL for the folder - $acl = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath).ACL - - try - { - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - - } - catch [Microsoft.PowerShell.Commands.WriteErrorException] - { - Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' - return - } - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - ###################### - # default scope - if ($IncludeDefaultScope) - { - Write-Verbose 'include default scope' - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope - $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl - } - - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to set the ACL for the default scope.' - Write-Error $result.FailedEntries - return - } - else - { - Write-Host 'Default Scope ACL set successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - - -} - -<# -.SYNOPSIS - Gets the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. - -.DESCRIPTION - The Get-DataLakeFolderACL function retrieves the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, and container name as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. - -.PARAMETER SubscriptionName - The name of the Azure subscription. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the storage account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the storage account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is optional. If omitted, the function will revert to the root of the container. - -.EXAMPLE - PS C:\> Get-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" - This example gets the ACL for the folder "/MyFolder" in the container "MyContainer" of the storage account "MyStorageAccount" in the resource group "MyResourceGroup" of the Azure subscription "MySubscription". - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Get-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $false)] - [string]$FolderPath = '/' # Path to the folder in the Data Lake - ) - - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue - - # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { - $FolderPath = $FolderPath.Substring(1) - } - - try - { - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - - # Get the ACLs for the folder - $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL - - # Process each ACL - $aclResults = foreach ($ace in $acls) - { - if ($ace.EntityId) - { - # Get the AD object for the entity - $adObject = Get-AzureADObjectByObjectId -ObjectIds $ace.EntityId - - # Create a custom object with the ACL info - [PSCustomObject]@{ - DisplayName = $adObject.DisplayName - ObjectId = $ace.EntityId - ObjectType = $adObject.ObjectType - Permissions = $ace.Permissions - DefaultScope = $ace.DefaultScope - } - } - } - - # Return the results - return $aclResults - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -<# -.SYNOPSIS - Moves a folder in Azure Data Lake Storage Gen2 to a new location. - -.DESCRIPTION - The Move-DataLakeFolder function moves a folder in Azure Data Lake Storage Gen2 to a new location. It requires the subscription name, resource group name, storage account name, source container name, source folder path, and destination folder path as input parameters. Optionally, it can also take a destination container name. If the destination container name is not provided, the function will use the source container name. - -.PARAMETER SubscriptionName - The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER SourceContainerName - The name of the source container for the move operation. This parameter is mandatory. - -.PARAMETER SourceFolderPath - The path of the folder to move. This parameter is mandatory. - -.PARAMETER DestinationContainerName - The name of the destination container for the move operation. This parameter is optional. If not specified, the function will use the source container name. - -.PARAMETER DestinationFolderPath - The path of the destination folder. This parameter is mandatory. - -.EXAMPLE - PS C:\> Move-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -SourceContainerName "MySourceContainer" -SourceFolderPath "/MySourceFolder" -DestinationFolderPath "/MyDestinationFolder" - This example moves the folder "/MySourceFolder" from the container "MySourceContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription" to the folder "/MyDestinationFolder" in the same container. - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Move-DataLakeFolder -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$SourceContainerName, # Source container name - - [Parameter(Mandatory = $true)] - [string]$SourceFolderPath, # Source folder path - - [Parameter(Mandatory = $false)] - [string]$DestinationContainerName, # Destination container name - - [Parameter(Mandatory = $true)] - [string]$DestinationFolderPath # Destination folder path - ) - - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue - - try - { - - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Check if the folder exists - $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath - - # If destination container name is not provided, use source container name - if (-not $DestinationContainerName) - { - $DestinationContainerName = $SourceContainerName - } - - # Move the folder - $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force - - # Write verbose output and return the result - Write-Verbose ('Function: Move-DataLakeFolder') - Write-Verbose "Folder moved: $DestinationFolderPath" - return $ret - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -<# -.SYNOPSIS - Removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. - -.DESCRIPTION - The Remove-DataLakeFolderACL function removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, container name, and identity as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. - -.PARAMETER SubscriptionName - The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ResourceGroupName - The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER StorageAccountName - The name of the Data Lake Storage Gen2 account. This parameter is mandatory. - -.PARAMETER ContainerName - The name of the container. This parameter is mandatory. - -.PARAMETER FolderPath - The path of the folder. This parameter is optional. If not specified, the function will revert to the root of the container. - -.PARAMETER Identity - The identity to remove from the ACL. This parameter is mandatory. - -.PARAMETER DoNotApplyACLRecursively - A switch parameter that specifies whether to remove the identity from the ACL recursively. This parameter is optional. - -.EXAMPLE - PS C:\> Remove-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -Identity "MyIdentity" - This example removes the identity "MyIdentity" from the ACL of the root of the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription". - -.NOTES - This function requires the Az.Storage and AzureAd modules and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. - - Author: Stephen Carroll - Microsoft - Date: 2021-08-31 -#> -function Remove-DataLakeFolderACL -{ - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$SubscriptionName, # Azure subscription name - - [Parameter(Mandatory = $true)] - [string]$ResourceGroupName, # Azure resource group name - - [Parameter(Mandatory = $true)] - [string]$StorageAccountName, # Azure storage account name - - [Parameter(Mandatory = $true)] - [string]$ContainerName, # Azure container name - - [Parameter(Mandatory = $false)] - [string]$FolderPath = '/', # Path to the folder in the Data Lake - - [Parameter(Mandatory = $true)] - [string]$Identity, # Identity to remove from the ACL - - [Parameter(Mandatory = $false)] - [switch]$DoNotApplyACLRecursively # Flag to indicate if the ACL should not be applied recursively - - ) - - # Import necessary modules - Import-Module -Name Az.Storage -ErrorAction SilentlyContinue - Import-Module -Name AzureAd -ErrorAction SilentlyContinue - - # Remove leading slash or backslash from the folder path - if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) - { - $FolderPath = $FolderPath.Substring(1) - } - - try - { - - $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName - - # Get the object ID of the identity to use in the ACL - $identityObj = Get-AADObjectId -Identity $Identity - $id = $identityObj.ObjectId - - # Get the folder - $folder = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath - - # Get the ACLs for the folder - $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL - - # Remove the specified identity from the ACL - $newacl = $acls | Where-Object { -not ($_.AccessControlType -eq $identityObj.ObjectType -and $_.EntityId -eq $id) } - - # Update the ACL - if(-not $DoNotApplyACLRecursively) - { - $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } - else - { - $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl - } - - # Check if the update was successful - if ($result.FailedEntries.Count -gt 0) - { - Write-Error 'Failed to update the ACL.' - Write-Error $result.FailedEntries - } - else - { - Write-Host 'ACL updated successfully.' - Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) - Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) - } - } - catch - { - # Write any errors to the console - Write-Error $_.Exception.Message - } -} - -Export-ModuleMember -Function * +#region Dependency Management Functions + +<# +.SYNOPSIS + Tests if required modules are available and optionally installs them. + +.DESCRIPTION + The Test-ModuleDependencies function checks if the required modules for AzureDataLakeManagement are available. + It can optionally install missing modules and provides user feedback about the dependency status. + +.PARAMETER AutoInstall + If specified, automatically installs missing required modules from PowerShell Gallery. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Test-ModuleDependencies + Checks for required modules and displays status information. + +.EXAMPLE + PS C:\> Test-ModuleDependencies -AutoInstall + Checks for required modules and automatically installs any that are missing. + +.NOTES + Required modules: Az.Storage, Microsoft.Graph.Users, Microsoft.Graph.Groups, Microsoft.Graph.DirectoryObjects, Microsoft.Graph.Applications + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Test-ModuleDependencies { + [CmdletBinding()] + [OutputType([System.Boolean])] + param( + [switch]$AutoInstall, + [switch]$Quiet + ) + + $requiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') + $missingModules = @() + $availableModules = @() + + if (-not $Quiet) { + Write-Host "Checking AzureDataLakeManagement module dependencies..." -ForegroundColor Yellow + } + + foreach ($moduleName in $requiredModules) { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $missingModules += $moduleName + if (-not $Quiet) { + Write-Warning "Missing required module: $moduleName" + } + } + else { + $availableModules += $moduleName + if (-not $Quiet) { + Write-Host "✓ Found module: $moduleName (Version: $($module[0].Version))" -ForegroundColor Green + } + } + } + + if ($missingModules.Count -eq 0) { + if (-not $Quiet) { + Write-Host "✓ All required modules are available." -ForegroundColor Green + } + return $true + } + + if ($AutoInstall) { + if (-not $Quiet) { + Write-Host "Installing missing modules..." -ForegroundColor Yellow + } + return Install-ModuleDependencies -Modules $missingModules -Quiet:$Quiet + } + else { + if (-not $Quiet) { + Write-Host "`nTo install missing modules, run:" -ForegroundColor Cyan + Write-Host "Test-ModuleDependencies -AutoInstall" -ForegroundColor White + Write-Host "`nOr install manually:" -ForegroundColor Cyan + foreach ($module in $missingModules) { + Write-Host "Install-Module -Name $module -Force" -ForegroundColor White + } + } + return $false + } +} + +<# +.SYNOPSIS + Installs required modules for AzureDataLakeManagement. + +.DESCRIPTION + The Install-ModuleDependencies function installs the specified required modules from PowerShell Gallery. + +.PARAMETER Modules + Array of module names to install. If not specified, installs all required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Install-ModuleDependencies + Installs all required modules for AzureDataLakeManagement. + +.EXAMPLE + PS C:\> Install-ModuleDependencies -Modules @('Az.Storage') + Installs only the Az.Storage module. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Install-ModuleDependencies { + [CmdletBinding()] + param( + [string[]]$Modules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), + [switch]$Quiet + ) + + $successCount = 0 + $failureCount = 0 + + foreach ($moduleName in $Modules) { + try { + if (-not $Quiet) { + Write-Host "Installing module: $moduleName..." -ForegroundColor Yellow + } + + Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber -ErrorAction Stop + + if (-not $Quiet) { + Write-Host "✓ Successfully installed: $moduleName" -ForegroundColor Green + } + $successCount++ + } + catch { + Write-Error "Failed to install module $moduleName`: $($_.Exception.Message)" + $failureCount++ + } + } + + if (-not $Quiet) { + if ($failureCount -eq 0) { + Write-Host "✓ All modules installed successfully." -ForegroundColor Green + } + else { + Write-Warning "Installed $successCount modules, failed to install $failureCount modules." + } + } + + return ($failureCount -eq 0) +} + +<# +.SYNOPSIS + Imports required modules with proper error handling. + +.DESCRIPTION + The Import-ModuleDependencies function imports the required modules for AzureDataLakeManagement functions. + It provides better error handling and user feedback compared to individual Import-Module calls. + +.PARAMETER RequiredModules + Array of module names to import. Defaults to the core required modules. + +.PARAMETER Quiet + If specified, suppresses informational output and only shows errors. + +.EXAMPLE + PS C:\> Import-ModuleDependencies + Imports all required modules for AzureDataLakeManagement. + +.NOTES + Author: Stephen Carroll - Microsoft + Date: 2025-01-09 +#> +function Import-ModuleDependencies { + [CmdletBinding()] + [OutputType([System.Boolean])] + param( + [string[]]$RequiredModules = @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects'), + [switch]$Quiet + ) + + $importFailures = @() + + foreach ($moduleName in $RequiredModules) { + try { + $module = Get-Module -Name $moduleName -ListAvailable -ErrorAction SilentlyContinue + if ($null -eq $module) { + $importFailures += $moduleName + Write-Error "Module $moduleName is not available. Please install it first." + continue + } + + Import-Module -Name $moduleName -ErrorAction Stop -Force + if (-not $Quiet) { + Write-Verbose "Successfully imported module: $moduleName" + } + } + catch { + $importFailures += $moduleName + Write-Error "Failed to import module $moduleName`: $($_.Exception.Message)" + } + } + + if ($importFailures.Count -gt 0) { + Write-Error "Failed to import modules: $($importFailures -join ', '). Some functions may not work correctly." + return $false + } + + return $true +} + +#endregion + +<# +.SYNOPSIS + This function retrieves the object ID, object type, and display name for a specified Azure AD user, group, or service principal. + +.DESCRIPTION + Get-AADObjectId is a function that takes an identity as a parameter and returns the object ID, object type, and display name of the corresponding Azure AD user, group, or service principal. It requires an active connection to Microsoft Graph. + +.PARAMETER Identity + The Identity parameter specifies the user principal name, group display name, or service principal display name of the object to retrieve. This parameter is mandatory. + +.EXAMPLE + PS C:\> Get-AADObjectId -Identity "johndoe@contoso.com" + ObjectId ObjectType DisplayName + -------- ---------- ----------- + 12345678-1234-1234-1234-1234567890ab User John Doe + + This example retrieves the object ID, object type, and display name for the Azure AD user with the user principal name "johndoe@contoso.com". + +.EXAMPLE + PS C:\> Get-AADObjectId -Identity "HR Group" + ObjectId ObjectType DisplayName + -------- ---------- ----------- + 87654321-4321-4321-4321-ba0987654321 Group HR Group + + This example retrieves the object ID, object type, and display name for the Azure AD group with the display name "HR Group". + +.NOTES + This function requires an active connection to Microsoft Graph using Connect-MgGraph. If the specified identity does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility +#> +function Get-AADObjectId { + param ( + # The identity for which the Azure AD Object ID is to be fetched + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$Identity + ) + + # Replacing single quotes in the identity with double single quotes for filter syntax + $Identity = $Identity.Replace("'", "''") + + try { + # Initializing user, group, and service principal to null + $user = $null + $group = $null + $sp = $null + + # Try to get the user, group, and service principal + # Using Microsoft Graph cmdlets instead of AzureAD + $user = Get-MgUser -Filter "UserPrincipalName eq '$Identity'" -ErrorAction SilentlyContinue + if ($null -eq $user) { + $group = Get-MgGroup -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + if ($null -eq $group) { + $sp = Get-MgServicePrincipal -Filter "DisplayName eq '$Identity'" -ErrorAction SilentlyContinue + } + } + + # Check which object is not null and assign the corresponding values + if ($null -ne $user) { + $objectType = 'User' + $objectId = $user.Id + $displayName = $user.DisplayName + } + elseif ($null -ne $group) { + $objectType = 'Group' + $objectId = $group.Id + $displayName = $group.DisplayName + } + elseif ($null -ne $sp) { + $objectType = 'ServicePrincipal' + $objectId = $sp.Id + $displayName = $sp.DisplayName + } + else { + Write-Error ('Object not found. Unable to find object "{0}" in Azure AD.' -f $Identity) + return + } + } + catch { + # Check if the error is due to missing authentication + # Microsoft Graph throws specific error codes for authentication issues + if ($_.Exception.GetType().Name -match 'AuthenticationException|UnauthorizedAccessException' -or + $_.Exception.Message -match '401|Unauthorized|authentication.*required' -or + $_.FullyQualifiedErrorId -match 'Authentication') { + Write-Error 'You must be authenticated to Microsoft Graph to run this command. Run Connect-MgGraph to authenticate.' + return + } + Write-Error $_.Exception.Message + return + } + + # Output the object details + Write-Verbose "Object ID: $objectId" + Write-Verbose "Object Type: $objectType" + Write-Verbose "Object Name: $displayName" + + # Create a custom object to return + $object = [PSCustomObject]@{ + ObjectId = $objectId + ObjectType = $objectType + DisplayName = $displayName + } + return $object +} + +<# +.SYNOPSIS + Retrieves the subscription ID and tenant ID for a specified Azure subscription. + +.DESCRIPTION + The Get-AzureSubscriptionInfo function takes a subscription name as a parameter and returns a custom object containing the subscription ID and tenant ID for the specified Azure subscription. It requires an active connection to Azure. + +.PARAMETER SubscriptionName + The SubscriptionName parameter specifies the name of the Azure subscription for which to retrieve the subscription ID and tenant ID. This parameter is mandatory. + +.EXAMPLE + PS C:\> Get-AzureSubscriptionInfo -SubscriptionName 'MySubscription' + SubscriptionId TenantId + -------------- -------- + 12345678-1234-1234-1234-1234567890ab 87654321-4321-4321-4321-ba0987654321 + + This example retrieves the subscription ID and tenant ID for the Azure subscription named 'MySubscription'. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Get-AzureSubscriptionInfo { + param ( + # The name of the Azure subscription + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$SubscriptionName + ) + + try { + # Get the subscription details + $subscription = Get-AzSubscription -SubscriptionName $SubscriptionName + + # Check if the subscription exists + if ($null -eq $subscription) { + Write-Error('Subscription "{0}" not found.', $SubscriptionName) + return + } + else { + # Write verbose messages for debugging + Write-Verbose 'Function: Get-AzureSubscriptionInfo: Subscription found.' + Write-Verbose "SubscriptionID: $subscription.id SubscriptionName: $subscription.Name" + } + } + catch { + # Handle exceptions and write an error message + Write-Error 'Ensure you have run Connect-AzAccount and that the subscription exists.' + return + } + + # Get the subscription ID and tenant ID + $subscriptionId = $subscription.SubscriptionId + $tenantId = $subscription.TenantId + + # Create a custom object to return + $object = [PSCustomObject]@{ + SubscriptionId = $subscriptionId + TenantId = $tenantId + } + + return $object +} + +<# +.SYNOPSIS + Creates a folder in a Data Lake Storage account. + +.DESCRIPTION + The Add-DataLakeFolder function creates a folder (or folder hierarchy) in a Data Lake storage account container. It requires an active connection to Azure. + +.PARAMETER SubscriptionName + The name of the Azure subscription to use. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container in the Data Lake Storage account. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder to create. May be a single folder or a folder hierarchy (e.g. 'folder1/folder2/folder3'). This parameter is mandatory. + +.PARAMETER ErrorIfFolderExists + Optional switch to throw error if folder exists. If not specified, will return the existing folder. + +.EXAMPLE + PS C:\> Add-DataLakeFolder -SubscriptionName 'MySubscription' -ResourceGroupName 'MyResourceGroup' -StorageAccountName 'MyStorageAccount' -ContainerName 'MyContainer' -FolderPath 'folder1/folder2/folder3' + This example creates a folder hierarchy 'folder1/folder2/folder3' in the specified Data Lake storage account container. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Add-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $true)] + [string]$FolderPath, # Path to the folder + + [switch]$ErrorIfFolderExists # Flag to indicate if an error should be thrown if the folder exists + ) + + # Get the subscription ID + $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId + if ($null -eq $subId) { + Write-Error 'Subscription not found.' + return + } + + # Set the current Azure context + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } + } + + # Check if the Az.Storage module is available and import it + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { + Write-Error 'Required module Az.Storage is not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) { + Write-Error 'Storage account not found.' + return + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # Create the folder + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", 'Create Data Lake Folder')) { + try { + $ret = New-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Directory -ErrorAction Stop + } + catch { + if ($ErrorIfFolderExists) { + Write-Error "Folder $FolderPath already exists." + return + } + $ret = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath # Get the folder if it already exists + return + } + + if ($null -eq $ret) { + Write-Error 'Failed to create the folder.' + return + } + else { + return $ret # Return the created folder + } + } + + +} + +<# +.SYNOPSIS + Deletes a folder from an Azure Data Lake Storage Gen2 account. + +.DESCRIPTION + The Remove-DataLakeFolder function deletes a folder from an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, and folder path as input parameters. If the folder does not exist, it will return an error unless the -ErrorIfFolderDoesNotExist switch is used. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container containing the folder. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder to delete. This parameter is mandatory. + +.PARAMETER ErrorIfFolderDoesNotExist + If this switch is used, the function will not return an error if the folder does not exist. + +.EXAMPLE + PS C:\> Remove-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "MyFolder" + This example deletes the folder "MyFolder" from the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the "MySubscription" Azure subscription. + +.NOTES + This function requires an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 +#> +function Remove-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $true)] + [string]$FolderPath, # Path to the folder + + [switch]$ErrorIfFolderDoesNotExist # Flag to indicate if an error should be thrown if the folder does not exist + ) + + # Get the subscription ID + $subId = (Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName).SubscriptionId + if ($null -eq $subId) { + Write-Error 'Subscription not found.' + return + } + + # Set the current Azure context + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } + } + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) { + Write-Error 'Storage account not found.' + return + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # Ensure the folder exists before deleting + try { + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -ErrorAction Stop + } + catch { + if ($ErrorIfFolderDoesNotExist) { + Write-Error "Folder '$FolderPath' does not exist to delete." + return + } + return + } + + # Delete the folder + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", 'Remove Data Lake Folder')) { + if ($null -ne $folderExists) { + $ret = Remove-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Force + } + } + + if ($null -ne $ret) { + Write-Error 'Failed to delete the folder.' + return + } + else { + Write-Host "Folder $ContainerName\$FolderPath deleted successfully." + return + } +} + +<# +.SYNOPSIS + Sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. + +.DESCRIPTION + The Set-DataLakeFolderACL function sets the Access Control List (ACL) for a folder in an Azure Data Lake Storage Gen2 account. It requires the subscription name, resource group name, storage account name, container name, folder path, identity, and access control type as input parameters. Optionally, it can also set the ACL for the container and include the default scope in the ACL. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container containing the folder. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is mandatory. + +.PARAMETER Identity + The identity to use in the ACL. This parameter is mandatory. + +.PARAMETER AccessControlType + The type of access control to apply to the folder. Valid values are 'Read' and 'Write'. This parameter is mandatory. + +.PARAMETER SetContainerACL + A switch parameter that specifies whether to set the ACL for the container. This parameter is optional. + +.PARAMETER IncludeDefaultScope + A switch parameter that specifies whether to include the default scope in the ACL. This parameter is optional. + +.PARAMETER DoNotApplyACLRecursively + A switch parameter that specifies whether to set the ACL recursively. This parameter is optional. + +.EXAMPLE + PS C:\> Set-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" -Identity "MyIdentity" -AccessControlType "Read" -IncludeDefaultScope + This example sets the ACL for the folder "/MyFolder" in the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" for the identity "MyIdentity" with read access and includes the default scope in the ACL. + +.NOTES + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. If the specified access control type is not 'Read' or 'Write', the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility +#> +function Set-DataLakeFolderACL { + [CmdletBinding(SupportsShouldProcess)] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, + + [Parameter(Mandatory = $true)] + [string]$ContainerName, + + [Parameter(Mandatory = $true)] + [string]$FolderPath, + + [Parameter(Mandatory = $true)] + [string]$Identity, + + [ValidateSet('Read', 'Write')] + [Parameter(Mandatory = $true)] + [string]$AccessControlType, + + [switch]$SetContainerACL, + + [switch]$IncludeDefaultScope, + + [switch]$DoNotApplyACLRecursively + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + $sub = Get-AzureSubscriptionInfo -SubscriptionName $SubscriptionName + if ($null -eq $sub) { + Write-Error 'Subscription not found. Ensure you have run Connect-AzAccount before execution.' + return + } + else { + $subId = $sub.SubscriptionId + } + + # Get the object ID of the identity to use in the ACL + $identityObj = Get-AADObjectId -Identity $Identity + if ($null -eq $identityObj) { + Write-Error 'Identity not found.' + return + } + else { + Write-Verbose ('{0} ID: {1} Display Name: {2}' -f $identityObj.ObjectType, $identityObj.ObjectId, $identityObj.DisplayName) + } + + # Set the current Azure context + if ($pscmdlet.ShouldProcess("Setting Azure context to subscription $SubscriptionName", 'Set Azure Context')) { + # Set the current Azure context + $subContext = Set-AzContext -Subscription $subId + if ($null -eq $subContext) { + Write-Error 'Failed to set the Azure context.' + return + } + else { + Write-Verbose $subContext.Name + } + } + + + # Get the Data Lake Storage account + $storageAccount = Get-AzStorageAccount -Name $StorageAccountName -ResourceGroup $ResourceGroupName + if ($null -eq $storageAccount) { + Write-Error 'Storage account not found.' + return + } + else { + Write-Verbose $storageAccount.StorageAccountName + } + + # Set the context to the Data Lake Storage account + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + if ($null -eq $ctx) { + Write-Error 'Failed to set the Data Lake Storage account context.' + return + } + + # verify the folder exists before setting the ACL + try { + $folderExists = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + if ($null -eq $folderExists) { + Write-Error('Folder not found.') + return + } + } + catch { + Write-Error('Folder not found.') + return + } + + # translate the access control type to applied permission + $permission = switch ( $AccessControlType ) { + 'Read' { + 'r-x' + } + 'Write' { + 'rwx' + } + default { + '' + } + } + + $identityType = switch ($identityObj.ObjectType) { + 'User' { + 'user' + } + 'Group' { + 'group' + } + 'ServicePrincipal' { + 'user' + } + 'ManagedIdentity' { + 'other' + } + default { + '' + } + } + + # set the ACL at the container level + if ($SetContainerACL) { + if ($PSCmdlet.ShouldProcess($ContainerName, "Set container ACL for identity '$($identityObj.DisplayName)' with Read access")) { + Write-Verbose 'set container ACL' + $containerACL = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName).ACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'r-x' -InputObject $containerACL + $containerACL = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission 'r-x' -InputObject $containerACL + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Acl $containerACL + + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL for the container.' + Write-Error $result.FailedEntries + return + } + else { + Write-Host 'Container ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + } + + # get the ACL for the folder + $acl = (Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath).ACL + + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Set ACL for identity '$($identityObj.DisplayName)' with $AccessControlType access")) { + try { + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + + } + catch [Microsoft.PowerShell.Commands.WriteErrorException] { + Write-Error 'Error communicating with Powershell module AZ.Storage. Ensure you have the latest version of the module installed. (Install-Module -Name Az.Storage -Force)' + return + } + + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL.' + Write-Error $result.FailedEntries + return + } + else { + Write-Host 'ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + ###################### + # default scope + if ($IncludeDefaultScope) { + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Set default scope ACL for identity '$($identityObj.DisplayName)' with $AccessControlType access")) { + Write-Verbose 'include default scope' + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType Mask -Permission 'rwx' -InputObject $acl -DefaultScope + $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType $identityType -EntityId $identityObj.ObjectId -Permission $permission -InputObject $acl -DefaultScope + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $acl + } + + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to set the ACL for the default scope.' + Write-Error $result.FailedEntries + return + } + else { + Write-Host 'Default Scope ACL set successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + } + + +} + +<# +.SYNOPSIS + Gets the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. + +.DESCRIPTION + The Get-DataLakeFolderACL function retrieves the Access Control List (ACL) for a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, and container name as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. + +.PARAMETER SubscriptionName + The name of the Azure subscription. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the storage account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the storage account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is optional. If omitted, the function will revert to the root of the container. + +.EXAMPLE + PS C:\> Get-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -FolderPath "/MyFolder" + This example gets the ACL for the folder "/MyFolder" in the container "MyContainer" of the storage account "MyStorageAccount" in the resource group "MyResourceGroup" of the Azure subscription "MySubscription". + +.NOTES + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, or container does not exist, the function will return an error message. If the specified folder does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility +#> +function Get-DataLakeFolderACL { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $false)] + [string]$FolderPath = '/' # Path to the folder in the Data Lake + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Remove leading slash or backslash from the folder path + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) { + $FolderPath = $FolderPath.Substring(1) + } + + try { + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Verify the folder exists (will throw error if not found) + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + + # Get the ACLs for the folder + $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL + + # Process each ACL + $aclResults = foreach ($ace in $acls) { + if ($ace.EntityId) { + # Get the AD object for the entity using Microsoft Graph + $adObject = Get-MgDirectoryObject -DirectoryObjectId $ace.EntityId -ErrorAction SilentlyContinue + + # Extract display name and object type from the directory object + $displayName = $null + $objectType = $null + + if ($adObject) { + # Try to get DisplayName from the object properties first, then from AdditionalProperties + if ($adObject.PSObject.Properties.Name -contains 'displayName') { + $displayName = $adObject.DisplayName + } + elseif ($adObject.AdditionalProperties.ContainsKey('displayName')) { + $displayName = $adObject.AdditionalProperties['displayName'] + } + # Extract object type from odata.type + if ($adObject.AdditionalProperties.ContainsKey('@odata.type')) { + $objectType = $adObject.AdditionalProperties['@odata.type'] -replace '#microsoft.graph.', '' + } + } + + # Create a custom object with the ACL info + [PSCustomObject]@{ + DisplayName = $displayName + ObjectId = $ace.EntityId + ObjectType = $objectType + Permissions = $ace.Permissions + DefaultScope = $ace.DefaultScope + } + } + } + + # Return the results + return $aclResults + } + catch { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +<# +.SYNOPSIS + Moves a folder in Azure Data Lake Storage Gen2 to a new location. + +.DESCRIPTION + The Move-DataLakeFolder function moves a folder in Azure Data Lake Storage Gen2 to a new location. It requires the subscription name, resource group name, storage account name, source container name, source folder path, and destination folder path as input parameters. Optionally, it can also take a destination container name. If the destination container name is not provided, the function will use the source container name. + +.PARAMETER SubscriptionName + The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER SourceContainerName + The name of the source container for the move operation. This parameter is mandatory. + +.PARAMETER SourceFolderPath + The path of the folder to move. This parameter is mandatory. + +.PARAMETER DestinationContainerName + The name of the destination container for the move operation. This parameter is optional. If not specified, the function will use the source container name. + +.PARAMETER DestinationFolderPath + The path of the destination folder. This parameter is mandatory. + +.EXAMPLE + PS C:\> Move-DataLakeFolder -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -SourceContainerName "MySourceContainer" -SourceFolderPath "/MySourceFolder" -DestinationFolderPath "/MyDestinationFolder" + This example moves the folder "/MySourceFolder" from the container "MySourceContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription" to the folder "/MyDestinationFolder" in the same container. + +.NOTES + This function requires the Az.Storage module and an active connection to Azure using Connect-AzAccount. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 + Updated: 2025-01-09 - Removed unnecessary AzureAD dependency +#> +function Move-DataLakeFolder { + [CmdletBinding(SupportsShouldProcess)] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$SourceContainerName, # Source container name + + [Parameter(Mandatory = $true)] + [string]$SourceFolderPath, # Source folder path + + [Parameter(Mandatory = $false)] + [string]$DestinationContainerName, # Destination container name + + [Parameter(Mandatory = $true)] + [string]$DestinationFolderPath # Destination folder path + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + try { + + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Check if the folder exists + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath + + # If destination container name is not provided, use source container name + if (-not $DestinationContainerName) { + $DestinationContainerName = $SourceContainerName + } + + # Move the folder + if ($PSCmdlet.ShouldProcess("$SourceContainerName\$SourceFolderPath", "Move folder to $DestinationContainerName\$DestinationFolderPath")) { + $ret = Move-AzDataLakeGen2Item -Context $ctx -FileSystem $SourceContainerName -Path $SourceFolderPath -DestFileSystem $DestinationContainerName -DestPath $DestinationFolderPath -Force + + # Write verbose output and return the result + Write-Verbose ('Function: Move-DataLakeFolder') + Write-Verbose "Folder moved: $DestinationFolderPath" + return $ret + } + } + catch { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +<# +.SYNOPSIS + Removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. + +.DESCRIPTION + The Remove-DataLakeFolderACL function removes an identity from the Access Control List (ACL) of a folder in Azure Data Lake Storage Gen2. It requires the subscription name, resource group name, storage account name, container name, and identity as input parameters. Optionally, it can also take a folder path. If the folder path is not provided, the function will revert to the root of the container. + +.PARAMETER SubscriptionName + The name of the Azure subscription containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ResourceGroupName + The name of the resource group containing the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER StorageAccountName + The name of the Data Lake Storage Gen2 account. This parameter is mandatory. + +.PARAMETER ContainerName + The name of the container. This parameter is mandatory. + +.PARAMETER FolderPath + The path of the folder. This parameter is optional. If not specified, the function will revert to the root of the container. + +.PARAMETER Identity + The identity to remove from the ACL. This parameter is mandatory. + +.PARAMETER DoNotApplyACLRecursively + A switch parameter that specifies whether to remove the identity from the ACL recursively. This parameter is optional. + +.EXAMPLE + PS C:\> Remove-DataLakeFolderACL -SubscriptionName "MySubscription" -ResourceGroupName "MyResourceGroup" -StorageAccountName "MyStorageAccount" -ContainerName "MyContainer" -Identity "MyIdentity" + This example removes the identity "MyIdentity" from the ACL of the root of the container "MyContainer" in the storage account "MyStorageAccount" in the resource group "MyResourceGroup" in the Azure subscription "MySubscription". + +.NOTES + This function requires the Az.Storage, Microsoft.Graph.Users, and Microsoft.Graph.Groups modules and an active connection to Azure using Connect-AzAccount and Connect-MgGraph. If the specified subscription, resource group, storage account, container, or folder does not exist, the function will return an error message. If the specified identity does not exist, the function will return an error message. + + Author: Stephen Carroll - Microsoft + Date: 2021-08-31 + Updated: 2025-01-09 - Migrated from AzureAD to Microsoft.Graph for PowerShell 7+ compatibility +#> +function Remove-DataLakeFolderACL { + [CmdletBinding(SupportsShouldProcess)] + param( + [Parameter(Mandatory = $true)] + [string]$SubscriptionName, # Azure subscription name + + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, # Azure resource group name + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, # Azure storage account name + + [Parameter(Mandatory = $true)] + [string]$ContainerName, # Azure container name + + [Parameter(Mandatory = $false)] + [string]$FolderPath = '/', # Path to the folder in the Data Lake + + [Parameter(Mandatory = $true)] + [string]$Identity, # Identity to remove from the ACL + + [Parameter(Mandatory = $false)] + [switch]$DoNotApplyACLRecursively # Flag to indicate if the ACL should not be applied recursively + + ) + + # Check if required modules are available and import them + if (-not (Import-ModuleDependencies -RequiredModules @('Az.Storage', 'Microsoft.Graph.Applications', 'Microsoft.Graph.Users', 'Microsoft.Graph.Groups', 'Microsoft.Graph.DirectoryObjects') -Quiet)) { + Write-Error 'Required modules are not available. Run Test-ModuleDependencies -AutoInstall to install missing dependencies.' + return + } + + # Remove leading slash or backslash from the folder path + if ($FolderPath.Length -gt 1 -and ($FolderPath.StartsWith('/') -or $FolderPath.StartsWith('\'))) { + $FolderPath = $FolderPath.Substring(1) + } + + try { + + $ctx = New-AzStorageContext -StorageAccountName $StorageAccountName + + # Get the object ID of the identity to use in the ACL + $identityObj = Get-AADObjectId -Identity $Identity + $id = $identityObj.ObjectId + + # Get the folder + $null = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath + + # Get the ACLs for the folder + $acls = Get-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath | Select-Object -ExpandProperty ACL + + # Remove the specified identity from the ACL + $newacl = $acls | Where-Object { -not ($_.AccessControlType -eq $identityObj.ObjectType -and $_.EntityId -eq $id) } + + # Update the ACL + if ($PSCmdlet.ShouldProcess("$ContainerName\$FolderPath", "Remove ACL for identity '$($identityObj.DisplayName)'")) { + if (-not $DoNotApplyACLRecursively) { + $result = Update-AzDataLakeGen2Item -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } + else { + $result = Update-AzDataLakeGen2AclRecursive -Context $ctx -FileSystem $ContainerName -Path $FolderPath -Acl $newacl + } + + # Check if the update was successful + if ($result.FailedEntries.Count -gt 0) { + Write-Error 'Failed to update the ACL.' + Write-Error $result.FailedEntries + } + else { + Write-Host 'ACL updated successfully.' + Write-Verbose ('Successful Directories: {0} ' -f $result.TotalDirectoriesSuccessfulCount) + Write-Verbose ('Successful Files: {0} ' -f $result.TotalFilesSuccessfulCount) + } + } + } + catch { + # Write any errors to the console + Write-Error $_.Exception.Message + } +} + +#region Module Initialization +# This code runs when the module is imported +# Check dependencies on module import +$dependencyCheckResult = Test-ModuleDependencies -Quiet + +if (-not $dependencyCheckResult) { + Write-Warning @" +AzureDataLakeManagement module loaded with missing dependencies. +Some functions may not work correctly until required modules are installed. +Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +"@ +} +#endregion + +Export-ModuleMember -Function * diff --git a/LICENSE b/LICENSE index 9c38f61..2c0bbe8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,21 @@ -MIT License - -Copyright (c) 2023 SteveCInVA - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License + +Copyright (c) 2023 SteveCInVA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..dd19445 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,214 @@ +# Migration Guide: AzureAD to Microsoft.Graph + +## Overview +Version 2025.11.2 of the AzureDataLakeManagement module migrates from the deprecated AzureAD PowerShell module to the Microsoft.Graph PowerShell SDK. This change is necessary for PowerShell 7+ compatibility and aligns with Microsoft's recommendations. + +## Why This Change? +- **AzureAD module is deprecated**: Microsoft announced the deprecation of the AzureAD and MSOnline modules with retirement scheduled for late 2025 +- **PowerShell 7+ incompatibility**: The AzureAD module only works with Windows PowerShell 5.1 and is not compatible with PowerShell Core (7+) +- **Modern authentication**: Microsoft Graph SDK uses MSAL (Microsoft Authentication Library) with better security and support for modern authentication methods +- **Future-proof**: Microsoft Graph is the unified endpoint for all Microsoft 365 services with regular updates + +## Breaking Changes + +### Module Dependencies +**Before (v2025.11.1 and earlier):** +```powershell +# Required modules +- Az.Storage +- AzureAD +- Az.Accounts + +``` + +**After (v2025.11.2 and later):** +```powershell +# Required modules +- Az.Storage +- Microsoft.Graph.Applications +- Microsoft.Graph.Users +- Microsoft.Graph.Groups +- Microsoft.Graph.DirectoryObjects + +``` + +### Authentication +**Before:** +```powershell +Connect-AzAccount +Connect-AzureAD +``` + +**After:** +```powershell +Connect-AzAccount +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +## Migration Steps + +### Step 1: Install New Dependencies +```powershell +# Uninstall old AzureAD module (optional) +Uninstall-Module -Name AzureAD + +# Install Microsoft Graph modules +Install-Module -Name Microsoft.Graph.Applications -Force +Install-Module -Name Microsoft.Graph.Users -Force +Install-Module -Name Microsoft.Graph.Groups -Force +Install-Module -Name Microsoft.Graph.DirectoryObjects -Force + +# Or use the module's built-in dependency management +Import-Module AzureDataLakeManagement +Test-ModuleDependencies -AutoInstall +``` + +### Step 2: Update Authentication in Scripts +Replace all instances of `Connect-AzureAD` with: +```powershell +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +**Note on Scopes:** +- `User.Read.All`: Required for reading user information +- `Group.Read.All`: Required for reading group information +- `Application.Read.All`: Required for reading service principal information + +For production/automation scenarios, consider using certificate-based authentication: +```powershell +Connect-MgGraph -ClientId "YOUR_CLIENT_ID" -TenantId "YOUR_TENANT_ID" -CertificateThumbprint "YOUR_CERT_THUMBPRINT" +``` + +### Step 3: Update Module Version +```powershell +# Update to the latest version +Update-Module -Name AzureDataLakeManagement + +# Verify version +Get-Module -Name AzureDataLakeManagement -ListAvailable | Select-Object Name, Version +``` + +## Function Changes + +### No Changes to Function Signatures +All public functions maintain the same parameters and behavior: +- `Get-AADObjectId` +- `Get-AzureSubscriptionInfo` +- `Add-DataLakeFolder` +- `Remove-DataLakeFolder` +- `Set-DataLakeFolderACL` +- `Get-DataLakeFolderACL` +- `Move-DataLakeFolder` +- `Remove-DataLakeFolderACL` + +### Internal Changes +The following cmdlet replacements were made internally: +- `Get-AzureADUser` → `Get-MgUser` +- `Get-AzureADGroup` → `Get-MgGroup` +- `Get-AzureADServicePrincipal` → `Get-MgServicePrincipal` +- `Get-AzureADObjectByObjectId` → `Get-MgDirectoryObject` + +## Troubleshooting + +### Error: "Module not found" +```powershell +# Solution: Install the required modules +Test-ModuleDependencies -AutoInstall +``` + +### Error: "Authentication needed" +```powershell +# Solution: Connect to Microsoft Graph with appropriate scopes +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +### Error: "Insufficient privileges" +```powershell +# Solution: Ensure your account has the required permissions +# Or use delegated permissions with admin consent +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +### Compatibility Issues +If you experience issues: +1. Verify PowerShell version: `$PSVersionTable.PSVersion` +2. Check installed modules: `Get-Module -ListAvailable | Where-Object Name -match "Graph|Az"` +3. Update all Az modules: `Update-Module -Name Az.*` +4. Restart PowerShell session + +## Testing Your Migration + +### Test Basic Functionality +```powershell +# Import module +Import-Module AzureDataLakeManagement + +# Check dependencies +Test-ModuleDependencies + +# Authenticate +Connect-AzAccount +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +# Test object lookup +Get-AADObjectId -Identity "user@yourdomain.com" +``` + +### Verify ACL Operations +```powershell +# Test ACL retrieval (replace with your values) +Get-DataLakeFolderACL -SubscriptionName "YourSubscription" ` + -ResourceGroupName "YourResourceGroup" ` + -StorageAccountName "yourstorageaccount" ` + -ContainerName "yourcontainer" ` + -FolderPath "yourfolder" +``` + +## Support and Resources + +### Documentation +- [Microsoft Graph PowerShell SDK Documentation](https://learn.microsoft.com/en-us/powershell/microsoftgraph/) +- [Upgrade from Azure AD PowerShell to Microsoft Graph PowerShell](https://learn.microsoft.com/en-us/powershell/microsoftgraph/migration-steps) + +### Common Use Cases + +#### Finding a User +```powershell +# Old way (AzureAD) +Get-AzureADUser -Filter "UserPrincipalName eq 'user@domain.com'" + +# New way (Microsoft Graph) +Get-MgUser -Filter "UserPrincipalName eq 'user@domain.com'" + +# Using the module (unchanged) +Get-AADObjectId -Identity "user@domain.com" +``` + +#### Finding a Group +```powershell +# Old way (AzureAD) +Get-AzureADGroup -Filter "DisplayName eq 'GroupName'" + +# New way (Microsoft Graph) +Get-MgGroup -Filter "DisplayName eq 'GroupName'" + +# Using the module (unchanged) +Get-AADObjectId -Identity "GroupName" +``` + +## Feedback and Issues +If you encounter any issues with the migration, please: +1. Review this migration guide +2. Check the [GitHub Issues](https://github.com/SteveCInVA/AzureDataLakeManagement/issues) +3. Create a new issue with details about your environment and the error + +## Rollback (Not Recommended) +If you need to temporarily roll back to the old version: +```powershell +# Install specific old version +Install-Module -Name AzureDataLakeManagement -RequiredVersion 2025.11.1 -Force + +# Note: This is only a temporary solution as AzureAD module will be retired +``` + +**Important**: The rollback is only temporary. You should plan to migrate to Microsoft.Graph as the AzureAD module will stop working when Microsoft completes the retirement. diff --git a/README.md b/README.md index 99ddf8a..d85ec43 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,182 @@ -# AzureDataLakeManagement -This project was created to help simplify the process of managing an Azure Datalake specifically around updating existing ACL's to child objects within the lake. -Yes, this can be accomplished with Azure Storage Explorer, however come customers don't like to install new software. - -My goal, is to make a straight forward set of functions that will assist a user in configuring folders and the associated ACL's in an ADLS Gen 2 storage container using the objects names rather than ID's. - -To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement - -*** - -## Version History: - -- 2025.1.1 - 01/09/2025 -Issue 27 - Added optional switch to set-DataLakeFolderACL and remove-DataLakeFolderACL functions to enable the user to not recursively apply permissions on children of the path specified. - -- 2024.1.1 - 01/09/2024 -Issue 22 - Fixed issue where a lack of Azure Permissions to Microsoft.Storage/storageAccounts/listKeys/action would cause failure to execute even with correct AzureAD permissions on objects. - -- 2023.12.3 - 12/01/2023 -Published via Github Actions to [PowershellGallery.com](https://www.powershellgallery.com/packages/AzureDataLakeManagement) - -- 2023.12.2 - 12/01/2023 -Removed - Github Action Publish testing - -- 2023.12.1 - 12/01/2023 -Function optimization and improved consistency of help content. - -- 2023.11.2 - 11/13/2023 -Added function to remove an ACL from a folder and all inherited children - +# AzureDataLakeManagement +This project was created to help simplify the process of managing an Azure Datalake specifically around updating existing ACL's to child objects within the lake. +Yes, this can be accomplished with Azure Storage Explorer, however some customers don't like to install new software. + +My goal, is to make a straight forward set of functions that will assist a user in configuring folders and the associated ACL's in an ADLS Gen 2 storage container using the objects names rather than ID's. + +To contribute to this project please view the GitHub project at https://github.com/SteveCInVA/AzureDataLakeManagement + +## Module Usage + +- For example usage see: [example.ps1](example.ps1) +- For module installation support see: [example-dependency-management.ps1](example-dependency-management.ps1) + +## Functions Supported: + +### Folder Management +- Add-DataLakeFolder +- Move-DataLakeFolder +- Remove-DataLakeFolder + +### ACL Management +- Get-DataLakeFolderACL +- Set-DataLakeFolderACL +- Remove-DataLakeFolderACL + +### Support Functions +- Get-AADObjectId +- Get-AzureSubscriptionInfo + +### Module Installation Support +- Import-ModuleDependencies +- Install-ModuleDependencies +- Test-ModuleDependencies + +## Development Environment + +### Using Dev Containers (Recommended) + +This repository includes support for Visual Studio Code dev containers, providing a consistent development environment with all required tools pre-installed. + +#### Prerequisites +- [Visual Studio Code](https://code.visualstudio.com/) +- [Docker Desktop](https://www.docker.com/products/docker-desktop) +- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VS Code + +#### Getting Started with Dev Containers +1. Clone the repository +2. Open the repository folder in Visual Studio Code +3. When prompted, click "Reopen in Container" (or use Command Palette: `Dev Containers: Reopen in Container`) +4. VS Code will build the container and install all dependencies automatically + +#### What's Included +The dev container includes: +- **PowerShell 7+** - Latest version installed automatically +- **Pre-installed VS Code Extensions:** + - PowerShell - Language support and debugging + - Pester Test - Testing framework support + - GitHub Copilot - AI-powered code assistance + - GitHub Actions - Workflow file support + - TODO Highlight v2 - Highlight TODO comments +- **PowerShell Modules:** + - PSScriptAnalyzer - For code quality checks + - Pester - For testing + +#### Working in the Dev Container +Once the container is running, you can: +- Import the module: `Import-Module ./AzureDataLakeManagement/AzureDataLakeManagement.psm1 -Force` +- Run code quality checks: `Invoke-ScriptAnalyzer -Path ./AzureDataLakeManagement/AzureDataLakeManagement.psm1` +- Test the module: `Test-ModuleManifest ./AzureDataLakeManagement/AzureDataLakeManagement.psd1` +- Run Pester tests: `Invoke-Pester -Path ./Tests` + +### Local Development (Without Dev Containers) + +If you prefer to develop locally without containers, ensure you have: +- PowerShell 7+ installed +- Required PowerShell modules (see Dependency Management section below) + +## Dependency Management + +Starting with version 2025.1.1, the module includes improved dependency management features. **Version 2025.11.2 migrates from the deprecated AzureAD module to Microsoft.Graph for PowerShell 7+ compatibility.** + +### Required Dependencies +The module requires the following PowerShell modules: +- `Az.Storage` - For Azure Storage operations +- `Microsoft.Graph.Applications` - For Microsoft Graph application operations (replaces AzureAD) +- `Microsoft.Graph.Users` - For Microsoft Graph user operations (replaces AzureAD) +- `Microsoft.Graph.Groups` - For Microsoft Graph group operations (replaces AzureAD) +- `Microsoft.Graph.DirectoryObjects` - For Microsoft Graph directory objects (replaces AzureAD) + + +**Important**: The legacy `AzureAD` module is no longer supported as it is incompatible with PowerShell 7+ and has been deprecated by Microsoft. This module now uses the Microsoft Graph PowerShell SDK. + +### Automatic Dependency Checking +When you import the module, it automatically checks for missing dependencies and provides helpful guidance: + +```powershell +Import-Module AzureDataLakeManagement +# Output: WARNING: AzureDataLakeManagement module loaded with missing dependencies. +# Some functions may not work correctly until required modules are installed. +# Run 'Test-ModuleDependencies -AutoInstall' to install missing dependencies automatically. +``` + +### Dependency Management Functions + +#### Test-ModuleDependencies +Check which dependencies are available: +```powershell +Test-ModuleDependencies +``` + +Automatically install missing dependencies: +```powershell +Test-ModuleDependencies -AutoInstall +``` + +#### Install-ModuleDependencies +Install all or specific dependencies: +```powershell +Install-ModuleDependencies +Install-ModuleDependencies -Modules @('Az.Storage') +``` + +#### Manual Installation +You can also install dependencies manually: +```powershell +Install-Module -Name Az.Storage -Force +Install-Module -Name Microsoft.Graph.Applications -Force +Install-Module -Name Microsoft.Graph.Users -Force +Install-Module -Name Microsoft.Graph.Groups -Force +Install-Module -Name Microsoft.Graph.DirectoryObjects -Force +``` + +### Authentication +Connect to both Azure and Microsoft Graph: +```powershell +# Connect to Azure +Connect-AzAccount -UseDeviceAuthentication + +# Connect to Microsoft Graph (replaces Connect-AzureAD) +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" +``` + +### Improved Error Handling +Functions now provide clearer error messages when dependencies are missing, guiding users to install the required modules. + +*** + +## Version History: + +- 2025.11.3 - 11/7/2025 +Code updates to support "ShouldProcess" functionality on all functions that cause changes. Functions now support the -whatif and -confirm flags in areas that would change data. Eliminated other build warnings identified from Invoke-ScriptAnalyzer. + +- 2025.11.2 - 11/04/2025 +**BREAKING CHANGE**: Migrated from deprecated AzureAD module to Microsoft.Graph PowerShell SDK for PowerShell 7+ compatibility. + - Replaced `AzureAD` dependency with `Microsoft.Graph.Applications`, `Microsoft.Graph.Users`, `Microsoft.Graph.Groups` and `Microsoft.Graph.DirectoryObjects` + - Updated `Get-AADObjectId` to use Microsoft Graph cmdlets (`Get-MgUser`, `Get-MgGroup`, `Get-MgServicePrincipal`) + - Updated `Get-DataLakeFolderACL` to use `Get-MgDirectoryObject` + - Updated authentication from `Connect-AzureAD` to `Connect-MgGraph` + - All ACL-related functions now support PowerShell 7+ + - Updated documentation and examples + +- 2025.11.1 - 11/04/2025 +Issue 34 - Added support for Visual Studio Code - Dev Containers to improve development / testing for solution. Improved documentation. + +- 2025.1.1 - 01/09/2025 +Issue 27 - Added optional switch to set-DataLakeFolderACL and remove-DataLakeFolderACL functions to enable the user to not recursively apply permissions on children of the path specified. + +- 2024.1.1 - 01/09/2024 +Issue 22 - Fixed issue where a lack of Azure Permissions to Microsoft.Storage/storageAccounts/listKeys/action would cause failure to execute even with correct AzureAD permissions on objects. + +- 2023.12.3 - 12/01/2023 +Published via Github Actions to [PowershellGallery.com](https://www.powershellgallery.com/packages/AzureDataLakeManagement) + +- 2023.12.2 - 12/01/2023 +Removed - Github Action Publish testing + +- 2023.12.1 - 12/01/2023 +Function optimization and improved consistency of help content. + +- 2023.11.2 - 11/13/2023 +Added function to remove an ACL from a folder and all inherited children + diff --git a/Tests/DependencyManagement.Tests.ps1 b/Tests/DependencyManagement.Tests.ps1 new file mode 100644 index 0000000..331bf0a --- /dev/null +++ b/Tests/DependencyManagement.Tests.ps1 @@ -0,0 +1,100 @@ +#Requires -Modules Pester + +BeforeAll { + # Import the module + $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' + Import-Module $ModulePath -Force +} + +Describe 'AzureDataLakeManagement Dependency Management' { + + Context 'Test-ModuleDependencies Function' { + It 'Should export Test-ModuleDependencies function' { + Get-Command -Name 'Test-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Test-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'AutoInstall' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Az.Storage' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Applications' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Users' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.Groups' } + Mock Get-Module { $null } -ParameterFilter { $Name -eq 'Microsoft.Graph.DirectoryObjects' } + + $result = Test-ModuleDependencies -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Install-ModuleDependencies Function' { + It 'Should export Install-ModuleDependencies function' { + Get-Command -Name 'Install-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Install-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'Modules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + } + + Context 'Import-ModuleDependencies Function' { + It 'Should export Import-ModuleDependencies function' { + Get-Command -Name 'Import-ModuleDependencies' -Module 'AzureDataLakeManagement' | Should -Not -BeNullOrEmpty + } + + It 'Should have the correct parameters' { + $command = Get-Command -Name 'Import-ModuleDependencies' + $command.Parameters.Keys | Should -Contain 'RequiredModules' + $command.Parameters.Keys | Should -Contain 'Quiet' + } + + It 'Should return boolean value' { + Mock Get-Module { $null } -ParameterFilter { $ListAvailable -eq $true } + $result = Import-ModuleDependencies -RequiredModules @('NonExistentModule') -Quiet + $result | Should -BeOfType [System.Boolean] + } + } + + Context 'Module Manifest' { + It 'Should declare external module dependencies in manifest' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Az.Storage' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Applications' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Users' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.Groups' + $manifest.PrivateData.PSData.ExternalModuleDependencies | Should -Contain 'Microsoft.Graph.DirectoryObjects' + } + + It 'Should export dependency management functions' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.ExportedFunctions.Keys | Should -Contain 'Test-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Install-ModuleDependencies' + $manifest.ExportedFunctions.Keys | Should -Contain 'Import-ModuleDependencies' + } + } + + Context 'Integration with Existing Functions' { + It 'Should have updated Add-DataLakeFolder to use centralized dependency checking' { + $functionContent = (Get-Command Add-DataLakeFolder).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + + It 'Should have updated Set-DataLakeFolderACL to use centralized dependency checking' { + $functionContent = (Get-Command Set-DataLakeFolderACL).Definition + $functionContent | Should -Match 'Import-ModuleDependencies' + $functionContent | Should -Match 'Test-ModuleDependencies -AutoInstall' + } + } +} + +AfterAll { + # Clean up + Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue +} \ No newline at end of file diff --git a/Tests/DevContainer.Tests.ps1 b/Tests/DevContainer.Tests.ps1 new file mode 100644 index 0000000..4e14d81 --- /dev/null +++ b/Tests/DevContainer.Tests.ps1 @@ -0,0 +1,85 @@ +#Requires -Modules Pester + +BeforeAll { + $DevContainerPath = Join-Path $PSScriptRoot '..' '.devcontainer' 'devcontainer.json' +} + +Describe 'DevContainer Configuration' { + + Context 'File Existence' { + It 'Should have a devcontainer.json file' { + Test-Path $DevContainerPath | Should -BeTrue + } + } + + Context 'Configuration Content' { + BeforeAll { + # Read and parse the devcontainer.json file (JSONC - JSON with Comments) + $content = Get-Content $DevContainerPath -Raw + # Remove single-line comments more carefully (avoid URLs with //) + $lines = $content -split "`n" + $cleanedLines = @() + foreach ($line in $lines) { + # Skip comment-only lines + if ($line -match '^\s*//') { + continue + } + # Remove trailing comments but preserve URLs and quoted strings + # Match everything before comment outside of quotes + elseif ($line -match '^([^"]*"[^"]*"[^"]*)\s*//' -or $line -match '^(.*[^:])\s*//') { + # Keep content before comment, but be careful with URLs + if ($line -notmatch '"https?://') { + $cleanedLines += $matches[1] + } + else { + # Line contains URL, keep it as-is + $cleanedLines += $line + } + } + else { + # Keep line as-is + $cleanedLines += $line + } + } + $jsonContent = $cleanedLines -join "`n" + $config = $jsonContent | ConvertFrom-Json + } + + It 'Should have a name property' { + $config.name | Should -Not -BeNullOrEmpty + } + + It 'Should specify PowerShell feature' { + $config.features.PSObject.Properties.Name | Should -Contain 'ghcr.io/devcontainers/features/powershell:1' + } + + It 'Should have PowerShell extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'ms-vscode.powershell' + } + + It 'Should have Pester Test extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'pspester.pester-test' + } + + It 'Should have GitHub Copilot extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.copilot' + } + + It 'Should have GitHub Actions extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'github.vscode-github-actions' + } + + It 'Should have TODO Highlight extension configured' { + $config.customizations.vscode.extensions | Should -Contain 'jgclark.vscode-todo-highlight' + } + + It 'Should install PSScriptAnalyzer and Pester in postCreateCommand' { + $config.postCreateCommand | Should -Match 'PSScriptAnalyzer' + $config.postCreateCommand | Should -Match 'Pester' + } + + It 'Should set PowerShell as default terminal' { + $config.customizations.vscode.settings.'terminal.integrated.defaultProfile.linux' | Should -Be 'pwsh' + } + } +} diff --git a/Tests/MicrosoftGraph.Tests.ps1 b/Tests/MicrosoftGraph.Tests.ps1 new file mode 100644 index 0000000..424fd3e --- /dev/null +++ b/Tests/MicrosoftGraph.Tests.ps1 @@ -0,0 +1,136 @@ +#Requires -Modules Pester + +BeforeAll { + # Import the module + $ModulePath = Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psd1' + Import-Module $ModulePath -Force +} + +Describe 'AzureDataLakeManagement Microsoft.Graph Migration' { + + Context 'Module Dependencies' { + It 'Should not reference AzureAD module in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + # Should not have AzureAD as a standalone module reference (but AzureDataLakeManagement is OK) + $moduleContent | Should -Not -Match "@\('Az\.Storage',\s*'AzureAD'" + } + + It 'Should reference Microsoft.Graph.Users in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + $moduleContent | Should -Match "Microsoft\.Graph\.Users" + } + + It 'Should reference Microsoft.Graph.Groups in dependencies' { + $moduleContent = Get-Content (Join-Path $PSScriptRoot '..' 'AzureDataLakeManagement' 'AzureDataLakeManagement.psm1') -Raw + $moduleContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'Get-AADObjectId Function Migration' { + It 'Should use Get-MgUser instead of Get-AzureADUser' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgUser" + $functionContent | Should -Not -Match "Get-AzureADUser" + } + + It 'Should use Get-MgGroup instead of Get-AzureADGroup' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgGroup" + $functionContent | Should -Not -Match "Get-AzureADGroup" + } + + It 'Should use Get-MgServicePrincipal instead of Get-AzureADServicePrincipal' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Get-MgServicePrincipal" + $functionContent | Should -Not -Match "Get-AzureADServicePrincipal" + } + + It 'Should use .Id property instead of .ObjectId' { + $functionContent = (Get-Command Get-AADObjectId).Definition + # Check that we assign to objectId from .Id property + $functionContent | Should -Match '\$objectId\s*=\s*\$\w+\.Id' + } + + It 'Should reference Connect-MgGraph in help/comments' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "Connect-MgGraph" + } + } + + Context 'Get-DataLakeFolderACL Function Migration' { + It 'Should use Get-MgDirectoryObject instead of Get-AzureADObjectByObjectId' { + $functionContent = (Get-Command Get-DataLakeFolderACL).Definition + $functionContent | Should -Match "Get-MgDirectoryObject" + $functionContent | Should -Not -Match "Get-AzureADObjectByObjectId" + } + + It 'Should require Microsoft.Graph modules' { + $functionContent = (Get-Command Get-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'ACL Functions Dependencies' { + It 'Set-DataLakeFolderACL should require Microsoft.Graph modules' { + $functionContent = (Get-Command Set-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + + It 'Remove-DataLakeFolderACL should require Microsoft.Graph modules' { + $functionContent = (Get-Command Remove-DataLakeFolderACL).Definition + $functionContent | Should -Match "Microsoft\.Graph\.Users" + $functionContent | Should -Match "Microsoft\.Graph\.Groups" + } + } + + Context 'Documentation Updates' { + It 'README should mention Microsoft.Graph modules' { + $readmeContent = Get-Content (Join-Path $PSScriptRoot '..' 'README.md') -Raw + $readmeContent | Should -Match "Microsoft\.Graph" + } + + It 'README should mention Connect-MgGraph' { + $readmeContent = Get-Content (Join-Path $PSScriptRoot '..' 'README.md') -Raw + $readmeContent | Should -Match "Connect-MgGraph" + } + + It 'example.ps1 should use Connect-MgGraph' { + $exampleContent = Get-Content (Join-Path $PSScriptRoot '..' 'example.ps1') -Raw + $exampleContent | Should -Match "Connect-MgGraph" + # The comment is OK, but we should not have an actual Connect-AzureAD command + $exampleContent | Should -Not -Match "^Connect-AzureAD" -Because "Connect-AzureAD should be commented out or replaced" + } + + It 'Module version should be updated to 2025.11.2 or higher' { + $manifest = Test-ModuleManifest -Path $ModulePath + $manifest.Version.Major | Should -BeGreaterOrEqual 2025 + $manifest.Version.Minor | Should -BeGreaterOrEqual 11 + $manifest.Version.Build | Should -BeGreaterOrEqual 2 + } + } + + Context 'Backward Compatibility' { + It 'Get-AADObjectId should still return ObjectId property' { + # This ensures backward compatibility - the property name should remain ObjectId + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "ObjectId\s*=" + } + + It 'Get-AADObjectId should still return ObjectType property' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "ObjectType\s*=" + } + + It 'Get-AADObjectId should still return DisplayName property' { + $functionContent = (Get-Command Get-AADObjectId).Definition + $functionContent | Should -Match "DisplayName\s*=" + } + } +} + +AfterAll { + # Clean up + Remove-Module 'AzureDataLakeManagement' -Force -ErrorAction SilentlyContinue +} diff --git a/example-dependency-management.ps1 b/example-dependency-management.ps1 new file mode 100644 index 0000000..8b6bbb0 --- /dev/null +++ b/example-dependency-management.ps1 @@ -0,0 +1,31 @@ +# Example script demonstrating improved dependency management in AzureDataLakeManagement + +# Import the module - it will now provide helpful feedback about missing dependencies +Import-Module .\AzureDataLakeManagement\AzureDataLakeManagement.psd1 + +# Check what dependencies are missing +Test-ModuleDependencies + +# To install missing dependencies automatically, you can run: +# Test-ModuleDependencies -AutoInstall + +# Or install them manually: +# Install-Module -Name Az.Storage -Force +# Install-Module -Name Microsoft.Graph.Applications -Force +# Install-Module -Name Microsoft.Graph.Users -Force +# Install-Module -Name Microsoft.Graph.Groups -Force +# Install-Module -Name Microsoft.Graph.DirectoryObjects -Force + +# The module will now provide better error messages when functions are called without required dependencies +# For example: +# Add-DataLakeFolder -SubscriptionName 'test' -ResourceGroupName 'test' -StorageAccountName 'test' -ContainerName 'test' -FolderPath 'test' + +Write-Host " +Dependency Management Features: +- Test-ModuleDependencies: Check which dependencies are available +- Test-ModuleDependencies -AutoInstall: Automatically install missing dependencies +- Install-ModuleDependencies: Install specific dependencies +- Import-ModuleDependencies: Import dependencies with better error handling + +The module will automatically check dependencies when imported and provide helpful guidance. +" -ForegroundColor Green \ No newline at end of file diff --git a/example.ps1 b/example.ps1 index f17ff80..13ed58f 100644 --- a/example.ps1 +++ b/example.ps1 @@ -1,46 +1,53 @@ -import-module AzureDatalakeManagement - -Connect-AzAccount -Connect-AzureAd - -$subName = '' -$rgName = 'resourceGroup01' -$storageAccountName = 'storage01' -$containerName = 'bronze' - -#create basic dataset folder structure -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleB\test1\subA\subB' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleC' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset2' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset3' -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' - -#add duplicate folder in error -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' - -#add duplicate folder in error & show error -add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -ErrorIfFolderExists - -#Set user acl at the root of the dataset but don't set default scope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity 'sam@contoso.com' -accessControlType Read - -#Set user acl at the root of the dataset and set default scope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity 'sam@contoso.com' -accessControlType Read -IncludeDefaultScope - -#Set user acl at the root of the dataset and configure the container for access by the user -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' -accessControlType Write -IncludeDefaultScope -SetContainerACL - -#set service acl at sub folder -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB\test1' -Identity '' -accessControlType Write -IncludeDefaultScope - -#set group acl at root of dataset -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity "" -accessControlType Read -IncludeDefaultScope -set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity "" -accessControlType Read -IncludeDefaultScope - -#remove folder from specified container -remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' - -#move sub folder from one dataset to another -move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'dataset1\sampleB' -DestinationContainerName $containerName -destinationFolderPath 'dataset2\sampleb' - +import-module AzureDatalakeManagement + +Connect-AzAccount -UseDeviceAuthentication +# Connect to Microsoft Graph (replaces Connect-AzureAD for PowerShell 7+ compatibility) +Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +$subName = '' +$rgName = 'resourceGroup01' +$storageAccountName = 'storage01' +$containerName = 'bronze' + +#create basic dataset folder structure +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleB\test1\subA\subB' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleC' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset2' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset3' +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' + +#add duplicate folder in error +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' + +#add duplicate folder in error & show error +add-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset1\sampleA' -ErrorIfFolderExists + +#Set user acl at the root of the dataset but don't set default scope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity 'sam@contoso.com' -accessControlType Read + +#Set user acl at the root of the dataset and set default scope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity 'sam@contoso.com' -accessControlType Read -IncludeDefaultScope + +#Set user acl at the root of the dataset and configure the container for access by the user +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' -accessControlType Write -IncludeDefaultScope -SetContainerACL + +#set service acl at sub folder +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB\test1' -Identity '' -accessControlType Write -IncludeDefaultScope + +#set group acl at root of dataset +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1' -Identity '' -accessControlType Read -IncludeDefaultScope +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset2' -Identity '' -accessControlType Read -IncludeDefaultScope + +#set acl with no recursion +set-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset1\sampleB' -Identity '' -accessControlType Write -IncludeDefaultScope -DoNotApplyACLRecursively + +#remove ACL from folder +remove-DataLakeFolderACL -SubscriptionName $subName -ResourceGroupName $rgName -StorageAccountName $storageAccountName -ContainerName $containerName -folderPath 'dataset3' -Identity 'bob@contoso.com' + +#remove folder from specified container +remove-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -containerName $containerName -folderPath 'dataset4' + +#move sub folder from one dataset to another +move-DataLakeFolder -SubscriptionName $subName -resourceGroup $rgName -storageAccountName $storageAccountName -SourceContainerName $containerName -sourceFolderPath 'dataset1\sampleB' -DestinationContainerName $containerName -destinationFolderPath 'dataset2\sampleb' + diff --git a/publish.ps1 b/publish.ps1 index ef02346..8b8db33 100644 --- a/publish.ps1 +++ b/publish.ps1 @@ -1,7 +1,7 @@ - -$parameters = @{ - NuGetApiKey = $env:PSGalleryKey - Path = "$PSScriptRoot\AzureDataLakeManagement" -} -Publish-Module @parameters - + +$parameters = @{ + NuGetApiKey = $env:PSGalleryKey + Path = "$PSScriptRoot\AzureDataLakeManagement" +} +Publish-Module @parameters +