From f39189a8dc0290e72d96974249f5969063bea269 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 14:49:16 -0800 Subject: [PATCH 01/90] Added test script for csv --- .github/workflows/test_script.ps1 | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/test_script.ps1 diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 new file mode 100644 index 0000000..092cb6e --- /dev/null +++ b/.github/workflows/test_script.ps1 @@ -0,0 +1,60 @@ +$csvPath = ".github\workflows\tracking_table.csv" +$githubAuthToken = args[0] + +function CreateAndPopulateCsv { + if (!(Test-Path $csvPath)) { + Add-Content -Path $csvPath -Value "FileName, CommitSha" + Write-Output "Created csv file." + } + $shaTable = GetCommitShaTable + #write all filename, sha to csv file + $shaTable.GetEnumerator() | ForEach-Object { + "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath + } +} + +function GetCommitShaTable { + $Header = @{ + "authorization" = "Bearer $githubAuthToken" + } + #get branch sha and use it to get tree with all commit shas and files + $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header + $shaTable = @{} + $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($_.path, $_.sha)}} + return $shaTable +} + +#we need token provided by workflow run to push file, not installationtoken, will test later +function PushCsvToRepo { + #if exists, we need sha of csv file before pushing updated file. If new, no need + $Header = @{ + "authorization" = "Bearer $githubAuthToken" + } + $path = ".github/workflows/tracking_table.csv" + Write-Output $path + $createFileUrl = "https://api.github.com/repos/aaroncorreya/SmartTrackingScriptDev/contents/$path" + $content = Get-Content -Path $csvPath | Out-String + $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) + Write-Output $encodedContent + $body = @{ + message = "trackingTable.csv created." + content = $encodedContent + branch = $branchName + } + + $Parameters = @{ + Method = "PUT" + Uri = $createFileUrl + Headers = $Header + Body = $body | ConvertTo-Json + } + #Commit csv file + Invoke-RestMethod @Parameters +} + +function main { + CreateAndPopulateCsv + PushCsvToRepo +} \ No newline at end of file From 54d2661824b8bfac434366b5bdadfab6b6b746da Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:02:47 -0800 Subject: [PATCH 02/90] Added environment variables --- .github/workflows/test_script.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 092cb6e..101df1c 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -1,5 +1,7 @@ $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = args[0] +$githubRepository = $Env:Repository +$branchName = $Env:Branch function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { @@ -57,4 +59,6 @@ function PushCsvToRepo { function main { CreateAndPopulateCsv PushCsvToRepo -} \ No newline at end of file +} + +main \ No newline at end of file From 88ea1d87f2cf956150193ec59c79be25fa218f73 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:16:31 -0800 Subject: [PATCH 03/90] Updated workflow and test script --- .github/workflows/testWorkflow.yml | 23 +++++++++++++++++------ .github/workflows/test_script.ps1 | 6 +++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 78dc3ec..2474968 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -4,7 +4,7 @@ name: CI # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the main branch + # Triggers the workflow on push or pull request events push: branches: [ main ] pull_request: @@ -18,7 +18,10 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-latest + runs-on: windows-latest + env: + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + repository: ${{GITHUB_REPOSITORY}} # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -26,11 +29,19 @@ jobs: - uses: actions/checkout@v2 # Runs a single command using the runners shell - - name: Run a one-line script - run: echo Hello, world! + - name: echo-default-env-variables + run: | + echo "Home: ${HOME}" + echo "GITHUB_WORKFLOW: ${GITHUB_WORKFLOW}" + echo "GITHUB_ACTIONS: ${GITHUB_ACTIONS}" + echo "GITHUB_ACTOR: ${GITHUB_ACTOR}" + echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" + echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}" + echo "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}" + echo "GITHUB_SHA: ${GITHUB_SHA}" + echo "GITHUB_REF: ${GITHUB_REF}" # Runs a set of commands using the runners shell - name: Run a multi-line script run: | - echo Add other actions to build, - echo test, and deploy your project. + ./.github/workflows/test_script.ps1 diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 101df1c..6022106 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -1,7 +1,7 @@ $csvPath = ".github\workflows\tracking_table.csv" -$githubAuthToken = args[0] -$githubRepository = $Env:Repository -$branchName = $Env:Branch +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:repository +$branchName = "testScript" function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { From b3e3000fb140901bbdb67ef11bde1fe22049678f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:20:42 -0800 Subject: [PATCH 04/90] Branch testScript: Added run script step --- .github/workflows/testWorkflow.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 2474968..5d717cc 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -28,19 +28,6 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - # Runs a single command using the runners shell - - name: echo-default-env-variables - run: | - echo "Home: ${HOME}" - echo "GITHUB_WORKFLOW: ${GITHUB_WORKFLOW}" - echo "GITHUB_ACTIONS: ${GITHUB_ACTIONS}" - echo "GITHUB_ACTOR: ${GITHUB_ACTOR}" - echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}" - echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}" - echo "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}" - echo "GITHUB_SHA: ${GITHUB_SHA}" - echo "GITHUB_REF: ${GITHUB_REF}" - # Runs a set of commands using the runners shell - name: Run a multi-line script run: | From 2abf8d93ef379b696d71c8fa2c7357b3177af89e Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:21:56 -0800 Subject: [PATCH 05/90] edited workflow to run on any push --- .github/workflows/testWorkflow.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 5d717cc..40713e5 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -3,12 +3,7 @@ name: CI # Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events - push: - branches: [ main ] - pull_request: - types: [assigned, opened, synchronize, reopened] +on: [push, pull_request] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 9b1e4ce56324763b3aa2ba10ceb5c858729786e4 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:24:53 -0800 Subject: [PATCH 06/90] reverted workflow due to error --- .github/workflows/testWorkflow.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 40713e5..42087a8 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -3,7 +3,12 @@ name: CI # Controls when the workflow will run -on: [push, pull_request] +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: * + pull_request: + types: [assigned, opened, synchronize, reopened] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -14,16 +19,18 @@ jobs: build: # The type of runner that the job will run on runs-on: windows-latest - env: - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - repository: ${{GITHUB_REPOSITORY}} # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + # Runs a single command using the runners shell + - name: Run a one-line script + run: echo Hello, world! + # Runs a set of commands using the runners shell - name: Run a multi-line script run: | - ./.github/workflows/test_script.ps1 + echo Add other actions to build, + echo test, and deploy your project. \ No newline at end of file From 0acf0c539817de37b02541121931a0679d25e5a9 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:25:55 -0800 Subject: [PATCH 07/90] Edited workflow to run on any branch --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 42087a8..bee4450 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -6,7 +6,7 @@ name: CI on: # Triggers the workflow on push or pull request events but only for the main branch push: - branches: * + branches: "**" pull_request: types: [assigned, opened, synchronize, reopened] From de1d06c236d69edc3aa3dff3ad4d20a685950336 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:27:27 -0800 Subject: [PATCH 08/90] Added env vars to workflow --- .github/workflows/testWorkflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index bee4450..36271dd 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -19,6 +19,9 @@ jobs: build: # The type of runner that the job will run on runs-on: windows-latest + env: + repository: ${GITHUB_REPOSITORY} + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} # Steps represent a sequence of tasks that will be executed as part of the job steps: From 049affc3b884a51524977441bc469d31ac43131c Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:30:55 -0800 Subject: [PATCH 09/90] Added repository print sanity check --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 36271dd..81a48b3 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo Hello, world! + run: echo "Repository is ${GITHUB_REPOSITORY}" # Runs a set of commands using the runners shell - name: Run a multi-line script From 5a0fec579d69da48b6428e251f825e7df18eb680 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:32:31 -0800 Subject: [PATCH 10/90] Retried repository print sanity check --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 81a48b3..a1b672f 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo "Repository is ${GITHUB_REPOSITORY}" + run: echo "${repository}" # Runs a set of commands using the runners shell - name: Run a multi-line script From 7b32cf7e1a72c64a0d9db25887906d9aa764dd48 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:35:21 -0800 Subject: [PATCH 11/90] Retried repository print sanity check 2 --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index a1b672f..a341398 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo "${repository}" + run: echo $env.repository # Runs a set of commands using the runners shell - name: Run a multi-line script From 347f6a3205fb549a4cce787dbcbad6c57174e0d9 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:37:36 -0800 Subject: [PATCH 12/90] Retried repository print sanity check 3 --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index a341398..82ddefa 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo $env.repository + run: echo ${{env.repository}} # Runs a set of commands using the runners shell - name: Run a multi-line script From 7de440107a122f4adbd3c8b4b29b04c9367e3fba Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:41:10 -0800 Subject: [PATCH 13/90] Reworked env vars --- .github/workflows/testWorkflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 82ddefa..5ef812c 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,6 +30,9 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script + env: + repository: ${GITHUB_REPOSITORY} + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} run: echo ${{env.repository}} # Runs a set of commands using the runners shell From 5d0883f1eb6dd2f4b057be8b1fef1ecdb0de36a5 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:45:02 -0800 Subject: [PATCH 14/90] Changed it back --- .github/workflows/testWorkflow.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 5ef812c..82ddefa 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,9 +30,6 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - env: - repository: ${GITHUB_REPOSITORY} - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} run: echo ${{env.repository}} # Runs a set of commands using the runners shell From b42c05b81b5ea93b81b96e3796e447eb291435cc Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:46:34 -0800 Subject: [PATCH 15/90] Removed brackets --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 82ddefa..a341398 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo ${{env.repository}} + run: echo $env.repository # Runs a set of commands using the runners shell - name: Run a multi-line script From f4eb796ff95c73c8a36762d8b780892067e63a3f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:48:28 -0800 Subject: [PATCH 16/90] Removed brackets 2 --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index a341398..7b63dea 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo $env.repository + run: echo "$env.repository" # Runs a set of commands using the runners shell - name: Run a multi-line script From 15b6914a95d06159898609ae5097ab8406d9e3bb Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:49:43 -0800 Subject: [PATCH 17/90] Added run script --- .github/workflows/testWorkflow.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 7b63dea..5023848 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -35,5 +35,4 @@ jobs: # Runs a set of commands using the runners shell - name: Run a multi-line script run: | - echo Add other actions to build, - echo test, and deploy your project. \ No newline at end of file + ./.github/workflows/smart_script.ps1 \ No newline at end of file From 28c840f9261c79e3990d102f5240e74d0bf4a50f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:50:40 -0800 Subject: [PATCH 18/90] Added run test script --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 5023848..4b32cfd 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -35,4 +35,4 @@ jobs: # Runs a set of commands using the runners shell - name: Run a multi-line script run: | - ./.github/workflows/smart_script.ps1 \ No newline at end of file + ./.github/workflows/test_script.ps1 \ No newline at end of file From accfc6a9969f143da0b8b0e7bc0b83eb157df095 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:53:18 -0800 Subject: [PATCH 19/90] Added run test script print repo --- .github/workflows/test_script.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 6022106..a973ce6 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -19,6 +19,7 @@ function GetCommitShaTable { $Header = @{ "authorization" = "Bearer $githubAuthToken" } + Write-Output $githubRepository #get branch sha and use it to get tree with all commit shas and files $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" From c391d9955c44eed4f4bc58a43d685194ffc22490 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:55:29 -0800 Subject: [PATCH 20/90] Added run test script print repo --- .github/workflows/test_script.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index a973ce6..c99e4b2 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -19,7 +19,6 @@ function GetCommitShaTable { $Header = @{ "authorization" = "Bearer $githubAuthToken" } - Write-Output $githubRepository #get branch sha and use it to get tree with all commit shas and files $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" @@ -58,8 +57,9 @@ function PushCsvToRepo { } function main { - CreateAndPopulateCsv - PushCsvToRepo + #CreateAndPopulateCsv + #PushCsvToRepo + Write-Output $githubRepository } main \ No newline at end of file From 671433072533d0b20058b03f60f0f104e7df805a Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 15:56:40 -0800 Subject: [PATCH 21/90] Removed unnecesary line --- .github/workflows/testWorkflow.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 4b32cfd..86829aa 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -25,9 +25,6 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - # Runs a single command using the runners shell - name: Run a one-line script run: echo "$env.repository" From 92058d62cda4e3b6c6f22be9982248e3b2aafa6a Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:01:10 -0800 Subject: [PATCH 22/90] Edited vars --- .github/workflows/testWorkflow.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 86829aa..6ee4ed4 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -20,16 +20,19 @@ jobs: # The type of runner that the job will run on runs-on: windows-latest env: - repository: ${GITHUB_REPOSITORY} + repository: ${{ GITHUB_REPOSITORY }} githubAuthToken: ${{ secrets.GITHUB_TOKEN }} # Steps represent a sequence of tasks that will be executed as part of the job steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + # Runs a single command using the runners shell - name: Run a one-line script run: echo "$env.repository" # Runs a set of commands using the runners shell - - name: Run a multi-line script + - name: Run script run: | ./.github/workflows/test_script.ps1 \ No newline at end of file From 534d8cc6fc6f0858e19fab79b614e3ebd1fae992 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:01:47 -0800 Subject: [PATCH 23/90] Edited vars --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 6ee4ed4..da2fbb4 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -20,7 +20,7 @@ jobs: # The type of runner that the job will run on runs-on: windows-latest env: - repository: ${{ GITHUB_REPOSITORY }} + repository: ${GITHUB_REPOSITORY} githubAuthToken: ${{ secrets.GITHUB_TOKEN }} # Steps represent a sequence of tasks that will be executed as part of the job From 3e6cb0c0cf644da60481d9d2df781607c786e824 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:05:05 -0800 Subject: [PATCH 24/90] Edited vars --- .github/workflows/testWorkflow.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index da2fbb4..6bd4887 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -19,14 +19,13 @@ jobs: build: # The type of runner that the job will run on runs-on: windows-latest - env: - repository: ${GITHUB_REPOSITORY} - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + env: + repository: ${GITHUB_REPOSITORY} + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} # Runs a single command using the runners shell - name: Run a one-line script From 0ea88eb3b54954dab71c2b5db4909d69a1057b92 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:08:06 -0800 Subject: [PATCH 25/90] Edited vars --- .github/workflows/testWorkflow.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index 6bd4887..a9bd31f 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -19,13 +19,14 @@ jobs: build: # The type of runner that the job will run on runs-on: windows-latest + env: + repository: aaroncorreya/SmartTrackingScriptDev + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - env: - repository: ${GITHUB_REPOSITORY} - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} # Runs a single command using the runners shell - name: Run a one-line script From da8b87a54e19db73d241287b2aa346a9daab2e9d Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:09:46 -0800 Subject: [PATCH 26/90] Edited env. --- .github/workflows/testWorkflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index a9bd31f..a8b3c66 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -30,7 +30,7 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo "$env.repository" + run: echo "${{ env.repository }}" # Runs a set of commands using the runners shell - name: Run script From 30270e2eeed5432f3c4a7d335b8aa0769eb130f3 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:13:10 -0800 Subject: [PATCH 27/90] Changed repo env in script --- .github/workflows/test_script.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index c99e4b2..3a47644 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -2,6 +2,7 @@ $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:repository $branchName = "testScript" +$repoTest = $Env:GITHUB_REPOSITORY function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { @@ -60,6 +61,8 @@ function main { #CreateAndPopulateCsv #PushCsvToRepo Write-Output $githubRepository + Write-Output $repoTest + } main \ No newline at end of file From 65ce7aa2be3884ec394183798709cf659f5bd7fb Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Thu, 16 Dec 2021 16:15:27 -0800 Subject: [PATCH 28/90] Script now runs csv functions --- .github/workflows/test_script.ps1 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 3a47644..fc7b455 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -1,8 +1,7 @@ $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $Env:githubAuthToken -$githubRepository = $Env:repository +$githubRepository = $Env:GITHUB_REPOSITORY $branchName = "testScript" -$repoTest = $Env:GITHUB_REPOSITORY function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { @@ -58,10 +57,9 @@ function PushCsvToRepo { } function main { - #CreateAndPopulateCsv - #PushCsvToRepo Write-Output $githubRepository - Write-Output $repoTest + CreateAndPopulateCsv + PushCsvToRepo } From a90ba6208e0a84802d309a7f6d035e6ff283c64c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 00:15:58 +0000 Subject: [PATCH 29/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..08425c9 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +Deployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From e4070dea22e96699fef36304476e8f3694e5987f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 10:24:13 -0800 Subject: [PATCH 30/90] Changed branch name variable to env --- .github/workflows/smart_script.ps1 | 7 +++++-- .github/workflows/test_script.ps1 | 3 ++- .github/workflows/tracking_table.csv | 3 --- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index cdaa6b2..f998f1d 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -302,5 +302,8 @@ function main() { } #main -CreateAndPopulateCsv -#PushCsvToRepo \ No newline at end of file +#CreateAndPopulateCsv +#PushCsvToRepo +$testStr = "refs/heads/main" +$testStr =$testStr.Replace("refs/heads/", "") +Write-Output $testStr \ No newline at end of file diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index fc7b455..4c991b2 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -1,7 +1,8 @@ $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY -$branchName = "testScript" +$refName = $Env:GITHUB_REF +$branchName = $refName.Replace("refs/heads/", "") function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 08425c9..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -Deployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 2eef122075a7acaba2b613874c8a6c80e1487dfc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 18:24:37 +0000 Subject: [PATCH 31/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d51bc8c --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +Deployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From ef8ea2117a1ba5d75b8232360f654ef61e40619f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 12:08:04 -0800 Subject: [PATCH 32/90] Checking paths in dict and in filesystem --- .github/workflows/smart_script.ps1 | 36 +++++++++++++++++------------- .github/workflows/test_script.ps1 | 8 ++++++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index f998f1d..b2ed1b3 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -56,7 +56,7 @@ function GetCommitShaTable { $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header $shaTable = @{} - $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($_.path, $_.sha)}} + $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($Directory + $_.path, $_.sha)}} return $shaTable } @@ -88,6 +88,15 @@ function PushCsvToRepo { Invoke-RestMethod @Parameters } +function ReadCsvToTable { + $mytable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $mytable) + { + $HashTable[$r.FileName]=$r.CommitSha + } + return $HashTable +} function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { $maxLoginRetries = 3 @@ -284,26 +293,23 @@ function main() { if ((-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true")) { Write-Output "Starting Full Deployment for Files in path: $Directory" CreateAndPopulateCsv - #TODO: push csv to repo - FullDeployment + #PushCsvToRepo + #FullDeployment } #else run smart tracking else { - #Import-Csv -Path $csvPath - $mytable = Import-Csv -Path $csvPath - $HashTable=@{} - foreach($r in $mytable) - { - $HashTable[$r.FileName]=$r.CommitSha - } - Write-Output $HashTable - } + $localCsvTable = ReadCsvToTable + $remoteShaTable = GetCommitShaTable + Get-ChildItem -Path $Directory -Recurse -Filter *.json | + ForEach-Object { + $path = $_.FullName + Write-Output $path + } + } } #main #CreateAndPopulateCsv #PushCsvToRepo -$testStr = "refs/heads/main" -$testStr =$testStr.Replace("refs/heads/", "") -Write-Output $testStr \ No newline at end of file +GetCommitShaTable diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 4c991b2..7d0e851 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -10,6 +10,7 @@ function CreateAndPopulateCsv { Write-Output "Created csv file." } $shaTable = GetCommitShaTable + Write-Output $shaTable #write all filename, sha to csv file $shaTable.GetEnumerator() | ForEach-Object { "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath @@ -25,7 +26,7 @@ function GetCommitShaTable { $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header $shaTable = @{} - $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($_.path, $_.sha)}} + $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($githubRepository + $_.path, $_.sha)}} return $shaTable } @@ -62,6 +63,11 @@ function main { CreateAndPopulateCsv PushCsvToRepo + Get-ChildItem -Path $Directory -Recurse -Filter *.json | + ForEach-Object { + $path = $_.FullName + Write-Output $path + } } main \ No newline at end of file From df9cfe23d7aeb96a4780f65a7dbd6bcaf2b1363d Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 12:08:34 -0800 Subject: [PATCH 33/90] Removed csv table and checking paths --- .github/workflows/tracking_table.csv | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d51bc8c..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -Deployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 55a4ddc481463532156c5135a037372e51ca9acd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 20:09:04 +0000 Subject: [PATCH 34/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..5939524 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 77584634471ef870f3d65f7c8760abb88f3dc688 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 12:13:15 -0800 Subject: [PATCH 35/90] Added workspace print --- .github/workflows/test_script.ps1 | 2 ++ .github/workflows/tracking_table.csv | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 7d0e851..d6593c9 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -3,6 +3,7 @@ $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY $refName = $Env:GITHUB_REF $branchName = $refName.Replace("refs/heads/", "") +$workspace = $Env:GITHUB_WORKSPACE function CreateAndPopulateCsv { if (!(Test-Path $csvPath)) { @@ -68,6 +69,7 @@ function main { $path = $_.FullName Write-Output $path } + Write-Output $workspace } main \ No newline at end of file diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 5939524..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 7651ed7fb5aa73c8d2bd392a5e44a104ec416164 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 20:13:37 +0000 Subject: [PATCH 36/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..5939524 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 367e89a4b521174507df33a703b4cb0e20ca7d4d Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 12:29:36 -0800 Subject: [PATCH 37/90] Reworked file path in dictionary --- .github/workflows/smart_script.ps1 | 9 ++++++++- .github/workflows/test_script.ps1 | 9 ++++++++- .github/workflows/tracking_table.csv | 3 --- 3 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index b2ed1b3..db4ffcc 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -56,7 +56,14 @@ function GetCommitShaTable { $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header $shaTable = @{} - $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($Directory + $_.path, $_.sha)}} + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-5) -eq ".json") + { + #needs to be $workplace in real implementation + $truePath = ($Directory + "\" + $_.path).Replace("/", "\") + $shaTable.Add($truePath, $_.sha) + } + } return $shaTable } diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index d6593c9..3787c73 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -27,7 +27,14 @@ function GetCommitShaTable { $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header $shaTable = @{} - $getTreeResponse.tree | ForEach-Object -Process {if ($_.path.Substring($_.path.Length-5) -eq ".json") {$shaTable.Add($githubRepository + $_.path, $_.sha)}} + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-5) -eq ".json") + { + #needs to be $workplace in real implementation + $truePath = ($Directory + "\" + $_.path).Replace("/", "\") + $shaTable.Add($truePath, $_.sha) + } + } return $shaTable } diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 5939524..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -aaroncorreya/SmartTrackingScriptDevDeployments/Parsers/ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From cd3679f11facbe55c8f3c4b7556747c166587c5e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 20:30:05 +0000 Subject: [PATCH 38/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..eed4410 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 7924b9b12fef7722d264f9f4cf67a24316992d42 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 17 Dec 2021 12:31:16 -0800 Subject: [PATCH 39/90] Changed it to workspace variable --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 3787c73..33144d9 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -31,7 +31,7 @@ function GetCommitShaTable { if ($_.path.Substring($_.path.Length-5) -eq ".json") { #needs to be $workplace in real implementation - $truePath = ($Directory + "\" + $_.path).Replace("/", "\") + $truePath = ($workspace + "\" + $_.path).Replace("/", "\") $shaTable.Add($truePath, $_.sha) } } diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index eed4410..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 9212e2daa676677a035e1273623456a3d310f49e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 20:31:42 +0000 Subject: [PATCH 40/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 120c0f823cfb2ac1ddcc954207d1b01e4ed6f237 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:09:40 -0800 Subject: [PATCH 41/90] Modified create csv function in test script --- .github/workflows/smart_script.ps1 | 158 ++++++++++++++++++++--------- .github/workflows/test_script.ps1 | 16 ++- 2 files changed, 124 insertions(+), 50 deletions(-) diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index db4ffcc..57b1d22 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -35,13 +35,18 @@ $resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping $MaxRetries = 3 $secondsBetweenAttempts = 5 -function CreateAndPopulateCsv { - if (!(Test-Path $csvPath)) { - Add-Content -Path $csvPath -Value "FileName, CommitSha" - Write-Output "Created csv file." - } - $shaTable = GetCommitShaTable - #write all filename, sha to csv file +function CreateCsv() { + if (Test-Path $csvPath) { + Clear-Content -Path $csvPath + } + Add-Content -Path $csvPath -Value "FileName, CommitSha" +} + +function WriteTableToCsv($shaTable) { + if (Test-Path $csvPath) { + Clear-Content -Path $csvPath + } + Add-Content -Path $csvPath -Value "FileName, CommitSha" $shaTable.GetEnumerator() | ForEach-Object { "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath } @@ -67,7 +72,6 @@ function GetCommitShaTable { return $shaTable } -#we need token provided by workflow run to push file, not installationtoken, will test later function PushCsvToRepo { #if exists, we need sha of csv file before pushing updated file. If new, no need $Header = @{ @@ -91,7 +95,6 @@ function PushCsvToRepo { Headers = $Header Body = $body | ConvertTo-Json } - #Commit csv file Invoke-RestMethod @Parameters } @@ -178,7 +181,6 @@ function IsRetryable($deploymentName) { return $false } } - function IsValidResourceType($template) { $isAllowedResources = $true $template.resources | ForEach-Object { @@ -246,8 +248,16 @@ function GenerateDeploymentName() { return "Sentinel_Deployment_$randomId" } -#modify this function to handle both manual deployment and smart tracking -function FullDeployment { +function CheckFullDeployment() { + $flag = $false + if ((-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true")) { + $flag = $true + } + return $flag +} + +function Deployment($fullDeploymentFlag, $localCsvTable, $remoteShaTable) { + Write-Output "Starting Deployment for Files in path: $Directory" if (Test-Path -Path $Directory) { $totalFiles = 0; @@ -255,27 +265,33 @@ function FullDeployment { Get-ChildItem -Path $Directory -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName - try { - #if manual deployment run this code - $totalFiles ++ - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + #put this into try catch + try { if (-not (IsValidResourceType $templateObject)) { Write-Output "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." return - } - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $_.FullName $deploymentName $templateObject - if (-not $isSuccess) - { - $totalFailed++ - } - #else run + } } - catch { - $totalFailed++ + catch { Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace + } + + if ($fullDeploymentFlag) { + $result = FullDeployment $path $templateObject + # if (-not $result.isSuccess) {$totalFailed++} + } + else { + $result = SmartDeployment $localCsvTable $remoteShaTable $path $templateObject + $localCsvTable = $result.csvTable + } + #convert to global variables + if ($result.isSuccess -eq $false) { + $totalFailed++ + } + if (-not $result.skip) { + $totalFiles++ } } if ($totalFiles -gt 0 -and $totalFailed -gt 0) @@ -283,6 +299,7 @@ function FullDeployment { $err = "$totalFailed of $totalFiles deployments failed." Throw $err } + return $localCsvTable } else { @@ -290,33 +307,78 @@ function FullDeployment { } } +function FullDeployment($path, $templateObject) { + try { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $deploymentName $templateObject + $result = @{ + skip = $false + isSuccess = $isSuccess + } + return $result + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function SmartDeployment($localCsvTable, $remoteShaTable, $path, $templateObject) { + try { + $skip = $false + $existingSha = $localCsvTable[$path] + $remoteSha = $remoteShaTable[$path] + if ((!$existingSha) -or ($existingSha -ne $remoteSha)) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $deploymentName $templateObject + $localCsvTable[$path] = $remoteSha + } + else { + $skip = $true + $isSuccess = $null + } + $result = @{ + skip = $skip + isSuccess = $isSuccess + csvTable = $localCsvTable + } + return $result + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + function main() { - # if ($CloudEnv -ne 'AzureCloud') - # { - # Write-Output "Attempting Sign In to Azure Cloud" - # ConnectAzCloud - # } + if ($CloudEnv -ne 'AzureCloud') + { + Write-Output "Attempting Sign In to Azure Cloud" + ConnectAzCloud + } - if ((-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true")) { - Write-Output "Starting Full Deployment for Files in path: $Directory" - CreateAndPopulateCsv - #PushCsvToRepo - #FullDeployment + $fullDeploymentFlag = CheckFullDeployment + Write-Output $fullDeploymentFlag + + if (-not (Test-Path $csvPath)) { + Write-Output "Creating csv and conducting full deployment." + $remoteShaTable = GetCommitShaTable + WriteTableToCsv($remoteShaTable) + # PushCsvToRepo + Deployment $fullDeploymentFlag $null $null } - #else run smart tracking else { $localCsvTable = ReadCsvToTable $remoteShaTable = GetCommitShaTable - - Get-ChildItem -Path $Directory -Recurse -Filter *.json | - ForEach-Object { - $path = $_.FullName - Write-Output $path - } + Write-Output "Local Csv Table" + Write-Output $localCsvTable + Write-Output "Remote Csv Table" + Write-Output $remoteShaTable + $updatedCsvTable = Deployment $fullDeploymentFlag $localCsvTable $remoteShaTable + WriteTableToCsv($updatedCsvTable) + #PushCsvToRepo } } -#main -#CreateAndPopulateCsv -#PushCsvToRepo -GetCommitShaTable +main + diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 33144d9..8668c1b 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -18,6 +18,16 @@ function CreateAndPopulateCsv { } } +function WriteTableToCsv($shaTable) { + if (Test-Path $csvPath) { + Clear-Content -Path $csvPath + } + Add-Content -Path $csvPath -Value "FileName, CommitSha" + $shaTable.GetEnumerator() | ForEach-Object { + "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath + } +} + function GetCommitShaTable { $Header = @{ "authorization" = "Bearer $githubAuthToken" @@ -30,7 +40,7 @@ function GetCommitShaTable { $getTreeResponse.tree | ForEach-Object { if ($_.path.Substring($_.path.Length-5) -eq ".json") { - #needs to be $workplace in real implementation + #needs to be $workspace in real implementation $truePath = ($workspace + "\" + $_.path).Replace("/", "\") $shaTable.Add($truePath, $_.sha) } @@ -68,7 +78,9 @@ function PushCsvToRepo { function main { Write-Output $githubRepository - CreateAndPopulateCsv + $shaTable = GetCommitShaTable + WriteTableToCsv $shaTable + # CreateAndPopulateCsv PushCsvToRepo Get-ChildItem -Path $Directory -Recurse -Filter *.json | From 595e7652bafe70fa23ff21607b0c31e03beb2119 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:10:14 -0800 Subject: [PATCH 42/90] Removed tracking table. Edited test script for csv funtions --- .github/workflows/tracking_table.csv | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From c2b1cf40b7089d88309a3c9e79338bd68b810d88 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 22:10:41 +0000 Subject: [PATCH 43/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 59d3ed47e7251c37276fbc5a3fe37bb02a7eb7b6 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:17:11 -0800 Subject: [PATCH 44/90] Changed pushcsv url to use githubRepository variable --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 8668c1b..d1fc738 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -56,7 +56,7 @@ function PushCsvToRepo { } $path = ".github/workflows/tracking_table.csv" Write-Output $path - $createFileUrl = "https://api.github.com/repos/aaroncorreya/SmartTrackingScriptDev/contents/$path" + $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" $content = Get-Content -Path $csvPath | Out-String $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) Write-Output $encodedContent diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 9274ce2f9726666c47df06651e14fab3925b6ecc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 22:17:38 +0000 Subject: [PATCH 45/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 249146b9661e87020a48204e711118faefbd4e30 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:20:44 -0800 Subject: [PATCH 46/90] Extrapolated header from functions --- .github/workflows/test_script.ps1 | 22 ++++------------------ .github/workflows/tracking_table.csv | 3 --- 2 files changed, 4 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index d1fc738..144c885 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -5,19 +5,11 @@ $refName = $Env:GITHUB_REF $branchName = $refName.Replace("refs/heads/", "") $workspace = $Env:GITHUB_WORKSPACE -function CreateAndPopulateCsv { - if (!(Test-Path $csvPath)) { - Add-Content -Path $csvPath -Value "FileName, CommitSha" - Write-Output "Created csv file." - } - $shaTable = GetCommitShaTable - Write-Output $shaTable - #write all filename, sha to csv file - $shaTable.GetEnumerator() | ForEach-Object { - "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath - } +$header = @{ + "authorization" = "Bearer $githubAuthToken" } + function WriteTableToCsv($shaTable) { if (Test-Path $csvPath) { Clear-Content -Path $csvPath @@ -29,9 +21,6 @@ function WriteTableToCsv($shaTable) { } function GetCommitShaTable { - $Header = @{ - "authorization" = "Bearer $githubAuthToken" - } #get branch sha and use it to get tree with all commit shas and files $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" @@ -51,9 +40,6 @@ function GetCommitShaTable { #we need token provided by workflow run to push file, not installationtoken, will test later function PushCsvToRepo { #if exists, we need sha of csv file before pushing updated file. If new, no need - $Header = @{ - "authorization" = "Bearer $githubAuthToken" - } $path = ".github/workflows/tracking_table.csv" Write-Output $path $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" @@ -69,7 +55,7 @@ function PushCsvToRepo { $Parameters = @{ Method = "PUT" Uri = $createFileUrl - Headers = $Header + Headers = $header Body = $body | ConvertTo-Json } #Commit csv file diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From b8a93a5f013fea850b70226fc0390a0ed1bcae54 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 22:21:08 +0000 Subject: [PATCH 47/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From c5583e8647fe1dfba983ac8846d171761f269b9e Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:52:42 -0800 Subject: [PATCH 48/90] Edited csv functions to call tree seperately --- .github/workflows/smart_script.ps1 | 43 ++++++++++++++++++---------- .github/workflows/test_script.ps1 | 11 +++++-- .github/workflows/tracking_table.csv | 3 -- 3 files changed, 36 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 57b1d22..637f7d7 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -20,9 +20,13 @@ $contentTypeMapping = @{ $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $json.githubAuthToken $githubRepository = $json.githubRepository -$branchName = "main" #change to variable passed through workflow +$branchName = "testScript" #change to variable passed through workflow $manualDeployment = $json.manualDeployment +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + if ([string]::IsNullOrEmpty($contentTypes)) { $contentTypes = "AnalyticsRule,Metadata" } @@ -35,13 +39,6 @@ $resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping $MaxRetries = 3 $secondsBetweenAttempts = 5 -function CreateCsv() { - if (Test-Path $csvPath) { - Clear-Content -Path $csvPath - } - Add-Content -Path $csvPath -Value "FileName, CommitSha" -} - function WriteTableToCsv($shaTable) { if (Test-Path $csvPath) { Clear-Content -Path $csvPath @@ -52,14 +49,30 @@ function WriteTableToCsv($shaTable) { } } -function GetCommitShaTable { - $Header = @{ - "authorization" = "Bearer $githubAuthToken" - } - #get branch sha and use it to get tree with all commit shas and files +function GetGithubTree { $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header + return $getTreeResponse +} + +function GetCsvCommitSha($getTreeResponse) { + $sha = $null + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-4) -eq ".csv") + { + $sha = $_.sha + } + } + return $sha +} + +function GetCommitShaTable($getTreeResponse) { + #get branch sha and use it to get tree with all commit shas and files + # $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header + # $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + # $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header + # $getTreeResponse = GetGithubTree $shaTable = @{} $getTreeResponse.tree | ForEach-Object { if ($_.path.Substring($_.path.Length-5) -eq ".json") @@ -362,14 +375,14 @@ function main() { if (-not (Test-Path $csvPath)) { Write-Output "Creating csv and conducting full deployment." - $remoteShaTable = GetCommitShaTable + $remoteShaTable = GetCommitShaTable $tree WriteTableToCsv($remoteShaTable) # PushCsvToRepo Deployment $fullDeploymentFlag $null $null } else { $localCsvTable = ReadCsvToTable - $remoteShaTable = GetCommitShaTable + $remoteShaTable = GetCommitShaTable $tree Write-Output "Local Csv Table" Write-Output $localCsvTable Write-Output "Remote Csv Table" diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 144c885..c6be5e5 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -20,11 +20,15 @@ function WriteTableToCsv($shaTable) { } } -function GetCommitShaTable { - #get branch sha and use it to get tree with all commit shas and files +function GetGithubTree { $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header + return $getTreeResponse +} + +function GetCommitShaTable($getTreeResponse) { + #get branch sha and use it to get tree with all commit shas and files $shaTable = @{} $getTreeResponse.tree | ForEach-Object { if ($_.path.Substring($_.path.Length-5) -eq ".json") @@ -64,7 +68,8 @@ function PushCsvToRepo { function main { Write-Output $githubRepository - $shaTable = GetCommitShaTable + $tree = GetGithubTree + $shaTable = GetCommitShaTable $tree WriteTableToCsv $shaTable # CreateAndPopulateCsv PushCsvToRepo diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From d74e5d237eb8adcfce169d511df1f859a2997a79 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 22:53:49 +0000 Subject: [PATCH 49/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 9fe468e69e7c40217bf2cd4574266a542449a64a Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 14:59:34 -0800 Subject: [PATCH 50/90] Added sha to pushcsv body request --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index c6be5e5..8f9e561 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -33,7 +33,6 @@ function GetCommitShaTable($getTreeResponse) { $getTreeResponse.tree | ForEach-Object { if ($_.path.Substring($_.path.Length-5) -eq ".json") { - #needs to be $workspace in real implementation $truePath = ($workspace + "\" + $_.path).Replace("/", "\") $shaTable.Add($truePath, $_.sha) } @@ -54,6 +53,7 @@ function PushCsvToRepo { message = "trackingTable.csv created." content = $encodedContent branch = $branchName + sha = $null } $Parameters = @{ diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 6c59658842706084936f5b18f0336f5edd0d3851 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 23:00:01 +0000 Subject: [PATCH 51/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 4286ce92206a98a8d05254e4718ac4f1f45ce8cc Mon Sep 17 00:00:00 2001 From: Aaron Correya <34196924+aaroncorreya@users.noreply.github.com> Date: Mon, 3 Jan 2022 15:13:31 -0800 Subject: [PATCH 52/90] Delete tracking_table.csv --- .github/workflows/tracking_table.csv | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From d52a04626b9d975c5d121e9a11b456b8fce006f5 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 15:15:11 -0800 Subject: [PATCH 53/90] Added get csv file sha to pushcsvtoRepo --- .github/workflows/test_script.ps1 | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 8f9e561..1f3e926 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -27,6 +27,17 @@ function GetGithubTree { return $getTreeResponse } +function GetCsvCommitSha($getTreeResponse) { + $sha = $null + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-4) -eq ".csv") + { + $sha = $_.sha + } + } + return $sha +} + function GetCommitShaTable($getTreeResponse) { #get branch sha and use it to get tree with all commit shas and files $shaTable = @{} @@ -41,10 +52,11 @@ function GetCommitShaTable($getTreeResponse) { } #we need token provided by workflow run to push file, not installationtoken, will test later -function PushCsvToRepo { +function PushCsvToRepo($getTreeResponse) { #if exists, we need sha of csv file before pushing updated file. If new, no need $path = ".github/workflows/tracking_table.csv" Write-Output $path + $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" $content = Get-Content -Path $csvPath | Out-String $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) @@ -53,7 +65,7 @@ function PushCsvToRepo { message = "trackingTable.csv created." content = $encodedContent branch = $branchName - sha = $null + sha = $sha } $Parameters = @{ From 3dac819a95d25ab0856f1a352befb628524be624 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 15:16:41 -0800 Subject: [PATCH 54/90] Added tree parameter to pushCsvToRepo --- .github/workflows/test_script.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 1f3e926..16a6cf7 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -84,7 +84,7 @@ function main { $shaTable = GetCommitShaTable $tree WriteTableToCsv $shaTable # CreateAndPopulateCsv - PushCsvToRepo + PushCsvToRepo $tree Get-ChildItem -Path $Directory -Recurse -Filter *.json | ForEach-Object { From 3f99376798dbc16c20bc5ab53dab3b7180357f32 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 23:17:04 +0000 Subject: [PATCH 55/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From ad739469eeb4c492e88ecce661c3d8f63f393693 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 16:43:27 -0800 Subject: [PATCH 56/90] Checking if update sha works --- .github/workflows/test_script.ps1 | 3 --- .github/workflows/tracking_table.csv | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 16a6cf7..0abc21b 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -51,9 +51,7 @@ function GetCommitShaTable($getTreeResponse) { return $shaTable } -#we need token provided by workflow run to push file, not installationtoken, will test later function PushCsvToRepo($getTreeResponse) { - #if exists, we need sha of csv file before pushing updated file. If new, no need $path = ".github/workflows/tracking_table.csv" Write-Output $path $sha = GetCsvCommitSha $getTreeResponse @@ -83,7 +81,6 @@ function main { $tree = GetGithubTree $shaTable = GetCommitShaTable $tree WriteTableToCsv $shaTable - # CreateAndPopulateCsv PushCsvToRepo $tree Get-ChildItem -Path $Directory -Recurse -Filter *.json | diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 4226fee3dd09ec7dc8b31a48c6c7b86324a971f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 00:43:53 +0000 Subject: [PATCH 57/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From c64b0b2adf068491e2380023b85ccec9ade8069f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 17:25:09 -0800 Subject: [PATCH 58/90] Commented out previous path --- .github/workflows/smart_script.ps1 | 1 + .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table.csv | 3 --- 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 637f7d7..5b3d684 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -372,6 +372,7 @@ function main() { $fullDeploymentFlag = CheckFullDeployment Write-Output $fullDeploymentFlag + $tree = GetGithubTree if (-not (Test-Path $csvPath)) { Write-Output "Creating csv and conducting full deployment." diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 0abc21b..0d06dca 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -52,7 +52,7 @@ function GetCommitShaTable($getTreeResponse) { } function PushCsvToRepo($getTreeResponse) { - $path = ".github/workflows/tracking_table.csv" + # $path = ".github/workflows/tracking_table.csv" Write-Output $path $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From b05724e19c0e382fb79b8fb247e33c640dcfea1c Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Mon, 3 Jan 2022 17:26:15 -0800 Subject: [PATCH 59/90] Uncommented path --- .github/workflows/test_script.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 0d06dca..0abc21b 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -52,7 +52,7 @@ function GetCommitShaTable($getTreeResponse) { } function PushCsvToRepo($getTreeResponse) { - # $path = ".github/workflows/tracking_table.csv" + $path = ".github/workflows/tracking_table.csv" Write-Output $path $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" From 87e87129d4ec7fcf7f00d59e3a3913b42ee6a4e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 01:26:39 +0000 Subject: [PATCH 60/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 4785c859cef2c0298d78f9d66a4835df308bbd37 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 12:36:07 -0800 Subject: [PATCH 61/90] Added comments to test script --- .github/workflows/test_script.ps1 | 10 +- .github/workflows/tracking_table.csv | 3 - .gitignore | 3 +- currentUpdateScript.ps1 | 300 +++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 7 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv create mode 100644 currentUpdateScript.ps1 diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 0abc21b..362f917 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -3,13 +3,14 @@ $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY $refName = $Env:GITHUB_REF $branchName = $refName.Replace("refs/heads/", "") +#$branchName = $Env:branch $workspace = $Env:GITHUB_WORKSPACE $header = @{ "authorization" = "Bearer $githubAuthToken" } - +#Writes sha dictionary object to csv file. Will delete any pre-existing content before writing. function WriteTableToCsv($shaTable) { if (Test-Path $csvPath) { Clear-Content -Path $csvPath @@ -20,6 +21,7 @@ function WriteTableToCsv($shaTable) { } } +#Gets all files and commit shas using Get Trees API function GetGithubTree { $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" @@ -27,6 +29,7 @@ function GetGithubTree { return $getTreeResponse } +#Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { $sha = $null $getTreeResponse.tree | ForEach-Object { @@ -38,8 +41,8 @@ function GetCsvCommitSha($getTreeResponse) { return $sha } +#Creates a table using the reponse from the tree api, creates a table function GetCommitShaTable($getTreeResponse) { - #get branch sha and use it to get tree with all commit shas and files $shaTable = @{} $getTreeResponse.tree | ForEach-Object { if ($_.path.Substring($_.path.Length-5) -eq ".json") @@ -51,6 +54,8 @@ function GetCommitShaTable($getTreeResponse) { return $shaTable } +#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. +#TODO: Add source control id to tracking_table name. function PushCsvToRepo($getTreeResponse) { $path = ".github/workflows/tracking_table.csv" Write-Output $path @@ -72,7 +77,6 @@ function PushCsvToRepo($getTreeResponse) { Headers = $header Body = $body | ConvertTo-Json } - #Commit csv file Invoke-RestMethod @Parameters } diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d diff --git a/.gitignore b/.gitignore index f925b1a..cca4362 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -environment* \ No newline at end of file +environment* +currentTemplate* diff --git a/currentUpdateScript.ps1 b/currentUpdateScript.ps1 new file mode 100644 index 0000000..dd9852d --- /dev/null +++ b/currentUpdateScript.ps1 @@ -0,0 +1,300 @@ +## Globals ## +$CloudEnv = $Env:cloudEnv +$ResourceGroupName = $Env:resourceGroupName +$WorkspaceName = $Env:workspaceName +$Directory = $Env:directory +$Creds = $Env:creds +$contentTypes = $Env:contentTypes +$contentTypeMapping = @{ + "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); + "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); + "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); + "Workbook"=@("Microsoft.Insights/workbooks"); + "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); +} +#TODO: Make path including sourceControlId for csv file +$csvPath = ".github\workflows\tracking_table.csv" +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:GITHUB_REPOSITORY +$refName = $Env:GITHUB_REF +$branchName = $refName.Replace("refs/heads/", "") +$workspace = $Env:GITHUB_WORKSPACE +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + +if ([string]::IsNullOrEmpty($contentTypes)) { + $contentTypes = "AnalyticsRule,Metadata" +} + +if (-not ($contentTypes.contains("Metadata"))) { + $contentTypes += ",Metadata" +} + +$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } +$MaxRetries = 3 +$secondsBetweenAttempts = 5 + +#Writes sha dictionary object to csv file. Will delete any pre-existing content before writing. +function WriteTableToCsv($shaTable) { + if (Test-Path $csvPath) { + Clear-Content -Path $csvPath + } + Add-Content -Path $csvPath -Value "FileName, CommitSha" + $shaTable.GetEnumerator() | ForEach-Object { + "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath + } +} + +#Gets all files and commit shas using Get Trees API +function GetGithubTree { + $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header + return $getTreeResponse +} + +#Gets blob commit sha of the csv file, used when updating csv file to repo +function GetCsvCommitSha($getTreeResponse) { + $sha = $null + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-4) -eq ".csv") + { + $sha = $_.sha + } + } + return $sha +} + +#Creates a table using the reponse from the tree api, creates a table +function GetCommitShaTable($getTreeResponse) { + $shaTable = @{} + $getTreeResponse.tree | ForEach-Object { + if ($_.path.Substring($_.path.Length-5) -eq ".json") + { + $truePath = ($workspace + "\" + $_.path).Replace("/", "\") + $shaTable.Add($truePath, $_.sha) + } + } + return $shaTable +} + +#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. +#TODO: Add source control id to tracking_table name. +function PushCsvToRepo($getTreeResponse) { + $path = ".github/workflows/tracking_table.csv" + Write-Output $path + $sha = GetCsvCommitSha $getTreeResponse + $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" + $content = Get-Content -Path $csvPath | Out-String + $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) + Write-Output $encodedContent + $body = @{ + message = "trackingTable.csv created." + content = $encodedContent + branch = $branchName + sha = $sha + } + + $Parameters = @{ + Method = "PUT" + Uri = $createFileUrl + Headers = $header + Body = $body | ConvertTo-Json + } + Invoke-RestMethod @Parameters +} + +function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { + $maxLoginRetries = 3 + $delayInSeconds = 30 + $retryCount = 1 + $stopTrying = $false + do { + try { + Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; + Write-Host "Login Successful" + $stopTrying = $true + } + catch { + if ($retryCount -ge $maxLoginRetries) { + Write-Host "Login failed after $maxLoginRetries attempts." + $stopTrying = $true + } + else { + Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." + Start-Sleep -Seconds $delayInSeconds + $retryCount++ + } + } + } + while (-not $stopTrying) +} + +function ConnectAzCloud { + $RawCreds = $Creds | ConvertFrom-Json + + Clear-AzContext -Scope Process; + Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; + + Add-AzEnvironment ` + -Name $CloudEnv ` + -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` + -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` + -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` + -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; + + $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force + $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) + + AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv + Set-AzContext -Tenant $RawCreds.tenantId | out-null; +} + +function IsValidTemplate($path, $templateObject) { + Try { + if (DoesContainWorkspaceParam $templateObject) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName + } + else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path + } + + return $true + } + Catch { + Write-Host "[Warning] The file $path is not valid: $_" + return $false + } +} + +function IsRetryable($deploymentName) { + $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" + Try { + $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop + return $retryableStatusCodes -contains $deploymentResult.StatusCode + } + Catch { + return $false + } +} + +function IsValidResourceType($template) { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + return $isAllowedResources +} + +function DoesContainWorkspaceParam($templateObject) { + $templateObject.parameters.PSobject.Properties.Name -contains "workspace" +} + +function AttemptDeployment($path, $deploymentName, $templateObject) { + Write-Host "[Info] Deploying $path with deployment name $deploymentName" + + $isValid = IsValidTemplate $path $templateObject + if (-not $isValid) { + return $false + } + $isSuccess = $false + $currentAttempt = 0 + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + if (DoesContainWorkspaceParam $templateObject) + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host + } + + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable $deploymentName)) + { + Write-Host "[Warning] Failed to deploy $path with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" + } + } + } + } + return $isSuccess +} + +function GenerateDeploymentName() { + $randomId = [guid]::NewGuid() + return "Sentinel_Deployment_$randomId" +} + +function main() { + if ($CloudEnv -ne 'AzureCloud') + { + Write-Output "Attempting Sign In to Azure Cloud" + ConnectAzCloud + } + + Write-Output "Starting Deployment for Files in path: $Directory" + + if (Test-Path -Path $Directory) + { + $totalFiles = 0; + $totalFailed = 0; + Get-ChildItem -Path $Directory -Recurse -Filter *.json | + ForEach-Object { + $path = $_.FullName + try { + $totalFiles ++ + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + if (-not (IsValidResourceType $templateObject)) + { + Write-Output "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $_.FullName $deploymentName $templateObject + if (-not $isSuccess) + { + $totalFailed++ + } + } + catch { + $totalFailed++ + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } + } + if ($totalFiles -gt 0 -and $totalFailed -gt 0) + { + $err = "$totalFailed of $totalFiles deployments failed." + Throw $err + } + } + else + { + Write-Output "[Warning] $Directory not found. nothing to deploy" + } +} + +main \ No newline at end of file From 432f69d76dfc535b3d16d8a09322e5d2e1006a1e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 20:36:31 +0000 Subject: [PATCH 62/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 09b31cfdde6ea23a1853501bd2f05b0c8adae29f Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 12:38:21 -0800 Subject: [PATCH 63/90] Added updated json query for GetCommitShaTable --- .github/workflows/test_script.ps1 | 3 ++- .github/workflows/tracking_table.csv | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 362f917..48f6c3f 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -45,7 +45,8 @@ function GetCsvCommitSha($getTreeResponse) { function GetCommitShaTable($getTreeResponse) { $shaTable = @{} $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-5) -eq ".json") + #if ($_.path.Substring($_.path.Length-5) -eq ".json") + if ([System.IO.Path]::GetExtension($_.path) -eq ".json") { $truePath = ($workspace + "\" + $_.path).Replace("/", "\") $shaTable.Add($truePath, $_.sha) diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From cea026117a5acc988d1c38f162088c1818417127 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 20:38:46 +0000 Subject: [PATCH 64/90] trackingTable.csv created. --- .github/workflows/tracking_table.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 7c8d56e9cb44d3ccd77c233952087dcd39bf2147 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 13:34:18 -0800 Subject: [PATCH 65/90] Added source control id to csv path name --- .github/workflows/testWorkflow.yml | 1 + .github/workflows/test_script.ps1 | 5 +++-- .github/workflows/tracking_table.csv | 3 --- 3 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 .github/workflows/tracking_table.csv diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml index a8b3c66..80e5be0 100644 --- a/.github/workflows/testWorkflow.yml +++ b/.github/workflows/testWorkflow.yml @@ -22,6 +22,7 @@ jobs: env: repository: aaroncorreya/SmartTrackingScriptDev githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + sourceControlId: "1234" # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 48f6c3f..bfba3b0 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -1,10 +1,11 @@ -$csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY $refName = $Env:GITHUB_REF $branchName = $refName.Replace("refs/heads/", "") #$branchName = $Env:branch $workspace = $Env:GITHUB_WORKSPACE +$sourceControlId = $Env:sourceControlId +$csvPath = ".github\workflows\tracking_table_$souceControlId.csv" $header = @{ "authorization" = "Bearer $githubAuthToken" @@ -58,7 +59,7 @@ function GetCommitShaTable($getTreeResponse) { #Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. #TODO: Add source control id to tracking_table name. function PushCsvToRepo($getTreeResponse) { - $path = ".github/workflows/tracking_table.csv" + $path = ".github/workflows/tracking_table_$sourceControlId.csv" Write-Output $path $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" diff --git a/.github/workflows/tracking_table.csv b/.github/workflows/tracking_table.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From ca2cd17865f49c3738f1e7555b1da37b204c7445 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 21:34:42 +0000 Subject: [PATCH 66/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 445f78ab882593389fe6ed3a7fcc7bd343557394 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 13:37:26 -0800 Subject: [PATCH 67/90] Edited typo for source control id --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table_1234.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index bfba3b0..c0b6e0c 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -5,7 +5,7 @@ $branchName = $refName.Replace("refs/heads/", "") #$branchName = $Env:branch $workspace = $Env:GITHUB_WORKSPACE $sourceControlId = $Env:sourceControlId -$csvPath = ".github\workflows\tracking_table_$souceControlId.csv" +$csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" $header = @{ "authorization" = "Bearer $githubAuthToken" diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 19694e56b724d0654f0b707556bb4199e695cd61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 21:37:52 +0000 Subject: [PATCH 68/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From a94a29ebd7bb5aa9079e6c3977ba481c4d947bbc Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 14:12:23 -0800 Subject: [PATCH 69/90] Update csvCommitSha function --- .github/workflows/smart_script.ps1 | 13 ++++++++++--- .github/workflows/test_script.ps1 | 4 +++- .github/workflows/tracking_table_1234.csv | 3 --- 3 files changed, 13 insertions(+), 7 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 5b3d684..7731d4a 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -17,11 +17,13 @@ $contentTypeMapping = @{ "Workbook"=@("Microsoft.Insights/workbooks"); "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); } -$csvPath = ".github\workflows\tracking_table.csv" +# $csvPath = ".github\workflows\tracking_table.csv" $githubAuthToken = $json.githubAuthToken $githubRepository = $json.githubRepository $branchName = "testScript" #change to variable passed through workflow $manualDeployment = $json.manualDeployment +$sourceControlId = $json.sourceControlId +$csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" $header = @{ "authorization" = "Bearer $githubAuthToken" @@ -58,8 +60,10 @@ function GetGithubTree { function GetCsvCommitSha($getTreeResponse) { $sha = $null + $path = ".github/workflows/tracking_table_$sourceControlId.csv" $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-4) -eq ".csv") + Write-Output $_.path + if ($_.path -eq $path) { $sha = $_.sha } @@ -90,7 +94,7 @@ function PushCsvToRepo { $Header = @{ "authorization" = "Bearer $githubAuthToken" } - $path = ".github/workflows/tracking_table.csv" + $path = ".github/workflows/tracking_table_$sourceControlId.csv" Write-Output $path $createFileUrl = "https://api.github.com/repos/aaroncorreya/SmartTrackingScriptDev/contents/$path" $content = Get-Content -Path $csvPath | Out-String @@ -392,6 +396,9 @@ function main() { WriteTableToCsv($updatedCsvTable) #PushCsvToRepo } + # Write-Output $tree + # $sha = GetCsvCommitSha $tree + # Write-Output $sha } main diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index c0b6e0c..52a9399 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -33,8 +33,10 @@ function GetGithubTree { #Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { $sha = $null + $path = ".github/workflows/tracking_table_$sourceControlId.csv" $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-4) -eq ".csv") + #if ($_.path.Substring($_.path.Length-4) -eq ".csv") + if ($_.path -eq $path) { $sha = $_.sha } diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From f15fa4023b541cc6c41487c31eb1d4363b22040d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 22:12:48 +0000 Subject: [PATCH 70/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 88b09de9eefcdd28853a485077a20cd7e189beb3 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 14:28:58 -0800 Subject: [PATCH 71/90] Added print of all json files in root --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table_1234.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 52a9399..ceabc95 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -91,7 +91,7 @@ function main { WriteTableToCsv $shaTable PushCsvToRepo $tree - Get-ChildItem -Path $Directory -Recurse -Filter *.json | + Get-ChildItem -Path $workspace -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName Write-Output $path diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 0aa8b70c001cc682e937875654a2b31aaa185dae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 22:29:29 +0000 Subject: [PATCH 72/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..d94168d --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 765b77b85e720a5599f8ea45894a32e025e3da43 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 14:30:49 -0800 Subject: [PATCH 73/90] added sha table print --- .github/workflows/test_script.ps1 | 4 +++- .github/workflows/tracking_table_1234.csv | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index ceabc95..ebb35e8 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -90,7 +90,9 @@ function main { $shaTable = GetCommitShaTable $tree WriteTableToCsv $shaTable PushCsvToRepo $tree - + Write-Output "SHA TABLE" + Write-Output $shaTable + Get-ChildItem -Path $workspace -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index d94168d..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From c057b58f15662522bfb6658e251a82232d745147 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 22:31:15 +0000 Subject: [PATCH 74/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..55bc0f2 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 497daf84f1eb92b5dd564da26c5e19e66b379661 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 16:03:00 -0800 Subject: [PATCH 75/90] Removed absolute paths --- .github/workflows/smart_script.ps1 | 19 +++++++++++-------- .github/workflows/test_script.ps1 | 5 +++-- .github/workflows/tracking_table_1234.csv | 3 --- 3 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 7731d4a..282478b 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -82,7 +82,8 @@ function GetCommitShaTable($getTreeResponse) { if ($_.path.Substring($_.path.Length-5) -eq ".json") { #needs to be $workplace in real implementation - $truePath = ($Directory + "\" + $_.path).Replace("/", "\") + # $truePath = ($Directory + "\" + $_.path).Replace("/", "\") + $truePath = $_.path.Replace("/", "\") $shaTable.Add($truePath, $_.sha) } } @@ -373,10 +374,7 @@ function main() { Write-Output "Attempting Sign In to Azure Cloud" ConnectAzCloud } - $fullDeploymentFlag = CheckFullDeployment - Write-Output $fullDeploymentFlag - $tree = GetGithubTree if (-not (Test-Path $csvPath)) { Write-Output "Creating csv and conducting full deployment." @@ -396,10 +394,15 @@ function main() { WriteTableToCsv($updatedCsvTable) #PushCsvToRepo } - # Write-Output $tree - # $sha = GetCsvCommitSha $tree - # Write-Output $sha + #make paths of both tables relative and the same } -main +# main +$fullDeploymentFlag = CheckFullDeployment +Write-Output $fullDeploymentFlag +$tree = GetGithubTree +$remoteShaTable = GetCommitShaTable $tree +$localCsvTable = ReadCsvToTable +Write-Output $remoteShaTable +Write-Output $localCsvTable \ No newline at end of file diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index ebb35e8..770991c 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -51,7 +51,8 @@ function GetCommitShaTable($getTreeResponse) { #if ($_.path.Substring($_.path.Length-5) -eq ".json") if ([System.IO.Path]::GetExtension($_.path) -eq ".json") { - $truePath = ($workspace + "\" + $_.path).Replace("/", "\") + #$truePath = ($workspace + "\" + $_.path).Replace("/", "\") + $truePath = $_.path.Replace("/", "\") $shaTable.Add($truePath, $_.sha) } } @@ -92,7 +93,7 @@ function main { PushCsvToRepo $tree Write-Output "SHA TABLE" Write-Output $shaTable - + Get-ChildItem -Path $workspace -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 55bc0f2..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -D:\a\SmartTrackingScriptDev\SmartTrackingScriptDev\Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 2aa37ca2a3c3a44accac8448340bb98e278de6a7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 00:03:28 +0000 Subject: [PATCH 76/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..9d741b2 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 6b2403385f871d9e1f0581d896576549a23bb634 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 16:06:31 -0800 Subject: [PATCH 77/90] Removed workspace from path --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table_1234.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 770991c..85a24e6 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -96,7 +96,7 @@ function main { Get-ChildItem -Path $workspace -Recurse -Filter *.json | ForEach-Object { - $path = $_.FullName + $path = $_.FullName.Replace($workspace, "") Write-Output $path } Write-Output $workspace diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 9d741b2..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 404d180d645e28595e2bd96707cf38aa09b08194 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 00:06:58 +0000 Subject: [PATCH 78/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..692ea13 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 0896b829c05b33027b239af807d34343521d4fae Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 16:09:05 -0800 Subject: [PATCH 79/90] Added back slash to workspace variable --- .github/workflows/test_script.ps1 | 2 +- .github/workflows/tracking_table_1234.csv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 85a24e6..8c946bf 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -3,7 +3,7 @@ $githubRepository = $Env:GITHUB_REPOSITORY $refName = $Env:GITHUB_REF $branchName = $refName.Replace("refs/heads/", "") #$branchName = $Env:branch -$workspace = $Env:GITHUB_WORKSPACE +$workspace = $Env:GITHUB_WORKSPACE + "\" $sourceControlId = $Env:sourceControlId $csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 692ea13..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From 3f513df29b668debddc805122e4fc46d4ecc1028 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 00:09:33 +0000 Subject: [PATCH 80/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..692ea13 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d +Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c From f5aef19dd12fefbcc9938520bf09cd4a4561c045 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Tue, 4 Jan 2022 16:46:41 -0800 Subject: [PATCH 81/90] Updated with where function for commit csv --- .github/workflows/smart_script.ps1 | 56 ++++++++++++----------- .github/workflows/test_script.ps1 | 12 +---- .github/workflows/tracking_table_1234.csv | 3 -- currentUpdateScript.ps1 | 29 ++++++------ 4 files changed, 46 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 282478b..5dbfdd0 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -283,6 +283,7 @@ function Deployment($fullDeploymentFlag, $localCsvTable, $remoteShaTable) { Get-ChildItem -Path $Directory -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName + # $path = $_.FullName.Replace($workspace, "") $templateObject = Get-Content $path | Out-String | ConvertFrom-Json #put this into try catch try { @@ -376,33 +377,36 @@ function main() { } $fullDeploymentFlag = CheckFullDeployment - if (-not (Test-Path $csvPath)) { - Write-Output "Creating csv and conducting full deployment." - $remoteShaTable = GetCommitShaTable $tree - WriteTableToCsv($remoteShaTable) - # PushCsvToRepo - Deployment $fullDeploymentFlag $null $null + # if (-not (Test-Path $csvPath)) { + # Write-Output "Creating csv and conducting full deployment." + # $remoteShaTable = GetCommitShaTable $tree + # WriteTableToCsv($remoteShaTable) + # # PushCsvToRepo + # Deployment $fullDeploymentFlag $null $null + # } + # else { + # $localCsvTable = ReadCsvToTable + # $remoteShaTable = GetCommitShaTable $tree + # Write-Output "Local Csv Table" + # Write-Output $localCsvTable + # Write-Output "Remote Csv Table" + # Write-Output $remoteShaTable + # $updatedCsvTable = Deployment $fullDeploymentFlag $localCsvTable $remoteShaTable + # WriteTableToCsv($updatedCsvTable) + # #PushCsvToRepo + # } + Get-ChildItem -Path $Directory -Recurse -Filter *.json | + ForEach-Object { + Write-Output $_.FullName.Replace($Directory + "\", "") } - else { - $localCsvTable = ReadCsvToTable - $remoteShaTable = GetCommitShaTable $tree - Write-Output "Local Csv Table" - Write-Output $localCsvTable - Write-Output "Remote Csv Table" - Write-Output $remoteShaTable - $updatedCsvTable = Deployment $fullDeploymentFlag $localCsvTable $remoteShaTable - WriteTableToCsv($updatedCsvTable) - #PushCsvToRepo - } - #make paths of both tables relative and the same } -# main +main -$fullDeploymentFlag = CheckFullDeployment -Write-Output $fullDeploymentFlag -$tree = GetGithubTree -$remoteShaTable = GetCommitShaTable $tree -$localCsvTable = ReadCsvToTable -Write-Output $remoteShaTable -Write-Output $localCsvTable \ No newline at end of file +# $fullDeploymentFlag = CheckFullDeployment +# Write-Output $fullDeploymentFlag +# $tree = GetGithubTree +# $remoteShaTable = GetCommitShaTable $tree +# $localCsvTable = ReadCsvToTable +# Write-Output $remoteShaTable +# Write-Output $localCsvTable \ No newline at end of file diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 8c946bf..8e38556 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -32,16 +32,7 @@ function GetGithubTree { #Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { - $sha = $null - $path = ".github/workflows/tracking_table_$sourceControlId.csv" - $getTreeResponse.tree | ForEach-Object { - #if ($_.path.Substring($_.path.Length-4) -eq ".csv") - if ($_.path -eq $path) - { - $sha = $_.sha - } - } - return $sha + return $getTreeResponse.tree | Where-Object { $_.path -eq ".github/workflows/tracking_table_$sourceControlId.csv" } } #Creates a table using the reponse from the tree api, creates a table @@ -94,6 +85,7 @@ function main { Write-Output "SHA TABLE" Write-Output $shaTable + #TODO: Make sure that the paths are the same when testing locally and remotely Get-ChildItem -Path $workspace -Recurse -Filter *.json | ForEach-Object { $path = $_.FullName.Replace($workspace, "") diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 692ea13..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d -Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c diff --git a/currentUpdateScript.ps1 b/currentUpdateScript.ps1 index dd9852d..f94b7a8 100644 --- a/currentUpdateScript.ps1 +++ b/currentUpdateScript.ps1 @@ -14,16 +14,12 @@ $contentTypeMapping = @{ "Workbook"=@("Microsoft.Insights/workbooks"); "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); } -#TODO: Make path including sourceControlId for csv file -$csvPath = ".github\workflows\tracking_table.csv" +$sourceControlId = $Env:sourceControlId $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY -$refName = $Env:GITHUB_REF -$branchName = $refName.Replace("refs/heads/", "") -$workspace = $Env:GITHUB_WORKSPACE -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} +$branchName = $Env:branch +$workspace = $Env:GITHUB_WORKSPACE + "\" +$csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" if ([string]::IsNullOrEmpty($contentTypes)) { $contentTypes = "AnalyticsRule,Metadata" @@ -48,6 +44,10 @@ function WriteTableToCsv($shaTable) { } } +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + #Gets all files and commit shas using Get Trees API function GetGithubTree { $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header @@ -59,8 +59,9 @@ function GetGithubTree { #Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { $sha = $null + $path = ".github/workflows/tracking_table_$sourceControlId.csv" $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-4) -eq ".csv") + if ($_.path -eq $path) { $sha = $_.sha } @@ -72,9 +73,9 @@ function GetCsvCommitSha($getTreeResponse) { function GetCommitShaTable($getTreeResponse) { $shaTable = @{} $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-5) -eq ".json") + if ([System.IO.Path]::GetExtension($_.path) -eq ".json") { - $truePath = ($workspace + "\" + $_.path).Replace("/", "\") + $truePath = $_.path.Replace("/", "\") $shaTable.Add($truePath, $_.sha) } } @@ -82,15 +83,13 @@ function GetCommitShaTable($getTreeResponse) { } #Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. -#TODO: Add source control id to tracking_table name. function PushCsvToRepo($getTreeResponse) { - $path = ".github/workflows/tracking_table.csv" - Write-Output $path + $path = ".github/workflows/tracking_table_$sourceControlId.csv" $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" $content = Get-Content -Path $csvPath | Out-String $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - Write-Output $encodedContent + $body = @{ message = "trackingTable.csv created." content = $encodedContent From 671e13523845064477386e29bd63ca2aa6504701 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 00:47:08 +0000 Subject: [PATCH 82/90] trackingTable.csv created. --- .github/workflows/tracking_table_1234.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv new file mode 100644 index 0000000..9d741b2 --- /dev/null +++ b/.github/workflows/tracking_table_1234.csv @@ -0,0 +1,3 @@ +FileName, CommitSha +Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c +Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 48bbecd3ba80d9bbbe071b35e37b0c543c8fdd05 Mon Sep 17 00:00:00 2001 From: Aaron Correya Date: Fri, 14 Jan 2022 15:31:50 -0800 Subject: [PATCH 83/90] Making repo up to date --- .github/workflows/smart_script.ps1 | 204 +++++++++++++---------------- .github/workflows/test_script.ps1 | 4 +- currentUpdateScript.ps1 | 167 ++++++++++++++++------- 3 files changed, 210 insertions(+), 165 deletions(-) diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 index 5dbfdd0..29a091f 100644 --- a/.github/workflows/smart_script.ps1 +++ b/.github/workflows/smart_script.ps1 @@ -17,13 +17,14 @@ $contentTypeMapping = @{ "Workbook"=@("Microsoft.Insights/workbooks"); "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); } -# $csvPath = ".github\workflows\tracking_table.csv" + $githubAuthToken = $json.githubAuthToken $githubRepository = $json.githubRepository -$branchName = "testScript" #change to variable passed through workflow +$branchName = "main" #change to variable passed through workflow $manualDeployment = $json.manualDeployment $sourceControlId = $json.sourceControlId $csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" +$global:localCsvTablefinal = @{} $header = @{ "authorization" = "Bearer $githubAuthToken" @@ -41,16 +42,20 @@ $resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping $MaxRetries = 3 $secondsBetweenAttempts = 5 -function WriteTableToCsv($shaTable) { - if (Test-Path $csvPath) { - Clear-Content -Path $csvPath - } - Add-Content -Path $csvPath -Value "FileName, CommitSha" - $shaTable.GetEnumerator() | ForEach-Object { - "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:localCsvTablefinal.GetEnumerator() | ForEach-Object { + $output += "{0},{1}`n" -f $_.Key, $_.Value } + return $output +} + +$header = @{ + "authorization" = "Bearer $githubAuthToken" } +#Gets all files and commit shas using Get Trees API function GetGithubTree { $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" @@ -58,31 +63,17 @@ function GetGithubTree { return $getTreeResponse } +#Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { - $sha = $null - $path = ".github/workflows/tracking_table_$sourceControlId.csv" - $getTreeResponse.tree | ForEach-Object { - Write-Output $_.path - if ($_.path -eq $path) - { - $sha = $_.sha - } - } - return $sha + return $getTreeResponse.tree | Where-Object { $_.path -eq ".github/workflows/tracking_table_$sourceControlId.csv" } } +#Creates a table using the reponse from the tree api, creates a table function GetCommitShaTable($getTreeResponse) { - #get branch sha and use it to get tree with all commit shas and files - # $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header - # $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - # $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header - # $getTreeResponse = GetGithubTree $shaTable = @{} $getTreeResponse.tree | ForEach-Object { - if ($_.path.Substring($_.path.Length-5) -eq ".json") + if ([System.IO.Path]::GetExtension($_.path) -eq ".json") { - #needs to be $workplace in real implementation - # $truePath = ($Directory + "\" + $_.path).Replace("/", "\") $truePath = $_.path.Replace("/", "\") $shaTable.Add($truePath, $_.sha) } @@ -90,42 +81,49 @@ function GetCommitShaTable($getTreeResponse) { return $shaTable } -function PushCsvToRepo { - #if exists, we need sha of csv file before pushing updated file. If new, no need - $Header = @{ - "authorization" = "Bearer $githubAuthToken" - } +#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. +function PushCsvToRepo($getTreeResponse) { $path = ".github/workflows/tracking_table_$sourceControlId.csv" - Write-Output $path - $createFileUrl = "https://api.github.com/repos/aaroncorreya/SmartTrackingScriptDev/contents/$path" - $content = Get-Content -Path $csvPath | Out-String + $sha = GetCsvCommitSha $getTreeResponse + $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" + $content = ConvertTableToString $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - Write-Output $encodedContent + $body = @{ message = "trackingTable.csv created." content = $encodedContent branch = $branchName + sha = $sha } $Parameters = @{ Method = "PUT" Uri = $createFileUrl - Headers = $Header + Headers = $header Body = $body | ConvertTo-Json } Invoke-RestMethod @Parameters } function ReadCsvToTable { - $mytable = Import-Csv -Path $csvPath + $csvTable = Import-Csv -Path $csvPath $HashTable=@{} - foreach($r in $mytable) + foreach($r in $csvTable) { $HashTable[$r.FileName]=$r.CommitSha } return $HashTable } +#Checks and removes any deleted content files +function CleanDeletedFilesFromTable { + $global:localCsvTablefinal.Clone().GetEnumerator() | ForEach-Object { + if (!(Test-Path -Path $_.Key)) { + $global:localCsvTablefinal.Remove($_.Key) + } + } +} + function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { $maxLoginRetries = 3 $delayInSeconds = 30 @@ -200,9 +198,15 @@ function IsRetryable($deploymentName) { } } function IsValidResourceType($template) { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "Failed to check valid resource type." + $isAllowedResources = $false } return $isAllowedResources } @@ -266,59 +270,44 @@ function GenerateDeploymentName() { return "Sentinel_Deployment_$randomId" } -function CheckFullDeployment() { - $flag = $false - if ((-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true")) { - $flag = $true - } - return $flag -} - -function Deployment($fullDeploymentFlag, $localCsvTable, $remoteShaTable) { - Write-Output "Starting Deployment for Files in path: $Directory" +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" if (Test-Path -Path $Directory) { $totalFiles = 0; $totalFailed = 0; Get-ChildItem -Path $Directory -Recurse -Filter *.json | ForEach-Object { - $path = $_.FullName - # $path = $_.FullName.Replace($workspace, "") + $path = $_.FullName.Replace($Directory + "\", "") $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - #put this into try catch - try { - if (-not (IsValidResourceType $templateObject)) - { - Write-Output "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - } - + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } if ($fullDeploymentFlag) { $result = FullDeployment $path $templateObject - # if (-not $result.isSuccess) {$totalFailed++} } else { - $result = SmartDeployment $localCsvTable $remoteShaTable $path $templateObject - $localCsvTable = $result.csvTable + $result = SmartDeployment $remoteShaTable $path $templateObject } - #convert to global variables if ($result.isSuccess -eq $false) { $totalFailed++ } if (-not $result.skip) { $totalFiles++ } - } + if ($result.isSuccess) { + $global:localCsvTablefinal[$path] = $remoteShaTable[$path] + } + } + CleanDeletedFilesFromTable + PushCsvToRepo $tree if ($totalFiles -gt 0 -and $totalFailed -gt 0) { $err = "$totalFailed of $totalFiles deployments failed." Throw $err } - return $localCsvTable } else { @@ -329,12 +318,10 @@ function Deployment($fullDeploymentFlag, $localCsvTable, $remoteShaTable) { function FullDeployment($path, $templateObject) { try { $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $path $deploymentName $templateObject - $result = @{ + return @{ skip = $false - isSuccess = $isSuccess + isSuccess = AttemptDeployment $path $deploymentName $templateObject } - return $result } catch { Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" @@ -342,26 +329,23 @@ function FullDeployment($path, $templateObject) { } } -function SmartDeployment($localCsvTable, $remoteShaTable, $path, $templateObject) { - try { +function SmartDeployment($remoteShaTable, $path, $templateObject) { + try { $skip = $false - $existingSha = $localCsvTable[$path] + $existingSha = $global:localCsvTablefinal[$path] $remoteSha = $remoteShaTable[$path] if ((!$existingSha) -or ($existingSha -ne $remoteSha)) { $deploymentName = GenerateDeploymentName $isSuccess = AttemptDeployment $path $deploymentName $templateObject - $localCsvTable[$path] = $remoteSha } else { $skip = $true $isSuccess = $null } - $result = @{ + return @{ skip = $skip isSuccess = $isSuccess - csvTable = $localCsvTable } - return $result } catch { Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" @@ -375,38 +359,30 @@ function main() { Write-Output "Attempting Sign In to Azure Cloud" ConnectAzCloud } - $fullDeploymentFlag = CheckFullDeployment - - # if (-not (Test-Path $csvPath)) { - # Write-Output "Creating csv and conducting full deployment." - # $remoteShaTable = GetCommitShaTable $tree - # WriteTableToCsv($remoteShaTable) - # # PushCsvToRepo - # Deployment $fullDeploymentFlag $null $null - # } - # else { - # $localCsvTable = ReadCsvToTable - # $remoteShaTable = GetCommitShaTable $tree - # Write-Output "Local Csv Table" - # Write-Output $localCsvTable - # Write-Output "Remote Csv Table" - # Write-Output $remoteShaTable - # $updatedCsvTable = Deployment $fullDeploymentFlag $localCsvTable $remoteShaTable - # WriteTableToCsv($updatedCsvTable) - # #PushCsvToRepo - # } - Get-ChildItem -Path $Directory -Recurse -Filter *.json | - ForEach-Object { - Write-Output $_.FullName.Replace($Directory + "\", "") + + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + } + + $fullDeploymentFlag = (-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true") + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + +# main + +function ConvertTableToString($table) { + $output = "" + $output += "FileName, CommitSha`n" + $table.GetEnumerator() | ForEach-Object { + $output += "{0},{1}`n" -f $_.Key, $_.Value } + Add-Content -path "output.txt" $output } -main -# $fullDeploymentFlag = CheckFullDeployment -# Write-Output $fullDeploymentFlag -# $tree = GetGithubTree -# $remoteShaTable = GetCommitShaTable $tree -# $localCsvTable = ReadCsvToTable -# Write-Output $remoteShaTable -# Write-Output $localCsvTable \ No newline at end of file +$tree = GetGithubTree +$table = GetCommitShaTable $tree +Write-Output $table +ConvertTableToString $table diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 index 8e38556..e79ce5b 100644 --- a/.github/workflows/test_script.ps1 +++ b/.github/workflows/test_script.ps1 @@ -94,4 +94,6 @@ function main { Write-Output $workspace } -main \ No newline at end of file +# main +$table = GetCommitShaTable +Write-Output $table \ No newline at end of file diff --git a/currentUpdateScript.ps1 b/currentUpdateScript.ps1 index f94b7a8..132e079 100644 --- a/currentUpdateScript.ps1 +++ b/currentUpdateScript.ps1 @@ -18,8 +18,9 @@ $sourceControlId = $Env:sourceControlId $githubAuthToken = $Env:githubAuthToken $githubRepository = $Env:GITHUB_REPOSITORY $branchName = $Env:branch -$workspace = $Env:GITHUB_WORKSPACE + "\" +$manualDeployment = $Env:manualDeployment $csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" +$global:localCsvTablefinal = @{} if ([string]::IsNullOrEmpty($contentTypes)) { $contentTypes = "AnalyticsRule,Metadata" @@ -33,15 +34,13 @@ $resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping $MaxRetries = 3 $secondsBetweenAttempts = 5 -#Writes sha dictionary object to csv file. Will delete any pre-existing content before writing. -function WriteTableToCsv($shaTable) { - if (Test-Path $csvPath) { - Clear-Content -Path $csvPath - } - Add-Content -Path $csvPath -Value "FileName, CommitSha" - $shaTable.GetEnumerator() | ForEach-Object { - "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:localCsvTablefinal.GetEnumerator() | ForEach-Object { + $output += "{0},{1}`n" -f $_.Key, $_.Value } + return $output } $header = @{ @@ -58,15 +57,7 @@ function GetGithubTree { #Gets blob commit sha of the csv file, used when updating csv file to repo function GetCsvCommitSha($getTreeResponse) { - $sha = $null - $path = ".github/workflows/tracking_table_$sourceControlId.csv" - $getTreeResponse.tree | ForEach-Object { - if ($_.path -eq $path) - { - $sha = $_.sha - } - } - return $sha + return $getTreeResponse.tree | Where-Object { $_.path -eq ".github/workflows/tracking_table_$sourceControlId.csv" } } #Creates a table using the reponse from the tree api, creates a table @@ -87,7 +78,7 @@ function PushCsvToRepo($getTreeResponse) { $path = ".github/workflows/tracking_table_$sourceControlId.csv" $sha = GetCsvCommitSha $getTreeResponse $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" - $content = Get-Content -Path $csvPath | Out-String + $content = ConvertTableToString $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) $body = @{ @@ -106,6 +97,25 @@ function PushCsvToRepo($getTreeResponse) { Invoke-RestMethod @Parameters } +function ReadCsvToTable { + $csvTable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $csvTable) + { + $HashTable[$r.FileName]=$r.CommitSha + } + return $HashTable +} + +#Checks and removes any deleted content files +function CleanDeletedFilesFromTable { + $global:localCsvTablefinal.Clone().GetEnumerator() | ForEach-Object { + if (!(Test-Path -Path $_.Key)) { + $global:localCsvTablefinal.Remove($_.Key) + } + } +} + function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { $maxLoginRetries = 3 $delayInSeconds = 30 @@ -181,9 +191,15 @@ function IsRetryable($deploymentName) { } function IsValidResourceType($template) { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "[Error] Failed to check valid resource type." + $isAllowedResources = $false } return $isAllowedResources } @@ -247,43 +263,39 @@ function GenerateDeploymentName() { return "Sentinel_Deployment_$randomId" } -function main() { - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - Write-Output "Starting Deployment for Files in path: $Directory" - +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" if (Test-Path -Path $Directory) { $totalFiles = 0; $totalFailed = 0; Get-ChildItem -Path $Directory -Recurse -Filter *.json | ForEach-Object { - $path = $_.FullName - try { - $totalFiles ++ - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Output "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $_.FullName $deploymentName $templateObject - if (-not $isSuccess) - { - $totalFailed++ - } + $path = $_.FullName.Replace($Directory + "\", "") + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + if ($fullDeploymentFlag) { + $result = FullDeployment $path $templateObject } - catch { + else { + $result = SmartDeployment $remoteShaTable $path $templateObject + } + if ($result.isSuccess -eq $false) { $totalFailed++ - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace } - } + if (-not $result.skip) { + $totalFiles++ + } + if ($result.isSuccess) { + $global:localCsvTablefinal[$path] = $remoteShaTable[$path] + } + } + CleanDeletedFilesFromTable + PushCsvToRepo $tree if ($totalFiles -gt 0 -and $totalFailed -gt 0) { $err = "$totalFailed of $totalFiles deployments failed." @@ -296,4 +308,59 @@ function main() { } } +function FullDeployment($path, $templateObject) { + try { + $deploymentName = GenerateDeploymentName + return @{ + skip = $false + isSuccess = AttemptDeployment $path $deploymentName $templateObject + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function SmartDeployment($remoteShaTable, $path, $templateObject) { + try { + $skip = $false + $existingSha = $global:localCsvTablefinal[$path] + $remoteSha = $remoteShaTable[$path] + if ((!$existingSha) -or ($existingSha -ne $remoteSha)) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $deploymentName $templateObject + } + else { + $skip = $true + $isSuccess = $null + } + return @{ + skip = $skip + isSuccess = $isSuccess + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function main() { + if ($CloudEnv -ne 'AzureCloud') + { + Write-Output "Attempting Sign In to Azure Cloud" + ConnectAzCloud + } + + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + } + + $fullDeploymentFlag = (-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true") + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + main \ No newline at end of file From dddead34c578685e72494f4214e3178b31460a6c Mon Sep 17 00:00:00 2001 From: Aaron Correya <34196924+aaroncorreya@users.noreply.github.com> Date: Mon, 14 Mar 2022 12:14:16 -0700 Subject: [PATCH 84/90] Delete .github directory --- .github/workflows/script.ps1 | 223 ------------- .github/workflows/smart_script.ps1 | 388 ---------------------- .github/workflows/testWorkflow.yml | 39 --- .github/workflows/test_script.ps1 | 99 ------ .github/workflows/tracking_table_1234.csv | 3 - 5 files changed, 752 deletions(-) delete mode 100644 .github/workflows/script.ps1 delete mode 100644 .github/workflows/smart_script.ps1 delete mode 100644 .github/workflows/testWorkflow.yml delete mode 100644 .github/workflows/test_script.ps1 delete mode 100644 .github/workflows/tracking_table_1234.csv diff --git a/.github/workflows/script.ps1 b/.github/workflows/script.ps1 deleted file mode 100644 index cb0657e..0000000 --- a/.github/workflows/script.ps1 +++ /dev/null @@ -1,223 +0,0 @@ -#read variables from json for dev -$json = (Get-Content "C:\One\SmartTrackingScriptDev\environment_df.json" -Raw) | ConvertFrom-Json -Write-Output $json -## Globals ## -$CloudEnv = $json.cloudEnv -$ResourceGroupName = $json.resourceGroupName -$WorkspaceName = $json.workspaceName -$Directory = $json.directory -$Creds = $json.creds -$contentTypes = $json.contentTypes -$contentTypeMapping = @{ - "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); - "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); - "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); - "Workbook"=@("Microsoft.Insights/workbooks"); - "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); -} - -if ([string]::IsNullOrEmpty($contentTypes)) { - $contentTypes = "AnalyticsRule,Metadata" -} - -if (-not ($contentTypes.contains("Metadata"))) { - $contentTypes += ",Metadata" -} - -$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } -$MaxRetries = 3 -$secondsBetweenAttempts = 5 - -function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { - $maxLoginRetries = 3 - $delayInSeconds = 30 - $retryCount = 1 - $stopTrying = $false - do { - try { - Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; - Write-Host "Login Successful" - $stopTrying = $true - } - catch { - if ($retryCount -ge $maxLoginRetries) { - Write-Host "Login failed after $maxLoginRetries attempts." - $stopTrying = $true - } - else { - Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." - Start-Sleep -Seconds $delayInSeconds - $retryCount++ - } - } - } - while (-not $stopTrying) -} - -function ConnectAzCloud { - $RawCreds = $Creds | ConvertFrom-Json - - Clear-AzContext -Scope Process; - Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; - - Add-AzEnvironment ` - -Name $CloudEnv ` - -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` - -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` - -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` - -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; - - $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force - $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) - - AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv - Set-AzContext -Tenant $RawCreds.tenantId | out-null; -} - -function IsValidTemplate($path, $templateObject) { - Try { - if (DoesContainWorkspaceParam $templateObject) { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName - } - else { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path - } - - return $true - } - Catch { - Write-Host "[Warning] The file $path is not valid: $_" - return $false - } -} - -function IsRetryable($deploymentName) { - $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" - Try { - $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop - return $retryableStatusCodes -contains $deploymentResult.StatusCode - } - Catch { - return $false - } -} - -function IsValidResourceType($template) { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources - } - return $isAllowedResources -} - -function DoesContainWorkspaceParam($templateObject) { - $templateObject.parameters.PSobject.Properties.Name -contains "workspace" -} - -function AttemptDeployment($path, $deploymentName, $templateObject) { - Write-Host "[Info] Deploying $path with deployment name $deploymentName" - - $isValid = IsValidTemplate $path $templateObject - if (-not $isValid) { - return $false - } - $isSuccess = $false - $currentAttempt = 0 - While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) - { - $currentAttempt ++ - Try - { - if (DoesContainWorkspaceParam $templateObject) - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host - } - - $isSuccess = $true - } - Catch [Exception] - { - $err = $_ - if (-not (IsRetryable $deploymentName)) - { - Write-Host "[Warning] Failed to deploy $path with error: $err" - break - } - else - { - if ($currentAttempt -le $MaxRetries) - { - Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." - Start-Sleep -Seconds $secondsBetweenAttempts - } - else - { - Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" - } - } - } - } - return $isSuccess -} - -function GenerateDeploymentName() { - $randomId = [guid]::NewGuid() - return "Sentinel_Deployment_$randomId" -} - -function main() { - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - Write-Output "Starting Deployment for Files in path: $Directory" - - if (Test-Path -Path $Directory) - { - $totalFiles = 0; - $totalFailed = 0; - Get-ChildItem -Path $Directory -Recurse -Filter *.json | - ForEach-Object { - $path = $_.FullName - try { - $totalFiles ++ - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Output "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $_.FullName $deploymentName $templateObject - if (-not $isSuccess) - { - $totalFailed++ - } - } - catch { - $totalFailed++ - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } - } - if ($totalFiles -gt 0 -and $totalFailed -gt 0) - { - $err = "$totalFailed of $totalFiles deployments failed." - Throw $err - } - } - else - { - Write-Output "[Warning] $Directory not found. nothing to deploy" - } -} - -main \ No newline at end of file diff --git a/.github/workflows/smart_script.ps1 b/.github/workflows/smart_script.ps1 deleted file mode 100644 index 29a091f..0000000 --- a/.github/workflows/smart_script.ps1 +++ /dev/null @@ -1,388 +0,0 @@ -#read variables from json for dev -$json = (Get-Content "C:\One\SmartTrackingScriptDev\environment_df.json" -Raw) | ConvertFrom-Json -Write-Output $json -## Globals ## -$CloudEnv = $json.cloudEnv -$ResourceGroupName = $json.resourceGroupName -$WorkspaceName = $json.workspaceName -$Directory = $json.directory -$Creds = $json.creds -$contentTypes = $json.contentTypes -$contentTypeMapping = @{ - "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); - "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); - "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); - "Workbook"=@("Microsoft.Insights/workbooks"); - "Metadata"=@("Microsoft.OperationalInsights/workspaces/providers/metadata"); -} - -$githubAuthToken = $json.githubAuthToken -$githubRepository = $json.githubRepository -$branchName = "main" #change to variable passed through workflow -$manualDeployment = $json.manualDeployment -$sourceControlId = $json.sourceControlId -$csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" -$global:localCsvTablefinal = @{} - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -if ([string]::IsNullOrEmpty($contentTypes)) { - $contentTypes = "AnalyticsRule,Metadata" -} - -if (-not ($contentTypes.contains("Metadata"))) { - $contentTypes += ",Metadata" -} - -$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } -$MaxRetries = 3 -$secondsBetweenAttempts = 5 - -#Converts hashtable to string that can be set as content when pushing csv file -function ConvertTableToString { - $output = "FileName, CommitSha`n" - $global:localCsvTablefinal.GetEnumerator() | ForEach-Object { - $output += "{0},{1}`n" -f $_.Key, $_.Value - } - return $output -} - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -#Gets all files and commit shas using Get Trees API -function GetGithubTree { - $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header - $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header - return $getTreeResponse -} - -#Gets blob commit sha of the csv file, used when updating csv file to repo -function GetCsvCommitSha($getTreeResponse) { - return $getTreeResponse.tree | Where-Object { $_.path -eq ".github/workflows/tracking_table_$sourceControlId.csv" } -} - -#Creates a table using the reponse from the tree api, creates a table -function GetCommitShaTable($getTreeResponse) { - $shaTable = @{} - $getTreeResponse.tree | ForEach-Object { - if ([System.IO.Path]::GetExtension($_.path) -eq ".json") - { - $truePath = $_.path.Replace("/", "\") - $shaTable.Add($truePath, $_.sha) - } - } - return $shaTable -} - -#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. -function PushCsvToRepo($getTreeResponse) { - $path = ".github/workflows/tracking_table_$sourceControlId.csv" - $sha = GetCsvCommitSha $getTreeResponse - $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" - $content = ConvertTableToString - $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - - $body = @{ - message = "trackingTable.csv created." - content = $encodedContent - branch = $branchName - sha = $sha - } - - $Parameters = @{ - Method = "PUT" - Uri = $createFileUrl - Headers = $header - Body = $body | ConvertTo-Json - } - Invoke-RestMethod @Parameters -} - -function ReadCsvToTable { - $csvTable = Import-Csv -Path $csvPath - $HashTable=@{} - foreach($r in $csvTable) - { - $HashTable[$r.FileName]=$r.CommitSha - } - return $HashTable -} - -#Checks and removes any deleted content files -function CleanDeletedFilesFromTable { - $global:localCsvTablefinal.Clone().GetEnumerator() | ForEach-Object { - if (!(Test-Path -Path $_.Key)) { - $global:localCsvTablefinal.Remove($_.Key) - } - } -} - -function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { - $maxLoginRetries = 3 - $delayInSeconds = 30 - $retryCount = 1 - $stopTrying = $false - do { - try { - Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; - Write-Host "Login Successful" - $stopTrying = $true - } - catch { - if ($retryCount -ge $maxLoginRetries) { - Write-Host "Login failed after $maxLoginRetries attempts." - $stopTrying = $true - } - else { - Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." - Start-Sleep -Seconds $delayInSeconds - $retryCount++ - } - } - } - while (-not $stopTrying) -} - -function ConnectAzCloud { - $RawCreds = $Creds | ConvertFrom-Json - - Clear-AzContext -Scope Process; - Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; - - Add-AzEnvironment ` - -Name $CloudEnv ` - -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` - -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` - -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` - -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; - - $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force - $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) - - AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv - Set-AzContext -Tenant $RawCreds.tenantId | out-null; -} - -function IsValidTemplate($path, $templateObject) { - Try { - if (DoesContainWorkspaceParam $templateObject) { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName - } - else { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path - } - - return $true - } - Catch { - Write-Host "[Warning] The file $path is not valid: $_" - return $false - } -} - -function IsRetryable($deploymentName) { - $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" - Try { - $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop - return $retryableStatusCodes -contains $deploymentResult.StatusCode - } - Catch { - return $false - } -} -function IsValidResourceType($template) { - try { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources - } - } - catch { - Write-Host "Failed to check valid resource type." - $isAllowedResources = $false - } - return $isAllowedResources -} - -function DoesContainWorkspaceParam($templateObject) { - $templateObject.parameters.PSobject.Properties.Name -contains "workspace" -} - -function AttemptDeployment($path, $deploymentName, $templateObject) { - Write-Host "[Info] Deploying $path with deployment name $deploymentName" - - $isValid = IsValidTemplate $path $templateObject - if (-not $isValid) { - return $false - } - $isSuccess = $false - $currentAttempt = 0 - While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) - { - $currentAttempt ++ - Try - { - if (DoesContainWorkspaceParam $templateObject) - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host - } - - $isSuccess = $true - } - Catch [Exception] - { - $err = $_ - if (-not (IsRetryable $deploymentName)) - { - Write-Host "[Warning] Failed to deploy $path with error: $err" - break - } - else - { - if ($currentAttempt -le $MaxRetries) - { - Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." - Start-Sleep -Seconds $secondsBetweenAttempts - } - else - { - Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" - } - } - } - } - return $isSuccess -} - -function GenerateDeploymentName() { - $randomId = [guid]::NewGuid() - return "Sentinel_Deployment_$randomId" -} - -function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { - Write-Host "Starting Deployment for Files in path: $Directory" - if (Test-Path -Path $Directory) - { - $totalFiles = 0; - $totalFailed = 0; - Get-ChildItem -Path $Directory -Recurse -Filter *.json | - ForEach-Object { - $path = $_.FullName.Replace($Directory + "\", "") - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - if ($fullDeploymentFlag) { - $result = FullDeployment $path $templateObject - } - else { - $result = SmartDeployment $remoteShaTable $path $templateObject - } - if ($result.isSuccess -eq $false) { - $totalFailed++ - } - if (-not $result.skip) { - $totalFiles++ - } - if ($result.isSuccess) { - $global:localCsvTablefinal[$path] = $remoteShaTable[$path] - } - } - CleanDeletedFilesFromTable - PushCsvToRepo $tree - if ($totalFiles -gt 0 -and $totalFailed -gt 0) - { - $err = "$totalFailed of $totalFiles deployments failed." - Throw $err - } - } - else - { - Write-Output "[Warning] $Directory not found. nothing to deploy" - } -} - -function FullDeployment($path, $templateObject) { - try { - $deploymentName = GenerateDeploymentName - return @{ - skip = $false - isSuccess = AttemptDeployment $path $deploymentName $templateObject - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function SmartDeployment($remoteShaTable, $path, $templateObject) { - try { - $skip = $false - $existingSha = $global:localCsvTablefinal[$path] - $remoteSha = $remoteShaTable[$path] - if ((!$existingSha) -or ($existingSha -ne $remoteSha)) { - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $path $deploymentName $templateObject - } - else { - $skip = $true - $isSuccess = $null - } - return @{ - skip = $skip - isSuccess = $isSuccess - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function main() { - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - if (Test-Path $csvPath) { - $global:localCsvTablefinal = ReadCsvToTable - } - - $fullDeploymentFlag = (-not (Test-Path $csvPath)) -or ($manualDeployment -eq "true") - $tree = GetGithubTree - $remoteShaTable = GetCommitShaTable $tree - Deployment $fullDeploymentFlag $remoteShaTable $tree -} - -# main - -function ConvertTableToString($table) { - $output = "" - $output += "FileName, CommitSha`n" - $table.GetEnumerator() | ForEach-Object { - $output += "{0},{1}`n" -f $_.Key, $_.Value - } - Add-Content -path "output.txt" $output -} - - -$tree = GetGithubTree -$table = GetCommitShaTable $tree -Write-Output $table -ConvertTableToString $table diff --git a/.github/workflows/testWorkflow.yml b/.github/workflows/testWorkflow.yml deleted file mode 100644 index 80e5be0..0000000 --- a/.github/workflows/testWorkflow.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: "**" - pull_request: - types: [assigned, opened, synchronize, reopened] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: windows-latest - env: - repository: aaroncorreya/SmartTrackingScriptDev - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - sourceControlId: "1234" - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Runs a single command using the runners shell - - name: Run a one-line script - run: echo "${{ env.repository }}" - - # Runs a set of commands using the runners shell - - name: Run script - run: | - ./.github/workflows/test_script.ps1 \ No newline at end of file diff --git a/.github/workflows/test_script.ps1 b/.github/workflows/test_script.ps1 deleted file mode 100644 index e79ce5b..0000000 --- a/.github/workflows/test_script.ps1 +++ /dev/null @@ -1,99 +0,0 @@ -$githubAuthToken = $Env:githubAuthToken -$githubRepository = $Env:GITHUB_REPOSITORY -$refName = $Env:GITHUB_REF -$branchName = $refName.Replace("refs/heads/", "") -#$branchName = $Env:branch -$workspace = $Env:GITHUB_WORKSPACE + "\" -$sourceControlId = $Env:sourceControlId -$csvPath = ".github\workflows\tracking_table_$sourceControlId.csv" - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -#Writes sha dictionary object to csv file. Will delete any pre-existing content before writing. -function WriteTableToCsv($shaTable) { - if (Test-Path $csvPath) { - Clear-Content -Path $csvPath - } - Add-Content -Path $csvPath -Value "FileName, CommitSha" - $shaTable.GetEnumerator() | ForEach-Object { - "{0},{1}" -f $_.Key, $_.Value | add-content -path $csvPath - } -} - -#Gets all files and commit shas using Get Trees API -function GetGithubTree { - $branchResponse = Invoke-RestMethod https://api.github.com/repos/$githubRepository/branches/$branchName -Headers $header - $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - $getTreeResponse = Invoke-RestMethod $treeUrl -Headers $header - return $getTreeResponse -} - -#Gets blob commit sha of the csv file, used when updating csv file to repo -function GetCsvCommitSha($getTreeResponse) { - return $getTreeResponse.tree | Where-Object { $_.path -eq ".github/workflows/tracking_table_$sourceControlId.csv" } -} - -#Creates a table using the reponse from the tree api, creates a table -function GetCommitShaTable($getTreeResponse) { - $shaTable = @{} - $getTreeResponse.tree | ForEach-Object { - #if ($_.path.Substring($_.path.Length-5) -eq ".json") - if ([System.IO.Path]::GetExtension($_.path) -eq ".json") - { - #$truePath = ($workspace + "\" + $_.path).Replace("/", "\") - $truePath = $_.path.Replace("/", "\") - $shaTable.Add($truePath, $_.sha) - } - } - return $shaTable -} - -#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. -#TODO: Add source control id to tracking_table name. -function PushCsvToRepo($getTreeResponse) { - $path = ".github/workflows/tracking_table_$sourceControlId.csv" - Write-Output $path - $sha = GetCsvCommitSha $getTreeResponse - $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" - $content = Get-Content -Path $csvPath | Out-String - $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - Write-Output $encodedContent - $body = @{ - message = "trackingTable.csv created." - content = $encodedContent - branch = $branchName - sha = $sha - } - - $Parameters = @{ - Method = "PUT" - Uri = $createFileUrl - Headers = $header - Body = $body | ConvertTo-Json - } - Invoke-RestMethod @Parameters -} - -function main { - Write-Output $githubRepository - $tree = GetGithubTree - $shaTable = GetCommitShaTable $tree - WriteTableToCsv $shaTable - PushCsvToRepo $tree - Write-Output "SHA TABLE" - Write-Output $shaTable - - #TODO: Make sure that the paths are the same when testing locally and remotely - Get-ChildItem -Path $workspace -Recurse -Filter *.json | - ForEach-Object { - $path = $_.FullName.Replace($workspace, "") - Write-Output $path - } - Write-Output $workspace -} - -# main -$table = GetCommitShaTable -Write-Output $table \ No newline at end of file diff --git a/.github/workflows/tracking_table_1234.csv b/.github/workflows/tracking_table_1234.csv deleted file mode 100644 index 9d741b2..0000000 --- a/.github/workflows/tracking_table_1234.csv +++ /dev/null @@ -1,3 +0,0 @@ -FileName, CommitSha -Deployments\Parsers\DnsEmpty.json,e8db6e88ccf4faa78170a1e814ce5c9d71b5ac0c -Deployments\Parsers\ASimAuthentication.json,b6e1709065f0917867f4cc0d1476aff4dadcb89d From 3b327d4365cd7906ff2c0c95562e62a703282d16 Mon Sep 17 00:00:00 2001 From: Aaron Correya <34196924+aaroncorreya@users.noreply.github.com> Date: Thu, 2 Jun 2022 16:31:54 -0700 Subject: [PATCH 85/90] Sentinel Content Deployment Script --- ...y-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 | 584 ++++++++++++++++++ 1 file changed, 584 insertions(+) create mode 100644 .github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 diff --git a/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 b/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 new file mode 100644 index 0000000..ba5aeee --- /dev/null +++ b/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 @@ -0,0 +1,584 @@ +## Globals ## +$CloudEnv = $Env:cloudEnv +$ResourceGroupName = $Env:resourceGroupName +$WorkspaceName = $Env:workspaceName +$WorkspaceId = $Env:workspaceId +$Directory = $Env:directory +$Creds = $Env:creds +$contentTypes = $Env:contentTypes +$contentTypeMapping = @{ + "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); + "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); + "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); + "Workbook"=@("Microsoft.Insights/workbooks"); +} +$sourceControlId = $Env:sourceControlId +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:GITHUB_REPOSITORY +$branchName = $Env:branch +$smartDeployment = $Env:smartDeployment +$csvPath = ".sentinel\tracking_table_$sourceControlId.csv" +$configPath = "sentinel-deployment.config" +$global:localCsvTablefinal = @{} +$global:updatedCsvTable = @{} +$global:parameterFileMapping = @{} +$global:prioritizedContentFiles = @() +$global:excludeContentFiles = @() + +$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' +$namePattern = '([-\w\._\(\)]+)' +$sentinelResourcePatterns = @{ + "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" + "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" + "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" + "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" +} + +if ([string]::IsNullOrEmpty($contentTypes)) { + $contentTypes = "AnalyticsRule" +} + +$metadataFilePath = "metadata.json" +@" +{ + "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "parentResourceId": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "sourceControlId": { + "type": "string" + }, + "workspace": { + "type": "string" + }, + "contentId": { + "type": "string" + } + }, + "variables": { + "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", + "properties": { + "parentId": "[parameters('parentResourceId')]", + "kind": "[parameters('kind')]", + "source": { + "kind": "SourceRepository", + "name": "Repositories", + "sourceId": "[parameters('sourceControlId')]" + } + } + } + ] +} +"@ | Out-File -FilePath $metadataFilePath + +$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } +$MaxRetries = 3 +$secondsBetweenAttempts = 5 + +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:updatedCsvTable.GetEnumerator() | ForEach-Object { + $output += "{0},{1}`n" -f $_.Key, $_.Value + } + return $output +} + +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + +#Gets all files and commit shas using Get Trees API +function GetGithubTree { + $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 + return $getTreeResponse +} + +#Gets blob commit sha of the csv file, used when updating csv file to repo +function GetCsvCommitSha($getTreeResponse) { + $shaObject = $getTreeResponse.tree | Where-Object { $_.path -eq $csvPath.Replace("\", "/") } + return $shaObject.sha +} + +#Creates a table using the reponse from the tree api, creates a table +function GetCommitShaTable($getTreeResponse) { + $shaTable = @{} + $getTreeResponse.tree | ForEach-Object { + if (([System.IO.Path]::GetExtension($_.path) -eq ".json") -or ($_.path -eq $configPath)) + { + $truePath = $_.path.Replace("/", "\") + $shaTable.Add($truePath, $_.sha) + } + } + return $shaTable +} + +#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. +function PushCsvToRepo($getTreeResponse) { + $path = $csvPath.Replace("\", "/") + $sha = GetCsvCommitSha $getTreeResponse + $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" + $content = ConvertTableToString + $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) + + $body = @{ + message = "trackingTable.csv created." + content = $encodedContent + branch = $branchName + sha = $sha + } | ConvertTo-Json + + $Parameters = @{ + Method = "PUT" + Uri = $createFileUrl + Headers = $header + Body = $body | ConvertTo-Json + } + AttemptInvokeRestMethod "Put" $createFileUrl $body $null 3 +} + +function ReadCsvToTable { + $csvTable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $csvTable) + { + $HashTable[$r.FileName]=$r.CommitSha + } + return $HashTable +} + +#Checks and removes any deleted content files +function CleanDeletedFilesFromTable { + $global:updatedCsvTable.Clone().GetEnumerator() | ForEach-Object { + if (!(Test-Path -Path $_.Key)) { + $global:updatedCsvTable.Remove($_.Key) + } + } +} + +function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { + $Stoploop = $false + $retryCount = 0 + do { + try { + $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes + $Stoploop = $true + } + catch { + if ($retryCount -gt $maxRetries) { + Write-Host "[Error] API call failed after $retryCount retries: $_" + $Stoploop = $true + } + else { + Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." + Start-Sleep -Seconds 5 + $retryCount = $retryCount + 1 + } + } + } + While ($Stoploop -eq $false) + return $result +} + +function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { + $maxLoginRetries = 3 + $delayInSeconds = 30 + $retryCount = 1 + $stopTrying = $false + do { + try { + Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; + Write-Host "Login Successful" + $stopTrying = $true + } + catch { + if ($retryCount -ge $maxLoginRetries) { + Write-Host "Login failed after $maxLoginRetries attempts." + $stopTrying = $true + } + else { + Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." + Start-Sleep -Seconds $delayInSeconds + $retryCount++ + } + } + } + while (-not $stopTrying) +} + +function ConnectAzCloud { + $RawCreds = $Creds | ConvertFrom-Json + + Clear-AzContext -Scope Process; + Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; + + Add-AzEnvironment ` + -Name $CloudEnv ` + -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` + -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` + -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` + -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; + + $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force + $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) + + AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv + Set-AzContext -Tenant $RawCreds.tenantId | out-null; +} + +function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject) { + $deploymentInfo = $null + try { + $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore + } + catch { + Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" + return + } + $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { + $resource = $_.TargetResource + $sentinelContentKinds = GetContentKinds $resource + if ($sentinelContentKinds.Count -gt 0) { + $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject + $contentId = $resource.Split("/")[-1] + try { + New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` + -parentResourceId $resource ` + -kind $contentKind ` + -contentId $contentId ` + -sourceControlId $sourceControlId ` + -workspace $workspaceName ` + -ErrorAction Stop | Out-Host + Write-Host "[Info] Created metadata metadata for $contentKind with parent resource id $resource" + } + catch { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error $_" + } + } + } +} + +function GetContentKinds($resource) { + return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } +} + +function ToContentKind($contentKinds, $resource, $templateObject) { + if ($contentKinds.Count -eq 1) { + return $contentKinds + } + if ($null -ne $resource -and $resource.Contains('savedSearches')) { + if ($templateObject.resources.properties.Category -eq "Hunting Queries") { + return "HuntingQuery" + } + return "Parser" + } + return $null +} + +function IsValidTemplate($path, $templateObject) { + Try { + if (DoesContainWorkspaceParam $templateObject) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName + } + else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path + } + + return $true + } + Catch { + Write-Host "[Warning] The file $path is not valid: $_" + return $false + } +} + +function IsRetryable($deploymentName) { + $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" + Try { + $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop + return $retryableStatusCodes -contains $deploymentResult.StatusCode + } + Catch { + return $false + } +} + +function IsValidResourceType($template) { + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "[Error] Failed to check valid resource type." + $isAllowedResources = $false + } + return $isAllowedResources +} + +function DoesContainWorkspaceParam($templateObject) { + $templateObject.parameters.PSobject.Properties.Name -contains "workspace" +} + +function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject) { + Write-Host "[Info] Deploying $path with deployment name $deploymentName" + + $isValid = IsValidTemplate $path $templateObject + if (-not $isValid) { + return $false + } + $isSuccess = $false + $currentAttempt = 0 + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" + if (DoesContainWorkspaceParam $templateObject) + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host + } + } + else + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host + } + } + AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject + + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable $deploymentName)) + { + Write-Host "[Warning] Failed to deploy $path with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" + } + } + } + } + return $isSuccess +} + +function GenerateDeploymentName() { + $randomId = [guid]::NewGuid() + return "Sentinel_Deployment_$randomId" +} + +#Load deployment configuration +function LoadDeploymentConfig() { + Write-Host "[Info] load the deployment configuration from [$configPath]" + $global:parameterFileMapping = @{} + $global:prioritizedContentFiles = @() + $global:excludeContentFiles = @() + try { + if (Test-Path $configPath) { + $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json + $parameterFileMappings = @{} + if ($deployment_config.parameterfilemappings) { + $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } + } + $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) + if ($null -ne $key) { + $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } + } + if ($deployment_config.prioritizedcontentfiles) { + $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles + } + $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles + if ($deployment_config.excludecontentfiles) { + $excludeList = $excludeList + $deployment_config.excludecontentfiles + } + $global:excludeContentFiles = $excludeList | Where-Object { Test-Path $_ } + } + } + catch { + Write-Host "[Warning] An error occurred while trying to load deployment configuration." + Write-Host "Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function filterContentFile($path) { + $temp = $path.Replace($Directory + "\", "").Replace("\", "/") + return $global:excludeContentFiles | ? {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} +} + +#resolve parameter file name, return $null if there is none. +function GetParameterFile($path) { + $index = $path.Replace("\", "/") + $key = ($global:parameterFileMapping.Keys | ? { $_ -eq $index }) + if ($key) { + $mappedParameterFile = $global:parameterFileMapping[$key].Replace("/", "\") + if (Test-Path $mappedParameterFile) { + return $mappedParameterFile + } + } + + $parameterFilePrefix = $path.TrimEnd(".json") + + $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + + $defaultParameterFile = $parameterFilePrefix + ".parameters.json" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + + return $null +} + +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" + if (Test-Path -Path $Directory) + { + $totalFiles = 0; + $totalFailed = 0; + $iterationList = @() + $global:prioritizedContentFiles | ForEach-Object { $iterationList += $_.Replace("/", "\") } + Get-ChildItem -Path $Directory -Recurse -Filter *.json -exclude *metadata.json, *.parameters*.json | + Where-Object { $null -eq ( filterContentFile $_.FullName ) } | + Select-Object -Property FullName | + ForEach-Object { $iterationList += $_.FullName.Replace($Directory + "\", "") } + $iterationList | ForEach-Object { + $path = $_ + Write-Host "[Info] Try to deploy $path" + if (-not (Test-Path $path)) { + Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." + return + } + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + #parameterFile = GetParameterFile $path + $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject + if ($result.isSuccess -eq $false) { + $totalFailed++ + } + if (-not $result.skip) { + $totalFiles++ + } + if ($result.isSuccess) { + $global:updatedCsvTable[$path] = $remoteShaTable[$path] + if ($parameterFile) { + $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] + } + } + } + CleanDeletedFilesFromTable + PushCsvToRepo $tree + if ($totalFiles -gt 0 -and $totalFailed -gt 0) + { + $err = "$totalFailed of $totalFiles deployments failed." + Throw $err + } + } + else + { + Write-Output "[Warning] $Directory not found. nothing to deploy" + } +} + +function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject) { + try { + $skip = $false + $isSuccess = $null + if (!$fullDeploymentFlag) { + $existingSha = $global:localCsvTablefinal[$path] + $remoteSha = $remoteShaTable[$path] + $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) + if ($skip -and $parameterFile) { + $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] + $remoteShaForParameterFile = $remoteShaTable[$parameterFile] + $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) + } + } + if (!$skip) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject + } + return @{ + skip = $skip + isSuccess = $isSuccess + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function main() { + if ($CloudEnv -ne 'AzureCloud') + { + Write-Output "Attempting Sign In to Azure Cloud" + ConnectAzCloud + } + + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + $global:updatedCsvTable = $global:localCsvTablefinal.Clone() + } + + LoadDeploymentConfig + + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + + $existingConfigSha = $global:localCsvTablefinal[$configPath] + $remoteConfigSha = $remoteShaTable[$configPath] + $modifiedConfig = ((!$existingConfigSha) -or ($existingConfigSha -ne $remoteConfigSha)) + $global:updatedCsvTable[$configPath] = $remoteConfigSha + + $fullDeploymentFlag = $modifiedConfig -or (-not (Test-Path $csvPath)) -or ($smartDeployment -eq "false") + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + +main \ No newline at end of file From 236d6684c706469359f84f93eb6e3d9fee4390b1 Mon Sep 17 00:00:00 2001 From: Aaron Correya <34196924+aaroncorreya@users.noreply.github.com> Date: Thu, 2 Jun 2022 16:31:55 -0700 Subject: [PATCH 86/90] Workflow file for Sentinel-Deploy --- ...y-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml diff --git a/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml b/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml new file mode 100644 index 0000000..862334a --- /dev/null +++ b/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml @@ -0,0 +1,80 @@ +name: Deploy Content to acorreya-test-workspace [7a5e8eb7-bfa9-467d-88e4-f4169e0718c3] +# Note: This workflow will deploy everything in the root directory. +# To deploy content only from a specific path (for example SentinelContent): +# 1. Add the target path to the "paths" property like such +# paths: +# - 'SentinelContent/**' +# - '!.github/workflows/**' +# - '.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml' +# 2. Append the path to the directory environment variable below +# directory: '${{ github.workspace }}/SentinelContent' + +on: + push: + branches: [ testScript ] + paths: + - '**' + - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow + - '.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml' + +jobs: + deploy-content: + runs-on: windows-latest + env: + resourceGroupName: 'sentineleco_canarytest_rg' + workspaceName: 'acorreya-test-workspace' + workspaceId: '6dfd26b3-bbeb-443c-a60d-e8da51a558eb' + directory: '${{ github.workspace }}' + cloudEnv: 'AzureCloud' + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} + contentTypes: 'AnalyticsRule' + branch: 'testScript' + sourceControlId: '7a5e8eb7-bfa9-467d-88e4-f4169e0718c3' + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + smartDeployment: 'true' + + steps: + - name: Login to Azure (Attempt 1) + continue-on-error: true + id: login1 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 1 failed + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 2) + continue-on-error: true + id: login2 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 2 failed + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 3) + continue-on-error: false + id: login3 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} + enable-AzPSSession: true + + - name: Checkout + uses: actions/checkout@v1 + + - name: Deploy Content to Azure Sentinel + uses: azure/powershell@v1 + with: + azPSVersion: 'latest' + inlineScript: | + ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 \ No newline at end of file From c02f943a8623b29877550ac3fe48faef4573d62b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Jun 2022 23:36:27 +0000 Subject: [PATCH 87/90] trackingTable.csv created. --- .../tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv | 1 + 1 file changed, 1 insertion(+) create mode 100644 .sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv diff --git a/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv b/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv new file mode 100644 index 0000000..408f4f9 --- /dev/null +++ b/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv @@ -0,0 +1 @@ +FileName, CommitSha From 54a3c6794330091e8b825334563c59bc775d0fc8 Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 23:42:32 +0000 Subject: [PATCH 88/90] Remove sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml --- ...y-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml | 80 ------------------- 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml diff --git a/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml b/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml deleted file mode 100644 index 862334a..0000000 --- a/.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Deploy Content to acorreya-test-workspace [7a5e8eb7-bfa9-467d-88e4-f4169e0718c3] -# Note: This workflow will deploy everything in the root directory. -# To deploy content only from a specific path (for example SentinelContent): -# 1. Add the target path to the "paths" property like such -# paths: -# - 'SentinelContent/**' -# - '!.github/workflows/**' -# - '.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml' -# 2. Append the path to the directory environment variable below -# directory: '${{ github.workspace }}/SentinelContent' - -on: - push: - branches: [ testScript ] - paths: - - '**' - - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow - - '.github/workflows/sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.yml' - -jobs: - deploy-content: - runs-on: windows-latest - env: - resourceGroupName: 'sentineleco_canarytest_rg' - workspaceName: 'acorreya-test-workspace' - workspaceId: '6dfd26b3-bbeb-443c-a60d-e8da51a558eb' - directory: '${{ github.workspace }}' - cloudEnv: 'AzureCloud' - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} - contentTypes: 'AnalyticsRule' - branch: 'testScript' - sourceControlId: '7a5e8eb7-bfa9-467d-88e4-f4169e0718c3' - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - smartDeployment: 'true' - - steps: - - name: Login to Azure (Attempt 1) - continue-on-error: true - id: login1 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 1 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 2) - continue-on-error: true - id: login2 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 2 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 3) - continue-on-error: false - id: login3 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_7a5e8eb7bfa9467d88e4f4169e0718c3 }} - enable-AzPSSession: true - - - name: Checkout - uses: actions/checkout@v1 - - - name: Deploy Content to Azure Sentinel - uses: azure/powershell@v1 - with: - azPSVersion: 'latest' - inlineScript: | - ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 \ No newline at end of file From d02a5d78593a5e2faacd77c5477b8e1e9c9367d6 Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 23:42:32 +0000 Subject: [PATCH 89/90] Remove azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 --- ...y-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 | 584 ------------------ 1 file changed, 584 deletions(-) delete mode 100644 .github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 diff --git a/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 b/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 deleted file mode 100644 index ba5aeee..0000000 --- a/.github/workflows/azure-sentinel-deploy-7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.ps1 +++ /dev/null @@ -1,584 +0,0 @@ -## Globals ## -$CloudEnv = $Env:cloudEnv -$ResourceGroupName = $Env:resourceGroupName -$WorkspaceName = $Env:workspaceName -$WorkspaceId = $Env:workspaceId -$Directory = $Env:directory -$Creds = $Env:creds -$contentTypes = $Env:contentTypes -$contentTypeMapping = @{ - "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); - "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); - "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); - "Workbook"=@("Microsoft.Insights/workbooks"); -} -$sourceControlId = $Env:sourceControlId -$githubAuthToken = $Env:githubAuthToken -$githubRepository = $Env:GITHUB_REPOSITORY -$branchName = $Env:branch -$smartDeployment = $Env:smartDeployment -$csvPath = ".sentinel\tracking_table_$sourceControlId.csv" -$configPath = "sentinel-deployment.config" -$global:localCsvTablefinal = @{} -$global:updatedCsvTable = @{} -$global:parameterFileMapping = @{} -$global:prioritizedContentFiles = @() -$global:excludeContentFiles = @() - -$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' -$namePattern = '([-\w\._\(\)]+)' -$sentinelResourcePatterns = @{ - "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" - "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" - "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" - "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" -} - -if ([string]::IsNullOrEmpty($contentTypes)) { - $contentTypes = "AnalyticsRule" -} - -$metadataFilePath = "metadata.json" -@" -{ - "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "parentResourceId": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "sourceControlId": { - "type": "string" - }, - "workspace": { - "type": "string" - }, - "contentId": { - "type": "string" - } - }, - "variables": { - "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", - "apiVersion": "2022-01-01-preview", - "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", - "properties": { - "parentId": "[parameters('parentResourceId')]", - "kind": "[parameters('kind')]", - "source": { - "kind": "SourceRepository", - "name": "Repositories", - "sourceId": "[parameters('sourceControlId')]" - } - } - } - ] -} -"@ | Out-File -FilePath $metadataFilePath - -$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } -$MaxRetries = 3 -$secondsBetweenAttempts = 5 - -#Converts hashtable to string that can be set as content when pushing csv file -function ConvertTableToString { - $output = "FileName, CommitSha`n" - $global:updatedCsvTable.GetEnumerator() | ForEach-Object { - $output += "{0},{1}`n" -f $_.Key, $_.Value - } - return $output -} - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -#Gets all files and commit shas using Get Trees API -function GetGithubTree { - $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 - $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 - return $getTreeResponse -} - -#Gets blob commit sha of the csv file, used when updating csv file to repo -function GetCsvCommitSha($getTreeResponse) { - $shaObject = $getTreeResponse.tree | Where-Object { $_.path -eq $csvPath.Replace("\", "/") } - return $shaObject.sha -} - -#Creates a table using the reponse from the tree api, creates a table -function GetCommitShaTable($getTreeResponse) { - $shaTable = @{} - $getTreeResponse.tree | ForEach-Object { - if (([System.IO.Path]::GetExtension($_.path) -eq ".json") -or ($_.path -eq $configPath)) - { - $truePath = $_.path.Replace("/", "\") - $shaTable.Add($truePath, $_.sha) - } - } - return $shaTable -} - -#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. -function PushCsvToRepo($getTreeResponse) { - $path = $csvPath.Replace("\", "/") - $sha = GetCsvCommitSha $getTreeResponse - $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$path" - $content = ConvertTableToString - $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - - $body = @{ - message = "trackingTable.csv created." - content = $encodedContent - branch = $branchName - sha = $sha - } | ConvertTo-Json - - $Parameters = @{ - Method = "PUT" - Uri = $createFileUrl - Headers = $header - Body = $body | ConvertTo-Json - } - AttemptInvokeRestMethod "Put" $createFileUrl $body $null 3 -} - -function ReadCsvToTable { - $csvTable = Import-Csv -Path $csvPath - $HashTable=@{} - foreach($r in $csvTable) - { - $HashTable[$r.FileName]=$r.CommitSha - } - return $HashTable -} - -#Checks and removes any deleted content files -function CleanDeletedFilesFromTable { - $global:updatedCsvTable.Clone().GetEnumerator() | ForEach-Object { - if (!(Test-Path -Path $_.Key)) { - $global:updatedCsvTable.Remove($_.Key) - } - } -} - -function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { - $Stoploop = $false - $retryCount = 0 - do { - try { - $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes - $Stoploop = $true - } - catch { - if ($retryCount -gt $maxRetries) { - Write-Host "[Error] API call failed after $retryCount retries: $_" - $Stoploop = $true - } - else { - Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." - Start-Sleep -Seconds 5 - $retryCount = $retryCount + 1 - } - } - } - While ($Stoploop -eq $false) - return $result -} - -function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { - $maxLoginRetries = 3 - $delayInSeconds = 30 - $retryCount = 1 - $stopTrying = $false - do { - try { - Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; - Write-Host "Login Successful" - $stopTrying = $true - } - catch { - if ($retryCount -ge $maxLoginRetries) { - Write-Host "Login failed after $maxLoginRetries attempts." - $stopTrying = $true - } - else { - Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." - Start-Sleep -Seconds $delayInSeconds - $retryCount++ - } - } - } - while (-not $stopTrying) -} - -function ConnectAzCloud { - $RawCreds = $Creds | ConvertFrom-Json - - Clear-AzContext -Scope Process; - Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; - - Add-AzEnvironment ` - -Name $CloudEnv ` - -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` - -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` - -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` - -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; - - $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force - $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) - - AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv - Set-AzContext -Tenant $RawCreds.tenantId | out-null; -} - -function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject) { - $deploymentInfo = $null - try { - $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore - } - catch { - Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" - return - } - $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { - $resource = $_.TargetResource - $sentinelContentKinds = GetContentKinds $resource - if ($sentinelContentKinds.Count -gt 0) { - $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject - $contentId = $resource.Split("/")[-1] - try { - New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` - -parentResourceId $resource ` - -kind $contentKind ` - -contentId $contentId ` - -sourceControlId $sourceControlId ` - -workspace $workspaceName ` - -ErrorAction Stop | Out-Host - Write-Host "[Info] Created metadata metadata for $contentKind with parent resource id $resource" - } - catch { - Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error $_" - } - } - } -} - -function GetContentKinds($resource) { - return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } -} - -function ToContentKind($contentKinds, $resource, $templateObject) { - if ($contentKinds.Count -eq 1) { - return $contentKinds - } - if ($null -ne $resource -and $resource.Contains('savedSearches')) { - if ($templateObject.resources.properties.Category -eq "Hunting Queries") { - return "HuntingQuery" - } - return "Parser" - } - return $null -} - -function IsValidTemplate($path, $templateObject) { - Try { - if (DoesContainWorkspaceParam $templateObject) { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName - } - else { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path - } - - return $true - } - Catch { - Write-Host "[Warning] The file $path is not valid: $_" - return $false - } -} - -function IsRetryable($deploymentName) { - $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" - Try { - $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop - return $retryableStatusCodes -contains $deploymentResult.StatusCode - } - Catch { - return $false - } -} - -function IsValidResourceType($template) { - try { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources - } - } - catch { - Write-Host "[Error] Failed to check valid resource type." - $isAllowedResources = $false - } - return $isAllowedResources -} - -function DoesContainWorkspaceParam($templateObject) { - $templateObject.parameters.PSobject.Properties.Name -contains "workspace" -} - -function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject) { - Write-Host "[Info] Deploying $path with deployment name $deploymentName" - - $isValid = IsValidTemplate $path $templateObject - if (-not $isValid) { - return $false - } - $isSuccess = $false - $currentAttempt = 0 - While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) - { - $currentAttempt ++ - Try - { - Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" - if (DoesContainWorkspaceParam $templateObject) - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host - } - } - else - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host - } - } - AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject - - $isSuccess = $true - } - Catch [Exception] - { - $err = $_ - if (-not (IsRetryable $deploymentName)) - { - Write-Host "[Warning] Failed to deploy $path with error: $err" - break - } - else - { - if ($currentAttempt -le $MaxRetries) - { - Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." - Start-Sleep -Seconds $secondsBetweenAttempts - } - else - { - Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" - } - } - } - } - return $isSuccess -} - -function GenerateDeploymentName() { - $randomId = [guid]::NewGuid() - return "Sentinel_Deployment_$randomId" -} - -#Load deployment configuration -function LoadDeploymentConfig() { - Write-Host "[Info] load the deployment configuration from [$configPath]" - $global:parameterFileMapping = @{} - $global:prioritizedContentFiles = @() - $global:excludeContentFiles = @() - try { - if (Test-Path $configPath) { - $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json - $parameterFileMappings = @{} - if ($deployment_config.parameterfilemappings) { - $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } - } - $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) - if ($null -ne $key) { - $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } - } - if ($deployment_config.prioritizedcontentfiles) { - $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles - } - $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles - if ($deployment_config.excludecontentfiles) { - $excludeList = $excludeList + $deployment_config.excludecontentfiles - } - $global:excludeContentFiles = $excludeList | Where-Object { Test-Path $_ } - } - } - catch { - Write-Host "[Warning] An error occurred while trying to load deployment configuration." - Write-Host "Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function filterContentFile($path) { - $temp = $path.Replace($Directory + "\", "").Replace("\", "/") - return $global:excludeContentFiles | ? {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} -} - -#resolve parameter file name, return $null if there is none. -function GetParameterFile($path) { - $index = $path.Replace("\", "/") - $key = ($global:parameterFileMapping.Keys | ? { $_ -eq $index }) - if ($key) { - $mappedParameterFile = $global:parameterFileMapping[$key].Replace("/", "\") - if (Test-Path $mappedParameterFile) { - return $mappedParameterFile - } - } - - $parameterFilePrefix = $path.TrimEnd(".json") - - $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" - if (Test-Path $workspaceParameterFile) { - return $workspaceParameterFile - } - - $defaultParameterFile = $parameterFilePrefix + ".parameters.json" - if (Test-Path $defaultParameterFile) { - return $defaultParameterFile - } - - return $null -} - -function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { - Write-Host "Starting Deployment for Files in path: $Directory" - if (Test-Path -Path $Directory) - { - $totalFiles = 0; - $totalFailed = 0; - $iterationList = @() - $global:prioritizedContentFiles | ForEach-Object { $iterationList += $_.Replace("/", "\") } - Get-ChildItem -Path $Directory -Recurse -Filter *.json -exclude *metadata.json, *.parameters*.json | - Where-Object { $null -eq ( filterContentFile $_.FullName ) } | - Select-Object -Property FullName | - ForEach-Object { $iterationList += $_.FullName.Replace($Directory + "\", "") } - $iterationList | ForEach-Object { - $path = $_ - Write-Host "[Info] Try to deploy $path" - if (-not (Test-Path $path)) { - Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." - return - } - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - #parameterFile = GetParameterFile $path - $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject - if ($result.isSuccess -eq $false) { - $totalFailed++ - } - if (-not $result.skip) { - $totalFiles++ - } - if ($result.isSuccess) { - $global:updatedCsvTable[$path] = $remoteShaTable[$path] - if ($parameterFile) { - $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] - } - } - } - CleanDeletedFilesFromTable - PushCsvToRepo $tree - if ($totalFiles -gt 0 -and $totalFailed -gt 0) - { - $err = "$totalFailed of $totalFiles deployments failed." - Throw $err - } - } - else - { - Write-Output "[Warning] $Directory not found. nothing to deploy" - } -} - -function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject) { - try { - $skip = $false - $isSuccess = $null - if (!$fullDeploymentFlag) { - $existingSha = $global:localCsvTablefinal[$path] - $remoteSha = $remoteShaTable[$path] - $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) - if ($skip -and $parameterFile) { - $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] - $remoteShaForParameterFile = $remoteShaTable[$parameterFile] - $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) - } - } - if (!$skip) { - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject - } - return @{ - skip = $skip - isSuccess = $isSuccess - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function main() { - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - if (Test-Path $csvPath) { - $global:localCsvTablefinal = ReadCsvToTable - $global:updatedCsvTable = $global:localCsvTablefinal.Clone() - } - - LoadDeploymentConfig - - $tree = GetGithubTree - $remoteShaTable = GetCommitShaTable $tree - - $existingConfigSha = $global:localCsvTablefinal[$configPath] - $remoteConfigSha = $remoteShaTable[$configPath] - $modifiedConfig = ((!$existingConfigSha) -or ($existingConfigSha -ne $remoteConfigSha)) - $global:updatedCsvTable[$configPath] = $remoteConfigSha - - $fullDeploymentFlag = $modifiedConfig -or (-not (Test-Path $csvPath)) -or ($smartDeployment -eq "false") - Deployment $fullDeploymentFlag $remoteShaTable $tree -} - -main \ No newline at end of file From 2e88528308d4f314ab0e65453d59ed0a00d2240a Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 23:42:33 +0000 Subject: [PATCH 90/90] Remove tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv --- .../tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv diff --git a/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv b/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv deleted file mode 100644 index 408f4f9..0000000 --- a/.sentinel/tracking_table_7a5e8eb7-bfa9-467d-88e4-f4169e0718c3.csv +++ /dev/null @@ -1 +0,0 @@ -FileName, CommitSha