diff --git a/.github/workflows/apply.yml b/.github/workflows/apply.yml index 4fe9bd6..36d33ac 100644 --- a/.github/workflows/apply.yml +++ b/.github/workflows/apply.yml @@ -58,6 +58,10 @@ on: description: "When true, pass `--commit ` to the Kosli commands via the COMMIT_ARG environment variable." default: false type: boolean + job_timeout_minutes: + description: "Minutes before GitHub cancels the apply job. Raise it together with aws_role_duration when a legitimate apply needs longer; see the README on timeouts." + default: 30 + type: number secrets: kosli_api_token: description: "Kosli API token. Required when kosli_template_file is set." @@ -96,12 +100,14 @@ jobs: kosli_flow: terraform-apply-${{ inputs.environment }}-${{ github.event.repository.name }} kosli_cli_version: ${{ inputs.kosli_cli_version }} pass_commit_to_kosli_commands: ${{ inputs.pass_commit_to_kosli_commands }} + job_timeout_minutes: ${{ inputs.job_timeout_minutes }} secrets: kosli_api_token: ${{ secrets.kosli_api_token }} reset-drift-detection: needs: apply runs-on: ubuntu-latest + timeout-minutes: 10 permissions: id-token: write contents: read @@ -121,12 +127,15 @@ jobs: ref: ${{ inputs.ref }} - name: Configure AWS credentials + timeout-minutes: 2 uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: ${{ inputs.aws_role_arn }} aws-region: ${{ inputs.aws_region }} role-duration-seconds: ${{ inputs.aws_role_duration }} role-session-name: ${{ github.event.repository.name }} + action-timeout-s: 45 + retry-max-attempts: 3 - name: Compute state bucket name run: | diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index b1b8a8e..290eeaf 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -21,6 +21,10 @@ on: tf_apply: default: false type: boolean + job_timeout_minutes: + description: "Minutes before GitHub cancels the plan/apply job. The default follows aws_role_duration: the OIDC credentials are static environment variables that are never refreshed, so AWS calls start failing with ExpiredToken 1200s (20 min) after the credentials step and a longer ceiling would buy nothing. Raise both together if a legitimate apply needs longer." + default: 30 + type: number environment: required: true type: string @@ -74,6 +78,7 @@ on: jobs: terraform: runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.job_timeout_minutes }} env: AWS_DEFAULT_REGION: ${{ inputs.aws_region }} environment: ${{ inputs.environment }} @@ -121,12 +126,15 @@ jobs: run: terraform fmt --recursive -check - name: Configure AWS credentials + timeout-minutes: 2 uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: ${{ inputs.aws_role_arn }} aws-region: ${{ inputs.aws_region }} role-duration-seconds: ${{ inputs.aws_role_duration }} role-session-name: ${{ github.event.repository.name }} + action-timeout-s: 45 + retry-max-attempts: 3 - name: Setup Kosli CLI if: inputs.kosli_template_file != '' diff --git a/.github/workflows/detect-drift.yml b/.github/workflows/detect-drift.yml index d25c008..4ab3cc0 100644 --- a/.github/workflows/detect-drift.yml +++ b/.github/workflows/detect-drift.yml @@ -25,6 +25,10 @@ on: description: "Extra environment variables, one KEY=VALUE per line, exported before the drift plan. To set Terraform variable `foo`, use `TF_VAR_foo=...`. Single-line values only. Note: for per-build values (e.g. an image tag) give the variable a default in variables.tf instead, otherwise drift detection reports false drift." default: "" type: string + job_timeout_minutes: + description: "Minutes before GitHub cancels the drift plan job. Raise it together with aws_role_duration when a legitimate plan needs longer; see the README on timeouts." + default: 30 + type: number concurrency: group: detect-drift-${{ github.repository }}-${{ inputs.environment }} @@ -33,6 +37,7 @@ concurrency: jobs: fetch-baseline: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: id-token: write contents: read @@ -41,12 +46,15 @@ jobs: drift: ${{ steps.read.outputs.drift }} steps: - name: Configure AWS credentials + timeout-minutes: 2 uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: ${{ inputs.aws_role_arn }} aws-region: ${{ inputs.aws_region }} role-duration-seconds: ${{ inputs.aws_role_duration }} role-session-name: ${{ github.event.repository.name }} + action-timeout-s: 45 + retry-max-attempts: 3 - name: Compute state bucket name run: | @@ -98,22 +106,27 @@ jobs: ref: ${{ needs.fetch-baseline.outputs.sha }} tf_apply: false tf_vars: ${{ inputs.tf_vars }} + job_timeout_minutes: ${{ inputs.job_timeout_minutes }} flag-drift: needs: [fetch-baseline, plan] if: needs.plan.outputs.has_changes == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 permissions: id-token: write contents: read steps: - name: Configure AWS credentials + timeout-minutes: 2 uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: ${{ inputs.aws_role_arn }} aws-region: ${{ inputs.aws_region }} role-duration-seconds: ${{ inputs.aws_role_duration }} role-session-name: ${{ github.event.repository.name }} + action-timeout-s: 45 + retry-max-attempts: 3 - name: Compute state bucket name run: | @@ -142,6 +155,7 @@ jobs: needs: [fetch-baseline, plan] if: needs.plan.outputs.has_changes == 'false' runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: No-drift summary run: | diff --git a/.github/workflows/plan.yml b/.github/workflows/plan.yml index 32f9a38..7053e5a 100644 --- a/.github/workflows/plan.yml +++ b/.github/workflows/plan.yml @@ -46,6 +46,10 @@ on: kosli_cli_version: default: "latest" type: string + job_timeout_minutes: + description: "Minutes before GitHub cancels the plan job. Raise it together with aws_role_duration when a legitimate plan needs longer; see the README on timeouts." + default: 30 + type: number secrets: kosli_api_token: description: "Kosli API token. Required when kosli_template_file is set." @@ -80,5 +84,6 @@ jobs: kosli_org: ${{ inputs.kosli_org }} kosli_flow: terraform-plan-${{ inputs.environment }}-${{ github.event.repository.name }} kosli_cli_version: ${{ inputs.kosli_cli_version }} + job_timeout_minutes: ${{ inputs.job_timeout_minutes }} secrets: kosli_api_token: ${{ secrets.kosli_api_token }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ff20b2..003a204 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ on: jobs: test: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/README.md b/README.md index c08c889..9c619a0 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Both `plan.yml` and `apply.yml` accept the same core inputs: | `working_directory` | no | `./` | Directory containing Terraform config | | `tf_version` | no | `1.14.6` | Terraform version to install | | `tf_vars` | no | `""` | Extra env vars (one `KEY=VALUE` per line) exported before plan/apply; see [Supplying Terraform variables](#supplying-terraform-variables) | +| `job_timeout_minutes` | no | `30` | Minutes before GitHub cancels the plan/apply job; see [Timeouts](#timeouts) | Plus, for opting into Kosli attestation (see [Kosli attestation](#kosli-attestation) below): @@ -169,6 +170,60 @@ image tag, which changes every run. | `kosli_api_token` | if `kosli_template_file` is set | Kosli API token for the attest steps. | | `kosli_github_token` | no (only `apply.yml`) | GitHub token used by `kosli attest pr github` to look up pull requests. When omitted, the pull-request attestation step is skipped. Typically passed as `${{ secrets.GITHUB_TOKEN }}` — in which case the **calling job must also declare `pull-requests: read`** in its `permissions:` block (see example below), otherwise the attestation step will fail with `Resource not accessible by integration`. | +### Timeouts + +Every job carries a `timeout-minutes` rather than inheriting GitHub's 360-minute default, and the +OIDC credential step is bounded so that an unreachable STS endpoint fails in under a minute instead +of holding a runner — and the environment's `concurrency` group — for over an hour: + +```yaml + - name: Configure AWS credentials + timeout-minutes: 2 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 + with: + # ... + action-timeout-s: 45 + retry-max-attempts: 3 +``` + +These are the standard values for **any** Kosli workflow using +`aws-actions/configure-aws-credentials`, not just the ones here. The reasoning: + +| Setting | Value | Why | +|---|---|---| +| `action-timeout-s` | `45` | Bounds the action as a whole, retries included. Across recent successful runs in `kosli-dev` and `cyber-dojo` the step took 0–2s, worst case 6s, so 45s is far above the real p99 and will not fail a slow-but-healthy authentication. | +| `retry-max-attempts` | `3` | The default is 12. STS throttling is genuinely transient and worth retrying, but 12 attempts is only useful if each attempt is fast — which is exactly what fails to hold when the endpoint is unreachable. | +| `timeout-minutes` (step) | `2` | A backstop that holds regardless of how the action behaves or what a future version changes. | +| `timeout-minutes` (job) | `30` plan/apply, `5`–`10` housekeeping | Bounded by `aws_role_duration`, not by how long Terraform might take — see below. | + +`disable-retry` is deliberately **not** used: a couple of quick retries is worth having, and +`action-timeout-s` already bounds the total cost of them. + +The plan/apply job's default of 30 minutes is **derived from `aws_role_duration`**, which defaults +to `1200` (20 minutes). The credentials the OIDC step exports are static environment variables and +are never refreshed, so 20 minutes after that step every AWS call starts failing with +`ExpiredToken` — a longer job ceiling would buy nothing, because the work cannot usefully continue. +Measured against real runs, setup before the credentials step takes 1–35s, so a job that consumes +its entire credential lifetime lands around 22–23 minutes; 30 leaves headroom without being +arbitrary. + +The two values are coupled, so **raise `job_timeout_minutes` and `aws_role_duration` together** when +a repository has a legitimately long apply. Raising only the session duration lets the job be +cancelled part-way through an `apply`, which can leave the state lock held — a worse outcome than a +slow run. Raising only the job ceiling buys time in which every AWS call fails: + +```yaml + with: + aws_role_duration: "3600" # 60 min session + job_timeout_minutes: 70 # 60 + setup + headroom +``` + +The role's own maximum session duration is the hard limit on `aws_role_duration`; if a longer +session is refused, that maximum needs raising on the IAM role first. + +A job that fails in 45 seconds can be re-run for nothing. A job that hangs for 82 minutes blocks +every apply queued behind it. + ### What it does **Plan** (`plan.yml`):