diff --git a/.github/README.md b/.github/README.md index 5e0d4574d..69c4084e7 100644 --- a/.github/README.md +++ b/.github/README.md @@ -141,6 +141,25 @@ repository can reference it explicitly: See [`../runbooks/dependency-management.md`](../runbooks/dependency-management.md) for onboarding, validation, migration, and rollback. +## Scheduled Workflow Inventory + +Use `scripts/audit-scheduled-workflows.rb` to produce a reproducible inventory +of scheduled workflows. The command performs read-only GitHub API requests. + +```sh +scripts/audit-scheduled-workflows.rb --org z-shell --public-only --format json +``` + +Disabled workflows are included unless `--active-only` is supplied. Use +`--public-only` when writing a report intended for public distribution. JSON +for a target set that includes private repositories must use `--output FILE`; +the command creates that destination with mode `0600`. Markdown always omits +private repository names. + +Follow the +[`recurring-operations` runbook](../runbooks/recurring-operations.md) to +classify the inventory, review failures, and draft evidence-backed follow-ups. + --- ## Common Use Cases diff --git a/.github/instruction-surfaces.json b/.github/instruction-surfaces.json index e50dffbc6..debfffdc9 100644 --- a/.github/instruction-surfaces.json +++ b/.github/instruction-surfaces.json @@ -471,6 +471,22 @@ "review_owner": "z-shell maintainers", "canonical_for": ["project-tracking"] }, + { + "id": "runbook-recurring-operations", + "path": "runbooks/recurring-operations.md", + "kind": "runbook", + "authority": "canonical-detail", + "consumers": ["codex", "claude-code", "copilot", "human"], + "tasks": [ + "recurring-operations", + "scheduled-workflow-audit", + "automation-review" + ], + "file_patterns": ["**"], + "required": true, + "review_owner": "z-shell maintainers", + "canonical_for": ["recurring-operations"] + }, { "id": "runbook-release", "path": "runbooks/release.md", diff --git a/runbooks/recurring-operations.md b/runbooks/recurring-operations.md new file mode 100644 index 000000000..23e144a7c --- /dev/null +++ b/runbooks/recurring-operations.md @@ -0,0 +1,306 @@ +# Runbook: Recurring Operations + +Use this runbook to classify and review scheduled workflows and other recurring +organization maintenance. + +## Purpose and boundary + +Recurring work should run only as often, and with only as much authority, as its +value requires. This runbook separates deterministic automation from +maintainer judgment and provides one evidence record for each classification. + +**Hard rule:** every review described here produces a draft only. Do not add +labels, post comments, close or lock issues or pull requests, dispatch +workflows, change settings, or modify repositories unless a maintainer approves +that action as a separate, scoped step. + +This runbook does not create a Codex schedule or any other scheduler. Evaluate +the scheduler's supported capabilities and authority before proposing one. + +The sources of truth are: + +- the live workflow definition, state, and recent runs in the owning repository; +- repository activity and the owning GitHub issue or pull request; +- Linear for cross-repository, security, release-blocking, strategic, or + organization-infrastructure work; +- the relevant organization runbook, accepted decision record, and + [recurring-operations coordination issue](https://github.com/z-shell/.github/issues/485). + +An inventory or review draft is evidence, not a replacement for those owners. + +## Choose scheduled, event-driven, reusable, or manual work + +Apply this decision tree to each operation: + +1. Does time change the operation's input or value? + - If yes, consider a schedule. + - If no, continue to the next question. +2. Does a source change, pull request, tag, label-definition change, or + repository creation supply the real input? + - If yes, use the narrow matching event-driven trigger. + - If no, continue to the next question. +3. Is the execution deterministic and repeated across repositories? + - If yes, centralize the tested execution in a reusable workflow. Each + caller still owns its trigger, permissions, concurrency, and target-branch + authority. + - If no, continue to the next question. +4. Does the work require judgment, private evidence, mutation, or credentials + broader than one narrow operation? + - If yes, keep it manual and require explicit approval. + - If no, document why automation provides observable value before adding it. + +Use a schedule only when elapsed time creates a meaningful new input, such as a +certificate approaching expiry or a periodic external data snapshot. A +schedule is not a substitute for a missing event trigger. + +## Record the classification + +Create one record per workflow. Refresh live state before filling it in. + +```text +Repository: +Workflow path: +Purpose: +Repository class: +Current workflow state: +Cron and timezone: +Event-driven triggers: +Owning runbook or issue: +Required permissions: +Concurrency behavior: +Latest scheduled result: +Repeated failure signature: +Observed value: +Classification: retain | event-driven | reusable | manual | remove +Decision evidence: +Follow-up owner: +Retirement condition: +``` + +Do not infer value from file presence or from one successful run. State what +the workflow detects, prevents, publishes, or maintains, and cite the evidence. + +## Keep public and private evidence separate + +A public report may contain public repository names, workflow paths, schedules, +documented permissions, aggregate results, and links to public issues or runs. + +A private report is required when the target set or evidence includes private +repository identities, administrative settings, credentials, private run +output, security-sensitive findings, or unpublished operational details. +Before publishing any derivative: + +1. remove private repository and workflow identities; +2. remove local paths, host details, tokens, credential names, and secret + values; +3. replace sensitive samples with aggregate counts or a public-safe + description; +4. verify every link and quoted field is already public; +5. save private artifacts with restricted access and keep them out of the + repository. + +If sanitization would remove the evidence needed to support a conclusion, keep +the entire conclusion private. + +## Design least-privilege automation + +Every workflow must declare explicit `permissions`. Start with no permissions +or read-only contents, then grant only the operations required by the specific +job. Prefer the repository-scoped `GITHUB_TOKEN`; use a narrowly scoped GitHub +App, OpenID Connect trust policy, or dedicated credential only when the +repository token cannot perform the required operation. + +Pin actions and reusable workflows to a full commit SHA. Treat a mutable branch +or tag reference as unreviewed code unless a documented platform constraint +requires it. Pass only the secrets a called workflow needs, and ensure nested +workflow permissions can only stay the same or become more restrictive. + +Never provision a broad unattended organization write token. Mutating +organization operations require a separately reviewed design, narrow +credentials, bounded targets, rollback instructions, and explicit maintainer +approval. + +## Apply schedule rules + +For every retained schedule: + +- declare UTC explicitly or use an intentional IANA timezone and document why + local civil time matters; +- use a non-zero minute offset unless a vendor requires an hour boundary; +- avoid shared high-load boundaries because scheduled runs can be delayed and, + under enough load, queued jobs can be dropped; +- bound runtime with timeouts, pagination, and finite retry behavior; +- add safe `workflow_dispatch` input where a maintainer benefits from a + controlled rerun; +- declare concurrency behavior and whether a newer run should cancel an older + one; +- name the maintainer or owning team that reviews failures and value. + +Scheduled workflows run from the default branch. Confirm the intended +definition is on that branch before relying on the next trigger. + +## Review inactivity state + +GitHub can automatically disable scheduled workflows in a public repository +after 60 days without repository activity. The API state +`disabled_inactivity` is live state, not evidence that the workflow should be +retired. + +Before acting on an inactive schedule: + +1. inspect its current workflow state; +2. inspect recent repository and default-branch activity; +3. determine whether new activity would reactivate an obsolete or unsafe + schedule; +4. classify its purpose and value using the complete record; +5. draft the smallest follow-up under the owning issue. + +Do not enable, delete, or rewrite a workflow solely because it is inactive. + +## Review failures and value + +Group failures by repository, workflow, job, and a stable signature such as the +failing step plus normalized error category. Count occurrences and record the +latest run for each group. Keep transient queue delay distinct from a repeated +deterministic failure. + +Refresh live workflow state and the latest runs before describing an old +failure as urgent. Record the observation window and any missing data. A file +on the default branch does not prove execution, and one green run does not +prove continuing value. + +Where available, use organization Actions usage and performance metrics to +compare queue time, duration, failure rate, and consumption. Metrics inform a +classification; they do not authorize a change. + +## Draft stale and lock dispositions + +The default stale and lock policy is `draft-disposition`. Reviews output +candidate lists only. They must not automatically add labels, post comments, +close items, or lock conversations. + +Evaluate issues and pull requests separately because their lifecycle signals +and recovery costs differ. Exclude these mandatory categories from routine +stale or lock candidates: + +- security reports; +- regressions; +- release blockers; +- roadmap items; +- pinned discussions; +- items labeled `status:blocked`. + +A repository-specific exception is allowed only after repository evidence and +explicit maintainer approval identify the alternate criteria, waiting period, +communication, recovery path, and owner. + +## Reconcile tracker ownership + +Every follow-up has one owning GitHub issue or pull request. Link to that owner +instead of opening duplicates. Use Linear in addition to the GitHub owner only +for cross-repository, security, release-blocking, strategic, or +organization-infrastructure work. + +Before proposing a new tracker item: + +1. search the owning repository for an existing issue or pull request; +2. search Linear for a linked or equivalent item; +3. attach new evidence to the existing owner when one exists; +4. record cross-links and keep status, owner, and next step consistent. + +## Use draft-only prompt templates + +### Scheduled-workflow health review + +```text +Review scheduled workflows using live definitions, current workflow state, +recent runs, repository activity, and the recurring-operations classification +record. + +- group failures by repository, workflow, job, and stable signature +- count occurrences and record the latest scheduled result +- distinguish queue delay from deterministic failure +- assess observed value rather than file presence +- classify each workflow as retain, event-driven, reusable, manual, or remove +- name an owner, decision evidence, and retirement condition +- keep private evidence in a restricted private artifact +- reconcile each follow-up with its owning GitHub and Linear records + +Return a draft only. Do not dispatch, enable, disable, edit, label, comment, +close, lock, or create tracker items. +``` + +### Stale and lock disposition review + +```text +Review issues and pull requests separately under the default +draft-disposition policy. + +- use live state and repository-specific evidence +- output candidate lists with age, activity, owner, and recovery context +- exclude security reports, regressions, release blockers, roadmap items, + pinned discussions, and status:blocked +- identify existing owning GitHub and Linear records +- state where maintainer judgment is required + +Return a draft only. Do not add labels, post comments, close items, lock +conversations, or create tracker items. +``` + +## Retire recurring work + +A workflow is ready for a removal proposal when its input no longer exists, +its outcome is owned by a reliable event-driven or reusable replacement, it +duplicates a canonical service, or repeated evidence shows no useful signal. +The proposal must identify the owner, replacement or reason, rollback path, and +retirement condition. Observe the replacement through its next expected +trigger before removing the old workflow. + +A recurring review is ready for retirement when its decision has been made, +its evidence is available through an owning operational process, or several +documented cycles produce no actionable signal. Close or update the owning +tracker records only after reconciliation. + +## Verify before, during, and after + +### Before + +- [ ] Confirm the repository class, default branch, live workflow state, and + recent repository activity. +- [ ] Identify the owning runbook, GitHub issue or pull request, Linear item, + maintainer, and retirement condition. +- [ ] Classify the operation with the decision tree and complete every record + field. +- [ ] Separate public-safe evidence from restricted evidence. +- [ ] Review permissions, credentials, immutable references, concurrency, + timeout, and rollback needs. + +### During + +- [ ] Keep output draft-only and targets bounded. +- [ ] Record schedules with UTC or an intentional IANA timezone. +- [ ] Group repeated failures by stable signature and retain occurrence counts + and the latest run. +- [ ] Preserve caller ownership of triggers, permissions, concurrency, and + target branches when proposing reusable execution. +- [ ] Stop before any mutation or private publication that lacks explicit + approval. + +### After + +- [ ] Run repository validation and review the complete diff or draft. +- [ ] Observe the next expected event or scheduled trigger before declaring a + replacement healthy. +- [ ] Recheck live state after the observation window. +- [ ] Reconcile GitHub and Linear owner, status, evidence, and next step. +- [ ] Confirm public output contains no private identity, administrative + detail, credential, secret, local path, or unresolved placeholder. +- [ ] Record whether the workflow or recurring review met its retirement + criteria. + +## Official references + +- [GitHub documentation for events that trigger workflows](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows) +- [GitHub documentation for reusing workflow configurations](https://docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations) +- [GitHub secure use reference for Actions](https://docs.github.com/en/actions/reference/security/secure-use) +- [GitHub documentation for viewing Actions metrics](https://docs.github.com/en/enterprise-cloud@latest/actions/how-tos/administer/view-metrics) diff --git a/scripts/audit-scheduled-workflows.rb b/scripts/audit-scheduled-workflows.rb new file mode 100755 index 000000000..e4b74a761 --- /dev/null +++ b/scripts/audit-scheduled-workflows.rb @@ -0,0 +1,339 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "open3" +require "optparse" +require "time" +require "yaml" + +module ScheduledWorkflowAudit + SCHEMA = "z-shell/scheduled-workflow-audit/v1" + + class GitHubError < StandardError + attr_reader :status + + def initialize(message, status: nil) + super(message) + @status = status + end + end + + class GitHubClient + def initialize(runner: nil) + @runner = runner || lambda { |command| Open3.capture3(*command) } + end + + def json(path) + command = ["gh", "api", "--method", "GET", path] + stdout, stderr, status = @runner.call(command) + return parse_json(stdout) if successful?(status) + + error = parse_error(stderr) + raise GitHubError.new(error.fetch("message", "GitHub API request failed"), status: error["status"]) + end + + private + + def successful?(status) + status.respond_to?(:success?) ? status.success? : status == true + end + + def parse_json(body) + parsed = JSON.parse(body) + return parsed if parsed.is_a?(Array) || parsed.is_a?(Hash) + + raise GitHubError, "GitHub API response must be an object or array" + rescue JSON::ParserError => error + raise GitHubError, "GitHub API returned invalid JSON: #{error.message}" + end + + def parse_error(body) + parsed = JSON.parse(body) + if parsed.is_a?(Hash) + parsed["status"] ||= status_from(body) + return parsed + end + + { "message" => body.to_s.strip.empty? ? "GitHub API request failed" : body.to_s.strip, "status" => status_from(body) } + rescue JSON::ParserError + { "message" => body.to_s.strip.empty? ? "GitHub API request failed" : body.to_s.strip, "status" => status_from(body) } + end + + def status_from(body) + body.to_s[/HTTP\s+(\d{3})/, 1]&.to_i + end + end + + class WorkflowParser + def parse(repository:, metadata:, content:) + workflow = YAML.safe_load(content, aliases: false) + workflow = {} if workflow.nil? + raise ArgumentError, "workflow must be a mapping" unless workflow.is_a?(Hash) + + triggers = workflow["on"] || workflow[true] || {} + triggers = {} unless triggers.is_a?(Hash) + schedules = Array(triggers["schedule"] || triggers[:schedule]).map do |entry| + raise ArgumentError, "schedule entry must be a mapping" unless entry.is_a?(Hash) + + cron = entry["cron"] || entry[:cron] + raise ArgumentError, "schedule entry must include cron" unless cron.is_a?(String) && !cron.empty? + + timezone = entry["timezone"] || entry[:timezone] + timezone = "UTC" unless timezone.is_a?(String) && !timezone.empty? + { "cron" => cron, "timezone" => timezone } + end + + jobs = workflow["jobs"].is_a?(Hash) ? workflow.fetch("jobs") : {} + { + "schema" => SCHEMA, + "repository" => repository.fetch("full_name"), + "visibility" => repository.fetch("visibility", "public"), + "default_branch" => repository.fetch("default_branch", "main"), + "path" => metadata.fetch("path"), + "name" => metadata.fetch("name"), + "state" => metadata.fetch("state", "unknown"), + "schedules" => schedules, + "workflow_dispatch" => triggers.key?("workflow_dispatch") || triggers.key?(:workflow_dispatch), + "permissions_locations" => locations(workflow, jobs, "permissions"), + "concurrency_locations" => locations(workflow, jobs, "concurrency"), + "reusable_calls" => reusable_calls(jobs), + "errors" => [] + } + rescue Psych::Exception => error + raise ArgumentError, "invalid workflow YAML: #{error.message}" + end + + private + + def locations(workflow, jobs, key) + locations = [] + locations << "workflow" if workflow.key?(key) || workflow.key?(key.to_sym) + locations << "job" if jobs.values.any? { |job| job.is_a?(Hash) && (job.key?(key) || job.key?(key.to_sym)) } + locations + end + + def reusable_calls(jobs) + jobs.values.filter_map { |job| job["uses"] if job.is_a?(Hash) && job["uses"].is_a?(String) } + end + end + + class Inventory + WORKER_COUNT = 12 + + def initialize(client:, org:, repo: nil, public_only: false, parser: WorkflowParser.new) + @client = client + @org = org + @repo = repo + @public_only = public_only + @parser = parser + end + + def run(active_only: false) + targets = target_repositories + queue = Queue.new + targets.each { |repository| queue << repository } + records = [] + lock = Mutex.new + workers = Array.new([targets.length, WORKER_COUNT].min) do + Thread.new do + loop do + repository = queue.pop(true) + result = begin + records_for(repository, active_only: active_only) + rescue StandardError => error + [error_record(repository, {}, error)] + end + lock.synchronize { records.concat(result) } + rescue ThreadError + break + end + end + end + workers.each(&:join) + records.sort_by { |record| [record.fetch("repository"), record["path"].to_s] } + end + + def private_target_set? + target_repositories.any? { |repository| repository.fetch("visibility", "public") != "public" } + end + + private + + def repositories + return [@client.json("/repos/#{@repo}")] if @repo + + page = 1 + repositories = [] + loop do + suffix = page == 1 ? "" : "&page=#{page}" + response = @client.json("/orgs/#{@org}/repos?type=all&per_page=100#{suffix}") + raise ArgumentError, "repository response must be an array" unless response.is_a?(Array) + + repositories.concat(response) + break if response.length < 100 + + page += 1 + end + repositories + end + + def target_repositories + @target_repositories ||= repositories.reject do |repository| + repository["archived"] || repository["fork"] || (@public_only && repository.fetch("visibility", "public") != "public") + end + end + + def records_for(repository, active_only:) + name = repository.fetch("full_name") + directory = begin + @client.json("/repos/#{name}/contents/.github/workflows?ref=#{repository.fetch("default_branch", "main")}") + rescue GitHubError => error + return [] if error.status == 404 + + raise + end + raise GitHubError, "workflow directory response must be an array" unless directory.is_a?(Array) + unless directory.all? { |entry| entry.is_a?(Hash) && entry["path"].is_a?(String) } + raise ArgumentError, "workflow directory entries must include paths" + end + + metadata_by_path = workflow_metadata(name).to_h { |workflow| [workflow.fetch("path"), workflow] } + directory.filter_map do |entry| + path = entry["path"] + next unless path&.match?(/\.ya?ml\z/) + + metadata = metadata_by_path[path] || { "path" => path, "name" => File.basename(path), "state" => "unknown" } + next if active_only && metadata.fetch("state", "unknown") != "active" + + content = @client.json("/repos/#{name}/contents/#{path}?ref=#{repository.fetch("default_branch", "main")}") + record = @parser.parse(repository: repository, metadata: metadata, content: decode_content(content)) + record.fetch("schedules").empty? ? nil : record + rescue GitHubError, ArgumentError => error + error_record(repository, metadata || { "path" => path, "name" => File.basename(path), "state" => "unknown" }, error) + end + rescue GitHubError, ArgumentError => error + [error_record(repository, {}, error)] + end + + def decode_content(response) + raise ArgumentError, "workflow content response must be an object" unless response.is_a?(Hash) + raise ArgumentError, "workflow content must use base64 encoding" unless response["encoding"] == "base64" + + response.fetch("content").delete("\n").unpack1("m0") + end + + def workflow_metadata(repository) + page = 1 + workflows = [] + loop do + suffix = page == 1 ? "" : "&page=#{page}" + response = @client.json("/repos/#{repository}/actions/workflows?per_page=100#{suffix}") + raise ArgumentError, "workflow metadata response must be an object" unless response.is_a?(Hash) + + current_page = response["workflows"] + raise ArgumentError, "workflow metadata response must include a workflows array" unless current_page.is_a?(Array) + unless current_page.all? { |workflow| workflow.is_a?(Hash) && workflow["path"].is_a?(String) } + raise ArgumentError, "workflow metadata entries must include paths" + end + + workflows.concat(current_page) + break if current_page.length < 100 + + page += 1 + end + workflows + end + + def error_record(repository, metadata, error) + { + "schema" => SCHEMA, + "repository" => repository.fetch("full_name"), + "visibility" => repository.fetch("visibility", "public"), + "default_branch" => repository.fetch("default_branch", "main"), + "path" => metadata["path"], + "name" => metadata["name"], + "state" => metadata["state"], + "schedules" => [], + "workflow_dispatch" => false, + "permissions_locations" => [], + "concurrency_locations" => [], + "reusable_calls" => [], + "errors" => [{ "status" => error.respond_to?(:status) ? error.status : nil, "message" => error.message }] + } + end + end + + class Renderer + def json(records) + JSON.pretty_generate(records) + "\n" + end + + def markdown(records, public_only:) + # Markdown is always public-safe, even when the caller did not filter targets. + visible = records.select { |record| record["visibility"] == "public" } + lines = ["# Scheduled workflow inventory", "", "| Repository | Workflow | State | Schedules |", "| --- | --- | --- | --- |"] + visible.each do |record| + schedules = record.fetch("schedules").map { |schedule| "#{schedule.fetch("cron")} (#{schedule.fetch("timezone")})" }.join("
") + lines << "| #{table_cell(record.fetch("repository"))} | #{table_cell(record["name"] || record["path"] || "Unavailable")} | #{table_cell(record["state"] || "unverified")} | #{table_cell(schedules)} |" + end + lines.join("\n") + "\n" + end + + private + + def table_cell(value) + value.to_s.gsub("|", "\\|").gsub(/\r?\n/, "
") + end + end + + class CLI + def self.run(argv, client: GitHubClient.new, stdout: $stdout, stderr: $stderr) + options = { org: nil, format: "json", public_only: false, active_only: false, output: nil, repo: nil } + parser = OptionParser.new do |option| + option.on("--org ORG") { |value| options[:org] = value } + option.on("--format FORMAT", ["json", "markdown"]) { |value| options[:format] = value } + option.on("--public-only") { options[:public_only] = true } + option.on("--repo OWNER/REPO") { |value| options[:repo] = value } + option.on("--active-only") { options[:active_only] = true } + option.on("--output FILE") { |value| options[:output] = value } + end + forbidden = argv.find { |argument| argument.match?(/\A--(?:apply|update|delete|enable|disable|dispatch)(?:=|\z)/) } + raise OptionParser::InvalidOption, forbidden if forbidden + + parser.parse!(argv) + raise OptionParser::InvalidArgument, "--repo must be OWNER/REPO" if options[:repo] && !options[:repo].match?(%r{\A[^/]+/[^/]+\z}) + raise OptionParser::MissingArgument, "--org" if options[:repo].nil? && (options[:org].nil? || options[:org].empty?) + + inventory = Inventory.new(client: client, org: options[:org], repo: options[:repo], public_only: options[:public_only]) + if options[:format] == "json" && inventory.private_target_set? && options[:output].nil? + stderr.puts "JSON for target sets that include private repositories requires --output FILE." + return 2 + end + records = inventory.run(active_only: options[:active_only]) + + rendered = options[:format] == "json" ? Renderer.new.json(records) : Renderer.new.markdown(records, public_only: options[:public_only]) + if options[:output] + write_output(options[:output], rendered) + else + stdout.write(rendered) + end + records.any? { |record| !record.fetch("errors").empty? } ? 1 : 0 + rescue OptionParser::ParseError, ArgumentError => error + stderr.puts error.message + 2 + rescue GitHubError => error + stderr.puts error.message + 1 + end + + def self.write_output(path, content) + File.open(path, File::WRONLY | File::CREAT | File::TRUNC, 0o600) do |file| + file.chmod(0o600) + file.write(content) + end + end + end +end + +exit(ScheduledWorkflowAudit::CLI.run(ARGV)) if $PROGRAM_NAME == __FILE__ diff --git a/scripts/fixtures/scheduled-workflows/active-iana-timezone.yml b/scripts/fixtures/scheduled-workflows/active-iana-timezone.yml new file mode 100644 index 000000000..4d7691eea --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/active-iana-timezone.yml @@ -0,0 +1,14 @@ +repository: + full_name: example/iana-demo + visibility: public + default_branch: main +workflow: + path: .github/workflows/timezone.yml + name: Timezone + state: active +content: | + name: Timezone + on: + schedule: + - cron: "0 8 * * 1-5" + timezone: Europe/London diff --git a/scripts/fixtures/scheduled-workflows/active-utc.yml b/scripts/fixtures/scheduled-workflows/active-utc.yml new file mode 100644 index 000000000..753222e1f --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/active-utc.yml @@ -0,0 +1,20 @@ +repository: + full_name: example/utc-demo + visibility: public + default_branch: main +workflow: + path: .github/workflows/check.yml + name: Check + state: active +content: | + name: Check + on: + schedule: + - cron: "17 9 * * 6" + workflow_dispatch: + permissions: + contents: read + concurrency: check-${{ github.ref }} + jobs: + lint: + uses: example/reusable/.github/workflows/lint.yml@main diff --git a/scripts/fixtures/scheduled-workflows/comment-only-workflow.yml b/scripts/fixtures/scheduled-workflows/comment-only-workflow.yml new file mode 100644 index 000000000..1f7967b58 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/comment-only-workflow.yml @@ -0,0 +1,10 @@ +repository: + full_name: example/comment-only + visibility: public + default_branch: main +workflow: + path: .github/workflows/comment-only.yml + name: Comment only + state: active +content: | + # This file intentionally has no active workflow definition. diff --git a/scripts/fixtures/scheduled-workflows/directory-api-error.yml b/scripts/fixtures/scheduled-workflows/directory-api-error.yml new file mode 100644 index 000000000..78963b3f5 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/directory-api-error.yml @@ -0,0 +1,6 @@ +repository: + full_name: example/directory-error + visibility: public + default_branch: main +directory_response: + message: expected directory array but received an API object diff --git a/scripts/fixtures/scheduled-workflows/disabled-inactivity.yml b/scripts/fixtures/scheduled-workflows/disabled-inactivity.yml new file mode 100644 index 000000000..a2afa1b79 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/disabled-inactivity.yml @@ -0,0 +1,13 @@ +repository: + full_name: example/inactive-demo + visibility: public + default_branch: main +workflow: + path: .github/workflows/inactive.yml + name: Inactive + state: disabled_inactivity +content: | + name: Inactive + on: + schedule: + - cron: "0 0 * * 0" diff --git a/scripts/fixtures/scheduled-workflows/job-level-controls.yml b/scripts/fixtures/scheduled-workflows/job-level-controls.yml new file mode 100644 index 000000000..e66933d47 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/job-level-controls.yml @@ -0,0 +1,21 @@ +repository: + full_name: example/job-controls + visibility: public + default_branch: main +workflow: + path: .github/workflows/job-controls.yml + name: Job controls + state: active +content: | + name: Job controls + on: + schedule: + - cron: "0 4 * * *" + jobs: + scan: + permissions: + contents: read + concurrency: scan-${{ github.ref }} + runs-on: ubuntu-latest + steps: + - run: true diff --git a/scripts/fixtures/scheduled-workflows/missing-workflow-directory.yml b/scripts/fixtures/scheduled-workflows/missing-workflow-directory.yml new file mode 100644 index 000000000..e5ae29e4f --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/missing-workflow-directory.yml @@ -0,0 +1,7 @@ +repository: + full_name: example/no-workflows + visibility: public + default_branch: main +error: + status: 404 + message: workflow directory not found diff --git a/scripts/fixtures/scheduled-workflows/multiple-cron.yml b/scripts/fixtures/scheduled-workflows/multiple-cron.yml new file mode 100644 index 000000000..053811543 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/multiple-cron.yml @@ -0,0 +1,14 @@ +repository: + full_name: example/multiple-cron + visibility: public + default_branch: main +workflow: + path: .github/workflows/multiple.yml + name: Multiple cron + state: active +content: | + name: Multiple cron + on: + schedule: + - cron: "0 1 * * *" + - cron: "0 13 * * *" diff --git a/scripts/fixtures/scheduled-workflows/private-no-schedule.yml b/scripts/fixtures/scheduled-workflows/private-no-schedule.yml new file mode 100644 index 000000000..753a04fa5 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/private-no-schedule.yml @@ -0,0 +1,12 @@ +repository: + full_name: example/private-no-schedule + visibility: private + default_branch: main +workflow: + path: .github/workflows/manual.yml + name: Manual + state: active +content: | + name: Manual + on: + workflow_dispatch: diff --git a/scripts/fixtures/scheduled-workflows/private-redaction.yml b/scripts/fixtures/scheduled-workflows/private-redaction.yml new file mode 100644 index 000000000..a07476383 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/private-redaction.yml @@ -0,0 +1,13 @@ +repository: + full_name: example/hidden-repository + visibility: private + default_branch: main +workflow: + path: .github/workflows/private.yml + name: Private schedule + state: active +content: | + name: Private schedule + on: + schedule: + - cron: "0 6 * * *" diff --git a/scripts/fixtures/scheduled-workflows/rate-limit-error.yml b/scripts/fixtures/scheduled-workflows/rate-limit-error.yml new file mode 100644 index 000000000..53c7ee651 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/rate-limit-error.yml @@ -0,0 +1,7 @@ +repository: + full_name: example/rate-limited + visibility: public + default_branch: main +error: + status: 403 + message: API rate limit exceeded diff --git a/scripts/fixtures/scheduled-workflows/reusable-workflow-caller.yml b/scripts/fixtures/scheduled-workflows/reusable-workflow-caller.yml new file mode 100644 index 000000000..4f5ba21f8 --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/reusable-workflow-caller.yml @@ -0,0 +1,16 @@ +repository: + full_name: example/reusable-caller + visibility: public + default_branch: main +workflow: + path: .github/workflows/caller.yml + name: Caller + state: active +content: | + name: Caller + on: + schedule: + - cron: "0 3 * * *" + jobs: + reusable: + uses: example/platform/.github/workflows/reusable.yml@v1 diff --git a/scripts/fixtures/scheduled-workflows/top-level-controls.yml b/scripts/fixtures/scheduled-workflows/top-level-controls.yml new file mode 100644 index 000000000..58f3fe75c --- /dev/null +++ b/scripts/fixtures/scheduled-workflows/top-level-controls.yml @@ -0,0 +1,17 @@ +repository: + full_name: example/top-controls + visibility: public + default_branch: main +workflow: + path: .github/workflows/top-controls.yml + name: Top controls + state: active +content: | + name: Top controls + on: + schedule: + - cron: "0 5 * * *" + permissions: read-all + concurrency: + group: top-${{ github.ref }} + cancel-in-progress: true diff --git a/scripts/test-audit-scheduled-workflows.rb b/scripts/test-audit-scheduled-workflows.rb new file mode 100755 index 000000000..b779aa9f3 --- /dev/null +++ b/scripts/test-audit-scheduled-workflows.rb @@ -0,0 +1,448 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "stringio" +require "tmpdir" +require "yaml" + +require_relative "audit-scheduled-workflows" + +class ScheduledWorkflowAuditTest + FIXTURES = File.expand_path("fixtures/scheduled-workflows", __dir__) + + def fixture(name) + YAML.safe_load_file(File.join(FIXTURES, "#{name}.yml"), aliases: false) + end + + def assert_equal(expected, actual) + raise "expected #{expected.inspect}, got #{actual.inspect}" unless expected == actual + end + + def assert(value) + raise "expected truthy value" unless value + end + + def refute(value) + raise "expected falsey value" if value + end + + def assert_includes(haystack, needle) + raise "expected #{haystack.inspect} to include #{needle.inspect}" unless haystack.include?(needle) + end + + def refute_includes(haystack, needle) + raise "expected #{haystack.inspect} not to include #{needle.inspect}" if haystack.include?(needle) + end + + def assert_match(pattern, actual) + raise "expected #{actual.inspect} to match #{pattern.inspect}" unless pattern.match?(actual) + end + + def assert_raises(error_class) + yield + raise "expected #{error_class}" + rescue error_class => error + error + end + + def parse(name) + data = fixture(name) + ScheduledWorkflowAudit::WorkflowParser.new.parse( + repository: data.fetch("repository"), + metadata: data.fetch("workflow"), + content: data.fetch("content") + ) + end + + def test_parser_returns_the_versioned_record_shape + assert_equal( + { + "schema" => "z-shell/scheduled-workflow-audit/v1", + "repository" => "example/utc-demo", + "visibility" => "public", + "default_branch" => "main", + "path" => ".github/workflows/check.yml", + "name" => "Check", + "state" => "active", + "schedules" => [{ "cron" => "17 9 * * 6", "timezone" => "UTC" }], + "workflow_dispatch" => true, + "permissions_locations" => ["workflow"], + "concurrency_locations" => ["workflow"], + "reusable_calls" => ["example/reusable/.github/workflows/lint.yml@main"], + "errors" => [] + }, + parse("active-utc") + ) + end + + def test_parser_preserves_an_explicit_iana_timezone_and_multiple_crons + assert_equal [{ "cron" => "0 8 * * 1-5", "timezone" => "Europe/London" }], parse("active-iana-timezone").fetch("schedules") + assert_equal 2, parse("multiple-cron").fetch("schedules").length + end + + def test_parser_defaults_invalid_timezones_to_utc + data = fixture("active-utc") + data["content"] = <<~YAML + on: + schedule: + - cron: "17 9 * * 6" + timezone: "" + - cron: "18 9 * * 6" + timezone: 42 + jobs: {} + YAML + + record = ScheduledWorkflowAudit::WorkflowParser.new.parse( + repository: data.fetch("repository"), + metadata: data.fetch("workflow"), + content: data.fetch("content") + ) + + assert_equal ["UTC", "UTC"], record.fetch("schedules").map { |schedule| schedule.fetch("timezone") } + end + + def test_parser_treats_a_comment_only_workflow_as_unscheduled + assert_equal [], parse("comment-only-workflow").fetch("schedules") + end + + def test_parser_rejects_a_non_mapping_workflow_document + data = fixture("comment-only-workflow") + error = assert_raises(ArgumentError) do + ScheduledWorkflowAudit::WorkflowParser.new.parse( + repository: data.fetch("repository"), metadata: data.fetch("workflow"), content: "[]\n" + ) + end + + assert_match(/mapping/, error.message) + end + + def test_parser_rejects_a_false_workflow_document + data = fixture("comment-only-workflow") + error = assert_raises(ArgumentError) do + ScheduledWorkflowAudit::WorkflowParser.new.parse( + repository: data.fetch("repository"), metadata: data.fetch("workflow"), content: "false\n" + ) + end + + assert_match(/mapping/, error.message) + end + + def test_parser_finds_job_and_workflow_controls + assert_equal ["job"], parse("job-level-controls").fetch("permissions_locations") + assert_equal ["job"], parse("job-level-controls").fetch("concurrency_locations") + assert_equal ["workflow"], parse("top-level-controls").fetch("permissions_locations") + assert_equal ["workflow"], parse("top-level-controls").fetch("concurrency_locations") + assert_equal ["example/platform/.github/workflows/reusable.yml@v1"], parse("reusable-workflow-caller").fetch("reusable_calls") + end + + def test_inventory_includes_disabled_workflows_unless_active_only_is_selected + inventory = ScheduledWorkflowAudit::Inventory.new( + client: FixtureClient.new([fixture("active-utc"), fixture("disabled-inactivity")]), + org: "example" + ) + + assert_equal ["example/inactive-demo", "example/utc-demo"], inventory.run.map { |record| record.fetch("repository") } + assert_equal 1, inventory.run(active_only: true).length + end + + def test_markdown_hides_private_repository_names + rendered = ScheduledWorkflowAudit::Renderer.new.markdown( + [parse("active-utc"), parse("private-redaction")], + public_only: false + ) + + assert_includes rendered, "example/utc-demo" + refute_includes rendered, "example/hidden-repository" + end + + def test_markdown_escapes_table_delimiters_and_newlines + record = parse("active-utc") + record["repository"] = "example/pipe|repo\ncontinued" + record["name"] = "Check|Now\nLater" + record["state"] = "active|queued\nstate" + record["schedules"] = [{ "cron" => "17|18\n19", "timezone" => "UTC|Local\nZone" }] + + rendered = ScheduledWorkflowAudit::Renderer.new.markdown([record], public_only: false) + + assert_includes rendered, "example/pipe\\|repo
continued" + assert_includes rendered, "Check\\|Now
Later" + assert_includes rendered, "active\\|queued
state" + assert_includes rendered, "17\\|18
19 (UTC\\|Local
Zone)" + end + + def test_repository_scope_does_not_require_an_organization + stdout = StringIO.new + stderr = StringIO.new + status = ScheduledWorkflowAudit::CLI.run( + ["--repo", "example/utc-demo", "--format", "markdown"], + client: FixtureClient.new([fixture("active-utc")]), stdout: stdout, stderr: stderr + ) + + assert_equal 0, status + assert_equal "", stderr.string + assert_includes stdout.string, "example/utc-demo" + end + + def test_private_json_requires_a_private_output_file + stdout = StringIO.new + stderr = StringIO.new + + status = ScheduledWorkflowAudit::CLI.run( + ["--org", "example", "--format", "json"], + client: FixtureClient.new([fixture("private-redaction")]), stdout: stdout, stderr: stderr + ) + + assert_equal 2, status + assert_match(/--output FILE/, stderr.string) + end + + def test_private_target_without_a_scheduled_workflow_still_requires_an_output_file + stderr = StringIO.new + status = ScheduledWorkflowAudit::CLI.run( + ["--org", "example", "--format", "json"], + client: FixtureClient.new([fixture("private-no-schedule")]), stdout: StringIO.new, stderr: stderr + ) + + assert_equal 2, status + assert_match(/--output FILE/, stderr.string) + end + + def test_private_json_output_is_created_with_mode_0600 + Dir.mktmpdir do |directory| + destination = File.join(directory, "inventory.json") + status = ScheduledWorkflowAudit::CLI.run( + ["--org", "example", "--format", "json", "--output", destination], + client: FixtureClient.new([fixture("private-redaction")]), stdout: StringIO.new, stderr: StringIO.new + ) + + assert_equal 0, status + assert_equal "600", format("%o", File.stat(destination).mode & 0o777) + end + end + + def test_client_uses_only_get_requests_and_reports_api_failures + client = ScheduledWorkflowAudit::GitHubClient.new(runner: lambda do |command| + assert_equal ["gh", "api", "--method", "GET", "/repos/example/no-workflows/contents/.github/workflows"], command + ["", "{\"message\":\"Not Found\"}", false] + end) + + error = assert_raises(ScheduledWorkflowAudit::GitHubError) do + client.json("/repos/example/no-workflows/contents/.github/workflows") + end + assert_equal "Not Found", error.message + end + + def test_client_turns_a_non_object_error_payload_into_a_github_error + client = ScheduledWorkflowAudit::GitHubClient.new(runner: lambda do |_command| + ["", "[]", false] + end) + + error = assert_raises(ScheduledWorkflowAudit::GitHubError) { client.json("/repos/example/demo") } + assert_equal "[]", error.message + end + + def test_client_preserves_status_from_a_json_error_payload + client = ScheduledWorkflowAudit::GitHubClient.new(runner: lambda do |_command| + ["", "{\"message\":\"Not Found\",\"status\":404}", false] + end) + + error = assert_raises(ScheduledWorkflowAudit::GitHubError) { client.json("/repos/example/demo") } + + assert_equal 404, error.status + end + + def test_inventory_returns_structured_errors_for_missing_or_failed_workflow_lookups + data = [fixture("missing-workflow-directory"), fixture("directory-api-error"), fixture("rate-limit-error")] + records = ScheduledWorkflowAudit::Inventory.new(client: FixtureClient.new(data), org: "example").run + + assert_equal 2, records.length + assert_equal [nil, 403], records.map { |record| record.fetch("errors").first.fetch("status") } + end + + def test_invalid_workflow_metadata_returns_a_structured_error_and_cli_exit_1 + data = fixture("active-utc") + data["workflows_response"] = { "message" => "unexpected workflow metadata object" } + records = ScheduledWorkflowAudit::Inventory.new(client: FixtureClient.new([data]), org: "example").run + + assert_equal 1, records.length + assert_match(/workflow metadata response/, records.fetch(0).fetch("errors").fetch(0).fetch("message")) + status = ScheduledWorkflowAudit::CLI.run( + ["--org", "example", "--format", "json"], + client: FixtureClient.new([data]), stdout: StringIO.new, stderr: StringIO.new + ) + assert_equal 1, status + end + + def test_missing_workflow_directory_means_zero_workflows + records = ScheduledWorkflowAudit::Inventory.new( + client: FixtureClient.new([fixture("missing-workflow-directory")]), org: "example" + ).run + + assert_equal [], records + end + + def test_directory_api_object_returns_one_structured_error_record + records = ScheduledWorkflowAudit::Inventory.new( + client: FixtureClient.new([fixture("directory-api-error")]), org: "example" + ).run + + assert_equal 1, records.length + assert_match(/workflow directory response must be an array/, records.fetch(0).fetch("errors").fetch(0).fetch("message")) + end + + def test_directory_entries_require_string_paths + data = fixture("active-utc") + data["directory_response"] = [nil, { "path" => 123 }] + + records = ScheduledWorkflowAudit::Inventory.new(client: FixtureClient.new([data]), org: "example").run + + assert_equal 1, records.length + assert_match(/workflow directory entries must include paths/, records.fetch(0).fetch("errors").fetch(0).fetch("message")) + end + + def test_inventory_turns_unexpected_worker_failures_into_error_records + failed = fixture("active-utc") + failed.fetch("repository")["full_name"] = "example/worker-failure" + client = UnexpectedFailureClient.new( + [failed, fixture("active-utc")], + failing_repository: "example/worker-failure" + ) + + records = ScheduledWorkflowAudit::Inventory.new(client: client, org: "example").run + + assert_equal ["example/utc-demo", "example/worker-failure"], records.map { |record| record.fetch("repository") } + failure = records.find { |record| record.fetch("repository") == "example/worker-failure" } + assert_match(/unexpected worker failure/, failure.fetch("errors").fetch(0).fetch("message")) + end + + def test_inventory_ignores_archived_and_fork_repositories + archived = fixture("active-utc") + archived.fetch("repository")["full_name"] = "example/archived" + archived.fetch("repository")["archived"] = true + fork = fixture("active-utc") + fork.fetch("repository")["full_name"] = "example/fork" + fork.fetch("repository")["fork"] = true + client = FixtureClient.new([fixture("active-utc"), archived, fork]) + + records = ScheduledWorkflowAudit::Inventory.new(client: client, org: "example").run + + assert_equal ["example/utc-demo"], records.map { |record| record.fetch("repository") } + refute client.requests.any? { |path| path.include?("example/archived") || path.include?("example/fork") } + end + + def test_inventory_requests_all_repository_pages + client = PagedClient.new + ScheduledWorkflowAudit::Inventory.new(client: client, org: "example").run + + assert_includes client.requests, "/orgs/example/repos?type=all&per_page=100&page=2" + end + + def test_inventory_paginates_workflow_metadata_for_active_only + data = fixture("active-utc") + data["workflow_pages"] = [ + { "workflows" => (1..100).map { |index| { "path" => ".github/workflows/page-#{index}.yml", "name" => "Page #{index}", "state" => "disabled_inactivity" } } }, + { "workflows" => [data.fetch("workflow")] } + ] + client = FixtureClient.new([data]) + + records = ScheduledWorkflowAudit::Inventory.new(client: client, org: "example").run(active_only: true) + + assert_equal ["Check"], records.map { |record| record.fetch("name") } + assert_includes client.requests, "/repos/example/utc-demo/actions/workflows?per_page=100&page=2" + end + + class FixtureClient + attr_reader :requests + + def initialize(fixtures) + @fixtures = fixtures + @requests = [] + end + + def json(path) + @requests << path + repository_name = path[%r{^/repos/([^/]+/[^/]+)}, 1] + if path == "/orgs/example/repos?type=all&per_page=100" + return @fixtures.map { |data| data.fetch("repository") } + end + + data = @fixtures.find { |fixture| fixture.fetch("repository").fetch("full_name") == repository_name } + raise "unexpected request: #{path}" unless data + + return data.fetch("repository") if path == "/repos/#{repository_name}" + + if data.key?("error") + error = data.fetch("error") + raise ScheduledWorkflowAudit::GitHubError.new(error.fetch("message"), status: error.fetch("status")) + end + + if path.match?(%r{/contents/\.github/workflows\?ref=}) + return data.fetch("directory_response", [{ "path" => data.fetch("workflow", {}).fetch("path", ".github/workflows/missing.yml") }]) + end + + if path.match?(%r{/actions/workflows\?per_page=100(?:&page=\d+)?\z}) + page = path[/&page=(\d+)\z/, 1]&.to_i || 1 + return data.fetch("workflow_pages")[page - 1] if data.key?("workflow_pages") + + return data.fetch("workflows_response", { "workflows" => [data.fetch("workflow")] }) + end + + if path.include?("/contents/") + return { + "content" => [data.fetch("content")].pack("m0"), + "encoding" => "base64" + } + end + + raise "unexpected request: #{path}" + end + end + + class PagedClient + attr_reader :requests + + def initialize + @requests = [] + @repositories = (1..101).map do |index| + { "full_name" => "example/page-#{index}", "visibility" => "public", "default_branch" => "main" } + end + end + + def json(path) + @requests << path + return @repositories.first(100) if path == "/orgs/example/repos?type=all&per_page=100" + return @repositories.last(1) if path == "/orgs/example/repos?type=all&per_page=100&page=2" + return [] if path.include?("/contents/.github/workflows?") + return { "workflows" => [] } if path.end_with?("/actions/workflows?per_page=100") + + raise "unexpected request: #{path}" + end + end + + class UnexpectedFailureClient < FixtureClient + def initialize(fixtures, failing_repository:) + super(fixtures) + @failing_repository = failing_repository + end + + def json(path) + raise "unexpected worker failure" if path.start_with?("/repos/#{@failing_repository}/") + + super + end + end +end + +tests = ScheduledWorkflowAuditTest.new +methods = ScheduledWorkflowAuditTest.instance_methods(false).grep(/^test_/).sort +failures = methods.filter_map do |method| + tests.public_send(method) + puts "PASS #{method}" + nil +rescue StandardError => error + warn "FAIL #{method}: #{error.message}" + error +end + +exit(failures.empty? ? 0 : 1) diff --git a/scripts/test_validate_agent_policy.py b/scripts/test_validate_agent_policy.py index 29a1a5668..da5ae4493 100644 --- a/scripts/test_validate_agent_policy.py +++ b/scripts/test_validate_agent_policy.py @@ -1477,6 +1477,38 @@ def test_cli_escapes_control_characters_in_discovered_paths(self) -> None: class PublicRepositoryTests(unittest.TestCase): + def test_public_manifest_routes_recurring_operations_runbook(self) -> None: + manifest = json.loads( + (PUBLIC_ROOT / ".github/instruction-surfaces.json").read_text() + ) + recurring_operations_surfaces = [ + surface + for surface in manifest["surfaces"] + if surface.get("canonical_for") == ["recurring-operations"] + ] + + self.assertEqual( + recurring_operations_surfaces, + [ + { + "id": "runbook-recurring-operations", + "path": "runbooks/recurring-operations.md", + "kind": "runbook", + "authority": "canonical-detail", + "consumers": ["codex", "claude-code", "copilot", "human"], + "tasks": [ + "recurring-operations", + "scheduled-workflow-audit", + "automation-review", + ], + "file_patterns": ["**"], + "required": True, + "review_owner": "z-shell maintainers", + "canonical_for": ["recurring-operations"], + } + ], + ) + def test_public_repository_documents_instruction_governance(self) -> None: adr = ( PUBLIC_ROOT / "decisions/0014-portable-agent-instruction-architecture.md"