diff --git a/.actionlint.yaml b/.actionlint.yaml index 7fdbbb5..f4098a3 100644 --- a/.actionlint.yaml +++ b/.actionlint.yaml @@ -1,8 +1,7 @@ ignore: - # NINJA_MAX_JOBS is intentionally optional — defined as "" at the workflow level + # NINJA_MAX_JOBS is intentionally optional -- defined as "" at the workflow level # so it can be overridden by uncommenting and setting a value. An empty string is # falsy in GHA expressions, so the build-arg is omitted when unset. actionlint # flags env.NINJA_MAX_JOBS because the key is commented out in the top-level env: # block and therefore cannot be statically verified. - 'Context access might be invalid: NINJA_MAX_JOBS' - \ No newline at end of file diff --git a/.claude/agents/dockerfile-reviewer.md b/.claude/agents/dockerfile-reviewer.md index 2bd30ed..0a9550d 100644 --- a/.claude/agents/dockerfile-reviewer.md +++ b/.claude/agents/dockerfile-reviewer.md @@ -31,6 +31,6 @@ Review `Dockerfile` and `Dockerfile.devtools` for the following: ## Context -Cross-reference with `.hadolint.yaml` for intentionally suppressed rules — do not flag issues that are already documented as acceptable. +Cross-reference with `.hadolint.yaml` for intentionally suppressed rules -- do not flag issues that are already documented as acceptable. Report findings grouped by severity: Critical, Warning, Info. diff --git a/.claude/hooks/block-pipe-to-shell.sh b/.claude/hooks/block-pipe-to-shell.sh deleted file mode 100755 index 9927f83..0000000 --- a/.claude/hooks/block-pipe-to-shell.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# PreToolUse hook: Block curl/wget pipe-to-shell patterns -# Source: Project CLAUDE.md — "prefer Debian-packaged software over curl|bash install scripts" -INPUT=$(cat) -COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command') - -# Block: curl ... | bash, curl ... | sh, wget ... | bash, wget ... | sh -# Also catches variants with sudo, env, or flags between the pipe and shell -if echo "$COMMAND" | grep -qEi '\b(curl|wget)\b.*\|\s*(sudo\s+)?(bash|sh|zsh|dash)\b'; then - echo "Blocked: Do not pipe curl/wget to a shell. Download the file first, verify its checksum, then execute it." >&2 - exit 2 -fi - -exit 0 diff --git a/.claude/hooks/check-dockerfile-heredoc-strict.sh b/.claude/hooks/check-dockerfile-heredoc-strict.sh new file mode 100755 index 0000000..6057bc8 --- /dev/null +++ b/.claude/hooks/check-dockerfile-heredoc-strict.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# PostToolUse hook: Ensure Dockerfile RUN heredocs contain 'set -euo pipefail'. +# Source: CLAUDE.md - "Dockerfile RUN heredocs use set -euo pipefail" +set -euo pipefail + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +case "$(basename "$CLAUDE_FILE_PATH")" in + Dockerfile*) ;; + *) exit 0 ;; +esac + +# Find RUN heredocs missing 'set -euo pipefail'. +# Tracks heredoc boundaries by matching the delimiter word after << and +# looking for it alone on a line to close the block. +violations=$(awk ' + /^[^#]*RUN.*<<[[:space:]]*[A-Za-z_]+/ { + tmp = $0 + sub(/.*<<[[:space:]]*/, "", tmp) + sub(/[^A-Za-z_].*/, "", tmp) + delim = tmp + in_heredoc = 1 + heredoc_start = NR + found_strict = 0 + next + } + in_heredoc { + if ($0 ~ /set -euo pipefail/ && $0 !~ /^[[:space:]]*#/) found_strict = 1 + if ($0 == delim) { + if (!found_strict) print heredoc_start": RUN heredoc (closed at line "NR") missing set -euo pipefail" + in_heredoc = 0 + } + } +' "$CLAUDE_FILE_PATH") + +if [[ -n "$violations" ]]; then + printf "Convention: Dockerfile RUN heredocs must include 'set -euo pipefail'. Violations in %s:\n" "$CLAUDE_FILE_PATH" >&2 + printf "%s\n" "$violations" >&2 + exit 1 +fi diff --git a/.claude/hooks/check-lint-registration.sh b/.claude/hooks/check-lint-registration.sh new file mode 100755 index 0000000..752a1fc --- /dev/null +++ b/.claude/hooks/check-lint-registration.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# PostToolUse hook: Warn when a lintable file is not registered in lint.sh. +# Also flags stale entries (files listed in lint.sh that no longer exist). +set -euo pipefail + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +cd "$(git rev-parse --show-toplevel)" || exit 1 + +# Only check lintable file types +case "$(basename "$CLAUDE_FILE_PATH")" in + Dockerfile*|*.sh|*.yml|*.yaml|*.json) ;; + *) exit 0 ;; +esac + +# Get path relative to repo root; skip files outside the repo +repo_root=$(pwd) +if [[ "$CLAUDE_FILE_PATH" == "$repo_root"/* ]]; then + rel_path="${CLAUDE_FILE_PATH#"$repo_root"/}" +else + exit 0 +fi + +# Extract all file entries from lint.sh arrays +extract_entries() { + awk ' + /^[A-Z_]+=\(/ && !/\)/ { in_arr=1; next } + in_arr && /^\)/ { in_arr=0; next } + in_arr { + gsub(/^[[:space:]]+/, "") + gsub(/[[:space:]]+$/, "") + if ($0 != "" && $0 !~ /^#/) print + } + ' lint.sh +} + +rc=0 + +# Check if this file is registered as an array entry (not just mentioned anywhere) +registered=false +while IFS= read -r entry; do + if [[ "$entry" == "$rel_path" ]]; then + registered=true + break + fi +done < <(extract_entries) + +if [[ "$registered" == false ]]; then + printf "Convention: '%s' is not registered in lint.sh. Add it to the appropriate file list.\n" "$rel_path" >&2 + rc=1 +fi + +# Check for stale entries (files listed but no longer on disk) +stale="" +while IFS= read -r entry; do + if [[ -n "$entry" && ! -f "$entry" ]]; then + stale+=" $entry"$'\n' + fi +done < <(extract_entries) + +if [[ -n "$stale" ]]; then + printf "Stale entries in lint.sh (files no longer exist). Remove them:\n" >&2 + printf "%s" "$stale" >&2 + rc=1 +fi + +exit "$rc" diff --git a/.claude/hooks/check-shell-strict-mode.sh b/.claude/hooks/check-shell-strict-mode.sh new file mode 100755 index 0000000..269ce4d --- /dev/null +++ b/.claude/hooks/check-shell-strict-mode.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# PostToolUse hook: Ensure shell scripts contain 'set -euo pipefail'. +# Exception: lint.sh uses 'set -uo pipefail' (omits -e for non-fail-fast design). + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +case "$(basename "$CLAUDE_FILE_PATH")" in + *.sh) ;; + *) exit 0 ;; +esac + +# lint.sh intentionally omits -e so all checks run before reporting a summary. +if [[ "$(basename "$CLAUDE_FILE_PATH")" == "lint.sh" ]]; then + if ! grep -qE '^[^#]*set -uo pipefail' "$CLAUDE_FILE_PATH"; then + echo "Convention: lint.sh must include 'set -uo pipefail'. Not found in $CLAUDE_FILE_PATH" >&2 + exit 1 + fi +else + if ! grep -qE '^[^#]*set -euo pipefail' "$CLAUDE_FILE_PATH"; then + echo "Convention: shell scripts must include 'set -euo pipefail'. Not found in $CLAUDE_FILE_PATH" >&2 + exit 1 + fi +fi diff --git a/.claude/hooks/check-trailing-whitespace.sh b/.claude/hooks/check-trailing-whitespace.sh new file mode 100755 index 0000000..b10c4eb --- /dev/null +++ b/.claude/hooks/check-trailing-whitespace.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# PostToolUse hook: Flag trailing whitespace in changed files. +# Markdown exception: trailing two spaces after text (line break) is allowed. +set -euo pipefail + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +# Skip binary files +case "$(basename "$CLAUDE_FILE_PATH")" in + *.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar*|*.gz|*.bz2|*.xz|*.woff*|*.ttf|*.eot|*.tgz) exit 0 ;; +esac + +base=$(basename "$CLAUDE_FILE_PATH") + +if [[ "$base" == *.md ]]; then + # In markdown, exactly 2 trailing spaces after non-whitespace text is a line break. + # Flag all other trailing whitespace (tabs, 1 space, 3+ spaces, whitespace-only lines). + violations=$(awk ' + /[[:space:]]$/ { + line = $0 + n = 0 + while (n < length(line) && substr(line, length(line) - n, 1) == " ") n++ + if (n == 2 && line ~ /[^[:space:]]/) next + print NR": "$0 + } + ' "$CLAUDE_FILE_PATH") +else + violations=$(grep -nE '[[:space:]]$' "$CLAUDE_FILE_PATH" || true) +fi + +if [[ -n "$violations" ]]; then + printf "Convention: remove trailing whitespace. Violations in %s:\n" "$CLAUDE_FILE_PATH" >&2 + printf "%s\n" "$violations" >&2 + exit 1 +fi diff --git a/.claude/hooks/check-unicode-lookalikes.sh b/.claude/hooks/check-unicode-lookalikes.sh new file mode 100755 index 0000000..f87f3a5 --- /dev/null +++ b/.claude/hooks/check-unicode-lookalikes.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# PostToolUse hook: Flag Unicode lookalikes that sneak in via autocorrect/copy-paste. +# Catches em/en dashes, smart quotes, non-breaking spaces, and ellipsis. +# Intentional Unicode (e.g., box-drawing chars in terminal output) is NOT matched. +set -euo pipefail + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +case "$(basename "$CLAUDE_FILE_PATH")" in + *.sh|*.yml|*.yaml|*.json|*.md|Dockerfile*) ;; + *) exit 0 ;; +esac + +# Build grep character class from UTF-8 byte sequences (keeps this script pure ASCII). +# U+00A0 non-breaking space +# U+2013 en dash +# U+2014 em dash +# U+2018 left single smart quote +# U+2019 right single smart quote +# U+201C left double smart quote +# U+201D right double smart quote +# U+2026 ellipsis +NBSP=$(printf '\xc2\xa0') +EN_DASH=$(printf '\xe2\x80\x93') +EM_DASH=$(printf '\xe2\x80\x94') +LSQUO=$(printf '\xe2\x80\x98') +RSQUO=$(printf '\xe2\x80\x99') +LDQUO=$(printf '\xe2\x80\x9c') +RDQUO=$(printf '\xe2\x80\x9d') +ELLIPSIS=$(printf '\xe2\x80\xa6') + +violations=$(grep -n "[${NBSP}${EN_DASH}${EM_DASH}${LSQUO}${RSQUO}${LDQUO}${RDQUO}${ELLIPSIS}]" "$CLAUDE_FILE_PATH" || true) + +if [[ -n "$violations" ]]; then + printf "Convention: avoid Unicode lookalikes (smart quotes, em/en dashes, etc.) in source files.\n" >&2 + printf "Use ASCII equivalents instead. Violations in %s:\n" "$CLAUDE_FILE_PATH" >&2 + printf "%s\n" "$violations" >&2 + exit 1 +fi diff --git a/.claude/hooks/check-workflow-expressions.sh b/.claude/hooks/check-workflow-expressions.sh new file mode 100755 index 0000000..96b3b49 --- /dev/null +++ b/.claude/hooks/check-workflow-expressions.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# PostToolUse hook: Flag ${{ github.* }} in workflow run: blocks. +# These should use shell env vars set via the step's env: key. +# +# Checks for ${{ github.* }} on lines inside run: blocks. These should be +# replaced with shell env vars (e.g., $GITHUB_REF) set via the step's env: key. + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +case "$(basename "$CLAUDE_FILE_PATH")" in + *.yml|*.yaml) ;; + *) exit 0 ;; +esac + +# Use awk to find ${{ github.* }} only inside run: blocks. +# Track indentation to detect when a run: block ends. +violations=$(awk ' + /^[[:space:]]+run:[[:space:]]*[|>]/ { + in_run = 1 + match($0, /^[[:space:]]*/) + run_indent = RLENGTH + next + } + /^[[:space:]]+run:[[:space:]]*[^|>]/ { + # Single-line run: value + if ($0 ~ /\$\{\{[[:space:]]*github\./) print NR": "$0 + next + } + in_run { + if (NF == 0) next + match($0, /^[[:space:]]*/) + cur_indent = RLENGTH + if (cur_indent <= run_indent && $0 ~ /[a-zA-Z_-]+:/) { in_run = 0; next } + if ($0 ~ /\$\{\{[[:space:]]*github\./) print NR": "$0 + } +' "$CLAUDE_FILE_PATH") + +if [[ -n "$violations" ]]; then + printf "Convention: use shell env vars (e.g., \$GITHUB_REF) instead of \${{ github.* }} in run: blocks.\n" >&2 + printf "Set values via the step's env: key. Violations in %s:\n" "$CLAUDE_FILE_PATH" >&2 + printf "%s\n" "$violations" >&2 + exit 1 +fi diff --git a/.claude/hooks/lint-changed-file.sh b/.claude/hooks/lint-changed-file.sh new file mode 100755 index 0000000..475ea12 --- /dev/null +++ b/.claude/hooks/lint-changed-file.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# PostToolUse hook: Lint only the file that was just edited/written. +# Delegates to lint.sh --file for consistent linter args. +set -euo pipefail + +[[ -z "$CLAUDE_FILE_PATH" ]] && exit 0 + +cd "$(git rev-parse --show-toplevel)" || exit 1 + +./lint.sh --file "$CLAUDE_FILE_PATH" 2>&1 | tail -20 diff --git a/.claude/settings.json b/.claude/settings.json index aeca89b..7dcec09 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,23 +1,36 @@ { "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "bash .claude/hooks/block-pipe-to-shell.sh" - } - ] - } - ], "PostToolUse": [ { "matcher": "Edit|Write", "hooks": [ { "type": "command", - "command": "bash -c 'cd \"$(git rev-parse --show-toplevel)\" && file=\"$CLAUDE_FILE_PATH\"; case \"$file\" in Dockerfile*|*.sh|*.yml|*.yaml|*.json) bash lint.sh 2>&1 | tail -20 ;; esac; exit 0'" + "command": "./.claude/hooks/lint-changed-file.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-shell-strict-mode.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-workflow-expressions.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-lint-registration.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-dockerfile-heredoc-strict.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-unicode-lookalikes.sh" + }, + { + "type": "command", + "command": "./.claude/hooks/check-trailing-whitespace.sh" } ] } diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index 22a416b..0306f4b 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -1,6 +1,6 @@ --- name: build -description: Build Docker images locally. Use after linting passes. +description: Build salt-dev Docker container images (base and devtools) via buildx with the salt-8cpu builder. Creates the builder if missing. disable-model-invocation: true --- @@ -9,8 +9,17 @@ disable-model-invocation: true Build the project Docker images: -1. First run `bash lint.sh` — abort if any check fails -2. Build base image: `docker buildx build --pull -t salt-dev --load .` -3. If $ARGUMENTS contains "devtools" or "all": - - Build devtools: `docker buildx build -f Dockerfile.devtools -t salt-dev-tools --load .` -4. Report build success/failure and image sizes via `docker images | grep salt-dev` +1. Run `./lint.sh` -- abort if any check fails +2. Ensure the `salt-8cpu` builder exists: + - Check: `docker buildx inspect salt-8cpu 2>/dev/null` + - If missing, create and constrain to 8 CPUs: + ```bash + docker buildx create --name salt-8cpu --driver docker-container --driver-opt default-load=true + docker buildx inspect --bootstrap salt-8cpu + docker update --cpus 8 "$(docker ps -qf 'name=buildx_buildkit_salt-8cpu')" + ``` + - Check memory: `docker info --format '{{.MemTotal}}'` -- warn user if < 22 GB +3. Build base image: `docker buildx build --builder salt-8cpu --pull -t salt-dev --load .` +4. If $ARGUMENTS contains "devtools" or "all": + - Build devtools: `docker buildx build --builder salt-8cpu -f Dockerfile.devtools -t salt-dev-tools --load .` +5. Report build success/failure and image sizes via `docker images | grep salt-dev` diff --git a/.claude/skills/lint/SKILL.md b/.claude/skills/lint/SKILL.md deleted file mode 100644 index 5a2de6a..0000000 --- a/.claude/skills/lint/SKILL.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: lint -description: Run all project linters and fix any issues found ---- - -## Current State -- Branch: !`git branch --show-current` -- Modified files: !`git diff --name-only` - -Run the project linters: - -1. Execute `bash lint.sh` from the repo root -2. If any check fails, read `.hadolint.yaml` for suppression policy -3. Fix issues in the source files — do NOT add inline `# hadolint ignore=` directives -4. For new hadolint suppressions, add to `.hadolint.yaml` with a rationale comment -5. Re-run `bash lint.sh` until all checks pass -6. Report which checks passed and any fixes applied diff --git a/.dockerignore b/.dockerignore index 13b1aaf..ad97791 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,9 +9,10 @@ LICENSE /patches/LICENSE /patches/.git/ -# Dev tooling — not used by any Dockerfile +# Dev tooling -- not used by any Dockerfile /.claude/ /.devcontainer/ +/.gitattributes /.gitignore /.gitmodules /.actionlint.yaml @@ -19,3 +20,6 @@ LICENSE /lint.sh /test-build-llvm.sh +# Loose archives -- fetched fresh during build +*.tgz + diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5e796e1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Normalize line endings to LF in the index, use native endings on checkout +* text=auto diff --git a/.github/actions/docker-cache/backup.sh b/.github/actions/docker-cache/backup.sh deleted file mode 100755 index d005b5b..0000000 --- a/.github/actions/docker-cache/backup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# shellcheck source=timing.sh -. "${BASH_SOURCE%/*}/timing.sh" - -main() { - local cache_tar=$1 - local cache_dir - cache_dir=$(dirname "$cache_tar") - - mkdir -p "$cache_dir" - rm -f "$cache_tar" - - timing sudo service docker stop - timing sudo /bin/tar -c -f "$cache_tar" -C /var/lib/docker . - sudo chown "$USER:$(id -g -n "$USER")" "$cache_tar" - ls -lh "$cache_tar" -} - -main "$@" diff --git a/.github/actions/docker-cache/restore.sh b/.github/actions/docker-cache/restore.sh deleted file mode 100755 index 73713c4..0000000 --- a/.github/actions/docker-cache/restore.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# shellcheck source=timing.sh -. "${BASH_SOURCE%/*}/timing.sh" - -main() { - local cache_tar=$1 - - if [[ -f "$cache_tar" ]]; then - ls -lh "$cache_tar" - timing sudo service docker stop - # mv is c. 25 seconds faster than rm -rf here - timing sudo mv /var/lib/docker "$(mktemp -d --dry-run)" - sudo mkdir -p /var/lib/docker - timing sudo tar -xf "$cache_tar" -C /var/lib/docker - timing sudo service docker start - else - # Slim docker down - comes with 3GB of data we don't want to backup - timing docker system prune -a -f --volumes - fi -} - -main "$@" diff --git a/.github/actions/docker-cache/timing.sh b/.github/actions/docker-cache/timing.sh deleted file mode 100755 index e807502..0000000 --- a/.github/actions/docker-cache/timing.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -timing() { - command time -f "[$*] took %E" "$@" -} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d49e3a5..a4756d1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -7,6 +7,7 @@ permissions: env: DOCKER_BUILDKIT: 1 + LLVM_VER: "20" # Uncomment to cap ninja parallelism (reduces peak memory for cold LLVM builds) # NINJA_MAX_JOBS: "2" @@ -22,6 +23,36 @@ on: branches: - "main" jobs: + changes: + # Detect whether build-relevant files changed. + # Skipped for schedule and tag events -- those always build. + if: >- + github.event_name != 'schedule' + && !startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + outputs: + build: ${{ steps.filter.outputs.build }} + steps: + - + name: Checkout + uses: actions/checkout@v6 + - + name: Check changed paths + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + build: + - 'Dockerfile' + - 'Dockerfile.devtools' + - 'build-llvm.sh' + - 'docker-entrypoint.sh' + - 'install-intel-ifx.sh' + - 'patches' + - 'patches/**' + - '.dockerignore' + - '.github/workflows/CI.yml' + lint: runs-on: ubuntu-latest steps: @@ -48,9 +79,21 @@ jobs: run: bash test-build-llvm.sh build-base: - needs: lint + needs: [lint, changes] + # Run when lint passed AND build-relevant files changed. + # Always run for schedule and tag events (changes job is skipped). + if: >- + always() + && needs.lint.result == 'success' + && (needs.changes.result == 'skipped' + || needs.changes.outputs.build == 'true') runs-on: ubuntu-latest steps: + - + name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive - name: Docker meta id: meta @@ -63,18 +106,18 @@ jobs: # generate Docker tags based on the following events/attributes tags: | type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest-llvm-${{ env.LLVM_VER }},enable={{is_default_branch}} type=ref,event=branch + type=ref,event=branch,suffix=-llvm-${{ env.LLVM_VER }} type=ref,event=pr type=semver,pattern={{version}} + type=semver,pattern={{version}}-llvm-${{ env.LLVM_VER }} type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}}.{{minor}}-llvm-${{ env.LLVM_VER }} type=semver,pattern={{major}} + type=semver,pattern={{major}}-llvm-${{ env.LLVM_VER }} type=sha type=schedule - - - name: Checkout - uses: actions/checkout@v6 - with: - submodules: recursive - name: Set up Docker Buildx id: setup-buildx @@ -124,8 +167,9 @@ jobs: file: ./Dockerfile pull: true build-args: | - CI=true + PHASED_BUILD=true AVAIL_MEM_KB=16567500 + LLVM_VER=${{ env.LLVM_VER }} ${{ env.NINJA_MAX_JOBS && format('NINJA_MAX_JOBS={0}', env.NINJA_MAX_JOBS) || '' }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} @@ -141,8 +185,9 @@ jobs: context: . file: ./Dockerfile build-args: | - CI=true + PHASED_BUILD=true AVAIL_MEM_KB=16567500 + LLVM_VER=${{ env.LLVM_VER }} ${{ env.NINJA_MAX_JOBS && format('NINJA_MAX_JOBS={0}', env.NINJA_MAX_JOBS) || '' }} push: false tags: salt-dev:cache-warmup @@ -162,6 +207,9 @@ jobs: if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: + - + name: Checkout + uses: actions/checkout@v6 - name: Docker meta id: meta @@ -172,16 +220,18 @@ jobs: ghcr.io/paratoolsinc/salt-dev-tools tags: | type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest-llvm-${{ env.LLVM_VER }},enable={{is_default_branch}} type=ref,event=branch + type=ref,event=branch,suffix=-llvm-${{ env.LLVM_VER }} type=ref,event=pr type=semver,pattern={{version}} + type=semver,pattern={{version}}-llvm-${{ env.LLVM_VER }} type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}}.{{minor}}-llvm-${{ env.LLVM_VER }} type=semver,pattern={{major}} + type=semver,pattern={{major}}-llvm-${{ env.LLVM_VER }} type=sha type=schedule - - - name: Checkout - uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.gitignore b/.gitignore index f090d24..2b9d7a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.claude/worktrees/** \ No newline at end of file +.claude/worktrees/** +.claude/settings.local.json diff --git a/CLAUDE.md b/CLAUDE.md index 829a0b2..65a388a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,34 +1,25 @@ # CLAUDE.md -Container definitions for [SALT](https://github.com/ParaToolsInc/salt) CI/CD and dev. Published to Docker Hub (`paratools/salt-dev`) and GHCR (`ghcr.io/paratoolsinc/salt-dev`). No test suite — only linting. +Container definitions for [SALT](https://github.com/ParaToolsInc/salt) CI/CD and dev. Published to Docker Hub (`paratools/salt-dev`) and GHCR (`ghcr.io/paratoolsinc/salt-dev`). No test suite -- only linting. ## Commands ```bash -# Build (requires BuildKit) -docker buildx build --pull -t salt-dev --load . -# Devtools image (requires base) -docker buildx build -f Dockerfile.devtools -t salt-dev-tools --load . -# Clone with submodules (required for patches/) -git clone --recursive git@github.com:ParaToolsInc/salt-dev.git -# Lint (run after any Dockerfile/shell/workflow/JSON change — LLVM builds take 2-3h so lint failures are expensive) -bash lint.sh +git clone --recursive git@github.com:ParaToolsInc/salt-dev.git # patches/ submodule required ``` +- `/build` -- build Docker images with the salt-8cpu builder + ## Conventions -- Shell scripts use `set -euo pipefail` - Hadolint suppressions: prefer inline `# hadolint ignore=DLxxxx` on the line directly above the instruction; put rationale on a separate comment line above -- When adding new Dockerfiles, shell scripts, workflows, or JSON files, add them to `lint.sh` -- Supply chain: verify downloads with sha256, pin GPG fingerprints, prefer Debian packages over `curl|bash` -- In workflow `run:` blocks, use `$GITHUB_REF` not `${{ github.ref }}` (script injection prevention) +- Supply chain: verify downloads with sha256, pin GPG fingerprints - Version tags: `v*.*.*` semver ## Gotchas -- `patches/` is a git submodule — clone with `--recursive` -- `mpich` branch still has `OMPI_ALLOW_RUN_AS_ROOT` env vars (OpenMPI leftovers, no effect with MPICH) -- GitHub CLI GPG key fingerprint (`2C61...6059`) pinned in `Dockerfile.devtools` — **expires 2026-09-04** -- PDT checksum (`2fc9e86...`) pinned in `Dockerfile` — update if upstream changes `pdt_lite.tgz` -- LLVM build: 150-200 min cold, ~1 min with ccache; `ARG CI=false` triggers shallow clones in GHA only +- Same-repo PRs can read base-branch `actions/cache` entries via `restore-keys`; fork PRs cannot +- GitHub CLI GPG key fingerprint (`2C61...6059`) pinned in `Dockerfile.devtools` -- **expires 2026-09-04** +- PDT checksum (`2fc9e86...`) pinned in `Dockerfile` -- update if upstream changes `pdt_lite.tgz` +- LLVM build: 150-200 min cold, ~4 min with ccache; `ARG PHASED_BUILD=true` enables OOM-aware phased build - Intel IFX APT repo has signature verification issues on Debian 13+ (sqv rejects Intel's OpenPGP format); `install-intel-ifx.sh` detects and prompts for `[trusted=yes]` diff --git a/Dockerfile b/Dockerfile index bdfea8f..4018198 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,8 @@ set -euo pipefail cmake --version EOC -ARG CI=false -ARG LLVM_VER=19 +ARG PHASED_BUILD=true +ARG LLVM_VER=20 # Clone LLVM repo. A shallow clone is faster, but pulling a cached repository is faster yet # cd inside heredoc script; WORKDIR can't replace it # RUN --mount=type=cache,target=/git <= 20 renamed the 'flang-new' binary to 'flang' and its install target accordingly + if [ "${LLVM_VER}" -ge 20 ]; then FLANG_BIN_TARGET=install-flang; else FLANG_BIN_TARGET=install-flang-new; fi FLANG_TARGETS=( tools/flang/install - install-flang-libraries install-flang-headers install-flang-new install-flang-cmake-exports + install-flang-libraries install-flang-headers "$FLANG_BIN_TARGET" install-flang-cmake-exports install-flangFrontend install-flangFrontendTool install-FortranCommon install-FortranDecimal install-FortranEvaluate install-FortranLower install-FortranParser install-FortranRuntime install-FortranSemantics @@ -144,9 +146,9 @@ set -euo pipefail ccache -s - if ${CI:-false}; then + if ${PHASED_BUILD:-true}; then - echo "=== CI mode: phased LLVM build to prevent OOM ===" + echo "=== Phased LLVM build (OOM-aware) ===" # Phase 1: Non-Flang targets at full parallelism (no OOM risk) echo "--- Phase 1: Non-Flang targets (parallel) ---" @@ -156,9 +158,14 @@ set -euo pipefail # Targets discovered from OOM failures on CI (-j4, 4.1 GB) and local (-j8, 2.5 GB). # Matched by CMake target directory; individual files within these dirs tend to be # memory-hungry due to heavy template instantiation in Flang/MLIR. + # Flang libs: FortranEvaluate, FortranSemantics, FortranLower, FortranParser + # Flang driver: flang (fc1_main.cpp.o, driver.cpp.o -- worst offender) + # Flang tools: bbc, fir-opt, fir-lsp-server, tco + # FIR/MLIR: FIRCodeGen, flangFrontend(Tool), MLIRMlirOptMain, + # MLIRCAPIRegisterEverything, MLIRLinalgTransforms echo "--- Phase 2: OOM-fragile object files (-j2) ---" mapfile -t OOM_TARGETS < <(ninja -C "$BUILD_DIR" -t targets all 2>/dev/null \ - | grep -E '(Fortran(Evaluate|Semantics|Lower|Parser)|FIRCodeGen|flangFrontend(Tool)?|MLIRMlirOptMain|bbc)\.dir/.*\.cpp\.o:' \ + | grep -E '(Fortran(Evaluate|Semantics|Lower|Parser)|FIRCodeGen|flangFrontend(Tool)?|flang|MLIRMlirOptMain|MLIRCAPIRegisterEverything|MLIRLinalgTransforms|bbc|fir-opt|fir-lsp-server|tco)\.dir/.*\.cpp\.o:' \ | cut -d: -f1 || true) if [ ${#OOM_TARGETS[@]} -gt 0 ]; then echo "Building ${#OOM_TARGETS[@]} OOM-fragile targets at -j2" @@ -172,7 +179,7 @@ set -euo pipefail echo "--- Phase 3: Flang targets (parallel, OOM files pre-built) ---" build-llvm.sh "${BUILD_LLVM_ARGS[@]}" "${FLANG_TARGETS[@]}" else - # Local: single pass (adequate memory + build-llvm.sh retry handles OOM) + # Single pass: all targets at once (use PHASED_BUILD=false to select) build-llvm.sh "${BUILD_LLVM_ARGS[@]}" \ "${NON_FLANG_TARGETS[@]}" "${FLANG_TARGETS[@]}" fi @@ -184,12 +191,22 @@ EOC RUN <&2 - exit 1 + if [ "${LLVM_VER}" -ge 20 ]; then + # LLVM >= 20: flang binary is versioned (flang-20) with a 'flang' symlink; + # use -L to follow symlinks so find matches both the real file and the symlink + FLANG="$(find -L /tmp/llvm -name flang -type f)" + if [ -z "$FLANG" ]; then + echo "ERROR: flang not found in /tmp/llvm -- Flang build failed?" >&2 + exit 1 + fi + else + FLANG_NEW="$(find /tmp/llvm -name flang-new -type f)" + if [ -z "$FLANG_NEW" ]; then + echo "ERROR: flang-new not found in /tmp/llvm -- Flang build failed?" >&2 + exit 1 + fi + ln -s flang-new "$(dirname "$FLANG_NEW")/flang" fi - ln -s flang-new "$(dirname "$FLANG_NEW")/flang" # remove for LLVM 20 EOC # Patch installed cmake exports/config files to not throw an error if not all components are installed @@ -260,6 +277,7 @@ ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 # http://tau.uoregon.edu/tau.tgz # http://fs.paratools.com/tau-mirror/tau.tgz # http://fs.paratools.com/tau-nightly.tgz +ARG LLVM_VER=20 # hadolint ignore=DL3003 RUN --mount=type=cache,id=ccache-tau,target=/home/salt/ccache <= 20 renamed 'flang-new' to 'flang' + if [ "${LLVM_VER}" -ge 20 ]; then FLANG_CMD=flang; else FLANG_CMD=flang-new; fi ./installtau -prefix=/usr/local -cc=gcc -c++=g++ -fortran=gfortran -pdt=/usr/local -pdt_c++=g++ \ -bfd=download -unwind=download -dwarf=download -otf=download -zlib=download -pthread -j ./installtau -prefix=/usr/local -cc=gcc -c++=g++ -fortran=gfortran -pdt=/usr/local -pdt_c++=g++ \ -bfd=download -unwind=download -dwarf=download -otf=download -zlib=download -pthread -mpi -j - ./installtau -prefix=/usr/local -cc=clang -c++=clang++ -fortran=flang-new -pdt=/usr/local -pdt_c++=g++ \ + ./installtau -prefix=/usr/local -cc=clang -c++=clang++ -fortran="$FLANG_CMD" -pdt=/usr/local -pdt_c++=g++ \ -bfd=download -unwind=download -dwarf=download -otf=download -zlib=download -pthread -j - ./installtau -prefix=/usr/local -cc=clang -c++=clang++ -fortran=flang-new -pdt=/usr/local -pdt_c++=g++ \ + ./installtau -prefix=/usr/local -cc=clang -c++=clang++ -fortran="$FLANG_CMD" -pdt=/usr/local -pdt_c++=g++ \ -bfd=download -unwind=download -dwarf=download -otf=download -zlib=download -pthread -mpi -j cd .. rm -rf tau* libdwarf-* otf2-* diff --git a/Dockerfile.devtools b/Dockerfile.devtools index a71d981..355652e 100644 --- a/Dockerfile.devtools +++ b/Dockerfile.devtools @@ -12,7 +12,7 @@ USER root # Create a named salt user with a real home directory # GID 967 matches the docker group already in the base image -# UID is not pinned — use --user at runtime to match host UID for bind mounts +# UID is not pinned -- use --user at runtime to match host UID for bind mounts # chown required: base image WORKDIR creates /home/salt owned by root, # and useradd -m does not chown pre-existing directories RUN useradd -m -s /bin/bash -g 967 salt \ diff --git a/README.md b/README.md index da7b292..a4e651c 100644 --- a/README.md +++ b/README.md @@ -41,15 +41,15 @@ htop, jq, bat, and python3. Build using the helper script: ``` shell -bash build-devtools.sh --no-push --no-intel # salt-dev-tools only, no Intel IFX (local) -bash build-devtools.sh --no-push # salt-dev-tools + Intel IFX installed (local) -bash build-devtools.sh # build, install IFX, and push to Docker Hub +./build-devtools.sh --no-push --no-intel # salt-dev-tools only, no Intel IFX (local) +./build-devtools.sh --no-push # salt-dev-tools + Intel IFX installed (local) +./build-devtools.sh # build, install IFX, and push to Docker Hub ``` To pin a specific IFX version: ``` shell -bash build-devtools.sh --no-push --ifx-version=2025.3 --tag=intel-2025.3 +./build-devtools.sh --no-push --ifx-version=2025.3 --tag=intel-2025.3 ``` Or build directly with BuildKit (requires a local `salt-dev` image): @@ -61,7 +61,7 @@ docker buildx build -f Dockerfile.devtools -t salt-dev-tools --load . Launch interactively (reads `CLAUDE_CODE_OAUTH_TOKEN` and `GH_TOKEN` from the environment): ``` shell -bash run-salt-dev.sh +./run-salt-dev.sh ``` ### VS Code Devcontainer @@ -73,12 +73,12 @@ The devcontainer configuration will build the image automatically and install Gi | Script | Description | Example | |---|---|---| -| `build-devtools.sh` | Builds `salt-dev-tools`, optionally installs Intel IFX, and pushes to Docker Hub | `bash build-devtools.sh --no-push` | -| `run-salt-dev.sh` | Launches a `salt-dev` or `salt-dev-tools` container with sensible defaults; resolves git identity and API tokens from the environment | `bash run-salt-dev.sh` | +| `build-devtools.sh` | Builds `salt-dev-tools`, optionally installs Intel IFX, and pushes to Docker Hub | `./build-devtools.sh --no-push` | +| `run-salt-dev.sh` | Launches a `salt-dev` or `salt-dev-tools` container with sensible defaults; resolves git identity and API tokens from the environment | `./run-salt-dev.sh` | | `install-intel-ifx.sh` | Installs Intel IFX/ICX/ICPX compilers inside `salt-dev-tools`; handles Debian 13+ APT signature quirks | `./install-intel-ifx.sh 2025.2` | -| `build-llvm.sh` | OOM-resilient LLVM build wrapper around ninja; maximizes parallelism and auto-recovers by retrying failed targets at progressively lower `-j` | `bash build-llvm.sh clang flang-new` | -| `test-build-llvm.sh` | Unit and integration tests for `build-llvm.sh` | `bash test-build-llvm.sh` | -| `lint.sh` | Runs all linters: hadolint, shellcheck, actionlint, jq | `bash lint.sh` | +| `build-llvm.sh` | OOM-resilient LLVM build wrapper around ninja; maximizes parallelism and auto-recovers by retrying failed targets at progressively lower `-j` | `./build-llvm.sh clang flang` | +| `test-build-llvm.sh` | Unit and integration tests for `build-llvm.sh` | `./test-build-llvm.sh` | +| `lint.sh` | Runs all linters: hadolint, shellcheck, actionlint, jq | `./lint.sh` | ## Optimizations for expensive build steps diff --git a/build-devtools.sh b/build-devtools.sh index 44d3a08..c54d055 100755 --- a/build-devtools.sh +++ b/build-devtools.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # -# build-devtools.sh — Build the salt-dev-tools image with Intel IFX compilers. +# build-devtools.sh -- Build the salt-dev-tools image with Intel IFX compilers. # # Automates the full pipeline: build devtools image from a base image, install # Intel IFX inside a temporary container, commit the result, and tag it. @@ -148,7 +148,7 @@ else fi ############################################################################### -# Stage 2: Install Intel IFX (local only — not pushed) +# Stage 2: Install Intel IFX (local only -- not pushed) ############################################################################### if [[ "$INSTALL_INTEL" == true ]]; then diff --git a/build-llvm.sh b/build-llvm.sh index 8c30ed4..f1f8f5c 100755 --- a/build-llvm.sh +++ b/build-llvm.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# build-llvm.sh — Adaptive OOM-resilient LLVM build wrapper around ninja. +# build-llvm.sh -- Adaptive OOM-resilient LLVM build wrapper around ninja. # # Maximizes parallelism, auto-recovers from OOM kills by retrying failed # targets at progressively lower -j values, and provides CI-friendly @@ -155,7 +155,7 @@ main() { done if [[ "$recovered" = false ]]; then - echo "*** Build failed even at -j1 — likely a real build error" >&2 + echo "*** Build failed even at -j1 -- likely a real build error" >&2 return 1 fi diff --git a/install-intel-ifx.sh b/install-intel-ifx.sh index 60c88ea..9f7f369 100755 --- a/install-intel-ifx.sh +++ b/install-intel-ifx.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# install-intel-ifx.sh — Install Intel IFX Fortran compiler (and icx/icpx C/C++) +# install-intel-ifx.sh -- Install Intel IFX Fortran compiler (and icx/icpx C/C++) # inside the salt-dev-tools Docker container. # # Usage: @@ -138,7 +138,7 @@ detect_debian_version() { warn "signature verifier (sqv) cannot process. APT signature verification will fail." warn "" warn "Options:" - warn " 1) Continue with [trusted=yes] — bypasses signature check (TLS still protects download)" + warn " 1) Continue with [trusted=yes] -- bypasses signature check (TLS still protects download)" warn " 2) Abort and wait for Intel to update their repository signing key" warn "" warn "To skip this prompt, re-run with --trust-intel-repo" @@ -225,7 +225,7 @@ install_packages() { else info "Installing Intel oneAPI compilers version ${pkg_version}..." - # Package names changed in 2024+: dpcpp-cpp-and-cpp-classic → dpcpp-cpp + # Package names changed in 2024+: dpcpp-cpp-and-cpp-classic -> dpcpp-cpp case "$pkg_version" in 2024* | 2025*) as_root apt-get install -y --no-install-recommends \ @@ -268,7 +268,7 @@ generate_env_file() { as_root mkdir -p "$(dirname "$ENV_FILE")" { echo "#!/usr/bin/env bash" - echo "# Intel oneAPI environment — auto-generated by install-intel-ifx.sh" + echo "# Intel oneAPI environment -- auto-generated by install-intel-ifx.sh" echo "# Source this file to activate Intel compilers:" echo "# source ${ENV_FILE}" echo "" @@ -332,7 +332,7 @@ FORTRAN if ifx -o "${tmpdir}/hello" "${tmpdir}/hello.f90" && "${tmpdir}/hello"; then info "Compile test passed." else - warn "Compile test failed — compiler is installed but may have runtime issues." + warn "Compile test failed -- compiler is installed but may have runtime issues." fi rm -rf "$tmpdir" } diff --git a/lint.sh b/lint.sh index b7b329d..1ed40de 100755 --- a/lint.sh +++ b/lint.sh @@ -1,30 +1,45 @@ #!/usr/bin/env bash # -# lint.sh — Run all linters for the salt-dev repository. +# lint.sh -- Run all linters for the salt-dev repository. # -# Usage: bash lint.sh +# Usage: ./lint.sh # lint all tracked files +# ./lint.sh --file PATH # lint a single file (for hooks) +# ./lint.sh --warn-untracked # also warn about unlisted files +# ./lint.sh -v # verbose output # # Requires: hadolint, shellcheck, actionlint, jq # Runs all checks (non-fail-fast) and reports a summary at the end. -# --- Color output when connected to a terminal --- +# -e intentionally omitted: linter failures must not abort the script so +# that all checks run and a full summary is reported at the end. +set -uo pipefail + # --- Parse flags --- VERBOSE=false -for arg in "$@"; do - case $arg in - -v|--verbose) VERBOSE=true ;; - *) printf "Unknown argument: %s\n" "$arg" >&2; exit 1 ;; +SINGLE_FILE="" +WARN_UNTRACKED=false +while [[ $# -gt 0 ]]; do + case $1 in + -v|--verbose) VERBOSE=true; shift ;; + --file) + if [[ $# -lt 2 || -z "$2" ]]; then + printf "Error: --file requires a path argument\n" >&2 + exit 1 + fi + SINGLE_FILE="$2"; shift 2 ;; + --warn-untracked) WARN_UNTRACKED=true; shift ;; + *) printf "Unknown argument: %s\n" "$1" >&2; exit 1 ;; esac done if [[ -t 1 ]]; then RED=$'\033[0;31m' GREEN=$'\033[0;32m' - # YELLOW=$'\033[0;33m' + YELLOW=$'\033[0;33m' BOLD=$'\033[1m' RESET=$'\033[0m' else - RED='' GREEN='' BOLD='' RESET='' # YELLOW='' + RED='' GREEN='' YELLOW='' BOLD='' RESET='' fi # --- File lists --- @@ -41,9 +56,13 @@ SHELL_SCRIPTS=( install-intel-ifx.sh run-salt-dev.sh test-build-llvm.sh - .github/actions/docker-cache/backup.sh - .github/actions/docker-cache/restore.sh - .github/actions/docker-cache/timing.sh + .claude/hooks/lint-changed-file.sh + .claude/hooks/check-shell-strict-mode.sh + .claude/hooks/check-workflow-expressions.sh + .claude/hooks/check-lint-registration.sh + .claude/hooks/check-dockerfile-heredoc-strict.sh + .claude/hooks/check-unicode-lookalikes.sh + .claude/hooks/check-trailing-whitespace.sh ) WORKFLOWS=( @@ -52,6 +71,13 @@ WORKFLOWS=( JSON_FILES=( .devcontainer/devcontainer.json + .claude/settings.json +) + +# Tool configs -- validated implicitly by their respective tools at runtime +CONFIG_FILES=( + .actionlint.yaml + .hadolint.yaml ) # --- Helpers --- @@ -100,7 +126,7 @@ fi # --- Change to repo root --- cd "$(git rev-parse --show-toplevel)" || exit 1 -# --- Run linters --- +# --- Linter args (shared between single-file and full modes) --- # Note: hadolint reports shellcheck violations inside RUN heredocs using the # line number of the RUN instruction, not the actual line within the heredoc. # Run shellcheck directly on extracted scripts for precise heredoc line numbers. @@ -111,6 +137,34 @@ if $VERBOSE; then HADOLINT_ARGS+=(--format json) ACTIONLINT_ARGS+=(-verbose) fi + +# --- Single-file mode (for hooks) --- +if [[ -n "$SINGLE_FILE" ]]; then + base=$(basename "$SINGLE_FILE") + case "$base" in + Dockerfile*) exec hadolint "${HADOLINT_ARGS[@]}" "$SINGLE_FILE" ;; + *.sh) exec shellcheck "${SHELLCHECK_ARGS[@]}" "$SINGLE_FILE" ;; + .actionlint.yaml|.hadolint.yaml) exit 0 ;; # validated implicitly by their tools + *.yml|*.yaml) exec actionlint "${ACTIONLINT_ARGS[@]}" "$SINGLE_FILE" ;; + *.json) exec jq empty "$SINGLE_FILE" ;; + *) printf "Unknown file type: %s\n" "$SINGLE_FILE" >&2; exit 1 ;; + esac +fi + +# --- Validate manifest entries exist --- +missing_files=() +for f in "${DOCKERFILES[@]}" "${SHELL_SCRIPTS[@]}" "${WORKFLOWS[@]}" "${JSON_FILES[@]}" "${CONFIG_FILES[@]}"; do + [[ -f "$f" ]] || missing_files+=("$f") +done +if [[ ${#missing_files[@]} -gt 0 ]]; then + printf '%s%sError: files listed in lint.sh but missing from disk:%s\n' "${RED}" "${BOLD}" "${RESET}" >&2 + printf ' %s\n' "${missing_files[@]}" >&2 + exit 1 +fi + +warnings=0 + +# --- Full run: all tracked files --- run_check "hadolint" hadolint "${HADOLINT_ARGS[@]}" "${DOCKERFILES[@]}" run_check "shellcheck" shellcheck "${SHELLCHECK_ARGS[@]}" "${SHELL_SCRIPTS[@]}" # Suppressions managed in .actionlint.yaml @@ -129,6 +183,135 @@ jq_check() { } run_check "jq (JSON syntax)" jq_check +# --- Verify CI path-filter covers Dockerfile COPY/ADD sources --- +# shellcheck disable=SC2329,SC2317 # invoked indirectly via run_check +ci_filter_check() { + local ci_yml=".github/workflows/CI.yml" + [[ -f "$ci_yml" ]] || return 0 + + # Extract dorny/paths-filter 'build' patterns from CI workflow + local filters=() + while IFS= read -r val; do + val="${val#\'}" ; val="${val%\'}" + val="${val#\"}" ; val="${val%\"}" + [[ -n "$val" ]] && filters+=("$val") + done < <(awk ' + /filters:[[:space:]]*\|/ { in_f=1; next } + in_f && /^[[:space:]]+build:[[:space:]]*$/ { in_b=1; next } + in_b && /^[[:space:]]*-[[:space:]]/ { + sub(/^[[:space:]]*-[[:space:]]+/, "") + sub(/[[:space:]]+$/, "") + if ($0 != "") print + next + } + in_b && /^[[:space:]]*$/ { next } + in_b { exit } + ' "$ci_yml") + + if [[ ${#filters[@]} -eq 0 ]]; then + printf " No 'build:' path-filter found in %s -- skipping\n" "$ci_yml" + return 0 + fi + + local rc=0 + + # Extract COPY/ADD source paths from Dockerfiles (skip inter-stage copies) + local sources=() + for df in "${DOCKERFILES[@]}"; do + while IFS= read -r src; do + [[ -n "$src" ]] && sources+=("$src") + done < <(awk ' + /^(COPY|ADD)/ && !/--from=/ { + line = $0 + while (line ~ /\\$/) { + sub(/\\$/, "", line) + if ((getline nl) > 0) line = line " " nl + } + sub(/^(COPY|ADD)[[:space:]]+/, "", line) + while (line ~ /^--[a-z]+=[^ \t]+[ \t]/) + sub(/^--[a-z]+=[^ \t]+[ \t]+/, "", line) + n = split(line, t) + for (i = 1; i < n; i++) + if (t[i] != "") print t[i] + } + ' "$df") + done + + # Verify each COPY/ADD source is matched by a filter pattern + for src in "${sources[@]}"; do + local matched=false + for pat in "${filters[@]}"; do + # shellcheck disable=SC2053 # intentional glob match + if [[ "$src" == $pat ]]; then + matched=true + break + fi + done + if [[ "$matched" == false ]]; then + printf " COPY/ADD source '%s' not covered by CI path-filter in %s\n" "$src" "$ci_yml" + rc=1 + fi + done + + # Verify each Dockerfile is in the filter + for df in "${DOCKERFILES[@]}"; do + local matched=false + for pat in "${filters[@]}"; do + # shellcheck disable=SC2053 + if [[ "$df" == $pat ]]; then + matched=true + break + fi + done + if [[ "$matched" == false ]]; then + printf " Dockerfile '%s' not in CI path-filter\n" "$df" + rc=1 + fi + done + + # Verify submodule paths are in the filter (both gitlink and contents) + if [[ -f .gitmodules ]]; then + while IFS= read -r sm; do + local has_exact=false has_glob=false + for pat in "${filters[@]}"; do + [[ "$pat" == "$sm" ]] && has_exact=true + [[ "$pat" == "${sm}/**" ]] && has_glob=true + done + if [[ "$has_exact" == false ]]; then + printf " Submodule '%s' (gitlink) not in CI path-filter -- misses pointer updates\n" "$sm" + rc=1 + fi + if [[ "$has_glob" == false ]]; then + printf " Submodule '%s/**' (contents) not in CI path-filter\n" "$sm" + rc=1 + fi + done < <(git config --file .gitmodules --get-regexp 'submodule\..*\.path' | awk '{print $2}') + fi + + return "$rc" +} +run_check "CI path-filter coverage" ci_filter_check + +# --- Warn about untracked lintable files --- +if [[ "$WARN_UNTRACKED" == true ]]; then + manifest_list=$(printf '%s\n' "${DOCKERFILES[@]}" "${SHELL_SCRIPTS[@]}" "${WORKFLOWS[@]}" "${JSON_FILES[@]}" "${CONFIG_FILES[@]}") + untracked=() + while IFS= read -r f; do + if ! printf '%s\n' "$manifest_list" | grep -qxF "$f"; then + untracked+=("$f") + fi + done < <(git ls-files --cached --others --exclude-standard \ + -- 'Dockerfile*' '**/Dockerfile*' '*.sh' '**/*.sh' '*.yml' '**/*.yml' \ + '*.yaml' '**/*.yaml' '*.json' '**/*.json' | sort -u) + if [[ ${#untracked[@]} -gt 0 ]]; then + warnings=${#untracked[@]} + printf '\n%s%sWarning: %d file(s) not in lint.sh manifest:%s\n' "${YELLOW}" "${BOLD}" "$warnings" "${RESET}" + for f in "${untracked[@]}"; do + printf '%s %s%s\n' "${YELLOW}" "$f" "${RESET}" + done + fi +fi + # --- Summary --- printf '%s━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━%s\n' "${BOLD}" "${RESET}" if [[ $failures -eq 0 ]]; then @@ -136,6 +319,9 @@ if [[ $failures -eq 0 ]]; then else printf '%s%s%d of %d checks failed.%s\n' "${RED}" "${BOLD}" "$failures" "$checked" "${RESET}" fi +if [[ $warnings -gt 0 ]]; then + printf '%s%s%d warning(s).%s\n' "${YELLOW}" "${BOLD}" "$warnings" "${RESET}" +fi printf '%s━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━%s\n' "${BOLD}" "${RESET}" exit "$failures" diff --git a/run-salt-dev.sh b/run-salt-dev.sh index 9f59118..31a756b 100755 --- a/run-salt-dev.sh +++ b/run-salt-dev.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # -# run-salt-dev.sh — Launch a salt-dev container with sensible defaults. +# run-salt-dev.sh -- Launch a salt-dev container with sensible defaults. # # Usage: # bash run-salt-dev.sh [options] [image[:tag]] diff --git a/test-build-llvm.sh b/test-build-llvm.sh index e18d5b8..f2862f4 100755 --- a/test-build-llvm.sh +++ b/test-build-llvm.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# test-build-llvm.sh — Unit and integration tests for build-llvm.sh. +# test-build-llvm.sh -- Unit and integration tests for build-llvm.sh. # # Tier 1: Unit tests (parallelism formula + target extraction) # Tier 2: Integration tests (mock ninja, OOM recovery flows)