From c650ccee5f1900099c6798df11e9d03d98c7439f Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 18:30:05 +0900 Subject: [PATCH 01/14] upgrade ci. add option --- README.md | 85 +++++++++++++++++--- docs/ci/QUICKSTART.md | 35 ++++++--- ops/ci/ci_self.sh | 173 ++++++++++++++++++++++++++++++++--------- ops/ci/ci_self_test.go | 69 ++++++++++++++++ 4 files changed, 302 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 75047e2..1d07c01 100644 --- a/README.md +++ b/README.md @@ -25,28 +25,88 @@ ci-self up - `ci-self` / `verify.yml` は `nix-daemon.sh` を自動読み込みして `nix` を検出（毎回の手動 `source` は不要） - 既存の `verify.yml` が古い場合は `bash ops/ci/scaffold_verify_workflow.sh --repo --apply --force` で更新 -## Mac mini ワンコマンド（推奨） +## 別端末の CI runner を 1 コマンドで使う（推奨） -MacBook から 1 コマンドで「鍵認証確認 -> 同期 -> Mac mini 実行 -> 結果回収」まで行う: +この導線は、特定の `~/dev/maakie-brainlab` 専用ではありません。 + +- マシンA: self-hosted runner / colima / docker を置いている端末 +- マシンB: 普段コードを書く端末。ここからマシンAへ verify を依頼する + +マシンB から 1 コマンドで「鍵認証確認 -> 同期 -> マシンA実行 -> 結果回収」まで行います。 + +```bash +ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / +``` + +例: ```bash -ci-self remote-ci --host @ --project-dir '~/dev/maakie-brainlab' --repo mt4110/maakie-brainlab +ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/maakie-brainlab' --repo mt4110/maakie-brainlab ``` `remote-ci` の実行内容: 1. SSH 公開鍵認証（password禁止）を検証 -2. ローカル作業ツリーを Mac mini へ `rsync` 同期 -3. （repo指定時）runner bootstrap をベストエフォート実行 -4. Mac mini で `ops/ci/run_verify_full.sh` を実行 -5. `verify-full.status` と `out/logs` をローカル `out/remote//` に回収 +2. マシンB のローカル作業ツリーをマシンA の `--project-dir` へ `rsync` 同期 +3. （`--repo` 指定時）マシンA 上で `ci-self register` をベストエフォート実行 +4. マシンA で `ops/ci/run_verify_full.sh` を実行 +5. `verify-full.status` と `out/logs` をマシンB の `out/remote//` に回収 + +### 初回だけ必要な準備（マシンB -> マシンA） + +1. マシンB で SSH 鍵を作る（未作成なら） + +```bash +ssh-keygen -t ed25519 -a 100 +``` + +2. マシンB の公開鍵をマシンA の `~/.ssh/authorized_keys` に登録する + +```bash +cat ~/.ssh/id_ed25519_for_ci_runner.pub | ssh -i ~/.ssh/id_ed25519_for_ci_runner @ 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys' +``` + +3. パスワードなし SSH を確認する + +```bash +ssh -i ~/.ssh/id_ed25519_for_ci_runner -o BatchMode=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no @ true +``` + +4. 通ったら `remote-ci` を実行する + +```bash +ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / +``` + +公開鍵未登録時は、`remote-ci` 自体が `authorized_keys` 登録のヒントを出して停止します。 + +### 外出先からでも使える？ + +使える場合があります。`remote-ci` が必要としているのは「同一LAN」ではなく「SSH 到達性」です。 + +- 使える: マシンA に外出先から SSH で到達できる場合 +- 典型例: Tailscale / VPN / ポート転送済みの自宅回線 / 固定IP など +- 使えない: マシンA へ SSH 経路が無い場合 + +`ci-self remote-ci` 自体は、外部公開やトンネル作成までは行いません。そこは別途ネットワーク設計が必要です。 + +### 今できること / まだできないこと -公開鍵未登録時は、`authorized_keys` 登録のヒントを出して停止します。 +- できる: `--project-dir` と `--repo` を切り替えて、任意の CI 対象リポジトリをマシンA で実行 +- できる: 未コミット変更を含むローカル作業ツリーを同期して verify を走らせる +- できる: `verify-full.status` と `out/logs` を手元へ回収する +- できる: 同一LANでも外出先でも、SSH 到達性があれば同じコマンドで使う +- まだできない: SSH パスワード認証での `remote-ci` 実行 +- まだできない: SSH 経路が無い状態からの自動疎通確立 +- まだできない: 複数プロジェクトの自動検出や自動振り分け。どの repo をどこで走らせるかは `--project-dir` / `.ci-self.env` で明示する 補足: -- `--host` は `ssh` の接続先文字列（`user@host` / IP / `~/.ssh/config` のHost別名） -- `--project-dir` に `~` を使う場合は `--project-dir '~/'` のようにクオート +- `--host` は `ssh` の接続先文字列（`user@host` / IP / `~/.ssh/config` の Host 別名） +- `-i` / `--identity` で SSH 鍵ファイルを指定できる。毎回省略したい場合は `.ci-self.env` の `CI_SELF_REMOTE_IDENTITY` を使う +- `--project-dir` はマシンA 側の配置先パス。`~` を使う場合は `--project-dir '~/'` のようにクオート +- `--local-dir` を使うと、マシンB 側の同期元を明示できる +- `--repo` を省略すると bootstrap は skip されるが、同期済みプロジェクト上での standalone verify は実行できる - runner 初期化/復旧専用の旧導線は `ci-self remote-up` ## さらに短縮する設定ファイル @@ -65,8 +125,9 @@ ci-self config-init CI_SELF_REPO=mt4110/maakie-brainlab CI_SELF_REF=main CI_SELF_PROJECT_DIR=/Users//dev/maakie-brainlab -CI_SELF_REMOTE_HOST=@mac-mini.local +CI_SELF_REMOTE_HOST=@ci-runner.local CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_REMOTE_IDENTITY=/Users//.ssh/id_ed25519_for_ci_runner CI_SELF_PR_BASE=main ``` @@ -76,7 +137,7 @@ CI_SELF_PR_BASE=main - `ci-self up`: ローカル最短（register + run-focus） - `ci-self focus`: run-focus 後、PR未作成なら自動作成し checks を監視 -- `ci-self remote-ci`: 鍵必須・同期・Mac mini実行・結果回収を1コマンドで実行 +- `ci-self remote-ci`: 鍵必須・同期・別端末での verify 実行・結果回収を1コマンドで実行 - `ci-self doctor --fix`: 依存/gh auth/colima/docker/runner_health を診断し可能な範囲で修復 - `ci-self doctor --repo-dir `: `flake.nix` リポジトリの Nix 到達性も含めて診断 - `ci-self remote-up`: SSH先で register + run-focus（同期しない旧導線） diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index 0734e22..34ca921 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -31,31 +31,40 @@ ci-self config-init CI_SELF_REPO=mt4110/maakie-brainlab CI_SELF_REF=main CI_SELF_PROJECT_DIR=/Users//dev/maakie-brainlab -CI_SELF_REMOTE_HOST=@mac-mini.local +CI_SELF_REMOTE_HOST=@ci-runner.local CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_REMOTE_IDENTITY=/Users//.ssh/id_ed25519_for_ci_runner CI_SELF_PR_BASE=main ``` -## ネットワーク別ワンコマンド +## 別端末の CI runner を使う -同一LAN / 外出先（SSHあり, 推奨）: +- マシンA: self-hosted runner / colima / docker を置く端末 +- マシンB: そこへ verify を依頼する手元端末 + +まずマシンB で SSH 鍵を用意し、公開鍵をマシンA の `~/.ssh/authorized_keys` に登録します。 + +```bash +ssh-keygen -t ed25519 -a 100 +cat ~/.ssh/id_ed25519_for_ci_runner.pub | ssh -i ~/.ssh/id_ed25519_for_ci_runner @ 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys' +ssh -i ~/.ssh/id_ed25519_for_ci_runner -o BatchMode=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no @ true +``` + +通ったら `remote-ci` を実行します。 ```bash -ci-self remote-ci +ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` `remote-ci` は以下を 1 コマンドで実行します: 1. SSH 鍵認証チェック（password不可） -2. ローカル変更を Mac mini に `rsync` 同期 -3. Mac mini 側 verify 実行 +2. ローカル変更をマシンA に `rsync` 同期 +3. マシンA 側 verify 実行 4. `out/remote//` へ結果回収 -未設定なら `--host --project-dir --repo` を明示してください。 - -```bash -ci-self remote-ci --host @ --project-dir '~/dev/maakie-brainlab' --repo mt4110/maakie-brainlab -``` +`remote-ci` は LAN 専用ではなく、外出先でも SSH 到達性があれば使えます。 +逆に SSH 経路が無い場合、`remote-ci` 自体は疎通を作れません。 runner 初期化/復旧専用の旧導線: @@ -63,12 +72,14 @@ runner 初期化/復旧専用の旧導線: ci-self remote-up ``` -外出先（SSHなし）: +外出先で SSH 到達性が無い場合: ```bash ci-self run-focus ``` +これは GitHub 側 workflow の dispatch/watch 用の別導線であり、マシンA での即時 `remote-ci` 実行そのものの代替ではありません。 + ## 事前診断と自己修復 ```bash diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index e6f5d5c..def5ca7 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -21,6 +21,7 @@ CONFIG_REF="" CONFIG_PROJECT_DIR="" CONFIG_REMOTE_HOST="" CONFIG_REMOTE_PROJECT_DIR="" +CONFIG_REMOTE_IDENTITY="" CONFIG_REMOTE_CLI="" CONFIG_LABELS="" CONFIG_RUNNER_NAME="" @@ -112,6 +113,7 @@ load_config() { CI_SELF_PROJECT_DIR) CONFIG_PROJECT_DIR="$val" ;; CI_SELF_REMOTE_HOST) CONFIG_REMOTE_HOST="$val" ;; CI_SELF_REMOTE_PROJECT_DIR) CONFIG_REMOTE_PROJECT_DIR="$val" ;; + CI_SELF_REMOTE_IDENTITY) CONFIG_REMOTE_IDENTITY="$val" ;; CI_SELF_REMOTE_CLI) CONFIG_REMOTE_CLI="$val" ;; CI_SELF_LABELS) CONFIG_LABELS="$val" ;; CI_SELF_RUNNER_NAME) CONFIG_RUNNER_NAME="$val" ;; @@ -154,8 +156,8 @@ Examples: ci-self register ci-self run-watch ci-self run-focus - ci-self remote-ci --host @ --project-dir '~/dev/maakie-brainlab' - ci-self remote-up --host mac-mini.local --project-dir ~/dev/maakie-brainlab + ci-self remote-ci --host @ -i ~/.ssh/id_ed25519 --project-dir '~/dev/project' + ci-self remote-up --host ci-runner.local --project-dir ~/dev/project USAGE } @@ -330,12 +332,41 @@ sync_pr_from_template() { return 0 fi - local body_file - body_file="$(mktemp)" - cp "$tmpl" "$body_file" - gh pr edit "$pr_number" -R "$repo" --title "$title" --body-file "$body_file" - rm -f "$body_file" - echo "OK: pr_template_sync pr=$pr_number template=$tmpl title=$title" + local current_title="" + local current_body="" + current_title="$(gh pr view "$pr_number" -R "$repo" --json title --jq '.title // ""' 2>/dev/null || true)" + current_body="$(gh pr view "$pr_number" -R "$repo" --json body --jq '.body // ""' 2>/dev/null || true)" + + local should_set_title=0 + local should_set_body=0 + if [[ -z "$(trim "$current_title")" ]]; then + should_set_title=1 + fi + if [[ -z "$(trim "$current_body")" ]]; then + should_set_body=1 + fi + + if [[ "$should_set_title" -eq 0 && "$should_set_body" -eq 0 ]]; then + echo "SKIP: pr_template_sync reason=pr_already_has_title_and_body pr=$pr_number" + return 0 + fi + + local body_file="" + if [[ "$should_set_body" -eq 1 ]]; then + body_file="$(mktemp)" + cp "$tmpl" "$body_file" + fi + + if [[ "$should_set_title" -eq 1 && "$should_set_body" -eq 1 ]]; then + gh pr edit "$pr_number" -R "$repo" --title "$title" --body-file "$body_file" + elif [[ "$should_set_title" -eq 1 ]]; then + gh pr edit "$pr_number" -R "$repo" --title "$title" + else + gh pr edit "$pr_number" -R "$repo" --body-file "$body_file" + fi + + [[ -n "$body_file" ]] && rm -f "$body_file" + echo "OK: pr_template_sync pr=$pr_number template=$tmpl title_sync=$should_set_title body_sync=$should_set_body" } ensure_branch_pushed() { @@ -869,6 +900,7 @@ CI_SELF_PROJECT_DIR=${project_dir} # Optional remote defaults CI_SELF_REMOTE_HOST= CI_SELF_REMOTE_PROJECT_DIR= +CI_SELF_REMOTE_IDENTITY= CI_SELF_REMOTE_CLI=ci-self # Optional runner defaults @@ -940,18 +972,21 @@ remote_path_for_shell() { run_remote_command_in_dir() { local host="$1" local project_dir="$2" - shift 2 + local identity="${3:-}" + shift 3 local remote_cmd_q local script_q local remote_script local remote_cd_q + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_cmd_q="$(quote_words "$@")" remote_cd_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; cd %s; %s' "$remote_cd_q" "$remote_cmd_q" script_q="$(quote_words "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=$*" - ssh "$host" "bash -lc $script_q" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } first_existing_public_key() { @@ -968,19 +1003,40 @@ first_existing_public_key() { return 1 } +preferred_public_key() { + local identity="${1:-}" + local identity_pub="" + if [[ -n "$identity" ]]; then + identity_pub="${identity}.pub" + if [[ -f "$identity_pub" ]]; then + printf '%s\n' "$identity_pub" + return 0 + fi + fi + first_existing_public_key +} + ensure_ssh_key_auth() { local host="$1" - if ssh -o BatchMode=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no "$host" "true" >/dev/null 2>&1; then + local identity="${2:-}" + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + + if "${ssh_cmd[@]}" -o BatchMode=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no "$host" "true" >/dev/null 2>&1; then echo "OK: ssh key_auth host=$host" return 0 fi echo "ERROR: ssh key-based auth failed for host=$host" >&2 local pub_key="" - pub_key="$(first_existing_public_key || true)" + pub_key="$(preferred_public_key "$identity" || true)" if [[ -n "$pub_key" ]]; then echo "HINT: register your public key to remote ~/.ssh/authorized_keys" >&2 - echo "HINT: cat $pub_key | ssh $host 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys'" >&2 + if [[ -n "$identity" ]]; then + echo "HINT: cat $pub_key | ssh -i $identity $host 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys'" >&2 + else + echo "HINT: cat $pub_key | ssh $host 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys'" >&2 + fi else echo "HINT: generate a key first: ssh-keygen -t ed25519 -a 100" >&2 fi @@ -990,46 +1046,66 @@ ensure_ssh_key_auth() { ensure_remote_project_dir() { local host="$1" local project_dir="$2" + local identity="${3:-}" local remote_dir_q local script_q local remote_script + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_dir_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; mkdir -p %s' "$remote_dir_q" script_q="$(quote_words "$remote_script")" echo "OK: ssh host=$host ensure_dir=$project_dir" - ssh "$host" "bash -lc $script_q" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } sync_local_project_to_remote() { local local_dir="$1" local host="$2" local project_dir="$3" + local identity="${4:-}" + local rsync_cmd=(rsync -az --delete) + local ssh_rsh="" + if [[ -n "$identity" ]]; then + ssh_rsh="$(quote_words ssh -i "$identity")" + rsync_cmd+=(-e "$ssh_rsh") + fi echo "OK: rsync host=$host src=$local_dir dst=$project_dir" - rsync -az --delete \ - --exclude ".local/" \ - --exclude "out/" \ - --exclude "cache/" \ - --exclude ".DS_Store" \ - "$local_dir/" "$host:$project_dir/" + rsync_cmd+=( + --exclude ".local/" + --exclude "out/" + --exclude "cache/" + --exclude ".DS_Store" + "$local_dir/" + "$host:$project_dir/" + ) + "${rsync_cmd[@]}" } fetch_remote_verify_artifacts() { local host="$1" local project_dir="$2" local out_dir="$3" + local identity="${4:-}" + local rsync_base=(rsync -a) + local ssh_rsh="" + if [[ -n "$identity" ]]; then + ssh_rsh="$(quote_words ssh -i "$identity")" + rsync_base+=(-e "$ssh_rsh") + fi mkdir -p "$out_dir" "$out_dir/logs" local failed=0 - if rsync -a "$host:$project_dir/out/verify-full.status" "$out_dir/"; then + if "${rsync_base[@]}" "$host:$project_dir/out/verify-full.status" "$out_dir/"; then echo "OK: fetch status_file=$out_dir/verify-full.status" else echo "ERROR: fetch status_file failed host=$host path=$project_dir/out/verify-full.status" >&2 failed=1 fi - if rsync -a "$host:$project_dir/out/logs/" "$out_dir/logs/"; then + if "${rsync_base[@]}" "$host:$project_dir/out/logs/" "$out_dir/logs/"; then echo "OK: fetch logs_dir=$out_dir/logs" else echo "ERROR: fetch logs failed host=$host path=$project_dir/out/logs/" >&2 @@ -1062,12 +1138,15 @@ run_remote_ci_self() { local host="$1" local project_dir="$2" local remote_cli="$3" - shift 3 + local identity="${4:-}" + shift 4 local remote_args=("$@") local remote_args_q local script_q local remote_script local remote_cd_q + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_args_q="$(quote_words "${remote_args[@]}")" if [[ "$project_dir" == "~/"* ]]; then @@ -1078,7 +1157,7 @@ run_remote_ci_self() { printf -v remote_script 'set -euo pipefail; cd %s; %q %s' "$remote_cd_q" "$remote_cli" "$remote_args_q" script_q="$(quote_words "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=$remote_cli ${remote_args[*]}" - ssh "$host" "bash -lc $script_q" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } cmd_remote_ci() { @@ -1086,6 +1165,7 @@ cmd_remote_ci() { local project_dir="" local local_dir="" local out_dir="" + local identity="" local remote_cli="ci-self" local repo="" local labels="" @@ -1100,6 +1180,7 @@ cmd_remote_ci() { while [[ $# -gt 0 ]]; do case "$1" in --host) host="${2:-}"; shift 2 ;; + -i|--identity) identity="${2:-}"; shift 2 ;; --project-dir) project_dir="${2:-}"; shift 2 ;; --local-dir) local_dir="${2:-}"; shift 2 ;; --out-dir) out_dir="${2:-}"; shift 2 ;; @@ -1115,7 +1196,7 @@ cmd_remote_ci() { --no-sync) no_sync=1; shift ;; -h|--help) cat <<'USAGE' -Usage: ci-self remote-ci --host [--project-dir path] [--local-dir path] [--out-dir path] +Usage: ci-self remote-ci --host [-i identity_file] [--project-dir path] [--local-dir path] [--out-dir path] [--repo owner/repo] [--remote-cli path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] @@ -1132,6 +1213,7 @@ USAGE done [[ -z "$host" ]] && host="$CONFIG_REMOTE_HOST" + [[ -z "$identity" && -n "$CONFIG_REMOTE_IDENTITY" ]] && identity="$CONFIG_REMOTE_IDENTITY" [[ "$remote_cli" == "ci-self" && -n "$CONFIG_REMOTE_CLI" ]] && remote_cli="$CONFIG_REMOTE_CLI" [[ -z "$repo" && -n "$CONFIG_REPO" ]] && repo="$CONFIG_REPO" [[ -z "$labels" && -n "$CONFIG_LABELS" ]] && labels="$CONFIG_LABELS" @@ -1143,7 +1225,9 @@ USAGE [[ -z "$project_dir" ]] && project_dir="$(default_remote_project_dir)" [[ -z "$local_dir" ]] && local_dir="$(default_local_project_dir)" local_dir="$(expand_local_path "$local_dir")" + [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -d "$local_dir" ]] || { echo "ERROR: --local-dir not found: $local_dir" >&2; return 2; } + [[ -z "$identity" || -f "$identity" ]] || { echo "ERROR: identity file not found: $identity" >&2; return 2; } if [[ -z "$out_dir" ]]; then out_dir="$local_dir/out/remote/$(sanitize_for_path_segment "$host")" @@ -1153,13 +1237,13 @@ USAGE command -v ssh >/dev/null 2>&1 || { echo "ERROR: ssh command not found" >&2; return 1; } command -v rsync >/dev/null 2>&1 || { echo "ERROR: rsync command not found" >&2; return 1; } - ensure_ssh_key_auth "$host" - ensure_remote_project_dir "$host" "$project_dir" + ensure_ssh_key_auth "$host" "$identity" + ensure_remote_project_dir "$host" "$project_dir" "$identity" if [[ "$no_sync" -eq 1 ]]; then echo "SKIP: sync reason=no_sync_flag" else - sync_local_project_to_remote "$local_dir" "$host" "$project_dir" + sync_local_project_to_remote "$local_dir" "$host" "$project_dir" "$identity" fi if [[ "$skip_bootstrap" -eq 1 ]]; then @@ -1172,7 +1256,7 @@ USAGE [[ -n "$runner_name" ]] && register_args+=(--runner-name "$runner_name") [[ -n "$runner_group" ]] && register_args+=(--runner-group "$runner_group") [[ -n "$discord_webhook_url" ]] && register_args+=(--discord-webhook-url "$discord_webhook_url") - if ! run_remote_ci_self "$host" "$project_dir" "$remote_cli" "${register_args[@]}"; then + if ! run_remote_ci_self "$host" "$project_dir" "$remote_cli" "$identity" "${register_args[@]}"; then echo "WARN: bootstrap failed; continuing standalone verify" >&2 fi fi @@ -1189,13 +1273,13 @@ USAGE remote_verify_args+=(sh ops/ci/run_verify_full.sh) local verify_failed=0 - if ! run_remote_command_in_dir "$host" "$project_dir" "${remote_verify_args[@]}"; then + if ! run_remote_command_in_dir "$host" "$project_dir" "$identity" "${remote_verify_args[@]}"; then echo "ERROR: remote verify command failed" >&2 verify_failed=1 fi local fetch_failed=0 - if ! fetch_remote_verify_artifacts "$host" "$project_dir" "$out_dir"; then + if ! fetch_remote_verify_artifacts "$host" "$project_dir" "$out_dir" "$identity"; then fetch_failed=1 fi @@ -1217,6 +1301,7 @@ USAGE cmd_remote_register() { local host="" local project_dir="" + local identity="" local remote_cli="ci-self" local repo="" local labels="" @@ -1229,6 +1314,7 @@ cmd_remote_register() { while [[ $# -gt 0 ]]; do case "$1" in --host) host="${2:-}"; shift 2 ;; + -i|--identity) identity="${2:-}"; shift 2 ;; --project-dir) project_dir="${2:-}"; shift 2 ;; --remote-cli) remote_cli="${2:-}"; shift 2 ;; --repo) repo="${2:-}"; shift 2 ;; @@ -1240,7 +1326,7 @@ cmd_remote_register() { --skip-workflow) skip_workflow=1; shift ;; -h|--help) cat <<'USAGE' -Usage: ci-self remote-register --host [--project-dir path] [--repo owner/repo] [--remote-cli path] +Usage: ci-self remote-register --host [-i identity_file] [--project-dir path] [--repo owner/repo] [--remote-cli path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--force-workflow] [--skip-workflow] USAGE @@ -1254,6 +1340,7 @@ USAGE done [[ -z "$host" ]] && host="$CONFIG_REMOTE_HOST" + [[ -z "$identity" && -n "$CONFIG_REMOTE_IDENTITY" ]] && identity="$CONFIG_REMOTE_IDENTITY" [[ "$remote_cli" == "ci-self" && -n "$CONFIG_REMOTE_CLI" ]] && remote_cli="$CONFIG_REMOTE_CLI" [[ -z "$repo" && -n "$CONFIG_REPO" ]] && repo="$CONFIG_REPO" [[ -z "$labels" && -n "$CONFIG_LABELS" ]] && labels="$CONFIG_LABELS" @@ -1262,8 +1349,10 @@ USAGE [[ -z "$discord_webhook_url" && -n "$CONFIG_DISCORD_WEBHOOK_URL" ]] && discord_webhook_url="$CONFIG_DISCORD_WEBHOOK_URL" [[ "$force_workflow" -eq 0 ]] && force_workflow="$(config_bool_to_int "$CONFIG_FORCE_WORKFLOW")" [[ "$skip_workflow" -eq 0 ]] && skip_workflow="$(config_bool_to_int "$CONFIG_SKIP_WORKFLOW")" + [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -n "$host" ]] || { echo "ERROR: --host is required" >&2; return 2; } + [[ -z "$identity" || -f "$identity" ]] || { echo "ERROR: identity file not found: $identity" >&2; return 2; } if [[ -z "$project_dir" ]]; then project_dir="$(default_remote_project_dir)" fi @@ -1277,12 +1366,13 @@ USAGE [[ "$force_workflow" -eq 1 ]] && args+=(--force-workflow) [[ "$skip_workflow" -eq 1 ]] && args+=(--skip-workflow) - run_remote_ci_self "$host" "$project_dir" "$remote_cli" "${args[@]}" + run_remote_ci_self "$host" "$project_dir" "$remote_cli" "$identity" "${args[@]}" } cmd_remote_run_focus() { local host="" local project_dir="" + local identity="" local remote_cli="ci-self" local repo="" local ref="" @@ -1290,13 +1380,14 @@ cmd_remote_run_focus() { while [[ $# -gt 0 ]]; do case "$1" in --host) host="${2:-}"; shift 2 ;; + -i|--identity) identity="${2:-}"; shift 2 ;; --project-dir) project_dir="${2:-}"; shift 2 ;; --remote-cli) remote_cli="${2:-}"; shift 2 ;; --repo) repo="${2:-}"; shift 2 ;; --ref) ref="${2:-}"; shift 2 ;; -h|--help) cat <<'USAGE' -Usage: ci-self remote-run-focus --host [--project-dir path] [--repo owner/repo] [--ref branch] [--remote-cli path] +Usage: ci-self remote-run-focus --host [-i identity_file] [--project-dir path] [--repo owner/repo] [--ref branch] [--remote-cli path] USAGE return 0 ;; @@ -1308,23 +1399,27 @@ USAGE done [[ -z "$host" ]] && host="$CONFIG_REMOTE_HOST" + [[ -z "$identity" && -n "$CONFIG_REMOTE_IDENTITY" ]] && identity="$CONFIG_REMOTE_IDENTITY" [[ "$remote_cli" == "ci-self" && -n "$CONFIG_REMOTE_CLI" ]] && remote_cli="$CONFIG_REMOTE_CLI" [[ -z "$repo" && -n "$CONFIG_REPO" ]] && repo="$CONFIG_REPO" [[ -z "$ref" ]] && ref="$(resolve_ref "$ref")" + [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -n "$host" ]] || { echo "ERROR: --host is required" >&2; return 2; } + [[ -z "$identity" || -f "$identity" ]] || { echo "ERROR: identity file not found: $identity" >&2; return 2; } if [[ -z "$project_dir" ]]; then project_dir="$(default_remote_project_dir)" fi local args=(run-focus --ref "$ref") [[ -n "$repo" ]] && args+=(--repo "$repo") - run_remote_ci_self "$host" "$project_dir" "$remote_cli" "${args[@]}" + run_remote_ci_self "$host" "$project_dir" "$remote_cli" "$identity" "${args[@]}" } cmd_remote_up() { local host="" local project_dir="" + local identity="" local remote_cli="ci-self" local repo="" local ref="" @@ -1338,6 +1433,7 @@ cmd_remote_up() { while [[ $# -gt 0 ]]; do case "$1" in --host) host="${2:-}"; shift 2 ;; + -i|--identity) identity="${2:-}"; shift 2 ;; --project-dir) project_dir="${2:-}"; shift 2 ;; --remote-cli) remote_cli="${2:-}"; shift 2 ;; --repo) repo="${2:-}"; shift 2 ;; @@ -1350,7 +1446,7 @@ cmd_remote_up() { --skip-workflow) skip_workflow=1; shift ;; -h|--help) cat <<'USAGE' -Usage: ci-self remote-up --host [--project-dir path] [--repo owner/repo] [--ref branch] +Usage: ci-self remote-up --host [-i identity_file] [--project-dir path] [--repo owner/repo] [--ref branch] [--remote-cli path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--force-workflow] [--skip-workflow] @@ -1365,6 +1461,7 @@ USAGE done [[ -z "$host" ]] && host="$CONFIG_REMOTE_HOST" + [[ -z "$identity" && -n "$CONFIG_REMOTE_IDENTITY" ]] && identity="$CONFIG_REMOTE_IDENTITY" [[ "$remote_cli" == "ci-self" && -n "$CONFIG_REMOTE_CLI" ]] && remote_cli="$CONFIG_REMOTE_CLI" [[ -z "$repo" && -n "$CONFIG_REPO" ]] && repo="$CONFIG_REPO" [[ -z "$ref" ]] && ref="$(resolve_ref "$ref")" @@ -1374,13 +1471,16 @@ USAGE [[ -z "$discord_webhook_url" && -n "$CONFIG_DISCORD_WEBHOOK_URL" ]] && discord_webhook_url="$CONFIG_DISCORD_WEBHOOK_URL" [[ "$force_workflow" -eq 0 ]] && force_workflow="$(config_bool_to_int "$CONFIG_FORCE_WORKFLOW")" [[ "$skip_workflow" -eq 0 ]] && skip_workflow="$(config_bool_to_int "$CONFIG_SKIP_WORKFLOW")" + [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -n "$host" ]] || { echo "ERROR: --host is required" >&2; return 2; } + [[ -z "$identity" || -f "$identity" ]] || { echo "ERROR: identity file not found: $identity" >&2; return 2; } if [[ -z "$project_dir" ]]; then project_dir="$(default_remote_project_dir)" fi local register_args=(--host "$host" --project-dir "$project_dir" --remote-cli "$remote_cli") + [[ -n "$identity" ]] && register_args+=(-i "$identity") [[ -n "$repo" ]] && register_args+=(--repo "$repo") [[ -n "$labels" ]] && register_args+=(--labels "$labels") [[ -n "$runner_name" ]] && register_args+=(--runner-name "$runner_name") @@ -1391,6 +1491,7 @@ USAGE cmd_remote_register "${register_args[@]}" local run_focus_args=(--host "$host" --project-dir "$project_dir" --remote-cli "$remote_cli" --ref "$ref") + [[ -n "$identity" ]] && run_focus_args+=(-i "$identity") [[ -n "$repo" ]] && run_focus_args+=(--repo "$repo") cmd_remote_run_focus "${run_focus_args[@]}" } diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index f496ea4..c9c4a0c 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -121,6 +121,9 @@ func TestConfigInitWritesFile(t *testing.T) { if !strings.Contains(content, "CI_SELF_PROJECT_DIR="+tmp) { t.Fatalf("missing project dir in config\ncontent:\n%s", content) } + if !strings.Contains(content, "CI_SELF_REMOTE_IDENTITY=") { + t.Fatalf("missing remote identity placeholder in config\ncontent:\n%s", content) + } } func TestRemoteUpUsesConfigHost(t *testing.T) { @@ -295,12 +298,16 @@ exit 0 func TestRemoteCIRunsSyncVerifyAndFetch(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "repo") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { t.Fatalf("mkdir local repo failed: %v", err) } if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { t.Fatalf("write local verify script failed: %v", err) } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } logPath := filepath.Join(tmp, "tool.log") sshPath := filepath.Join(tmp, "ssh") @@ -347,6 +354,8 @@ exit 0 "remote-ci", "--host", "mini-user@192.168.1.9", + "-i", + identityPath, "--local-dir", localDir, "--project-dir", @@ -360,8 +369,68 @@ exit 0 t.Fatalf("expected success status output\noutput:\n%s", out) } + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + logText := string(logBody) + if !strings.Contains(logText, "ssh -i "+identityPath) { + t.Fatalf("expected ssh to receive identity file\nlog:\n%s", logText) + } + if !strings.Contains(logText, "rsync -az --delete -e ssh -i "+identityPath) { + t.Fatalf("expected rsync sync to receive identity file\nlog:\n%s", logText) + } + if !strings.Contains(logText, "rsync -a -e ssh -i "+identityPath) { + t.Fatalf("expected rsync fetch to receive identity file\nlog:\n%s", logText) + } + statusPath := filepath.Join(localDir, "out", "remote", "mini-user-192.168.1.9", "verify-full.status") if _, statErr := os.Stat(statusPath); statErr != nil { t.Fatalf("expected fetched status file at %s: %v", statusPath, statErr) } } + +func TestRemoteUpAcceptsIdentity(t *testing.T) { + tmp := t.TempDir() + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "ssh.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf("#!/usr/bin/env bash\nset -euo pipefail\necho \"$*\" >> %q\nexit 42\n", logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-up", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--project-dir", + "~/dev/zt-gateway", + "--skip-workflow", + "--ref", + "main", + ) + if err == nil { + t.Fatalf("expected failure from fake ssh, got success\noutput:\n%s", out) + } + if strings.Contains(out, "unknown option for remote-up: -i") { + t.Fatalf("identity option was not parsed\noutput:\n%s", out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read ssh log: %v", readErr) + } + if !strings.Contains(string(logBody), "-i "+identityPath) { + t.Fatalf("expected ssh call to include identity file\nlog:\n%s", string(logBody)) + } +} From 5d88fa0fae77eee9f1c60fa217e0a5c3830dc29f Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 18:39:35 +0900 Subject: [PATCH 02/14] Add SSH identity support to remote-ci --- README.md | 3 ++ docs/ci/QUICKSTART.md | 2 + ops/ci/ci_self.sh | 28 +++++++++-- ops/ci/ci_self_test.go | 108 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1d07c01..0246c28 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ ci-self up マシンB から 1 コマンドで「鍵認証確認 -> 同期 -> マシンA実行 -> 結果回収」まで行います。 +`remote-ci` の同期元は、現在の作業リポジトリです。対象 repo のルートで実行するか、`--local-dir ` で明示してください。 + ```bash ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` @@ -104,6 +106,7 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run - `--host` は `ssh` の接続先文字列（`user@host` / IP / `~/.ssh/config` の Host 別名） - `-i` / `--identity` で SSH 鍵ファイルを指定できる。毎回省略したい場合は `.ci-self.env` の `CI_SELF_REMOTE_IDENTITY` を使う +- `remote-ci` の同期元はデフォルトで「今いる repo」。別 repo を送りたい場合は `--local-dir` を使う - `--project-dir` はマシンA 側の配置先パス。`~` を使う場合は `--project-dir '~/'` のようにクオート - `--local-dir` を使うと、マシンB 側の同期元を明示できる - `--repo` を省略すると bootstrap は skip されるが、同期済みプロジェクト上での standalone verify は実行できる diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index 34ca921..f75f8ea 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -52,6 +52,8 @@ ssh -i ~/.ssh/id_ed25519_for_ci_runner -o BatchMode=yes -o PasswordAuthenticatio 通ったら `remote-ci` を実行します。 +`remote-ci` の同期元は、現在の作業リポジトリです。対象 repo のルートで実行するか、`--local-dir ` で明示してください。 + ```bash ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index def5ca7..1105aea 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -41,7 +41,7 @@ trim() { expand_local_path() { local p="$1" if [[ "$p" == "~/"* ]]; then - printf '%s\n' "${HOME}/${p#~/}" + printf '%s\n' "${HOME}/${p#"~/"}" else printf '%s\n' "$p" fi @@ -960,10 +960,28 @@ sanitize_for_path_segment() { printf '%s\n' "$out" } +ensure_default_local_dir_matches_repo() { + local local_dir="$1" + local repo="$2" + local local_dir_was_explicit="$3" + [[ -n "$repo" ]] || return 0 + [[ "$local_dir_was_explicit" -eq 1 ]] && return 0 + + local expected_name="${repo##*/}" + local actual_name + actual_name="$(basename "$local_dir")" + if [[ "$actual_name" != "$expected_name" ]]; then + echo "ERROR: default local-dir appears to be the wrong project: $local_dir" >&2 + echo "HINT: repo=$repo expects a local dir like .../$expected_name" >&2 + echo "HINT: run ci-self from the target repo root, or pass --local-dir " >&2 + return 1 + fi +} + remote_path_for_shell() { local path="$1" if [[ "$path" == "~/"* ]]; then - printf '\$HOME/%s\n' "${path#~/}" + printf '\$HOME/%s\n' "${path#"~/"}" else printf '%q\n' "$path" fi @@ -1150,7 +1168,7 @@ run_remote_ci_self() { remote_args_q="$(quote_words "${remote_args[@]}")" if [[ "$project_dir" == "~/"* ]]; then - remote_cd_q="\$HOME/${project_dir#~/}" + remote_cd_q="\$HOME/${project_dir#"~/"}" else printf -v remote_cd_q '%q' "$project_dir" fi @@ -1166,6 +1184,7 @@ cmd_remote_ci() { local local_dir="" local out_dir="" local identity="" + local local_dir_was_explicit=0 local remote_cli="ci-self" local repo="" local labels="" @@ -1182,7 +1201,7 @@ cmd_remote_ci() { --host) host="${2:-}"; shift 2 ;; -i|--identity) identity="${2:-}"; shift 2 ;; --project-dir) project_dir="${2:-}"; shift 2 ;; - --local-dir) local_dir="${2:-}"; shift 2 ;; + --local-dir) local_dir="${2:-}"; local_dir_was_explicit=1; shift 2 ;; --out-dir) out_dir="${2:-}"; shift 2 ;; --remote-cli) remote_cli="${2:-}"; shift 2 ;; --repo) repo="${2:-}"; shift 2 ;; @@ -1228,6 +1247,7 @@ USAGE [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -d "$local_dir" ]] || { echo "ERROR: --local-dir not found: $local_dir" >&2; return 2; } [[ -z "$identity" || -f "$identity" ]] || { echo "ERROR: identity file not found: $identity" >&2; return 2; } + ensure_default_local_dir_matches_repo "$local_dir" "$repo" "$local_dir_was_explicit" if [[ -z "$out_dir" ]]; then out_dir="$local_dir/out/remote/$(sanitize_for_path_segment "$host")" diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index c9c4a0c..ba633ef 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -295,6 +295,30 @@ exit 0 } } +func TestRemoteCIRejectsWrongDefaultLocalDir(t *testing.T) { + tmp := t.TempDir() + out, err := runCiSelfInDirEnv( + t, + tmp, + nil, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "--repo", + "mt4110/veil-rs", + "--skip-bootstrap", + ) + if err == nil { + t.Fatalf("expected remote-ci to reject mismatched default local dir\noutput:\n%s", out) + } + if !strings.Contains(out, "ERROR: default local-dir appears to be the wrong project") { + t.Fatalf("expected mismatched-local-dir error\noutput:\n%s", out) + } + if !strings.Contains(out, "pass --local-dir ") { + t.Fatalf("expected local-dir hint\noutput:\n%s", out) + } +} + func TestRemoteCIRunsSyncVerifyAndFetch(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "repo") @@ -390,6 +414,90 @@ exit 0 } } +func TestRemoteCIWithTildeProjectDirUsesRemoteHome(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "veil-rs") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write local verify script failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/_workspace/veil-rs", + "--skip-bootstrap", + ) + if err != nil { + t.Fatalf("remote-ci with tilde project dir failed: %v\noutput:\n%s", err, out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + logText := string(logBody) + if strings.Contains(logText, "$HOME/~/_workspace/veil-rs") { + t.Fatalf("tilde path was expanded incorrectly\nlog:\n%s", logText) + } + if !strings.Contains(logText, "_workspace/veil-rs") { + t.Fatalf("expected remote command to target remote project dir\nlog:\n%s", logText) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From ac16798ac29e935539f5905810c900d20ac980db Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:12:31 +0900 Subject: [PATCH 03/14] Exclude generated dirs from remote sync --- README.md | 8 ++++ docs/ci/QUICKSTART.md | 3 ++ ops/ci/ci_self.sh | 37 +++++++++++++-- ops/ci/ci_self_test.go | 105 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 149 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0246c28..b39190b 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,12 @@ ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --pr 4. マシンA で `ops/ci/run_verify_full.sh` を実行 5. `verify-full.status` と `out/logs` をマシンB の `out/remote//` に回収 +同期時の既定: + +- `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` は同期しない +- `rsync --info=progress2` で進捗を表示する +- build/test が Git メタデータを直接参照する repo だけは `--sync-git-dir` で `.git/` 同期を有効化する + ### 初回だけ必要な準備（マシンB -> マシンA） 1. マシンB で SSH 鍵を作る（未作成なら） @@ -109,6 +115,8 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run - `remote-ci` の同期元はデフォルトで「今いる repo」。別 repo を送りたい場合は `--local-dir` を使う - `--project-dir` はマシンA 側の配置先パス。`~` を使う場合は `--project-dir '~/'` のようにクオート - `--local-dir` を使うと、マシンB 側の同期元を明示できる +- `--sync-git-dir` を付けると `.git/` も同期する。通常は不要 +- `*.md`, `*.log`, `*.txt`, `build/`, `bin/`, `10MB超ファイル` は既定除外していない。tracked asset / fixture / source の可能性があるため - `--repo` を省略すると bootstrap は skip されるが、同期済みプロジェクト上での standalone verify は実行できる - runner 初期化/復旧専用の旧導線は `ci-self remote-up` diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index f75f8ea..8c444a0 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -65,6 +65,9 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run 3. マシンA 側 verify 実行 4. `out/remote//` へ結果回収 +既定では `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` を同期せず、`rsync --info=progress2` で進捗を表示します。 +repo 側の build/test が Git メタデータを直接読む場合だけ `--sync-git-dir` を付けてください。 + `remote-ci` は LAN 専用ではなく、外出先でも SSH 到達性があれば使えます。 逆に SSH 経路が無い場合、`remote-ci` 自体は疎通を作れません。 diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index 1105aea..126ef28 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1083,7 +1083,8 @@ sync_local_project_to_remote() { local host="$2" local project_dir="$3" local identity="${4:-}" - local rsync_cmd=(rsync -az --delete) + local sync_git_dir="${5:-0}" + local rsync_cmd=(rsync -az --delete --human-readable --info=progress2) local ssh_rsh="" if [[ -n "$identity" ]]; then ssh_rsh="$(quote_words ssh -i "$identity")" @@ -1094,11 +1095,39 @@ sync_local_project_to_remote() { --exclude ".local/" --exclude "out/" --exclude "cache/" + --exclude ".cache/" + --exclude "target/" + --exclude "dist/" + --exclude "node_modules/" + --exclude ".next/" + --exclude ".nuxt/" + --exclude ".svelte-kit/" + --exclude ".turbo/" + --exclude ".parcel-cache/" + --exclude ".venv/" + --exclude "venv/" + --exclude "__pycache__/" + --exclude ".pytest_cache/" + --exclude ".mypy_cache/" + --exclude ".ruff_cache/" + --exclude ".tox/" + --exclude ".nox/" + --exclude ".eggs/" + --exclude "*.egg-info/" + --exclude "coverage/" + --exclude "htmlcov/" + --exclude ".gradle/" --exclude ".DS_Store" + ) + if [[ "$sync_git_dir" -ne 1 ]]; then + rsync_cmd+=(--exclude ".git/") + fi + rsync_cmd+=( "$local_dir/" "$host:$project_dir/" ) "${rsync_cmd[@]}" + echo "OK: rsync_complete host=$host dst=$project_dir" } fetch_remote_verify_artifacts() { @@ -1185,6 +1214,7 @@ cmd_remote_ci() { local out_dir="" local identity="" local local_dir_was_explicit=0 + local sync_git_dir=0 local remote_cli="ci-self" local repo="" local labels="" @@ -1211,6 +1241,7 @@ cmd_remote_ci() { --discord-webhook-url) discord_webhook_url="${2:-}"; shift 2 ;; --verify-dry-run) verify_dry_run="$(config_bool_to_int "${2:-}")"; shift 2 ;; --verify-gha-sync) verify_gha_sync="$(config_bool_to_int "${2:-}")"; shift 2 ;; + --sync-git-dir) sync_git_dir=1; shift ;; --skip-bootstrap) skip_bootstrap=1; shift ;; --no-sync) no_sync=1; shift ;; -h|--help) @@ -1220,7 +1251,7 @@ Usage: ci-self remote-ci --host [-i identity_file] [--project-dir pat [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--verify-dry-run 0|1] [--verify-gha-sync 0|1] - [--skip-bootstrap] [--no-sync] + [--sync-git-dir] [--skip-bootstrap] [--no-sync] USAGE return 0 ;; @@ -1263,7 +1294,7 @@ USAGE if [[ "$no_sync" -eq 1 ]]; then echo "SKIP: sync reason=no_sync_flag" else - sync_local_project_to_remote "$local_dir" "$host" "$project_dir" "$identity" + sync_local_project_to_remote "$local_dir" "$host" "$project_dir" "$identity" "$sync_git_dir" fi if [[ "$skip_bootstrap" -eq 1 ]]; then diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index ba633ef..88d5de8 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -401,9 +401,30 @@ exit 0 if !strings.Contains(logText, "ssh -i "+identityPath) { t.Fatalf("expected ssh to receive identity file\nlog:\n%s", logText) } - if !strings.Contains(logText, "rsync -az --delete -e ssh -i "+identityPath) { + if !strings.Contains(logText, "rsync -az --delete --human-readable --info=progress2 -e ssh -i "+identityPath) { t.Fatalf("expected rsync sync to receive identity file\nlog:\n%s", logText) } + if !strings.Contains(logText, "--exclude target/") { + t.Fatalf("expected rsync sync to exclude target dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude dist/") { + t.Fatalf("expected rsync sync to exclude dist dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude node_modules/") { + t.Fatalf("expected rsync sync to exclude node_modules dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude .venv/") { + t.Fatalf("expected rsync sync to exclude python virtualenv dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude coverage/") { + t.Fatalf("expected rsync sync to exclude coverage dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude .next/") { + t.Fatalf("expected rsync sync to exclude next build dir\nlog:\n%s", logText) + } + if !strings.Contains(logText, "--exclude .git/") { + t.Fatalf("expected rsync sync to exclude git dir by default\nlog:\n%s", logText) + } if !strings.Contains(logText, "rsync -a -e ssh -i "+identityPath) { t.Fatalf("expected rsync fetch to receive identity file\nlog:\n%s", logText) } @@ -498,6 +519,88 @@ exit 0 } } +func TestRemoteCISyncGitDirOptIn(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "repo") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write local verify script failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/dev/zt-gateway", + "--skip-bootstrap", + "--sync-git-dir", + ) + if err != nil { + t.Fatalf("remote-ci failed: %v\noutput:\n%s", err, out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + logText := string(logBody) + if strings.Contains(logText, "--exclude .git/") { + t.Fatalf("expected sync-git-dir to keep .git in sync set\nlog:\n%s", logText) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From 72aff6c933c87e1f1ec0f8fafd1ad96118e18402 Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:33:09 +0900 Subject: [PATCH 04/14] Document rsync requirements and add fallback --- README.md | 39 ++++++++++++++++++ docs/ci/QUICKSTART.md | 1 + ops/ci/ci_self.sh | 8 +++- ops/ci/ci_self_test.go | 91 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b39190b..e2bbcdd 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ ci-self up `remote-ci` の同期元は、現在の作業リポジトリです。対象 repo のルートで実行するか、`--local-dir ` で明示してください。 +`remote-ci` はローカル `rsync` コマンドをそのまま使います。macOS 既定の `openrsync` だと古く、進捗表示や互換性で不利です。 + ```bash ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` @@ -58,6 +60,7 @@ ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --pr - `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` は同期しない - `rsync --info=progress2` で進捗を表示する +- ローカル `rsync` が `--info=progress2` 非対応なら `-h --progress` へ自動フォールバックする - build/test が Git メタデータを直接参照する repo だけは `--sync-git-dir` で `.git/` 同期を有効化する ### 初回だけ必要な準備（マシンB -> マシンA） @@ -86,6 +89,42 @@ ssh -i ~/.ssh/id_ed25519_for_ci_runner -o BatchMode=yes -o PasswordAuthenticatio ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` +### rsync バージョン注意 + +macOS 既定の `rsync` が古い場合があります。 + +```bash +rsync --version +``` + +古い例: + +```text +openrsync: protocol version 29 +rsync version 2.6.9 compatible +``` + +推奨: + +```bash +brew install rsync +``` + +`alias rsync=...` だけでは `ci-self` の bash スクリプトには効かないことがあります。`PATH` の先頭に Homebrew 版を入れてください。 + +Apple Silicon の例: + +```bash +echo 'export PATH="/opt/homebrew/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc +``` + +確認例: + +```text +rsync version 3.4.1 protocol version 32 +``` + 公開鍵未登録時は、`remote-ci` 自体が `authorized_keys` 登録のヒントを出して停止します。 ### 外出先からでも使える？ diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index 8c444a0..6b568d7 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -66,6 +66,7 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run 4. `out/remote//` へ結果回収 既定では `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` を同期せず、`rsync --info=progress2` で進捗を表示します。 +ローカル `rsync` が古い場合は `-h --progress` に自動フォールバックしますが、Homebrew の新しい `rsync` を推奨します。 repo 側の build/test が Git メタデータを直接読む場合だけ `--sync-git-dir` を付けてください。 `remote-ci` は LAN 専用ではなく、外出先でも SSH 到達性があれば使えます。 diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index 126ef28..ab50cdc 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1084,12 +1084,18 @@ sync_local_project_to_remote() { local project_dir="$3" local identity="${4:-}" local sync_git_dir="${5:-0}" - local rsync_cmd=(rsync -az --delete --human-readable --info=progress2) + local rsync_cmd=(rsync -az --delete) local ssh_rsh="" if [[ -n "$identity" ]]; then ssh_rsh="$(quote_words ssh -i "$identity")" rsync_cmd+=(-e "$ssh_rsh") fi + if rsync --info=progress2 --version >/dev/null 2>&1; then + rsync_cmd+=(--human-readable --info=progress2) + else + rsync_cmd+=(-h --progress) + echo "WARN: local rsync does not support --info=progress2; falling back to -h --progress" >&2 + fi echo "OK: rsync host=$host src=$local_dir dst=$project_dir" rsync_cmd+=( --exclude ".local/" diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 88d5de8..50b56c9 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -401,7 +401,8 @@ exit 0 if !strings.Contains(logText, "ssh -i "+identityPath) { t.Fatalf("expected ssh to receive identity file\nlog:\n%s", logText) } - if !strings.Contains(logText, "rsync -az --delete --human-readable --info=progress2 -e ssh -i "+identityPath) { + if !strings.Contains(logText, "rsync -az --delete -e ssh -i "+identityPath+" --human-readable --info=progress2") && + !strings.Contains(logText, "rsync -az --delete --human-readable --info=progress2 -e ssh -i "+identityPath) { t.Fatalf("expected rsync sync to receive identity file\nlog:\n%s", logText) } if !strings.Contains(logText, "--exclude target/") { @@ -601,6 +602,94 @@ exit 0 } } +func TestRemoteCIFallsBackForLegacyRsync(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "repo") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write local verify script failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +if [[ "$*" == *"--info=progress2 --version"* ]]; then + exit 1 +fi +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/dev/zt-gateway", + "--skip-bootstrap", + ) + if err != nil { + t.Fatalf("remote-ci failed: %v\noutput:\n%s", err, out) + } + if !strings.Contains(out, "falling back to -h --progress") { + t.Fatalf("expected fallback warning in output\noutput:\n%s", out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + logText := string(logBody) + if !strings.Contains(logText, "rsync -az --delete -e ssh -i "+identityPath+" -h --progress") && + !strings.Contains(logText, "rsync -az --delete -h --progress -e ssh -i "+identityPath) { + t.Fatalf("expected legacy rsync fallback flags\nlog:\n%s", logText) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From 448f88c41a379e37cbbdd22bbba7eaaee9ebb1aa Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:39:44 +0900 Subject: [PATCH 05/14] Run remote verify without repo wrapper scripts --- README.md | 5 +++-- ops/ci/ci_self.sh | 36 ++++++++++++++++++++++++++++++------ ops/ci/ci_self_test.go | 26 ++++++++++---------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e2bbcdd..f93a6e3 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,8 @@ ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --pr 1. SSH 公開鍵認証（password禁止）を検証 2. マシンB のローカル作業ツリーをマシンA の `--project-dir` へ `rsync` 同期 -3. （`--repo` 指定時）マシンA 上で `ci-self register` をベストエフォート実行 -4. マシンA で `ops/ci/run_verify_full.sh` を実行 +3. （`--repo` 指定時かつ remote `ci-self` 利用可能時）マシンA 上で `ci-self register` をベストエフォート実行 +4. マシンA で、同梱の verify wrapper を SSH 経由で実行 5. `verify-full.status` と `out/logs` をマシンB の `out/remote//` に回収 同期時の既定: @@ -156,6 +156,7 @@ rsync version 3.4.1 protocol version 32 - `--local-dir` を使うと、マシンB 側の同期元を明示できる - `--sync-git-dir` を付けると `.git/` も同期する。通常は不要 - `*.md`, `*.log`, `*.txt`, `build/`, `bin/`, `10MB超ファイル` は既定除外していない。tracked asset / fixture / source の可能性があるため +- remote `ci-self` 未導入でも verify 自体は動く。影響するのは bootstrap だけ - `--repo` を省略すると bootstrap は skip されるが、同期済みプロジェクト上での standalone verify は実行できる - runner 初期化/復旧専用の旧導線は `ci-self remote-up` diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index ab50cdc..ed1fc61 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1007,6 +1007,35 @@ run_remote_command_in_dir() { "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } +run_remote_verify_wrapper() { + local host="$1" + local project_dir="$2" + local identity="${3:-}" + local verify_dry_run="${4:-1}" + local verify_gha_sync="${5:-1}" + local github_sha="${6:-}" + local github_ref="${7:-}" + local remote_cd_q + local script_q + local remote_script + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + + remote_cd_q="$(remote_path_for_shell "$project_dir")" + printf -v remote_script 'set -euo pipefail; cd %s; export REPO_DIR="$PWD" OUT_DIR="$PWD/out" VERIFY_DRY_RUN=%q VERIFY_GHA_SYNC=%q GITHUB_ACTIONS=%q' \ + "$remote_cd_q" "$verify_dry_run" "$verify_gha_sync" "true" + if [[ -n "$github_sha" ]]; then + printf -v remote_script '%s GITHUB_SHA=%q' "$remote_script" "$github_sha" + fi + if [[ -n "$github_ref" ]]; then + printf -v remote_script '%s GITHUB_REF_NAME=%q' "$remote_script" "$github_ref" + fi + printf -v remote_script '%s; sh -s' "$remote_script" + script_q="$(quote_words "$remote_script")" + echo "OK: ssh host=$host dir=$project_dir cmd=remote_verify_wrapper" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" < "$ROOT_DIR/ops/ci/run_verify_full.sh" +} + first_existing_public_key() { local key="" for key in \ @@ -1324,13 +1353,8 @@ USAGE ref="$(git -C "$local_dir" rev-parse --abbrev-ref HEAD 2>/dev/null || true)" [[ "$ref" == "HEAD" ]] && ref="" - local remote_verify_args=(env "VERIFY_DRY_RUN=$verify_dry_run" "VERIFY_GHA_SYNC=$verify_gha_sync" "GITHUB_ACTIONS=true") - [[ -n "$sha" ]] && remote_verify_args+=("GITHUB_SHA=$sha") - [[ -n "$ref" ]] && remote_verify_args+=("GITHUB_REF_NAME=$ref") - remote_verify_args+=(sh ops/ci/run_verify_full.sh) - local verify_failed=0 - if ! run_remote_command_in_dir "$host" "$project_dir" "$identity" "${remote_verify_args[@]}"; then + if ! run_remote_verify_wrapper "$host" "$project_dir" "$identity" "$verify_dry_run" "$verify_gha_sync" "$sha" "$ref"; then echo "ERROR: remote verify command failed" >&2 verify_failed=1 fi diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 50b56c9..c644552 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -323,12 +323,9 @@ func TestRemoteCIRunsSyncVerifyAndFetch(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "repo") identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") - if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + if err := os.MkdirAll(localDir, 0o755); err != nil { t.Fatalf("mkdir local repo failed: %v", err) } - if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { - t.Fatalf("write local verify script failed: %v", err) - } if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { t.Fatalf("write identity file failed: %v", err) } @@ -426,6 +423,12 @@ exit 0 if !strings.Contains(logText, "--exclude .git/") { t.Fatalf("expected rsync sync to exclude git dir by default\nlog:\n%s", logText) } + if !strings.Contains(out, "cmd=remote_verify_wrapper") { + t.Fatalf("expected remote verify wrapper invocation\noutput:\n%s", out) + } + if !strings.Contains(logText, "sh\\ -s") { + t.Fatalf("expected remote verify wrapper to stream script over ssh\nlog:\n%s", logText) + } if !strings.Contains(logText, "rsync -a -e ssh -i "+identityPath) { t.Fatalf("expected rsync fetch to receive identity file\nlog:\n%s", logText) } @@ -440,12 +443,9 @@ func TestRemoteCIWithTildeProjectDirUsesRemoteHome(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "veil-rs") identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") - if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + if err := os.MkdirAll(localDir, 0o755); err != nil { t.Fatalf("mkdir local repo failed: %v", err) } - if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { - t.Fatalf("write local verify script failed: %v", err) - } if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { t.Fatalf("write identity file failed: %v", err) } @@ -524,12 +524,9 @@ func TestRemoteCISyncGitDirOptIn(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "repo") identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") - if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + if err := os.MkdirAll(localDir, 0o755); err != nil { t.Fatalf("mkdir local repo failed: %v", err) } - if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { - t.Fatalf("write local verify script failed: %v", err) - } if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { t.Fatalf("write identity file failed: %v", err) } @@ -606,12 +603,9 @@ func TestRemoteCIFallsBackForLegacyRsync(t *testing.T) { tmp := t.TempDir() localDir := filepath.Join(tmp, "repo") identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") - if err := os.MkdirAll(filepath.Join(localDir, "ops", "ci"), 0o755); err != nil { + if err := os.MkdirAll(localDir, 0o755); err != nil { t.Fatalf("mkdir local repo failed: %v", err) } - if err := os.WriteFile(filepath.Join(localDir, "ops", "ci", "run_verify_full.sh"), []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755); err != nil { - t.Fatalf("write local verify script failed: %v", err) - } if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { t.Fatalf("write identity file failed: %v", err) } From 993ea760d78be3aac47055f19c1897a32b282f30 Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 20:16:28 +0900 Subject: [PATCH 06/14] Add SSH fallback for remote artifact fetch --- ops/ci/ci_self.sh | 53 +++++++++++++++++++-- ops/ci/ci_self_test.go | 101 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 4 deletions(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index ed1fc61..ee3cf88 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1036,6 +1036,23 @@ run_remote_verify_wrapper() { "${ssh_cmd[@]}" "$host" "bash -lc $script_q" < "$ROOT_DIR/ops/ci/run_verify_full.sh" } +probe_remote_verify_artifacts() { + local host="$1" + local project_dir="$2" + local identity="${3:-}" + local remote_cd_q + local script_q + local remote_script + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + + remote_cd_q="$(remote_path_for_shell "$project_dir")" + printf -v remote_script 'set -euo pipefail; cd %s; if [[ -f out/verify-full.status ]]; then echo "OK: remote_artifacts status_file=$PWD/out/verify-full.status"; else echo "WARN: remote_artifacts status_file_missing=$PWD/out/verify-full.status"; fi; if [[ -d out/logs ]]; then echo "OK: remote_artifacts logs_dir=$PWD/out/logs"; else echo "WARN: remote_artifacts logs_dir_missing=$PWD/out/logs"; fi' \ + "$remote_cd_q" + script_q="$(quote_words "$remote_script")" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" +} + first_existing_public_key() { local key="" for key in \ @@ -1183,15 +1200,42 @@ fetch_remote_verify_artifacts() { if "${rsync_base[@]}" "$host:$project_dir/out/verify-full.status" "$out_dir/"; then echo "OK: fetch status_file=$out_dir/verify-full.status" else - echo "ERROR: fetch status_file failed host=$host path=$project_dir/out/verify-full.status" >&2 - failed=1 + local remote_cd_q + local remote_script + local script_q + local ssh_cmd=(ssh) + local tmp_status="$out_dir/verify-full.status.tmp" + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + remote_cd_q="$(remote_path_for_shell "$project_dir")" + printf -v remote_script 'set -euo pipefail; cd %s; test -f out/verify-full.status; cat out/verify-full.status' "$remote_cd_q" + script_q="$(quote_words "$remote_script")" + if "${ssh_cmd[@]}" "$host" "bash -lc $script_q" > "$tmp_status" && grep -q 'status=' "$tmp_status"; then + mv "$tmp_status" "$out_dir/verify-full.status" + echo "OK: fetch status_file=$out_dir/verify-full.status source=ssh_fallback" + else + rm -f "$tmp_status" + echo "ERROR: fetch status_file failed host=$host path=$project_dir/out/verify-full.status" >&2 + failed=1 + fi fi if "${rsync_base[@]}" "$host:$project_dir/out/logs/" "$out_dir/logs/"; then echo "OK: fetch logs_dir=$out_dir/logs" else - echo "ERROR: fetch logs failed host=$host path=$project_dir/out/logs/" >&2 - failed=1 + local remote_cd_q + local remote_script + local script_q + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + remote_cd_q="$(remote_path_for_shell "$project_dir")" + printf -v remote_script 'set -euo pipefail; cd %s; test -d out/logs; tar -cf - -C out logs' "$remote_cd_q" + script_q="$(quote_words "$remote_script")" + if "${ssh_cmd[@]}" "$host" "bash -lc $script_q" | tar -xf - -C "$out_dir"; then + echo "OK: fetch logs_dir=$out_dir/logs source=ssh_fallback" + else + echo "ERROR: fetch logs failed host=$host path=$project_dir/out/logs/" >&2 + failed=1 + fi fi return "$failed" @@ -1358,6 +1402,7 @@ USAGE echo "ERROR: remote verify command failed" >&2 verify_failed=1 fi + probe_remote_verify_artifacts "$host" "$project_dir" "$identity" || true local fetch_failed=0 if ! fetch_remote_verify_artifacts "$host" "$project_dir" "$out_dir" "$identity"; then diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index c644552..25b71d9 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -684,6 +684,107 @@ exit 0 } } +func TestRemoteCIFetchFallsBackToSSH(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "repo") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + fixtureDir := filepath.Join(tmp, "remote-fixture") + fixtureLogsDir := filepath.Join(fixtureDir, "logs") + if err := os.MkdirAll(localDir, 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + if err := os.MkdirAll(fixtureLogsDir, 0o755); err != nil { + t.Fatalf("mkdir fixture logs failed: %v", err) + } + if err := os.WriteFile(filepath.Join(fixtureDir, "verify-full.status"), []byte("status=OK\n"), 0o644); err != nil { + t.Fatalf("write fixture status failed: %v", err) + } + if err := os.WriteFile(filepath.Join(fixtureLogsDir, "verify.log"), []byte("ok\n"), 0o644); err != nil { + t.Fatalf("write fixture log failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +if [[ "$*" == *"verify-full.status"* && "$*" == *"cat"* ]]; then + cat %q + exit 0 +fi +if [[ "$*" == *"tar"* && "$*" == *"logs"* ]]; then + tar -cf - -C %q logs + exit 0 +fi +if [[ "$*" == *"mkdir -p"* || "$*" == *"sh"* || "$*" == *"status_file="* || "$*" == *"logs_dir="* ]]; then + exit 0 +fi +exit 1 +`, logPath, filepath.Join(fixtureDir, "verify-full.status"), fixtureDir) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + exit 23 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + exit 23 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/dev/zt-gateway", + "--skip-bootstrap", + ) + if err != nil { + t.Fatalf("remote-ci failed: %v\noutput:\n%s", err, out) + } + if !strings.Contains(out, "source=ssh_fallback") { + t.Fatalf("expected ssh fallback fetch output\noutput:\n%s", out) + } + + statusPath := filepath.Join(localDir, "out", "remote", "mini-user-192.168.1.9", "verify-full.status") + if content, readErr := os.ReadFile(statusPath); readErr != nil { + t.Fatalf("expected fetched status file via ssh fallback: %v", readErr) + } else if !strings.Contains(string(content), "status=OK") { + t.Fatalf("unexpected status file content: %s", string(content)) + } + + logFilePath := filepath.Join(localDir, "out", "remote", "mini-user-192.168.1.9", "logs", "verify.log") + if content, readErr := os.ReadFile(logFilePath); readErr != nil { + t.Fatalf("expected fetched log via ssh fallback: %v", readErr) + } else if strings.TrimSpace(string(content)) != "ok" { + t.Fatalf("unexpected fetched log content: %s", string(content)) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From 36f9b54496c23497f3cda4cfe383a6e41b5ee92b Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 20:57:49 +0900 Subject: [PATCH 07/14] Fix remote tilde path handling --- ops/ci/ci_self.sh | 24 +++++++++++++++--------- ops/ci/ci_self_test.go | 5 ++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index ee3cf88..167c551 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -930,6 +930,12 @@ quote_words() { printf '%s\n' "$out" } +quote_bash_lc_script() { + local script="$1" + script="${script//\'/\'\"\'\"\'}" + printf "'%s'\n" "$script" +} + default_remote_project_dir() { if [[ -n "$CONFIG_REMOTE_PROJECT_DIR" ]]; then printf '%s\n' "$CONFIG_REMOTE_PROJECT_DIR" @@ -981,7 +987,7 @@ ensure_default_local_dir_matches_repo() { remote_path_for_shell() { local path="$1" if [[ "$path" == "~/"* ]]; then - printf '\$HOME/%s\n' "${path#"~/"}" + printf '$HOME/%s\n' "${path#"~/"}" else printf '%q\n' "$path" fi @@ -1002,7 +1008,7 @@ run_remote_command_in_dir() { remote_cmd_q="$(quote_words "$@")" remote_cd_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; cd %s; %s' "$remote_cd_q" "$remote_cmd_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=$*" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } @@ -1031,7 +1037,7 @@ run_remote_verify_wrapper() { printf -v remote_script '%s GITHUB_REF_NAME=%q' "$remote_script" "$github_ref" fi printf -v remote_script '%s; sh -s' "$remote_script" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=remote_verify_wrapper" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" < "$ROOT_DIR/ops/ci/run_verify_full.sh" } @@ -1049,7 +1055,7 @@ probe_remote_verify_artifacts() { remote_cd_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; cd %s; if [[ -f out/verify-full.status ]]; then echo "OK: remote_artifacts status_file=$PWD/out/verify-full.status"; else echo "WARN: remote_artifacts status_file_missing=$PWD/out/verify-full.status"; fi; if [[ -d out/logs ]]; then echo "OK: remote_artifacts logs_dir=$PWD/out/logs"; else echo "WARN: remote_artifacts logs_dir_missing=$PWD/out/logs"; fi' \ "$remote_cd_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } @@ -1119,7 +1125,7 @@ ensure_remote_project_dir() { remote_dir_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; mkdir -p %s' "$remote_dir_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" echo "OK: ssh host=$host ensure_dir=$project_dir" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } @@ -1208,7 +1214,7 @@ fetch_remote_verify_artifacts() { [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_cd_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; cd %s; test -f out/verify-full.status; cat out/verify-full.status' "$remote_cd_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" if "${ssh_cmd[@]}" "$host" "bash -lc $script_q" > "$tmp_status" && grep -q 'status=' "$tmp_status"; then mv "$tmp_status" "$out_dir/verify-full.status" echo "OK: fetch status_file=$out_dir/verify-full.status source=ssh_fallback" @@ -1229,7 +1235,7 @@ fetch_remote_verify_artifacts() { [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_cd_q="$(remote_path_for_shell "$project_dir")" printf -v remote_script 'set -euo pipefail; cd %s; test -d out/logs; tar -cf - -C out logs' "$remote_cd_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" if "${ssh_cmd[@]}" "$host" "bash -lc $script_q" | tar -xf - -C "$out_dir"; then echo "OK: fetch logs_dir=$out_dir/logs source=ssh_fallback" else @@ -1276,12 +1282,12 @@ run_remote_ci_self() { remote_args_q="$(quote_words "${remote_args[@]}")" if [[ "$project_dir" == "~/"* ]]; then - remote_cd_q="\$HOME/${project_dir#"~/"}" + printf -v remote_cd_q '$HOME/%s' "${project_dir#"~/"}" else printf -v remote_cd_q '%q' "$project_dir" fi printf -v remote_script 'set -euo pipefail; cd %s; %q %s' "$remote_cd_q" "$remote_cli" "$remote_args_q" - script_q="$(quote_words "$remote_script")" + script_q="$(quote_bash_lc_script "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=$remote_cli ${remote_args[*]}" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 25b71d9..4a9f01d 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -426,7 +426,7 @@ exit 0 if !strings.Contains(out, "cmd=remote_verify_wrapper") { t.Fatalf("expected remote verify wrapper invocation\noutput:\n%s", out) } - if !strings.Contains(logText, "sh\\ -s") { + if !strings.Contains(logText, "sh\\ -s") && !strings.Contains(logText, "sh -s") { t.Fatalf("expected remote verify wrapper to stream script over ssh\nlog:\n%s", logText) } if !strings.Contains(logText, "rsync -a -e ssh -i "+identityPath) { @@ -515,6 +515,9 @@ exit 0 if strings.Contains(logText, "$HOME/~/_workspace/veil-rs") { t.Fatalf("tilde path was expanded incorrectly\nlog:\n%s", logText) } + if strings.Contains(logText, "\\$HOME/_workspace/veil-rs") { + t.Fatalf("tilde path should expand on remote, not remain escaped\nlog:\n%s", logText) + } if !strings.Contains(logText, "_workspace/veil-rs") { t.Fatalf("expected remote command to target remote project dir\nlog:\n%s", logText) } From df9dead54d672931220b0e8d8322532148d0ebfa Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:21:54 +0900 Subject: [PATCH 08/14] Prefer installed remote cli and homebrew rsync --- ops/ci/ci_self.sh | 40 +++++++++++++++++++++++++++++----- ops/ci/ci_self_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index 167c551..f513000 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -47,6 +47,29 @@ expand_local_path() { fi } +preferred_rsync_bin() { + local current="" + current="$(command -v rsync 2>/dev/null || true)" + if [[ -n "$current" && "$current" != "/usr/bin/rsync" ]]; then + printf '%s\n' "$current" + return 0 + fi + if [[ "$(uname -s 2>/dev/null || true)" == "Darwin" ]]; then + local candidate="" + for candidate in /opt/homebrew/bin/rsync /usr/local/bin/rsync; do + if [[ -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + fi + if [[ -n "$current" ]]; then + printf '%s\n' "$current" + return 0 + fi + return 1 +} + run_go_cmd() { if command -v go >/dev/null 2>&1; then if go "$@"; then @@ -1136,13 +1159,15 @@ sync_local_project_to_remote() { local project_dir="$3" local identity="${4:-}" local sync_git_dir="${5:-0}" - local rsync_cmd=(rsync -az --delete) + local rsync_bin="" + rsync_bin="$(preferred_rsync_bin)" || { echo "ERROR: rsync command not found" >&2; return 1; } + local rsync_cmd=("$rsync_bin" -az --delete) local ssh_rsh="" if [[ -n "$identity" ]]; then ssh_rsh="$(quote_words ssh -i "$identity")" rsync_cmd+=(-e "$ssh_rsh") fi - if rsync --info=progress2 --version >/dev/null 2>&1; then + if "$rsync_bin" --info=progress2 --version >/dev/null 2>&1; then rsync_cmd+=(--human-readable --info=progress2) else rsync_cmd+=(-h --progress) @@ -1193,7 +1218,9 @@ fetch_remote_verify_artifacts() { local project_dir="$2" local out_dir="$3" local identity="${4:-}" - local rsync_base=(rsync -a) + local rsync_bin="" + rsync_bin="$(preferred_rsync_bin)" || { echo "ERROR: rsync command not found" >&2; return 1; } + local rsync_base=("$rsync_bin" -a) local ssh_rsh="" if [[ -n "$identity" ]]; then ssh_rsh="$(quote_words ssh -i "$identity")" @@ -1274,6 +1301,7 @@ run_remote_ci_self() { shift 4 local remote_args=("$@") local remote_args_q + local remote_cli_q local script_q local remote_script local remote_cd_q @@ -1281,12 +1309,14 @@ run_remote_ci_self() { [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") remote_args_q="$(quote_words "${remote_args[@]}")" + remote_cli_q="$(remote_path_for_shell "$remote_cli")" if [[ "$project_dir" == "~/"* ]]; then printf -v remote_cd_q '$HOME/%s' "${project_dir#"~/"}" else printf -v remote_cd_q '%q' "$project_dir" fi - printf -v remote_script 'set -euo pipefail; cd %s; %q %s' "$remote_cd_q" "$remote_cli" "$remote_args_q" + printf -v remote_script 'set -euo pipefail; remote_cli=%s; if [[ "$remote_cli" != */* ]] && ! command -v "$remote_cli" >/dev/null 2>&1 && [[ -x "$HOME/.local/bin/$remote_cli" ]]; then remote_cli="$HOME/.local/bin/$remote_cli"; fi; cd %s; "$remote_cli" %s' \ + "$remote_cli_q" "$remote_cd_q" "$remote_args_q" script_q="$(quote_bash_lc_script "$remote_script")" echo "OK: ssh host=$host dir=$project_dir cmd=$remote_cli ${remote_args[*]}" "${ssh_cmd[@]}" "$host" "bash -lc $script_q" @@ -1371,7 +1401,7 @@ USAGE out_dir="$(expand_local_path "$out_dir")" command -v ssh >/dev/null 2>&1 || { echo "ERROR: ssh command not found" >&2; return 1; } - command -v rsync >/dev/null 2>&1 || { echo "ERROR: rsync command not found" >&2; return 1; } + preferred_rsync_bin >/dev/null 2>&1 || { echo "ERROR: rsync command not found" >&2; return 1; } ensure_ssh_key_auth "$host" "$identity" ensure_remote_project_dir "$host" "$project_dir" "$identity" diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 4a9f01d..4f118ab 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -73,6 +73,55 @@ func TestRemoteRegisterRequiresHost(t *testing.T) { } } +func TestRemoteRegisterFallsBackToHomeLocalBin(t *testing.T) { + tmp := t.TempDir() + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "ssh.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "$*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 42 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-register", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--project-dir", + "~/dev/zt-gateway", + "--repo", + "mt4110/zt-gateway", + "--skip-workflow", + ) + if err == nil { + t.Fatalf("expected failure from fake ssh, got success\noutput:\n%s", out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read ssh log: %v", readErr) + } + if !strings.Contains(string(logBody), "$HOME/.local/bin/$remote_cli") { + t.Fatalf("expected remote register to fall back to ~/.local/bin/ci-self\nlog:\n%s", string(logBody)) + } +} + func TestUpHelp(t *testing.T) { out, err := runCiSelf(t, "up", "--help") if err != nil { From 78a34eca354892bc07af2222f0f579ba81ac00b3 Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:39:11 +0900 Subject: [PATCH 09/14] Make remote bootstrap backward compatible --- ops/ci/ci_self.sh | 9 +++-- ops/ci/ci_self_test.go | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index f513000..d04bba9 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -461,6 +461,7 @@ cmd_register() { local discord_webhook_url="" local force_workflow=0 local skip_workflow=0 + local skip_dispatch=1 while [[ $# -gt 0 ]]; do case "$1" in @@ -473,9 +474,10 @@ cmd_register() { --discord-webhook-url) discord_webhook_url="${2:-}"; shift 2 ;; --force-workflow) force_workflow=1; shift ;; --skip-workflow) skip_workflow=1; shift ;; + --skip-dispatch) skip_dispatch=1; shift ;; -h|--help) cat <<'USAGE' -Usage: ci-self register [--repo owner/repo] [--repo-dir path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--force-workflow] [--skip-workflow] +Usage: ci-self register [--repo owner/repo] [--repo-dir path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--force-workflow] [--skip-workflow] [--skip-dispatch] USAGE return 0 ;; @@ -498,11 +500,12 @@ USAGE [[ "$skip_workflow" -eq 0 ]] && skip_workflow="$(config_bool_to_int "$CONFIG_SKIP_WORKFLOW")" repo="$(resolve_repo "$repo")" - local args=(--repo "$repo" --repo-dir "$repo_dir" --ref "$ref" --labels "$labels" --runner-group "$runner_group" --skip-dispatch) + local args=(--repo "$repo" --repo-dir "$repo_dir" --ref "$ref" --labels "$labels" --runner-group "$runner_group") [[ -n "$runner_name" ]] && args+=(--runner-name "$runner_name") [[ -n "$discord_webhook_url" ]] && args+=(--discord-webhook-url "$discord_webhook_url") [[ "$force_workflow" -eq 1 ]] && args+=(--force-workflow) [[ "$skip_workflow" -eq 1 ]] && args+=(--skip-workflow) + [[ "$skip_dispatch" -eq 1 ]] && args+=(--skip-dispatch) bash "$ROOT_DIR/ops/ci/onboard_and_verify.sh" "${args[@]}" } @@ -1417,7 +1420,7 @@ USAGE elif [[ -z "$repo" ]]; then echo "SKIP: bootstrap reason=repo_not_set" else - local register_args=(register --repo "$repo" --repo-dir "$project_dir" --skip-workflow --skip-dispatch) + local register_args=(register --repo "$repo" --repo-dir "$project_dir" --skip-workflow) [[ -n "$labels" ]] && register_args+=(--labels "$labels") [[ -n "$runner_name" ]] && register_args+=(--runner-name "$runner_name") [[ -n "$runner_group" ]] && register_args+=(--runner-group "$runner_group") diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 4f118ab..9ab260d 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -837,6 +837,88 @@ exit 0 } } +func TestRemoteCIBootstrapOmitsSkipDispatch(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "zt-gateway") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.MkdirAll(localDir, 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/dev/zt-gateway", + "--repo", + "mt4110/zt-gateway", + ) + if err != nil { + t.Fatalf("remote-ci failed: %v\noutput:\n%s", err, out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + logText := string(logBody) + if !strings.Contains(logText, "register --repo mt4110/zt-gateway") || !strings.Contains(logText, "--skip-workflow") { + t.Fatalf("expected bootstrap register invocation\nlog:\n%s", logText) + } + if strings.Contains(logText, "--skip-dispatch") { + t.Fatalf("expected remote bootstrap to omit --skip-dispatch for compatibility\nlog:\n%s", logText) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From 4d511c65ce997ce4ea7d0f2b2a29fd962e2c6590 Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:42:45 +0900 Subject: [PATCH 10/14] Skip remote bootstrap when gh auth is missing --- README.md | 5 +++ docs/ci/QUICKSTART.md | 6 ++- ops/ci/ci_self.sh | 41 ++++++++++++++++---- ops/ci/ci_self_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f93a6e3..9eb4918 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,11 @@ ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --pr 4. マシンA で、同梱の verify wrapper を SSH 経由で実行 5. `verify-full.status` と `out/logs` をマシンB の `out/remote//` に回収 +注: + +- bootstrap は remote 側で `gh auth status` が通るときだけ実行する +- remote 側で GitHub CLI 未導入または未ログインなら、bootstrap は warning ではなく skip され、standalone verify は続行する + 同期時の既定: - `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` は同期しない diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index 6b568d7..04c3370 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -62,12 +62,14 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run 1. SSH 鍵認証チェック（password不可） 2. ローカル変更をマシンA に `rsync` 同期 -3. マシンA 側 verify 実行 -4. `out/remote//` へ結果回収 +3. （`--repo` 指定時かつ remote 側 `gh auth status` 成功時のみ）bootstrap 実行 +4. マシンA 側 verify 実行 +5. `out/remote//` へ結果回収 既定では `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, `.next/` などの生成物ディレクトリと `.git/` を同期せず、`rsync --info=progress2` で進捗を表示します。 ローカル `rsync` が古い場合は `-h --progress` に自動フォールバックしますが、Homebrew の新しい `rsync` を推奨します。 repo 側の build/test が Git メタデータを直接読む場合だけ `--sync-git-dir` を付けてください。 +remote 側で GitHub CLI 未導入または未ログインなら bootstrap は skip されますが、verify 自体は続行します。 `remote-ci` は LAN 専用ではなく、外出先でも SSH 到達性があれば使えます。 逆に SSH 経路が無い場合、`remote-ci` 自体は疎通を作れません。 diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index d04bba9..4c91222 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1085,6 +1085,19 @@ probe_remote_verify_artifacts() { "${ssh_cmd[@]}" "$host" "bash -lc $script_q" } +remote_bootstrap_status() { + local host="$1" + local identity="${2:-}" + local script_q + local remote_script + local ssh_cmd=(ssh) + [[ -n "$identity" ]] && ssh_cmd+=(-i "$identity") + + remote_script='set -euo pipefail; if ! command -v gh >/dev/null 2>&1; then echo gh_missing; elif gh auth status >/dev/null 2>&1; then echo ok; else echo gh_auth_missing; fi' + script_q="$(quote_bash_lc_script "$remote_script")" + "${ssh_cmd[@]}" "$host" "bash -lc $script_q" 2>/dev/null || true +} + first_existing_public_key() { local key="" for key in \ @@ -1420,14 +1433,26 @@ USAGE elif [[ -z "$repo" ]]; then echo "SKIP: bootstrap reason=repo_not_set" else - local register_args=(register --repo "$repo" --repo-dir "$project_dir" --skip-workflow) - [[ -n "$labels" ]] && register_args+=(--labels "$labels") - [[ -n "$runner_name" ]] && register_args+=(--runner-name "$runner_name") - [[ -n "$runner_group" ]] && register_args+=(--runner-group "$runner_group") - [[ -n "$discord_webhook_url" ]] && register_args+=(--discord-webhook-url "$discord_webhook_url") - if ! run_remote_ci_self "$host" "$project_dir" "$remote_cli" "$identity" "${register_args[@]}"; then - echo "WARN: bootstrap failed; continuing standalone verify" >&2 - fi + local bootstrap_status="" + bootstrap_status="$(remote_bootstrap_status "$host" "$identity")" + case "$bootstrap_status" in + gh_missing) + echo "SKIP: bootstrap reason=remote_gh_missing" + ;; + gh_auth_missing) + echo "SKIP: bootstrap reason=remote_gh_auth_missing" + ;; + *) + local register_args=(register --repo "$repo" --repo-dir "$project_dir" --skip-workflow) + [[ -n "$labels" ]] && register_args+=(--labels "$labels") + [[ -n "$runner_name" ]] && register_args+=(--runner-name "$runner_name") + [[ -n "$runner_group" ]] && register_args+=(--runner-group "$runner_group") + [[ -n "$discord_webhook_url" ]] && register_args+=(--discord-webhook-url "$discord_webhook_url") + if ! run_remote_ci_self "$host" "$project_dir" "$remote_cli" "$identity" "${register_args[@]}"; then + echo "WARN: bootstrap failed; continuing standalone verify" >&2 + fi + ;; + esac fi local sha="" diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 9ab260d..0efdfcc 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -919,6 +919,94 @@ exit 0 } } +func TestRemoteCISkipsBootstrapWhenRemoteGHAuthMissing(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "zt-gateway") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + if err := os.MkdirAll(localDir, 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +if [[ "$*" == *"gh auth status"* ]]; then + echo gh_auth_missing + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--host", + "mini-user@192.168.1.9", + "-i", + identityPath, + "--local-dir", + localDir, + "--project-dir", + "~/dev/zt-gateway", + "--repo", + "mt4110/zt-gateway", + ) + if err != nil { + t.Fatalf("remote-ci failed: %v\noutput:\n%s", err, out) + } + if !strings.Contains(out, "SKIP: bootstrap reason=remote_gh_auth_missing") { + t.Fatalf("expected bootstrap skip for missing remote gh auth\noutput:\n%s", out) + } + if strings.Contains(out, "WARN: bootstrap failed") { + t.Fatalf("did not expect bootstrap failure warning when auth is missing\noutput:\n%s", out) + } + + logBody, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatalf("failed to read tool log: %v", readErr) + } + if strings.Contains(string(logBody), " register --repo ") { + t.Fatalf("expected bootstrap to stop before remote register when gh auth is missing\nlog:\n%s", string(logBody)) + } +} + func TestRemoteUpAcceptsIdentity(t *testing.T) { tmp := t.TempDir() identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") From 605e55af8a722bf05c4c34398f560d50b05a62aa Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:46:35 +0900 Subject: [PATCH 11/14] Generalize docs examples and add README_EN --- README.md | 15 ++-- README_EN.md | 168 ++++++++++++++++++++++++++++++++++++++++++ docs/ci/QUICKSTART.md | 11 ++- 3 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 README_EN.md diff --git a/README.md b/README.md index 9eb4918..533fddb 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ bash ops/ci/install_cli.sh CI対象リポジトリで（ローカル）: ```bash -cd ~/dev/maakie-brainlab +cd ~/dev/ ci-self up ``` @@ -27,11 +27,14 @@ ci-self up ## 別端末の CI runner を 1 コマンドで使う（推奨） -この導線は、特定の `~/dev/maakie-brainlab` 専用ではありません。 +この導線は、特定のリポジトリ専用ではありません。 - マシンA: self-hosted runner / colima / docker を置いている端末 - マシンB: 普段コードを書く端末。ここからマシンAへ verify を依頼する +比喩で言うと、マシンB は普段の机、マシンA は重い作業を引き受ける工房です。 +`remote-ci` は、机の上で書いたものを工房へ持ち込み、確認が終わった成果物だけを持ち帰る導線です。 + マシンB から 1 コマンドで「鍵認証確認 -> 同期 -> マシンA実行 -> 結果回収」まで行います。 `remote-ci` の同期元は、現在の作業リポジトリです。対象 repo のルートで実行するか、`--local-dir ` で明示してください。 @@ -45,7 +48,7 @@ ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_run 例: ```bash -ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/maakie-brainlab' --repo mt4110/maakie-brainlab +ci-self remote-ci --host ci@192.168.1.20 -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / ``` `remote-ci` の実行内容: @@ -178,11 +181,11 @@ ci-self config-init 例: ```env -CI_SELF_REPO=mt4110/maakie-brainlab +CI_SELF_REPO=/ CI_SELF_REF=main -CI_SELF_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_PROJECT_DIR=/Users//dev/ CI_SELF_REMOTE_HOST=@ci-runner.local -CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/ CI_SELF_REMOTE_IDENTITY=/Users//.ssh/id_ed25519_for_ci_runner CI_SELF_PR_BASE=main ``` diff --git a/README_EN.md b/README_EN.md new file mode 100644 index 0000000..e601037 --- /dev/null +++ b/README_EN.md @@ -0,0 +1,168 @@ +# runner-kit (self-hosted runner + colima + docker) + +`runner-kit` is a macOS-oriented toolkit for operating a self-hosted GitHub Actions runner with Colima and Docker. + +## Quick Start + +Install the CLI once: + +```bash +cd ~/dev/ci-self-runner +bash ops/ci/install_cli.sh +``` + +Run it from the CI target repository: + +```bash +cd ~/dev/ +ci-self up +``` + +`ci-self up` runs `register + run-focus` in sequence. + +## Use A Remote CI Runner In One Command + +- Machine A: the box that hosts the self-hosted runner, Colima, and Docker +- Machine B: the box where you normally write code + +Think of Machine B as your desk, and Machine A as the workshop. +`remote-ci` moves your current worktree from the desk to the workshop, runs verification there, then brings back only the results. + +```bash +ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / +``` + +What `remote-ci` does: + +1. Verifies SSH public-key auth +2. Syncs the local worktree from Machine B to `--project-dir` on Machine A +3. Runs bootstrap on Machine A only when `--repo` is set and remote `gh auth status` succeeds +4. Runs the bundled verify wrapper over SSH on Machine A +5. Collects `verify-full.status` and `out/logs` into `out/remote//` on Machine B + +Defaults during sync: + +- Generated directories such as `target/`, `dist/`, `node_modules/`, `.venv/`, `coverage/`, and `.next/` are excluded +- `.git/` is excluded by default +- `rsync --info=progress2` is used when available +- If local `rsync` is too old, it falls back to `-h --progress` +- Use `--sync-git-dir` only when your build or tests need Git metadata directly + +Notes: + +- If remote `gh` is missing or not authenticated, bootstrap is skipped instead of treated as a hard failure +- Verification itself still continues even when bootstrap is skipped +- `remote-ci` syncs the current repository by default; use `--local-dir ` when you want to sync a different local path + +## First-Time Setup For Machine B -> Machine A + +Generate an SSH key on Machine B if needed: + +```bash +ssh-keygen -t ed25519 -a 100 +``` + +Append the public key to `~/.ssh/authorized_keys` on Machine A: + +```bash +cat ~/.ssh/id_ed25519_for_ci_runner.pub | ssh -i ~/.ssh/id_ed25519_for_ci_runner @ 'mkdir -p ~/.ssh && chmod 700 ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys' +``` + +Confirm passwordless SSH: + +```bash +ssh -i ~/.ssh/id_ed25519_for_ci_runner -o BatchMode=yes -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no @ true +``` + +Then run: + +```bash +ci-self remote-ci --host @ -i ~/.ssh/id_ed25519_for_ci_runner --project-dir '~/dev/' --repo / +``` + +## rsync Note + +macOS ships an older `rsync` in some environments. + +```bash +rsync --version +``` + +Recommended: + +```bash +brew install rsync +``` + +An `alias rsync=...` is often not enough for the `ci-self` bash script. Prefer putting Homebrew first in `PATH`. + +Apple Silicon example: + +```bash +echo 'export PATH="/opt/homebrew/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc +``` + +## Can I Use It Away From Home? + +Sometimes, yes. The real requirement is not “same LAN”, but “SSH reachability”. + +- Works when Machine A is reachable over SSH +- Common setups: Tailscale, VPN, port forwarding, fixed-IP home network +- Does not work when no SSH route exists + +`remote-ci` does not create tunnels or expose the machine by itself. + +## What It Can And Cannot Do + +- Can: run any target repository on Machine A by changing `--project-dir` and `--repo` +- Can: sync uncommitted local changes and verify them remotely +- Can: collect `verify-full.status` and `out/logs` back to Machine B +- Can: use the same command both on a local network and remotely, as long as SSH works +- Cannot yet: run over password-based SSH +- Cannot yet: create SSH reachability for you when no route exists +- Cannot yet: auto-detect or auto-route multiple repositories without explicit config + +## Optional Config File + +`ci-self` auto-loads `.ci-self.env`. + +Create it with: + +```bash +ci-self config-init +``` + +Example: + +```env +CI_SELF_REPO=/ +CI_SELF_REF=main +CI_SELF_PROJECT_DIR=/Users//dev/ +CI_SELF_REMOTE_HOST=@ci-runner.local +CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/ +CI_SELF_REMOTE_IDENTITY=/Users//.ssh/id_ed25519_for_ci_runner +CI_SELF_PR_BASE=main +``` + +## Main Commands + +- `ci-self up`: fastest local path (`register + run-focus`) +- `ci-self focus`: runs `run-focus`, creates a PR if missing, then watches checks +- `ci-self remote-ci`: SSH-required sync + remote verify + result collection in one command +- `ci-self doctor --fix`: checks dependencies, `gh auth`, Colima, Docker, and runner health +- `ci-self remote-up`: older SSH path for `register + run-focus` without syncing +- `ci-self config-init`: generates a `.ci-self.env` template + +## Security Assumptions + +- Intended for single-owner personal operation +- Self-hosted execution is gated by `SELF_HOSTED_OWNER` +- For external collaborators or fork PRs, review `docs/ci/SECURITY_HARDENING_TASK.md` first + +## More + +- `README.md` +- `docs/ci/QUICKSTART.md` +- `docs/ci/RUNBOOK.md` +- `docs/ci/SECURITY_HARDENING_TASK.md` diff --git a/docs/ci/QUICKSTART.md b/docs/ci/QUICKSTART.md index 04c3370..3a0f7a4 100644 --- a/docs/ci/QUICKSTART.md +++ b/docs/ci/QUICKSTART.md @@ -10,7 +10,7 @@ bash ops/ci/install_cli.sh ## 2) CI対象リポジトリで 1 コマンド実行 ```bash -cd ~/dev/maakie-brainlab +cd ~/dev/ ci-self up ``` @@ -28,11 +28,11 @@ ci-self config-init `.ci-self.env` 例: ```env -CI_SELF_REPO=mt4110/maakie-brainlab +CI_SELF_REPO=/ CI_SELF_REF=main -CI_SELF_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_PROJECT_DIR=/Users//dev/ CI_SELF_REMOTE_HOST=@ci-runner.local -CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/maakie-brainlab +CI_SELF_REMOTE_PROJECT_DIR=/Users//dev/ CI_SELF_REMOTE_IDENTITY=/Users//.ssh/id_ed25519_for_ci_runner CI_SELF_PR_BASE=main ``` @@ -42,6 +42,9 @@ CI_SELF_PR_BASE=main - マシンA: self-hosted runner / colima / docker を置く端末 - マシンB: そこへ verify を依頼する手元端末 +比喩で言うと、マシンB は普段の机、マシンA は重い作業を引き受ける工房です。 +`remote-ci` は、机の上の作業ツリーを工房へ運び、検証後の結果だけを机へ戻すイメージです。 + まずマシンB で SSH 鍵を用意し、公開鍵をマシンA の `~/.ssh/authorized_keys` に登録します。 ```bash From c093b3b5822053a994b4f3f7f73d0066cd27df94 Mon Sep 17 00:00:00 2001 From: mt4110 <4221282+mt4110@users.noreply.github.com> Date: Tue, 31 Mar 2026 00:10:56 +0900 Subject: [PATCH 12/14] Address PR review comments --- ops/ci/ci_self.sh | 8 +++-- ops/ci/ci_self_test.go | 73 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index 4c91222..1805572 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -461,7 +461,6 @@ cmd_register() { local discord_webhook_url="" local force_workflow=0 local skip_workflow=0 - local skip_dispatch=1 while [[ $# -gt 0 ]]; do case "$1" in @@ -474,7 +473,7 @@ cmd_register() { --discord-webhook-url) discord_webhook_url="${2:-}"; shift 2 ;; --force-workflow) force_workflow=1; shift ;; --skip-workflow) skip_workflow=1; shift ;; - --skip-dispatch) skip_dispatch=1; shift ;; + --skip-dispatch) shift ;; -h|--help) cat <<'USAGE' Usage: ci-self register [--repo owner/repo] [--repo-dir path] [--labels csv] [--runner-name name] [--runner-group name] [--discord-webhook-url url] [--force-workflow] [--skip-workflow] [--skip-dispatch] @@ -505,7 +504,7 @@ USAGE [[ -n "$discord_webhook_url" ]] && args+=(--discord-webhook-url "$discord_webhook_url") [[ "$force_workflow" -eq 1 ]] && args+=(--force-workflow) [[ "$skip_workflow" -eq 1 ]] && args+=(--skip-workflow) - [[ "$skip_dispatch" -eq 1 ]] && args+=(--skip-dispatch) + args+=(--skip-dispatch) bash "$ROOT_DIR/ops/ci/onboard_and_verify.sh" "${args[@]}" } @@ -1405,6 +1404,9 @@ USAGE [[ -n "$host" ]] || { echo "ERROR: --host is required" >&2; return 2; } [[ -z "$project_dir" ]] && project_dir="$(default_remote_project_dir)" [[ -z "$local_dir" ]] && local_dir="$(default_local_project_dir)" + if [[ "$local_dir_was_explicit" -eq 0 && -n "$CONFIG_PROJECT_DIR" && "$local_dir" == "$CONFIG_PROJECT_DIR" ]]; then + local_dir_was_explicit=1 + fi local_dir="$(expand_local_path "$local_dir")" [[ -n "$identity" ]] && identity="$(expand_local_path "$identity")" [[ -d "$local_dir" ]] || { echo "ERROR: --local-dir not found: $local_dir" >&2; return 2; } diff --git a/ops/ci/ci_self_test.go b/ops/ci/ci_self_test.go index 0efdfcc..9935cbe 100644 --- a/ops/ci/ci_self_test.go +++ b/ops/ci/ci_self_test.go @@ -213,6 +213,79 @@ func TestRemoteUpUsesConfigHost(t *testing.T) { } } +func TestRemoteCIConfiguredProjectDirSkipsRepoNameGuard(t *testing.T) { + tmp := t.TempDir() + localDir := filepath.Join(tmp, "backend-v2") + identityPath := filepath.Join(tmp, "id_ed25519_for_mac_mini") + cfg := filepath.Join(tmp, ".ci-self.env") + if err := os.MkdirAll(localDir, 0o755); err != nil { + t.Fatalf("mkdir local repo failed: %v", err) + } + if err := os.WriteFile(identityPath, []byte("dummy-private-key"), 0o600); err != nil { + t.Fatalf("write identity file failed: %v", err) + } + cfgContent := "CI_SELF_PROJECT_DIR=" + localDir + "\nCI_SELF_REMOTE_HOST=mini-user@192.168.1.9\nCI_SELF_REMOTE_PROJECT_DIR=~/dev/backend\n" + if err := os.WriteFile(cfg, []byte(cfgContent), 0o644); err != nil { + t.Fatalf("write config failed: %v", err) + } + + logPath := filepath.Join(tmp, "tool.log") + sshPath := filepath.Join(tmp, "ssh") + sshScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "ssh $*" >> %q +if [[ "$*" == *"BatchMode=yes"* ]]; then + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(sshPath, []byte(sshScript), 0o755); err != nil { + t.Fatalf("write fake ssh failed: %v", err) + } + + rsyncPath := filepath.Join(tmp, "rsync") + rsyncScript := fmt.Sprintf(`#!/usr/bin/env bash +set -euo pipefail +echo "rsync $*" >> %q +src="${@: -2:1}" +dst="${@: -1}" +if printf '%%s' "$src" | grep -q '/out/verify-full.status$'; then + mkdir -p "$dst" + cat > "${dst%%/}/verify-full.status" <<'EOF' +status=OK +EOF + exit 0 +fi +if printf '%%s' "$src" | grep -q '/out/logs/$'; then + mkdir -p "${dst%%/}" + echo "ok" > "${dst%%/}/verify.log" + exit 0 +fi +exit 0 +`, logPath) + if err := os.WriteFile(rsyncPath, []byte(rsyncScript), 0o755); err != nil { + t.Fatalf("write fake rsync failed: %v", err) + } + + out, err := runCiSelfInDirEnv( + t, + tmp, + []string{"PATH=" + tmp + ":" + os.Getenv("PATH")}, + "remote-ci", + "--repo", + "mt4110/backend", + "-i", + identityPath, + "--skip-bootstrap", + ) + if err != nil { + t.Fatalf("remote-ci should accept configured local project dir with mismatched basename: %v\noutput:\n%s", err, out) + } + if strings.Contains(out, "ERROR: default local-dir appears to be the wrong project") { + t.Fatalf("repo-name guard should not fire for configured project dir\noutput:\n%s", out) + } +} + func TestRunWatchResolvesVerifyWorkflowID(t *testing.T) { tmp := t.TempDir() logPath := filepath.Join(tmp, "gh.log") From d541f071fa1e4fba4f9896bbaf405c3cb5352ac8 Mon Sep 17 00:00:00 2001 From: "@mt4110" <4221282+mt4110@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:33:55 +0900 Subject: [PATCH 13/14] Update ops/ci/ci_self.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ops/ci/ci_self.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index 1805572..d48c342 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -1012,7 +1012,8 @@ ensure_default_local_dir_matches_repo() { remote_path_for_shell() { local path="$1" if [[ "$path" == "~/"* ]]; then - printf '$HOME/%s\n' "${path#"~/"}" + # Emit a quoted path that expands $HOME on the remote side: cd "$HOME/..." + printf '%s\n' "\"\$HOME/${path#"~/"}\"" else printf '%q\n' "$path" fi From 7b0adee88736c43403a961dba7a8321dfcf0cba2 Mon Sep 17 00:00:00 2001 From: "@mt4110" <4221282+mt4110@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:34:04 +0900 Subject: [PATCH 14/14] Update ops/ci/ci_self.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ops/ci/ci_self.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/ci/ci_self.sh b/ops/ci/ci_self.sh index d48c342..d37d16c 100755 --- a/ops/ci/ci_self.sh +++ b/ops/ci/ci_self.sh @@ -957,7 +957,7 @@ quote_words() { quote_bash_lc_script() { local script="$1" - script="${script//\'/\'\"\'\"\'}" + script=${script//\'/\'"\'"\'} printf "'%s'\n" "$script" }