diff --git a/docs/usage/commands/new.md b/docs/usage/commands/new.md index 04389ca3..705ed25d 100644 --- a/docs/usage/commands/new.md +++ b/docs/usage/commands/new.md @@ -72,6 +72,12 @@ bits and symlinks are preserved), falling back to a plain ZIP download otherwise. As ZIP does not preserve executable bits nor symlinks, `chmod +x` and symlink fixes might be necessary after scaffolding. +Private repos work when `git` is on `PATH`, using whatever credentials +your local `git` already has configured for them (SSH key, HTTPS +credential helper, etc.) — there's no separate token or login step. This +applies to `--list` as well as scaffolding. Without `git` on `PATH`, only +public repos are reachable. + ## App Scaffold `jo new my-agent` prints: diff --git a/stack-lang/tool/Test.scala b/stack-lang/tool/Test.scala index 265929db..0c4552c4 100644 --- a/stack-lang/tool/Test.scala +++ b/stack-lang/tool/Test.scala @@ -694,6 +694,16 @@ private def runGithubTemplateProviderTests(): List[Path] = server.createContext("/acme/hashref/v1#odd/jo-templates.jsonl", (ex: HttpExchange) => respond(ex, 200, manifestBytes)) server.start() + // Any request gets a bare 401 with no body — enough to make git's HTTPS + // clone attempt bail out with "could not read Username ... terminal + // prompts disabled" (GIT_TERMINAL_PROMPT=0), the same failure a real + // private GitHub repo produces over HTTPS with no credential helper + // configured. Used below as a clone candidate that's guaranteed to fail, + // to exercise the fall-through-to-the-next-candidate logic. + val authServer = HttpServer.create(InetSocketAddress("127.0.0.1", 0), 0) + authServer.createContext("/", (ex: HttpExchange) => respond(ex, 401, Array.emptyByteArray)) + authServer.start() + try val baseUrl = s"http://127.0.0.1:${server.getAddress.getPort}" // gitAvailable = false so these checks deterministically exercise the @@ -765,7 +775,7 @@ private def runGithubTemplateProviderTests(): List[Path] = val gitProvider = GithubTemplateProvider( rawBaseUrl = baseUrl, archiveBaseUrl = baseUrl, - cloneBaseUrl = gitReposParent.toString, + cloneBaseUrls = List(s"$gitReposParent/"), gitAvailable = true, ) @@ -791,8 +801,45 @@ private def runGithubTemplateProviderTests(): List[Path] = case Result.Err(_) => true case Result.Ok(_) => false + check("manifest (git): reads jo-templates.jsonl from a git checkout, not raw HTTP, when git is available"): + gitProvider.manifest("acme/repo", "HEAD") == Result.Ok(List(TemplateEntry("default", ".", None))) + + val unauthedHttpsUrl = s"http://127.0.0.1:${authServer.getAddress.getPort}/" + + // The second candidate here is a plain filesystem path, not a real + // ssh:// / git@host: URL — GIT_SSH_COMMAND has no effect on a + // local-path remote, so this doesn't exercise real SSH transport, only + // the candidate-list logic itself (first candidate fails, second one is + // tried and its result is what's returned). + val multiCandidateProvider = GithubTemplateProvider( + rawBaseUrl = baseUrl, + archiveBaseUrl = baseUrl, + cloneBaseUrls = List(unauthedHttpsUrl, s"$gitReposParent/"), + gitAvailable = true, + ) + + check("fetch (git): a first candidate URL needing credentials falls through to the next one, which succeeds"): + val dest = Files.createTempDirectory("jo-template-multi-candidate-test-") + multiCandidateProvider.fetch("acme/repo", "HEAD", None, dest) match + case Result.Ok(_) => Files.exists(dest.resolve("jo-templates.jsonl")) + case Result.Err(_) => false + + val allCandidatesFailProvider = GithubTemplateProvider( + rawBaseUrl = baseUrl, + archiveBaseUrl = baseUrl, + cloneBaseUrls = List(unauthedHttpsUrl, s"${gitReposParent.resolve("no-such-dir")}/"), + gitAvailable = true, + ) + + check("fetch (git): every candidate URL failing reports all of their failures, not just the last one"): + val dest = Files.createTempDirectory("jo-template-multi-candidate-fail-test-") + allCandidatesFailProvider.fetch("acme/repo", "HEAD", None, dest) match + case Result.Err(msg) => msg.contains("could not fetch acme/repo from any configured clone URL") && msg.lines.count() >= 3 + case Result.Ok(_) => false + finally server.stop(0) + authServer.stop(0) if failed then List(Paths.get("GithubTemplateProvider")) else Nil diff --git a/stack-lang/tool/template/GithubTemplateProvider.scala b/stack-lang/tool/template/GithubTemplateProvider.scala index 066a380e..94b19c89 100644 --- a/stack-lang/tool/template/GithubTemplateProvider.scala +++ b/stack-lang/tool/template/GithubTemplateProvider.scala @@ -9,20 +9,30 @@ import tool.Result /** [[TemplateProvider]] for GitHub (`gh:owner/repo`). * - * `manifest` (used only by `--list`) always uses the raw-content HTTP - * endpoint — cheap, and safe as a standalone fetch since nothing else gets - * combined with it. `fetch` prefers a shallow `git` checkout when `git` is - * on `PATH`, since only a real checkout reproduces Unix executable bits - * and symlinks correctly (`java.util.zip` exposes neither — see - * `TemplateArchive`); it falls back to downloading and unzipping a - * codeload archive when `git` isn't available. Base URLs are constructor - * parameters, not hardcoded, so tests can point this at a local server or - * a local git repo instead of the real GitHub hosts. + * `manifest` (used only by `--list`) and `fetch` both prefer a shallow + * `git` checkout when `git` is on `PATH`: a real checkout is what lets + * either operation reach a private repo using whatever credentials the + * user's local `git` already has configured (SSH key, credential helper, + * `.netrc`, `insteadOf` rewrites, ...) — this tool never manages + * credentials itself. `git` doesn't care which transport a clone URL + * names, so rather than hardcoding one URL, `cloneBaseUrls` is an ordered + * list of candidates (HTTPS, then SSH, by default) tried in turn as + * `origin` until one of them authenticates; if every candidate fails, all + * of their failures are folded into the final error, not just the last + * one. A real checkout also reproduces Unix executable bits and symlinks + * correctly, which matters for `fetch` specifically (`java.util.zip` + * exposes neither — see `TemplateArchive`). When `git` isn't available, + * `manifest` falls back to an unauthenticated raw-content HTTP request and + * `fetch` falls back to downloading and unzipping a codeload archive — + * both public-repos-only, since there's no git credential store to draw + * on. Base URLs are constructor parameters, not hardcoded, so tests can + * point this at a local server or a local git repo instead of the real + * GitHub hosts. */ class GithubTemplateProvider( rawBaseUrl: String = GithubTemplateProvider.rawUrl, archiveBaseUrl: String = GithubTemplateProvider.archiveUrl, - cloneBaseUrl: String = GithubTemplateProvider.cloneUrl, + cloneBaseUrls: List[String] = GithubTemplateProvider.cloneUrls, gitAvailable: Boolean = GithubTemplateProvider.detectGit(), ) extends TemplateProvider: private val http = HttpClient.newBuilder() @@ -30,6 +40,30 @@ class GithubTemplateProvider( .build() def manifest(identifier: String, gitref: String): Result[List[TemplateEntry]] = + if gitAvailable then manifestViaGit(identifier, gitref) + else manifestViaHttp(identifier, gitref) + + def fetch(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = + if gitAvailable then fetchViaGit(identifier, gitref, name, destDir) + else fetchViaZip(identifier, gitref, name, destDir) + + // ---- Internals --------------------------------------------------------------- + + private def manifestViaGit(identifier: String, gitref: String): Result[List[TemplateEntry]] = + parseIdentifier(identifier).flatMap: (owner, repo) => + checkoutViaGit(owner, repo, gitref).flatMap: checkoutDir => + try + val manifestFile = checkoutDir.resolve("jo-templates.jsonl") + + if !Files.exists(manifestFile) then + Result.Err(s"$identifier has no jo-templates.jsonl — not a valid Jo template repo") + else + TemplateManifest.parse(Files.readString(manifestFile)) + + finally + deleteRecursively(checkoutDir) + + private def manifestViaHttp(identifier: String, gitref: String): Result[List[TemplateEntry]] = parseIdentifier(identifier).flatMap: (owner, repo) => val url = uri(rawBaseUrl, s"/$owner/$repo/$gitref/jo-templates.jsonl") @@ -43,35 +77,73 @@ class GithubTemplateProvider( case FetchResult.Failure(msg) => Result.Err(msg) - def fetch(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = - if gitAvailable then fetchViaGit(identifier, gitref, name, destDir) - else fetchViaZip(identifier, gitref, name, destDir) - - // ---- Internals --------------------------------------------------------------- - private def fetchViaGit(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = parseIdentifier(identifier).flatMap: (owner, repo) => - val url = s"$cloneBaseUrl/$owner/$repo.git" - val checkoutDir = Files.createTempDirectory("jo-template-") - - try - val checkedOut = - for - _ <- runGit("init", "-q", checkoutDir.toString) - _ <- runGit("-C", checkoutDir.toString, "remote", "add", "origin", url) - _ <- runGit("-C", checkoutDir.toString, "fetch", "--depth", "1", "-q", "origin", gitref) - _ <- runGit("-C", checkoutDir.toString, "checkout", "-q", "FETCH_HEAD") - yield () - - checkedOut.flatMap: _ => - // Never ship git's own metadata into the scaffolded project — the - // user-facing result is a plain directory either way, exactly - // like the zip path, regardless of how it was fetched internally. - deleteRecursively(checkoutDir.resolve(".git")) + checkoutViaGit(owner, repo, gitref).flatMap: checkoutDir => + try TemplateArchive.resolveAndCopy(checkoutDir, name, destDir, s"$identifier at $gitref") + finally + deleteRecursively(checkoutDir) - finally + /** Shallow single-commit checkout of `owner/repo` at `gitref` into a fresh + * temp directory, returned to the caller (who owns cleanup). + * + * `.git` is deleted from a successful checkout immediately: the result + * must be a plain directory either way, exactly like the zip path, + * regardless of how it was fetched internally. + */ + private def checkoutViaGit(owner: String, repo: String, gitref: String): Result[Path] = + val checkoutDir = Files.createTempDirectory("jo-template-") + + val checkedOut = + for + _ <- runGit("init", "-q", checkoutDir.toString) + _ <- fetchFromFirstWorkingUrl(checkoutDir, owner, repo, gitref) + _ <- runGit("-C", checkoutDir.toString, "checkout", "-q", "FETCH_HEAD") + yield () + + checkedOut match + case Result.Ok(_) => + deleteRecursively(checkoutDir.resolve(".git")) + Result.Ok(checkoutDir) + + case Result.Err(msg) => deleteRecursively(checkoutDir) + Result.Err(msg) + + /** Tries each of `cloneBaseUrls` as `origin`, in order, until one fetches + * successfully. + * + * Each candidate is just a different transport for the same operation as + * far as git is concerned — nothing here needs to know HTTPS from SSH + * from a plain local path, or inspect a failure's error text to decide + * whether to move on. If every candidate fails, their failures are all + * reported together, not just the last one, since with only the last + * error a genuinely private repo (every candidate needs credentials + * nothing has) and a typo'd repo name (every candidate agrees it doesn't + * exist) would otherwise look identical. + */ + private def fetchFromFirstWorkingUrl(checkoutDir: Path, owner: String, repo: String, gitref: String): Result[Unit] = + def attempt(remaining: List[String], errors: List[String]): Result[Unit] = + remaining match + case Nil => + Result.Err(s"could not fetch $owner/$repo from any configured clone URL:\n" + errors.reverse.map(e => s" - $e").mkString("\n")) + + case base :: rest => + val url = s"$base$owner/$repo.git" + val remoteOp = if errors.isEmpty then "add" else "set-url" + + val result = + for + _ <- runGit("-C", checkoutDir.toString, "remote", remoteOp, "origin", url) + _ <- runGit("-C", checkoutDir.toString, "fetch", "--depth", "1", "-q", "origin", gitref) + yield () + + result match + case Result.Ok(_) => Result.Ok(()) + case Result.Err(e) => attempt(rest, e :: errors) + + attempt(cloneBaseUrls, Nil) private def fetchViaZip(identifier: String, gitref: String, name: Option[String], destDir: Path): Result[Unit] = parseIdentifier(identifier).flatMap: (owner, repo) => @@ -92,14 +164,21 @@ class GithubTemplateProvider( finally Files.deleteIfExists(tempZip) - /** Runs `git` with `args`, capturing combined output. `GIT_TERMINAL_PROMPT=0` - * ensures a private or nonexistent repo fails fast with an error instead - * of hanging on a credential prompt with nothing to answer it. + /** Runs `git` with `args`, capturing combined output. + * + * Fails fast rather than hanging on any prompt for credentials this + * process has nothing to answer with: `GIT_TERMINAL_PROMPT=0` covers + * git's own HTTPS credential prompt, and `GIT_SSH_COMMAND`'s + * `BatchMode=yes` covers the `ssh` subprocess's own passphrase / + * unknown-host-key prompts. The latter is simply inert whenever the URL + * being operated on isn't an SSH one, so it's always set rather than + * only for SSH-shaped candidates. */ private def runGit(args: String*): Result[Unit] = try val pb = ProcessBuilder(("git" +: args)*).redirectErrorStream(true) pb.environment().put("GIT_TERMINAL_PROMPT", "0") + pb.environment().put("GIT_SSH_COMMAND", "ssh -o BatchMode=yes") val proc = pb.start() val output = String(proc.getInputStream.readAllBytes(), "UTF-8") val exit = proc.waitFor() @@ -193,9 +272,21 @@ class GithubTemplateProvider( FetchResult.Failure(s"failed to fetch $url: ${describe(e)}") object GithubTemplateProvider: - val rawUrl = "https://raw.githubusercontent.com" - val archiveUrl = "https://codeload.github.com" - val cloneUrl = "https://github.com" + val rawUrl = "https://raw.githubusercontent.com" + val archiveUrl = "https://codeload.github.com" + + /** Each entry is a prefix `owner/repo.git` is appended to directly (no + * separator inserted), so every candidate must already end in whatever + * its own transport needs: a trailing `/` for HTTPS, a trailing `:` for + * SSH's `user@host:path` scp-like syntax. + * + * Tried in this order — HTTPS first since it's the more common case + * (public repos, or an HTTPS credential helper already configured), SSH + * second as the fallback for repos only reachable that way. + */ + val cloneUrl = "https://github.com/" + val sshCloneUrl = "git@github.com:" + val cloneUrls = List(cloneUrl, sshCloneUrl) /** Checked once, at construction — not on every `fetch` call. */ def detectGit(): Boolean =