From e2b690eb31242dc5dcf14a24bfd1402bac31d255 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 7 Aug 2026 14:27:37 +0800 Subject: [PATCH 01/39] feat(crosscompile): add Linux phase 1 MVP --- cmd/llar/internal/crosscompile.go | 199 ++++++++++ cmd/llar/internal/crosscompile_test.go | 121 ++++++ cmd/llar/internal/make.go | 15 + internal/build/crosscompile/crosscompile.go | 361 ++++++++++++++++++ .../build/crosscompile/crosscompile_test.go | 149 ++++++++ x/cmake/cmake.go | 5 + x/cmake/cmake_test.go | 23 +- 7 files changed, 868 insertions(+), 5 deletions(-) create mode 100644 cmd/llar/internal/crosscompile.go create mode 100644 cmd/llar/internal/crosscompile_test.go create mode 100644 internal/build/crosscompile/crosscompile.go create mode 100644 internal/build/crosscompile/crosscompile_test.go diff --git a/cmd/llar/internal/crosscompile.go b/cmd/llar/internal/crosscompile.go new file mode 100644 index 0000000..7d8e780 --- /dev/null +++ b/cmd/llar/internal/crosscompile.go @@ -0,0 +1,199 @@ +package internal + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "runtime" + + "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/crosscompile" + "github.com/goplus/llar/internal/execbroker" + "github.com/goplus/llar/internal/formula/repo" + "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/mod/module" +) + +type llvmPathToolchain struct { + cc string + cxx string + archiver string + ranlib string + nm string + strip string +} + +func (t llvmPathToolchain) CC() string { return t.cc } +func (t llvmPathToolchain) CXX() string { return t.cxx } +func (t llvmPathToolchain) Archiver() string { return t.archiver } +func (t llvmPathToolchain) Ranlib() string { return t.ranlib } +func (t llvmPathToolchain) NM() string { return t.nm } +func (t llvmPathToolchain) Strip() string { return t.strip } + +var newCrossCompileToolchain = func() (crosscompile.Toolchain, error) { + find := func(name string) (string, error) { + path, err := exec.LookPath(name) + if err != nil { + return "", fmt.Errorf("find prepared LLVM command %q: %w", name, err) + } + return path, nil + } + cc, err := find("clang") + if err != nil { + return nil, err + } + cxx, err := find("clang++") + if err != nil { + return nil, err + } + archiver, err := find("llvm-ar") + if err != nil { + return nil, err + } + ranlib, err := find("llvm-ranlib") + if err != nil { + return nil, err + } + nm, err := find("llvm-nm") + if err != nil { + return nil, err + } + strip, err := find("llvm-strip") + if err != nil { + return nil, err + } + return llvmPathToolchain{ + cc: cc, + cxx: cxx, + archiver: archiver, + ranlib: ranlib, + nm: nm, + strip: strip, + }, nil +} + +func prepareCrossCompile( + ctx context.Context, + store repo.Store, + root module.Version, + matrix formula.Matrix, + buildOpts build.Options, +) (*crosscompile.CrossCompile, *modules.Module, error) { + targetOS, targetArch := matrixTarget(matrix) + sysroot, ok := crosscompile.Sysroot(targetOS, targetArch) + if !ok || targetOS == runtime.GOOS && targetArch == runtime.GOARCH || root.Path == sysroot.Path { + return nil, nil, nil + } + + sysrootMods, err := modules.Load(ctx, sysroot, modules.Options{ + FormulaStore: store, + Matrix: matrix, + }) + if err != nil { + return nil, nil, fmt.Errorf("load cross compile sysroot %s@%s: %w", sysroot.Path, sysroot.Version, err) + } + buildOpts.RunTest = false + builder, err := build.NewBuilder(buildOpts) + if err != nil { + return nil, nil, fmt.Errorf("create sysroot builder: %w", err) + } + results, err := builder.Build(ctx, sysrootMods) + if err != nil { + return nil, nil, fmt.Errorf("build cross compile sysroot %s@%s: %w", sysroot.Path, sysroot.Version, err) + } + + toolchain, err := newCrossCompileToolchain() + if err != nil { + return nil, nil, err + } + rewriter, err := crosscompile.New(targetOS, targetArch, toolchain, results[len(results)-1].Metadata) + if err != nil { + return nil, nil, fmt.Errorf("prepare cross compile target %s/%s: %w", targetOS, targetArch, err) + } + return rewriter, sysrootMods[0], nil +} + +func matrixTarget(matrix formula.Matrix) (targetOS, targetArch string) { + if values := matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + return +} + +func injectSysroot(mods []*modules.Module, sysroot *modules.Module) []*modules.Module { + for _, mod := range mods { + mod.Deps = append(mod.Deps, sysroot) + } + return append(mods, sysroot) +} + +func wrapCrossCompileHooks(mods []*modules.Module, rewriter *crosscompile.CrossCompile, stdout, stderr io.Writer) { + middleware := func(req execbroker.Request) execbroker.Request { + patch := rewriter.Use(crosscompile.Command{ + Name: req.Name, + Args: req.Args, + Env: effectiveCommandEnv(req.Env), + Dir: req.Dir, + }) + return applyCrossCompilePatch(req, patch) + } + for _, mod := range mods { + if hook := mod.OnBuild; hook != nil { + mod.OnBuild = func(ctx *formula.Context) { + _ = execbroker.Do(execbroker.Scope{ + Dir: ctx.SourceDir, + Stdin: os.Stdin, + Stdout: stdout, + Stderr: stderr, + Middleware: middleware, + }, func() error { + hook(ctx) + return nil + }) + } + } + if hook := mod.OnTest; hook != nil { + mod.OnTest = func(ctx *formula.Context) { + _ = execbroker.Do(execbroker.Scope{ + Dir: ctx.SourceDir, + Stdin: os.Stdin, + Stdout: stdout, + Stderr: stderr, + Middleware: middleware, + }, func() error { + hook(ctx) + return nil + }) + } + } + } +} + +func applyCrossCompilePatch(req execbroker.Request, patch crosscompile.Patch) execbroker.Request { + if patch.Name != "" { + req.Name = patch.Name + } + if len(patch.PrependArg) > 0 { + req.Args = append(append([]string(nil), patch.PrependArg...), req.Args...) + } + if len(patch.AppendArg) > 0 { + req.Args = append(req.Args, patch.AppendArg...) + } + if patch.Env != nil { + req.Env = append([]string(nil), patch.Env...) + } + return req +} + +func effectiveCommandEnv(env []string) []string { + if env != nil { + return append([]string(nil), env...) + } + return os.Environ() +} diff --git a/cmd/llar/internal/crosscompile_test.go b/cmd/llar/internal/crosscompile_test.go new file mode 100644 index 0000000..e030dd9 --- /dev/null +++ b/cmd/llar/internal/crosscompile_test.go @@ -0,0 +1,121 @@ +package internal + +import ( + "io" + "os" + "reflect" + "testing" + + "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build/crosscompile" + "github.com/goplus/llar/internal/execbroker" + "github.com/goplus/llar/internal/modules" +) + +type testLLVMToolchain struct{} + +func (testLLVMToolchain) CC() string { return "/llvm/bin/clang" } +func (testLLVMToolchain) CXX() string { return "/llvm/bin/clang++" } +func (testLLVMToolchain) Archiver() string { return "/llvm/bin/llvm-ar" } +func (testLLVMToolchain) Ranlib() string { return "/llvm/bin/llvm-ranlib" } +func (testLLVMToolchain) NM() string { return "/llvm/bin/llvm-nm" } +func (testLLVMToolchain) Strip() string { return "/llvm/bin/llvm-strip" } + +func TestMatrixTarget(t *testing.T) { + matrix := formula.Matrix{Require: map[string][]string{ + "os": {"linux"}, + "arch": {"arm64"}, + }} + if targetOS, targetArch := matrixTarget(matrix); targetOS != "linux" || targetArch != "arm64" { + t.Fatalf("matrixTarget = %s/%s, want linux/arm64", targetOS, targetArch) + } +} + +func TestInjectSysroot(t *testing.T) { + root := &modules.Module{Path: "owner/root", Version: "v1"} + dep := &modules.Module{Path: "owner/dep", Version: "v2"} + sysroot := &modules.Module{Path: "bminor/glibc", Version: "glibc-2.17"} + root.Deps = []*modules.Module{dep} + + got := injectSysroot([]*modules.Module{root, dep}, sysroot) + if want := []*modules.Module{root, dep, sysroot}; !reflect.DeepEqual(got, want) { + t.Fatalf("modules = %+v, want %+v", got, want) + } + if want := []*modules.Module{dep, sysroot}; !reflect.DeepEqual(root.Deps, want) { + t.Fatalf("root deps = %+v, want %+v", root.Deps, want) + } + if want := []*modules.Module{sysroot}; !reflect.DeepEqual(dep.Deps, want) { + t.Fatalf("dep deps = %+v, want %+v", dep.Deps, want) + } +} + +func TestApplyCrossCompilePatch(t *testing.T) { + req := execbroker.Request{ + Name: "cc", + Args: []string{"-c", "a.c"}, + Env: []string{"CFLAGS=-O2"}, + } + got := applyCrossCompilePatch(req, crosscompile.Patch{ + Name: "/llvm/bin/clang", + PrependArg: []string{"--target=aarch64-linux-gnu"}, + AppendArg: []string{"--sysroot=/sdk"}, + Env: []string{"CFLAGS=-O2 --sysroot=/sdk"}, + }) + if got.Name != "/llvm/bin/clang" { + t.Fatalf("Name = %q", got.Name) + } + if want := []string{"--target=aarch64-linux-gnu", "-c", "a.c", "--sysroot=/sdk"}; !reflect.DeepEqual(got.Args, want) { + t.Fatalf("Args = %q, want %q", got.Args, want) + } + if want := []string{"CFLAGS=-O2 --sysroot=/sdk"}; !reflect.DeepEqual(got.Env, want) { + t.Fatalf("Env = %q, want %q", got.Env, want) + } +} + +func TestWrapCrossCompileHooks(t *testing.T) { + rewriter, err := crosscompile.New("linux", "arm64", testLLVMToolchain{}, "--sysroot=/sdk") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rewriter.Close() }) + + var commandName string + var commandArgs []string + mod := &modules.Module{Path: "owner/root", Version: "v1"} + mod.OnBuild = func(*formula.Context) { + cmd := execbroker.Command("cc", "-c", "a.c") + commandName = cmd.Path + commandArgs = append([]string(nil), cmd.Args...) + } + wrapCrossCompileHooks([]*modules.Module{mod}, rewriter, io.Discard, io.Discard) + mod.OnBuild(formula.NewContext(nil, t.TempDir(), "", "", nil)) + + if commandName != "/llvm/bin/clang" { + t.Fatalf("command path = %q", commandName) + } + want := []string{"/llvm/bin/clang", "--target=aarch64-linux-gnu", "--sysroot=/sdk", "-c", "a.c"} + if !reflect.DeepEqual(commandArgs, want) { + t.Fatalf("command args = %q, want %q", commandArgs, want) + } + if cmd := execbroker.Command("cc"); cmd.Path == "/llvm/bin/clang" { + t.Fatal("cross compile middleware leaked after Formula hook") + } +} + +func TestNewCrossCompileToolchainUsesPreparedPath(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"clang", "clang++", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + path := dir + string(os.PathSeparator) + name + if err := os.WriteFile(path, []byte("tool"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir) + toolchain, err := newCrossCompileToolchain() + if err != nil { + t.Fatal(err) + } + if toolchain.CC() != dir+string(os.PathSeparator)+"clang" { + t.Fatalf("CC = %q", toolchain.CC()) + } +} diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index c16f343..499f445 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -158,6 +158,21 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } + rewriter, sysrootMod, err := prepareCrossCompile( + ctx, + store, + module.Version{Path: mods[0].Path, Version: mods[0].Version}, + matrix, + buildOpts, + ) + if err != nil { + return err + } + if rewriter != nil { + defer rewriter.Close() + wrapCrossCompileHooks(mods, rewriter, buildOutput, buildOutput) + mods = injectSysroot(mods, sysrootMod) + } builder, err := build.NewBuilder(buildOpts) if err != nil { diff --git a/internal/build/crosscompile/crosscompile.go b/internal/build/crosscompile/crosscompile.go new file mode 100644 index 0000000..d9436f6 --- /dev/null +++ b/internal/build/crosscompile/crosscompile.go @@ -0,0 +1,361 @@ +package crosscompile + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/goplus/llar/mod/module" + ccmetadata "github.com/goplus/llar/x/metadata/cc" +) + +const ( + linuxSysrootPath = "bminor/glibc" + linuxSysrootVersion = "glibc-2.17" +) + +// Toolchain supplies prepared C-family tools for the build host. +type Toolchain interface { + CC() string + CXX() string + Archiver() string + Ranlib() string + NM() string + Strip() string +} + +// Command describes a command before cross-compilation defaults are applied. +type Command struct { + Name string + Args []string + Env []string + Dir string +} + +// Patch contains the changes required for one command. +type Patch struct { + Name string + PrependArg []string + AppendArg []string + Env []string +} + +// CrossCompile contains prepared command-rewrite facts for one target. +type CrossCompile struct { + systemProcessor string + targetTriple string + sysroot string + toolchain Toolchain + toolchainFile string + tempDir string +} + +// Sysroot returns the fixed compatibility sysroot Formula for a built-in +// Linux target. +func Sysroot(targetOS, targetArch string) (module.Version, bool) { + if targetOS != "linux" { + return module.Version{}, false + } + switch targetArch { + case "amd64", "arm64": + return module.Version{Path: linuxSysrootPath, Version: linuxSysrootVersion}, true + default: + return module.Version{}, false + } +} + +// New prepares command rewriting for a built-in Linux target. +func New(targetOS, targetArch string, toolchain Toolchain, sysrootMetadata string) (*CrossCompile, error) { + triple, processor, err := linuxTarget(targetOS, targetArch) + if err != nil { + return nil, err + } + if err := validateToolchain(toolchain); err != nil { + return nil, fmt.Errorf("prepare cross compiler for %s/%s: %w", targetOS, targetArch, err) + } + metadata, err := ccmetadata.Parse(sysrootMetadata) + if err != nil { + return nil, fmt.Errorf("parse sysroot metadata for %s/%s: %w", targetOS, targetArch, err) + } + if metadata.Sysroot() == "" { + return nil, fmt.Errorf("sysroot metadata for %s/%s has no sysroot", targetOS, targetArch) + } + + tempDir, err := os.MkdirTemp("", "llar-crosscompile-*") + if err != nil { + return nil, fmt.Errorf("prepare CMake toolchain for %s/%s: %w", targetOS, targetArch, err) + } + c := &CrossCompile{ + systemProcessor: processor, + targetTriple: triple, + sysroot: metadata.Sysroot(), + toolchain: toolchain, + tempDir: tempDir, + } + c.toolchainFile = filepath.Join(tempDir, "toolchain.cmake") + if err := os.WriteFile(c.toolchainFile, []byte(c.cmakeToolchain()), 0o600); err != nil { + _ = os.RemoveAll(tempDir) + return nil, fmt.Errorf("prepare CMake toolchain for %s/%s: %w", targetOS, targetArch, err) + } + return c, nil +} + +// Close removes generated build-system configuration. +func (c *CrossCompile) Close() error { + return os.RemoveAll(c.tempDir) +} + +// Use returns cross-compilation defaults for cmd. Explicit Formula settings +// are preserved. +func (c *CrossCompile) Use(cmd Command) Patch { + base := filepath.Base(cmd.Name) + if base == "configure" { + return c.autotoolsPatch(cmd) + } + if filepath.Base(cmd.Name) != cmd.Name { + return Patch{} + } + + switch base { + case "cmake": + if isCMakeConfigure(cmd.Args) && !hasCMakeToolchain(cmd.Args) { + return Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} + } + case "pkg-config": + return c.pkgConfigPatch(cmd.Env) + case "cc", "gcc", "clang": + return c.compilerPatch(c.toolchain.CC(), cmd.Args) + case "c++", "g++", "clang++": + return c.compilerPatch(c.toolchain.CXX(), cmd.Args) + case "ar", "llvm-ar": + return Patch{Name: c.toolchain.Archiver()} + case "ranlib", "llvm-ranlib": + return Patch{Name: c.toolchain.Ranlib()} + case "nm", "llvm-nm": + return Patch{Name: c.toolchain.NM()} + case "strip", "llvm-strip": + return Patch{Name: c.toolchain.Strip()} + } + return Patch{} +} + +func (c *CrossCompile) compilerPatch(name string, args []string) Patch { + flags := missingCompilerFlags(args, c.targetTriple, c.sysroot) + return Patch{Name: name, PrependArg: flags} +} + +func (c *CrossCompile) autotoolsPatch(cmd Command) Patch { + env := append([]string(nil), cmd.Env...) + env = setMissingEnv(env, "CC", c.toolchain.CC()) + env = setMissingEnv(env, "CXX", c.toolchain.CXX()) + env = setMissingEnv(env, "AR", c.toolchain.Archiver()) + env = setMissingEnv(env, "RANLIB", c.toolchain.Ranlib()) + env = setMissingEnv(env, "NM", c.toolchain.NM()) + env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) + + compilerFlags := []string{"--target=" + c.targetTriple, "--sysroot=" + c.sysroot} + env = setEnvFlags(env, "CFLAGS", compilerFlags) + env = setEnvFlags(env, "CXXFLAGS", compilerFlags) + env = setEnvFlags(env, "CPPFLAGS", []string{"--sysroot=" + c.sysroot}) + env = setEnvFlags(env, "LDFLAGS", compilerFlags) + + var args []string + if !hasOption(cmd.Args, "--host") { + args = append(args, "--host="+c.targetTriple) + } + return Patch{AppendArg: args, Env: env} +} + +func (c *CrossCompile) pkgConfigPatch(commandEnv []string) Patch { + env := append([]string(nil), commandEnv...) + env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) + libDirs, _ := envValue(env, "PKG_CONFIG_PATH") + paths := filepath.SplitList(libDirs) + paths = append(paths, + filepath.Join(c.sysroot, "usr", "lib", c.targetTriple, "pkgconfig"), + filepath.Join(c.sysroot, "usr", "lib64", "pkgconfig"), + filepath.Join(c.sysroot, "usr", "lib", "pkgconfig"), + filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), + ) + env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) + return Patch{Env: env} +} + +func (c *CrossCompile) cmakeToolchain() string { + values := [][2]string{ + {"CMAKE_SYSTEM_NAME", "Linux"}, + {"CMAKE_SYSTEM_PROCESSOR", c.systemProcessor}, + {"CMAKE_SYSROOT", c.sysroot}, + {"CMAKE_C_COMPILER", c.toolchain.CC()}, + {"CMAKE_CXX_COMPILER", c.toolchain.CXX()}, + {"CMAKE_AR", c.toolchain.Archiver()}, + {"CMAKE_RANLIB", c.toolchain.Ranlib()}, + {"CMAKE_NM", c.toolchain.NM()}, + {"CMAKE_STRIP", c.toolchain.Strip()}, + {"CMAKE_C_COMPILER_TARGET", c.targetTriple}, + {"CMAKE_CXX_COMPILER_TARGET", c.targetTriple}, + {"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER"}, + {"CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY"}, + {"CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY"}, + {"CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY"}, + } + var out strings.Builder + for _, value := range values { + fmt.Fprintf(&out, "if(NOT DEFINED %s)\n set(%s \"%s\")\nendif()\n", value[0], value[0], cmakeEscape(value[1])) + } + return out.String() +} + +func linuxTarget(targetOS, targetArch string) (triple, processor string, err error) { + if targetOS != "linux" { + return "", "", fmt.Errorf("unsupported cross compile target %s/%s", targetOS, targetArch) + } + switch targetArch { + case "amd64": + return "x86_64-linux-gnu", "x86_64", nil + case "arm64": + return "aarch64-linux-gnu", "aarch64", nil + default: + return "", "", fmt.Errorf("unsupported cross compile target %s/%s", targetOS, targetArch) + } +} + +func validateToolchain(toolchain Toolchain) error { + if toolchain == nil { + return fmt.Errorf("toolchain is required") + } + tools := []struct { + name string + path string + }{ + {"CC", toolchain.CC()}, + {"CXX", toolchain.CXX()}, + {"archiver", toolchain.Archiver()}, + {"ranlib", toolchain.Ranlib()}, + {"nm", toolchain.NM()}, + {"strip", toolchain.Strip()}, + } + for _, tool := range tools { + if tool.path == "" { + return fmt.Errorf("%s is required", tool.name) + } + } + return nil +} + +func isCMakeConfigure(args []string) bool { + if len(args) == 0 { + return true + } + switch args[0] { + case "--build", "--install", "--open", "--workflow": + return false + default: + return true + } +} + +func hasCMakeToolchain(args []string) bool { + for _, arg := range args { + if strings.HasPrefix(arg, "-DCMAKE_TOOLCHAIN_FILE") || arg == "--toolchain" || strings.HasPrefix(arg, "--toolchain=") { + return true + } + } + return false +} + +func missingCompilerFlags(args []string, triple, sysroot string) []string { + var flags []string + if !hasTargetFlag(args) { + flags = append(flags, "--target="+triple) + } + if !hasSysrootFlag(args) { + flags = append(flags, "--sysroot="+sysroot) + } + return flags +} + +func hasTargetFlag(args []string) bool { + return hasFlag(args, "--target", "-target") +} + +func hasSysrootFlag(args []string) bool { + return hasFlag(args, "--sysroot", "-sysroot", "-isysroot") +} + +func hasFlag(args []string, names ...string) bool { + for _, arg := range args { + for _, name := range names { + if arg == name || strings.HasPrefix(arg, name+"=") || name == "-isysroot" && strings.HasPrefix(arg, name) { + return true + } + } + } + return false +} + +func hasOption(args []string, name string) bool { + for _, arg := range args { + if arg == name || strings.HasPrefix(arg, name+"=") { + return true + } + } + return false +} + +func envValue(env []string, key string) (string, bool) { + prefix := key + "=" + for i := len(env) - 1; i >= 0; i-- { + if strings.HasPrefix(env[i], prefix) { + return strings.TrimPrefix(env[i], prefix), true + } + } + return "", false +} + +func setEnv(env []string, key, value string) []string { + prefix := key + "=" + for i := len(env) - 1; i >= 0; i-- { + if strings.HasPrefix(env[i], prefix) { + env[i] = prefix + value + return env + } + } + return append(env, prefix+value) +} + +func setMissingEnv(env []string, key, value string) []string { + if _, ok := envValue(env, key); ok { + return env + } + return append(env, key+"="+value) +} + +func setEnvFlags(env []string, key string, defaults []string) []string { + value, _ := envValue(env, key) + original := value + args := strings.Fields(value) + for _, flag := range defaults { + if strings.HasPrefix(flag, "--target=") && hasTargetFlag(args) { + continue + } + if strings.HasPrefix(flag, "--sysroot=") && hasSysrootFlag(args) { + continue + } + if value != "" { + value += " " + } + value += flag + args = append(args, flag) + } + if value != original { + return setEnv(env, key, value) + } + return env +} + +func cmakeEscape(value string) string { + value = strings.ReplaceAll(value, "\\", "/") + return strings.ReplaceAll(value, "\"", "\\\"") +} diff --git a/internal/build/crosscompile/crosscompile_test.go b/internal/build/crosscompile/crosscompile_test.go new file mode 100644 index 0000000..7eb8682 --- /dev/null +++ b/internal/build/crosscompile/crosscompile_test.go @@ -0,0 +1,149 @@ +package crosscompile + +import ( + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/goplus/llar/mod/module" +) + +type fakeToolchain struct{} + +func (fakeToolchain) CC() string { return "/llvm/bin/clang" } +func (fakeToolchain) CXX() string { return "/llvm/bin/clang++" } +func (fakeToolchain) Archiver() string { return "/llvm/bin/llvm-ar" } +func (fakeToolchain) Ranlib() string { return "/llvm/bin/llvm-ranlib" } +func (fakeToolchain) NM() string { return "/llvm/bin/llvm-nm" } +func (fakeToolchain) Strip() string { return "/llvm/bin/llvm-strip" } + +func TestSysroot(t *testing.T) { + want := module.Version{Path: "bminor/glibc", Version: "glibc-2.17"} + for _, arch := range []string{"amd64", "arm64"} { + got, ok := Sysroot("linux", arch) + if !ok || got != want { + t.Fatalf("Sysroot(linux, %s) = %+v, %v; want %+v, true", arch, got, ok, want) + } + } + for _, target := range [][2]string{{"darwin", "arm64"}, {"linux", "riscv64"}, {"", "esp32"}} { + if got, ok := Sysroot(target[0], target[1]); ok { + t.Fatalf("Sysroot(%q, %q) = %+v, true; want unsupported", target[0], target[1], got) + } + } +} + +func TestNewRequiresSysrootMetadata(t *testing.T) { + if _, err := New("linux", "arm64", fakeToolchain{}, "-L/lib"); err == nil || !strings.Contains(err.Error(), "has no sysroot") { + t.Fatalf("New error = %v, want missing sysroot", err) + } +} + +func TestNewWritesCMakeToolchain(t *testing.T) { + c, err := New("linux", "arm64", fakeToolchain{}, "--sysroot=/sdk") + if err != nil { + t.Fatal(err) + } + path := c.toolchainFile + t.Cleanup(func() { _ = c.Close() }) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + content := string(data) + for _, want := range []string{"CMAKE_SYSTEM_NAME", "aarch64", "/llvm/bin/clang", "aarch64-linux-gnu", "/sdk"} { + if !strings.Contains(content, want) { + t.Fatalf("toolchain file does not contain %q:\n%s", want, content) + } + } + if err := c.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("toolchain file still exists after Close: %v", err) + } +} + +func TestUseCMake(t *testing.T) { + c := newTestCrossCompile(t) + patch := c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { + t.Fatalf("AppendArg = %q, want %q", got, want) + } + patch = c.Use(Command{Name: "cmake", Args: []string{"--build", "build"}}) + if len(patch.AppendArg) != 0 { + t.Fatalf("build Patch = %+v, want no toolchain argument", patch) + } + patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) + if len(patch.AppendArg) != 0 { + t.Fatalf("explicit toolchain Patch = %+v", patch) + } +} + +func TestUseDirectCommands(t *testing.T) { + c := newTestCrossCompile(t) + patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) + if patch.Name != "/llvm/bin/clang" { + t.Fatalf("Name = %q", patch.Name) + } + want := []string{"--target=aarch64-linux-gnu", "--sysroot=/sdk"} + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) + } + patch = c.Use(Command{Name: "cc", Args: []string{"--target=custom", "--sysroot=/custom"}}) + if len(patch.PrependArg) != 0 { + t.Fatalf("explicit compiler flags were duplicated: %q", patch.PrependArg) + } + if patch := c.Use(Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { + t.Fatalf("explicit compiler path was rewritten: %+v", patch) + } +} + +func TestUseAutotools(t *testing.T) { + c := newTestCrossCompile(t) + patch := c.Use(Command{ + Name: "/src/configure", + Args: []string{"--build=x86_64-apple-darwin"}, + Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, + }) + if got, _ := envValue(patch.Env, "CC"); got != "/custom/cc" { + t.Fatalf("CC override = %q, want /custom/cc", got) + } + if got, _ := envValue(patch.Env, "CFLAGS"); got != "-O2 --target=custom --sysroot=/sdk" { + t.Fatalf("CFLAGS = %q", got) + } + if got, want := patch.AppendArg, []string{"--host=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { + t.Fatalf("AppendArg = %q, want %q", got, want) + } +} + +func TestUsePkgConfig(t *testing.T) { + c := newTestCrossCompile(t) + depPaths := strings.Join([]string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig"}, string(os.PathListSeparator)) + patch := c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) + if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) + } + got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") + for _, want := range []string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "aarch64-linux-gnu", "pkgconfig")} { + if !slices.Contains(filepath.SplitList(got), want) { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) + } + } + patch = c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) + if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/custom" { + t.Fatalf("PKG_CONFIG_LIBDIR override = %q, want /custom", got) + } +} + +func newTestCrossCompile(t *testing.T) *CrossCompile { + t.Helper() + c, err := New("linux", "arm64", fakeToolchain{}, "--sysroot=/sdk") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + return c +} diff --git a/x/cmake/cmake.go b/x/cmake/cmake.go index 1c1625e..f0575af 100644 --- a/x/cmake/cmake.go +++ b/x/cmake/cmake.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "sort" + "strings" "github.com/goplus/llar/internal/execbroker" ) @@ -81,6 +82,7 @@ func (c *CMake) Use(root string) { } } prependPath("CMAKE_PREFIX_PATH", root) + prependPath("CMAKE_FIND_ROOT_PATH", root) if hasInclude { prependPath("CMAKE_INCLUDE_PATH", includeDir) } @@ -124,6 +126,9 @@ func (c *CMake) Configure(args ...string) { if c.buildType != "" { cmakeArgs = append(cmakeArgs, "-DCMAKE_BUILD_TYPE:STRING="+c.buildType) } + if roots := os.Getenv("CMAKE_FIND_ROOT_PATH"); roots != "" { + cmakeArgs = append(cmakeArgs, "-DCMAKE_FIND_ROOT_PATH:STRING="+strings.Join(filepath.SplitList(roots), ";")) + } cmakeArgs = append(cmakeArgs, c.definesArgs()...) cmakeArgs = append(cmakeArgs, args...) c.run("cmake", cmakeArgs) diff --git a/x/cmake/cmake_test.go b/x/cmake/cmake_test.go index 1b87ece..45f3113 100644 --- a/x/cmake/cmake_test.go +++ b/x/cmake/cmake_test.go @@ -21,7 +21,7 @@ func TestUseSetsEnv(t *testing.T) { } for _, key := range []string{ - "PKG_CONFIG_PATH", "CMAKE_PREFIX_PATH", "CMAKE_INCLUDE_PATH", + "PKG_CONFIG_PATH", "CMAKE_PREFIX_PATH", "CMAKE_FIND_ROOT_PATH", "CMAKE_INCLUDE_PATH", "CMAKE_LIBRARY_PATH", "INCLUDE", "LIB", "CPPFLAGS", "LDFLAGS", } { t.Setenv(key, "") @@ -31,10 +31,11 @@ func TestUseSetsEnv(t *testing.T) { c.Use(root) for key, want := range map[string]string{ - "PKG_CONFIG_PATH": pkgconfigDir, - "CMAKE_PREFIX_PATH": root, - "CMAKE_INCLUDE_PATH": includeDir, - "CMAKE_LIBRARY_PATH": libDir, + "PKG_CONFIG_PATH": pkgconfigDir, + "CMAKE_PREFIX_PATH": root, + "CMAKE_FIND_ROOT_PATH": root, + "CMAKE_INCLUDE_PATH": includeDir, + "CMAKE_LIBRARY_PATH": libDir, } { if got := os.Getenv(key); got != want { t.Errorf("%s = %q, want %q", key, got, want) @@ -79,6 +80,18 @@ func TestUsePartialDirs(t *testing.T) { } } +func TestUseMultipleRoots(t *testing.T) { + t.Setenv("CMAKE_FIND_ROOT_PATH", "") + c := New("", "", "") + c.Use("/deps/one") + c.Use("/deps/two") + + sep := string(os.PathListSeparator) + if got, want := os.Getenv("CMAKE_FIND_ROOT_PATH"), "/deps/two"+sep+"/deps/one"; got != want { + t.Fatalf("CMAKE_FIND_ROOT_PATH = %q, want %q", got, want) + } +} + func TestOutputDir(t *testing.T) { if got := New("", "build", "").OutputDir(); got != "build" { t.Errorf("OutputDir = %q, want %q", got, "build") From 8d9c899183a2d7c3a0751096a9bc3600e2ae1295 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 00:13:07 +0800 Subject: [PATCH 02/39] feat(crosscompile): align C target build flow --- cmd/llar/internal/crosscompile.go | 186 ++--------------- cmd/llar/internal/crosscompile_test.go | 122 +++-------- cmd/llar/internal/make.go | 83 ++++++-- internal/build/build.go | 17 +- internal/build/c/llvm/toolchain.go | 49 +++++ internal/build/c/llvm/toolchain_test.go | 25 +++ .../crosscompile.go => c/target.go} | 195 +++++++++--------- .../crosscompile_test.go => c/target_test.go} | 85 +++++--- internal/build/c/toolchain.go | 30 +++ internal/build/c/toolchain_test.go | 23 +++ internal/build/http/http.go | 4 +- internal/build/http/http_test.go | 8 +- internal/build/target.go | 56 +++++ internal/build/target_test.go | 78 +++++++ internal/modules/load.go | 3 + internal/modules/load_test.go | 20 ++ 16 files changed, 563 insertions(+), 421 deletions(-) create mode 100644 internal/build/c/llvm/toolchain.go create mode 100644 internal/build/c/llvm/toolchain_test.go rename internal/build/{crosscompile/crosscompile.go => c/target.go} (61%) rename internal/build/{crosscompile/crosscompile_test.go => c/target_test.go} (59%) create mode 100644 internal/build/c/toolchain.go create mode 100644 internal/build/c/toolchain_test.go create mode 100644 internal/build/target.go create mode 100644 internal/build/target_test.go diff --git a/cmd/llar/internal/crosscompile.go b/cmd/llar/internal/crosscompile.go index 7d8e780..0c9d1f6 100644 --- a/cmd/llar/internal/crosscompile.go +++ b/cmd/llar/internal/crosscompile.go @@ -1,119 +1,37 @@ package internal import ( - "context" - "fmt" - "io" - "os" - "os/exec" "runtime" "github.com/goplus/llar/formula" - "github.com/goplus/llar/internal/build" - "github.com/goplus/llar/internal/build/crosscompile" - "github.com/goplus/llar/internal/execbroker" - "github.com/goplus/llar/internal/formula/repo" - "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/internal/build/c" "github.com/goplus/llar/mod/module" ) -type llvmPathToolchain struct { - cc string - cxx string - archiver string - ranlib string - nm string - strip string -} - -func (t llvmPathToolchain) CC() string { return t.cc } -func (t llvmPathToolchain) CXX() string { return t.cxx } -func (t llvmPathToolchain) Archiver() string { return t.archiver } -func (t llvmPathToolchain) Ranlib() string { return t.ranlib } -func (t llvmPathToolchain) NM() string { return t.nm } -func (t llvmPathToolchain) Strip() string { return t.strip } - -var newCrossCompileToolchain = func() (crosscompile.Toolchain, error) { - find := func(name string) (string, error) { - path, err := exec.LookPath(name) - if err != nil { - return "", fmt.Errorf("find prepared LLVM command %q: %w", name, err) - } - return path, nil - } - cc, err := find("clang") - if err != nil { - return nil, err - } - cxx, err := find("clang++") - if err != nil { - return nil, err - } - archiver, err := find("llvm-ar") - if err != nil { - return nil, err - } - ranlib, err := find("llvm-ranlib") - if err != nil { - return nil, err - } - nm, err := find("llvm-nm") - if err != nil { - return nil, err +func crossCompileTarget(matrix formula.Matrix) (string, bool) { + targetOS, targetArch := matrixTarget(matrix) + if targetOS == runtime.GOOS && targetArch == runtime.GOARCH { + return "", false } - strip, err := find("llvm-strip") - if err != nil { - return nil, err + if _, ok := c.Sysroot(targetOS, targetArch); !ok { + return "", false } - return llvmPathToolchain{ - cc: cc, - cxx: cxx, - archiver: archiver, - ranlib: ranlib, - nm: nm, - strip: strip, - }, nil + return targetArch + "-" + targetOS, true } -func prepareCrossCompile( - ctx context.Context, - store repo.Store, +func crossCompileSysroot( root module.Version, matrix formula.Matrix, - buildOpts build.Options, -) (*crosscompile.CrossCompile, *modules.Module, error) { +) (module.Version, bool) { + if _, ok := matrix.Require["libc"]; ok { + return module.Version{}, false + } targetOS, targetArch := matrixTarget(matrix) - sysroot, ok := crosscompile.Sysroot(targetOS, targetArch) + sysroot, ok := c.Sysroot(targetOS, targetArch) if !ok || targetOS == runtime.GOOS && targetArch == runtime.GOARCH || root.Path == sysroot.Path { - return nil, nil, nil - } - - sysrootMods, err := modules.Load(ctx, sysroot, modules.Options{ - FormulaStore: store, - Matrix: matrix, - }) - if err != nil { - return nil, nil, fmt.Errorf("load cross compile sysroot %s@%s: %w", sysroot.Path, sysroot.Version, err) - } - buildOpts.RunTest = false - builder, err := build.NewBuilder(buildOpts) - if err != nil { - return nil, nil, fmt.Errorf("create sysroot builder: %w", err) - } - results, err := builder.Build(ctx, sysrootMods) - if err != nil { - return nil, nil, fmt.Errorf("build cross compile sysroot %s@%s: %w", sysroot.Path, sysroot.Version, err) - } - - toolchain, err := newCrossCompileToolchain() - if err != nil { - return nil, nil, err - } - rewriter, err := crosscompile.New(targetOS, targetArch, toolchain, results[len(results)-1].Metadata) - if err != nil { - return nil, nil, fmt.Errorf("prepare cross compile target %s/%s: %w", targetOS, targetArch, err) + return module.Version{}, false } - return rewriter, sysrootMods[0], nil + return sysroot, true } func matrixTarget(matrix formula.Matrix) (targetOS, targetArch string) { @@ -125,75 +43,3 @@ func matrixTarget(matrix formula.Matrix) (targetOS, targetArch string) { } return } - -func injectSysroot(mods []*modules.Module, sysroot *modules.Module) []*modules.Module { - for _, mod := range mods { - mod.Deps = append(mod.Deps, sysroot) - } - return append(mods, sysroot) -} - -func wrapCrossCompileHooks(mods []*modules.Module, rewriter *crosscompile.CrossCompile, stdout, stderr io.Writer) { - middleware := func(req execbroker.Request) execbroker.Request { - patch := rewriter.Use(crosscompile.Command{ - Name: req.Name, - Args: req.Args, - Env: effectiveCommandEnv(req.Env), - Dir: req.Dir, - }) - return applyCrossCompilePatch(req, patch) - } - for _, mod := range mods { - if hook := mod.OnBuild; hook != nil { - mod.OnBuild = func(ctx *formula.Context) { - _ = execbroker.Do(execbroker.Scope{ - Dir: ctx.SourceDir, - Stdin: os.Stdin, - Stdout: stdout, - Stderr: stderr, - Middleware: middleware, - }, func() error { - hook(ctx) - return nil - }) - } - } - if hook := mod.OnTest; hook != nil { - mod.OnTest = func(ctx *formula.Context) { - _ = execbroker.Do(execbroker.Scope{ - Dir: ctx.SourceDir, - Stdin: os.Stdin, - Stdout: stdout, - Stderr: stderr, - Middleware: middleware, - }, func() error { - hook(ctx) - return nil - }) - } - } - } -} - -func applyCrossCompilePatch(req execbroker.Request, patch crosscompile.Patch) execbroker.Request { - if patch.Name != "" { - req.Name = patch.Name - } - if len(patch.PrependArg) > 0 { - req.Args = append(append([]string(nil), patch.PrependArg...), req.Args...) - } - if len(patch.AppendArg) > 0 { - req.Args = append(req.Args, patch.AppendArg...) - } - if patch.Env != nil { - req.Env = append([]string(nil), patch.Env...) - } - return req -} - -func effectiveCommandEnv(env []string) []string { - if env != nil { - return append([]string(nil), env...) - } - return os.Environ() -} diff --git a/cmd/llar/internal/crosscompile_test.go b/cmd/llar/internal/crosscompile_test.go index e030dd9..bdb145e 100644 --- a/cmd/llar/internal/crosscompile_test.go +++ b/cmd/llar/internal/crosscompile_test.go @@ -1,26 +1,13 @@ package internal import ( - "io" - "os" - "reflect" + "runtime" "testing" "github.com/goplus/llar/formula" - "github.com/goplus/llar/internal/build/crosscompile" - "github.com/goplus/llar/internal/execbroker" - "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/mod/module" ) -type testLLVMToolchain struct{} - -func (testLLVMToolchain) CC() string { return "/llvm/bin/clang" } -func (testLLVMToolchain) CXX() string { return "/llvm/bin/clang++" } -func (testLLVMToolchain) Archiver() string { return "/llvm/bin/llvm-ar" } -func (testLLVMToolchain) Ranlib() string { return "/llvm/bin/llvm-ranlib" } -func (testLLVMToolchain) NM() string { return "/llvm/bin/llvm-nm" } -func (testLLVMToolchain) Strip() string { return "/llvm/bin/llvm-strip" } - func TestMatrixTarget(t *testing.T) { matrix := formula.Matrix{Require: map[string][]string{ "os": {"linux"}, @@ -31,91 +18,40 @@ func TestMatrixTarget(t *testing.T) { } } -func TestInjectSysroot(t *testing.T) { - root := &modules.Module{Path: "owner/root", Version: "v1"} - dep := &modules.Module{Path: "owner/dep", Version: "v2"} - sysroot := &modules.Module{Path: "bminor/glibc", Version: "glibc-2.17"} - root.Deps = []*modules.Module{dep} - - got := injectSysroot([]*modules.Module{root, dep}, sysroot) - if want := []*modules.Module{root, dep, sysroot}; !reflect.DeepEqual(got, want) { - t.Fatalf("modules = %+v, want %+v", got, want) - } - if want := []*modules.Module{dep, sysroot}; !reflect.DeepEqual(root.Deps, want) { - t.Fatalf("root deps = %+v, want %+v", root.Deps, want) - } - if want := []*modules.Module{sysroot}; !reflect.DeepEqual(dep.Deps, want) { - t.Fatalf("dep deps = %+v, want %+v", dep.Deps, want) - } -} - -func TestApplyCrossCompilePatch(t *testing.T) { - req := execbroker.Request{ - Name: "cc", - Args: []string{"-c", "a.c"}, - Env: []string{"CFLAGS=-O2"}, - } - got := applyCrossCompilePatch(req, crosscompile.Patch{ - Name: "/llvm/bin/clang", - PrependArg: []string{"--target=aarch64-linux-gnu"}, - AppendArg: []string{"--sysroot=/sdk"}, - Env: []string{"CFLAGS=-O2 --sysroot=/sdk"}, - }) - if got.Name != "/llvm/bin/clang" { - t.Fatalf("Name = %q", got.Name) +func TestCrossCompileTarget(t *testing.T) { + targetArch := "amd64" + if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { + targetArch = "arm64" } - if want := []string{"--target=aarch64-linux-gnu", "-c", "a.c", "--sysroot=/sdk"}; !reflect.DeepEqual(got.Args, want) { - t.Fatalf("Args = %q, want %q", got.Args, want) + matrix := formula.Matrix{Require: map[string][]string{ + "os": {"linux"}, + "arch": {targetArch}, + }} + if got, ok := crossCompileTarget(matrix); !ok || got != targetArch+"-linux" { + t.Fatalf("crossCompileTarget = %q, %v; want %q, true", got, ok, targetArch+"-linux") } - if want := []string{"CFLAGS=-O2 --sysroot=/sdk"}; !reflect.DeepEqual(got.Env, want) { - t.Fatalf("Env = %q, want %q", got.Env, want) + matrix.Require["libc"] = []string{"glibc-2.13"} + if got, ok := crossCompileTarget(matrix); !ok || got != targetArch+"-linux" { + t.Fatalf("crossCompileTarget with libc = %q, %v; want %q, true", got, ok, targetArch+"-linux") } } -func TestWrapCrossCompileHooks(t *testing.T) { - rewriter, err := crosscompile.New("linux", "arm64", testLLVMToolchain{}, "--sysroot=/sdk") - if err != nil { - t.Fatal(err) +func TestCrossCompileSysrootSkipsDefaultForLibcRequirement(t *testing.T) { + targetArch := "amd64" + if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { + targetArch = "arm64" } - t.Cleanup(func() { _ = rewriter.Close() }) - - var commandName string - var commandArgs []string - mod := &modules.Module{Path: "owner/root", Version: "v1"} - mod.OnBuild = func(*formula.Context) { - cmd := execbroker.Command("cc", "-c", "a.c") - commandName = cmd.Path - commandArgs = append([]string(nil), cmd.Args...) - } - wrapCrossCompileHooks([]*modules.Module{mod}, rewriter, io.Discard, io.Discard) - mod.OnBuild(formula.NewContext(nil, t.TempDir(), "", "", nil)) - - if commandName != "/llvm/bin/clang" { - t.Fatalf("command path = %q", commandName) - } - want := []string{"/llvm/bin/clang", "--target=aarch64-linux-gnu", "--sysroot=/sdk", "-c", "a.c"} - if !reflect.DeepEqual(commandArgs, want) { - t.Fatalf("command args = %q, want %q", commandArgs, want) - } - if cmd := execbroker.Command("cc"); cmd.Path == "/llvm/bin/clang" { - t.Fatal("cross compile middleware leaked after Formula hook") - } -} + matrix := formula.Matrix{Require: map[string][]string{ + "os": {"linux"}, + "arch": {targetArch}, + }} + root := module.Version{Path: "owner/root", Version: "v1"} -func TestNewCrossCompileToolchainUsesPreparedPath(t *testing.T) { - dir := t.TempDir() - for _, name := range []string{"clang", "clang++", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { - path := dir + string(os.PathSeparator) + name - if err := os.WriteFile(path, []byte("tool"), 0o755); err != nil { - t.Fatal(err) - } - } - t.Setenv("PATH", dir) - toolchain, err := newCrossCompileToolchain() - if err != nil { - t.Fatal(err) + if _, ok := crossCompileSysroot(root, matrix); !ok { + t.Fatal("crossCompileSysroot did not select the default sysroot") } - if toolchain.CC() != dir+string(os.PathSeparator)+"clang" { - t.Fatalf("CC = %q", toolchain.CC()) + matrix.Require["libc"] = nil + if got, ok := crossCompileSysroot(root, matrix); ok || got != (module.Version{}) { + t.Fatalf("crossCompileSysroot = %+v, %v; want no default sysroot", got, ok) } } diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 499f445..7193f79 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -12,11 +12,14 @@ import ( "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/c" + "github.com/goplus/llar/internal/build/c/llvm" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/modules/modlocal" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" + ccmetadata "github.com/goplus/llar/x/metadata/cc" "github.com/spf13/cobra" ) @@ -129,10 +132,17 @@ func hostMatrix() formula.Matrix { // hooks triggered — each dependency is verified by its own // `llar test ` invocation. func buildModule(ctx context.Context, store repo.Store, modPath, version string, matrix formula.Matrix, runTest bool) error { - mods, err := modules.Load(ctx, module.Version{Path: modPath, Version: version}, modules.Options{ + root := module.Version{Path: modPath, Version: version} + targetMatrix, crossCompile := crossCompileTarget(matrix) + sysroot, useDefaultSysroot := crossCompileSysroot(root, matrix) + loadOpts := modules.Options{ FormulaStore: store, Matrix: matrix, - }) + } + if useDefaultSysroot { + loadOpts.Roots = []module.Version{sysroot} + } + mods, err := modules.Load(ctx, root, loadOpts) if err != nil { return fmt.Errorf("failed to load modules: %w", err) } @@ -158,22 +168,61 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } - rewriter, sysrootMod, err := prepareCrossCompile( - ctx, - store, - module.Version{Path: mods[0].Path, Version: mods[0].Version}, - matrix, - buildOpts, - ) - if err != nil { - return err - } - if rewriter != nil { - defer rewriter.Close() - wrapCrossCompileHooks(mods, rewriter, buildOutput, buildOutput) - mods = injectSysroot(mods, sysrootMod) - } + if crossCompile { + llvmToolchain, err := llvm.New() + if err != nil { + return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + bootstrapTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: llvmToolchain.Toolchain, + }) + if err != nil { + return err + } + defer bootstrapTarget.Close() + buildOpts.Target = bootstrapTarget + if useDefaultSysroot { + // A successful Load includes every Options.Roots path in the selected graph. + var selectedSysroot *modules.Module + for _, mod := range mods { + if mod.Path == sysroot.Path { + selectedSysroot = mod + break + } + } + + sysrootOpts := buildOpts + sysrootOpts.RunTest = false + sysrootBuilder, err := build.NewBuilder(sysrootOpts) + if err != nil { + return fmt.Errorf("failed to create sysroot builder: %w", err) + } + sysrootResults, err := sysrootBuilder.Build(ctx, []*modules.Module{selectedSysroot}) + if err != nil { + return fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + metadata, err := ccmetadata.Parse(sysrootResults[0].Metadata) + if err != nil { + return fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + if metadata.Sysroot() == "" { + return fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) + } + + configuredTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: llvmToolchain.Toolchain, + Sysroot: metadata.Sysroot(), + }) + if err != nil { + return err + } + defer configuredTarget.Close() + buildOpts.Target = configuredTarget + } + } builder, err := build.NewBuilder(buildOpts) if err != nil { return fmt.Errorf("failed to create builder: %w", err) diff --git a/internal/build/build.go b/internal/build/build.go index 47268a3..ee5e0c3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -28,6 +28,7 @@ type Builder struct { stderr io.Writer workspaceDir string cache cache.Cache + target Target newRepo func(repoPath string) (vcs.Repo, error) // defaults to vcs.NewRepo } @@ -51,6 +52,7 @@ type Options struct { Stderr io.Writer WorkspaceDir string Cache cache.Cache + Target Target } func runFormulaHook(fn func()) (err error) { @@ -110,6 +112,7 @@ func NewBuilder(opts Options) (*Builder, error) { stderr: stderr, workspaceDir: workspaceDir, cache: c, + target: opts.Target, newRepo: vcs.NewRepo, }, nil } @@ -243,6 +246,11 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul rootID = module.Version{Path: targets[0].Path, Version: targets[0].Version} } + var commandMiddleware execbroker.Middleware + if b.target != nil { + commandMiddleware = targetMiddleware(b.target) + } + build := func(mod *modules.Module) (Result, error) { isRoot := mod.Path == rootID.Path && mod.Version == rootID.Version testThisMod := b.runTest && isRoot && mod.OnTest != nil @@ -306,10 +314,11 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul var metadata string if err := execbroker.Do(execbroker.Scope{ - Dir: tmpSourceDir, - Stdin: os.Stdin, - Stdout: b.stdout, - Stderr: b.stderr, + Dir: tmpSourceDir, + Stdin: os.Stdin, + Stdout: b.stdout, + Stderr: b.stderr, + Middleware: commandMiddleware, }, func() error { // Run OnBuild only on cache miss; reuse cached metadata otherwise. if cacheHit { diff --git a/internal/build/c/llvm/toolchain.go b/internal/build/c/llvm/toolchain.go new file mode 100644 index 0000000..3f2216c --- /dev/null +++ b/internal/build/c/llvm/toolchain.go @@ -0,0 +1,49 @@ +package llvm + +import ( + "fmt" + "os/exec" + + "github.com/goplus/llar/internal/build/c" +) + +// Toolchain is a prepared LLVM C-family toolchain. +type Toolchain struct { + c.Toolchain +} + +// New prepares an LLVM Toolchain from commands available in PATH. +func New() (*Toolchain, error) { + find := func(name string) (string, error) { + path, err := exec.LookPath(name) + if err != nil { + return "", fmt.Errorf("find prepared LLVM command %q: %w", name, err) + } + return path, nil + } + cc, err := find("clang") + if err != nil { + return nil, err + } + cxx, err := find("clang++") + if err != nil { + return nil, err + } + archiver, err := find("llvm-ar") + if err != nil { + return nil, err + } + ranlib, err := find("llvm-ranlib") + if err != nil { + return nil, err + } + nm, err := find("llvm-nm") + if err != nil { + return nil, err + } + strip, err := find("llvm-strip") + if err != nil { + return nil, err + } + return &Toolchain{Toolchain: c.NewToolchain(cc, cxx, archiver, ranlib, nm, strip)}, nil +} diff --git a/internal/build/c/llvm/toolchain_test.go b/internal/build/c/llvm/toolchain_test.go new file mode 100644 index 0000000..514be3c --- /dev/null +++ b/internal/build/c/llvm/toolchain_test.go @@ -0,0 +1,25 @@ +package llvm + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNewUsesPreparedPath(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"clang", "clang++", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("tool"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir) + + toolchain, err := New() + if err != nil { + t.Fatal(err) + } + if toolchain.CC() != filepath.Join(dir, "clang") { + t.Fatalf("CC = %q", toolchain.CC()) + } +} diff --git a/internal/build/crosscompile/crosscompile.go b/internal/build/c/target.go similarity index 61% rename from internal/build/crosscompile/crosscompile.go rename to internal/build/c/target.go index d9436f6..e62597b 100644 --- a/internal/build/crosscompile/crosscompile.go +++ b/internal/build/c/target.go @@ -1,4 +1,4 @@ -package crosscompile +package c import ( "fmt" @@ -6,8 +6,8 @@ import ( "path/filepath" "strings" + "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" - ccmetadata "github.com/goplus/llar/x/metadata/cc" ) const ( @@ -15,34 +15,16 @@ const ( linuxSysrootVersion = "glibc-2.17" ) -// Toolchain supplies prepared C-family tools for the build host. -type Toolchain interface { - CC() string - CXX() string - Archiver() string - Ranlib() string - NM() string - Strip() string +// Config contains the facts required to prepare a C target. +type Config struct { + Matrix string + Toolchain Toolchain + Sysroot string } -// Command describes a command before cross-compilation defaults are applied. -type Command struct { - Name string - Args []string - Env []string - Dir string -} - -// Patch contains the changes required for one command. -type Patch struct { - Name string - PrependArg []string - AppendArg []string - Env []string -} - -// CrossCompile contains prepared command-rewrite facts for one target. -type CrossCompile struct { +// Target contains prepared C command defaults for one build target. +type Target struct { + target string systemProcessor string targetTriple string sysroot string @@ -65,62 +47,65 @@ func Sysroot(targetOS, targetArch string) (module.Version, bool) { } } -// New prepares command rewriting for a built-in Linux target. -func New(targetOS, targetArch string, toolchain Toolchain, sysrootMetadata string) (*CrossCompile, error) { - triple, processor, err := linuxTarget(targetOS, targetArch) - if err != nil { - return nil, err +// NewTarget prepares C command defaults from config. +func NewTarget(config Config) (*Target, error) { + target := config.Matrix + if value, _, ok := strings.Cut(config.Matrix, "|"); ok { + target = value } - if err := validateToolchain(toolchain); err != nil { - return nil, fmt.Errorf("prepare cross compiler for %s/%s: %w", targetOS, targetArch, err) - } - metadata, err := ccmetadata.Parse(sysrootMetadata) + triple, processor, err := linuxTarget(target) if err != nil { - return nil, fmt.Errorf("parse sysroot metadata for %s/%s: %w", targetOS, targetArch, err) + return nil, err } - if metadata.Sysroot() == "" { - return nil, fmt.Errorf("sysroot metadata for %s/%s has no sysroot", targetOS, targetArch) + if err := validateToolchain(config.Toolchain); err != nil { + return nil, fmt.Errorf("prepare C target %s: %w", target, err) } - tempDir, err := os.MkdirTemp("", "llar-crosscompile-*") - if err != nil { - return nil, fmt.Errorf("prepare CMake toolchain for %s/%s: %w", targetOS, targetArch, err) - } - c := &CrossCompile{ + return &Target{ + target: target, systemProcessor: processor, targetTriple: triple, - sysroot: metadata.Sysroot(), - toolchain: toolchain, - tempDir: tempDir, - } - c.toolchainFile = filepath.Join(tempDir, "toolchain.cmake") - if err := os.WriteFile(c.toolchainFile, []byte(c.cmakeToolchain()), 0o600); err != nil { - _ = os.RemoveAll(tempDir) - return nil, fmt.Errorf("prepare CMake toolchain for %s/%s: %w", targetOS, targetArch, err) - } - return c, nil + sysroot: config.Sysroot, + toolchain: config.Toolchain, + }, nil } // Close removes generated build-system configuration. -func (c *CrossCompile) Close() error { +func (c *Target) Close() error { + if c.tempDir == "" { + return nil + } return os.RemoveAll(c.tempDir) } -// Use returns cross-compilation defaults for cmd. Explicit Formula settings -// are preserved. -func (c *CrossCompile) Use(cmd Command) Patch { +// Use returns C target defaults for cmd. Explicit Formula settings are +// preserved. +func (c *Target) Use(cmd build.Command) build.Patch { base := filepath.Base(cmd.Name) if base == "configure" { return c.autotoolsPatch(cmd) } if filepath.Base(cmd.Name) != cmd.Name { - return Patch{} + return build.Patch{} } switch base { case "cmake": if isCMakeConfigure(cmd.Args) && !hasCMakeToolchain(cmd.Args) { - return Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} + if c.toolchainFile == "" { + tempDir, err := os.MkdirTemp("", "llar-c-target-*") + if err != nil { + panic(fmt.Errorf("prepare CMake toolchain for %s: %w", c.target, err)) + } + toolchainFile := filepath.Join(tempDir, "toolchain.cmake") + if err := os.WriteFile(toolchainFile, []byte(c.cmakeToolchain()), 0o600); err != nil { + _ = os.RemoveAll(tempDir) + panic(fmt.Errorf("prepare CMake toolchain for %s: %w", c.target, err)) + } + c.tempDir = tempDir + c.toolchainFile = toolchainFile + } + return build.Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} } case "pkg-config": return c.pkgConfigPatch(cmd.Env) @@ -129,23 +114,23 @@ func (c *CrossCompile) Use(cmd Command) Patch { case "c++", "g++", "clang++": return c.compilerPatch(c.toolchain.CXX(), cmd.Args) case "ar", "llvm-ar": - return Patch{Name: c.toolchain.Archiver()} + return build.Patch{Name: c.toolchain.Archiver()} case "ranlib", "llvm-ranlib": - return Patch{Name: c.toolchain.Ranlib()} + return build.Patch{Name: c.toolchain.Ranlib()} case "nm", "llvm-nm": - return Patch{Name: c.toolchain.NM()} + return build.Patch{Name: c.toolchain.NM()} case "strip", "llvm-strip": - return Patch{Name: c.toolchain.Strip()} + return build.Patch{Name: c.toolchain.Strip()} } - return Patch{} + return build.Patch{} } -func (c *CrossCompile) compilerPatch(name string, args []string) Patch { +func (c *Target) compilerPatch(name string, args []string) build.Patch { flags := missingCompilerFlags(args, c.targetTriple, c.sysroot) - return Patch{Name: name, PrependArg: flags} + return build.Patch{Name: name, PrependArg: flags} } -func (c *CrossCompile) autotoolsPatch(cmd Command) Patch { +func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env := append([]string(nil), cmd.Env...) env = setMissingEnv(env, "CC", c.toolchain.CC()) env = setMissingEnv(env, "CXX", c.toolchain.CXX()) @@ -154,20 +139,28 @@ func (c *CrossCompile) autotoolsPatch(cmd Command) Patch { env = setMissingEnv(env, "NM", c.toolchain.NM()) env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) - compilerFlags := []string{"--target=" + c.targetTriple, "--sysroot=" + c.sysroot} + compilerFlags := []string{"--target=" + c.targetTriple} + if c.sysroot != "" { + compilerFlags = append(compilerFlags, "--sysroot="+c.sysroot) + } env = setEnvFlags(env, "CFLAGS", compilerFlags) env = setEnvFlags(env, "CXXFLAGS", compilerFlags) - env = setEnvFlags(env, "CPPFLAGS", []string{"--sysroot=" + c.sysroot}) + if c.sysroot != "" { + env = setEnvFlags(env, "CPPFLAGS", []string{"--sysroot=" + c.sysroot}) + } env = setEnvFlags(env, "LDFLAGS", compilerFlags) var args []string if !hasOption(cmd.Args, "--host") { args = append(args, "--host="+c.targetTriple) } - return Patch{AppendArg: args, Env: env} + return build.Patch{AppendArg: args, Env: env} } -func (c *CrossCompile) pkgConfigPatch(commandEnv []string) Patch { +func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { + if c.sysroot == "" { + return build.Patch{} + } env := append([]string(nil), commandEnv...) env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) libDirs, _ := envValue(env, "PKG_CONFIG_PATH") @@ -179,26 +172,34 @@ func (c *CrossCompile) pkgConfigPatch(commandEnv []string) Patch { filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), ) env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) - return Patch{Env: env} + return build.Patch{Env: env} } -func (c *CrossCompile) cmakeToolchain() string { +func (c *Target) cmakeToolchain() string { values := [][2]string{ {"CMAKE_SYSTEM_NAME", "Linux"}, {"CMAKE_SYSTEM_PROCESSOR", c.systemProcessor}, - {"CMAKE_SYSROOT", c.sysroot}, - {"CMAKE_C_COMPILER", c.toolchain.CC()}, - {"CMAKE_CXX_COMPILER", c.toolchain.CXX()}, - {"CMAKE_AR", c.toolchain.Archiver()}, - {"CMAKE_RANLIB", c.toolchain.Ranlib()}, - {"CMAKE_NM", c.toolchain.NM()}, - {"CMAKE_STRIP", c.toolchain.Strip()}, - {"CMAKE_C_COMPILER_TARGET", c.targetTriple}, - {"CMAKE_CXX_COMPILER_TARGET", c.targetTriple}, - {"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER"}, - {"CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY"}, - {"CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY"}, - {"CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY"}, + } + if c.sysroot != "" { + values = append(values, [2]string{"CMAKE_SYSROOT", c.sysroot}) + } + values = append(values, + [2]string{"CMAKE_C_COMPILER", c.toolchain.CC()}, + [2]string{"CMAKE_CXX_COMPILER", c.toolchain.CXX()}, + [2]string{"CMAKE_AR", c.toolchain.Archiver()}, + [2]string{"CMAKE_RANLIB", c.toolchain.Ranlib()}, + [2]string{"CMAKE_NM", c.toolchain.NM()}, + [2]string{"CMAKE_STRIP", c.toolchain.Strip()}, + [2]string{"CMAKE_C_COMPILER_TARGET", c.targetTriple}, + [2]string{"CMAKE_CXX_COMPILER_TARGET", c.targetTriple}, + ) + if c.sysroot != "" { + values = append(values, + [2]string{"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER"}, + [2]string{"CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY"}, + [2]string{"CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY"}, + [2]string{"CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY"}, + ) } var out strings.Builder for _, value := range values { @@ -207,24 +208,18 @@ func (c *CrossCompile) cmakeToolchain() string { return out.String() } -func linuxTarget(targetOS, targetArch string) (triple, processor string, err error) { - if targetOS != "linux" { - return "", "", fmt.Errorf("unsupported cross compile target %s/%s", targetOS, targetArch) - } - switch targetArch { - case "amd64": +func linuxTarget(target string) (triple, processor string, err error) { + switch target { + case "amd64-linux": return "x86_64-linux-gnu", "x86_64", nil - case "arm64": + case "arm64-linux": return "aarch64-linux-gnu", "aarch64", nil default: - return "", "", fmt.Errorf("unsupported cross compile target %s/%s", targetOS, targetArch) + return "", "", fmt.Errorf("unsupported C target %s", target) } } func validateToolchain(toolchain Toolchain) error { - if toolchain == nil { - return fmt.Errorf("toolchain is required") - } tools := []struct { name string path string @@ -270,7 +265,7 @@ func missingCompilerFlags(args []string, triple, sysroot string) []string { if !hasTargetFlag(args) { flags = append(flags, "--target="+triple) } - if !hasSysrootFlag(args) { + if sysroot != "" && !hasSysrootFlag(args) { flags = append(flags, "--sysroot="+sysroot) } return flags diff --git a/internal/build/crosscompile/crosscompile_test.go b/internal/build/c/target_test.go similarity index 59% rename from internal/build/crosscompile/crosscompile_test.go rename to internal/build/c/target_test.go index 7eb8682..e6be90b 100644 --- a/internal/build/crosscompile/crosscompile_test.go +++ b/internal/build/c/target_test.go @@ -1,4 +1,4 @@ -package crosscompile +package c import ( "os" @@ -8,17 +8,20 @@ import ( "strings" "testing" + "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" ) -type fakeToolchain struct{} - -func (fakeToolchain) CC() string { return "/llvm/bin/clang" } -func (fakeToolchain) CXX() string { return "/llvm/bin/clang++" } -func (fakeToolchain) Archiver() string { return "/llvm/bin/llvm-ar" } -func (fakeToolchain) Ranlib() string { return "/llvm/bin/llvm-ranlib" } -func (fakeToolchain) NM() string { return "/llvm/bin/llvm-nm" } -func (fakeToolchain) Strip() string { return "/llvm/bin/llvm-strip" } +func fakeToolchain() Toolchain { + return NewToolchain( + "/llvm/bin/clang", + "/llvm/bin/clang++", + "/llvm/bin/llvm-ar", + "/llvm/bin/llvm-ranlib", + "/llvm/bin/llvm-nm", + "/llvm/bin/llvm-strip", + ) +} func TestSysroot(t *testing.T) { want := module.Version{Path: "bminor/glibc", Version: "glibc-2.17"} @@ -35,19 +38,39 @@ func TestSysroot(t *testing.T) { } } -func TestNewRequiresSysrootMetadata(t *testing.T) { - if _, err := New("linux", "arm64", fakeToolchain{}, "-L/lib"); err == nil || !strings.Contains(err.Error(), "has no sysroot") { - t.Fatalf("New error = %v, want missing sysroot", err) +func TestBootstrapTargetOmitsSysroot(t *testing.T) { + c, err := NewTarget(Config{Matrix: "arm64-linux", Toolchain: fakeToolchain()}) + if err != nil { + t.Fatal(err) } -} + t.Cleanup(func() { _ = c.Close() }) -func TestNewWritesCMakeToolchain(t *testing.T) { - c, err := New("linux", "arm64", fakeToolchain{}, "--sysroot=/sdk") + patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) + if got, want := patch.PrependArg, []string{"--target=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { + t.Fatalf("PrependArg = %q, want %q", got, want) + } + if patch := c.Use(build.Command{Name: "pkg-config"}); patch.Env != nil { + t.Fatalf("pkg-config Patch = %+v, want no sysroot environment", patch) + } + + patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) if err != nil { t.Fatal(err) } + if strings.Contains(string(data), "CMAKE_SYSROOT") { + t.Fatalf("bootstrap CMake toolchain contains CMAKE_SYSROOT:\n%s", data) + } +} + +func TestUseCMakeWritesToolchainLazily(t *testing.T) { + c := newTestTarget(t) + if c.toolchainFile != "" || c.tempDir != "" { + t.Fatalf("New created CMake files: toolchainFile=%q tempDir=%q", c.toolchainFile, c.tempDir) + } + + c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) path := c.toolchainFile - t.Cleanup(func() { _ = c.Close() }) data, err := os.ReadFile(path) if err != nil { t.Fatal(err) @@ -67,24 +90,24 @@ func TestNewWritesCMakeToolchain(t *testing.T) { } func TestUseCMake(t *testing.T) { - c := newTestCrossCompile(t) - patch := c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + c := newTestTarget(t) + patch := c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } - patch = c.Use(Command{Name: "cmake", Args: []string{"--build", "build"}}) + patch = c.Use(build.Command{Name: "cmake", Args: []string{"--build", "build"}}) if len(patch.AppendArg) != 0 { t.Fatalf("build Patch = %+v, want no toolchain argument", patch) } - patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) + patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) if len(patch.AppendArg) != 0 { t.Fatalf("explicit toolchain Patch = %+v", patch) } } func TestUseDirectCommands(t *testing.T) { - c := newTestCrossCompile(t) - patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) + c := newTestTarget(t) + patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) if patch.Name != "/llvm/bin/clang" { t.Fatalf("Name = %q", patch.Name) } @@ -92,18 +115,18 @@ func TestUseDirectCommands(t *testing.T) { if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(Command{Name: "cc", Args: []string{"--target=custom", "--sysroot=/custom"}}) + patch = c.Use(build.Command{Name: "cc", Args: []string{"--target=custom", "--sysroot=/custom"}}) if len(patch.PrependArg) != 0 { t.Fatalf("explicit compiler flags were duplicated: %q", patch.PrependArg) } - if patch := c.Use(Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { + if patch := c.Use(build.Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { t.Fatalf("explicit compiler path was rewritten: %+v", patch) } } func TestUseAutotools(t *testing.T) { - c := newTestCrossCompile(t) - patch := c.Use(Command{ + c := newTestTarget(t) + patch := c.Use(build.Command{ Name: "/src/configure", Args: []string{"--build=x86_64-apple-darwin"}, Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, @@ -120,9 +143,9 @@ func TestUseAutotools(t *testing.T) { } func TestUsePkgConfig(t *testing.T) { - c := newTestCrossCompile(t) + c := newTestTarget(t) depPaths := strings.Join([]string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig"}, string(os.PathListSeparator)) - patch := c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) + patch := c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) } @@ -132,15 +155,15 @@ func TestUsePkgConfig(t *testing.T) { t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) } } - patch = c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) + patch = c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/custom" { t.Fatalf("PKG_CONFIG_LIBDIR override = %q, want /custom", got) } } -func newTestCrossCompile(t *testing.T) *CrossCompile { +func newTestTarget(t *testing.T) *Target { t.Helper() - c, err := New("linux", "arm64", fakeToolchain{}, "--sysroot=/sdk") + c, err := NewTarget(Config{Matrix: "arm64-linux|shared", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) if err != nil { t.Fatal(err) } diff --git a/internal/build/c/toolchain.go b/internal/build/c/toolchain.go new file mode 100644 index 0000000..156a3a0 --- /dev/null +++ b/internal/build/c/toolchain.go @@ -0,0 +1,30 @@ +package c + +// Toolchain contains prepared C-family command paths. +type Toolchain struct { + cc string + cxx string + archiver string + ranlib string + nm string + strip string +} + +// NewToolchain creates a Toolchain from prepared command paths. +func NewToolchain(cc, cxx, archiver, ranlib, nm, strip string) Toolchain { + return Toolchain{ + cc: cc, + cxx: cxx, + archiver: archiver, + ranlib: ranlib, + nm: nm, + strip: strip, + } +} + +func (t Toolchain) CC() string { return t.cc } +func (t Toolchain) CXX() string { return t.cxx } +func (t Toolchain) Archiver() string { return t.archiver } +func (t Toolchain) Ranlib() string { return t.ranlib } +func (t Toolchain) NM() string { return t.nm } +func (t Toolchain) Strip() string { return t.strip } diff --git a/internal/build/c/toolchain_test.go b/internal/build/c/toolchain_test.go new file mode 100644 index 0000000..755d121 --- /dev/null +++ b/internal/build/c/toolchain_test.go @@ -0,0 +1,23 @@ +package c + +import "testing" + +func TestToolchain(t *testing.T) { + toolchain := NewToolchain("cc", "c++", "ar", "ranlib", "nm", "strip") + for _, test := range []struct { + name string + got string + want string + }{ + {name: "CC", got: toolchain.CC(), want: "cc"}, + {name: "CXX", got: toolchain.CXX(), want: "c++"}, + {name: "Archiver", got: toolchain.Archiver(), want: "ar"}, + {name: "Ranlib", got: toolchain.Ranlib(), want: "ranlib"}, + {name: "NM", got: toolchain.NM(), want: "nm"}, + {name: "Strip", got: toolchain.Strip(), want: "strip"}, + } { + if test.got != test.want { + t.Fatalf("%s = %q, want %q", test.name, test.got, test.want) + } + } +} diff --git a/internal/build/http/http.go b/internal/build/http/http.go index 39740e7..b3d54cd 100644 --- a/internal/build/http/http.go +++ b/internal/build/http/http.go @@ -200,9 +200,9 @@ func parseRequest(r *http.Request) (request, error) { if len(values) != 1 || values[0] == "" { return request{}, fmt.Errorf("matrix %q requires exactly one value", key) } - // Requests carry a flat matrix. Platform dimensions propagate to + // Requests carry a flat matrix. Target requirements propagate to // dependencies; all other dimensions are package build options. - if key == "os" || key == "arch" { + if key == "os" || key == "arch" || key == "libc" { if require == nil { require = make(map[string][]string) } diff --git a/internal/build/http/http_test.go b/internal/build/http/http_test.go index b0b430a..9dc3361 100644 --- a/internal/build/http/http_test.go +++ b/internal/build/http/http_test.go @@ -53,11 +53,11 @@ func TestParseRequest(t *testing.T) { }, { name: "require and options", - target: "/v1/artifacts/madler/zlib@v1.3.1?os=linux&arch=amd64&debug=OFF&shared=ON", + target: "/v1/artifacts/madler/zlib@v1.3.1?os=linux&arch=amd64&libc=glibc-2.13&debug=OFF&shared=ON", wantModule: "madler/zlib", wantVer: "v1.3.1", - wantMatrix: "amd64-linux|OFF-ON", - wantRequire: map[string][]string{"arch": {"amd64"}, "os": {"linux"}}, + wantMatrix: "amd64-glibc-2.13-linux|OFF-ON", + wantRequire: map[string][]string{"arch": {"amd64"}, "libc": {"glibc-2.13"}, "os": {"linux"}}, wantOptions: map[string][]string{"debug": {"OFF"}, "shared": {"ON"}}, }, {name: "wrong path", target: "/v1/modules/madler/zlib?os=linux", wantErr: "artifact path not found"}, @@ -94,7 +94,7 @@ func TestParseRequest(t *testing.T) { for key, values := range req.query { values[0] = "changed" matrixValues := req.matrix.Options[key] - if key == "os" || key == "arch" { + if key == "os" || key == "arch" || key == "libc" { matrixValues = req.matrix.Require[key] } if matrixValues[0] == "changed" { diff --git a/internal/build/target.go b/internal/build/target.go new file mode 100644 index 0000000..8e3182f --- /dev/null +++ b/internal/build/target.go @@ -0,0 +1,56 @@ +package build + +import ( + "os" + + "github.com/goplus/llar/internal/execbroker" +) + +// Command describes a command before target defaults are applied. +type Command struct { + Name string + Args []string + Env []string + Dir string +} + +// Patch contains target-specific changes for one command. +type Patch struct { + Name string + PrependArg []string + AppendArg []string + Env []string +} + +// Target applies language-specific target defaults to build commands. +type Target interface { + Use(Command) Patch +} + +func targetMiddleware(target Target) execbroker.Middleware { + return func(req execbroker.Request) execbroker.Request { + env := req.Env + if env == nil { + env = os.Environ() + } + patch := target.Use(Command{ + Name: req.Name, + Args: req.Args, + Env: env, + Dir: req.Dir, + }) + if patch.Name != "" { + req.Name = patch.Name + } + if len(patch.PrependArg) > 0 { + req.Args = append(append([]string(nil), patch.PrependArg...), req.Args...) + } + if len(patch.AppendArg) > 0 { + req.Args = append(req.Args, patch.AppendArg...) + } + if patch.Env != nil { + req.Env = append([]string(nil), patch.Env...) + } + return req + } +} diff --git a/internal/build/target_test.go b/internal/build/target_test.go new file mode 100644 index 0000000..1591704 --- /dev/null +++ b/internal/build/target_test.go @@ -0,0 +1,78 @@ +package build + +import ( + "context" + "reflect" + "testing" + "testing/fstest" + + classfile "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/execbroker" + internalformula "github.com/goplus/llar/internal/formula" + "github.com/goplus/llar/internal/modules" +) + +type testTarget struct { + command Command +} + +func (t *testTarget) Use(command Command) Patch { + t.command = command + return Patch{ + Name: "/toolchain/cc", + PrependArg: []string{"--target=aarch64-linux-gnu"}, + AppendArg: []string{"--sysroot=/sdk"}, + Env: []string{"CC=/toolchain/cc"}, + } +} + +func TestTargetMiddleware(t *testing.T) { + target := new(testTarget) + got := targetMiddleware(target)(execbroker.Request{ + Name: "cc", + Args: []string{"-c", "a.c"}, + Env: []string{"CFLAGS=-O2"}, + Dir: "/src", + }) + if got.Name != "/toolchain/cc" { + t.Fatalf("Name = %q", got.Name) + } + if want := []string{"--target=aarch64-linux-gnu", "-c", "a.c", "--sysroot=/sdk"}; !reflect.DeepEqual(got.Args, want) { + t.Fatalf("Args = %q, want %q", got.Args, want) + } + if want := []string{"CC=/toolchain/cc"}; !reflect.DeepEqual(got.Env, want) { + t.Fatalf("Env = %q, want %q", got.Env, want) + } + if target.command.Name != "cc" || target.command.Dir != "/src" { + t.Fatalf("Command = %+v", target.command) + } + if want := []string{"CFLAGS=-O2"}; !reflect.DeepEqual(target.command.Env, want) { + t.Fatalf("Command.Env = %q, want %q", target.command.Env, want) + } +} + +func TestBuildAppliesTargetToFormulaCommands(t *testing.T) { + store := setupTestStore(t) + builder := setupBuilder(t, store, "arm64-linux") + target := new(testTarget) + builder.target = target + + var commandName string + root := &modules.Module{ + Formula: &internalformula.Formula{OnBuild: func(*classfile.Context) { + commandName = execbroker.Command("cc", "-c", "a.c").Args[0] + }}, + FS: fstest.MapFS{"README": {Data: []byte("test")}}, + Path: "test/liba", + Version: "1.0.0", + } + if _, err := builder.Build(context.Background(), []*modules.Module{root}); err != nil { + t.Fatal(err) + } + if commandName != "/toolchain/cc" { + t.Fatalf("command name = %q, want /toolchain/cc", commandName) + } + if target.command.Name != "cc" { + t.Fatalf("target command = %+v", target.command) + } +} diff --git a/internal/modules/load.go b/internal/modules/load.go index 185ba01..e2cc2a1 100644 --- a/internal/modules/load.go +++ b/internal/modules/load.go @@ -61,6 +61,8 @@ type Options struct { // FormulaStore is the store for downloading and caching formulas. FormulaStore repo.Store Matrix classfile.Matrix + // Roots adds caller-supplied requirements to the main module before MVS. + Roots []module.Version } func latestVersion(ctx context.Context, modPath string, repo vcs.Repo, comparator func(v1, v2 module.Version) int) (version string, err error) { @@ -196,6 +198,7 @@ func Load(ctx context.Context, main module.Version, opts Options) ([]*Module, er if err != nil { return nil, err } + mainDeps = append(mainDeps, opts.Roots...) cmp := func(p, v1, v2 string) int { // none is an internal version for MVS, which means the smallest if v1 == "none" && v2 != "none" { diff --git a/internal/modules/load_test.go b/internal/modules/load_test.go index 49a7727..79ab9b1 100644 --- a/internal/modules/load_test.go +++ b/internal/modules/load_test.go @@ -457,6 +457,26 @@ func TestLoad_InjectsTargetBeforeFilterAndOnRequire(t *testing.T) { } } +func TestLoad_AddsRootsToMainRequirements(t *testing.T) { + store := setupTestStore(t, "testdata/load") + main := module.Version{Path: "towner/withdeps", Version: "1.0.0"} + root := module.Version{Path: "towner/leafmod", Version: "2.0.0"} + + mods, err := Load(context.Background(), main, Options{ + FormulaStore: store, + Roots: []module.Version{root}, + }) + if err != nil { + t.Fatal(err) + } + if got := findModule(mods, root.Path); got == nil || got.Version != root.Version { + t.Fatalf("root module = %+v, want %+v", got, root) + } + if got, want := depVersions(mods[0]), []module.Version{root}; !slices.Equal(got, want) { + t.Fatalf("main deps = %+v, want %+v", got, want) + } +} + func TestLoad_FilterRejectsSelectedMatrix(t *testing.T) { store := setupTestStore(t, "testdata/load") ctx := context.Background() From 6323bf71ec1e5112621410233d3d64a17b931932 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 10:09:50 +0800 Subject: [PATCH 03/39] docs(crosscompile): mark multi-language extension points --- cmd/llar/internal/crosscompile.go | 4 ++++ cmd/llar/internal/make.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/cmd/llar/internal/crosscompile.go b/cmd/llar/internal/crosscompile.go index 0c9d1f6..b038d9c 100644 --- a/cmd/llar/internal/crosscompile.go +++ b/cmd/llar/internal/crosscompile.go @@ -13,6 +13,8 @@ func crossCompileTarget(matrix formula.Matrix) (string, bool) { if targetOS == runtime.GOOS && targetArch == runtime.GOARCH { return "", false } + // TODO: Select support across language implementations when another + // build.Target is added; c.Sysroot currently defines the supported set. if _, ok := c.Sysroot(targetOS, targetArch); !ok { return "", false } @@ -27,6 +29,8 @@ func crossCompileSysroot( return module.Version{}, false } targetOS, targetArch := matrixTarget(matrix) + // TODO: Add language-specific bootstrap inputs alongside this C sysroot + // policy when another build.Target requires its own preparation. sysroot, ok := c.Sysroot(targetOS, targetArch) if !ok || targetOS == runtime.GOOS && targetArch == runtime.GOARCH || root.Path == sysroot.Path { return module.Version{}, false diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 7193f79..445df49 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -168,6 +168,8 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } + // TODO: Prepare the matching language build.Target here when implementations + // beyond c.NewTarget are added. if crossCompile { llvmToolchain, err := llvm.New() if err != nil { From 889f85038d0bdc9905b5b791862a34187bc3a734 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 10:28:52 +0800 Subject: [PATCH 04/39] refactor(crosscompile): keep target preparation inline --- cmd/llar/internal/crosscompile.go | 49 ---------------------- cmd/llar/internal/crosscompile_test.go | 57 -------------------------- cmd/llar/internal/make.go | 42 ++++++++++++++----- 3 files changed, 31 insertions(+), 117 deletions(-) delete mode 100644 cmd/llar/internal/crosscompile.go delete mode 100644 cmd/llar/internal/crosscompile_test.go diff --git a/cmd/llar/internal/crosscompile.go b/cmd/llar/internal/crosscompile.go deleted file mode 100644 index b038d9c..0000000 --- a/cmd/llar/internal/crosscompile.go +++ /dev/null @@ -1,49 +0,0 @@ -package internal - -import ( - "runtime" - - "github.com/goplus/llar/formula" - "github.com/goplus/llar/internal/build/c" - "github.com/goplus/llar/mod/module" -) - -func crossCompileTarget(matrix formula.Matrix) (string, bool) { - targetOS, targetArch := matrixTarget(matrix) - if targetOS == runtime.GOOS && targetArch == runtime.GOARCH { - return "", false - } - // TODO: Select support across language implementations when another - // build.Target is added; c.Sysroot currently defines the supported set. - if _, ok := c.Sysroot(targetOS, targetArch); !ok { - return "", false - } - return targetArch + "-" + targetOS, true -} - -func crossCompileSysroot( - root module.Version, - matrix formula.Matrix, -) (module.Version, bool) { - if _, ok := matrix.Require["libc"]; ok { - return module.Version{}, false - } - targetOS, targetArch := matrixTarget(matrix) - // TODO: Add language-specific bootstrap inputs alongside this C sysroot - // policy when another build.Target requires its own preparation. - sysroot, ok := c.Sysroot(targetOS, targetArch) - if !ok || targetOS == runtime.GOOS && targetArch == runtime.GOARCH || root.Path == sysroot.Path { - return module.Version{}, false - } - return sysroot, true -} - -func matrixTarget(matrix formula.Matrix) (targetOS, targetArch string) { - if values := matrix.Require["os"]; len(values) > 0 { - targetOS = values[0] - } - if values := matrix.Require["arch"]; len(values) > 0 { - targetArch = values[0] - } - return -} diff --git a/cmd/llar/internal/crosscompile_test.go b/cmd/llar/internal/crosscompile_test.go deleted file mode 100644 index bdb145e..0000000 --- a/cmd/llar/internal/crosscompile_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package internal - -import ( - "runtime" - "testing" - - "github.com/goplus/llar/formula" - "github.com/goplus/llar/mod/module" -) - -func TestMatrixTarget(t *testing.T) { - matrix := formula.Matrix{Require: map[string][]string{ - "os": {"linux"}, - "arch": {"arm64"}, - }} - if targetOS, targetArch := matrixTarget(matrix); targetOS != "linux" || targetArch != "arm64" { - t.Fatalf("matrixTarget = %s/%s, want linux/arm64", targetOS, targetArch) - } -} - -func TestCrossCompileTarget(t *testing.T) { - targetArch := "amd64" - if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { - targetArch = "arm64" - } - matrix := formula.Matrix{Require: map[string][]string{ - "os": {"linux"}, - "arch": {targetArch}, - }} - if got, ok := crossCompileTarget(matrix); !ok || got != targetArch+"-linux" { - t.Fatalf("crossCompileTarget = %q, %v; want %q, true", got, ok, targetArch+"-linux") - } - matrix.Require["libc"] = []string{"glibc-2.13"} - if got, ok := crossCompileTarget(matrix); !ok || got != targetArch+"-linux" { - t.Fatalf("crossCompileTarget with libc = %q, %v; want %q, true", got, ok, targetArch+"-linux") - } -} - -func TestCrossCompileSysrootSkipsDefaultForLibcRequirement(t *testing.T) { - targetArch := "amd64" - if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { - targetArch = "arm64" - } - matrix := formula.Matrix{Require: map[string][]string{ - "os": {"linux"}, - "arch": {targetArch}, - }} - root := module.Version{Path: "owner/root", Version: "v1"} - - if _, ok := crossCompileSysroot(root, matrix); !ok { - t.Fatal("crossCompileSysroot did not select the default sysroot") - } - matrix.Require["libc"] = nil - if got, ok := crossCompileSysroot(root, matrix); ok || got != (module.Version{}) { - t.Fatalf("crossCompileSysroot = %+v, %v; want no default sysroot", got, ok) - } -} diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 445df49..7d4dcc6 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -133,14 +133,31 @@ func hostMatrix() formula.Matrix { // `llar test ` invocation. func buildModule(ctx context.Context, store repo.Store, modPath, version string, matrix formula.Matrix, runTest bool) error { root := module.Version{Path: modPath, Version: version} - targetMatrix, crossCompile := crossCompileTarget(matrix) - sysroot, useDefaultSysroot := crossCompileSysroot(root, matrix) + var targetOS, targetArch string + if values := matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + var targetRoot module.Version + var useCTarget bool + if targetOS != runtime.GOOS || targetArch != runtime.GOARCH { + // TODO: Add other language target policies alongside this C case when + // they provide build.Target implementations. + cSysroot, ok := c.Sysroot(targetOS, targetArch) + useCTarget = ok + _, customLibc := matrix.Require["libc"] + if useCTarget && !customLibc && root.Path != cSysroot.Path { + targetRoot = cSysroot + } + } loadOpts := modules.Options{ FormulaStore: store, Matrix: matrix, } - if useDefaultSysroot { - loadOpts.Roots = []module.Version{sysroot} + if targetRoot != (module.Version{}) { + loadOpts.Roots = []module.Version{targetRoot} } mods, err := modules.Load(ctx, root, loadOpts) if err != nil { @@ -168,9 +185,10 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } - // TODO: Prepare the matching language build.Target here when implementations - // beyond c.NewTarget are added. - if crossCompile { + var target build.Target + // TODO: Add other language build.Target preparation alongside this C case. + if useCTarget { + targetMatrix := targetArch + "-" + targetOS llvmToolchain, err := llvm.New() if err != nil { return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) @@ -183,13 +201,13 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, return err } defer bootstrapTarget.Close() - buildOpts.Target = bootstrapTarget + target = bootstrapTarget - if useDefaultSysroot { + if targetRoot != (module.Version{}) { // A successful Load includes every Options.Roots path in the selected graph. var selectedSysroot *modules.Module for _, mod := range mods { - if mod.Path == sysroot.Path { + if mod.Path == targetRoot.Path { selectedSysroot = mod break } @@ -197,6 +215,7 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, sysrootOpts := buildOpts sysrootOpts.RunTest = false + sysrootOpts.Target = bootstrapTarget sysrootBuilder, err := build.NewBuilder(sysrootOpts) if err != nil { return fmt.Errorf("failed to create sysroot builder: %w", err) @@ -222,9 +241,10 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, return err } defer configuredTarget.Close() - buildOpts.Target = configuredTarget + target = configuredTarget } } + buildOpts.Target = target builder, err := build.NewBuilder(buildOpts) if err != nil { return fmt.Errorf("failed to create builder: %w", err) From 3c213451e2f8b5d1c3369a9bd9243c525e33b95f Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 10:51:58 +0800 Subject: [PATCH 05/39] feat(crosscompile): add Darwin C targets --- cmd/llar/internal/make.go | 2 +- internal/build/c/target.go | 186 ++++++++++++++++++++++---------- internal/build/c/target_test.go | 88 ++++++++++++++- 3 files changed, 216 insertions(+), 60 deletions(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 7d4dcc6..ca8690e 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -148,7 +148,7 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, cSysroot, ok := c.Sysroot(targetOS, targetArch) useCTarget = ok _, customLibc := matrix.Require["libc"] - if useCTarget && !customLibc && root.Path != cSysroot.Path { + if useCTarget && (targetOS != "linux" || !customLibc) && root.Path != cSysroot.Path { targetRoot = cSysroot } } diff --git a/internal/build/c/target.go b/internal/build/c/target.go index e62597b..f3909a0 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -11,8 +11,10 @@ import ( ) const ( - linuxSysrootPath = "bminor/glibc" - linuxSysrootVersion = "glibc-2.17" + linuxSysrootPath = "bminor/glibc" + linuxSysrootVersion = "glibc-2.17" + darwinSysrootPath = "joseluisq/macosx-sdks" + darwinSysrootVersion = "14.5" ) // Config contains the facts required to prepare a C target. @@ -24,27 +26,33 @@ type Config struct { // Target contains prepared C command defaults for one build target. type Target struct { - target string - systemProcessor string - targetTriple string - sysroot string - toolchain Toolchain - toolchainFile string - tempDir string + target string + systemName string + systemProcessor string + targetTriple string + deploymentTarget string + sysroot string + toolchain Toolchain + toolchainFile string + tempDir string } -// Sysroot returns the fixed compatibility sysroot Formula for a built-in -// Linux target. +// Sysroot returns the fixed compatibility sysroot Formula for a built-in C +// target. func Sysroot(targetOS, targetArch string) (module.Version, bool) { - if targetOS != "linux" { - return module.Version{}, false - } - switch targetArch { - case "amd64", "arm64": - return module.Version{Path: linuxSysrootPath, Version: linuxSysrootVersion}, true - default: - return module.Version{}, false + switch targetOS { + case "linux": + switch targetArch { + case "amd64", "arm64": + return module.Version{Path: linuxSysrootPath, Version: linuxSysrootVersion}, true + } + case "darwin": + switch targetArch { + case "amd64", "arm64": + return module.Version{Path: darwinSysrootPath, Version: darwinSysrootVersion}, true + } } + return module.Version{}, false } // NewTarget prepares C command defaults from config. @@ -53,20 +61,41 @@ func NewTarget(config Config) (*Target, error) { if value, _, ok := strings.Cut(config.Matrix, "|"); ok { target = value } - triple, processor, err := linuxTarget(target) - if err != nil { - return nil, err + var systemName, processor, triple, deploymentTarget string + switch target { + case "amd64-linux": + systemName = "Linux" + processor = "x86_64" + triple = "x86_64-linux-gnu" + case "arm64-linux": + systemName = "Linux" + processor = "aarch64" + triple = "aarch64-linux-gnu" + case "amd64-darwin": + systemName = "Darwin" + processor = "x86_64" + triple = "x86_64-apple-darwin" + deploymentTarget = "10.13" + case "arm64-darwin": + systemName = "Darwin" + processor = "arm64" + triple = "arm64-apple-darwin" + deploymentTarget = "11.0" + default: + return nil, fmt.Errorf("unsupported C target %s", target) } if err := validateToolchain(config.Toolchain); err != nil { return nil, fmt.Errorf("prepare C target %s: %w", target, err) } return &Target{ - target: target, - systemProcessor: processor, - targetTriple: triple, - sysroot: config.Sysroot, - toolchain: config.Toolchain, + target: target, + systemName: systemName, + systemProcessor: processor, + targetTriple: triple, + deploymentTarget: deploymentTarget, + sysroot: config.Sysroot, + toolchain: config.Toolchain, }, nil } @@ -126,10 +155,35 @@ func (c *Target) Use(cmd build.Command) build.Patch { } func (c *Target) compilerPatch(name string, args []string) build.Patch { - flags := missingCompilerFlags(args, c.targetTriple, c.sysroot) + linker := true + for _, arg := range args { + switch arg { + case "-c", "-E", "-S": + linker = false + } + } + flags := missingCompilerFlags(args, c.compilerFlags(linker)) return build.Patch{Name: name, PrependArg: flags} } +func (c *Target) compilerFlags(linker bool) []string { + flags := []string{"--target=" + c.targetTriple} + if c.sysroot != "" { + if c.systemName == "Darwin" { + flags = append(flags, "-isysroot"+c.sysroot) + } else { + flags = append(flags, "--sysroot="+c.sysroot) + } + } + if c.deploymentTarget != "" { + flags = append(flags, "-mmacosx-version-min="+c.deploymentTarget) + } + if linker && c.systemName == "Darwin" { + flags = append(flags, "-fuse-ld=lld") + } + return flags +} + func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env := append([]string(nil), cmd.Env...) env = setMissingEnv(env, "CC", c.toolchain.CC()) @@ -139,16 +193,17 @@ func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env = setMissingEnv(env, "NM", c.toolchain.NM()) env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) - compilerFlags := []string{"--target=" + c.targetTriple} - if c.sysroot != "" { - compilerFlags = append(compilerFlags, "--sysroot="+c.sysroot) - } + compilerFlags := c.compilerFlags(false) env = setEnvFlags(env, "CFLAGS", compilerFlags) env = setEnvFlags(env, "CXXFLAGS", compilerFlags) if c.sysroot != "" { - env = setEnvFlags(env, "CPPFLAGS", []string{"--sysroot=" + c.sysroot}) + sysrootFlag := "--sysroot=" + c.sysroot + if c.systemName == "Darwin" { + sysrootFlag = "-isysroot" + c.sysroot + } + env = setEnvFlags(env, "CPPFLAGS", []string{sysrootFlag}) } - env = setEnvFlags(env, "LDFLAGS", compilerFlags) + env = setEnvFlags(env, "LDFLAGS", c.compilerFlags(true)) var args []string if !hasOption(cmd.Args, "--host") { @@ -165,8 +220,10 @@ func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) libDirs, _ := envValue(env, "PKG_CONFIG_PATH") paths := filepath.SplitList(libDirs) + if c.systemName == "Linux" { + paths = append(paths, filepath.Join(c.sysroot, "usr", "lib", c.targetTriple, "pkgconfig")) + } paths = append(paths, - filepath.Join(c.sysroot, "usr", "lib", c.targetTriple, "pkgconfig"), filepath.Join(c.sysroot, "usr", "lib64", "pkgconfig"), filepath.Join(c.sysroot, "usr", "lib", "pkgconfig"), filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), @@ -177,10 +234,18 @@ func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { func (c *Target) cmakeToolchain() string { values := [][2]string{ - {"CMAKE_SYSTEM_NAME", "Linux"}, + {"CMAKE_SYSTEM_NAME", c.systemName}, {"CMAKE_SYSTEM_PROCESSOR", c.systemProcessor}, } - if c.sysroot != "" { + if c.systemName == "Darwin" { + values = append(values, + [2]string{"CMAKE_OSX_ARCHITECTURES", c.systemProcessor}, + [2]string{"CMAKE_OSX_DEPLOYMENT_TARGET", c.deploymentTarget}, + ) + if c.sysroot != "" { + values = append(values, [2]string{"CMAKE_OSX_SYSROOT", c.sysroot}) + } + } else if c.sysroot != "" { values = append(values, [2]string{"CMAKE_SYSROOT", c.sysroot}) } values = append(values, @@ -193,6 +258,13 @@ func (c *Target) cmakeToolchain() string { [2]string{"CMAKE_C_COMPILER_TARGET", c.targetTriple}, [2]string{"CMAKE_CXX_COMPILER_TARGET", c.targetTriple}, ) + if c.systemName == "Darwin" { + values = append(values, + [2]string{"CMAKE_EXE_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, + [2]string{"CMAKE_SHARED_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, + [2]string{"CMAKE_MODULE_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, + ) + } if c.sysroot != "" { values = append(values, [2]string{"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER"}, @@ -208,17 +280,6 @@ func (c *Target) cmakeToolchain() string { return out.String() } -func linuxTarget(target string) (triple, processor string, err error) { - switch target { - case "amd64-linux": - return "x86_64-linux-gnu", "x86_64", nil - case "arm64-linux": - return "aarch64-linux-gnu", "aarch64", nil - default: - return "", "", fmt.Errorf("unsupported C target %s", target) - } -} - func validateToolchain(toolchain Toolchain) error { tools := []struct { name string @@ -260,13 +321,22 @@ func hasCMakeToolchain(args []string) bool { return false } -func missingCompilerFlags(args []string, triple, sysroot string) []string { +func missingCompilerFlags(args, defaults []string) []string { var flags []string - if !hasTargetFlag(args) { - flags = append(flags, "--target="+triple) - } - if sysroot != "" && !hasSysrootFlag(args) { - flags = append(flags, "--sysroot="+sysroot) + for _, flag := range defaults { + if strings.HasPrefix(flag, "--target=") && hasTargetFlag(args) { + continue + } + if (strings.HasPrefix(flag, "--sysroot=") || strings.HasPrefix(flag, "-isysroot")) && hasSysrootFlag(args) { + continue + } + if strings.HasPrefix(flag, "-mmacosx-version-min=") && hasFlag(args, "-mmacosx-version-min") { + continue + } + if strings.HasPrefix(flag, "-fuse-ld=") && hasFlag(args, "-fuse-ld") { + continue + } + flags = append(flags, flag) } return flags } @@ -335,7 +405,13 @@ func setEnvFlags(env []string, key string, defaults []string) []string { if strings.HasPrefix(flag, "--target=") && hasTargetFlag(args) { continue } - if strings.HasPrefix(flag, "--sysroot=") && hasSysrootFlag(args) { + if (strings.HasPrefix(flag, "--sysroot=") || strings.HasPrefix(flag, "-isysroot")) && hasSysrootFlag(args) { + continue + } + if strings.HasPrefix(flag, "-mmacosx-version-min=") && hasFlag(args, "-mmacosx-version-min") { + continue + } + if strings.HasPrefix(flag, "-fuse-ld=") && hasFlag(args, "-fuse-ld") { continue } if value != "" { diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index e6be90b..da36ce7 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -24,20 +24,100 @@ func fakeToolchain() Toolchain { } func TestSysroot(t *testing.T) { - want := module.Version{Path: "bminor/glibc", Version: "glibc-2.17"} + linux := module.Version{Path: "bminor/glibc", Version: "glibc-2.17"} + darwin := module.Version{Path: "joseluisq/macosx-sdks", Version: "14.5"} for _, arch := range []string{"amd64", "arm64"} { got, ok := Sysroot("linux", arch) - if !ok || got != want { - t.Fatalf("Sysroot(linux, %s) = %+v, %v; want %+v, true", arch, got, ok, want) + if !ok || got != linux { + t.Fatalf("Sysroot(linux, %s) = %+v, %v; want %+v, true", arch, got, ok, linux) + } + got, ok = Sysroot("darwin", arch) + if !ok || got != darwin { + t.Fatalf("Sysroot(darwin, %s) = %+v, %v; want %+v, true", arch, got, ok, darwin) } } - for _, target := range [][2]string{{"darwin", "arm64"}, {"linux", "riscv64"}, {"", "esp32"}} { + for _, target := range [][2]string{{"darwin", "riscv64"}, {"linux", "riscv64"}, {"", "esp32"}} { if got, ok := Sysroot(target[0], target[1]); ok { t.Fatalf("Sysroot(%q, %q) = %+v, true; want unsupported", target[0], target[1], got) } } } +func TestDarwinTarget(t *testing.T) { + target, err := NewTarget(Config{Matrix: "amd64-darwin|shared", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = target.Close() }) + + patch := target.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) + want := []string{"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13"} + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) + } + patch = target.Use(build.Command{Name: "cc", Args: []string{"a.o", "-shared"}}) + if !slices.Contains(patch.PrependArg, "-fuse-ld=lld") { + t.Fatalf("link PrependArg = %q, want LLD selection", patch.PrependArg) + } + + patch = target.Use(build.Command{ + Name: "cc", + Args: []string{"--target=custom", "-isysroot/custom", "-mmacosx-version-min=12.0", "-fuse-ld=custom"}, + }) + if len(patch.PrependArg) != 0 { + t.Fatalf("explicit Darwin flags were duplicated: %q", patch.PrependArg) + } + + patch = target.Use(build.Command{Name: "/src/configure"}) + if got, want := patch.AppendArg, []string{"--host=x86_64-apple-darwin"}; !reflect.DeepEqual(got, want) { + t.Fatalf("configure AppendArg = %q, want %q", got, want) + } + for key, values := range map[string][]string{ + "CFLAGS": {"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13"}, + "CPPFLAGS": {"-isysroot/sdk"}, + "LDFLAGS": {"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13", "-fuse-ld=lld"}, + } { + got, _ := envValue(patch.Env, key) + for _, value := range values { + if !slices.Contains(strings.Fields(got), value) { + t.Fatalf("%s = %q, want %q", key, got, value) + } + } + } + + patch = target.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) + if err != nil { + t.Fatal(err) + } + content := string(data) + for _, want := range []string{ + "set(CMAKE_SYSTEM_NAME \"Darwin\")", + "CMAKE_OSX_ARCHITECTURES", + "CMAKE_OSX_SYSROOT", + "CMAKE_OSX_DEPLOYMENT_TARGET", + "x86_64-apple-darwin", + "-fuse-ld=lld", + } { + if !strings.Contains(content, want) { + t.Fatalf("toolchain file does not contain %q:\n%s", want, content) + } + } +} + +func TestDarwinArm64DeploymentTarget(t *testing.T) { + target, err := NewTarget(Config{Matrix: "arm64-darwin", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = target.Close() }) + + patch := target.Use(build.Command{Name: "cc"}) + if !slices.Contains(patch.PrependArg, "-mmacosx-version-min=11.0") { + t.Fatalf("PrependArg = %q, want arm64 deployment target", patch.PrependArg) + } +} + func TestBootstrapTargetOmitsSysroot(t *testing.T) { c, err := NewTarget(Config{Matrix: "arm64-linux", Toolchain: fakeToolchain()}) if err != nil { From 61a4c8097ec17ce1b0f9c081b7da018d51b4df51 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 14:47:11 +0800 Subject: [PATCH 06/39] refactor(crosscompile): prepare target commands in LLVM --- cmd/llar/internal/make.go | 10 +- internal/build/c/llvm/toolchain.go | 47 ++++- internal/build/c/llvm/toolchain_test.go | 23 ++- internal/build/c/target.go | 219 ++++++------------------ internal/build/c/target_test.go | 81 +++++---- internal/build/c/toolchain.go | 21 ++- internal/build/c/toolchain_test.go | 22 ++- 7 files changed, 201 insertions(+), 222 deletions(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index ca8690e..f723081 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -189,13 +189,13 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, // TODO: Add other language build.Target preparation alongside this C case. if useCTarget { targetMatrix := targetArch + "-" + targetOS - llvmToolchain, err := llvm.New() + bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) if err != nil { return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) } bootstrapTarget, err := c.NewTarget(c.Config{ Matrix: targetMatrix, - Toolchain: llvmToolchain.Toolchain, + Toolchain: bootstrapToolchain.Toolchain, }) if err != nil { return err @@ -232,9 +232,13 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, return fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) } + configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) + if err != nil { + return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } configuredTarget, err := c.NewTarget(c.Config{ Matrix: targetMatrix, - Toolchain: llvmToolchain.Toolchain, + Toolchain: configuredToolchain.Toolchain, Sysroot: metadata.Sysroot(), }) if err != nil { diff --git a/internal/build/c/llvm/toolchain.go b/internal/build/c/llvm/toolchain.go index 3f2216c..4814611 100644 --- a/internal/build/c/llvm/toolchain.go +++ b/internal/build/c/llvm/toolchain.go @@ -12,8 +12,32 @@ type Toolchain struct { c.Toolchain } +// Config contains the target facts required to prepare LLVM commands. +type Config struct { + OS string + Arch string + Sysroot string +} + // New prepares an LLVM Toolchain from commands available in PATH. -func New() (*Toolchain, error) { +func New(config Config) (*Toolchain, error) { + var triple, linkerName string + switch config.Arch + "-" + config.OS { + case "amd64-linux": + triple = "x86_64-linux-gnu" + linkerName = "ld.lld" + case "arm64-linux": + triple = "aarch64-linux-gnu" + linkerName = "ld.lld" + case "amd64-darwin": + triple = "x86_64-apple-macos10.13" + linkerName = "ld64.lld" + case "arm64-darwin": + triple = "arm64-apple-macos11.0" + linkerName = "ld64.lld" + default: + return nil, fmt.Errorf("unsupported LLVM target %s/%s", config.OS, config.Arch) + } find := func(name string) (string, error) { path, err := exec.LookPath(name) if err != nil { @@ -21,11 +45,15 @@ func New() (*Toolchain, error) { } return path, nil } - cc, err := find("clang") + ccPath, err := find("clang") + if err != nil { + return nil, err + } + cxxPath, err := find("clang++") if err != nil { return nil, err } - cxx, err := find("clang++") + linker, err := find(linkerName) if err != nil { return nil, err } @@ -45,5 +73,16 @@ func New() (*Toolchain, error) { if err != nil { return nil, err } - return &Toolchain{Toolchain: c.NewToolchain(cc, cxx, archiver, ranlib, nm, strip)}, nil + cc := []string{ccPath, "--target=" + triple, "-fuse-ld=lld"} + cxx := []string{cxxPath, "--target=" + triple, "-fuse-ld=lld"} + if config.Sysroot != "" { + flag := "--sysroot=" + config.Sysroot + if config.OS == "darwin" { + flag = "-isysroot" + config.Sysroot + } + cc = append(cc, flag) + cxx = append(cxx, flag) + } + toolchain := c.NewToolchain(cc, cxx, []string{linker}, archiver, ranlib, nm, strip) + return &Toolchain{Toolchain: toolchain}, nil } diff --git a/internal/build/c/llvm/toolchain_test.go b/internal/build/c/llvm/toolchain_test.go index 514be3c..cca60a5 100644 --- a/internal/build/c/llvm/toolchain_test.go +++ b/internal/build/c/llvm/toolchain_test.go @@ -3,23 +3,38 @@ package llvm import ( "os" "path/filepath" + "reflect" "testing" ) func TestNewUsesPreparedPath(t *testing.T) { dir := t.TempDir() - for _, name := range []string{"clang", "clang++", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + for _, name := range []string{"clang", "clang++", "ld.lld", "ld64.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { if err := os.WriteFile(filepath.Join(dir, name), []byte("tool"), 0o755); err != nil { t.Fatal(err) } } t.Setenv("PATH", dir) - toolchain, err := New() + toolchain, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: "/sdk"}) if err != nil { t.Fatal(err) } - if toolchain.CC() != filepath.Join(dir, "clang") { - t.Fatalf("CC = %q", toolchain.CC()) + if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=aarch64-linux-gnu", "-fuse-ld=lld", "--sysroot=/sdk"}; !reflect.DeepEqual(got, want) { + t.Fatalf("CC = %q, want %q", got, want) + } + if got, want := toolchain.Linker(), []string{filepath.Join(dir, "ld.lld")}; !reflect.DeepEqual(got, want) { + t.Fatalf("Linker = %q, want %q", got, want) + } + + toolchain, err = New(Config{OS: "darwin", Arch: "amd64", Sysroot: "/sdk"}) + if err != nil { + t.Fatal(err) + } + if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"}; !reflect.DeepEqual(got, want) { + t.Fatalf("Darwin CC = %q, want %q", got, want) + } + if got, want := toolchain.Linker(), []string{filepath.Join(dir, "ld64.lld")}; !reflect.DeepEqual(got, want) { + t.Fatalf("Darwin Linker = %q, want %q", got, want) } } diff --git a/internal/build/c/target.go b/internal/build/c/target.go index f3909a0..5c971e2 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -8,6 +8,7 @@ import ( "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" + "github.com/kballard/go-shellquote" ) const ( @@ -26,15 +27,14 @@ type Config struct { // Target contains prepared C command defaults for one build target. type Target struct { - target string - systemName string - systemProcessor string - targetTriple string - deploymentTarget string - sysroot string - toolchain Toolchain - toolchainFile string - tempDir string + target string + systemName string + systemProcessor string + autotoolsHost string + sysroot string + toolchain Toolchain + toolchainFile string + tempDir string } // Sysroot returns the fixed compatibility sysroot Formula for a built-in C @@ -61,41 +61,37 @@ func NewTarget(config Config) (*Target, error) { if value, _, ok := strings.Cut(config.Matrix, "|"); ok { target = value } - var systemName, processor, triple, deploymentTarget string + var systemName, systemProcessor, autotoolsHost string switch target { case "amd64-linux": systemName = "Linux" - processor = "x86_64" - triple = "x86_64-linux-gnu" + systemProcessor = "x86_64" + autotoolsHost = "x86_64-linux-gnu" case "arm64-linux": systemName = "Linux" - processor = "aarch64" - triple = "aarch64-linux-gnu" + systemProcessor = "aarch64" + autotoolsHost = "aarch64-linux-gnu" case "amd64-darwin": systemName = "Darwin" - processor = "x86_64" - triple = "x86_64-apple-darwin" - deploymentTarget = "10.13" + systemProcessor = "x86_64" + autotoolsHost = "x86_64-apple-darwin" case "arm64-darwin": systemName = "Darwin" - processor = "arm64" - triple = "arm64-apple-darwin" - deploymentTarget = "11.0" + systemProcessor = "arm64" + autotoolsHost = "aarch64-apple-darwin" default: return nil, fmt.Errorf("unsupported C target %s", target) } if err := validateToolchain(config.Toolchain); err != nil { return nil, fmt.Errorf("prepare C target %s: %w", target, err) } - return &Target{ - target: target, - systemName: systemName, - systemProcessor: processor, - targetTriple: triple, - deploymentTarget: deploymentTarget, - sysroot: config.Sysroot, - toolchain: config.Toolchain, + target: target, + systemName: systemName, + systemProcessor: systemProcessor, + autotoolsHost: autotoolsHost, + sysroot: config.Sysroot, + toolchain: config.Toolchain, }, nil } @@ -139,9 +135,11 @@ func (c *Target) Use(cmd build.Command) build.Patch { case "pkg-config": return c.pkgConfigPatch(cmd.Env) case "cc", "gcc", "clang": - return c.compilerPatch(c.toolchain.CC(), cmd.Args) + return commandPatch(c.toolchain.CC()) case "c++", "g++", "clang++": - return c.compilerPatch(c.toolchain.CXX(), cmd.Args) + return commandPatch(c.toolchain.CXX()) + case "ld", "ld.lld", "ld64.lld": + return commandPatch(c.toolchain.Linker()) case "ar", "llvm-ar": return build.Patch{Name: c.toolchain.Archiver()} case "ranlib", "llvm-ranlib": @@ -154,60 +152,23 @@ func (c *Target) Use(cmd build.Command) build.Patch { return build.Patch{} } -func (c *Target) compilerPatch(name string, args []string) build.Patch { - linker := true - for _, arg := range args { - switch arg { - case "-c", "-E", "-S": - linker = false - } - } - flags := missingCompilerFlags(args, c.compilerFlags(linker)) - return build.Patch{Name: name, PrependArg: flags} -} - -func (c *Target) compilerFlags(linker bool) []string { - flags := []string{"--target=" + c.targetTriple} - if c.sysroot != "" { - if c.systemName == "Darwin" { - flags = append(flags, "-isysroot"+c.sysroot) - } else { - flags = append(flags, "--sysroot="+c.sysroot) - } - } - if c.deploymentTarget != "" { - flags = append(flags, "-mmacosx-version-min="+c.deploymentTarget) - } - if linker && c.systemName == "Darwin" { - flags = append(flags, "-fuse-ld=lld") - } - return flags +func commandPatch(command []string) build.Patch { + return build.Patch{Name: command[0], PrependArg: command[1:]} } func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env := append([]string(nil), cmd.Env...) - env = setMissingEnv(env, "CC", c.toolchain.CC()) - env = setMissingEnv(env, "CXX", c.toolchain.CXX()) + env = setMissingEnv(env, "CC", shellquote.Join(c.toolchain.CC()...)) + env = setMissingEnv(env, "CXX", shellquote.Join(c.toolchain.CXX()...)) + env = setMissingEnv(env, "LD", shellquote.Join(c.toolchain.Linker()...)) env = setMissingEnv(env, "AR", c.toolchain.Archiver()) env = setMissingEnv(env, "RANLIB", c.toolchain.Ranlib()) env = setMissingEnv(env, "NM", c.toolchain.NM()) env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) - compilerFlags := c.compilerFlags(false) - env = setEnvFlags(env, "CFLAGS", compilerFlags) - env = setEnvFlags(env, "CXXFLAGS", compilerFlags) - if c.sysroot != "" { - sysrootFlag := "--sysroot=" + c.sysroot - if c.systemName == "Darwin" { - sysrootFlag = "-isysroot" + c.sysroot - } - env = setEnvFlags(env, "CPPFLAGS", []string{sysrootFlag}) - } - env = setEnvFlags(env, "LDFLAGS", c.compilerFlags(true)) - var args []string if !hasOption(cmd.Args, "--host") { - args = append(args, "--host="+c.targetTriple) + args = append(args, "--host="+c.autotoolsHost) } return build.Patch{AppendArg: args, Env: env} } @@ -220,9 +181,6 @@ func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) libDirs, _ := envValue(env, "PKG_CONFIG_PATH") paths := filepath.SplitList(libDirs) - if c.systemName == "Linux" { - paths = append(paths, filepath.Join(c.sysroot, "usr", "lib", c.targetTriple, "pkgconfig")) - } paths = append(paths, filepath.Join(c.sysroot, "usr", "lib64", "pkgconfig"), filepath.Join(c.sysroot, "usr", "lib", "pkgconfig"), @@ -240,7 +198,6 @@ func (c *Target) cmakeToolchain() string { if c.systemName == "Darwin" { values = append(values, [2]string{"CMAKE_OSX_ARCHITECTURES", c.systemProcessor}, - [2]string{"CMAKE_OSX_DEPLOYMENT_TARGET", c.deploymentTarget}, ) if c.sysroot != "" { values = append(values, [2]string{"CMAKE_OSX_SYSROOT", c.sysroot}) @@ -249,22 +206,14 @@ func (c *Target) cmakeToolchain() string { values = append(values, [2]string{"CMAKE_SYSROOT", c.sysroot}) } values = append(values, - [2]string{"CMAKE_C_COMPILER", c.toolchain.CC()}, - [2]string{"CMAKE_CXX_COMPILER", c.toolchain.CXX()}, + [2]string{"CMAKE_C_COMPILER", strings.Join(c.toolchain.CC(), ";")}, + [2]string{"CMAKE_CXX_COMPILER", strings.Join(c.toolchain.CXX(), ";")}, + [2]string{"CMAKE_LINKER", strings.Join(c.toolchain.Linker(), ";")}, [2]string{"CMAKE_AR", c.toolchain.Archiver()}, [2]string{"CMAKE_RANLIB", c.toolchain.Ranlib()}, [2]string{"CMAKE_NM", c.toolchain.NM()}, [2]string{"CMAKE_STRIP", c.toolchain.Strip()}, - [2]string{"CMAKE_C_COMPILER_TARGET", c.targetTriple}, - [2]string{"CMAKE_CXX_COMPILER_TARGET", c.targetTriple}, ) - if c.systemName == "Darwin" { - values = append(values, - [2]string{"CMAKE_EXE_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, - [2]string{"CMAKE_SHARED_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, - [2]string{"CMAKE_MODULE_LINKER_FLAGS_INIT", "-fuse-ld=lld"}, - ) - } if c.sysroot != "" { values = append(values, [2]string{"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER"}, @@ -281,12 +230,23 @@ func (c *Target) cmakeToolchain() string { } func validateToolchain(toolchain Toolchain) error { + commands := []struct { + name string + command []string + }{ + {"CC", toolchain.CC()}, + {"CXX", toolchain.CXX()}, + {"linker", toolchain.Linker()}, + } + for _, command := range commands { + if len(command.command) == 0 || command.command[0] == "" { + return fmt.Errorf("%s is required", command.name) + } + } tools := []struct { name string path string }{ - {"CC", toolchain.CC()}, - {"CXX", toolchain.CXX()}, {"archiver", toolchain.Archiver()}, {"ranlib", toolchain.Ranlib()}, {"nm", toolchain.NM()}, @@ -321,45 +281,6 @@ func hasCMakeToolchain(args []string) bool { return false } -func missingCompilerFlags(args, defaults []string) []string { - var flags []string - for _, flag := range defaults { - if strings.HasPrefix(flag, "--target=") && hasTargetFlag(args) { - continue - } - if (strings.HasPrefix(flag, "--sysroot=") || strings.HasPrefix(flag, "-isysroot")) && hasSysrootFlag(args) { - continue - } - if strings.HasPrefix(flag, "-mmacosx-version-min=") && hasFlag(args, "-mmacosx-version-min") { - continue - } - if strings.HasPrefix(flag, "-fuse-ld=") && hasFlag(args, "-fuse-ld") { - continue - } - flags = append(flags, flag) - } - return flags -} - -func hasTargetFlag(args []string) bool { - return hasFlag(args, "--target", "-target") -} - -func hasSysrootFlag(args []string) bool { - return hasFlag(args, "--sysroot", "-sysroot", "-isysroot") -} - -func hasFlag(args []string, names ...string) bool { - for _, arg := range args { - for _, name := range names { - if arg == name || strings.HasPrefix(arg, name+"=") || name == "-isysroot" && strings.HasPrefix(arg, name) { - return true - } - } - } - return false -} - func hasOption(args []string, name string) bool { for _, arg := range args { if arg == name || strings.HasPrefix(arg, name+"=") { @@ -379,17 +300,6 @@ func envValue(env []string, key string) (string, bool) { return "", false } -func setEnv(env []string, key, value string) []string { - prefix := key + "=" - for i := len(env) - 1; i >= 0; i-- { - if strings.HasPrefix(env[i], prefix) { - env[i] = prefix + value - return env - } - } - return append(env, prefix+value) -} - func setMissingEnv(env []string, key, value string) []string { if _, ok := envValue(env, key); ok { return env @@ -397,35 +307,6 @@ func setMissingEnv(env []string, key, value string) []string { return append(env, key+"="+value) } -func setEnvFlags(env []string, key string, defaults []string) []string { - value, _ := envValue(env, key) - original := value - args := strings.Fields(value) - for _, flag := range defaults { - if strings.HasPrefix(flag, "--target=") && hasTargetFlag(args) { - continue - } - if (strings.HasPrefix(flag, "--sysroot=") || strings.HasPrefix(flag, "-isysroot")) && hasSysrootFlag(args) { - continue - } - if strings.HasPrefix(flag, "-mmacosx-version-min=") && hasFlag(args, "-mmacosx-version-min") { - continue - } - if strings.HasPrefix(flag, "-fuse-ld=") && hasFlag(args, "-fuse-ld") { - continue - } - if value != "" { - value += " " - } - value += flag - args = append(args, flag) - } - if value != original { - return setEnv(env, key, value) - } - return env -} - func cmakeEscape(value string) string { value = strings.ReplaceAll(value, "\\", "/") return strings.ReplaceAll(value, "\"", "\\\"") diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index da36ce7..ec8f02a 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -12,10 +12,18 @@ import ( "github.com/goplus/llar/mod/module" ) -func fakeToolchain() Toolchain { +func fakeToolchain(t *testing.T, targetOS string, compilerArgs ...string) Toolchain { + t.Helper() + cc := append([]string{"/llvm/bin/clang"}, compilerArgs...) + cxx := append([]string{"/llvm/bin/clang++"}, compilerArgs...) + linker := "/llvm/bin/ld.lld" + if targetOS == "darwin" { + linker = "/llvm/bin/ld64.lld" + } return NewToolchain( - "/llvm/bin/clang", - "/llvm/bin/clang++", + cc, + cxx, + []string{linker}, "/llvm/bin/llvm-ar", "/llvm/bin/llvm-ranlib", "/llvm/bin/llvm-nm", @@ -44,28 +52,30 @@ func TestSysroot(t *testing.T) { } func TestDarwinTarget(t *testing.T) { - target, err := NewTarget(Config{Matrix: "amd64-darwin|shared", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) + toolchain := fakeToolchain(t, "darwin", "--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk") + target, err := NewTarget(Config{Matrix: "amd64-darwin|shared", Toolchain: toolchain, Sysroot: "/sdk"}) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = target.Close() }) patch := target.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) - want := []string{"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13"} + want := []string{"--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"} if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } patch = target.Use(build.Command{Name: "cc", Args: []string{"a.o", "-shared"}}) - if !slices.Contains(patch.PrependArg, "-fuse-ld=lld") { - t.Fatalf("link PrependArg = %q, want LLD selection", patch.PrependArg) + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) } patch = target.Use(build.Command{ Name: "cc", - Args: []string{"--target=custom", "-isysroot/custom", "-mmacosx-version-min=12.0", "-fuse-ld=custom"}, + Args: []string{"--target=custom", "-isysroot/custom", "-fuse-ld=custom"}, }) - if len(patch.PrependArg) != 0 { - t.Fatalf("explicit Darwin flags were duplicated: %q", patch.PrependArg) + want = []string{"--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"} + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("PrependArg = %q, want prepared defaults %q", patch.PrependArg, want) } patch = target.Use(build.Command{Name: "/src/configure"}) @@ -73,9 +83,8 @@ func TestDarwinTarget(t *testing.T) { t.Fatalf("configure AppendArg = %q, want %q", got, want) } for key, values := range map[string][]string{ - "CFLAGS": {"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13"}, - "CPPFLAGS": {"-isysroot/sdk"}, - "LDFLAGS": {"--target=x86_64-apple-darwin", "-isysroot/sdk", "-mmacosx-version-min=10.13", "-fuse-ld=lld"}, + "CC": {"/llvm/bin/clang", "--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"}, + "LD": {"/llvm/bin/ld64.lld"}, } { got, _ := envValue(patch.Env, key) for _, value := range values { @@ -95,8 +104,9 @@ func TestDarwinTarget(t *testing.T) { "set(CMAKE_SYSTEM_NAME \"Darwin\")", "CMAKE_OSX_ARCHITECTURES", "CMAKE_OSX_SYSROOT", - "CMAKE_OSX_DEPLOYMENT_TARGET", - "x86_64-apple-darwin", + "CMAKE_LINKER", + "/llvm/bin/ld64.lld", + "x86_64-apple-macos10.13", "-fuse-ld=lld", } { if !strings.Contains(content, want) { @@ -105,28 +115,29 @@ func TestDarwinTarget(t *testing.T) { } } -func TestDarwinArm64DeploymentTarget(t *testing.T) { - target, err := NewTarget(Config{Matrix: "arm64-darwin", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) +func TestDarwinArm64Compiler(t *testing.T) { + toolchain := fakeToolchain(t, "darwin", "--target=arm64-apple-macos11.0", "-fuse-ld=lld", "-isysroot/sdk") + target, err := NewTarget(Config{Matrix: "arm64-darwin", Toolchain: toolchain, Sysroot: "/sdk"}) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = target.Close() }) patch := target.Use(build.Command{Name: "cc"}) - if !slices.Contains(patch.PrependArg, "-mmacosx-version-min=11.0") { - t.Fatalf("PrependArg = %q, want arm64 deployment target", patch.PrependArg) + if !slices.Contains(patch.PrependArg, "--target=arm64-apple-macos11.0") { + t.Fatalf("PrependArg = %q, want prepared arm64 compiler target", patch.PrependArg) } } func TestBootstrapTargetOmitsSysroot(t *testing.T) { - c, err := NewTarget(Config{Matrix: "arm64-linux", Toolchain: fakeToolchain()}) + c, err := NewTarget(Config{Matrix: "arm64-linux", Toolchain: fakeToolchain(t, "linux", "--target=aarch64-linux-gnu", "-fuse-ld=lld")}) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = c.Close() }) patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) - if got, want := patch.PrependArg, []string{"--target=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { + if got, want := patch.PrependArg, []string{"--target=aarch64-linux-gnu", "-fuse-ld=lld"}; !reflect.DeepEqual(got, want) { t.Fatalf("PrependArg = %q, want %q", got, want) } if patch := c.Use(build.Command{Name: "pkg-config"}); patch.Env != nil { @@ -156,7 +167,7 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { t.Fatal(err) } content := string(data) - for _, want := range []string{"CMAKE_SYSTEM_NAME", "aarch64", "/llvm/bin/clang", "aarch64-linux-gnu", "/sdk"} { + for _, want := range []string{"CMAKE_SYSTEM_NAME", "CMAKE_LINKER", "aarch64", "/llvm/bin/clang", "/llvm/bin/ld.lld", "aarch64-linux-gnu", "/sdk", "-fuse-ld=lld"} { if !strings.Contains(content, want) { t.Fatalf("toolchain file does not contain %q:\n%s", want, content) } @@ -191,17 +202,25 @@ func TestUseDirectCommands(t *testing.T) { if patch.Name != "/llvm/bin/clang" { t.Fatalf("Name = %q", patch.Name) } - want := []string{"--target=aarch64-linux-gnu", "--sysroot=/sdk"} + want := []string{"--target=aarch64-linux-gnu", "-fuse-ld=lld", "--sysroot=/sdk"} if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(build.Command{Name: "cc", Args: []string{"--target=custom", "--sysroot=/custom"}}) - if len(patch.PrependArg) != 0 { - t.Fatalf("explicit compiler flags were duplicated: %q", patch.PrependArg) + patch = c.Use(build.Command{Name: "cc", Args: []string{"a.o", "-o", "a"}}) + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) + } + patch = c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c", "--target=custom", "--sysroot=/custom"}}) + if !reflect.DeepEqual(patch.PrependArg, want) { + t.Fatalf("PrependArg = %q, want prepared defaults %q", patch.PrependArg, want) } if patch := c.Use(build.Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { t.Fatalf("explicit compiler path was rewritten: %+v", patch) } + patch = c.Use(build.Command{Name: "ld"}) + if patch.Name != "/llvm/bin/ld.lld" { + t.Fatalf("linker Name = %q, want /llvm/bin/ld.lld", patch.Name) + } } func TestUseAutotools(t *testing.T) { @@ -214,9 +233,12 @@ func TestUseAutotools(t *testing.T) { if got, _ := envValue(patch.Env, "CC"); got != "/custom/cc" { t.Fatalf("CC override = %q, want /custom/cc", got) } - if got, _ := envValue(patch.Env, "CFLAGS"); got != "-O2 --target=custom --sysroot=/sdk" { + if got, _ := envValue(patch.Env, "CFLAGS"); got != "-O2 --target=custom" { t.Fatalf("CFLAGS = %q", got) } + if got, _ := envValue(patch.Env, "LD"); got != "/llvm/bin/ld.lld" { + t.Fatalf("LD = %q, want /llvm/bin/ld.lld", got) + } if got, want := patch.AppendArg, []string{"--host=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } @@ -230,7 +252,7 @@ func TestUsePkgConfig(t *testing.T) { t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) } got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") - for _, want := range []string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "aarch64-linux-gnu", "pkgconfig")} { + for _, want := range []string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { if !slices.Contains(filepath.SplitList(got), want) { t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) } @@ -243,7 +265,8 @@ func TestUsePkgConfig(t *testing.T) { func newTestTarget(t *testing.T) *Target { t.Helper() - c, err := NewTarget(Config{Matrix: "arm64-linux|shared", Toolchain: fakeToolchain(), Sysroot: "/sdk"}) + toolchain := fakeToolchain(t, "linux", "--target=aarch64-linux-gnu", "-fuse-ld=lld", "--sysroot=/sdk") + c, err := NewTarget(Config{Matrix: "arm64-linux|shared", Toolchain: toolchain, Sysroot: "/sdk"}) if err != nil { t.Fatal(err) } diff --git a/internal/build/c/toolchain.go b/internal/build/c/toolchain.go index 156a3a0..5181035 100644 --- a/internal/build/c/toolchain.go +++ b/internal/build/c/toolchain.go @@ -1,20 +1,22 @@ package c -// Toolchain contains prepared C-family command paths. +// Toolchain contains prepared C-family commands. type Toolchain struct { - cc string - cxx string + cc []string + cxx []string + linker []string archiver string ranlib string nm string strip string } -// NewToolchain creates a Toolchain from prepared command paths. -func NewToolchain(cc, cxx, archiver, ranlib, nm, strip string) Toolchain { +// NewToolchain creates a Toolchain from prepared target commands. +func NewToolchain(cc, cxx, linker []string, archiver, ranlib, nm, strip string) Toolchain { return Toolchain{ - cc: cc, - cxx: cxx, + cc: append([]string(nil), cc...), + cxx: append([]string(nil), cxx...), + linker: append([]string(nil), linker...), archiver: archiver, ranlib: ranlib, nm: nm, @@ -22,8 +24,9 @@ func NewToolchain(cc, cxx, archiver, ranlib, nm, strip string) Toolchain { } } -func (t Toolchain) CC() string { return t.cc } -func (t Toolchain) CXX() string { return t.cxx } +func (t Toolchain) CC() []string { return append([]string(nil), t.cc...) } +func (t Toolchain) CXX() []string { return append([]string(nil), t.cxx...) } +func (t Toolchain) Linker() []string { return append([]string(nil), t.linker...) } func (t Toolchain) Archiver() string { return t.archiver } func (t Toolchain) Ranlib() string { return t.ranlib } func (t Toolchain) NM() string { return t.nm } diff --git a/internal/build/c/toolchain_test.go b/internal/build/c/toolchain_test.go index 755d121..a911635 100644 --- a/internal/build/c/toolchain_test.go +++ b/internal/build/c/toolchain_test.go @@ -1,16 +1,30 @@ package c -import "testing" +import ( + "reflect" + "testing" +) func TestToolchain(t *testing.T) { - toolchain := NewToolchain("cc", "c++", "ar", "ranlib", "nm", "strip") + toolchain := NewToolchain([]string{"cc", "--target=aarch64-linux-gnu"}, []string{"c++", "--target=aarch64-linux-gnu"}, []string{"ld.lld"}, "ar", "ranlib", "nm", "strip") + for _, test := range []struct { + name string + got []string + want []string + }{ + {name: "CC", got: toolchain.CC(), want: []string{"cc", "--target=aarch64-linux-gnu"}}, + {name: "CXX", got: toolchain.CXX(), want: []string{"c++", "--target=aarch64-linux-gnu"}}, + {name: "Linker", got: toolchain.Linker(), want: []string{"ld.lld"}}, + } { + if !reflect.DeepEqual(test.got, test.want) { + t.Fatalf("%s = %q, want %q", test.name, test.got, test.want) + } + } for _, test := range []struct { name string got string want string }{ - {name: "CC", got: toolchain.CC(), want: "cc"}, - {name: "CXX", got: toolchain.CXX(), want: "c++"}, {name: "Archiver", got: toolchain.Archiver(), want: "ar"}, {name: "Ranlib", got: toolchain.Ranlib(), want: "ranlib"}, {name: "NM", got: toolchain.NM(), want: "nm"}, From cb88b29a910942e1b57ecd87cacae54122a7da3f Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 17:51:22 +0800 Subject: [PATCH 07/39] fix(test): reject cross-target execution --- cmd/llar/internal/make.go | 6 +++++- cmd/llar/internal/make_test.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index f723081..0caba3d 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -140,9 +140,13 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, if values := matrix.Require["arch"]; len(values) > 0 { targetArch = values[0] } + crossCompile := targetOS != runtime.GOOS || targetArch != runtime.GOARCH + if runTest && crossCompile { + return fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + } var targetRoot module.Version var useCTarget bool - if targetOS != runtime.GOOS || targetArch != runtime.GOARCH { + if crossCompile { // TODO: Add other language target policies alongside this C case when // they provide build.Target implementations. cSysroot, ok := c.Sysroot(targetOS, targetArch) diff --git a/cmd/llar/internal/make_test.go b/cmd/llar/internal/make_test.go index 8b330cb..34a338e 100644 --- a/cmd/llar/internal/make_test.go +++ b/cmd/llar/internal/make_test.go @@ -167,6 +167,23 @@ func TestRunMakeReturnsMatrixErrorBeforeStore(t *testing.T) { } } +func TestBuildModuleRejectsCrossTargetTest(t *testing.T) { + targetOS, targetArch := "linux", "amd64" + if runtime.GOOS == targetOS && runtime.GOARCH == targetArch { + targetArch = "arm64" + } + matrix := formula.Matrix{Require: map[string][]string{ + "os": {targetOS}, + "arch": {targetArch}, + }} + + err := buildModule(context.Background(), nil, "owner/repo", "v1.0.0", matrix, true) + want := fmt.Sprintf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + if err == nil || err.Error() != want { + t.Fatalf("buildModule error = %v, want %q", err, want) + } +} + func TestNewRemoteStore(t *testing.T) { isolatedWorkspaceDir(t) store, err := newRemoteStore() From 3a41331b7f148f8a1c786061c73b7763cb13f390 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 17:57:02 +0800 Subject: [PATCH 08/39] refactor(crosscompile): hide default sysroot graph --- cmd/llar/internal/make.go | 23 ++++++++++------------- internal/modules/load.go | 3 --- internal/modules/load_test.go | 20 -------------------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 0caba3d..525ec1b 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -160,13 +160,17 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, FormulaStore: store, Matrix: matrix, } - if targetRoot != (module.Version{}) { - loadOpts.Roots = []module.Version{targetRoot} - } mods, err := modules.Load(ctx, root, loadOpts) if err != nil { return fmt.Errorf("failed to load modules: %w", err) } + var sysrootMods []*modules.Module + if targetRoot != (module.Version{}) { + sysrootMods, err = modules.Load(ctx, targetRoot, loadOpts) + if err != nil { + return fmt.Errorf("failed to load sysroot %s@%s: %w", targetRoot.Path, targetRoot.Version, err) + } + } var buildOutput io.Writer = io.Discard if makeVerbose { @@ -208,14 +212,7 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, target = bootstrapTarget if targetRoot != (module.Version{}) { - // A successful Load includes every Options.Roots path in the selected graph. - var selectedSysroot *modules.Module - for _, mod := range mods { - if mod.Path == targetRoot.Path { - selectedSysroot = mod - break - } - } + selectedSysroot := sysrootMods[0] sysrootOpts := buildOpts sysrootOpts.RunTest = false @@ -224,11 +221,11 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, if err != nil { return fmt.Errorf("failed to create sysroot builder: %w", err) } - sysrootResults, err := sysrootBuilder.Build(ctx, []*modules.Module{selectedSysroot}) + sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) if err != nil { return fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) } - metadata, err := ccmetadata.Parse(sysrootResults[0].Metadata) + metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) if err != nil { return fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) } diff --git a/internal/modules/load.go b/internal/modules/load.go index e2cc2a1..185ba01 100644 --- a/internal/modules/load.go +++ b/internal/modules/load.go @@ -61,8 +61,6 @@ type Options struct { // FormulaStore is the store for downloading and caching formulas. FormulaStore repo.Store Matrix classfile.Matrix - // Roots adds caller-supplied requirements to the main module before MVS. - Roots []module.Version } func latestVersion(ctx context.Context, modPath string, repo vcs.Repo, comparator func(v1, v2 module.Version) int) (version string, err error) { @@ -198,7 +196,6 @@ func Load(ctx context.Context, main module.Version, opts Options) ([]*Module, er if err != nil { return nil, err } - mainDeps = append(mainDeps, opts.Roots...) cmp := func(p, v1, v2 string) int { // none is an internal version for MVS, which means the smallest if v1 == "none" && v2 != "none" { diff --git a/internal/modules/load_test.go b/internal/modules/load_test.go index 79ab9b1..49a7727 100644 --- a/internal/modules/load_test.go +++ b/internal/modules/load_test.go @@ -457,26 +457,6 @@ func TestLoad_InjectsTargetBeforeFilterAndOnRequire(t *testing.T) { } } -func TestLoad_AddsRootsToMainRequirements(t *testing.T) { - store := setupTestStore(t, "testdata/load") - main := module.Version{Path: "towner/withdeps", Version: "1.0.0"} - root := module.Version{Path: "towner/leafmod", Version: "2.0.0"} - - mods, err := Load(context.Background(), main, Options{ - FormulaStore: store, - Roots: []module.Version{root}, - }) - if err != nil { - t.Fatal(err) - } - if got := findModule(mods, root.Path); got == nil || got.Version != root.Version { - t.Fatalf("root module = %+v, want %+v", got, root) - } - if got, want := depVersions(mods[0]), []module.Version{root}; !slices.Equal(got, want) { - t.Fatalf("main deps = %+v, want %+v", got, want) - } -} - func TestLoad_FilterRejectsSelectedMatrix(t *testing.T) { store := setupTestStore(t, "testdata/load") ctx := context.Background() From ee7e86b96ac756d522c8ba66c6ac4c9d5d0212a7 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 18:10:54 +0800 Subject: [PATCH 09/39] feat(crosscompile): support formula-owned sysroots --- cmd/llar/internal/make.go | 2 +- x/autotools/autotools.go | 23 ++++++++++++++++++ x/autotools/autotools_test.go | 44 +++++++++++++++++++++++++++++++++++ x/cmake/cmake.go | 10 ++++++++ x/cmake/cmake_test.go | 19 +++++++++++++++ 5 files changed, 97 insertions(+), 1 deletion(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 525ec1b..29335f4 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -152,7 +152,7 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, cSysroot, ok := c.Sysroot(targetOS, targetArch) useCTarget = ok _, customLibc := matrix.Require["libc"] - if useCTarget && (targetOS != "linux" || !customLibc) && root.Path != cSysroot.Path { + if useCTarget && !customLibc && root.Path != cSysroot.Path { targetRoot = cSysroot } } diff --git a/x/autotools/autotools.go b/x/autotools/autotools.go index bfa59b4..c2b6001 100644 --- a/x/autotools/autotools.go +++ b/x/autotools/autotools.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "github.com/goplus/llar/internal/execbroker" "github.com/goplus/llar/x/pkgconfig" @@ -29,6 +30,28 @@ func New(sourceDir, buildDir, installDir string) *AutoTools { // Source overrides the source directory. func (a *AutoTools) Source(dir string) { a.sourceDir = dir } +// Sysroot sets the target system root for compiler, linker, and pkg-config +// lookups. Both compiler spellings are supplied so the same Formula works for +// generic and Apple targets. +func (a *AutoTools) Sysroot(root string) { + for _, key := range []string{"CPPFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS"} { + appendFlag(key, "--sysroot="+root) + appendFlag(key, "-isysroot"+root) + } + if _, ok := os.LookupEnv("PKG_CONFIG_SYSROOT_DIR"); !ok { + os.Setenv("PKG_CONFIG_SYSROOT_DIR", root) + } + if _, ok := os.LookupEnv("PKG_CONFIG_LIBDIR"); !ok { + paths := filepath.SplitList(os.Getenv("PKG_CONFIG_PATH")) + paths = append(paths, + filepath.Join(root, "usr", "lib64", "pkgconfig"), + filepath.Join(root, "usr", "lib", "pkgconfig"), + filepath.Join(root, "usr", "share", "pkgconfig"), + ) + os.Setenv("PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) + } +} + // Use configures the process environment so that Autotools, compilers, and // pkg-config find a non-system dependency installed at root. func (a *AutoTools) Use(root string) { diff --git a/x/autotools/autotools_test.go b/x/autotools/autotools_test.go index 2f24a47..b94abe6 100644 --- a/x/autotools/autotools_test.go +++ b/x/autotools/autotools_test.go @@ -5,6 +5,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "testing" ) @@ -115,6 +116,49 @@ func TestUsePartialDirs(t *testing.T) { } } +func TestSysroot(t *testing.T) { + for _, key := range []string{"CPPFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS"} { + t.Setenv(key, "-existing") + } + for _, key := range []string{"PKG_CONFIG_SYSROOT_DIR", "PKG_CONFIG_LIBDIR"} { + value, ok := os.LookupEnv(key) + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if ok { + _ = os.Setenv(key, value) + } else { + _ = os.Unsetenv(key) + } + }) + } + t.Setenv("PKG_CONFIG_PATH", "/deps/lib/pkgconfig") + + a := New("", "", "") + a.Sysroot("/sdk") + + for _, key := range []string{"CPPFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS"} { + if got, want := os.Getenv(key), "-existing --sysroot=/sdk -isysroot/sdk"; got != want { + t.Errorf("%s = %q, want %q", key, got, want) + } + } + if got := os.Getenv("PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { + t.Errorf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + } + libDirs := filepath.SplitList(os.Getenv("PKG_CONFIG_LIBDIR")) + for _, want := range []string{ + "/deps/lib/pkgconfig", + filepath.Join("/sdk", "usr", "lib64", "pkgconfig"), + filepath.Join("/sdk", "usr", "lib", "pkgconfig"), + filepath.Join("/sdk", "usr", "share", "pkgconfig"), + } { + if !slices.Contains(libDirs, want) { + t.Errorf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) + } + } +} + func TestSource(t *testing.T) { a := New("original", "", "") a.Source("/new/src") diff --git a/x/cmake/cmake.go b/x/cmake/cmake.go index f0575af..e7c2865 100644 --- a/x/cmake/cmake.go +++ b/x/cmake/cmake.go @@ -49,6 +49,16 @@ func (c *CMake) BuildType(name string) { c.buildType = name } // Toolchain sets CMAKE_TOOLCHAIN_FILE. func (c *CMake) Toolchain(path string) { c.toolchain = path } +// Sysroot sets the target system root for both generic and Apple targets. +func (c *CMake) Sysroot(root string) { + c.Define("CMAKE_SYSROOT", root) + c.Define("CMAKE_OSX_SYSROOT", root) + c.Define("CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER") + c.Define("CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY") + c.Define("CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY") + c.Define("CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY") +} + // Define adds a -D:STRING= definition. func (c *CMake) Define(key, value string) { c.defines[key] = defineValue{value: value, typeName: "STRING"} diff --git a/x/cmake/cmake_test.go b/x/cmake/cmake_test.go index 45f3113..e56fc34 100644 --- a/x/cmake/cmake_test.go +++ b/x/cmake/cmake_test.go @@ -126,6 +126,25 @@ func TestDefinesArgs(t *testing.T) { } } +func TestSysroot(t *testing.T) { + c := New("", "", "") + c.Sysroot("/sdk") + + joined := strings.Join(c.definesArgs(), " ") + for _, want := range []string{ + "-DCMAKE_SYSROOT:STRING=/sdk", + "-DCMAKE_OSX_SYSROOT:STRING=/sdk", + "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM:STRING=NEVER", + "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY:STRING=ONLY", + "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE:STRING=ONLY", + "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE:STRING=ONLY", + } { + if !strings.Contains(joined, want) { + t.Errorf("definesArgs missing %q, got %q", want, joined) + } + } +} + func TestDefinesArgsEmpty(t *testing.T) { c := New("", "", "") if args := c.definesArgs(); args != nil { From 7ea2eeda1d381d3da3cfce62976ff9ca69f2ea52 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 18:14:21 +0800 Subject: [PATCH 10/39] fix(crosscompile): detect configure host support --- internal/build/c/target.go | 34 ++++++++++++++++++++------------- internal/build/c/target_test.go | 21 ++++++++++++++++---- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/internal/build/c/target.go b/internal/build/c/target.go index 5c971e2..e03c430 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -1,6 +1,7 @@ package c import ( + "bytes" "fmt" "os" "path/filepath" @@ -166,11 +167,27 @@ func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env = setMissingEnv(env, "NM", c.toolchain.NM()) env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) - var args []string - if !hasOption(cmd.Args, "--host") { - args = append(args, "--host="+c.autotoolsHost) + for _, arg := range cmd.Args { + if arg == "--host" || strings.HasPrefix(arg, "--host=") { + return build.Patch{Env: env} + } + } + + configurePath := cmd.Name + if !filepath.IsAbs(configurePath) { + configurePath = filepath.Join(cmd.Dir, configurePath) } - return build.Patch{AppendArg: args, Env: env} + data, err := os.ReadFile(configurePath) + if err != nil { + panic(fmt.Errorf("inspect configure options for %s: %w", c.target, err)) + } + // Only scripts that declare --host receive the Autoconf target tuple. + // For example, zlib's custom configure declares CHOST but rejects --host. + patch := build.Patch{Env: env} + if bytes.Contains(data, []byte("--host")) { + patch.AppendArg = []string{"--host=" + c.autotoolsHost} + } + return patch } func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { @@ -281,15 +298,6 @@ func hasCMakeToolchain(args []string) bool { return false } -func hasOption(args []string, name string) bool { - for _, arg := range args { - if arg == name || strings.HasPrefix(arg, name+"=") { - return true - } - } - return false -} - func envValue(env []string, key string) (string, bool) { prefix := key + "=" for i := len(env) - 1; i >= 0; i-- { diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index ec8f02a..131d4ca 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -78,9 +78,13 @@ func TestDarwinTarget(t *testing.T) { t.Fatalf("PrependArg = %q, want prepared defaults %q", patch.PrependArg, want) } - patch = target.Use(build.Command{Name: "/src/configure"}) - if got, want := patch.AppendArg, []string{"--host=x86_64-apple-darwin"}; !reflect.DeepEqual(got, want) { - t.Fatalf("configure AppendArg = %q, want %q", got, want) + configure := filepath.Join(t.TempDir(), "configure") + if err := os.WriteFile(configure, []byte("#!/bin/sh\nCHOST=${CHOST-}\n"), 0o755); err != nil { + t.Fatal(err) + } + patch = target.Use(build.Command{Name: configure}) + if len(patch.AppendArg) != 0 { + t.Fatalf("configure AppendArg = %q, want no unsupported arguments", patch.AppendArg) } for key, values := range map[string][]string{ "CC": {"/llvm/bin/clang", "--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"}, @@ -225,8 +229,12 @@ func TestUseDirectCommands(t *testing.T) { func TestUseAutotools(t *testing.T) { c := newTestTarget(t) + configure := filepath.Join(t.TempDir(), "configure") + if err := os.WriteFile(configure, []byte("#!/bin/sh\n# options: --build=BUILD --host=HOST\n"), 0o755); err != nil { + t.Fatal(err) + } patch := c.Use(build.Command{ - Name: "/src/configure", + Name: configure, Args: []string{"--build=x86_64-apple-darwin"}, Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, }) @@ -242,6 +250,11 @@ func TestUseAutotools(t *testing.T) { if got, want := patch.AppendArg, []string{"--host=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } + + patch = c.Use(build.Command{Name: configure, Args: []string{"--host=custom-linux"}}) + if len(patch.AppendArg) != 0 { + t.Fatalf("explicit host AppendArg = %q, want no duplicate host", patch.AppendArg) + } } func TestUsePkgConfig(t *testing.T) { From 257fecbe02c96d930c33fe82de2a784b1a940bbb Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 18:29:53 +0800 Subject: [PATCH 11/39] docs(crosscompile): explain sysroot formula loading --- cmd/llar/internal/make.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 29335f4..d31ae3a 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -166,6 +166,8 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, } var sysrootMods []*modules.Module if targetRoot != (module.Version{}) { + // The default sysroot has no dependencies, but modules.Load still owns + // selecting the Formula whose fromVer applies to targetRoot.Version. sysrootMods, err = modules.Load(ctx, targetRoot, loadOpts) if err != nil { return fmt.Errorf("failed to load sysroot %s@%s: %w", targetRoot.Path, targetRoot.Version, err) From 35a3d6404479844d94b0b2383870f613d2d7bcc4 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 19:11:07 +0800 Subject: [PATCH 12/39] test(crosscompile): add cross-runner zlib e2e --- .github/workflows/crosscompile-e2e.yml | 227 ++++++++++++++++++ testdata/crosscompile-e2e/consumer.c | 28 +++ .../bminor/glibc/glibc-2.17/Glibc_llar.gox | 8 + .../formulas/bminor/glibc/versions.json | 4 + .../macosx-sdks/sdk-14_5/MacosxSdks_llar.gox | 8 + .../joseluisq/macosx-sdks/versions.json | 4 + testdata/crosscompile-e2e/install-sysroot | 5 + 7 files changed, 284 insertions(+) create mode 100644 .github/workflows/crosscompile-e2e.yml create mode 100644 testdata/crosscompile-e2e/consumer.c create mode 100644 testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox create mode 100644 testdata/crosscompile-e2e/formulas/bminor/glibc/versions.json create mode 100644 testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox create mode 100644 testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/versions.json create mode 100644 testdata/crosscompile-e2e/install-sysroot diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml new file mode 100644 index 0000000..a0245fc --- /dev/null +++ b/.github/workflows/crosscompile-e2e.yml @@ -0,0 +1,227 @@ +name: Cross Compile E2E + +on: + push: + pull_request: + +permissions: + contents: read + +concurrency: + group: crosscompile-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-linux-arm64: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.x + + - name: Install LLVM and the arm64 runtime + run: | + sudo apt-get update + sudo apt-get install --yes clang-18 lld-18 llvm-18 gcc-aarch64-linux-gnu rpm2cpio cpio + echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" + + - name: Prepare glibc 2.17 arm64 sysroot + run: | + package_dir="$RUNNER_TEMP/glibc-packages" + sysroot_dir="$RUNNER_TEMP/glibc-sysroot" + base_url="https://vault.centos.org/altarch/7.9.2009/os/aarch64/Packages" + mkdir -p "$package_dir" "$sysroot_dir" + + for package in \ + glibc-2.17-317.el7.aarch64.rpm \ + glibc-devel-2.17-317.el7.aarch64.rpm \ + glibc-headers-2.17-317.el7.aarch64.rpm \ + kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm + do + curl --fail --location --retry 3 \ + "$base_url/$package" \ + --output "$package_dir/$package" + done + + cd "$package_dir" + sha256sum --check <<'EOF' + ca1976545bcd4a6c099f1b3e2505469083b6bb4901d2517275a8da4138c0b21a glibc-2.17-317.el7.aarch64.rpm + 458738f7b415fb73473725b6bef340234954e4eb00731cbf8aa1afad5ccd12f6 glibc-devel-2.17-317.el7.aarch64.rpm + 58a5802d026435b36d7080ea4943868a82fc41642a2b0e475239bd6aaa5882cd glibc-headers-2.17-317.el7.aarch64.rpm + 56fffc40800cd7c30830009c860f04c8dba1110a6271efd804a0bb01e0bdf8a4 kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm + EOF + for package in *.rpm; do + rpm2cpio "$package" | (cd "$sysroot_dir" && cpio -idm --quiet) + done + echo "LLAR_E2E_SYSROOT=$sysroot_dir" >> "$GITHUB_ENV" + + - name: Prepare formula and source repositories + run: | + formula_repo="$RUNNER_TEMP/llarhub" + source_repo="$RUNNER_TEMP/glibc-source" + + mkdir -p "$formula_repo" "$source_repo" + cp -R testdata/kodo-e2e/formulas/. "$formula_repo/" + cp -R testdata/crosscompile-e2e/formulas/. "$formula_repo/" + git -C "$formula_repo" init --initial-branch=main + git -C "$formula_repo" config user.email crosscompile-e2e@example.com + git -C "$formula_repo" config user.name "Cross Compile E2E" + git -C "$formula_repo" add . + git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" + + cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" + chmod +x "$source_repo/install-sysroot" + git -C "$source_repo" init --initial-branch=main + git -C "$source_repo" config user.email crosscompile-e2e@example.com + git -C "$source_repo" config user.name "Cross Compile E2E" + git -C "$source_repo" add . + git -C "$source_repo" commit --quiet -m "Add sysroot installer" + git -C "$source_repo" tag glibc-2.17 + + git config --global protocol.file.allow always + git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" + git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" + + - name: Cross compile zlib + run: | + go build -ldflags="-checklinkname=0" -o "$RUNNER_TEMP/llar" ./cmd/llar + "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ + --os linux \ + --arch arm64 \ + --output "$RUNNER_TEMP/zlib-linux-arm64" \ + --verbose + + - name: Upload zlib artifact + uses: actions/upload-artifact@v4 + with: + name: zlib-linux-arm64 + path: ${{ runner.temp }}/zlib-linux-arm64 + if-no-files-found: error + + run-linux-arm64: + needs: build-linux-arm64 + runs-on: ubuntu-24.04-arm + timeout-minutes: 10 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Download zlib artifact + uses: actions/download-artifact@v4 + with: + name: zlib-linux-arm64 + path: ${{ runner.temp }}/zlib-linux-arm64 + + - name: Build and run consumer + run: | + test "$(uname -m)" = "aarch64" + cc \ + -I"$RUNNER_TEMP/zlib-linux-arm64/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" \ + -o "$RUNNER_TEMP/zlib-consumer" + file "$RUNNER_TEMP/zlib-consumer" + "$RUNNER_TEMP/zlib-consumer" + + build-darwin-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 25 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.x + + - name: Install LLVM + run: | + sudo apt-get update + sudo apt-get install --yes clang-18 lld-18 llvm-18 + echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" + + - name: Prepare macOS 14.5 SDK + run: | + archive="$RUNNER_TEMP/MacOSX14.5.sdk.tar.xz" + sdk_root="$RUNNER_TEMP/macos-sdk" + curl --fail --location --retry 3 \ + "https://github.com/joseluisq/macosx-sdks/releases/download/14.5/MacOSX14.5.sdk.tar.xz" \ + --output "$archive" + echo "6e146275d19f027faa2e8354da5e0267513abf013b8f16ad65a231653a2b1c5d $archive" | sha256sum --check + mkdir -p "$sdk_root" + tar -xJf "$archive" -C "$sdk_root" + echo "LLAR_E2E_SYSROOT=$sdk_root/MacOSX14.5.sdk" >> "$GITHUB_ENV" + + - name: Prepare formula and source repositories + run: | + formula_repo="$RUNNER_TEMP/llarhub" + source_repo="$RUNNER_TEMP/macosx-sdks-source" + + mkdir -p "$formula_repo" "$source_repo" + cp -R testdata/kodo-e2e/formulas/. "$formula_repo/" + cp -R testdata/crosscompile-e2e/formulas/. "$formula_repo/" + git -C "$formula_repo" init --initial-branch=main + git -C "$formula_repo" config user.email crosscompile-e2e@example.com + git -C "$formula_repo" config user.name "Cross Compile E2E" + git -C "$formula_repo" add . + git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" + + cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" + chmod +x "$source_repo/install-sysroot" + git -C "$source_repo" init --initial-branch=main + git -C "$source_repo" config user.email crosscompile-e2e@example.com + git -C "$source_repo" config user.name "Cross Compile E2E" + git -C "$source_repo" add . + git -C "$source_repo" commit --quiet -m "Add sysroot installer" + git -C "$source_repo" tag 14.5 + + git config --global protocol.file.allow always + git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" + git config --global url."file://$source_repo".insteadOf "https://github.com/joseluisq/macosx-sdks.git" + + - name: Cross compile zlib + run: | + go build -ldflags="-checklinkname=0" -o "$RUNNER_TEMP/llar" ./cmd/llar + "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ + --os darwin \ + --arch arm64 \ + --output "$RUNNER_TEMP/zlib-darwin-arm64" \ + --verbose + + - name: Upload zlib artifact + uses: actions/upload-artifact@v4 + with: + name: zlib-darwin-arm64 + path: ${{ runner.temp }}/zlib-darwin-arm64 + if-no-files-found: error + + run-darwin-arm64: + needs: build-darwin-arm64 + runs-on: macos-14 + timeout-minutes: 10 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Download zlib artifact + uses: actions/download-artifact@v4 + with: + name: zlib-darwin-arm64 + path: ${{ runner.temp }}/zlib-darwin-arm64 + + - name: Build and run consumer + run: | + test "$(uname -m)" = "arm64" + cc \ + -I"$RUNNER_TEMP/zlib-darwin-arm64/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$RUNNER_TEMP/zlib-darwin-arm64/lib/libz.a" \ + -o "$RUNNER_TEMP/zlib-consumer" + file "$RUNNER_TEMP/zlib-consumer" + "$RUNNER_TEMP/zlib-consumer" diff --git a/testdata/crosscompile-e2e/consumer.c b/testdata/crosscompile-e2e/consumer.c new file mode 100644 index 0000000..6d2311f --- /dev/null +++ b/testdata/crosscompile-e2e/consumer.c @@ -0,0 +1,28 @@ +#include +#include +#include + +int main(void) { + static const Bytef input[] = "LLAR cross-compiled zlib"; + Bytef compressed[128]; + Bytef restored[128]; + uLongf compressed_len = sizeof(compressed); + uLongf restored_len = sizeof(restored); + + if (compress(compressed, &compressed_len, input, sizeof(input)) != Z_OK) { + return 1; + } + if (uncompress(restored, &restored_len, compressed, compressed_len) != Z_OK) { + return 2; + } + if (restored_len != sizeof(input) || memcmp(restored, input, sizeof(input)) != 0) { + return 3; + } + if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) { + return 4; + } + + printf("zlib %s: compressed %lu bytes to %lu bytes\n", + zlibVersion(), (unsigned long)sizeof(input), (unsigned long)compressed_len); + return 0; +} diff --git a/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox new file mode 100644 index 0000000..6940edd --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox @@ -0,0 +1,8 @@ +id "bminor/glibc" + +fromVer "glibc-2.17" + +onBuild ctx => { + exec! "./install-sysroot", ctx.outputDir + ctx.setMetadata "--sysroot="+ctx.outputDir +} diff --git a/testdata/crosscompile-e2e/formulas/bminor/glibc/versions.json b/testdata/crosscompile-e2e/formulas/bminor/glibc/versions.json new file mode 100644 index 0000000..4b06491 --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/bminor/glibc/versions.json @@ -0,0 +1,4 @@ +{ + "path": "bminor/glibc", + "deps": {} +} diff --git a/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox new file mode 100644 index 0000000..19335e2 --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox @@ -0,0 +1,8 @@ +id "joseluisq/macosx-sdks" + +fromVer "14.5" + +onBuild ctx => { + exec! "./install-sysroot", ctx.outputDir + ctx.setMetadata "--sysroot="+ctx.outputDir +} diff --git a/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/versions.json b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/versions.json new file mode 100644 index 0000000..67d412f --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/versions.json @@ -0,0 +1,4 @@ +{ + "path": "joseluisq/macosx-sdks", + "deps": {} +} diff --git a/testdata/crosscompile-e2e/install-sysroot b/testdata/crosscompile-e2e/install-sysroot new file mode 100644 index 0000000..b324d86 --- /dev/null +++ b/testdata/crosscompile-e2e/install-sysroot @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +: "${LLAR_E2E_SYSROOT:?LLAR_E2E_SYSROOT is required}" +cp -a "$LLAR_E2E_SYSROOT/." "$1/" From 15dc46826e4bc6849c9857f61177644ef60a4f68 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 19:25:24 +0800 Subject: [PATCH 13/39] fix(crosscompile): use Bootlin glibc sysroot --- .github/workflows/crosscompile-e2e.yml | 55 ++++++++----------- internal/build/c/target.go | 2 +- internal/build/c/target_test.go | 2 +- .../{glibc-2.17 => glibc-2.24}/Glibc_llar.gox | 2 +- testdata/crosscompile-e2e/install-sysroot | 4 +- 5 files changed, 28 insertions(+), 37 deletions(-) rename testdata/crosscompile-e2e/formulas/bminor/glibc/{glibc-2.17 => glibc-2.24}/Glibc_llar.gox (85%) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index a0245fc..625ede7 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -24,41 +24,29 @@ jobs: with: go-version: 1.24.x - - name: Install LLVM and the arm64 runtime + - name: Install LLVM run: | sudo apt-get update - sudo apt-get install --yes clang-18 lld-18 llvm-18 gcc-aarch64-linux-gnu rpm2cpio cpio + sudo apt-get install --yes clang-18 lld-18 llvm-18 echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" - - name: Prepare glibc 2.17 arm64 sysroot + - name: Prepare Bootlin glibc 2.24 arm64 sysroot run: | - package_dir="$RUNNER_TEMP/glibc-packages" - sysroot_dir="$RUNNER_TEMP/glibc-sysroot" - base_url="https://vault.centos.org/altarch/7.9.2009/os/aarch64/Packages" - mkdir -p "$package_dir" "$sysroot_dir" - - for package in \ - glibc-2.17-317.el7.aarch64.rpm \ - glibc-devel-2.17-317.el7.aarch64.rpm \ - glibc-headers-2.17-317.el7.aarch64.rpm \ - kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm - do - curl --fail --location --retry 3 \ - "$base_url/$package" \ - --output "$package_dir/$package" - done - - cd "$package_dir" - sha256sum --check <<'EOF' - ca1976545bcd4a6c099f1b3e2505469083b6bb4901d2517275a8da4138c0b21a glibc-2.17-317.el7.aarch64.rpm - 458738f7b415fb73473725b6bef340234954e4eb00731cbf8aa1afad5ccd12f6 glibc-devel-2.17-317.el7.aarch64.rpm - 58a5802d026435b36d7080ea4943868a82fc41642a2b0e475239bd6aaa5882cd glibc-headers-2.17-317.el7.aarch64.rpm - 56fffc40800cd7c30830009c860f04c8dba1110a6271efd804a0bb01e0bdf8a4 kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm - EOF - for package in *.rpm; do - rpm2cpio "$package" | (cd "$sysroot_dir" && cpio -idm --quiet) - done - echo "LLAR_E2E_SYSROOT=$sysroot_dir" >> "$GITHUB_ENV" + archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" + toolchain_root="$RUNNER_TEMP/bootlin" + curl --fail --location --retry 3 \ + "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" \ + --output "$archive" + echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $archive" | sha256sum --check + mkdir -p "$toolchain_root" + tar -xjf "$archive" -C "$toolchain_root" + + root="$toolchain_root/aarch64--glibc--stable" + sysroot="$root/aarch64-buildroot-linux-gnu/sysroot" + gcc_runtime="$root/lib/gcc/aarch64-buildroot-linux-gnu/5.4.0" + # Clang receives only the sysroot path, so install Bootlin's target + # runtime, for example crtbeginS.o and libgcc.a, into its library path. + cp -a "$gcc_runtime/." "$sysroot/usr/lib/" - name: Prepare formula and source repositories run: | @@ -75,13 +63,15 @@ jobs: git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" + tar -cJf "$source_repo/sysroot.tar.xz" \ + -C "$RUNNER_TEMP/bootlin/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" . chmod +x "$source_repo/install-sysroot" git -C "$source_repo" init --initial-branch=main git -C "$source_repo" config user.email crosscompile-e2e@example.com git -C "$source_repo" config user.name "Cross Compile E2E" git -C "$source_repo" add . git -C "$source_repo" commit --quiet -m "Add sysroot installer" - git -C "$source_repo" tag glibc-2.17 + git -C "$source_repo" tag glibc-2.24 git config --global protocol.file.allow always git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" @@ -156,7 +146,6 @@ jobs: echo "6e146275d19f027faa2e8354da5e0267513abf013b8f16ad65a231653a2b1c5d $archive" | sha256sum --check mkdir -p "$sdk_root" tar -xJf "$archive" -C "$sdk_root" - echo "LLAR_E2E_SYSROOT=$sdk_root/MacOSX14.5.sdk" >> "$GITHUB_ENV" - name: Prepare formula and source repositories run: | @@ -173,6 +162,8 @@ jobs: git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" + tar -cJf "$source_repo/sysroot.tar.xz" \ + -C "$RUNNER_TEMP/macos-sdk/MacOSX14.5.sdk" . chmod +x "$source_repo/install-sysroot" git -C "$source_repo" init --initial-branch=main git -C "$source_repo" config user.email crosscompile-e2e@example.com diff --git a/internal/build/c/target.go b/internal/build/c/target.go index e03c430..2da0a7b 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -14,7 +14,7 @@ import ( const ( linuxSysrootPath = "bminor/glibc" - linuxSysrootVersion = "glibc-2.17" + linuxSysrootVersion = "glibc-2.24" darwinSysrootPath = "joseluisq/macosx-sdks" darwinSysrootVersion = "14.5" ) diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index 131d4ca..d0ba054 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -32,7 +32,7 @@ func fakeToolchain(t *testing.T, targetOS string, compilerArgs ...string) Toolch } func TestSysroot(t *testing.T) { - linux := module.Version{Path: "bminor/glibc", Version: "glibc-2.17"} + linux := module.Version{Path: "bminor/glibc", Version: "glibc-2.24"} darwin := module.Version{Path: "joseluisq/macosx-sdks", Version: "14.5"} for _, arch := range []string{"amd64", "arm64"} { got, ok := Sysroot("linux", arch) diff --git a/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox similarity index 85% rename from testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox rename to testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox index 6940edd..6d17b74 100644 --- a/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.17/Glibc_llar.gox +++ b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox @@ -1,6 +1,6 @@ id "bminor/glibc" -fromVer "glibc-2.17" +fromVer "glibc-2.24" onBuild ctx => { exec! "./install-sysroot", ctx.outputDir diff --git a/testdata/crosscompile-e2e/install-sysroot b/testdata/crosscompile-e2e/install-sysroot index b324d86..0e50d71 100644 --- a/testdata/crosscompile-e2e/install-sysroot +++ b/testdata/crosscompile-e2e/install-sysroot @@ -1,5 +1,5 @@ #!/bin/sh set -eu -: "${LLAR_E2E_SYSROOT:?LLAR_E2E_SYSROOT is required}" -cp -a "$LLAR_E2E_SYSROOT/." "$1/" +source_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +tar -xJf "$source_dir/sysroot.tar.xz" -C "$1" From 13617a6d85676e70cb71281145ef24ab38cb9b60 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 21:03:29 +0800 Subject: [PATCH 14/39] test(crosscompile): exercise zlib file IO --- testdata/crosscompile-e2e/consumer.c | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/testdata/crosscompile-e2e/consumer.c b/testdata/crosscompile-e2e/consumer.c index 6d2311f..d3ccb9a 100644 --- a/testdata/crosscompile-e2e/consumer.c +++ b/testdata/crosscompile-e2e/consumer.c @@ -4,10 +4,13 @@ int main(void) { static const Bytef input[] = "LLAR cross-compiled zlib"; + static const char gzip_path[] = "llar-zlib-e2e.gz"; Bytef compressed[128]; Bytef restored[128]; uLongf compressed_len = sizeof(compressed); uLongf restored_len = sizeof(restored); + gzFile gzip; + int gzip_len; if (compress(compressed, &compressed_len, input, sizeof(input)) != Z_OK) { return 1; @@ -22,7 +25,36 @@ int main(void) { return 4; } - printf("zlib %s: compressed %lu bytes to %lu bytes\n", + gzip = gzopen(gzip_path, "wb"); + if (gzip == NULL) { + return 5; + } + if (gzwrite(gzip, input, (unsigned)sizeof(input)) != (int)sizeof(input)) { + return 6; + } + if (gzclose(gzip) != Z_OK) { + return 7; + } + + gzip = gzopen(gzip_path, "rb"); + if (gzip == NULL) { + return 8; + } + gzip_len = gzread(gzip, restored, (unsigned)sizeof(restored)); + if (gzip_len != (int)sizeof(input)) { + return 9; + } + if (gzclose(gzip) != Z_OK) { + return 10; + } + if (memcmp(restored, input, sizeof(input)) != 0) { + return 11; + } + if (remove(gzip_path) != 0) { + return 12; + } + + printf("zlib %s: compressed %lu bytes to %lu bytes and verified gzip I/O\n", zlibVersion(), (unsigned long)sizeof(input), (unsigned long)compressed_len); return 0; } From 53bb76b0c01f8b03d67992534d42911e6fef4d4c Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 21:44:26 +0800 Subject: [PATCH 15/39] ci(crosscompile): show build and target platforms --- .github/workflows/crosscompile-e2e.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 625ede7..8dfc938 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -1,4 +1,4 @@ -name: Cross Compile E2E +name: Cross Compile E2E (Linux hosts to arm64 targets) on: push: @@ -13,6 +13,7 @@ concurrency: jobs: build-linux-arm64: + name: "Build zlib: Linux amd64 host to Linux arm64 target" runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -94,6 +95,7 @@ jobs: if-no-files-found: error run-linux-arm64: + name: "Run zlib: Linux arm64 runner consumes Linux arm64 artifact" needs: build-linux-arm64 runs-on: ubuntu-24.04-arm timeout-minutes: 10 @@ -119,6 +121,7 @@ jobs: "$RUNNER_TEMP/zlib-consumer" build-darwin-arm64: + name: "Build zlib: Linux arm64 host to Darwin arm64 target" runs-on: ubuntu-24.04-arm timeout-minutes: 25 steps: @@ -193,6 +196,7 @@ jobs: if-no-files-found: error run-darwin-arm64: + name: "Run zlib: macOS arm64 runner consumes Darwin arm64 artifact" needs: build-darwin-arm64 runs-on: macos-14 timeout-minutes: 10 From 60563afb75dd01aeb492648996f9b3c8c7f0d6de Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 21:53:28 +0800 Subject: [PATCH 16/39] ci(crosscompile): verify target system libraries --- .github/workflows/crosscompile-e2e.yml | 78 +++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 8dfc938..b51b43e 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -87,6 +87,25 @@ jobs: --output "$RUNNER_TEMP/zlib-linux-arm64" \ --verbose + - name: Link consumer with Bootlin target libc + run: | + output="$RUNNER_TEMP/zlib-linux-arm64" + sysroot="$RUNNER_TEMP/bootlin/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" + trace="$RUNNER_TEMP/linux-arm64-link.trace" + mkdir -p "$output/bin" + clang \ + --target=aarch64-linux-gnu \ + -fuse-ld=lld \ + --sysroot="$sysroot" \ + -I"$output/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$output/lib/libz.a" \ + -Wl,--trace \ + -o "$output/bin/zlib-consumer" \ + 2>&1 | tee "$trace" + grep -F "$sysroot/usr/lib/Scrt1.o" "$trace" + grep -F "$sysroot/usr/lib/libc.so" "$trace" + - name: Upload zlib artifact uses: actions/upload-artifact@v4 with: @@ -109,16 +128,29 @@ jobs: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 - - name: Build and run consumer + - name: Inspect and run cross-linked consumer + run: | + consumer="$RUNNER_TEMP/zlib-linux-arm64/bin/zlib-consumer" + test "$(uname -m)" = "aarch64" + file "$consumer" + readelf --program-headers "$consumer" | tee "$RUNNER_TEMP/program-headers.txt" + grep -F "Requesting program interpreter: /lib/ld-linux-aarch64.so.1" "$RUNNER_TEMP/program-headers.txt" + readelf --dynamic "$consumer" | tee "$RUNNER_TEMP/dynamic-section.txt" + grep -F "Shared library: [libc.so.6]" "$RUNNER_TEMP/dynamic-section.txt" + readelf --version-info "$consumer" + chmod +x "$consumer" + "$consumer" + + - name: Build and run consumer with runner toolchain run: | test "$(uname -m)" = "aarch64" cc \ -I"$RUNNER_TEMP/zlib-linux-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ "$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer" - file "$RUNNER_TEMP/zlib-consumer" - "$RUNNER_TEMP/zlib-consumer" + -o "$RUNNER_TEMP/zlib-consumer-native" + file "$RUNNER_TEMP/zlib-consumer-native" + "$RUNNER_TEMP/zlib-consumer-native" build-darwin-arm64: name: "Build zlib: Linux arm64 host to Darwin arm64 target" @@ -188,6 +220,24 @@ jobs: --output "$RUNNER_TEMP/zlib-darwin-arm64" \ --verbose + - name: Link consumer with macOS 14.5 SDK + run: | + output="$RUNNER_TEMP/zlib-darwin-arm64" + sysroot="$RUNNER_TEMP/macos-sdk/MacOSX14.5.sdk" + trace="$RUNNER_TEMP/darwin-arm64-link.trace" + mkdir -p "$output/bin" + clang \ + --target=arm64-apple-macos11.0 \ + -fuse-ld=lld \ + -isysroot "$sysroot" \ + -I"$output/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$output/lib/libz.a" \ + -Wl,-t \ + -o "$output/bin/zlib-consumer" \ + 2>&1 | tee "$trace" + grep -F "$sysroot/usr/lib/libSystem.tbd" "$trace" + - name: Upload zlib artifact uses: actions/upload-artifact@v4 with: @@ -210,13 +260,25 @@ jobs: name: zlib-darwin-arm64 path: ${{ runner.temp }}/zlib-darwin-arm64 - - name: Build and run consumer + - name: Inspect and run cross-linked consumer + run: | + consumer="$RUNNER_TEMP/zlib-darwin-arm64/bin/zlib-consumer" + test "$(uname -m)" = "arm64" + file "$consumer" + otool -L "$consumer" | tee "$RUNNER_TEMP/load-commands.txt" + grep -F "/usr/lib/libSystem.B.dylib" "$RUNNER_TEMP/load-commands.txt" + otool -l "$consumer" | sed -n '/cmd LC_BUILD_VERSION/,+5p' | tee "$RUNNER_TEMP/build-version.txt" + grep -E "sdk +14\\.5" "$RUNNER_TEMP/build-version.txt" + chmod +x "$consumer" + "$consumer" + + - name: Build and run consumer with runner toolchain run: | test "$(uname -m)" = "arm64" cc \ -I"$RUNNER_TEMP/zlib-darwin-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ "$RUNNER_TEMP/zlib-darwin-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer" - file "$RUNNER_TEMP/zlib-consumer" - "$RUNNER_TEMP/zlib-consumer" + -o "$RUNNER_TEMP/zlib-consumer-native" + file "$RUNNER_TEMP/zlib-consumer-native" + "$RUNNER_TEMP/zlib-consumer-native" From a8ddf3dd682c49e874b919c46adbf03b39e57ba1 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 22:21:02 +0800 Subject: [PATCH 17/39] ci(crosscompile): inspect target dynamic libraries --- .github/workflows/crosscompile-e2e.yml | 36 +++++++------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index b51b43e..1a4cd68 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -87,11 +87,10 @@ jobs: --output "$RUNNER_TEMP/zlib-linux-arm64" \ --verbose - - name: Link consumer with Bootlin target libc + - name: Link cross-compiled consumer run: | output="$RUNNER_TEMP/zlib-linux-arm64" sysroot="$RUNNER_TEMP/bootlin/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" - trace="$RUNNER_TEMP/linux-arm64-link.trace" mkdir -p "$output/bin" clang \ --target=aarch64-linux-gnu \ @@ -100,11 +99,7 @@ jobs: -I"$output/include" \ testdata/crosscompile-e2e/consumer.c \ "$output/lib/libz.a" \ - -Wl,--trace \ - -o "$output/bin/zlib-consumer" \ - 2>&1 | tee "$trace" - grep -F "$sysroot/usr/lib/Scrt1.o" "$trace" - grep -F "$sysroot/usr/lib/libc.so" "$trace" + -o "$output/bin/zlib-consumer" - name: Upload zlib artifact uses: actions/upload-artifact@v4 @@ -128,17 +123,12 @@ jobs: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 - - name: Inspect and run cross-linked consumer + - name: Check dynamic libraries and run cross-linked consumer run: | consumer="$RUNNER_TEMP/zlib-linux-arm64/bin/zlib-consumer" - test "$(uname -m)" = "aarch64" - file "$consumer" - readelf --program-headers "$consumer" | tee "$RUNNER_TEMP/program-headers.txt" - grep -F "Requesting program interpreter: /lib/ld-linux-aarch64.so.1" "$RUNNER_TEMP/program-headers.txt" - readelf --dynamic "$consumer" | tee "$RUNNER_TEMP/dynamic-section.txt" - grep -F "Shared library: [libc.so.6]" "$RUNNER_TEMP/dynamic-section.txt" - readelf --version-info "$consumer" chmod +x "$consumer" + ldd "$consumer" | tee "$RUNNER_TEMP/dynamic-libraries.txt" + grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/dynamic-libraries.txt" "$consumer" - name: Build and run consumer with runner toolchain @@ -220,11 +210,10 @@ jobs: --output "$RUNNER_TEMP/zlib-darwin-arm64" \ --verbose - - name: Link consumer with macOS 14.5 SDK + - name: Link cross-compiled consumer run: | output="$RUNNER_TEMP/zlib-darwin-arm64" sysroot="$RUNNER_TEMP/macos-sdk/MacOSX14.5.sdk" - trace="$RUNNER_TEMP/darwin-arm64-link.trace" mkdir -p "$output/bin" clang \ --target=arm64-apple-macos11.0 \ @@ -233,10 +222,7 @@ jobs: -I"$output/include" \ testdata/crosscompile-e2e/consumer.c \ "$output/lib/libz.a" \ - -Wl,-t \ - -o "$output/bin/zlib-consumer" \ - 2>&1 | tee "$trace" - grep -F "$sysroot/usr/lib/libSystem.tbd" "$trace" + -o "$output/bin/zlib-consumer" - name: Upload zlib artifact uses: actions/upload-artifact@v4 @@ -260,16 +246,12 @@ jobs: name: zlib-darwin-arm64 path: ${{ runner.temp }}/zlib-darwin-arm64 - - name: Inspect and run cross-linked consumer + - name: Check dynamic libraries and run cross-linked consumer run: | consumer="$RUNNER_TEMP/zlib-darwin-arm64/bin/zlib-consumer" - test "$(uname -m)" = "arm64" - file "$consumer" + chmod +x "$consumer" otool -L "$consumer" | tee "$RUNNER_TEMP/load-commands.txt" grep -F "/usr/lib/libSystem.B.dylib" "$RUNNER_TEMP/load-commands.txt" - otool -l "$consumer" | sed -n '/cmd LC_BUILD_VERSION/,+5p' | tee "$RUNNER_TEMP/build-version.txt" - grep -E "sdk +14\\.5" "$RUNNER_TEMP/build-version.txt" - chmod +x "$consumer" "$consumer" - name: Build and run consumer with runner toolchain From 27e750d5b5d80117b979f4a34d997e95633e0476 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 22:27:09 +0800 Subject: [PATCH 18/39] ci(crosscompile): use amd64 Darwin build host --- .github/workflows/crosscompile-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 1a4cd68..64816f5 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -143,8 +143,8 @@ jobs: "$RUNNER_TEMP/zlib-consumer-native" build-darwin-arm64: - name: "Build zlib: Linux arm64 host to Darwin arm64 target" - runs-on: ubuntu-24.04-arm + name: "Build zlib: Linux amd64 host to Darwin arm64 target" + runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - name: Check out code From ba6ef79127467e92c5398d967a69718c7b1e9502 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 22:48:07 +0800 Subject: [PATCH 19/39] ci(crosscompile): link consumers on target runners --- .github/workflows/crosscompile-e2e.yml | 60 +++----------------------- 1 file changed, 6 insertions(+), 54 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 64816f5..43638e4 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -87,20 +87,6 @@ jobs: --output "$RUNNER_TEMP/zlib-linux-arm64" \ --verbose - - name: Link cross-compiled consumer - run: | - output="$RUNNER_TEMP/zlib-linux-arm64" - sysroot="$RUNNER_TEMP/bootlin/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" - mkdir -p "$output/bin" - clang \ - --target=aarch64-linux-gnu \ - -fuse-ld=lld \ - --sysroot="$sysroot" \ - -I"$output/include" \ - testdata/crosscompile-e2e/consumer.c \ - "$output/lib/libz.a" \ - -o "$output/bin/zlib-consumer" - - name: Upload zlib artifact uses: actions/upload-artifact@v4 with: @@ -123,24 +109,14 @@ jobs: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 - - name: Check dynamic libraries and run cross-linked consumer - run: | - consumer="$RUNNER_TEMP/zlib-linux-arm64/bin/zlib-consumer" - chmod +x "$consumer" - ldd "$consumer" | tee "$RUNNER_TEMP/dynamic-libraries.txt" - grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/dynamic-libraries.txt" - "$consumer" - - - name: Build and run consumer with runner toolchain + - name: Link and run consumer run: | - test "$(uname -m)" = "aarch64" cc \ -I"$RUNNER_TEMP/zlib-linux-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ "$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer-native" - file "$RUNNER_TEMP/zlib-consumer-native" - "$RUNNER_TEMP/zlib-consumer-native" + -o "$RUNNER_TEMP/zlib-consumer" + "$RUNNER_TEMP/zlib-consumer" build-darwin-arm64: name: "Build zlib: Linux amd64 host to Darwin arm64 target" @@ -210,20 +186,6 @@ jobs: --output "$RUNNER_TEMP/zlib-darwin-arm64" \ --verbose - - name: Link cross-compiled consumer - run: | - output="$RUNNER_TEMP/zlib-darwin-arm64" - sysroot="$RUNNER_TEMP/macos-sdk/MacOSX14.5.sdk" - mkdir -p "$output/bin" - clang \ - --target=arm64-apple-macos11.0 \ - -fuse-ld=lld \ - -isysroot "$sysroot" \ - -I"$output/include" \ - testdata/crosscompile-e2e/consumer.c \ - "$output/lib/libz.a" \ - -o "$output/bin/zlib-consumer" - - name: Upload zlib artifact uses: actions/upload-artifact@v4 with: @@ -246,21 +208,11 @@ jobs: name: zlib-darwin-arm64 path: ${{ runner.temp }}/zlib-darwin-arm64 - - name: Check dynamic libraries and run cross-linked consumer - run: | - consumer="$RUNNER_TEMP/zlib-darwin-arm64/bin/zlib-consumer" - chmod +x "$consumer" - otool -L "$consumer" | tee "$RUNNER_TEMP/load-commands.txt" - grep -F "/usr/lib/libSystem.B.dylib" "$RUNNER_TEMP/load-commands.txt" - "$consumer" - - - name: Build and run consumer with runner toolchain + - name: Link and run consumer run: | - test "$(uname -m)" = "arm64" cc \ -I"$RUNNER_TEMP/zlib-darwin-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ "$RUNNER_TEMP/zlib-darwin-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer-native" - file "$RUNNER_TEMP/zlib-consumer-native" - "$RUNNER_TEMP/zlib-consumer-native" + -o "$RUNNER_TEMP/zlib-consumer" + "$RUNNER_TEMP/zlib-consumer" From 50e34724bba7e189a5ff86cb044191898adb2c3c Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 22:51:17 +0800 Subject: [PATCH 20/39] ci(crosscompile): reuse macOS SDK archive --- .github/workflows/crosscompile-e2e.yml | 6 +----- .../formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox | 2 +- .../joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox | 2 +- testdata/crosscompile-e2e/install-sysroot | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 43638e4..dc6f3f1 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -140,13 +140,10 @@ jobs: - name: Prepare macOS 14.5 SDK run: | archive="$RUNNER_TEMP/MacOSX14.5.sdk.tar.xz" - sdk_root="$RUNNER_TEMP/macos-sdk" curl --fail --location --retry 3 \ "https://github.com/joseluisq/macosx-sdks/releases/download/14.5/MacOSX14.5.sdk.tar.xz" \ --output "$archive" echo "6e146275d19f027faa2e8354da5e0267513abf013b8f16ad65a231653a2b1c5d $archive" | sha256sum --check - mkdir -p "$sdk_root" - tar -xJf "$archive" -C "$sdk_root" - name: Prepare formula and source repositories run: | @@ -163,8 +160,7 @@ jobs: git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" - tar -cJf "$source_repo/sysroot.tar.xz" \ - -C "$RUNNER_TEMP/macos-sdk/MacOSX14.5.sdk" . + cp "$RUNNER_TEMP/MacOSX14.5.sdk.tar.xz" "$source_repo/sysroot.tar.xz" chmod +x "$source_repo/install-sysroot" git -C "$source_repo" init --initial-branch=main git -C "$source_repo" config user.email crosscompile-e2e@example.com diff --git a/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox index 6d17b74..ff7ef81 100644 --- a/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox +++ b/testdata/crosscompile-e2e/formulas/bminor/glibc/glibc-2.24/Glibc_llar.gox @@ -3,6 +3,6 @@ id "bminor/glibc" fromVer "glibc-2.24" onBuild ctx => { - exec! "./install-sysroot", ctx.outputDir + exec! "./install-sysroot", ctx.outputDir, "0" ctx.setMetadata "--sysroot="+ctx.outputDir } diff --git a/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox index 19335e2..e1adeaf 100644 --- a/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox +++ b/testdata/crosscompile-e2e/formulas/joseluisq/macosx-sdks/sdk-14_5/MacosxSdks_llar.gox @@ -3,6 +3,6 @@ id "joseluisq/macosx-sdks" fromVer "14.5" onBuild ctx => { - exec! "./install-sysroot", ctx.outputDir + exec! "./install-sysroot", ctx.outputDir, "1" ctx.setMetadata "--sysroot="+ctx.outputDir } diff --git a/testdata/crosscompile-e2e/install-sysroot b/testdata/crosscompile-e2e/install-sysroot index 0e50d71..12d411f 100644 --- a/testdata/crosscompile-e2e/install-sysroot +++ b/testdata/crosscompile-e2e/install-sysroot @@ -2,4 +2,4 @@ set -eu source_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -tar -xJf "$source_dir/sysroot.tar.xz" -C "$1" +tar -xJf "$source_dir/sysroot.tar.xz" -C "$1" --strip-components="$2" From c496e092a8edb79494c321679397e0374e09d7fb Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Mon, 10 Aug 2026 23:00:28 +0800 Subject: [PATCH 21/39] ci(crosscompile): inspect target zlib artifacts --- .github/workflows/crosscompile-e2e.yml | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index dc6f3f1..3b4be62 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -109,14 +109,28 @@ jobs: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 - - name: Link and run consumer + - name: Inspect, link, and run consumer run: | + lib="$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" + member_dir="$RUNNER_TEMP/zlib-members" + consumer="$RUNNER_TEMP/zlib-consumer" + + mkdir -p "$member_dir" + ( + cd "$member_dir" + ar x "$lib" adler32.o + ) + file "$member_dir/adler32.o" | tee "$RUNNER_TEMP/zlib-format.txt" + grep -F "ELF 64-bit LSB relocatable, ARM aarch64" "$RUNNER_TEMP/zlib-format.txt" + cc \ -I"$RUNNER_TEMP/zlib-linux-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ - "$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer" - "$RUNNER_TEMP/zlib-consumer" + "$lib" \ + -o "$consumer" + ldd "$consumer" | tee "$RUNNER_TEMP/dynamic-libraries.txt" + grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/dynamic-libraries.txt" + "$consumer" build-darwin-arm64: name: "Build zlib: Linux amd64 host to Darwin arm64 target" @@ -204,11 +218,19 @@ jobs: name: zlib-darwin-arm64 path: ${{ runner.temp }}/zlib-darwin-arm64 - - name: Link and run consumer + - name: Inspect, link, and run consumer run: | + lib="$RUNNER_TEMP/zlib-darwin-arm64/lib/libz.a" + consumer="$RUNNER_TEMP/zlib-consumer" + + lipo -info "$lib" | tee "$RUNNER_TEMP/zlib-format.txt" + grep -F "architecture: arm64" "$RUNNER_TEMP/zlib-format.txt" + cc \ -I"$RUNNER_TEMP/zlib-darwin-arm64/include" \ testdata/crosscompile-e2e/consumer.c \ - "$RUNNER_TEMP/zlib-darwin-arm64/lib/libz.a" \ - -o "$RUNNER_TEMP/zlib-consumer" - "$RUNNER_TEMP/zlib-consumer" + "$lib" \ + -o "$consumer" + otool -L "$consumer" | tee "$RUNNER_TEMP/dynamic-libraries.txt" + grep -F "/usr/lib/libSystem.B.dylib" "$RUNNER_TEMP/dynamic-libraries.txt" + "$consumer" From fbbae81d7ad01ebbd4025b9924c07ad61a6ee2f2 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 10:46:46 +0800 Subject: [PATCH 22/39] refactor(build): own cross compile orchestration --- cmd/llar/internal/make.go | 118 ++++---------------------------- cmd/llar/internal/make_test.go | 2 +- internal/build/build.go | 117 ++++++++++++++++++++++++++++--- internal/build/build_test.go | 20 +++--- internal/build/c/target.go | 49 ++++++++----- internal/build/c/target_test.go | 45 ++++++------ internal/build/cache.go | 2 +- internal/build/cache_test.go | 22 +++--- internal/build/e2e_test.go | 8 +-- internal/build/http/http.go | 2 +- internal/build/target.go | 26 +------ internal/build/target_test.go | 77 ++++++++++++++------- testdata/kodo-e2e/main.go | 2 +- 13 files changed, 257 insertions(+), 233 deletions(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index d31ae3a..774cf87 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -12,14 +12,11 @@ import ( "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build" - "github.com/goplus/llar/internal/build/c" - "github.com/goplus/llar/internal/build/c/llvm" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/modules/modlocal" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" - ccmetadata "github.com/goplus/llar/x/metadata/cc" "github.com/spf13/cobra" ) @@ -133,59 +130,18 @@ func hostMatrix() formula.Matrix { // `llar test ` invocation. func buildModule(ctx context.Context, store repo.Store, modPath, version string, matrix formula.Matrix, runTest bool) error { root := module.Version{Path: modPath, Version: version} - var targetOS, targetArch string - if values := matrix.Require["os"]; len(values) > 0 { - targetOS = values[0] - } - if values := matrix.Require["arch"]; len(values) > 0 { - targetArch = values[0] - } - crossCompile := targetOS != runtime.GOOS || targetArch != runtime.GOARCH - if runTest && crossCompile { - return fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) - } - var targetRoot module.Version - var useCTarget bool - if crossCompile { - // TODO: Add other language target policies alongside this C case when - // they provide build.Target implementations. - cSysroot, ok := c.Sysroot(targetOS, targetArch) - useCTarget = ok - _, customLibc := matrix.Require["libc"] - if useCTarget && !customLibc && root.Path != cSysroot.Path { - targetRoot = cSysroot - } - } - loadOpts := modules.Options{ - FormulaStore: store, - Matrix: matrix, - } - mods, err := modules.Load(ctx, root, loadOpts) - if err != nil { - return fmt.Errorf("failed to load modules: %w", err) - } - var sysrootMods []*modules.Module - if targetRoot != (module.Version{}) { - // The default sysroot has no dependencies, but modules.Load still owns - // selecting the Formula whose fromVer applies to targetRoot.Version. - sysrootMods, err = modules.Load(ctx, targetRoot, loadOpts) - if err != nil { - return fmt.Errorf("failed to load sysroot %s@%s: %w", targetRoot.Path, targetRoot.Version, err) - } - } var buildOutput io.Writer = io.Discard if makeVerbose { buildOutput = os.Stderr } - matrixStr := matrix.Combinations()[0] buildOpts := build.Options{ - Store: store, - MatrixStr: matrixStr, - RunTest: runTest, - Stdout: buildOutput, - Stderr: buildOutput, + Store: store, + Matrix: matrix, + RunTest: runTest, + Stdout: buildOutput, + Stderr: buildOutput, } if makeOutput != "" { tmpDir, err := os.MkdirTemp("", "llar-make-*") @@ -195,67 +151,17 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } - var target build.Target - // TODO: Add other language build.Target preparation alongside this C case. - if useCTarget { - targetMatrix := targetArch + "-" + targetOS - bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) - if err != nil { - return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - bootstrapTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: bootstrapToolchain.Toolchain, - }) - if err != nil { - return err - } - defer bootstrapTarget.Close() - target = bootstrapTarget - - if targetRoot != (module.Version{}) { - selectedSysroot := sysrootMods[0] - - sysrootOpts := buildOpts - sysrootOpts.RunTest = false - sysrootOpts.Target = bootstrapTarget - sysrootBuilder, err := build.NewBuilder(sysrootOpts) - if err != nil { - return fmt.Errorf("failed to create sysroot builder: %w", err) - } - sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) - if err != nil { - return fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) - if err != nil { - return fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - if metadata.Sysroot() == "" { - return fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) - } - - configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) - if err != nil { - return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - configuredTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: configuredToolchain.Toolchain, - Sysroot: metadata.Sysroot(), - }) - if err != nil { - return err - } - defer configuredTarget.Close() - target = configuredTarget - } - } - buildOpts.Target = target builder, err := build.NewBuilder(buildOpts) if err != nil { return fmt.Errorf("failed to create builder: %w", err) } + mods, err := modules.Load(ctx, root, modules.Options{ + FormulaStore: store, + Matrix: matrix, + }) + if err != nil { + return fmt.Errorf("failed to load modules: %w", err) + } results, err := builder.Build(ctx, mods) if err != nil { diff --git a/cmd/llar/internal/make_test.go b/cmd/llar/internal/make_test.go index 34a338e..77c2fde 100644 --- a/cmd/llar/internal/make_test.go +++ b/cmd/llar/internal/make_test.go @@ -178,7 +178,7 @@ func TestBuildModuleRejectsCrossTargetTest(t *testing.T) { }} err := buildModule(context.Background(), nil, "owner/repo", "v1.0.0", matrix, true) - want := fmt.Sprintf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + want := fmt.Sprintf("failed to create builder: llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) if err == nil || err.Error() != want { t.Fatalf("buildModule error = %v, want %q", err, want) } diff --git a/internal/build/build.go b/internal/build/build.go index ee5e0c3..d24baec 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -8,27 +8,31 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "sort" "strings" classfile "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build/c" + "github.com/goplus/llar/internal/build/c/llvm" "github.com/goplus/llar/internal/build/cache" "github.com/goplus/llar/internal/execbroker" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" + ccmetadata "github.com/goplus/llar/x/metadata/cc" ) type Builder struct { store repo.Store - matrix string + matrix classfile.Matrix runTest bool stdout io.Writer stderr io.Writer workspaceDir string cache cache.Cache - target Target + target *c.Target newRepo func(repoPath string) (vcs.Repo, error) // defaults to vcs.NewRepo } @@ -38,8 +42,10 @@ type Result struct { } type Options struct { - Store repo.Store - MatrixStr string + Store repo.Store + // Matrix is the selected build matrix used for cache identity and target + // preparation. + Matrix classfile.Matrix // RunTest, when true, causes Build to invoke OnTest on the root target // after OnBuild (or after reusing cached build metadata). The build // cache is consulted as usual: on a cache hit the root's OnBuild is @@ -52,7 +58,8 @@ type Options struct { Stderr io.Writer WorkspaceDir string Cache cache.Cache - Target Target + // Target overrides automatic C target and default sysroot preparation. + Target *c.Target } func runFormulaHook(fn func()) (err error) { @@ -84,6 +91,16 @@ func defaultWorkspaceDir() (string, error) { // NewBuilder creates a new Builder. func NewBuilder(opts Options) (*Builder, error) { + var targetOS, targetArch string + if values := opts.Matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := opts.Matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + if opts.RunTest && (targetOS != runtime.GOOS || targetArch != runtime.GOARCH) { + return nil, fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + } workspaceDir := opts.WorkspaceDir if workspaceDir == "" { var err error @@ -106,7 +123,7 @@ func NewBuilder(opts Options) (*Builder, error) { } return &Builder{ store: opts.Store, - matrix: opts.MatrixStr, + matrix: opts.Matrix, runTest: opts.RunTest, stdout: stdout, stderr: stderr, @@ -229,6 +246,84 @@ func (b *Builder) resolveModTransitiveDeps(targets []*modules.Module, mod *modul } func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Result, error) { + buildTarget := b.target + var targetOS, targetArch string + if values := b.matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := b.matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + // A caller-supplied target owns all target and sysroot preparation. Without + // one, cross builds use LLAR's default C target policy. + if len(targets) > 0 && buildTarget == nil && (targetOS != runtime.GOOS || targetArch != runtime.GOARCH) { + cSysroot, useCTarget := c.Sysroot(targetOS, targetArch) + if useCTarget { + var sysrootMods []*modules.Module + _, customLibc := b.matrix.Require["libc"] + if !customLibc && targets[0].Path != cSysroot.Path { + var err error + // modules.Load selects the sysroot Formula whose fromVer applies to + // cSysroot.Version; the sysroot graph remains separate from targets. + sysrootMods, err = modules.Load(ctx, cSysroot, modules.Options{ + FormulaStore: b.store, + Matrix: b.matrix, + }) + if err != nil { + return nil, fmt.Errorf("failed to load sysroot %s@%s: %w", cSysroot.Path, cSysroot.Version, err) + } + } + + targetMatrix := targetArch + "-" + targetOS + bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) + if err != nil { + return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + bootstrapTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: bootstrapToolchain.Toolchain, + }) + if err != nil { + return nil, err + } + defer bootstrapTarget.Close() + buildTarget = bootstrapTarget + + if len(sysrootMods) > 0 { + selectedSysroot := sysrootMods[0] + sysrootBuilder := *b + sysrootBuilder.runTest = false + sysrootBuilder.target = bootstrapTarget + sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) + if err != nil { + return nil, fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) + if err != nil { + return nil, fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + if metadata.Sysroot() == "" { + return nil, fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) + } + + configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) + if err != nil { + return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + configuredTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: configuredToolchain.Toolchain, + Sysroot: metadata.Sysroot(), + }) + if err != nil { + return nil, err + } + defer configuredTarget.Close() + buildTarget = configuredTarget + } + } + } + builtResults := make(map[module.Version]classfile.BuildResult) // Identify the root target. By MVS convention (see constructBuildList @@ -247,8 +342,8 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul } var commandMiddleware execbroker.Middleware - if b.target != nil { - commandMiddleware = targetMiddleware(b.target) + if buildTarget != nil { + commandMiddleware = targetMiddleware(buildTarget) } build := func(mod *modules.Module) (Result, error) { @@ -265,7 +360,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul // Consult the build cache. A hit means we already have the // module's build metadata and its installDir is populated from a // previous successful build. - entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix}) + entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}) if err != nil { return Result{}, err } @@ -305,7 +400,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul return b.installDir(m.Path, m.Version) } project := &classfile.Project{Deps: deps, SourceFS: mod.FS.(fs.ReadFileFS)} - buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix, getOutputDir) + buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix.Combinations()[0], getOutputDir) // Inject results of already-built dependencies for modVer, result := range builtResults { @@ -356,7 +451,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul // Save cache only on cache miss. A cache hit means the entry is // already present and current; OnTest does not modify metadata. if !cacheHit { - entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix}, os.DirFS(installDir), cache.Entry{ + entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}, os.DirFS(installDir), cache.Entry{ Metadata: metadata, Deps: deps, }) diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 58931b3..212a00a 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -41,6 +41,10 @@ func paths(mods []*modules.Module) string { return strings.Join(s, " ") } +func testMatrix(value string) classfile.Matrix { + return classfile.Matrix{Require: map[string][]string{"matrix": {value}}} +} + // versions returns the "Path@Version" strings for []module.Version. func versions(vers []module.Version) string { var s []string @@ -308,7 +312,7 @@ func setupBuilder(t *testing.T, store repo.Store, matrix string) *Builder { workspaceDir := t.TempDir() return &Builder{ store: store, - matrix: matrix, + matrix: testMatrix(matrix), workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -512,7 +516,7 @@ func TestNewBuilder(t *testing.T) { store := setupTestStore(t) b, err := NewBuilder(Options{ Store: store, - MatrixStr: "amd64-linux", + Matrix: classfile.Matrix{Require: map[string][]string{"matrix": {"amd64-linux"}}}, WorkspaceDir: tmpDir, }) if err != nil { @@ -521,8 +525,8 @@ func TestNewBuilder(t *testing.T) { if b.workspaceDir != tmpDir { t.Errorf("workspaceDir = %q, want %q", b.workspaceDir, tmpDir) } - if b.matrix != "amd64-linux" { - t.Errorf("matrix = %q, want %q", b.matrix, "amd64-linux") + if got := b.matrix.Combinations()[0]; got != "amd64-linux" { + t.Errorf("matrix = %q, want %q", got, "amd64-linux") } if b.store != store { t.Error("store not set correctly") @@ -534,7 +538,7 @@ func TestNewBuilder(t *testing.T) { t.Run("default workspace dir", func(t *testing.T) { b, err := NewBuilder(Options{ - MatrixStr: "arm64-darwin", + Matrix: classfile.Matrix{Require: map[string][]string{"matrix": {"arm64-darwin"}}}, }) if err != nil { t.Fatalf("NewBuilder() error = %v", err) @@ -641,7 +645,7 @@ func TestBuild_LocksEntireDependencyGraph(t *testing.T) { buildCache := &graphLockCache{store: store, paths: paths} b := &Builder{ store: store, - matrix: "amd64-linux", + matrix: testMatrix("amd64-linux"), workspaceDir: t.TempDir(), cache: buildCache, } @@ -673,7 +677,7 @@ func TestBuild_ReleasesGraphLocksAfterLockError(t *testing.T) { buildCache := &graphLockCache{store: store, paths: []string{"a/dep", "m/root"}} b := &Builder{ store: store, - matrix: "amd64-linux", + matrix: testMatrix("amd64-linux"), workspaceDir: t.TempDir(), cache: buildCache, } @@ -708,7 +712,7 @@ func TestBuild_OppositeGraphOrdersDoNotDeadlock(t *testing.T) { newBuilder := func(id int) *Builder { return &Builder{ store: &oppositeGraphStore{id: id, locks: locks}, - matrix: "amd64-linux", + matrix: testMatrix("amd64-linux"), workspaceDir: t.TempDir(), cache: &recordingCache{hits: map[module.Version]cache.Entry{ {Path: "test/x", Version: "1.0.0"}: {Metadata: "x"}, diff --git a/internal/build/c/target.go b/internal/build/c/target.go index 2da0a7b..9b3fd55 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strings" - "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" "github.com/kballard/go-shellquote" ) @@ -19,6 +18,22 @@ const ( darwinSysrootVersion = "14.5" ) +// Command describes a command before C target defaults are applied. +type Command struct { + Name string + Args []string + Env []string + Dir string +} + +// Patch contains C target changes for one command. +type Patch struct { + Name string + PrependArg []string + AppendArg []string + Env []string +} + // Config contains the facts required to prepare a C target. type Config struct { Matrix string @@ -106,13 +121,13 @@ func (c *Target) Close() error { // Use returns C target defaults for cmd. Explicit Formula settings are // preserved. -func (c *Target) Use(cmd build.Command) build.Patch { +func (c *Target) Use(cmd Command) Patch { base := filepath.Base(cmd.Name) if base == "configure" { return c.autotoolsPatch(cmd) } if filepath.Base(cmd.Name) != cmd.Name { - return build.Patch{} + return Patch{} } switch base { @@ -131,7 +146,7 @@ func (c *Target) Use(cmd build.Command) build.Patch { c.tempDir = tempDir c.toolchainFile = toolchainFile } - return build.Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} + return Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} } case "pkg-config": return c.pkgConfigPatch(cmd.Env) @@ -142,22 +157,22 @@ func (c *Target) Use(cmd build.Command) build.Patch { case "ld", "ld.lld", "ld64.lld": return commandPatch(c.toolchain.Linker()) case "ar", "llvm-ar": - return build.Patch{Name: c.toolchain.Archiver()} + return Patch{Name: c.toolchain.Archiver()} case "ranlib", "llvm-ranlib": - return build.Patch{Name: c.toolchain.Ranlib()} + return Patch{Name: c.toolchain.Ranlib()} case "nm", "llvm-nm": - return build.Patch{Name: c.toolchain.NM()} + return Patch{Name: c.toolchain.NM()} case "strip", "llvm-strip": - return build.Patch{Name: c.toolchain.Strip()} + return Patch{Name: c.toolchain.Strip()} } - return build.Patch{} + return Patch{} } -func commandPatch(command []string) build.Patch { - return build.Patch{Name: command[0], PrependArg: command[1:]} +func commandPatch(command []string) Patch { + return Patch{Name: command[0], PrependArg: command[1:]} } -func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { +func (c *Target) autotoolsPatch(cmd Command) Patch { env := append([]string(nil), cmd.Env...) env = setMissingEnv(env, "CC", shellquote.Join(c.toolchain.CC()...)) env = setMissingEnv(env, "CXX", shellquote.Join(c.toolchain.CXX()...)) @@ -169,7 +184,7 @@ func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { for _, arg := range cmd.Args { if arg == "--host" || strings.HasPrefix(arg, "--host=") { - return build.Patch{Env: env} + return Patch{Env: env} } } @@ -183,16 +198,16 @@ func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { } // Only scripts that declare --host receive the Autoconf target tuple. // For example, zlib's custom configure declares CHOST but rejects --host. - patch := build.Patch{Env: env} + patch := Patch{Env: env} if bytes.Contains(data, []byte("--host")) { patch.AppendArg = []string{"--host=" + c.autotoolsHost} } return patch } -func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { +func (c *Target) pkgConfigPatch(commandEnv []string) Patch { if c.sysroot == "" { - return build.Patch{} + return Patch{} } env := append([]string(nil), commandEnv...) env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) @@ -204,7 +219,7 @@ func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), ) env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) - return build.Patch{Env: env} + return Patch{Env: env} } func (c *Target) cmakeToolchain() string { diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index d0ba054..4c8da15 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" ) @@ -59,17 +58,17 @@ func TestDarwinTarget(t *testing.T) { } t.Cleanup(func() { _ = target.Close() }) - patch := target.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := target.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) want := []string{"--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"} if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = target.Use(build.Command{Name: "cc", Args: []string{"a.o", "-shared"}}) + patch = target.Use(Command{Name: "cc", Args: []string{"a.o", "-shared"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) } - patch = target.Use(build.Command{ + patch = target.Use(Command{ Name: "cc", Args: []string{"--target=custom", "-isysroot/custom", "-fuse-ld=custom"}, }) @@ -82,7 +81,7 @@ func TestDarwinTarget(t *testing.T) { if err := os.WriteFile(configure, []byte("#!/bin/sh\nCHOST=${CHOST-}\n"), 0o755); err != nil { t.Fatal(err) } - patch = target.Use(build.Command{Name: configure}) + patch = target.Use(Command{Name: configure}) if len(patch.AppendArg) != 0 { t.Fatalf("configure AppendArg = %q, want no unsupported arguments", patch.AppendArg) } @@ -98,7 +97,7 @@ func TestDarwinTarget(t *testing.T) { } } - patch = target.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch = target.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) if err != nil { t.Fatal(err) @@ -127,7 +126,7 @@ func TestDarwinArm64Compiler(t *testing.T) { } t.Cleanup(func() { _ = target.Close() }) - patch := target.Use(build.Command{Name: "cc"}) + patch := target.Use(Command{Name: "cc"}) if !slices.Contains(patch.PrependArg, "--target=arm64-apple-macos11.0") { t.Fatalf("PrependArg = %q, want prepared arm64 compiler target", patch.PrependArg) } @@ -140,15 +139,15 @@ func TestBootstrapTargetOmitsSysroot(t *testing.T) { } t.Cleanup(func() { _ = c.Close() }) - patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) if got, want := patch.PrependArg, []string{"--target=aarch64-linux-gnu", "-fuse-ld=lld"}; !reflect.DeepEqual(got, want) { t.Fatalf("PrependArg = %q, want %q", got, want) } - if patch := c.Use(build.Command{Name: "pkg-config"}); patch.Env != nil { + if patch := c.Use(Command{Name: "pkg-config"}); patch.Env != nil { t.Fatalf("pkg-config Patch = %+v, want no sysroot environment", patch) } - patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) if err != nil { t.Fatal(err) @@ -164,7 +163,7 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { t.Fatalf("New created CMake files: toolchainFile=%q tempDir=%q", c.toolchainFile, c.tempDir) } - c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) path := c.toolchainFile data, err := os.ReadFile(path) if err != nil { @@ -186,15 +185,15 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { func TestUseCMake(t *testing.T) { c := newTestTarget(t) - patch := c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch := c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } - patch = c.Use(build.Command{Name: "cmake", Args: []string{"--build", "build"}}) + patch = c.Use(Command{Name: "cmake", Args: []string{"--build", "build"}}) if len(patch.AppendArg) != 0 { t.Fatalf("build Patch = %+v, want no toolchain argument", patch) } - patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) + patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) if len(patch.AppendArg) != 0 { t.Fatalf("explicit toolchain Patch = %+v", patch) } @@ -202,7 +201,7 @@ func TestUseCMake(t *testing.T) { func TestUseDirectCommands(t *testing.T) { c := newTestTarget(t) - patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) if patch.Name != "/llvm/bin/clang" { t.Fatalf("Name = %q", patch.Name) } @@ -210,18 +209,18 @@ func TestUseDirectCommands(t *testing.T) { if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(build.Command{Name: "cc", Args: []string{"a.o", "-o", "a"}}) + patch = c.Use(Command{Name: "cc", Args: []string{"a.o", "-o", "a"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c", "--target=custom", "--sysroot=/custom"}}) + patch = c.Use(Command{Name: "cc", Args: []string{"-c", "a.c", "--target=custom", "--sysroot=/custom"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want prepared defaults %q", patch.PrependArg, want) } - if patch := c.Use(build.Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { + if patch := c.Use(Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { t.Fatalf("explicit compiler path was rewritten: %+v", patch) } - patch = c.Use(build.Command{Name: "ld"}) + patch = c.Use(Command{Name: "ld"}) if patch.Name != "/llvm/bin/ld.lld" { t.Fatalf("linker Name = %q, want /llvm/bin/ld.lld", patch.Name) } @@ -233,7 +232,7 @@ func TestUseAutotools(t *testing.T) { if err := os.WriteFile(configure, []byte("#!/bin/sh\n# options: --build=BUILD --host=HOST\n"), 0o755); err != nil { t.Fatal(err) } - patch := c.Use(build.Command{ + patch := c.Use(Command{ Name: configure, Args: []string{"--build=x86_64-apple-darwin"}, Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, @@ -251,7 +250,7 @@ func TestUseAutotools(t *testing.T) { t.Fatalf("AppendArg = %q, want %q", got, want) } - patch = c.Use(build.Command{Name: configure, Args: []string{"--host=custom-linux"}}) + patch = c.Use(Command{Name: configure, Args: []string{"--host=custom-linux"}}) if len(patch.AppendArg) != 0 { t.Fatalf("explicit host AppendArg = %q, want no duplicate host", patch.AppendArg) } @@ -260,7 +259,7 @@ func TestUseAutotools(t *testing.T) { func TestUsePkgConfig(t *testing.T) { c := newTestTarget(t) depPaths := strings.Join([]string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig"}, string(os.PathListSeparator)) - patch := c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) + patch := c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) } @@ -270,7 +269,7 @@ func TestUsePkgConfig(t *testing.T) { t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) } } - patch = c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) + patch = c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/custom" { t.Fatalf("PKG_CONFIG_LIBDIR override = %q, want /custom", got) } diff --git a/internal/build/cache.go b/internal/build/cache.go index 9a47a14..487966b 100644 --- a/internal/build/cache.go +++ b/internal/build/cache.go @@ -80,7 +80,7 @@ func (b *Builder) installDir(modPath, version string) (string, error) { if err != nil { return "", err } - return filepath.Join(b.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, version, b.matrix)), nil + return filepath.Join(b.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, version, b.matrix.Combinations()[0])), nil } // loadCache reads the cache file for a module from the workspace directory. diff --git a/internal/build/cache_test.go b/internal/build/cache_test.go index d6de8e7..980c133 100644 --- a/internal/build/cache_test.go +++ b/internal/build/cache_test.go @@ -97,7 +97,7 @@ func TestBuildCache_Overwrite(t *testing.T) { } func TestBuilder_InstallDir(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} dir, err := b.installDir("madler/zlib", "1.0.0") if err != nil { @@ -110,7 +110,7 @@ func TestBuilder_InstallDir(t *testing.T) { } func TestBuilder_CacheDir(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} dir, err := b.cacheDir("madler/zlib") if err != nil { @@ -124,7 +124,7 @@ func TestBuilder_CacheDir(t *testing.T) { func TestBuilder_SaveLoadCache(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} + b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} now := time.Now().Truncate(time.Second) original := &buildCache{} @@ -188,7 +188,7 @@ func TestBuilder_SaveLoadCache(t *testing.T) { // --------------------------------------------------------------------------- func TestBuilder_CacheDir_InvalidPath(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} // Empty path should fail EscapePath (filepath.Localize) _, err := b.cacheDir("") @@ -210,7 +210,7 @@ func TestBuilder_CacheDir_InvalidPath(t *testing.T) { } func TestBuilder_InstallDir_InvalidPath(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} _, err := b.installDir("", "1.0.0") if err == nil { @@ -225,7 +225,7 @@ func TestBuilder_InstallDir_InvalidPath(t *testing.T) { func TestBuilder_LoadCache_InvalidPath(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} + b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} _, err := b.loadCache("") if err == nil { @@ -235,7 +235,7 @@ func TestBuilder_LoadCache_InvalidPath(t *testing.T) { func TestBuilder_SaveCache_InvalidPath(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} + b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} cache := &buildCache{} cache.set("1.0.0", "amd64-linux", &buildEntry{BuildTime: time.Now()}) @@ -247,8 +247,8 @@ func TestBuilder_SaveCache_InvalidPath(t *testing.T) { } func TestBuilder_InstallDir_DifferentMatrices(t *testing.T) { - b1 := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} - b2 := &Builder{workspaceDir: "/tmp/ws", matrix: "arm64-darwin"} + b1 := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b2 := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("arm64-darwin")} dir1, _ := b1.installDir("test/lib", "1.0.0") dir2, _ := b2.installDir("test/lib", "1.0.0") @@ -265,7 +265,7 @@ func TestBuilder_InstallDir_DifferentMatrices(t *testing.T) { } func TestBuilder_InstallDir_DifferentVersions(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} dir1, _ := b.installDir("test/lib", "1.0.0") dir2, _ := b.installDir("test/lib", "2.0.0") @@ -277,7 +277,7 @@ func TestBuilder_InstallDir_DifferentVersions(t *testing.T) { func TestBuilder_SaveCache_CreatesDir(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} + b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} cache := &buildCache{} cache.set("1.0.0", "amd64-linux", &buildEntry{ diff --git a/internal/build/e2e_test.go b/internal/build/e2e_test.go index 6968d0d..4393c99 100644 --- a/internal/build/e2e_test.go +++ b/internal/build/e2e_test.go @@ -128,7 +128,7 @@ func TestE2E_MatrixVariation(t *testing.T) { // Verify each matrix has its own install directory for _, matrix := range matrices { - b := &Builder{workspaceDir: wsDir, matrix: matrix} + b := &Builder{workspaceDir: wsDir, matrix: testMatrix(matrix)} dir, _ := b.installDir("test/ctxcheck", "1.0.0") if _, err := os.Stat(dir); err != nil { t.Errorf("installDir not created for matrix %q: %v", matrix, err) @@ -305,7 +305,7 @@ func TestE2E_RealZlibBuild(t *testing.T) { b := &Builder{ store: store, - matrix: matrix, + matrix: testMatrix(matrix), workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -379,7 +379,7 @@ func TestE2E_RealLibpngBuild(t *testing.T) { b := &Builder{ store: store, - matrix: matrix, + matrix: testMatrix(matrix), workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -477,7 +477,7 @@ func TestE2E_RealFreetypeBuild(t *testing.T) { b := &Builder{ store: store, - matrix: matrix, + matrix: testMatrix(matrix), workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { diff --git a/internal/build/http/http.go b/internal/build/http/http.go index b3d54cd..886f2bb 100644 --- a/internal/build/http/http.go +++ b/internal/build/http/http.go @@ -130,7 +130,7 @@ func (h *handler) build(ctx context.Context, req request, info io.Writer) (resul builder, err := build.NewBuilder(build.Options{ Store: h.formulaStore, - MatrixStr: req.matrixStr, + Matrix: req.matrix, Stdout: info, Stderr: info, WorkspaceDir: h.workspaceDir, diff --git a/internal/build/target.go b/internal/build/target.go index 8e3182f..2510caa 100644 --- a/internal/build/target.go +++ b/internal/build/target.go @@ -3,37 +3,17 @@ package build import ( "os" + "github.com/goplus/llar/internal/build/c" "github.com/goplus/llar/internal/execbroker" ) -// Command describes a command before target defaults are applied. -type Command struct { - Name string - Args []string - Env []string - Dir string -} - -// Patch contains target-specific changes for one command. -type Patch struct { - Name string - PrependArg []string - AppendArg []string - Env []string -} - -// Target applies language-specific target defaults to build commands. -type Target interface { - Use(Command) Patch -} - -func targetMiddleware(target Target) execbroker.Middleware { +func targetMiddleware(target *c.Target) execbroker.Middleware { return func(req execbroker.Request) execbroker.Request { env := req.Env if env == nil { env = os.Environ() } - patch := target.Use(Command{ + patch := target.Use(c.Command{ Name: req.Name, Args: req.Args, Env: env, diff --git a/internal/build/target_test.go b/internal/build/target_test.go index 1591704..5a7a952 100644 --- a/internal/build/target_test.go +++ b/internal/build/target_test.go @@ -2,32 +2,48 @@ package build import ( "context" + "path/filepath" "reflect" + "runtime" "testing" "testing/fstest" classfile "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build/c" "github.com/goplus/llar/internal/execbroker" internalformula "github.com/goplus/llar/internal/formula" "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/internal/vcs" ) -type testTarget struct { - command Command -} - -func (t *testTarget) Use(command Command) Patch { - t.command = command - return Patch{ - Name: "/toolchain/cc", - PrependArg: []string{"--target=aarch64-linux-gnu"}, - AppendArg: []string{"--sysroot=/sdk"}, - Env: []string{"CC=/toolchain/cc"}, +func newCTarget(t *testing.T, targetOS, targetArch string) *c.Target { + t.Helper() + triple := "x86_64-linux-gnu" + if targetArch == "arm64" { + triple = "aarch64-linux-gnu" } + target, err := c.NewTarget(c.Config{ + Matrix: targetArch + "-" + targetOS, + Toolchain: c.NewToolchain( + []string{"/toolchain/cc", "--target=" + triple, "--sysroot=/sdk"}, + []string{"/toolchain/c++", "--target=" + triple, "--sysroot=/sdk"}, + []string{"/toolchain/ld.lld"}, + "/toolchain/ar", + "/toolchain/ranlib", + "/toolchain/nm", + "/toolchain/strip", + ), + Sysroot: "/sdk", + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = target.Close() }) + return target } func TestTargetMiddleware(t *testing.T) { - target := new(testTarget) + target := newCTarget(t, "linux", "arm64") got := targetMiddleware(target)(execbroker.Request{ Name: "cc", Args: []string{"-c", "a.c"}, @@ -37,25 +53,37 @@ func TestTargetMiddleware(t *testing.T) { if got.Name != "/toolchain/cc" { t.Fatalf("Name = %q", got.Name) } - if want := []string{"--target=aarch64-linux-gnu", "-c", "a.c", "--sysroot=/sdk"}; !reflect.DeepEqual(got.Args, want) { + if want := []string{"--target=aarch64-linux-gnu", "--sysroot=/sdk", "-c", "a.c"}; !reflect.DeepEqual(got.Args, want) { t.Fatalf("Args = %q, want %q", got.Args, want) } - if want := []string{"CC=/toolchain/cc"}; !reflect.DeepEqual(got.Env, want) { + if want := []string{"CFLAGS=-O2"}; !reflect.DeepEqual(got.Env, want) { t.Fatalf("Env = %q, want %q", got.Env, want) } - if target.command.Name != "cc" || target.command.Dir != "/src" { - t.Fatalf("Command = %+v", target.command) - } - if want := []string{"CFLAGS=-O2"}; !reflect.DeepEqual(target.command.Env, want) { - t.Fatalf("Command.Env = %q, want %q", target.command.Env, want) - } } func TestBuildAppliesTargetToFormulaCommands(t *testing.T) { store := setupTestStore(t) - builder := setupBuilder(t, store, "arm64-linux") - target := new(testTarget) - builder.target = target + targetOS, targetArch := "linux", "amd64" + if runtime.GOOS == targetOS && runtime.GOARCH == targetArch { + targetArch = "arm64" + } + target := newCTarget(t, targetOS, targetArch) + builder, err := NewBuilder(Options{ + Store: store, + Matrix: classfile.Matrix{Require: map[string][]string{ + "os": {targetOS}, + "arch": {targetArch}, + }}, + WorkspaceDir: t.TempDir(), + Target: target, + }) + if err != nil { + t.Fatal(err) + } + builder.newRepo = func(string) (vcs.Repo, error) { + return newMockRepo(filepath.Join(testSourceDir, "test", "liba")), nil + } + t.Setenv("PATH", "") var commandName string root := &modules.Module{ @@ -72,7 +100,4 @@ func TestBuildAppliesTargetToFormulaCommands(t *testing.T) { if commandName != "/toolchain/cc" { t.Fatalf("command name = %q, want /toolchain/cc", commandName) } - if target.command.Name != "cc" { - t.Fatalf("target command = %+v", target.command) - } } diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index 8e8f762..7c5e4e5 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -541,7 +541,7 @@ func (s *suite) build(ctx context.Context, target module.Version, matrix, worksp } builder, err := build.NewBuilder(build.Options{ Store: s.formulas, - MatrixStr: matrix, + Matrix: targetMatrix, WorkspaceDir: workspaceDir, Cache: c, }) From 2edfa6531229845ce2135cd61485ac7d96a461d7 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 10:55:26 +0800 Subject: [PATCH 23/39] refactor(build): reuse graph build closure for sysroot --- internal/build/build.go | 435 ++++++++++++++++++++-------------------- 1 file changed, 218 insertions(+), 217 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index d24baec..3802b16 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -246,6 +246,222 @@ func (b *Builder) resolveModTransitiveDeps(targets []*modules.Module, mod *modul } func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Result, error) { + build := func(targets []*modules.Module, buildTarget *c.Target, runTest bool) ([]Result, error) { + builtResults := make(map[module.Version]classfile.BuildResult) + + // Identify the root target. By MVS convention (see constructBuildList + // and modules.Load), targets[0] is the main module requested by the + // caller; runTest semantics (fresh build + OnTest invocation) only + // apply to it. + // + // Root identity is tracked by (Path, Version) rather than pointer + // equality so the comparison survives any future refactor of + // constructBuildList that stops reusing *modules.Module pointers + // (e.g. parallel builds that clone module structs). The pair is + // unique in an MVS build list, so it is a safe identity key. + var rootID module.Version + if len(targets) > 0 { + rootID = module.Version{Path: targets[0].Path, Version: targets[0].Version} + } + + var commandMiddleware execbroker.Middleware + if buildTarget != nil { + commandMiddleware = targetMiddleware(buildTarget) + } + + buildModule := func(mod *modules.Module) (Result, error) { + isRoot := mod.Path == rootID.Path && mod.Version == rootID.Version + testThisMod := runTest && isRoot && mod.OnTest != nil + + installDir, err := b.installDir(mod.Path, mod.Version) + if err != nil { + return Result{}, err + } + deps := b.resolveModTransitiveDeps(targets, mod) + modVer := module.Version{Path: mod.Path, Version: mod.Version} + + // Consult the build cache. A hit means we already have the + // module's build metadata and its installDir is populated from a + // previous successful build. + entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}) + if err != nil { + return Result{}, err + } + + // Fast path: cache hit and no OnTest to run. Skip source clone + // and OnBuild entirely. + if cacheHit && !testThisMod { + return Result{Metadata: entry.Metadata, OutputDir: installDir}, nil + } + + // At this point we need to run OnBuild, OnTest, or both. All of + // them expect a source checkout and a prepared build context, so + // set those up uniformly regardless of cache state. + + // TODO(MeteorsLiu): Source cache dir (belongs in the vcs layer) + tmpSourceDir, err := os.MkdirTemp("", fmt.Sprintf("source-%s-%s*", strings.ReplaceAll(mod.Path, "/", "-"), mod.Version)) + if err != nil { + return Result{}, err + } + defer os.RemoveAll(tmpSourceDir) + + // Before we start to build, clone source to tmpSourceDir. + // TODO(MeteorsLiu): Support different code host + repo, err := b.newRepo(fmt.Sprintf("github.com/%s", mod.Path)) + if err != nil { + return Result{}, err + } + if err := repo.Sync(ctx, mod.Version, "", tmpSourceDir); err != nil { + return Result{}, err + } + + if err := os.MkdirAll(installDir, 0o755); err != nil { + return Result{}, err + } + + getOutputDir := func(_ string, m module.Version) (string, error) { + return b.installDir(m.Path, m.Version) + } + project := &classfile.Project{Deps: deps, SourceFS: mod.FS.(fs.ReadFileFS)} + buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix.Combinations()[0], getOutputDir) + + // Inject results of already-built dependencies + for modVer, result := range builtResults { + buildContext.AddBuildResult(modVer, result) + } + + var metadata string + if err := execbroker.Do(execbroker.Scope{ + Dir: tmpSourceDir, + Stdin: os.Stdin, + Stdout: b.stdout, + Stderr: b.stderr, + Middleware: commandMiddleware, + }, func() error { + // Run OnBuild only on cache miss; reuse cached metadata otherwise. + if cacheHit { + metadata = entry.Metadata + } else { + if err := runFormulaHook(func() { + mod.OnBuild(buildContext) + }); err != nil { + return err + } + if len(buildContext.Errs) > 0 { + return errors.Join(buildContext.Errs...) + } + metadata = buildContext.Out.Metadata() + } + + // Run OnTest (root only) against the just-built or cached + // artifacts, reusing the same build context so tests see a + // consistent environment either way. + if testThisMod { + if err := runFormulaHook(func() { + mod.OnTest(buildContext) + }); err != nil { + return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, err) + } + if len(buildContext.Errs) > 0 { + return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, buildContext.Errs.ToError()) + } + } + return nil + }); err != nil { + return Result{}, err + } + + // Save cache only on cache miss. A cache hit means the entry is + // already present and current; OnTest does not modify metadata. + if !cacheHit { + entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}, os.DirFS(installDir), cache.Entry{ + Metadata: metadata, + Deps: deps, + }) + if err != nil { + return Result{}, err + } + metadata = entry.Metadata + } + + return Result{Metadata: metadata, OutputDir: installDir}, nil + } + + var results []Result + + buildList := b.constructBuildList(targets) + lockPaths := make([]string, 0, len(buildList)) + for _, target := range buildList { + lockPaths = append(lockPaths, target.Path) + } + // A dependent keeps reading its dependencies' install directories after + // their own build steps return, so hold the whole graph until Build completes. + // + // Case 1 - Disjoint graphs: + // Request A builds libpng -> zlib and request B builds curl -> openssl. + // They lock different module paths and remain parallel. + // + // Case 2 - Overlapping graphs: + // Request A builds libpng -> zlib and request B builds freetype -> zlib. + // Because both graphs contain zlib, the later request waits for the earlier + // Build to finish, then reuses its published zlib artifact instead of observing + // a replaced install tree. + // + // Lock ordering: + // Use a stable order so overlapping graphs cannot deadlock. For example, + // X -> Y produces build order [Y, X], while another matrix with Y -> X produces + // [X, Y]. Locking in build order can leave each request holding one lock and + // waiting for the other; sorting makes both lock [X, Y]. + sort.Strings(lockPaths) + unlocks := make([]func(), 0, len(lockPaths)) + for _, path := range lockPaths { + unlock, err := b.store.LockModule(path) + if err != nil { + for i := len(unlocks) - 1; i >= 0; i-- { + unlocks[i]() + } + return nil, err + } + unlocks = append(unlocks, unlock) + } + defer func() { + for i := len(unlocks) - 1; i >= 0; i-- { + unlocks[i]() + } + }() + + // Save current environment and restore it after OnBuild, + // that's because OnBuild may break environment + // TODO(MeteorsLiu): Switch to sandbox to run OnBuild + savedEnv := os.Environ() + defer func() { + os.Clearenv() + for _, env := range savedEnv { + k, v, _ := strings.Cut(env, "=") + os.Setenv(k, v) + } + }() + + // TODO(MeteorsLiu): Parallel build + for _, target := range buildList { + result, err := buildModule(target) + if err != nil { + return nil, err + } + + // Track result for downstream dependencies + modVer := module.Version{Path: target.Path, Version: target.Version} + br := classfile.BuildResult{} + if result.Metadata != "" { + br.SetMetadata(result.Metadata) + } + builtResults[modVer] = br + + results = append(results, result) + } + return results, nil + } + buildTarget := b.target var targetOS, targetArch string if values := b.matrix.Require["os"]; len(values) > 0 { @@ -291,10 +507,7 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul if len(sysrootMods) > 0 { selectedSysroot := sysrootMods[0] - sysrootBuilder := *b - sysrootBuilder.runTest = false - sysrootBuilder.target = bootstrapTarget - sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) + sysrootResults, err := build(sysrootMods, bootstrapTarget, false) if err != nil { return nil, fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) } @@ -324,217 +537,5 @@ func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Resul } } - builtResults := make(map[module.Version]classfile.BuildResult) - - // Identify the root target. By MVS convention (see constructBuildList - // and modules.Load), targets[0] is the main module requested by the - // caller; runTest semantics (fresh build + OnTest invocation) only - // apply to it. - // - // Root identity is tracked by (Path, Version) rather than pointer - // equality so the comparison survives any future refactor of - // constructBuildList that stops reusing *modules.Module pointers - // (e.g. parallel builds that clone module structs). The pair is - // unique in an MVS build list, so it is a safe identity key. - var rootID module.Version - if len(targets) > 0 { - rootID = module.Version{Path: targets[0].Path, Version: targets[0].Version} - } - - var commandMiddleware execbroker.Middleware - if buildTarget != nil { - commandMiddleware = targetMiddleware(buildTarget) - } - - build := func(mod *modules.Module) (Result, error) { - isRoot := mod.Path == rootID.Path && mod.Version == rootID.Version - testThisMod := b.runTest && isRoot && mod.OnTest != nil - - installDir, err := b.installDir(mod.Path, mod.Version) - if err != nil { - return Result{}, err - } - deps := b.resolveModTransitiveDeps(targets, mod) - modVer := module.Version{Path: mod.Path, Version: mod.Version} - - // Consult the build cache. A hit means we already have the - // module's build metadata and its installDir is populated from a - // previous successful build. - entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}) - if err != nil { - return Result{}, err - } - - // Fast path: cache hit and no OnTest to run. Skip source clone - // and OnBuild entirely. - if cacheHit && !testThisMod { - return Result{Metadata: entry.Metadata, OutputDir: installDir}, nil - } - - // At this point we need to run OnBuild, OnTest, or both. All of - // them expect a source checkout and a prepared build context, so - // set those up uniformly regardless of cache state. - - // TODO(MeteorsLiu): Source cache dir (belongs in the vcs layer) - tmpSourceDir, err := os.MkdirTemp("", fmt.Sprintf("source-%s-%s*", strings.ReplaceAll(mod.Path, "/", "-"), mod.Version)) - if err != nil { - return Result{}, err - } - defer os.RemoveAll(tmpSourceDir) - - // Before we start to build, clone source to tmpSourceDir. - // TODO(MeteorsLiu): Support different code host - repo, err := b.newRepo(fmt.Sprintf("github.com/%s", mod.Path)) - if err != nil { - return Result{}, err - } - if err := repo.Sync(ctx, mod.Version, "", tmpSourceDir); err != nil { - return Result{}, err - } - - if err := os.MkdirAll(installDir, 0o755); err != nil { - return Result{}, err - } - - getOutputDir := func(_ string, m module.Version) (string, error) { - return b.installDir(m.Path, m.Version) - } - project := &classfile.Project{Deps: deps, SourceFS: mod.FS.(fs.ReadFileFS)} - buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix.Combinations()[0], getOutputDir) - - // Inject results of already-built dependencies - for modVer, result := range builtResults { - buildContext.AddBuildResult(modVer, result) - } - - var metadata string - if err := execbroker.Do(execbroker.Scope{ - Dir: tmpSourceDir, - Stdin: os.Stdin, - Stdout: b.stdout, - Stderr: b.stderr, - Middleware: commandMiddleware, - }, func() error { - // Run OnBuild only on cache miss; reuse cached metadata otherwise. - if cacheHit { - metadata = entry.Metadata - } else { - if err := runFormulaHook(func() { - mod.OnBuild(buildContext) - }); err != nil { - return err - } - if len(buildContext.Errs) > 0 { - return errors.Join(buildContext.Errs...) - } - metadata = buildContext.Out.Metadata() - } - - // Run OnTest (root only) against the just-built or cached - // artifacts, reusing the same build context so tests see a - // consistent environment either way. - if testThisMod { - if err := runFormulaHook(func() { - mod.OnTest(buildContext) - }); err != nil { - return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, err) - } - if len(buildContext.Errs) > 0 { - return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, buildContext.Errs.ToError()) - } - } - return nil - }); err != nil { - return Result{}, err - } - - // Save cache only on cache miss. A cache hit means the entry is - // already present and current; OnTest does not modify metadata. - if !cacheHit { - entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}, os.DirFS(installDir), cache.Entry{ - Metadata: metadata, - Deps: deps, - }) - if err != nil { - return Result{}, err - } - metadata = entry.Metadata - } - - return Result{Metadata: metadata, OutputDir: installDir}, nil - } - - var results []Result - - buildList := b.constructBuildList(targets) - lockPaths := make([]string, 0, len(buildList)) - for _, target := range buildList { - lockPaths = append(lockPaths, target.Path) - } - // A dependent keeps reading its dependencies' install directories after - // their own build steps return, so hold the whole graph until Build completes. - // - // Case 1 - Disjoint graphs: - // Request A builds libpng -> zlib and request B builds curl -> openssl. - // They lock different module paths and remain parallel. - // - // Case 2 - Overlapping graphs: - // Request A builds libpng -> zlib and request B builds freetype -> zlib. - // Because both graphs contain zlib, the later request waits for the earlier - // Build to finish, then reuses its published zlib artifact instead of observing - // a replaced install tree. - // - // Lock ordering: - // Use a stable order so overlapping graphs cannot deadlock. For example, - // X -> Y produces build order [Y, X], while another matrix with Y -> X produces - // [X, Y]. Locking in build order can leave each request holding one lock and - // waiting for the other; sorting makes both lock [X, Y]. - sort.Strings(lockPaths) - unlocks := make([]func(), 0, len(lockPaths)) - for _, path := range lockPaths { - unlock, err := b.store.LockModule(path) - if err != nil { - for i := len(unlocks) - 1; i >= 0; i-- { - unlocks[i]() - } - return nil, err - } - unlocks = append(unlocks, unlock) - } - defer func() { - for i := len(unlocks) - 1; i >= 0; i-- { - unlocks[i]() - } - }() - - // Save current environment and restore it after OnBuild, - // that's because OnBuild may break environment - // TODO(MeteorsLiu): Switch to sandbox to run OnBuild - savedEnv := os.Environ() - defer func() { - os.Clearenv() - for _, env := range savedEnv { - k, v, _ := strings.Cut(env, "=") - os.Setenv(k, v) - } - }() - - // TODO(MeteorsLiu): Parallel build - for _, target := range buildList { - result, err := build(target) - if err != nil { - return nil, err - } - - // Track result for downstream dependencies - modVer := module.Version{Path: target.Path, Version: target.Version} - br := classfile.BuildResult{} - if result.Metadata != "" { - br.SetMetadata(result.Metadata) - } - builtResults[modVer] = br - - results = append(results, result) - } - return results, nil + return build(targets, buildTarget, b.runTest) } From 6ade7fe533f852fdca93b9f74336e0fa359331c3 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 11:11:50 +0800 Subject: [PATCH 24/39] revert: keep sysroot orchestration outside build --- cmd/llar/internal/make.go | 118 +++++++- cmd/llar/internal/make_test.go | 2 +- internal/build/build.go | 468 +++++++++++++------------------- internal/build/build_test.go | 20 +- internal/build/c/target.go | 49 ++-- internal/build/c/target_test.go | 45 +-- internal/build/cache.go | 2 +- internal/build/cache_test.go | 22 +- internal/build/e2e_test.go | 8 +- internal/build/http/http.go | 2 +- internal/build/target.go | 26 +- internal/build/target_test.go | 77 ++---- testdata/kodo-e2e/main.go | 2 +- 13 files changed, 408 insertions(+), 433 deletions(-) diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index 774cf87..d31ae3a 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -12,11 +12,14 @@ import ( "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/c" + "github.com/goplus/llar/internal/build/c/llvm" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/modules/modlocal" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" + ccmetadata "github.com/goplus/llar/x/metadata/cc" "github.com/spf13/cobra" ) @@ -130,18 +133,59 @@ func hostMatrix() formula.Matrix { // `llar test ` invocation. func buildModule(ctx context.Context, store repo.Store, modPath, version string, matrix formula.Matrix, runTest bool) error { root := module.Version{Path: modPath, Version: version} + var targetOS, targetArch string + if values := matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + crossCompile := targetOS != runtime.GOOS || targetArch != runtime.GOARCH + if runTest && crossCompile { + return fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + } + var targetRoot module.Version + var useCTarget bool + if crossCompile { + // TODO: Add other language target policies alongside this C case when + // they provide build.Target implementations. + cSysroot, ok := c.Sysroot(targetOS, targetArch) + useCTarget = ok + _, customLibc := matrix.Require["libc"] + if useCTarget && !customLibc && root.Path != cSysroot.Path { + targetRoot = cSysroot + } + } + loadOpts := modules.Options{ + FormulaStore: store, + Matrix: matrix, + } + mods, err := modules.Load(ctx, root, loadOpts) + if err != nil { + return fmt.Errorf("failed to load modules: %w", err) + } + var sysrootMods []*modules.Module + if targetRoot != (module.Version{}) { + // The default sysroot has no dependencies, but modules.Load still owns + // selecting the Formula whose fromVer applies to targetRoot.Version. + sysrootMods, err = modules.Load(ctx, targetRoot, loadOpts) + if err != nil { + return fmt.Errorf("failed to load sysroot %s@%s: %w", targetRoot.Path, targetRoot.Version, err) + } + } var buildOutput io.Writer = io.Discard if makeVerbose { buildOutput = os.Stderr } + matrixStr := matrix.Combinations()[0] buildOpts := build.Options{ - Store: store, - Matrix: matrix, - RunTest: runTest, - Stdout: buildOutput, - Stderr: buildOutput, + Store: store, + MatrixStr: matrixStr, + RunTest: runTest, + Stdout: buildOutput, + Stderr: buildOutput, } if makeOutput != "" { tmpDir, err := os.MkdirTemp("", "llar-make-*") @@ -151,17 +195,67 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } + var target build.Target + // TODO: Add other language build.Target preparation alongside this C case. + if useCTarget { + targetMatrix := targetArch + "-" + targetOS + bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) + if err != nil { + return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + bootstrapTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: bootstrapToolchain.Toolchain, + }) + if err != nil { + return err + } + defer bootstrapTarget.Close() + target = bootstrapTarget + + if targetRoot != (module.Version{}) { + selectedSysroot := sysrootMods[0] + + sysrootOpts := buildOpts + sysrootOpts.RunTest = false + sysrootOpts.Target = bootstrapTarget + sysrootBuilder, err := build.NewBuilder(sysrootOpts) + if err != nil { + return fmt.Errorf("failed to create sysroot builder: %w", err) + } + sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) + if err != nil { + return fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) + if err != nil { + return fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + if metadata.Sysroot() == "" { + return fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) + } + + configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) + if err != nil { + return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + configuredTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: configuredToolchain.Toolchain, + Sysroot: metadata.Sysroot(), + }) + if err != nil { + return err + } + defer configuredTarget.Close() + target = configuredTarget + } + } + buildOpts.Target = target builder, err := build.NewBuilder(buildOpts) if err != nil { return fmt.Errorf("failed to create builder: %w", err) } - mods, err := modules.Load(ctx, root, modules.Options{ - FormulaStore: store, - Matrix: matrix, - }) - if err != nil { - return fmt.Errorf("failed to load modules: %w", err) - } results, err := builder.Build(ctx, mods) if err != nil { diff --git a/cmd/llar/internal/make_test.go b/cmd/llar/internal/make_test.go index 77c2fde..34a338e 100644 --- a/cmd/llar/internal/make_test.go +++ b/cmd/llar/internal/make_test.go @@ -178,7 +178,7 @@ func TestBuildModuleRejectsCrossTargetTest(t *testing.T) { }} err := buildModule(context.Background(), nil, "owner/repo", "v1.0.0", matrix, true) - want := fmt.Sprintf("failed to create builder: llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) + want := fmt.Sprintf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) if err == nil || err.Error() != want { t.Fatalf("buildModule error = %v, want %q", err, want) } diff --git a/internal/build/build.go b/internal/build/build.go index 3802b16..ee5e0c3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -8,31 +8,27 @@ import ( "io/fs" "os" "path/filepath" - "runtime" "sort" "strings" classfile "github.com/goplus/llar/formula" - "github.com/goplus/llar/internal/build/c" - "github.com/goplus/llar/internal/build/c/llvm" "github.com/goplus/llar/internal/build/cache" "github.com/goplus/llar/internal/execbroker" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" - ccmetadata "github.com/goplus/llar/x/metadata/cc" ) type Builder struct { store repo.Store - matrix classfile.Matrix + matrix string runTest bool stdout io.Writer stderr io.Writer workspaceDir string cache cache.Cache - target *c.Target + target Target newRepo func(repoPath string) (vcs.Repo, error) // defaults to vcs.NewRepo } @@ -42,10 +38,8 @@ type Result struct { } type Options struct { - Store repo.Store - // Matrix is the selected build matrix used for cache identity and target - // preparation. - Matrix classfile.Matrix + Store repo.Store + MatrixStr string // RunTest, when true, causes Build to invoke OnTest on the root target // after OnBuild (or after reusing cached build metadata). The build // cache is consulted as usual: on a cache hit the root's OnBuild is @@ -58,8 +52,7 @@ type Options struct { Stderr io.Writer WorkspaceDir string Cache cache.Cache - // Target overrides automatic C target and default sysroot preparation. - Target *c.Target + Target Target } func runFormulaHook(fn func()) (err error) { @@ -91,16 +84,6 @@ func defaultWorkspaceDir() (string, error) { // NewBuilder creates a new Builder. func NewBuilder(opts Options) (*Builder, error) { - var targetOS, targetArch string - if values := opts.Matrix.Require["os"]; len(values) > 0 { - targetOS = values[0] - } - if values := opts.Matrix.Require["arch"]; len(values) > 0 { - targetArch = values[0] - } - if opts.RunTest && (targetOS != runtime.GOOS || targetArch != runtime.GOARCH) { - return nil, fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) - } workspaceDir := opts.WorkspaceDir if workspaceDir == "" { var err error @@ -123,7 +106,7 @@ func NewBuilder(opts Options) (*Builder, error) { } return &Builder{ store: opts.Store, - matrix: opts.Matrix, + matrix: opts.MatrixStr, runTest: opts.RunTest, stdout: stdout, stderr: stderr, @@ -246,296 +229,217 @@ func (b *Builder) resolveModTransitiveDeps(targets []*modules.Module, mod *modul } func (b *Builder) Build(ctx context.Context, targets []*modules.Module) ([]Result, error) { - build := func(targets []*modules.Module, buildTarget *c.Target, runTest bool) ([]Result, error) { - builtResults := make(map[module.Version]classfile.BuildResult) - - // Identify the root target. By MVS convention (see constructBuildList - // and modules.Load), targets[0] is the main module requested by the - // caller; runTest semantics (fresh build + OnTest invocation) only - // apply to it. - // - // Root identity is tracked by (Path, Version) rather than pointer - // equality so the comparison survives any future refactor of - // constructBuildList that stops reusing *modules.Module pointers - // (e.g. parallel builds that clone module structs). The pair is - // unique in an MVS build list, so it is a safe identity key. - var rootID module.Version - if len(targets) > 0 { - rootID = module.Version{Path: targets[0].Path, Version: targets[0].Version} - } + builtResults := make(map[module.Version]classfile.BuildResult) + + // Identify the root target. By MVS convention (see constructBuildList + // and modules.Load), targets[0] is the main module requested by the + // caller; runTest semantics (fresh build + OnTest invocation) only + // apply to it. + // + // Root identity is tracked by (Path, Version) rather than pointer + // equality so the comparison survives any future refactor of + // constructBuildList that stops reusing *modules.Module pointers + // (e.g. parallel builds that clone module structs). The pair is + // unique in an MVS build list, so it is a safe identity key. + var rootID module.Version + if len(targets) > 0 { + rootID = module.Version{Path: targets[0].Path, Version: targets[0].Version} + } - var commandMiddleware execbroker.Middleware - if buildTarget != nil { - commandMiddleware = targetMiddleware(buildTarget) - } + var commandMiddleware execbroker.Middleware + if b.target != nil { + commandMiddleware = targetMiddleware(b.target) + } - buildModule := func(mod *modules.Module) (Result, error) { - isRoot := mod.Path == rootID.Path && mod.Version == rootID.Version - testThisMod := runTest && isRoot && mod.OnTest != nil + build := func(mod *modules.Module) (Result, error) { + isRoot := mod.Path == rootID.Path && mod.Version == rootID.Version + testThisMod := b.runTest && isRoot && mod.OnTest != nil - installDir, err := b.installDir(mod.Path, mod.Version) - if err != nil { - return Result{}, err - } - deps := b.resolveModTransitiveDeps(targets, mod) - modVer := module.Version{Path: mod.Path, Version: mod.Version} + installDir, err := b.installDir(mod.Path, mod.Version) + if err != nil { + return Result{}, err + } + deps := b.resolveModTransitiveDeps(targets, mod) + modVer := module.Version{Path: mod.Path, Version: mod.Version} - // Consult the build cache. A hit means we already have the - // module's build metadata and its installDir is populated from a - // previous successful build. - entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}) - if err != nil { - return Result{}, err - } + // Consult the build cache. A hit means we already have the + // module's build metadata and its installDir is populated from a + // previous successful build. + entry, cacheHit, err := b.cache.Get(ctx, cache.Key{Module: modVer, Matrix: b.matrix}) + if err != nil { + return Result{}, err + } - // Fast path: cache hit and no OnTest to run. Skip source clone - // and OnBuild entirely. - if cacheHit && !testThisMod { - return Result{Metadata: entry.Metadata, OutputDir: installDir}, nil - } + // Fast path: cache hit and no OnTest to run. Skip source clone + // and OnBuild entirely. + if cacheHit && !testThisMod { + return Result{Metadata: entry.Metadata, OutputDir: installDir}, nil + } - // At this point we need to run OnBuild, OnTest, or both. All of - // them expect a source checkout and a prepared build context, so - // set those up uniformly regardless of cache state. + // At this point we need to run OnBuild, OnTest, or both. All of + // them expect a source checkout and a prepared build context, so + // set those up uniformly regardless of cache state. - // TODO(MeteorsLiu): Source cache dir (belongs in the vcs layer) - tmpSourceDir, err := os.MkdirTemp("", fmt.Sprintf("source-%s-%s*", strings.ReplaceAll(mod.Path, "/", "-"), mod.Version)) - if err != nil { - return Result{}, err - } - defer os.RemoveAll(tmpSourceDir) + // TODO(MeteorsLiu): Source cache dir (belongs in the vcs layer) + tmpSourceDir, err := os.MkdirTemp("", fmt.Sprintf("source-%s-%s*", strings.ReplaceAll(mod.Path, "/", "-"), mod.Version)) + if err != nil { + return Result{}, err + } + defer os.RemoveAll(tmpSourceDir) - // Before we start to build, clone source to tmpSourceDir. - // TODO(MeteorsLiu): Support different code host - repo, err := b.newRepo(fmt.Sprintf("github.com/%s", mod.Path)) - if err != nil { - return Result{}, err - } - if err := repo.Sync(ctx, mod.Version, "", tmpSourceDir); err != nil { - return Result{}, err - } + // Before we start to build, clone source to tmpSourceDir. + // TODO(MeteorsLiu): Support different code host + repo, err := b.newRepo(fmt.Sprintf("github.com/%s", mod.Path)) + if err != nil { + return Result{}, err + } + if err := repo.Sync(ctx, mod.Version, "", tmpSourceDir); err != nil { + return Result{}, err + } - if err := os.MkdirAll(installDir, 0o755); err != nil { - return Result{}, err - } + if err := os.MkdirAll(installDir, 0o755); err != nil { + return Result{}, err + } - getOutputDir := func(_ string, m module.Version) (string, error) { - return b.installDir(m.Path, m.Version) - } - project := &classfile.Project{Deps: deps, SourceFS: mod.FS.(fs.ReadFileFS)} - buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix.Combinations()[0], getOutputDir) + getOutputDir := func(_ string, m module.Version) (string, error) { + return b.installDir(m.Path, m.Version) + } + project := &classfile.Project{Deps: deps, SourceFS: mod.FS.(fs.ReadFileFS)} + buildContext := classfile.NewContext(project, tmpSourceDir, installDir, b.matrix, getOutputDir) - // Inject results of already-built dependencies - for modVer, result := range builtResults { - buildContext.AddBuildResult(modVer, result) - } + // Inject results of already-built dependencies + for modVer, result := range builtResults { + buildContext.AddBuildResult(modVer, result) + } - var metadata string - if err := execbroker.Do(execbroker.Scope{ - Dir: tmpSourceDir, - Stdin: os.Stdin, - Stdout: b.stdout, - Stderr: b.stderr, - Middleware: commandMiddleware, - }, func() error { - // Run OnBuild only on cache miss; reuse cached metadata otherwise. - if cacheHit { - metadata = entry.Metadata - } else { - if err := runFormulaHook(func() { - mod.OnBuild(buildContext) - }); err != nil { - return err - } - if len(buildContext.Errs) > 0 { - return errors.Join(buildContext.Errs...) - } - metadata = buildContext.Out.Metadata() + var metadata string + if err := execbroker.Do(execbroker.Scope{ + Dir: tmpSourceDir, + Stdin: os.Stdin, + Stdout: b.stdout, + Stderr: b.stderr, + Middleware: commandMiddleware, + }, func() error { + // Run OnBuild only on cache miss; reuse cached metadata otherwise. + if cacheHit { + metadata = entry.Metadata + } else { + if err := runFormulaHook(func() { + mod.OnBuild(buildContext) + }); err != nil { + return err } - - // Run OnTest (root only) against the just-built or cached - // artifacts, reusing the same build context so tests see a - // consistent environment either way. - if testThisMod { - if err := runFormulaHook(func() { - mod.OnTest(buildContext) - }); err != nil { - return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, err) - } - if len(buildContext.Errs) > 0 { - return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, buildContext.Errs.ToError()) - } + if len(buildContext.Errs) > 0 { + return errors.Join(buildContext.Errs...) } - return nil - }); err != nil { - return Result{}, err + metadata = buildContext.Out.Metadata() } - // Save cache only on cache miss. A cache hit means the entry is - // already present and current; OnTest does not modify metadata. - if !cacheHit { - entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix.Combinations()[0]}, os.DirFS(installDir), cache.Entry{ - Metadata: metadata, - Deps: deps, - }) - if err != nil { - return Result{}, err + // Run OnTest (root only) against the just-built or cached + // artifacts, reusing the same build context so tests see a + // consistent environment either way. + if testThisMod { + if err := runFormulaHook(func() { + mod.OnTest(buildContext) + }); err != nil { + return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, err) + } + if len(buildContext.Errs) > 0 { + return fmt.Errorf("onTest failed for %s@%s: %w", mod.Path, mod.Version, buildContext.Errs.ToError()) } - metadata = entry.Metadata } - - return Result{Metadata: metadata, OutputDir: installDir}, nil + return nil + }); err != nil { + return Result{}, err } - var results []Result - - buildList := b.constructBuildList(targets) - lockPaths := make([]string, 0, len(buildList)) - for _, target := range buildList { - lockPaths = append(lockPaths, target.Path) - } - // A dependent keeps reading its dependencies' install directories after - // their own build steps return, so hold the whole graph until Build completes. - // - // Case 1 - Disjoint graphs: - // Request A builds libpng -> zlib and request B builds curl -> openssl. - // They lock different module paths and remain parallel. - // - // Case 2 - Overlapping graphs: - // Request A builds libpng -> zlib and request B builds freetype -> zlib. - // Because both graphs contain zlib, the later request waits for the earlier - // Build to finish, then reuses its published zlib artifact instead of observing - // a replaced install tree. - // - // Lock ordering: - // Use a stable order so overlapping graphs cannot deadlock. For example, - // X -> Y produces build order [Y, X], while another matrix with Y -> X produces - // [X, Y]. Locking in build order can leave each request holding one lock and - // waiting for the other; sorting makes both lock [X, Y]. - sort.Strings(lockPaths) - unlocks := make([]func(), 0, len(lockPaths)) - for _, path := range lockPaths { - unlock, err := b.store.LockModule(path) + // Save cache only on cache miss. A cache hit means the entry is + // already present and current; OnTest does not modify metadata. + if !cacheHit { + entry, err := b.cache.Put(ctx, cache.Key{Module: modVer, Matrix: b.matrix}, os.DirFS(installDir), cache.Entry{ + Metadata: metadata, + Deps: deps, + }) if err != nil { - for i := len(unlocks) - 1; i >= 0; i-- { - unlocks[i]() - } - return nil, err + return Result{}, err } - unlocks = append(unlocks, unlock) + metadata = entry.Metadata } - defer func() { - for i := len(unlocks) - 1; i >= 0; i-- { - unlocks[i]() - } - }() - - // Save current environment and restore it after OnBuild, - // that's because OnBuild may break environment - // TODO(MeteorsLiu): Switch to sandbox to run OnBuild - savedEnv := os.Environ() - defer func() { - os.Clearenv() - for _, env := range savedEnv { - k, v, _ := strings.Cut(env, "=") - os.Setenv(k, v) - } - }() - // TODO(MeteorsLiu): Parallel build - for _, target := range buildList { - result, err := buildModule(target) - if err != nil { - return nil, err - } + return Result{Metadata: metadata, OutputDir: installDir}, nil + } - // Track result for downstream dependencies - modVer := module.Version{Path: target.Path, Version: target.Version} - br := classfile.BuildResult{} - if result.Metadata != "" { - br.SetMetadata(result.Metadata) - } - builtResults[modVer] = br + var results []Result - results = append(results, result) + buildList := b.constructBuildList(targets) + lockPaths := make([]string, 0, len(buildList)) + for _, target := range buildList { + lockPaths = append(lockPaths, target.Path) + } + // A dependent keeps reading its dependencies' install directories after + // their own build steps return, so hold the whole graph until Build completes. + // + // Case 1 - Disjoint graphs: + // Request A builds libpng -> zlib and request B builds curl -> openssl. + // They lock different module paths and remain parallel. + // + // Case 2 - Overlapping graphs: + // Request A builds libpng -> zlib and request B builds freetype -> zlib. + // Because both graphs contain zlib, the later request waits for the earlier + // Build to finish, then reuses its published zlib artifact instead of observing + // a replaced install tree. + // + // Lock ordering: + // Use a stable order so overlapping graphs cannot deadlock. For example, + // X -> Y produces build order [Y, X], while another matrix with Y -> X produces + // [X, Y]. Locking in build order can leave each request holding one lock and + // waiting for the other; sorting makes both lock [X, Y]. + sort.Strings(lockPaths) + unlocks := make([]func(), 0, len(lockPaths)) + for _, path := range lockPaths { + unlock, err := b.store.LockModule(path) + if err != nil { + for i := len(unlocks) - 1; i >= 0; i-- { + unlocks[i]() + } + return nil, err } - return results, nil + unlocks = append(unlocks, unlock) } + defer func() { + for i := len(unlocks) - 1; i >= 0; i-- { + unlocks[i]() + } + }() - buildTarget := b.target - var targetOS, targetArch string - if values := b.matrix.Require["os"]; len(values) > 0 { - targetOS = values[0] - } - if values := b.matrix.Require["arch"]; len(values) > 0 { - targetArch = values[0] - } - // A caller-supplied target owns all target and sysroot preparation. Without - // one, cross builds use LLAR's default C target policy. - if len(targets) > 0 && buildTarget == nil && (targetOS != runtime.GOOS || targetArch != runtime.GOARCH) { - cSysroot, useCTarget := c.Sysroot(targetOS, targetArch) - if useCTarget { - var sysrootMods []*modules.Module - _, customLibc := b.matrix.Require["libc"] - if !customLibc && targets[0].Path != cSysroot.Path { - var err error - // modules.Load selects the sysroot Formula whose fromVer applies to - // cSysroot.Version; the sysroot graph remains separate from targets. - sysrootMods, err = modules.Load(ctx, cSysroot, modules.Options{ - FormulaStore: b.store, - Matrix: b.matrix, - }) - if err != nil { - return nil, fmt.Errorf("failed to load sysroot %s@%s: %w", cSysroot.Path, cSysroot.Version, err) - } - } + // Save current environment and restore it after OnBuild, + // that's because OnBuild may break environment + // TODO(MeteorsLiu): Switch to sandbox to run OnBuild + savedEnv := os.Environ() + defer func() { + os.Clearenv() + for _, env := range savedEnv { + k, v, _ := strings.Cut(env, "=") + os.Setenv(k, v) + } + }() - targetMatrix := targetArch + "-" + targetOS - bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) - if err != nil { - return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - bootstrapTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: bootstrapToolchain.Toolchain, - }) - if err != nil { - return nil, err - } - defer bootstrapTarget.Close() - buildTarget = bootstrapTarget - - if len(sysrootMods) > 0 { - selectedSysroot := sysrootMods[0] - sysrootResults, err := build(sysrootMods, bootstrapTarget, false) - if err != nil { - return nil, fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) - if err != nil { - return nil, fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - if metadata.Sysroot() == "" { - return nil, fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) - } + // TODO(MeteorsLiu): Parallel build + for _, target := range buildList { + result, err := build(target) + if err != nil { + return nil, err + } - configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) - if err != nil { - return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - configuredTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: configuredToolchain.Toolchain, - Sysroot: metadata.Sysroot(), - }) - if err != nil { - return nil, err - } - defer configuredTarget.Close() - buildTarget = configuredTarget - } + // Track result for downstream dependencies + modVer := module.Version{Path: target.Path, Version: target.Version} + br := classfile.BuildResult{} + if result.Metadata != "" { + br.SetMetadata(result.Metadata) } - } + builtResults[modVer] = br - return build(targets, buildTarget, b.runTest) + results = append(results, result) + } + return results, nil } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 212a00a..58931b3 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -41,10 +41,6 @@ func paths(mods []*modules.Module) string { return strings.Join(s, " ") } -func testMatrix(value string) classfile.Matrix { - return classfile.Matrix{Require: map[string][]string{"matrix": {value}}} -} - // versions returns the "Path@Version" strings for []module.Version. func versions(vers []module.Version) string { var s []string @@ -312,7 +308,7 @@ func setupBuilder(t *testing.T, store repo.Store, matrix string) *Builder { workspaceDir := t.TempDir() return &Builder{ store: store, - matrix: testMatrix(matrix), + matrix: matrix, workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -516,7 +512,7 @@ func TestNewBuilder(t *testing.T) { store := setupTestStore(t) b, err := NewBuilder(Options{ Store: store, - Matrix: classfile.Matrix{Require: map[string][]string{"matrix": {"amd64-linux"}}}, + MatrixStr: "amd64-linux", WorkspaceDir: tmpDir, }) if err != nil { @@ -525,8 +521,8 @@ func TestNewBuilder(t *testing.T) { if b.workspaceDir != tmpDir { t.Errorf("workspaceDir = %q, want %q", b.workspaceDir, tmpDir) } - if got := b.matrix.Combinations()[0]; got != "amd64-linux" { - t.Errorf("matrix = %q, want %q", got, "amd64-linux") + if b.matrix != "amd64-linux" { + t.Errorf("matrix = %q, want %q", b.matrix, "amd64-linux") } if b.store != store { t.Error("store not set correctly") @@ -538,7 +534,7 @@ func TestNewBuilder(t *testing.T) { t.Run("default workspace dir", func(t *testing.T) { b, err := NewBuilder(Options{ - Matrix: classfile.Matrix{Require: map[string][]string{"matrix": {"arm64-darwin"}}}, + MatrixStr: "arm64-darwin", }) if err != nil { t.Fatalf("NewBuilder() error = %v", err) @@ -645,7 +641,7 @@ func TestBuild_LocksEntireDependencyGraph(t *testing.T) { buildCache := &graphLockCache{store: store, paths: paths} b := &Builder{ store: store, - matrix: testMatrix("amd64-linux"), + matrix: "amd64-linux", workspaceDir: t.TempDir(), cache: buildCache, } @@ -677,7 +673,7 @@ func TestBuild_ReleasesGraphLocksAfterLockError(t *testing.T) { buildCache := &graphLockCache{store: store, paths: []string{"a/dep", "m/root"}} b := &Builder{ store: store, - matrix: testMatrix("amd64-linux"), + matrix: "amd64-linux", workspaceDir: t.TempDir(), cache: buildCache, } @@ -712,7 +708,7 @@ func TestBuild_OppositeGraphOrdersDoNotDeadlock(t *testing.T) { newBuilder := func(id int) *Builder { return &Builder{ store: &oppositeGraphStore{id: id, locks: locks}, - matrix: testMatrix("amd64-linux"), + matrix: "amd64-linux", workspaceDir: t.TempDir(), cache: &recordingCache{hits: map[module.Version]cache.Entry{ {Path: "test/x", Version: "1.0.0"}: {Metadata: "x"}, diff --git a/internal/build/c/target.go b/internal/build/c/target.go index 9b3fd55..2da0a7b 100644 --- a/internal/build/c/target.go +++ b/internal/build/c/target.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" "github.com/kballard/go-shellquote" ) @@ -18,22 +19,6 @@ const ( darwinSysrootVersion = "14.5" ) -// Command describes a command before C target defaults are applied. -type Command struct { - Name string - Args []string - Env []string - Dir string -} - -// Patch contains C target changes for one command. -type Patch struct { - Name string - PrependArg []string - AppendArg []string - Env []string -} - // Config contains the facts required to prepare a C target. type Config struct { Matrix string @@ -121,13 +106,13 @@ func (c *Target) Close() error { // Use returns C target defaults for cmd. Explicit Formula settings are // preserved. -func (c *Target) Use(cmd Command) Patch { +func (c *Target) Use(cmd build.Command) build.Patch { base := filepath.Base(cmd.Name) if base == "configure" { return c.autotoolsPatch(cmd) } if filepath.Base(cmd.Name) != cmd.Name { - return Patch{} + return build.Patch{} } switch base { @@ -146,7 +131,7 @@ func (c *Target) Use(cmd Command) Patch { c.tempDir = tempDir c.toolchainFile = toolchainFile } - return Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} + return build.Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} } case "pkg-config": return c.pkgConfigPatch(cmd.Env) @@ -157,22 +142,22 @@ func (c *Target) Use(cmd Command) Patch { case "ld", "ld.lld", "ld64.lld": return commandPatch(c.toolchain.Linker()) case "ar", "llvm-ar": - return Patch{Name: c.toolchain.Archiver()} + return build.Patch{Name: c.toolchain.Archiver()} case "ranlib", "llvm-ranlib": - return Patch{Name: c.toolchain.Ranlib()} + return build.Patch{Name: c.toolchain.Ranlib()} case "nm", "llvm-nm": - return Patch{Name: c.toolchain.NM()} + return build.Patch{Name: c.toolchain.NM()} case "strip", "llvm-strip": - return Patch{Name: c.toolchain.Strip()} + return build.Patch{Name: c.toolchain.Strip()} } - return Patch{} + return build.Patch{} } -func commandPatch(command []string) Patch { - return Patch{Name: command[0], PrependArg: command[1:]} +func commandPatch(command []string) build.Patch { + return build.Patch{Name: command[0], PrependArg: command[1:]} } -func (c *Target) autotoolsPatch(cmd Command) Patch { +func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env := append([]string(nil), cmd.Env...) env = setMissingEnv(env, "CC", shellquote.Join(c.toolchain.CC()...)) env = setMissingEnv(env, "CXX", shellquote.Join(c.toolchain.CXX()...)) @@ -184,7 +169,7 @@ func (c *Target) autotoolsPatch(cmd Command) Patch { for _, arg := range cmd.Args { if arg == "--host" || strings.HasPrefix(arg, "--host=") { - return Patch{Env: env} + return build.Patch{Env: env} } } @@ -198,16 +183,16 @@ func (c *Target) autotoolsPatch(cmd Command) Patch { } // Only scripts that declare --host receive the Autoconf target tuple. // For example, zlib's custom configure declares CHOST but rejects --host. - patch := Patch{Env: env} + patch := build.Patch{Env: env} if bytes.Contains(data, []byte("--host")) { patch.AppendArg = []string{"--host=" + c.autotoolsHost} } return patch } -func (c *Target) pkgConfigPatch(commandEnv []string) Patch { +func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { if c.sysroot == "" { - return Patch{} + return build.Patch{} } env := append([]string(nil), commandEnv...) env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) @@ -219,7 +204,7 @@ func (c *Target) pkgConfigPatch(commandEnv []string) Patch { filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), ) env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) - return Patch{Env: env} + return build.Patch{Env: env} } func (c *Target) cmakeToolchain() string { diff --git a/internal/build/c/target_test.go b/internal/build/c/target_test.go index 4c8da15..d0ba054 100644 --- a/internal/build/c/target_test.go +++ b/internal/build/c/target_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/goplus/llar/internal/build" "github.com/goplus/llar/mod/module" ) @@ -58,17 +59,17 @@ func TestDarwinTarget(t *testing.T) { } t.Cleanup(func() { _ = target.Close() }) - patch := target.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := target.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) want := []string{"--target=x86_64-apple-macos10.13", "-fuse-ld=lld", "-isysroot/sdk"} if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = target.Use(Command{Name: "cc", Args: []string{"a.o", "-shared"}}) + patch = target.Use(build.Command{Name: "cc", Args: []string{"a.o", "-shared"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) } - patch = target.Use(Command{ + patch = target.Use(build.Command{ Name: "cc", Args: []string{"--target=custom", "-isysroot/custom", "-fuse-ld=custom"}, }) @@ -81,7 +82,7 @@ func TestDarwinTarget(t *testing.T) { if err := os.WriteFile(configure, []byte("#!/bin/sh\nCHOST=${CHOST-}\n"), 0o755); err != nil { t.Fatal(err) } - patch = target.Use(Command{Name: configure}) + patch = target.Use(build.Command{Name: configure}) if len(patch.AppendArg) != 0 { t.Fatalf("configure AppendArg = %q, want no unsupported arguments", patch.AppendArg) } @@ -97,7 +98,7 @@ func TestDarwinTarget(t *testing.T) { } } - patch = target.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch = target.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) if err != nil { t.Fatal(err) @@ -126,7 +127,7 @@ func TestDarwinArm64Compiler(t *testing.T) { } t.Cleanup(func() { _ = target.Close() }) - patch := target.Use(Command{Name: "cc"}) + patch := target.Use(build.Command{Name: "cc"}) if !slices.Contains(patch.PrependArg, "--target=arm64-apple-macos11.0") { t.Fatalf("PrependArg = %q, want prepared arm64 compiler target", patch.PrependArg) } @@ -139,15 +140,15 @@ func TestBootstrapTargetOmitsSysroot(t *testing.T) { } t.Cleanup(func() { _ = c.Close() }) - patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) if got, want := patch.PrependArg, []string{"--target=aarch64-linux-gnu", "-fuse-ld=lld"}; !reflect.DeepEqual(got, want) { t.Fatalf("PrependArg = %q, want %q", got, want) } - if patch := c.Use(Command{Name: "pkg-config"}); patch.Env != nil { + if patch := c.Use(build.Command{Name: "pkg-config"}); patch.Env != nil { t.Fatalf("pkg-config Patch = %+v, want no sysroot environment", patch) } - patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) data, err := os.ReadFile(strings.TrimPrefix(patch.AppendArg[0], "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=")) if err != nil { t.Fatal(err) @@ -163,7 +164,7 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { t.Fatalf("New created CMake files: toolchainFile=%q tempDir=%q", c.toolchainFile, c.tempDir) } - c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) path := c.toolchainFile data, err := os.ReadFile(path) if err != nil { @@ -185,15 +186,15 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { func TestUseCMake(t *testing.T) { c := newTestTarget(t) - patch := c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch := c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } - patch = c.Use(Command{Name: "cmake", Args: []string{"--build", "build"}}) + patch = c.Use(build.Command{Name: "cmake", Args: []string{"--build", "build"}}) if len(patch.AppendArg) != 0 { t.Fatalf("build Patch = %+v, want no toolchain argument", patch) } - patch = c.Use(Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) + patch = c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "--toolchain", "/custom.cmake"}}) if len(patch.AppendArg) != 0 { t.Fatalf("explicit toolchain Patch = %+v", patch) } @@ -201,7 +202,7 @@ func TestUseCMake(t *testing.T) { func TestUseDirectCommands(t *testing.T) { c := newTestTarget(t) - patch := c.Use(Command{Name: "cc", Args: []string{"-c", "a.c"}}) + patch := c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c"}}) if patch.Name != "/llvm/bin/clang" { t.Fatalf("Name = %q", patch.Name) } @@ -209,18 +210,18 @@ func TestUseDirectCommands(t *testing.T) { if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(Command{Name: "cc", Args: []string{"a.o", "-o", "a"}}) + patch = c.Use(build.Command{Name: "cc", Args: []string{"a.o", "-o", "a"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("link PrependArg = %q, want %q", patch.PrependArg, want) } - patch = c.Use(Command{Name: "cc", Args: []string{"-c", "a.c", "--target=custom", "--sysroot=/custom"}}) + patch = c.Use(build.Command{Name: "cc", Args: []string{"-c", "a.c", "--target=custom", "--sysroot=/custom"}}) if !reflect.DeepEqual(patch.PrependArg, want) { t.Fatalf("PrependArg = %q, want prepared defaults %q", patch.PrependArg, want) } - if patch := c.Use(Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { + if patch := c.Use(build.Command{Name: filepath.Join("custom", "cc")}); patch.Name != "" { t.Fatalf("explicit compiler path was rewritten: %+v", patch) } - patch = c.Use(Command{Name: "ld"}) + patch = c.Use(build.Command{Name: "ld"}) if patch.Name != "/llvm/bin/ld.lld" { t.Fatalf("linker Name = %q, want /llvm/bin/ld.lld", patch.Name) } @@ -232,7 +233,7 @@ func TestUseAutotools(t *testing.T) { if err := os.WriteFile(configure, []byte("#!/bin/sh\n# options: --build=BUILD --host=HOST\n"), 0o755); err != nil { t.Fatal(err) } - patch := c.Use(Command{ + patch := c.Use(build.Command{ Name: configure, Args: []string{"--build=x86_64-apple-darwin"}, Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, @@ -250,7 +251,7 @@ func TestUseAutotools(t *testing.T) { t.Fatalf("AppendArg = %q, want %q", got, want) } - patch = c.Use(Command{Name: configure, Args: []string{"--host=custom-linux"}}) + patch = c.Use(build.Command{Name: configure, Args: []string{"--host=custom-linux"}}) if len(patch.AppendArg) != 0 { t.Fatalf("explicit host AppendArg = %q, want no duplicate host", patch.AppendArg) } @@ -259,7 +260,7 @@ func TestUseAutotools(t *testing.T) { func TestUsePkgConfig(t *testing.T) { c := newTestTarget(t) depPaths := strings.Join([]string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig"}, string(os.PathListSeparator)) - patch := c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) + patch := c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) } @@ -269,7 +270,7 @@ func TestUsePkgConfig(t *testing.T) { t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) } } - patch = c.Use(Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) + patch = c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/custom" { t.Fatalf("PKG_CONFIG_LIBDIR override = %q, want /custom", got) } diff --git a/internal/build/cache.go b/internal/build/cache.go index 487966b..9a47a14 100644 --- a/internal/build/cache.go +++ b/internal/build/cache.go @@ -80,7 +80,7 @@ func (b *Builder) installDir(modPath, version string) (string, error) { if err != nil { return "", err } - return filepath.Join(b.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, version, b.matrix.Combinations()[0])), nil + return filepath.Join(b.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, version, b.matrix)), nil } // loadCache reads the cache file for a module from the workspace directory. diff --git a/internal/build/cache_test.go b/internal/build/cache_test.go index 980c133..d6de8e7 100644 --- a/internal/build/cache_test.go +++ b/internal/build/cache_test.go @@ -97,7 +97,7 @@ func TestBuildCache_Overwrite(t *testing.T) { } func TestBuilder_InstallDir(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} dir, err := b.installDir("madler/zlib", "1.0.0") if err != nil { @@ -110,7 +110,7 @@ func TestBuilder_InstallDir(t *testing.T) { } func TestBuilder_CacheDir(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} dir, err := b.cacheDir("madler/zlib") if err != nil { @@ -124,7 +124,7 @@ func TestBuilder_CacheDir(t *testing.T) { func TestBuilder_SaveLoadCache(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} now := time.Now().Truncate(time.Second) original := &buildCache{} @@ -188,7 +188,7 @@ func TestBuilder_SaveLoadCache(t *testing.T) { // --------------------------------------------------------------------------- func TestBuilder_CacheDir_InvalidPath(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} // Empty path should fail EscapePath (filepath.Localize) _, err := b.cacheDir("") @@ -210,7 +210,7 @@ func TestBuilder_CacheDir_InvalidPath(t *testing.T) { } func TestBuilder_InstallDir_InvalidPath(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} _, err := b.installDir("", "1.0.0") if err == nil { @@ -225,7 +225,7 @@ func TestBuilder_InstallDir_InvalidPath(t *testing.T) { func TestBuilder_LoadCache_InvalidPath(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} _, err := b.loadCache("") if err == nil { @@ -235,7 +235,7 @@ func TestBuilder_LoadCache_InvalidPath(t *testing.T) { func TestBuilder_SaveCache_InvalidPath(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} cache := &buildCache{} cache.set("1.0.0", "amd64-linux", &buildEntry{BuildTime: time.Now()}) @@ -247,8 +247,8 @@ func TestBuilder_SaveCache_InvalidPath(t *testing.T) { } func TestBuilder_InstallDir_DifferentMatrices(t *testing.T) { - b1 := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} - b2 := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("arm64-darwin")} + b1 := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} + b2 := &Builder{workspaceDir: "/tmp/ws", matrix: "arm64-darwin"} dir1, _ := b1.installDir("test/lib", "1.0.0") dir2, _ := b2.installDir("test/lib", "1.0.0") @@ -265,7 +265,7 @@ func TestBuilder_InstallDir_DifferentMatrices(t *testing.T) { } func TestBuilder_InstallDir_DifferentVersions(t *testing.T) { - b := &Builder{workspaceDir: "/tmp/ws", matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: "/tmp/ws", matrix: "amd64-linux"} dir1, _ := b.installDir("test/lib", "1.0.0") dir2, _ := b.installDir("test/lib", "2.0.0") @@ -277,7 +277,7 @@ func TestBuilder_InstallDir_DifferentVersions(t *testing.T) { func TestBuilder_SaveCache_CreatesDir(t *testing.T) { tmpDir := t.TempDir() - b := &Builder{workspaceDir: tmpDir, matrix: testMatrix("amd64-linux")} + b := &Builder{workspaceDir: tmpDir, matrix: "amd64-linux"} cache := &buildCache{} cache.set("1.0.0", "amd64-linux", &buildEntry{ diff --git a/internal/build/e2e_test.go b/internal/build/e2e_test.go index 4393c99..6968d0d 100644 --- a/internal/build/e2e_test.go +++ b/internal/build/e2e_test.go @@ -128,7 +128,7 @@ func TestE2E_MatrixVariation(t *testing.T) { // Verify each matrix has its own install directory for _, matrix := range matrices { - b := &Builder{workspaceDir: wsDir, matrix: testMatrix(matrix)} + b := &Builder{workspaceDir: wsDir, matrix: matrix} dir, _ := b.installDir("test/ctxcheck", "1.0.0") if _, err := os.Stat(dir); err != nil { t.Errorf("installDir not created for matrix %q: %v", matrix, err) @@ -305,7 +305,7 @@ func TestE2E_RealZlibBuild(t *testing.T) { b := &Builder{ store: store, - matrix: testMatrix(matrix), + matrix: matrix, workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -379,7 +379,7 @@ func TestE2E_RealLibpngBuild(t *testing.T) { b := &Builder{ store: store, - matrix: testMatrix(matrix), + matrix: matrix, workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { @@ -477,7 +477,7 @@ func TestE2E_RealFreetypeBuild(t *testing.T) { b := &Builder{ store: store, - matrix: testMatrix(matrix), + matrix: matrix, workspaceDir: workspaceDir, cache: &localCache{workspaceDir: workspaceDir}, newRepo: func(repoPath string) (vcs.Repo, error) { diff --git a/internal/build/http/http.go b/internal/build/http/http.go index 886f2bb..b3d54cd 100644 --- a/internal/build/http/http.go +++ b/internal/build/http/http.go @@ -130,7 +130,7 @@ func (h *handler) build(ctx context.Context, req request, info io.Writer) (resul builder, err := build.NewBuilder(build.Options{ Store: h.formulaStore, - Matrix: req.matrix, + MatrixStr: req.matrixStr, Stdout: info, Stderr: info, WorkspaceDir: h.workspaceDir, diff --git a/internal/build/target.go b/internal/build/target.go index 2510caa..8e3182f 100644 --- a/internal/build/target.go +++ b/internal/build/target.go @@ -3,17 +3,37 @@ package build import ( "os" - "github.com/goplus/llar/internal/build/c" "github.com/goplus/llar/internal/execbroker" ) -func targetMiddleware(target *c.Target) execbroker.Middleware { +// Command describes a command before target defaults are applied. +type Command struct { + Name string + Args []string + Env []string + Dir string +} + +// Patch contains target-specific changes for one command. +type Patch struct { + Name string + PrependArg []string + AppendArg []string + Env []string +} + +// Target applies language-specific target defaults to build commands. +type Target interface { + Use(Command) Patch +} + +func targetMiddleware(target Target) execbroker.Middleware { return func(req execbroker.Request) execbroker.Request { env := req.Env if env == nil { env = os.Environ() } - patch := target.Use(c.Command{ + patch := target.Use(Command{ Name: req.Name, Args: req.Args, Env: env, diff --git a/internal/build/target_test.go b/internal/build/target_test.go index 5a7a952..1591704 100644 --- a/internal/build/target_test.go +++ b/internal/build/target_test.go @@ -2,48 +2,32 @@ package build import ( "context" - "path/filepath" "reflect" - "runtime" "testing" "testing/fstest" classfile "github.com/goplus/llar/formula" - "github.com/goplus/llar/internal/build/c" "github.com/goplus/llar/internal/execbroker" internalformula "github.com/goplus/llar/internal/formula" "github.com/goplus/llar/internal/modules" - "github.com/goplus/llar/internal/vcs" ) -func newCTarget(t *testing.T, targetOS, targetArch string) *c.Target { - t.Helper() - triple := "x86_64-linux-gnu" - if targetArch == "arm64" { - triple = "aarch64-linux-gnu" - } - target, err := c.NewTarget(c.Config{ - Matrix: targetArch + "-" + targetOS, - Toolchain: c.NewToolchain( - []string{"/toolchain/cc", "--target=" + triple, "--sysroot=/sdk"}, - []string{"/toolchain/c++", "--target=" + triple, "--sysroot=/sdk"}, - []string{"/toolchain/ld.lld"}, - "/toolchain/ar", - "/toolchain/ranlib", - "/toolchain/nm", - "/toolchain/strip", - ), - Sysroot: "/sdk", - }) - if err != nil { - t.Fatal(err) +type testTarget struct { + command Command +} + +func (t *testTarget) Use(command Command) Patch { + t.command = command + return Patch{ + Name: "/toolchain/cc", + PrependArg: []string{"--target=aarch64-linux-gnu"}, + AppendArg: []string{"--sysroot=/sdk"}, + Env: []string{"CC=/toolchain/cc"}, } - t.Cleanup(func() { _ = target.Close() }) - return target } func TestTargetMiddleware(t *testing.T) { - target := newCTarget(t, "linux", "arm64") + target := new(testTarget) got := targetMiddleware(target)(execbroker.Request{ Name: "cc", Args: []string{"-c", "a.c"}, @@ -53,37 +37,25 @@ func TestTargetMiddleware(t *testing.T) { if got.Name != "/toolchain/cc" { t.Fatalf("Name = %q", got.Name) } - if want := []string{"--target=aarch64-linux-gnu", "--sysroot=/sdk", "-c", "a.c"}; !reflect.DeepEqual(got.Args, want) { + if want := []string{"--target=aarch64-linux-gnu", "-c", "a.c", "--sysroot=/sdk"}; !reflect.DeepEqual(got.Args, want) { t.Fatalf("Args = %q, want %q", got.Args, want) } - if want := []string{"CFLAGS=-O2"}; !reflect.DeepEqual(got.Env, want) { + if want := []string{"CC=/toolchain/cc"}; !reflect.DeepEqual(got.Env, want) { t.Fatalf("Env = %q, want %q", got.Env, want) } + if target.command.Name != "cc" || target.command.Dir != "/src" { + t.Fatalf("Command = %+v", target.command) + } + if want := []string{"CFLAGS=-O2"}; !reflect.DeepEqual(target.command.Env, want) { + t.Fatalf("Command.Env = %q, want %q", target.command.Env, want) + } } func TestBuildAppliesTargetToFormulaCommands(t *testing.T) { store := setupTestStore(t) - targetOS, targetArch := "linux", "amd64" - if runtime.GOOS == targetOS && runtime.GOARCH == targetArch { - targetArch = "arm64" - } - target := newCTarget(t, targetOS, targetArch) - builder, err := NewBuilder(Options{ - Store: store, - Matrix: classfile.Matrix{Require: map[string][]string{ - "os": {targetOS}, - "arch": {targetArch}, - }}, - WorkspaceDir: t.TempDir(), - Target: target, - }) - if err != nil { - t.Fatal(err) - } - builder.newRepo = func(string) (vcs.Repo, error) { - return newMockRepo(filepath.Join(testSourceDir, "test", "liba")), nil - } - t.Setenv("PATH", "") + builder := setupBuilder(t, store, "arm64-linux") + target := new(testTarget) + builder.target = target var commandName string root := &modules.Module{ @@ -100,4 +72,7 @@ func TestBuildAppliesTargetToFormulaCommands(t *testing.T) { if commandName != "/toolchain/cc" { t.Fatalf("command name = %q, want /toolchain/cc", commandName) } + if target.command.Name != "cc" { + t.Fatalf("target command = %+v", target.command) + } } diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index 7c5e4e5..8e8f762 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -541,7 +541,7 @@ func (s *suite) build(ctx context.Context, target module.Version, matrix, worksp } builder, err := build.NewBuilder(build.Options{ Store: s.formulas, - Matrix: targetMatrix, + MatrixStr: matrix, WorkspaceDir: workspaceDir, Cache: c, }) From f5128bf475a388140e7f22f4cb458f9de9c78917 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 11:50:44 +0800 Subject: [PATCH 25/39] refactor(crosscompile): share target loading --- cmd/llar/internal/make.go | 95 +++------------ internal/build/http/http.go | 24 +++- .../c/llvm/toolchain.go | 2 +- .../c/llvm/toolchain_test.go | 0 internal/{build => crosscompile}/c/target.go | 0 .../{build => crosscompile}/c/target_test.go | 0 .../{build => crosscompile}/c/toolchain.go | 0 .../c/toolchain_test.go | 0 internal/crosscompile/crosscompile.go | 112 ++++++++++++++++++ 9 files changed, 151 insertions(+), 82 deletions(-) rename internal/{build => crosscompile}/c/llvm/toolchain.go (97%) rename internal/{build => crosscompile}/c/llvm/toolchain_test.go (100%) rename internal/{build => crosscompile}/c/target.go (100%) rename internal/{build => crosscompile}/c/target_test.go (100%) rename internal/{build => crosscompile}/c/toolchain.go (100%) rename internal/{build => crosscompile}/c/toolchain_test.go (100%) create mode 100644 internal/crosscompile/crosscompile.go diff --git a/cmd/llar/internal/make.go b/cmd/llar/internal/make.go index d31ae3a..39771f0 100644 --- a/cmd/llar/internal/make.go +++ b/cmd/llar/internal/make.go @@ -12,14 +12,12 @@ import ( "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build" - "github.com/goplus/llar/internal/build/c" - "github.com/goplus/llar/internal/build/c/llvm" + "github.com/goplus/llar/internal/crosscompile" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/internal/modules/modlocal" "github.com/goplus/llar/internal/vcs" "github.com/goplus/llar/mod/module" - ccmetadata "github.com/goplus/llar/x/metadata/cc" "github.com/spf13/cobra" ) @@ -144,18 +142,6 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, if runTest && crossCompile { return fmt.Errorf("llar test cannot run %s/%s target on %s/%s host", targetOS, targetArch, runtime.GOOS, runtime.GOARCH) } - var targetRoot module.Version - var useCTarget bool - if crossCompile { - // TODO: Add other language target policies alongside this C case when - // they provide build.Target implementations. - cSysroot, ok := c.Sysroot(targetOS, targetArch) - useCTarget = ok - _, customLibc := matrix.Require["libc"] - if useCTarget && !customLibc && root.Path != cSysroot.Path { - targetRoot = cSysroot - } - } loadOpts := modules.Options{ FormulaStore: store, Matrix: matrix, @@ -164,15 +150,6 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, if err != nil { return fmt.Errorf("failed to load modules: %w", err) } - var sysrootMods []*modules.Module - if targetRoot != (module.Version{}) { - // The default sysroot has no dependencies, but modules.Load still owns - // selecting the Formula whose fromVer applies to targetRoot.Version. - sysrootMods, err = modules.Load(ctx, targetRoot, loadOpts) - if err != nil { - return fmt.Errorf("failed to load sysroot %s@%s: %w", targetRoot.Path, targetRoot.Version, err) - } - } var buildOutput io.Writer = io.Discard if makeVerbose { @@ -195,63 +172,23 @@ func buildModule(ctx context.Context, store repo.Store, modPath, version string, defer os.RemoveAll(tmpDir) buildOpts.WorkspaceDir = tmpDir } - var target build.Target - // TODO: Add other language build.Target preparation alongside this C case. - if useCTarget { - targetMatrix := targetArch + "-" + targetOS - bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) - if err != nil { - return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - bootstrapTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: bootstrapToolchain.Toolchain, - }) - if err != nil { - return err - } - defer bootstrapTarget.Close() - target = bootstrapTarget - - if targetRoot != (module.Version{}) { - selectedSysroot := sysrootMods[0] - - sysrootOpts := buildOpts - sysrootOpts.RunTest = false - sysrootOpts.Target = bootstrapTarget - sysrootBuilder, err := build.NewBuilder(sysrootOpts) - if err != nil { - return fmt.Errorf("failed to create sysroot builder: %w", err) - } - sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) - if err != nil { - return fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) - if err != nil { - return fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) - } - if metadata.Sysroot() == "" { - return fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) - } - - configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) - if err != nil { - return fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) - } - configuredTarget, err := c.NewTarget(c.Config{ - Matrix: targetMatrix, - Toolchain: configuredToolchain.Toolchain, - Sysroot: metadata.Sysroot(), - }) - if err != nil { - return err - } - defer configuredTarget.Close() - target = configuredTarget + target, err := crosscompile.Load(ctx, root, crosscompile.Config{ + Store: store, + Matrix: matrix, + Stdout: buildOpts.Stdout, + Stderr: buildOpts.Stderr, + WorkspaceDir: buildOpts.WorkspaceDir, + Cache: buildOpts.Cache, + }) + if err != nil { + return err + } + if target != nil { + if closer, ok := target.(io.Closer); ok { + defer closer.Close() } + buildOpts.Target = target } - buildOpts.Target = target builder, err := build.NewBuilder(buildOpts) if err != nil { return fmt.Errorf("failed to create builder: %w", err) diff --git a/internal/build/http/http.go b/internal/build/http/http.go index b3d54cd..ac9a395 100644 --- a/internal/build/http/http.go +++ b/internal/build/http/http.go @@ -19,6 +19,7 @@ import ( "github.com/goplus/llar/internal/artifact" "github.com/goplus/llar/internal/build" "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/internal/crosscompile" "github.com/goplus/llar/internal/formula/repo" "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/mod/module" @@ -120,7 +121,8 @@ func (h *handler) do(ctx context.Context, req request, info io.Writer) (result, } func (h *handler) build(ctx context.Context, req request, info io.Writer) (result, error) { - mods, err := modules.Load(ctx, module.Version{Path: req.module, Version: req.version}, modules.Options{ + root := module.Version{Path: req.module, Version: req.version} + mods, err := modules.Load(ctx, root, modules.Options{ FormulaStore: h.formulaStore, Matrix: req.matrix, }) @@ -128,17 +130,35 @@ func (h *handler) build(ctx context.Context, req request, info io.Writer) (resul return result{}, err } - builder, err := build.NewBuilder(build.Options{ + buildOpts := build.Options{ Store: h.formulaStore, MatrixStr: req.matrixStr, Stdout: info, Stderr: info, WorkspaceDir: h.workspaceDir, Cache: h.cache, + } + target, err := crosscompile.Load(ctx, root, crosscompile.Config{ + Store: h.formulaStore, + Matrix: req.matrix, + Stdout: info, + Stderr: info, + WorkspaceDir: h.workspaceDir, + Cache: h.cache, }) if err != nil { return result{}, err } + if target != nil { + if closer, ok := target.(io.Closer); ok { + defer closer.Close() + } + buildOpts.Target = target + } + builder, err := build.NewBuilder(buildOpts) + if err != nil { + return result{}, err + } if _, err := builder.Build(ctx, mods); err != nil { return result{}, err } diff --git a/internal/build/c/llvm/toolchain.go b/internal/crosscompile/c/llvm/toolchain.go similarity index 97% rename from internal/build/c/llvm/toolchain.go rename to internal/crosscompile/c/llvm/toolchain.go index 4814611..60dad1d 100644 --- a/internal/build/c/llvm/toolchain.go +++ b/internal/crosscompile/c/llvm/toolchain.go @@ -4,7 +4,7 @@ import ( "fmt" "os/exec" - "github.com/goplus/llar/internal/build/c" + "github.com/goplus/llar/internal/crosscompile/c" ) // Toolchain is a prepared LLVM C-family toolchain. diff --git a/internal/build/c/llvm/toolchain_test.go b/internal/crosscompile/c/llvm/toolchain_test.go similarity index 100% rename from internal/build/c/llvm/toolchain_test.go rename to internal/crosscompile/c/llvm/toolchain_test.go diff --git a/internal/build/c/target.go b/internal/crosscompile/c/target.go similarity index 100% rename from internal/build/c/target.go rename to internal/crosscompile/c/target.go diff --git a/internal/build/c/target_test.go b/internal/crosscompile/c/target_test.go similarity index 100% rename from internal/build/c/target_test.go rename to internal/crosscompile/c/target_test.go diff --git a/internal/build/c/toolchain.go b/internal/crosscompile/c/toolchain.go similarity index 100% rename from internal/build/c/toolchain.go rename to internal/crosscompile/c/toolchain.go diff --git a/internal/build/c/toolchain_test.go b/internal/crosscompile/c/toolchain_test.go similarity index 100% rename from internal/build/c/toolchain_test.go rename to internal/crosscompile/c/toolchain_test.go diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go new file mode 100644 index 0000000..6e193fe --- /dev/null +++ b/internal/crosscompile/crosscompile.go @@ -0,0 +1,112 @@ +package crosscompile + +import ( + "context" + "fmt" + "io" + "runtime" + + "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/internal/crosscompile/c" + "github.com/goplus/llar/internal/crosscompile/c/llvm" + "github.com/goplus/llar/internal/formula/repo" + "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/mod/module" + ccmetadata "github.com/goplus/llar/x/metadata/cc" +) + +// Config contains the build resources used to prepare a cross-compile target. +type Config struct { + Store repo.Store + Matrix formula.Matrix + Stdout io.Writer + Stderr io.Writer + WorkspaceDir string + Cache cache.Cache +} + +// Load returns the target used to cross-compile root. A nil target means the +// requested matrix is native or has no built-in C target policy. +func Load(ctx context.Context, root module.Version, config Config) (build.Target, error) { + var targetOS, targetArch string + if values := config.Matrix.Require["os"]; len(values) > 0 { + targetOS = values[0] + } + if values := config.Matrix.Require["arch"]; len(values) > 0 { + targetArch = values[0] + } + if targetOS == runtime.GOOS && targetArch == runtime.GOARCH { + return nil, nil + } + + // TODO: Add other language target policies alongside this C case when + // they provide build.Target implementations. + cSysroot, ok := c.Sysroot(targetOS, targetArch) + if !ok { + return nil, nil + } + targetMatrix := targetArch + "-" + targetOS + bootstrapToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch}) + if err != nil { + return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + bootstrapTarget, err := c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: bootstrapToolchain.Toolchain, + }) + if err != nil { + return nil, err + } + + _, customLibc := config.Matrix.Require["libc"] + if customLibc || root.Path == cSysroot.Path { + return bootstrapTarget, nil + } + defer bootstrapTarget.Close() + + // The default sysroot has no dependencies, but modules.Load still owns + // selecting the Formula whose fromVer applies to cSysroot.Version. + sysrootMods, err := modules.Load(ctx, cSysroot, modules.Options{ + FormulaStore: config.Store, + Matrix: config.Matrix, + }) + if err != nil { + return nil, fmt.Errorf("failed to load sysroot %s@%s: %w", cSysroot.Path, cSysroot.Version, err) + } + selectedSysroot := sysrootMods[0] + sysrootBuilder, err := build.NewBuilder(build.Options{ + Store: config.Store, + MatrixStr: config.Matrix.Combinations()[0], + Stdout: config.Stdout, + Stderr: config.Stderr, + WorkspaceDir: config.WorkspaceDir, + Cache: config.Cache, + Target: bootstrapTarget, + }) + if err != nil { + return nil, fmt.Errorf("failed to create sysroot builder: %w", err) + } + sysrootResults, err := sysrootBuilder.Build(ctx, sysrootMods) + if err != nil { + return nil, fmt.Errorf("failed to build sysroot %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + metadata, err := ccmetadata.Parse(sysrootResults[len(sysrootResults)-1].Metadata) + if err != nil { + return nil, fmt.Errorf("failed to parse sysroot metadata for %s@%s: %w", selectedSysroot.Path, selectedSysroot.Version, err) + } + if metadata.Sysroot() == "" { + return nil, fmt.Errorf("sysroot metadata for %s@%s has no sysroot", selectedSysroot.Path, selectedSysroot.Version) + } + + configuredToolchain, err := llvm.New(llvm.Config{OS: targetOS, Arch: targetArch, Sysroot: metadata.Sysroot()}) + if err != nil { + return nil, fmt.Errorf("prepare C toolchain for %s: %w", targetMatrix, err) + } + return c.NewTarget(c.Config{ + Matrix: targetMatrix, + Toolchain: configuredToolchain.Toolchain, + Sysroot: metadata.Sysroot(), + }) +} From e7a32e166e5d6046530612fc51bb5628a95c9b31 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 12:15:47 +0800 Subject: [PATCH 26/39] test(crosscompile): improve coverage --- internal/build/http/http_test.go | 61 +++- .../crosscompile/c/llvm/toolchain_test.go | 55 +++- internal/crosscompile/crosscompile_test.go | 262 ++++++++++++++++++ 3 files changed, 367 insertions(+), 11 deletions(-) create mode 100644 internal/crosscompile/crosscompile_test.go diff --git a/internal/build/http/http_test.go b/internal/build/http/http_test.go index 9dc3361..f00f02e 100644 --- a/internal/build/http/http_test.go +++ b/internal/build/http/http_test.go @@ -12,6 +12,7 @@ import ( "io/fs" "net/http" "net/http/httptest" + "os" "path/filepath" "reflect" "runtime" @@ -163,7 +164,8 @@ func TestServeHTTPSingleflight(t *testing.T) { Artifacts: artifacts, WorkspaceDir: t.TempDir(), }).(*handler) - const target = "/v1/artifacts/DaveGamble/cJSON@v1.7.18?os=linux&arch=amd64" + query := "arch=" + runtime.GOARCH + "&os=" + runtime.GOOS + target := "/v1/artifacts/DaveGamble/cJSON@v1.7.18?" + query var recorders [2]*httptest.ResponseRecorder var wg sync.WaitGroup @@ -182,7 +184,7 @@ func TestServeHTTPSingleflight(t *testing.T) { wg.Add(1) go serve(1) - key := "DaveGamble/cJSON@v1.7.18?arch=amd64&os=linux" + key := "DaveGamble/cJSON@v1.7.18?" + query waitFor(t, func() bool { stream, ok := h.infos.Load(key) if !ok { @@ -206,10 +208,11 @@ func TestServeHTTPSingleflight(t *testing.T) { t.Fatalf("artifact Get calls = %d, want 2", got) } + jsonQuery := strings.ReplaceAll(query, "&", `\u0026`) wantLines := []string{ - `info "resolving DaveGamble/cJSON@v1.7.18?arch=amd64\u0026os=linux"`, - `artifact {"id":"madler/zlib@v1.3.1?arch=amd64\u0026os=linux","type":"tar.gz","url":"https://artifacts.example/madler/zlib"}`, - `artifact {"id":"DaveGamble/cJSON@v1.7.18?arch=amd64\u0026os=linux","type":"tar.gz","url":"https://artifacts.example/DaveGamble/cJSON","deps":["madler/zlib@v1.3.1?arch=amd64\u0026os=linux"]}`, + `info "resolving DaveGamble/cJSON@v1.7.18?` + jsonQuery + `"`, + `artifact {"id":"madler/zlib@v1.3.1?` + jsonQuery + `","type":"tar.gz","url":"https://artifacts.example/madler/zlib"}`, + `artifact {"id":"DaveGamble/cJSON@v1.7.18?` + jsonQuery + `","type":"tar.gz","url":"https://artifacts.example/DaveGamble/cJSON","deps":["madler/zlib@v1.3.1?` + jsonQuery + `"]}`, } gotLines := strings.Split(strings.TrimSpace(recorders[0].Body.String()), "\n") if len(gotLines) != len(wantLines) { @@ -262,6 +265,54 @@ func TestServeHTTPBuildErrors(t *testing.T) { } } +func TestBuildCrossCompileTarget(t *testing.T) { + targetArch := "amd64" + if runtime.GOOS == "linux" && runtime.GOARCH == targetArch { + targetArch = "arm64" + } + target := "/v1/artifacts/madler/zlib@v1.3.1?arch=" + targetArch + "&libc=custom&os=linux" + req, err := parseRequest(httptest.NewRequest(http.MethodGet, target, nil)) + if err != nil { + t.Fatal(err) + } + + t.Run("success", func(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"clang", "clang++", "ld.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("tool"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir) + h := New(Options{ + FormulaStore: localFormulas(t), + Cache: &testCache{}, + Artifacts: &testArtifactStore{}, + WorkspaceDir: t.TempDir(), + }).(*handler) + result, err := h.build(context.Background(), req, io.Discard) + if err != nil { + t.Fatal(err) + } + if len(result.artifacts) != 1 || !strings.HasPrefix(result.artifacts[0].ID, "madler/zlib@v1.3.1?") { + t.Fatalf("build artifacts = %+v", result.artifacts) + } + }) + + t.Run("toolchain error", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + h := New(Options{ + FormulaStore: localFormulas(t), + Cache: &testCache{}, + WorkspaceDir: t.TempDir(), + }).(*handler) + _, err := h.build(context.Background(), req, io.Discard) + if err == nil || !strings.Contains(err.Error(), "prepare C toolchain") { + t.Fatalf("build error = %v, want toolchain error", err) + } + }) +} + func TestDoReturnsCanceledContext(t *testing.T) { store := &blockingFormulaStore{ started: make(chan struct{}), diff --git a/internal/crosscompile/c/llvm/toolchain_test.go b/internal/crosscompile/c/llvm/toolchain_test.go index cca60a5..6cf20e4 100644 --- a/internal/crosscompile/c/llvm/toolchain_test.go +++ b/internal/crosscompile/c/llvm/toolchain_test.go @@ -4,16 +4,12 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" ) func TestNewUsesPreparedPath(t *testing.T) { - dir := t.TempDir() - for _, name := range []string{"clang", "clang++", "ld.lld", "ld64.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { - if err := os.WriteFile(filepath.Join(dir, name), []byte("tool"), 0o755); err != nil { - t.Fatal(err) - } - } + dir := fakeToolDir(t) t.Setenv("PATH", dir) toolchain, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: "/sdk"}) @@ -37,4 +33,51 @@ func TestNewUsesPreparedPath(t *testing.T) { if got, want := toolchain.Linker(), []string{filepath.Join(dir, "ld64.lld")}; !reflect.DeepEqual(got, want) { t.Fatalf("Darwin Linker = %q, want %q", got, want) } + + toolchain, err = New(Config{OS: "linux", Arch: "amd64"}) + if err != nil { + t.Fatal(err) + } + if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=x86_64-linux-gnu", "-fuse-ld=lld"}; !reflect.DeepEqual(got, want) { + t.Fatalf("Linux amd64 CC = %q, want %q", got, want) + } + + toolchain, err = New(Config{OS: "darwin", Arch: "arm64"}) + if err != nil { + t.Fatal(err) + } + if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=arm64-apple-macos11.0", "-fuse-ld=lld"}; !reflect.DeepEqual(got, want) { + t.Fatalf("Darwin arm64 CC = %q, want %q", got, want) + } +} + +func TestNewErrors(t *testing.T) { + if _, err := New(Config{OS: "plan9", Arch: "amd64"}); err == nil || !strings.Contains(err.Error(), "unsupported LLVM target") { + t.Fatalf("New unsupported target error = %v", err) + } + + for _, missing := range []string{"clang", "clang++", "ld.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + t.Run(missing, func(t *testing.T) { + dir := fakeToolDir(t) + if err := os.Remove(filepath.Join(dir, missing)); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + _, err := New(Config{OS: "linux", Arch: "arm64"}) + if err == nil || !strings.Contains(err.Error(), `find prepared LLVM command "`+missing+`"`) { + t.Fatalf("New missing %s error = %v", missing, err) + } + }) + } +} + +func fakeToolDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"clang", "clang++", "ld.lld", "ld64.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("tool"), 0o755); err != nil { + t.Fatal(err) + } + } + return dir } diff --git a/internal/crosscompile/crosscompile_test.go b/internal/crosscompile/crosscompile_test.go new file mode 100644 index 0000000..44c1fd9 --- /dev/null +++ b/internal/crosscompile/crosscompile_test.go @@ -0,0 +1,262 @@ +package crosscompile + +import ( + "context" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/goplus/llar/formula" + "github.com/goplus/llar/internal/build" + "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/mod/module" +) + +func TestLoadWithoutCrossCompileTarget(t *testing.T) { + tests := []struct { + name string + matrix formula.Matrix + }{ + { + name: "native", + matrix: formula.Matrix{Require: map[string][]string{ + "os": {runtime.GOOS}, + "arch": {runtime.GOARCH}, + }}, + }, + { + name: "unsupported", + matrix: formula.Matrix{Require: map[string][]string{ + "os": {"unsupported"}, + "arch": {"unsupported"}, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + target, err := Load(context.Background(), module.Version{Path: "owner/repo", Version: "v1.0.0"}, Config{Matrix: tt.matrix}) + if err != nil { + t.Fatal(err) + } + if target != nil { + t.Fatalf("Load target = %T, want nil", target) + } + }) + } +} + +func TestLoadBootstrapTarget(t *testing.T) { + installFakeLLVM(t) + _, triple := linuxCrossMatrix() + tests := []struct { + name string + root module.Version + libc bool + }{ + { + name: "custom libc", + root: module.Version{Path: "owner/repo", Version: "v1.0.0"}, + libc: true, + }, + { + name: "sysroot formula", + root: module.Version{Path: "bminor/glibc", Version: "glibc-2.24"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selected, _ := linuxCrossMatrix() + if tt.libc { + selected.Require["libc"] = []string{"custom"} + } + target, err := Load(context.Background(), tt.root, Config{Matrix: selected}) + if err != nil { + t.Fatal(err) + } + if target == nil { + t.Fatal("Load target = nil") + } + if closer, ok := target.(io.Closer); ok { + t.Cleanup(func() { _ = closer.Close() }) + } + patch := target.Use(build.Command{Name: "cc"}) + if filepath.Base(patch.Name) != "clang" { + t.Fatalf("compiler = %q, want fake clang", patch.Name) + } + args := strings.Join(patch.PrependArg, " ") + if !strings.Contains(args, "--target="+triple) { + t.Fatalf("compiler args = %q, want target %q", args, triple) + } + if strings.Contains(args, "sysroot") { + t.Fatalf("bootstrap compiler args = %q, want no sysroot", args) + } + }) + } +} + +func TestLoadDefaultSysroot(t *testing.T) { + installFakeLLVM(t) + matrix, _ := linuxCrossMatrix() + target, err := Load(context.Background(), module.Version{Path: "owner/repo", Version: "v1.0.0"}, Config{ + Store: localSysrootFormulas(t), + Matrix: matrix, + Stdout: io.Discard, + Stderr: io.Discard, + WorkspaceDir: t.TempDir(), + Cache: metadataCache{metadata: "--sysroot=/target-sdk"}, + }) + if err != nil { + t.Fatal(err) + } + if target == nil { + t.Fatal("Load target = nil") + } + if closer, ok := target.(io.Closer); ok { + t.Cleanup(func() { _ = closer.Close() }) + } + patch := target.Use(build.Command{Name: "cc"}) + if args := strings.Join(patch.PrependArg, " "); !strings.Contains(args, "--sysroot=/target-sdk") { + t.Fatalf("compiler args = %q, want configured sysroot", args) + } +} + +func TestLoadErrors(t *testing.T) { + matrix, _ := linuxCrossMatrix() + root := module.Version{Path: "owner/repo", Version: "v1.0.0"} + cacheErr := errors.New("cache unavailable") + + t.Run("toolchain", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + selected, _ := linuxCrossMatrix() + selected.Require["libc"] = []string{"custom"} + _, err := Load(context.Background(), root, Config{Matrix: selected}) + if err == nil || !strings.Contains(err.Error(), "prepare C toolchain") { + t.Fatalf("Load error = %v, want toolchain error", err) + } + }) + + t.Run("formula", func(t *testing.T) { + installFakeLLVM(t) + formulaErr := errors.New("formula unavailable") + _, err := Load(context.Background(), root, Config{ + Store: formulaStore{err: formulaErr}, + Matrix: matrix, + }) + if !errors.Is(err, formulaErr) || !strings.Contains(err.Error(), "failed to load sysroot") { + t.Fatalf("Load error = %v, want formula error", err) + } + }) + + tests := []struct { + name string + cache cache.Cache + wantText string + wantErr error + }{ + { + name: "cache", + cache: metadataCache{err: cacheErr}, + wantText: "failed to build sysroot", + wantErr: cacheErr, + }, + { + name: "metadata", + cache: metadataCache{metadata: "--sysroot"}, + wantText: "failed to parse sysroot metadata", + }, + { + name: "missing sysroot", + cache: metadataCache{metadata: "-I/include"}, + wantText: "has no sysroot", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + installFakeLLVM(t) + _, err := Load(context.Background(), root, Config{ + Store: localSysrootFormulas(t), + Matrix: matrix, + Stdout: io.Discard, + Stderr: io.Discard, + WorkspaceDir: t.TempDir(), + Cache: tt.cache, + }) + if err == nil || !strings.Contains(err.Error(), tt.wantText) { + t.Fatalf("Load error = %v, want %q", err, tt.wantText) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("Load error = %v, want wrapped %v", err, tt.wantErr) + } + }) + } +} + +func linuxCrossMatrix() (formula.Matrix, string) { + arch := "amd64" + triple := "x86_64-linux-gnu" + if runtime.GOOS == "linux" && runtime.GOARCH == arch { + arch = "arm64" + triple = "aarch64-linux-gnu" + } + return formula.Matrix{Require: map[string][]string{ + "os": {"linux"}, + "arch": {arch}, + }}, triple +} + +func installFakeLLVM(t *testing.T) { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"clang", "clang++", "ld.lld", "llvm-ar", "llvm-ranlib", "llvm-nm", "llvm-strip"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir) +} + +func localSysrootFormulas(t *testing.T) formulaStore { + t.Helper() + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return formulaStore{root: filepath.Join(filepath.Dir(filename), "..", "..", "testdata", "crosscompile-e2e", "formulas")} +} + +type formulaStore struct { + root string + err error +} + +func (s formulaStore) ModuleFS(_ context.Context, modPath string) (fs.FS, error) { + if s.err != nil { + return nil, s.err + } + return os.DirFS(filepath.Join(s.root, filepath.FromSlash(modPath))), nil +} + +func (formulaStore) LockModule(string) (func(), error) { + return func() {}, nil +} + +type metadataCache struct { + metadata string + err error +} + +func (c metadataCache) Get(context.Context, cache.Key) (cache.Entry, bool, error) { + if c.err != nil { + return cache.Entry{}, false, c.err + } + return cache.Entry{Metadata: c.metadata}, true, nil +} + +func (metadataCache) Put(context.Context, cache.Key, fs.FS, cache.Entry) (cache.Entry, error) { + return cache.Entry{}, errors.New("unexpected cache Put") +} From 53d723f2755b7f554a9208e1b2262698801e51cb Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 12:16:05 +0800 Subject: [PATCH 27/39] ci(crosscompile): test formula-managed libc --- .github/workflows/crosscompile-e2e.yml | 61 +++++++++++++++++-- .../formulas/madler/zlib/v1.3.1/Zlib_llar.gox | 11 ++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 3b4be62..44f5dcf 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -13,7 +13,7 @@ concurrency: jobs: build-linux-arm64: - name: "Build zlib: Linux amd64 host to Linux arm64 target" + name: "Build zlib: Linux amd64 -> Linux arm64 (default/custom libc)" runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -78,7 +78,7 @@ jobs: git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" - - name: Cross compile zlib + - name: Cross compile zlib with default sysroot run: | go build -ldflags="-checklinkname=0" -o "$RUNNER_TEMP/llar" ./cmd/llar "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ @@ -87,15 +87,35 @@ jobs: --output "$RUNNER_TEMP/zlib-linux-arm64" \ --verbose - - name: Upload zlib artifact + - name: Cross compile zlib with Formula-managed libc + run: | + "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ + --os linux \ + --arch arm64 \ + --require libc=glibc-2.24 \ + --output "$RUNNER_TEMP/zlib-linux-arm64-custom-libc" \ + --json \ + --verbose \ + > "$RUNNER_TEMP/zlib-linux-arm64-custom-libc.json" + grep -F '"path":"bminor/glibc","version":"glibc-2.24"' \ + "$RUNNER_TEMP/zlib-linux-arm64-custom-libc.json" + + - name: Upload default-sysroot zlib artifact uses: actions/upload-artifact@v4 with: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 if-no-files-found: error + - name: Upload custom-libc zlib artifact + uses: actions/upload-artifact@v4 + with: + name: zlib-linux-arm64-custom-libc + path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc + if-no-files-found: error + run-linux-arm64: - name: "Run zlib: Linux arm64 runner consumes Linux arm64 artifact" + name: "Run zlib: Linux arm64 (default/custom libc)" needs: build-linux-arm64 runs-on: ubuntu-24.04-arm timeout-minutes: 10 @@ -103,13 +123,19 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Download zlib artifact + - name: Download default-sysroot zlib artifact uses: actions/download-artifact@v4 with: name: zlib-linux-arm64 path: ${{ runner.temp }}/zlib-linux-arm64 - - name: Inspect, link, and run consumer + - name: Download custom-libc zlib artifact + uses: actions/download-artifact@v4 + with: + name: zlib-linux-arm64-custom-libc + path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc + + - name: Inspect, link, and run default-sysroot consumer run: | lib="$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" member_dir="$RUNNER_TEMP/zlib-members" @@ -132,6 +158,29 @@ jobs: grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/dynamic-libraries.txt" "$consumer" + - name: Inspect, link, and run custom-libc consumer + run: | + lib="$RUNNER_TEMP/zlib-linux-arm64-custom-libc/lib/libz.a" + member_dir="$RUNNER_TEMP/zlib-custom-libc-members" + consumer="$RUNNER_TEMP/zlib-custom-libc-consumer" + + mkdir -p "$member_dir" + ( + cd "$member_dir" + ar x "$lib" adler32.o + ) + file "$member_dir/adler32.o" | tee "$RUNNER_TEMP/zlib-custom-libc-format.txt" + grep -F "ELF 64-bit LSB relocatable, ARM aarch64" "$RUNNER_TEMP/zlib-custom-libc-format.txt" + + cc \ + -I"$RUNNER_TEMP/zlib-linux-arm64-custom-libc/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$lib" \ + -o "$consumer" + ldd "$consumer" | tee "$RUNNER_TEMP/custom-libc-dynamic-libraries.txt" + grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/custom-libc-dynamic-libraries.txt" + "$consumer" + build-darwin-arm64: name: "Build zlib: Linux amd64 host to Darwin arm64 target" runs-on: ubuntu-24.04 diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox index 11a9297..ab637b6 100644 --- a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -2,10 +2,21 @@ id "madler/zlib" fromVer "v1.3.1" +onRequire (proj, deps) => { + if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "glibc-2.24" { + deps.require "bminor/glibc", libc[0] + } +} + onBuild ctx => { installDir := ctx.outputDir a := autotools.new(ctx.SourceDir, ctx.SourceDir+"/_build", installDir) + for _, dep := range ctx.Proj.Deps { + if dep.Path == "bminor/glibc" { + a.sysroot ctx.outputDir(dep) + } + } a.configure "--static" a.build From 48f39a4a593847b242796b66d5a84a7cb3a4f200 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 12:53:49 +0800 Subject: [PATCH 28/39] ci(crosscompile): distinguish custom libc sysroot --- .github/workflows/crosscompile-e2e.yml | 58 ++++++++++++++----- .../formulas/madler/zlib/v1.3.1/Zlib_llar.gox | 2 +- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 44f5dcf..eb343bc 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -31,23 +31,36 @@ jobs: sudo apt-get install --yes clang-18 lld-18 llvm-18 echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" - - name: Prepare Bootlin glibc 2.24 arm64 sysroot + - name: Prepare Bootlin glibc 2.24 and 2.27 arm64 sysroots run: | - archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" - toolchain_root="$RUNNER_TEMP/bootlin" + default_archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" + custom_archive="$RUNNER_TEMP/aarch64--glibc--stable-2018.11-1.tar.bz2" + default_toolchain_root="$RUNNER_TEMP/bootlin/glibc-2.24" + custom_toolchain_root="$RUNNER_TEMP/bootlin/glibc-2.27" + curl --fail --location --retry 3 \ "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" \ - --output "$archive" - echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $archive" | sha256sum --check - mkdir -p "$toolchain_root" - tar -xjf "$archive" -C "$toolchain_root" - - root="$toolchain_root/aarch64--glibc--stable" - sysroot="$root/aarch64-buildroot-linux-gnu/sysroot" - gcc_runtime="$root/lib/gcc/aarch64-buildroot-linux-gnu/5.4.0" + --output "$default_archive" + curl --fail --location --retry 3 \ + "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2018.11-1.tar.bz2" \ + --output "$custom_archive" + echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $default_archive" | sha256sum --check + echo "abae0522480b9f37ff6cee4249e147e7cb78e1997cc6f76dba7e0fb8ec04221d $custom_archive" | sha256sum --check + + mkdir -p "$default_toolchain_root" "$custom_toolchain_root" + tar -xjf "$default_archive" -C "$default_toolchain_root" + tar -xjf "$custom_archive" -C "$custom_toolchain_root" + + default_root="$default_toolchain_root/aarch64--glibc--stable" + default_sysroot="$default_root/aarch64-buildroot-linux-gnu/sysroot" + default_gcc_runtime="$default_root/lib/gcc/aarch64-buildroot-linux-gnu/5.4.0" + custom_root="$custom_toolchain_root/aarch64--glibc--stable-2018.11-1" + custom_sysroot="$custom_root/aarch64-buildroot-linux-gnu/sysroot" + custom_gcc_runtime="$custom_root/lib/gcc/aarch64-buildroot-linux-gnu/7.3.0" # Clang receives only the sysroot path, so install Bootlin's target # runtime, for example crtbeginS.o and libgcc.a, into its library path. - cp -a "$gcc_runtime/." "$sysroot/usr/lib/" + cp -a "$default_gcc_runtime/." "$default_sysroot/usr/lib/" + cp -a "$custom_gcc_runtime/." "$custom_sysroot/usr/lib/" - name: Prepare formula and source repositories run: | @@ -65,7 +78,7 @@ jobs: cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" tar -cJf "$source_repo/sysroot.tar.xz" \ - -C "$RUNNER_TEMP/bootlin/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" . + -C "$RUNNER_TEMP/bootlin/glibc-2.24/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" . chmod +x "$source_repo/install-sysroot" git -C "$source_repo" init --initial-branch=main git -C "$source_repo" config user.email crosscompile-e2e@example.com @@ -74,6 +87,12 @@ jobs: git -C "$source_repo" commit --quiet -m "Add sysroot installer" git -C "$source_repo" tag glibc-2.24 + tar -cJf "$source_repo/sysroot.tar.xz" \ + -C "$RUNNER_TEMP/bootlin/glibc-2.27/aarch64--glibc--stable-2018.11-1/aarch64-buildroot-linux-gnu/sysroot" . + git -C "$source_repo" add sysroot.tar.xz + git -C "$source_repo" commit --quiet -m "Update sysroot to glibc 2.27" + git -C "$source_repo" tag glibc-2.27 + git config --global protocol.file.allow always git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" @@ -89,16 +108,23 @@ jobs: - name: Cross compile zlib with Formula-managed libc run: | + custom_log="$RUNNER_TEMP/zlib-linux-arm64-custom-libc.log" "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ --os linux \ --arch arm64 \ - --require libc=glibc-2.24 \ + --require libc=glibc-2.27 \ --output "$RUNNER_TEMP/zlib-linux-arm64-custom-libc" \ --json \ --verbose \ - > "$RUNNER_TEMP/zlib-linux-arm64-custom-libc.json" - grep -F '"path":"bminor/glibc","version":"glibc-2.24"' \ + > "$RUNNER_TEMP/zlib-linux-arm64-custom-libc.json" \ + 2> >(tee "$custom_log" >&2) + grep -F '"path":"bminor/glibc","version":"glibc-2.27"' \ "$RUNNER_TEMP/zlib-linux-arm64-custom-libc.json" + grep -E -- '--sysroot=[^ ]*/bminor/glibc@glibc-2\.27-' "$custom_log" + if grep -F -- 'bminor/glibc@glibc-2.24-' "$custom_log"; then + echo "custom libc build used the default glibc 2.24 sysroot" >&2 + exit 1 + fi - name: Upload default-sysroot zlib artifact uses: actions/upload-artifact@v4 diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox index ab637b6..85c4bd3 100644 --- a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -3,7 +3,7 @@ id "madler/zlib" fromVer "v1.3.1" onRequire (proj, deps) => { - if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "glibc-2.24" { + if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "glibc-2.27" { deps.require "bminor/glibc", libc[0] } } From bd50f4e1c999c5da7c2d4a92546091ed69ea60ec Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 13:04:55 +0800 Subject: [PATCH 29/39] ci(crosscompile): install Bootlin sysroots directly --- .github/workflows/crosscompile-e2e.yml | 34 ++++------------ .../crosscompile-e2e/install-bootlin-sysroot | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 26 deletions(-) create mode 100755 testdata/crosscompile-e2e/install-bootlin-sysroot diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index eb343bc..8401132 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -31,12 +31,10 @@ jobs: sudo apt-get install --yes clang-18 lld-18 llvm-18 echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" - - name: Prepare Bootlin glibc 2.24 and 2.27 arm64 sysroots + - name: Download Bootlin glibc 2.24 and 2.27 arm64 toolchains run: | default_archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" custom_archive="$RUNNER_TEMP/aarch64--glibc--stable-2018.11-1.tar.bz2" - default_toolchain_root="$RUNNER_TEMP/bootlin/glibc-2.24" - custom_toolchain_root="$RUNNER_TEMP/bootlin/glibc-2.27" curl --fail --location --retry 3 \ "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" \ @@ -47,21 +45,6 @@ jobs: echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $default_archive" | sha256sum --check echo "abae0522480b9f37ff6cee4249e147e7cb78e1997cc6f76dba7e0fb8ec04221d $custom_archive" | sha256sum --check - mkdir -p "$default_toolchain_root" "$custom_toolchain_root" - tar -xjf "$default_archive" -C "$default_toolchain_root" - tar -xjf "$custom_archive" -C "$custom_toolchain_root" - - default_root="$default_toolchain_root/aarch64--glibc--stable" - default_sysroot="$default_root/aarch64-buildroot-linux-gnu/sysroot" - default_gcc_runtime="$default_root/lib/gcc/aarch64-buildroot-linux-gnu/5.4.0" - custom_root="$custom_toolchain_root/aarch64--glibc--stable-2018.11-1" - custom_sysroot="$custom_root/aarch64-buildroot-linux-gnu/sysroot" - custom_gcc_runtime="$custom_root/lib/gcc/aarch64-buildroot-linux-gnu/7.3.0" - # Clang receives only the sysroot path, so install Bootlin's target - # runtime, for example crtbeginS.o and libgcc.a, into its library path. - cp -a "$default_gcc_runtime/." "$default_sysroot/usr/lib/" - cp -a "$custom_gcc_runtime/." "$custom_sysroot/usr/lib/" - - name: Prepare formula and source repositories run: | formula_repo="$RUNNER_TEMP/llarhub" @@ -76,21 +59,20 @@ jobs: git -C "$formula_repo" add . git -C "$formula_repo" commit --quiet -m "Cross compile E2E formulas" - cp testdata/crosscompile-e2e/install-sysroot "$source_repo/" - tar -cJf "$source_repo/sysroot.tar.xz" \ - -C "$RUNNER_TEMP/bootlin/glibc-2.24/aarch64--glibc--stable/aarch64-buildroot-linux-gnu/sysroot" . + cp testdata/crosscompile-e2e/install-bootlin-sysroot "$source_repo/install-sysroot" + cp "$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" "$source_repo/" chmod +x "$source_repo/install-sysroot" git -C "$source_repo" init --initial-branch=main git -C "$source_repo" config user.email crosscompile-e2e@example.com git -C "$source_repo" config user.name "Cross Compile E2E" git -C "$source_repo" add . - git -C "$source_repo" commit --quiet -m "Add sysroot installer" + git -C "$source_repo" commit --quiet -m "Add glibc 2.24 toolchain" git -C "$source_repo" tag glibc-2.24 - tar -cJf "$source_repo/sysroot.tar.xz" \ - -C "$RUNNER_TEMP/bootlin/glibc-2.27/aarch64--glibc--stable-2018.11-1/aarch64-buildroot-linux-gnu/sysroot" . - git -C "$source_repo" add sysroot.tar.xz - git -C "$source_repo" commit --quiet -m "Update sysroot to glibc 2.27" + git -C "$source_repo" rm --quiet aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2 + cp "$RUNNER_TEMP/aarch64--glibc--stable-2018.11-1.tar.bz2" "$source_repo/" + git -C "$source_repo" add aarch64--glibc--stable-2018.11-1.tar.bz2 + git -C "$source_repo" commit --quiet -m "Update toolchain to glibc 2.27" git -C "$source_repo" tag glibc-2.27 git config --global protocol.file.allow always diff --git a/testdata/crosscompile-e2e/install-bootlin-sysroot b/testdata/crosscompile-e2e/install-bootlin-sysroot new file mode 100755 index 0000000..1de6dcb --- /dev/null +++ b/testdata/crosscompile-e2e/install-bootlin-sysroot @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu + +output_dir=$1 +source_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +set -- "$source_dir"/aarch64--glibc--stable-*.tar.bz2 +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + echo "expected one Bootlin glibc toolchain archive" >&2 + exit 1 +fi +archive=$1 + +case $(basename -- "$archive") in + aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2) + root=aarch64--glibc--stable + gcc_version=5.4.0 + ;; + aarch64--glibc--stable-2018.11-1.tar.bz2) + root=aarch64--glibc--stable-2018.11-1 + gcc_version=7.3.0 + ;; + *) + echo "unsupported Bootlin glibc toolchain archive: $(basename -- "$archive")" >&2 + exit 1 + ;; +esac + +mkdir -p "$output_dir" +LC_ALL=C tar -xjf "$archive" -C "$output_dir" --strip-components=3 \ + "$root/aarch64-buildroot-linux-gnu/sysroot" \ + "$root/lib/gcc/aarch64-buildroot-linux-gnu/$gcc_version" + +# Clang needs the target GCC runtime, for example crtbeginS.o and libgcc.a, +# beside the target libc when configure links compiler probes. +runtime_dir="$output_dir/aarch64-buildroot-linux-gnu/$gcc_version" +mkdir -p "$output_dir/usr/lib" +cp -a "$runtime_dir/." "$output_dir/usr/lib/" +rm -r "$output_dir/aarch64-buildroot-linux-gnu" From b529172652ee96ade2fd81c80b880fb0fe3b8b64 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 14:53:47 +0800 Subject: [PATCH 30/39] feat(crosscompile): select Linux environment from sysroot --- .github/workflows/crosscompile-e2e.yml | 85 +++++++++++++++- internal/crosscompile/c/llvm/toolchain.go | 48 ++++++++- .../crosscompile/c/llvm/toolchain_test.go | 51 +++++++++- internal/crosscompile/crosscompile.go | 97 ++++++++++++++++++- internal/crosscompile/crosscompile_test.go | 92 +++++++++++++++++- .../ifduyue/musl/v1.1.19/Musl_llar.gox | 8 ++ .../formulas/ifduyue/musl/versions.json | 4 + .../crosscompile-e2e/install-bootlin-sysroot | 21 ++-- .../formulas/madler/zlib/v1.3.1/Zlib_llar.gox | 11 ++- 9 files changed, 395 insertions(+), 22 deletions(-) create mode 100644 testdata/crosscompile-e2e/formulas/ifduyue/musl/v1.1.19/Musl_llar.gox create mode 100644 testdata/crosscompile-e2e/formulas/ifduyue/musl/versions.json diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 8401132..6b2ef2e 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -13,7 +13,7 @@ concurrency: jobs: build-linux-arm64: - name: "Build zlib: Linux amd64 -> Linux arm64 (default/custom libc)" + name: "Build zlib: Linux amd64 -> Linux arm64 (glibc/musl)" runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -31,10 +31,11 @@ jobs: sudo apt-get install --yes clang-18 lld-18 llvm-18 echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" - - name: Download Bootlin glibc 2.24 and 2.27 arm64 toolchains + - name: Download Bootlin glibc and musl arm64 toolchains run: | default_archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" custom_archive="$RUNNER_TEMP/aarch64--glibc--stable-2018.11-1.tar.bz2" + musl_archive="$RUNNER_TEMP/aarch64--musl--stable-2018.11-1.tar.bz2" curl --fail --location --retry 3 \ "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" \ @@ -42,15 +43,20 @@ jobs: curl --fail --location --retry 3 \ "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2018.11-1.tar.bz2" \ --output "$custom_archive" + curl --fail --location --retry 3 \ + "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--musl--stable-2018.11-1.tar.bz2" \ + --output "$musl_archive" echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $default_archive" | sha256sum --check echo "abae0522480b9f37ff6cee4249e147e7cb78e1997cc6f76dba7e0fb8ec04221d $custom_archive" | sha256sum --check + echo "ce2d60c4f33cdc1c09ab8e10316b76639da71c4af1ab1948f3547fc05b814796 $musl_archive" | sha256sum --check - name: Prepare formula and source repositories run: | formula_repo="$RUNNER_TEMP/llarhub" source_repo="$RUNNER_TEMP/glibc-source" + musl_source_repo="$RUNNER_TEMP/musl-source" - mkdir -p "$formula_repo" "$source_repo" + mkdir -p "$formula_repo" "$source_repo" "$musl_source_repo" cp -R testdata/kodo-e2e/formulas/. "$formula_repo/" cp -R testdata/crosscompile-e2e/formulas/. "$formula_repo/" git -C "$formula_repo" init --initial-branch=main @@ -75,9 +81,20 @@ jobs: git -C "$source_repo" commit --quiet -m "Update toolchain to glibc 2.27" git -C "$source_repo" tag glibc-2.27 + cp testdata/crosscompile-e2e/install-bootlin-sysroot "$musl_source_repo/install-sysroot" + cp "$RUNNER_TEMP/aarch64--musl--stable-2018.11-1.tar.bz2" "$musl_source_repo/" + chmod +x "$musl_source_repo/install-sysroot" + git -C "$musl_source_repo" init --initial-branch=main + git -C "$musl_source_repo" config user.email crosscompile-e2e@example.com + git -C "$musl_source_repo" config user.name "Cross Compile E2E" + git -C "$musl_source_repo" add . + git -C "$musl_source_repo" commit --quiet -m "Add musl 1.1.19 toolchain" + git -C "$musl_source_repo" tag v1.1.19 + git config --global protocol.file.allow always git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" + git config --global url."file://$musl_source_repo".insteadOf "https://github.com/ifduyue/musl.git" - name: Cross compile zlib with default sysroot run: | @@ -88,7 +105,7 @@ jobs: --output "$RUNNER_TEMP/zlib-linux-arm64" \ --verbose - - name: Cross compile zlib with Formula-managed libc + - name: Cross compile zlib with Formula-managed glibc run: | custom_log="$RUNNER_TEMP/zlib-linux-arm64-custom-libc.log" "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ @@ -108,6 +125,23 @@ jobs: exit 1 fi + - name: Cross compile zlib with Formula-managed musl + run: | + musl_log="$RUNNER_TEMP/zlib-linux-arm64-musl.log" + "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ + --os linux \ + --arch arm64 \ + --require libc=musl-1.1.19 \ + --output "$RUNNER_TEMP/zlib-linux-arm64-musl" \ + --json \ + --verbose \ + > "$RUNNER_TEMP/zlib-linux-arm64-musl.json" \ + 2> >(tee "$musl_log" >&2) + grep -F '"path":"ifduyue/musl","version":"v1.1.19"' \ + "$RUNNER_TEMP/zlib-linux-arm64-musl.json" + grep -E -- '--sysroot=[^ ]*/ifduyue/musl@v1\.1\.19-' "$musl_log" + grep -F -- '--target=aarch64-linux-musl' "$musl_log" + - name: Upload default-sysroot zlib artifact uses: actions/upload-artifact@v4 with: @@ -122,8 +156,15 @@ jobs: path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc if-no-files-found: error + - name: Upload musl zlib artifact + uses: actions/upload-artifact@v4 + with: + name: zlib-linux-arm64-musl + path: ${{ runner.temp }}/zlib-linux-arm64-musl + if-no-files-found: error + run-linux-arm64: - name: "Run zlib: Linux arm64 (default/custom libc)" + name: "Run zlib: Linux arm64 (glibc/musl)" needs: build-linux-arm64 runs-on: ubuntu-24.04-arm timeout-minutes: 10 @@ -131,6 +172,11 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Install musl tools + run: | + sudo apt-get update + sudo apt-get install --yes musl-tools + - name: Download default-sysroot zlib artifact uses: actions/download-artifact@v4 with: @@ -143,6 +189,12 @@ jobs: name: zlib-linux-arm64-custom-libc path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc + - name: Download musl zlib artifact + uses: actions/download-artifact@v4 + with: + name: zlib-linux-arm64-musl + path: ${{ runner.temp }}/zlib-linux-arm64-musl + - name: Inspect, link, and run default-sysroot consumer run: | lib="$RUNNER_TEMP/zlib-linux-arm64/lib/libz.a" @@ -189,6 +241,29 @@ jobs: grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/custom-libc-dynamic-libraries.txt" "$consumer" + - name: Inspect, link, and run musl consumer + run: | + lib="$RUNNER_TEMP/zlib-linux-arm64-musl/lib/libz.a" + member_dir="$RUNNER_TEMP/zlib-musl-members" + consumer="$RUNNER_TEMP/zlib-musl-consumer" + + mkdir -p "$member_dir" + ( + cd "$member_dir" + ar x "$lib" adler32.o + ) + file "$member_dir/adler32.o" | tee "$RUNNER_TEMP/zlib-musl-format.txt" + grep -F "ELF 64-bit LSB relocatable, ARM aarch64" "$RUNNER_TEMP/zlib-musl-format.txt" + + musl-gcc \ + -I"$RUNNER_TEMP/zlib-linux-arm64-musl/include" \ + testdata/crosscompile-e2e/consumer.c \ + "$lib" \ + -o "$consumer" + readelf --program-headers "$consumer" | tee "$RUNNER_TEMP/musl-program-headers.txt" + grep -F "/lib/ld-musl-aarch64.so.1" "$RUNNER_TEMP/musl-program-headers.txt" + "$consumer" + build-darwin-arm64: name: "Build zlib: Linux amd64 host to Darwin arm64 target" runs-on: ubuntu-24.04 diff --git a/internal/crosscompile/c/llvm/toolchain.go b/internal/crosscompile/c/llvm/toolchain.go index 60dad1d..3c3b901 100644 --- a/internal/crosscompile/c/llvm/toolchain.go +++ b/internal/crosscompile/c/llvm/toolchain.go @@ -2,7 +2,10 @@ package llvm import ( "fmt" + "os" "os/exec" + "path/filepath" + "strings" "github.com/goplus/llar/internal/crosscompile/c" ) @@ -24,10 +27,10 @@ func New(config Config) (*Toolchain, error) { var triple, linkerName string switch config.Arch + "-" + config.OS { case "amd64-linux": - triple = "x86_64-linux-gnu" + triple = "x86_64-linux-" linkerName = "ld.lld" case "arm64-linux": - triple = "aarch64-linux-gnu" + triple = "aarch64-linux-" linkerName = "ld.lld" case "amd64-darwin": triple = "x86_64-apple-macos10.13" @@ -38,6 +41,47 @@ func New(config Config) (*Toolchain, error) { default: return nil, fmt.Errorf("unsupported LLVM target %s/%s", config.OS, config.Arch) } + if config.OS == "linux" { + environment := "gnu" + if config.Sysroot != "" { + // The loader family is the sysroot's target-environment fact. For + // example, any ld-musl* loader requires a Linux musl Clang triple. + hasLoader := func(prefix string) (bool, error) { + for _, dir := range []string{"lib", "lib64"} { + entries, err := os.ReadDir(filepath.Join(config.Sysroot, dir)) + if os.IsNotExist(err) { + continue + } + if err != nil { + return false, err + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), prefix) { + return true, nil + } + } + } + return false, nil + } + environment = "" + if ok, err := hasLoader("ld-linux"); err != nil { + return nil, fmt.Errorf("inspect GNU loader in %s: %w", config.Sysroot, err) + } else if ok { + environment = "gnu" + } + if environment == "" { + if ok, err := hasLoader("ld-musl"); err != nil { + return nil, fmt.Errorf("inspect musl loader in %s: %w", config.Sysroot, err) + } else if ok { + environment = "musl" + } + } + if environment == "" { + return nil, fmt.Errorf("cannot determine LLVM target environment from sysroot %s: neither ld-linux* nor ld-musl* exists", config.Sysroot) + } + } + triple += environment + } find := func(name string) (string, error) { path, err := exec.LookPath(name) if err != nil { diff --git a/internal/crosscompile/c/llvm/toolchain_test.go b/internal/crosscompile/c/llvm/toolchain_test.go index 6cf20e4..901a2d2 100644 --- a/internal/crosscompile/c/llvm/toolchain_test.go +++ b/internal/crosscompile/c/llvm/toolchain_test.go @@ -11,12 +11,13 @@ import ( func TestNewUsesPreparedPath(t *testing.T) { dir := fakeToolDir(t) t.Setenv("PATH", dir) + gnuSysroot := fakeLinuxSysroot(t, "gnu") - toolchain, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: "/sdk"}) + toolchain, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: gnuSysroot}) if err != nil { t.Fatal(err) } - if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=aarch64-linux-gnu", "-fuse-ld=lld", "--sysroot=/sdk"}; !reflect.DeepEqual(got, want) { + if got, want := toolchain.CC(), []string{filepath.Join(dir, "clang"), "--target=aarch64-linux-gnu", "-fuse-ld=lld", "--sysroot=" + gnuSysroot}; !reflect.DeepEqual(got, want) { t.Fatalf("CC = %q, want %q", got, want) } if got, want := toolchain.Linker(), []string{filepath.Join(dir, "ld.lld")}; !reflect.DeepEqual(got, want) { @@ -51,6 +52,30 @@ func TestNewUsesPreparedPath(t *testing.T) { } } +func TestNewSelectsLinuxEnvironmentFromSysroot(t *testing.T) { + dir := fakeToolDir(t) + t.Setenv("PATH", dir) + + for _, environment := range []string{"gnu", "musl"} { + t.Run(environment, func(t *testing.T) { + sysroot := fakeLinuxSysroot(t, environment) + toolchain, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: sysroot}) + if err != nil { + t.Fatal(err) + } + args := strings.Join(toolchain.CC(), " ") + if !strings.Contains(args, "--target=aarch64-linux-"+environment) { + t.Fatalf("CC = %q, want %s target", args, environment) + } + }) + } + + unknown := t.TempDir() + if _, err := New(Config{OS: "linux", Arch: "arm64", Sysroot: unknown}); err == nil || !strings.Contains(err.Error(), "cannot determine LLVM target environment") { + t.Fatalf("New unknown sysroot error = %v", err) + } +} + func TestNewErrors(t *testing.T) { if _, err := New(Config{OS: "plan9", Arch: "amd64"}); err == nil || !strings.Contains(err.Error(), "unsupported LLVM target") { t.Fatalf("New unsupported target error = %v", err) @@ -81,3 +106,25 @@ func fakeToolDir(t *testing.T) string { } return dir } + +func fakeLinuxSysroot(t *testing.T, environment string) string { + t.Helper() + root := t.TempDir() + var loader string + switch environment { + case "gnu": + loader = "lib64/ld-linux-test.so.37" + case "musl": + loader = "lib/ld-musl-test.so.42" + default: + t.Fatalf("unsupported fake Linux sysroot %s", environment) + } + path := filepath.Join(root, filepath.FromSlash(loader)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0o755); err != nil { + t.Fatal(err) + } + return root +} diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 6e193fe..894a736 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -2,9 +2,11 @@ package crosscompile import ( "context" + "errors" "fmt" "io" "runtime" + "strings" "github.com/goplus/llar/formula" "github.com/goplus/llar/internal/build" @@ -15,6 +17,7 @@ import ( "github.com/goplus/llar/internal/modules" "github.com/goplus/llar/mod/module" ccmetadata "github.com/goplus/llar/x/metadata/cc" + "github.com/kballard/go-shellquote" ) // Config contains the build resources used to prepare a cross-compile target. @@ -27,6 +30,90 @@ type Config struct { Cache cache.Cache } +// customTarget uses the bootstrap toolchain to build libc itself. When a +// consumer such as zlib supplies cmake.Sysroot or autotools.Sysroot, it switches +// to the toolchain selected from that sysroot. +type customTarget struct { + targetOS string + targetArch string + targetMatrix string + bootstrap *c.Target + configured *c.Target +} + +func (t *customTarget) Use(command build.Command) build.Patch { + root, err := commandSysroot(command) + if err != nil { + panic(err) + } + if root == "" && t.configured == nil { + return t.bootstrap.Use(command) + } + if t.configured == nil { + toolchain, err := llvm.New(llvm.Config{OS: t.targetOS, Arch: t.targetArch, Sysroot: root}) + if err != nil { + panic(fmt.Errorf("prepare C toolchain for %s: %w", t.targetMatrix, err)) + } + target, err := c.NewTarget(c.Config{ + Matrix: t.targetMatrix, + Toolchain: toolchain.Toolchain, + Sysroot: root, + }) + if err != nil { + panic(err) + } + t.configured = target + } + return t.configured.Use(command) +} + +func (t *customTarget) Close() error { + var configuredErr error + if t.configured != nil { + configuredErr = t.configured.Close() + } + return errors.Join(t.bootstrap.Close(), configuredErr) +} + +func commandSysroot(command build.Command) (string, error) { + var root string + for _, entry := range command.Env { + key, value, ok := strings.Cut(entry, "=") + if !ok { + continue + } + switch key { + case "CPPFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS": + metadata, err := ccmetadata.Parse(value) + if err != nil { + return "", fmt.Errorf("parse %s for cross compile target: %w", key, err) + } + if metadata.Sysroot() != "" { + root = metadata.Sysroot() + } + } + } + metadata, err := ccmetadata.Parse(shellquote.Join(command.Args...)) + if err != nil { + return "", fmt.Errorf("parse command flags for cross compile target: %w", err) + } + if metadata.Sysroot() != "" { + root = metadata.Sysroot() + } + for _, arg := range command.Args { + key, value, ok := strings.Cut(arg, "=") + if !ok { + continue + } + key = strings.TrimPrefix(key, "-D") + key, _, _ = strings.Cut(key, ":") + if key == "CMAKE_SYSROOT" || key == "CMAKE_OSX_SYSROOT" { + root = value + } + } + return root, nil +} + // Load returns the target used to cross-compile root. A nil target means the // requested matrix is native or has no built-in C target policy. func Load(ctx context.Context, root module.Version, config Config) (build.Target, error) { @@ -61,9 +148,17 @@ func Load(ctx context.Context, root module.Version, config Config) (build.Target } _, customLibc := config.Matrix.Require["libc"] - if customLibc || root.Path == cSysroot.Path { + if root.Path == cSysroot.Path { return bootstrapTarget, nil } + if customLibc { + return &customTarget{ + targetOS: targetOS, + targetArch: targetArch, + targetMatrix: targetMatrix, + bootstrap: bootstrapTarget, + }, nil + } defer bootstrapTarget.Close() // The default sysroot has no dependencies, but modules.Load still owns diff --git a/internal/crosscompile/crosscompile_test.go b/internal/crosscompile/crosscompile_test.go index 44c1fd9..527f740 100644 --- a/internal/crosscompile/crosscompile_test.go +++ b/internal/crosscompile/crosscompile_test.go @@ -102,13 +102,15 @@ func TestLoadBootstrapTarget(t *testing.T) { func TestLoadDefaultSysroot(t *testing.T) { installFakeLLVM(t) matrix, _ := linuxCrossMatrix() + arch := matrix.Require["arch"][0] + sysroot := fakeCrossSysroot(t, arch, "gnu") target, err := Load(context.Background(), module.Version{Path: "owner/repo", Version: "v1.0.0"}, Config{ Store: localSysrootFormulas(t), Matrix: matrix, Stdout: io.Discard, Stderr: io.Discard, WorkspaceDir: t.TempDir(), - Cache: metadataCache{metadata: "--sysroot=/target-sdk"}, + Cache: metadataCache{metadata: "--sysroot=" + sysroot}, }) if err != nil { t.Fatal(err) @@ -120,11 +122,71 @@ func TestLoadDefaultSysroot(t *testing.T) { t.Cleanup(func() { _ = closer.Close() }) } patch := target.Use(build.Command{Name: "cc"}) - if args := strings.Join(patch.PrependArg, " "); !strings.Contains(args, "--sysroot=/target-sdk") { + if args := strings.Join(patch.PrependArg, " "); !strings.Contains(args, "--sysroot="+sysroot) { t.Fatalf("compiler args = %q, want configured sysroot", args) } } +func TestLoadCustomLibcUsesCommandSysroot(t *testing.T) { + installFakeLLVM(t) + matrix, _ := linuxCrossMatrix() + arch := matrix.Require["arch"][0] + matrix.Require["libc"] = []string{"custom"} + target, err := Load(context.Background(), module.Version{Path: "owner/repo", Version: "v1.0.0"}, Config{Matrix: matrix}) + if err != nil { + t.Fatal(err) + } + if closer, ok := target.(io.Closer); ok { + t.Cleanup(func() { _ = closer.Close() }) + } + + sysroot := fakeCrossSysroot(t, arch, "musl") + patch := target.Use(build.Command{Name: "cc", Args: []string{"--sysroot=" + sysroot}}) + args := strings.Join(patch.PrependArg, " ") + wantTriple := "x86_64-linux-musl" + if arch == "arm64" { + wantTriple = "aarch64-linux-musl" + } + if !strings.Contains(args, "--target="+wantTriple) { + t.Fatalf("compiler args = %q, want target %q", args, wantTriple) + } +} + +func TestCommandSysroot(t *testing.T) { + tests := []struct { + name string + command build.Command + want string + }{ + { + name: "autotools environment", + command: build.Command{Env: []string{"CFLAGS=-O2 --sysroot=/autotools-sdk -isysroot/autotools-sdk"}}, + want: "/autotools-sdk", + }, + { + name: "cmake definition", + command: build.Command{Args: []string{"-DCMAKE_SYSROOT:STRING=/cmake-sdk"}}, + want: "/cmake-sdk", + }, + { + name: "compiler option", + command: build.Command{Args: []string{"-c", "source.c", "-isysroot/compiler-sdk"}}, + want: "/compiler-sdk", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := commandSysroot(tt.command) + if err != nil { + t.Fatal(err) + } + if got != tt.want { + t.Fatalf("commandSysroot = %q, want %q", got, tt.want) + } + }) + } +} + func TestLoadErrors(t *testing.T) { matrix, _ := linuxCrossMatrix() root := module.Version{Path: "owner/repo", Version: "v1.0.0"} @@ -220,6 +282,32 @@ func installFakeLLVM(t *testing.T) { t.Setenv("PATH", dir) } +func fakeCrossSysroot(t *testing.T, arch, environment string) string { + t.Helper() + root := t.TempDir() + var loader string + switch arch + "-" + environment { + case "amd64-gnu": + loader = "lib64/ld-linux-x86-64.so.2" + case "amd64-musl": + loader = "lib/ld-musl-x86_64.so.1" + case "arm64-gnu": + loader = "lib/ld-linux-aarch64.so.1" + case "arm64-musl": + loader = "lib/ld-musl-aarch64.so.1" + default: + t.Fatalf("unsupported fake Linux sysroot %s/%s", arch, environment) + } + path := filepath.Join(root, filepath.FromSlash(loader)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0o755); err != nil { + t.Fatal(err) + } + return root +} + func localSysrootFormulas(t *testing.T) formulaStore { t.Helper() _, filename, _, ok := runtime.Caller(0) diff --git a/testdata/crosscompile-e2e/formulas/ifduyue/musl/v1.1.19/Musl_llar.gox b/testdata/crosscompile-e2e/formulas/ifduyue/musl/v1.1.19/Musl_llar.gox new file mode 100644 index 0000000..1f10c74 --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/ifduyue/musl/v1.1.19/Musl_llar.gox @@ -0,0 +1,8 @@ +id "ifduyue/musl" + +fromVer "v1.1.19" + +onBuild ctx => { + exec! "./install-sysroot", ctx.outputDir + ctx.setMetadata "--sysroot="+ctx.outputDir +} diff --git a/testdata/crosscompile-e2e/formulas/ifduyue/musl/versions.json b/testdata/crosscompile-e2e/formulas/ifduyue/musl/versions.json new file mode 100644 index 0000000..97b6460 --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/ifduyue/musl/versions.json @@ -0,0 +1,4 @@ +{ + "path": "ifduyue/musl", + "deps": {} +} diff --git a/testdata/crosscompile-e2e/install-bootlin-sysroot b/testdata/crosscompile-e2e/install-bootlin-sysroot index 1de6dcb..10cf77d 100755 --- a/testdata/crosscompile-e2e/install-bootlin-sysroot +++ b/testdata/crosscompile-e2e/install-bootlin-sysroot @@ -4,9 +4,9 @@ set -eu output_dir=$1 source_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -set -- "$source_dir"/aarch64--glibc--stable-*.tar.bz2 +set -- "$source_dir"/aarch64--*--stable-*.tar.bz2 if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then - echo "expected one Bootlin glibc toolchain archive" >&2 + echo "expected one Bootlin toolchain archive" >&2 exit 1 fi archive=$1 @@ -14,26 +14,33 @@ archive=$1 case $(basename -- "$archive") in aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2) root=aarch64--glibc--stable + tuple=aarch64-buildroot-linux-gnu gcc_version=5.4.0 ;; aarch64--glibc--stable-2018.11-1.tar.bz2) root=aarch64--glibc--stable-2018.11-1 + tuple=aarch64-buildroot-linux-gnu + gcc_version=7.3.0 + ;; + aarch64--musl--stable-2018.11-1.tar.bz2) + root=aarch64--musl--stable-2018.11-1 + tuple=aarch64-buildroot-linux-musl gcc_version=7.3.0 ;; *) - echo "unsupported Bootlin glibc toolchain archive: $(basename -- "$archive")" >&2 + echo "unsupported Bootlin toolchain archive: $(basename -- "$archive")" >&2 exit 1 ;; esac mkdir -p "$output_dir" LC_ALL=C tar -xjf "$archive" -C "$output_dir" --strip-components=3 \ - "$root/aarch64-buildroot-linux-gnu/sysroot" \ - "$root/lib/gcc/aarch64-buildroot-linux-gnu/$gcc_version" + "$root/$tuple/sysroot" \ + "$root/lib/gcc/$tuple/$gcc_version" # Clang needs the target GCC runtime, for example crtbeginS.o and libgcc.a, # beside the target libc when configure links compiler probes. -runtime_dir="$output_dir/aarch64-buildroot-linux-gnu/$gcc_version" +runtime_dir="$output_dir/$tuple/$gcc_version" mkdir -p "$output_dir/usr/lib" cp -a "$runtime_dir/." "$output_dir/usr/lib/" -rm -r "$output_dir/aarch64-buildroot-linux-gnu" +rm -r "$output_dir/$tuple" diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox index 85c4bd3..2e5941e 100644 --- a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -3,8 +3,13 @@ id "madler/zlib" fromVer "v1.3.1" onRequire (proj, deps) => { - if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "glibc-2.27" { - deps.require "bminor/glibc", libc[0] + if libc := target.require["libc"]; len(libc) > 0 { + if libc[0] == "glibc-2.27" { + deps.require "bminor/glibc", libc[0] + } + if libc[0] == "musl-1.1.19" { + deps.require "ifduyue/musl", "v1.1.19" + } } } @@ -13,7 +18,7 @@ onBuild ctx => { a := autotools.new(ctx.SourceDir, ctx.SourceDir+"/_build", installDir) for _, dep := range ctx.Proj.Deps { - if dep.Path == "bminor/glibc" { + if dep.Path == "bminor/glibc" || dep.Path == "ifduyue/musl" { a.sysroot ctx.outputDir(dep) } } From efa3997cba067afa96c4feff259177068da636b0 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 15:28:56 +0800 Subject: [PATCH 31/39] test(crosscompile): verify musl libuv ABI --- .github/workflows/crosscompile-e2e.yml | 64 ++++++++++--------- testdata/crosscompile-e2e/libuv-consumer.c | 20 ++++++ .../libuv/libuv/v1.52.1/Libuv_llar.gox | 28 ++++++++ .../formulas/libuv/libuv/versions.json | 4 ++ .../formulas/madler/zlib/v1.3.1/Zlib_llar.gox | 5 +- 5 files changed, 87 insertions(+), 34 deletions(-) create mode 100644 testdata/crosscompile-e2e/libuv-consumer.c create mode 100644 testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox create mode 100644 testdata/kodo-e2e/formulas/libuv/libuv/versions.json diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 6b2ef2e..5e2cce0 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -13,7 +13,7 @@ concurrency: jobs: build-linux-arm64: - name: "Build zlib: Linux amd64 -> Linux arm64 (glibc/musl)" + name: "Build C libs: Linux amd64 -> Linux arm64" runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -125,20 +125,20 @@ jobs: exit 1 fi - - name: Cross compile zlib with Formula-managed musl + - name: Cross compile libuv with Formula-managed musl run: | - musl_log="$RUNNER_TEMP/zlib-linux-arm64-musl.log" - "$RUNNER_TEMP/llar" make madler/zlib@v1.3.1 \ + musl_log="$RUNNER_TEMP/libuv-linux-arm64-musl.log" + "$RUNNER_TEMP/llar" make libuv/libuv@v1.52.1 \ --os linux \ --arch arm64 \ --require libc=musl-1.1.19 \ - --output "$RUNNER_TEMP/zlib-linux-arm64-musl" \ + --output "$RUNNER_TEMP/libuv-linux-arm64-musl" \ --json \ --verbose \ - > "$RUNNER_TEMP/zlib-linux-arm64-musl.json" \ + > "$RUNNER_TEMP/libuv-linux-arm64-musl.json" \ 2> >(tee "$musl_log" >&2) grep -F '"path":"ifduyue/musl","version":"v1.1.19"' \ - "$RUNNER_TEMP/zlib-linux-arm64-musl.json" + "$RUNNER_TEMP/libuv-linux-arm64-musl.json" grep -E -- '--sysroot=[^ ]*/ifduyue/musl@v1\.1\.19-' "$musl_log" grep -F -- '--target=aarch64-linux-musl' "$musl_log" @@ -156,15 +156,15 @@ jobs: path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc if-no-files-found: error - - name: Upload musl zlib artifact + - name: Upload musl libuv artifact uses: actions/upload-artifact@v4 with: - name: zlib-linux-arm64-musl - path: ${{ runner.temp }}/zlib-linux-arm64-musl + name: libuv-linux-arm64-musl + path: ${{ runner.temp }}/libuv-linux-arm64-musl if-no-files-found: error run-linux-arm64: - name: "Run zlib: Linux arm64 (glibc/musl)" + name: "Run C libs: Linux arm64" needs: build-linux-arm64 runs-on: ubuntu-24.04-arm timeout-minutes: 10 @@ -189,11 +189,11 @@ jobs: name: zlib-linux-arm64-custom-libc path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc - - name: Download musl zlib artifact + - name: Download musl libuv artifact uses: actions/download-artifact@v4 with: - name: zlib-linux-arm64-musl - path: ${{ runner.temp }}/zlib-linux-arm64-musl + name: libuv-linux-arm64-musl + path: ${{ runner.temp }}/libuv-linux-arm64-musl - name: Inspect, link, and run default-sysroot consumer run: | @@ -241,28 +241,32 @@ jobs: grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/custom-libc-dynamic-libraries.txt" "$consumer" - - name: Inspect, link, and run musl consumer + - name: Compare glibc and musl libuv ABIs run: | - lib="$RUNNER_TEMP/zlib-linux-arm64-musl/lib/libz.a" - member_dir="$RUNNER_TEMP/zlib-musl-members" - consumer="$RUNNER_TEMP/zlib-musl-consumer" + root="$RUNNER_TEMP/libuv-linux-arm64-musl" + lib="$root/lib/libuv.a" + glibc_consumer="$RUNNER_TEMP/libuv-glibc-consumer" + musl_consumer="$RUNNER_TEMP/libuv-musl-consumer" - mkdir -p "$member_dir" - ( - cd "$member_dir" - ar x "$lib" adler32.o - ) - file "$member_dir/adler32.o" | tee "$RUNNER_TEMP/zlib-musl-format.txt" - grep -F "ELF 64-bit LSB relocatable, ARM aarch64" "$RUNNER_TEMP/zlib-musl-format.txt" + cc \ + -DEXPECT_ABI_MATCH=0 \ + -I"$root/include" \ + testdata/crosscompile-e2e/libuv-consumer.c \ + "$lib" \ + -pthread -ldl -lrt \ + -o "$glibc_consumer" + "$glibc_consumer" musl-gcc \ - -I"$RUNNER_TEMP/zlib-linux-arm64-musl/include" \ - testdata/crosscompile-e2e/consumer.c \ + -DEXPECT_ABI_MATCH=1 \ + -I"$root/include" \ + testdata/crosscompile-e2e/libuv-consumer.c \ "$lib" \ - -o "$consumer" - readelf --program-headers "$consumer" | tee "$RUNNER_TEMP/musl-program-headers.txt" + -pthread -ldl -lrt \ + -o "$musl_consumer" + readelf --program-headers "$musl_consumer" | tee "$RUNNER_TEMP/musl-program-headers.txt" grep -F "/lib/ld-musl-aarch64.so.1" "$RUNNER_TEMP/musl-program-headers.txt" - "$consumer" + "$musl_consumer" build-darwin-arm64: name: "Build zlib: Linux amd64 host to Darwin arm64 target" diff --git a/testdata/crosscompile-e2e/libuv-consumer.c b/testdata/crosscompile-e2e/libuv-consumer.c new file mode 100644 index 0000000..65c40ae --- /dev/null +++ b/testdata/crosscompile-e2e/libuv-consumer.c @@ -0,0 +1,20 @@ +#include + +#include + +#ifndef EXPECT_ABI_MATCH +#error EXPECT_ABI_MATCH must be defined +#endif + +int main(void) { + size_t built_size = uv_loop_size(); + size_t consumer_size = sizeof(uv_loop_t); + int matches = built_size == consumer_size; + + printf("libuv loop size: library=%zu consumer=%zu\n", + built_size, consumer_size); + if (matches != EXPECT_ABI_MATCH) { + return 1; + } + return 0; +} diff --git a/testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox b/testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox new file mode 100644 index 0000000..c65c866 --- /dev/null +++ b/testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox @@ -0,0 +1,28 @@ +id "libuv/libuv" + +fromVer "v1.52.1" + +onRequire (proj, deps) => { + if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "musl-1.1.19" { + deps.require "ifduyue/musl", "v1.1.19" + } +} + +onBuild ctx => { + installDir := ctx.outputDir + + c := cmake.new(ctx.SourceDir, ctx.SourceDir+"/_build", installDir) + for _, dep := range ctx.Proj.Deps { + if dep.Path == "ifduyue/musl" { + c.sysroot ctx.outputDir(dep) + } + } + c.buildType "Release" + c.defineBool "LIBUV_BUILD_SHARED", false + c.defineBool "BUILD_TESTING", false + c.configure + c.build + c.install + + ctx.setMetadata "-luv -pthread -ldl -lrt" +} diff --git a/testdata/kodo-e2e/formulas/libuv/libuv/versions.json b/testdata/kodo-e2e/formulas/libuv/libuv/versions.json new file mode 100644 index 0000000..275b882 --- /dev/null +++ b/testdata/kodo-e2e/formulas/libuv/libuv/versions.json @@ -0,0 +1,4 @@ +{ + "path": "libuv/libuv", + "deps": {} +} diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox index 2e5941e..d081ecd 100644 --- a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -7,9 +7,6 @@ onRequire (proj, deps) => { if libc[0] == "glibc-2.27" { deps.require "bminor/glibc", libc[0] } - if libc[0] == "musl-1.1.19" { - deps.require "ifduyue/musl", "v1.1.19" - } } } @@ -18,7 +15,7 @@ onBuild ctx => { a := autotools.new(ctx.SourceDir, ctx.SourceDir+"/_build", installDir) for _, dep := range ctx.Proj.Deps { - if dep.Path == "bminor/glibc" || dep.Path == "ifduyue/musl" { + if dep.Path == "bminor/glibc" { a.sysroot ctx.outputDir(dep) } } From 424633b688eab6af106e277ac58d195638e0cfd0 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 15:35:53 +0800 Subject: [PATCH 32/39] test(crosscompile): use musl-compatible libuv --- .github/workflows/crosscompile-e2e.yml | 2 +- .../formulas/libuv/libuv/{v1.52.1 => v1.49.2}/Libuv_llar.gox | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename testdata/kodo-e2e/formulas/libuv/libuv/{v1.52.1 => v1.49.2}/Libuv_llar.gox (96%) diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 5e2cce0..1c95510 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -128,7 +128,7 @@ jobs: - name: Cross compile libuv with Formula-managed musl run: | musl_log="$RUNNER_TEMP/libuv-linux-arm64-musl.log" - "$RUNNER_TEMP/llar" make libuv/libuv@v1.52.1 \ + "$RUNNER_TEMP/llar" make libuv/libuv@v1.49.2 \ --os linux \ --arch arm64 \ --require libc=musl-1.1.19 \ diff --git a/testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox b/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox similarity index 96% rename from testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox rename to testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox index c65c866..9819b44 100644 --- a/testdata/kodo-e2e/formulas/libuv/libuv/v1.52.1/Libuv_llar.gox +++ b/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox @@ -1,6 +1,6 @@ id "libuv/libuv" -fromVer "v1.52.1" +fromVer "v1.49.2" onRequire (proj, deps) => { if libc := target.require["libc"]; len(libc) > 0 && libc[0] == "musl-1.1.19" { From 3567e752209ba3933d6b0c9e0d13be4f9800cfe7 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 15:40:27 +0800 Subject: [PATCH 33/39] test(crosscompile): show libuv compile commands --- testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox b/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox index 9819b44..eda93e3 100644 --- a/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox +++ b/testdata/kodo-e2e/formulas/libuv/libuv/v1.49.2/Libuv_llar.gox @@ -18,6 +18,7 @@ onBuild ctx => { } } c.buildType "Release" + c.defineBool "CMAKE_VERBOSE_MAKEFILE", true c.defineBool "LIBUV_BUILD_SHARED", false c.defineBool "BUILD_TESTING", false c.configure From 9b27abd42939af6ca3cc939811e48e35b4e39242 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 15:47:48 +0800 Subject: [PATCH 34/39] test(crosscompile): clarify libuv ABI assertions --- testdata/crosscompile-e2e/libuv-consumer.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/testdata/crosscompile-e2e/libuv-consumer.c b/testdata/crosscompile-e2e/libuv-consumer.c index 65c40ae..49cfabd 100644 --- a/testdata/crosscompile-e2e/libuv-consumer.c +++ b/testdata/crosscompile-e2e/libuv-consumer.c @@ -9,12 +9,21 @@ int main(void) { size_t built_size = uv_loop_size(); size_t consumer_size = sizeof(uv_loop_t); - int matches = built_size == consumer_size; printf("libuv loop size: library=%zu consumer=%zu\n", built_size, consumer_size); - if (matches != EXPECT_ABI_MATCH) { + +#if EXPECT_ABI_MATCH + // The musl-built library and musl consumer must use the same pthread layout. + if (built_size != consumer_size) { + return 1; + } +#else + // The glibc consumer is the negative control: uv_loop_t contains libc-owned + // pthread types, so matching sizes would fail to distinguish the two ABIs. + if (built_size == consumer_size) { return 1; } +#endif return 0; } From 7e097f9840f5a013feb9fe3e633c22234552d0e9 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 16:02:02 +0800 Subject: [PATCH 35/39] test(llard): cover cross compilation --- .github/workflows/llard-cluster-e2e.yml | 32 +++++++++- testdata/llard-e2e/e2e_test.go | 82 +++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/.github/workflows/llard-cluster-e2e.yml b/.github/workflows/llard-cluster-e2e.yml index 88e3c23..8f69d92 100644 --- a/.github/workflows/llard-cluster-e2e.yml +++ b/.github/workflows/llard-cluster-e2e.yml @@ -34,6 +34,12 @@ jobs: with: go-version: 1.24.x + - name: Install LLVM + run: | + sudo apt-get update + sudo apt-get install --yes clang-18 lld-18 llvm-18 + echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" + - name: Join private network uses: tailscale/github-action@v4 with: @@ -47,18 +53,40 @@ jobs: go build -ldflags="-checklinkname=0" -o "$RUNNER_TEMP/llard" ./cmd/llard go build -o "$RUNNER_TEMP/llard-e2e-control" ./testdata/llard-e2e/control - - name: Prepare formula repository + - name: Download Bootlin glibc arm64 toolchain + run: | + archive="$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" + curl --fail --location --retry 3 \ + "https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/tarballs/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" \ + --output "$archive" + echo "530137589c4588599ebd115c12630c75e79c935ac4f34b0941404036f5e25026 $archive" | sha256sum --check + + - name: Prepare formula and source repositories run: | formula_repo="$RUNNER_TEMP/llarhub" - mkdir -p "$formula_repo" + source_repo="$RUNNER_TEMP/glibc-source" + mkdir -p "$formula_repo" "$source_repo" cp -R testdata/kodo-e2e/formulas/. "$formula_repo/" + cp -R testdata/crosscompile-e2e/formulas/bminor "$formula_repo/" git -C "$formula_repo" init --initial-branch=main git -C "$formula_repo" config user.email llard-e2e@example.com git -C "$formula_repo" config user.name "LLARD E2E" git -C "$formula_repo" add . git -C "$formula_repo" commit -m "LLARD E2E formulas" + + cp testdata/crosscompile-e2e/install-bootlin-sysroot "$source_repo/install-sysroot" + cp "$RUNNER_TEMP/aarch64--glibc--stable-2017.05-toolchains-1-1.tar.bz2" "$source_repo/" + chmod +x "$source_repo/install-sysroot" + git -C "$source_repo" init --initial-branch=main + git -C "$source_repo" config user.email llard-e2e@example.com + git -C "$source_repo" config user.name "LLARD E2E" + git -C "$source_repo" add . + git -C "$source_repo" commit --quiet -m "Add glibc 2.24 toolchain" + git -C "$source_repo" tag glibc-2.24 + git config --global protocol.file.allow always git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" + git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" - name: Run llard env: diff --git a/testdata/llard-e2e/e2e_test.go b/testdata/llard-e2e/e2e_test.go index 5378322..575a3df 100644 --- a/testdata/llard-e2e/e2e_test.go +++ b/testdata/llard-e2e/e2e_test.go @@ -133,15 +133,32 @@ func TestLLARDE2E(t *testing.T) { if err != nil { t.Fatal(err) } + crossMatrix := classfile.Matrix{Require: map[string][]string{ + "arch": {"arm64"}, + "os": {"linux"}, + }} + cross := *s + cross.matrix = crossMatrix + cross.matrixStr = crossMatrix.Combinations()[0] + cross.query = url.Values{ + "arch": {"arm64"}, + "os": {"linux"}, + }.Encode() if err := s.deleteArtifacts(ctx, zlibTarget, libpngTarget, cjsonTarget); err != nil { t.Fatal(err) } + if err := cross.deleteArtifacts(ctx, zlibTarget); err != nil { + t.Fatal(err) + } t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() if err := s.deleteArtifacts(ctx, zlibTarget, libpngTarget, cjsonTarget); err != nil { t.Errorf("cleanup artifacts: %v", err) } + if err := cross.deleteArtifacts(ctx, zlibTarget); err != nil { + t.Errorf("cleanup cross-compile artifact: %v", err) + } }) for _, step := range []struct { @@ -152,6 +169,7 @@ func TestLLARDE2E(t *testing.T) { {"warm zlib request reuses stored artifact", s.warmZlibBuild}, {"concurrent zlib requests run one build", s.concurrentZlibBuild}, {"concurrent roots share canonical zlib artifact", s.concurrentSharedDependency}, + {"cross-compile zlib for Linux arm64", cross.crossCompileZlib}, {"protocol errors use command JSON lines", s.protocolErrors}, } { start := time.Now() @@ -439,6 +457,70 @@ func (s *suite) concurrentSharedDependency(ctx context.Context) error { return nil } +func (s *suite) crossCompileZlib(ctx context.Context) error { + before, err := s.buildCounts(ctx) + if err != nil { + return err + } + got, err := s.get(ctx, s.cfg.baseURL, zlibTarget) + if err != nil { + return err + } + if err := s.requireSuccess(got, zlibTarget); err != nil { + return err + } + after, err := s.buildCounts(ctx) + if err != nil { + return err + } + if delta := after[zlibTarget.key()] - before[zlibTarget.key()]; delta != 1 { + return fmt.Errorf("cross-compile zlib build count = %d, want 1", delta) + } + if got.upstream == "" { + return errors.New("cross-compile nginx response has no X-Upstream-Addr") + } + if _, ok := s.upstreams[got.upstream]; !ok { + return fmt.Errorf("cross-compile nginx upstream = %q, workers = %v", got.upstream, s.upstreams) + } + + hasTarget := false + hasSysroot := false + for _, line := range got.infos { + if strings.Contains(line, "--target=aarch64-linux-gnu") { + hasTarget = true + } + if strings.Contains(line, "bminor/glibc@glibc-2.24-") { + hasSysroot = true + } + } + if !hasTarget { + return fmt.Errorf("cross-compile info has no ARM64 GNU target:\n%s", strings.Join(got.infos, "\n")) + } + if !hasSysroot { + return fmt.Errorf("cross-compile info has no default glibc sysroot:\n%s", strings.Join(got.infos, "\n")) + } + + message := got.artifacts[0] + if len(message.Deps) != 0 { + return fmt.Errorf("cross-compile zlib deps = %q, want hidden sysroot", message.Deps) + } + info, checksum, err := s.downloadArtifact(ctx, message) + if err != nil { + return err + } + if info.Metadata != zlibTarget.metadata { + return fmt.Errorf("cross-compile zlib metadata = %q, want %q", info.Metadata, zlibTarget.metadata) + } + stored, err := s.artifacts.Get(ctx, s.artifactKey(zlibTarget)) + if err != nil { + return err + } + if stored.Source.URL != message.URL || stored.Checksum != checksum { + return fmt.Errorf("stored cross-compile zlib artifact = %+v, response URL %q checksum %q", stored, message.URL, checksum) + } + return nil +} + func (s *suite) protocolErrors(ctx context.Context) error { missingMatrix, err := s.do(ctx, http.MethodGet, s.cfg.baseURL+"/v1/artifacts/"+zlibTarget.path+"@"+zlibTarget.version) if err != nil { From e510874226c1e13beeea5cbc4725ac981a6840a6 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 17:41:38 +0800 Subject: [PATCH 36/39] fix(crosscompile): pass pkg-config env to build systems --- internal/crosscompile/c/target.go | 7 ++++++- internal/crosscompile/c/target_test.go | 26 ++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/internal/crosscompile/c/target.go b/internal/crosscompile/c/target.go index 2da0a7b..a069126 100644 --- a/internal/crosscompile/c/target.go +++ b/internal/crosscompile/c/target.go @@ -131,7 +131,9 @@ func (c *Target) Use(cmd build.Command) build.Patch { c.tempDir = tempDir c.toolchainFile = toolchainFile } - return build.Patch{AppendArg: []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}} + patch := c.pkgConfigPatch(cmd.Env) + patch.AppendArg = []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile} + return patch } case "pkg-config": return c.pkgConfigPatch(cmd.Env) @@ -166,6 +168,9 @@ func (c *Target) autotoolsPatch(cmd build.Command) build.Patch { env = setMissingEnv(env, "RANLIB", c.toolchain.Ranlib()) env = setMissingEnv(env, "NM", c.toolchain.NM()) env = setMissingEnv(env, "STRIP", c.toolchain.Strip()) + if c.sysroot != "" { + env = c.pkgConfigPatch(env).Env + } for _, arg := range cmd.Args { if arg == "--host" || strings.HasPrefix(arg, "--host=") { diff --git a/internal/crosscompile/c/target_test.go b/internal/crosscompile/c/target_test.go index d0ba054..35af076 100644 --- a/internal/crosscompile/c/target_test.go +++ b/internal/crosscompile/c/target_test.go @@ -186,10 +186,23 @@ func TestUseCMakeWritesToolchainLazily(t *testing.T) { func TestUseCMake(t *testing.T) { c := newTestTarget(t) - patch := c.Use(build.Command{Name: "cmake", Args: []string{"-S", ".", "-B", "build"}}) + patch := c.Use(build.Command{ + Name: "cmake", + Args: []string{"-S", ".", "-B", "build"}, + Env: []string{"PKG_CONFIG_PATH=/deps/lib/pkgconfig"}, + }) if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } + if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + } + libDirs, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") + for _, want := range []string{"/deps/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { + if !slices.Contains(filepath.SplitList(libDirs), want) { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) + } + } patch = c.Use(build.Command{Name: "cmake", Args: []string{"--build", "build"}}) if len(patch.AppendArg) != 0 { t.Fatalf("build Patch = %+v, want no toolchain argument", patch) @@ -236,7 +249,7 @@ func TestUseAutotools(t *testing.T) { patch := c.Use(build.Command{ Name: configure, Args: []string{"--build=x86_64-apple-darwin"}, - Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom"}, + Env: []string{"CC=/custom/cc", "CFLAGS=-O2 --target=custom", "PKG_CONFIG_PATH=/deps/lib/pkgconfig"}, }) if got, _ := envValue(patch.Env, "CC"); got != "/custom/cc" { t.Fatalf("CC override = %q, want /custom/cc", got) @@ -250,6 +263,15 @@ func TestUseAutotools(t *testing.T) { if got, want := patch.AppendArg, []string{"--host=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } + if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + } + libDirs, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") + for _, want := range []string{"/deps/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { + if !slices.Contains(filepath.SplitList(libDirs), want) { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) + } + } patch = c.Use(build.Command{Name: configure, Args: []string{"--host=custom-linux"}}) if len(patch.AppendArg) != 0 { From 96558f86c6d9d762969e1f1585629d4bea99cc06 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 18:25:25 +0800 Subject: [PATCH 37/39] fix(crosscompile): isolate pkg-config dependency lookup --- .github/workflows/crosscompile-e2e.yml | 47 ++++++++++++++++++- internal/crosscompile/c/target.go | 11 ++--- internal/crosscompile/c/target_test.go | 32 +++++-------- .../v1.0.0/Consumer_llar.gox | 18 +++++++ .../llar/pkgconfig-consumer/versions.json | 8 ++++ .../pkgconfig-consumer/project.cmake | 23 +++++++++ x/autotools/autotools.go | 13 ++--- x/autotools/autotools_test.go | 17 ++----- 8 files changed, 116 insertions(+), 53 deletions(-) create mode 100644 testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/v1.0.0/Consumer_llar.gox create mode 100644 testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/versions.json create mode 100644 testdata/crosscompile-e2e/pkgconfig-consumer/project.cmake diff --git a/.github/workflows/crosscompile-e2e.yml b/.github/workflows/crosscompile-e2e.yml index 1c95510..4307d45 100644 --- a/.github/workflows/crosscompile-e2e.yml +++ b/.github/workflows/crosscompile-e2e.yml @@ -28,7 +28,7 @@ jobs: - name: Install LLVM run: | sudo apt-get update - sudo apt-get install --yes clang-18 lld-18 llvm-18 + sudo apt-get install --yes clang-18 lld-18 llvm-18 pkg-config echo "/usr/lib/llvm-18/bin" >> "$GITHUB_PATH" - name: Download Bootlin glibc and musl arm64 toolchains @@ -55,8 +55,9 @@ jobs: formula_repo="$RUNNER_TEMP/llarhub" source_repo="$RUNNER_TEMP/glibc-source" musl_source_repo="$RUNNER_TEMP/musl-source" + consumer_source_repo="$RUNNER_TEMP/pkgconfig-consumer-source" - mkdir -p "$formula_repo" "$source_repo" "$musl_source_repo" + mkdir -p "$formula_repo" "$source_repo" "$musl_source_repo" "$consumer_source_repo" cp -R testdata/kodo-e2e/formulas/. "$formula_repo/" cp -R testdata/crosscompile-e2e/formulas/. "$formula_repo/" git -C "$formula_repo" init --initial-branch=main @@ -91,10 +92,20 @@ jobs: git -C "$musl_source_repo" commit --quiet -m "Add musl 1.1.19 toolchain" git -C "$musl_source_repo" tag v1.1.19 + cp testdata/crosscompile-e2e/pkgconfig-consumer/project.cmake "$consumer_source_repo/CMakeLists.txt" + cp testdata/crosscompile-e2e/consumer.c "$consumer_source_repo/main.c" + git -C "$consumer_source_repo" init --initial-branch=main + git -C "$consumer_source_repo" config user.email crosscompile-e2e@example.com + git -C "$consumer_source_repo" config user.name "Cross Compile E2E" + git -C "$consumer_source_repo" add . + git -C "$consumer_source_repo" commit --quiet -m "Add pkg-config dependency consumer" + git -C "$consumer_source_repo" tag v1.0.0 + git config --global protocol.file.allow always git config --global url."file://$formula_repo".insteadOf "https://github.com/goplus/llarhub.git" git config --global url."file://$source_repo".insteadOf "https://github.com/bminor/glibc.git" git config --global url."file://$musl_source_repo".insteadOf "https://github.com/ifduyue/musl.git" + git config --global url."file://$consumer_source_repo".insteadOf "https://github.com/llar/pkgconfig-consumer.git" - name: Cross compile zlib with default sysroot run: | @@ -125,6 +136,14 @@ jobs: exit 1 fi + - name: Cross compile pkg-config dependency consumer + run: | + "$RUNNER_TEMP/llar" make llar/pkgconfig-consumer@v1.0.0 \ + --os linux \ + --arch arm64 \ + --output "$RUNNER_TEMP/pkgconfig-consumer-linux-arm64" \ + --verbose + - name: Cross compile libuv with Formula-managed musl run: | musl_log="$RUNNER_TEMP/libuv-linux-arm64-musl.log" @@ -156,6 +175,13 @@ jobs: path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc if-no-files-found: error + - name: Upload pkg-config consumer artifact + uses: actions/upload-artifact@v4 + with: + name: pkgconfig-consumer-linux-arm64 + path: ${{ runner.temp }}/pkgconfig-consumer-linux-arm64 + if-no-files-found: error + - name: Upload musl libuv artifact uses: actions/upload-artifact@v4 with: @@ -189,6 +215,12 @@ jobs: name: zlib-linux-arm64-custom-libc path: ${{ runner.temp }}/zlib-linux-arm64-custom-libc + - name: Download pkg-config consumer artifact + uses: actions/download-artifact@v4 + with: + name: pkgconfig-consumer-linux-arm64 + path: ${{ runner.temp }}/pkgconfig-consumer-linux-arm64 + - name: Download musl libuv artifact uses: actions/download-artifact@v4 with: @@ -241,6 +273,17 @@ jobs: grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/custom-libc-dynamic-libraries.txt" "$consumer" + - name: Inspect and run pkg-config dependency consumer + run: | + consumer="$RUNNER_TEMP/pkgconfig-consumer-linux-arm64/bin/llar-pkgconfig-consumer" + + chmod +x "$consumer" + file "$consumer" | tee "$RUNNER_TEMP/pkgconfig-consumer-format.txt" + grep -E "ELF 64-bit LSB.*ARM aarch64" "$RUNNER_TEMP/pkgconfig-consumer-format.txt" + ldd "$consumer" | tee "$RUNNER_TEMP/pkgconfig-consumer-dynamic-libraries.txt" + grep -E "libc\\.so\\.6 => /" "$RUNNER_TEMP/pkgconfig-consumer-dynamic-libraries.txt" + "$consumer" + - name: Compare glibc and musl libuv ABIs run: | root="$RUNNER_TEMP/libuv-linux-arm64-musl" diff --git a/internal/crosscompile/c/target.go b/internal/crosscompile/c/target.go index a069126..757cd1b 100644 --- a/internal/crosscompile/c/target.go +++ b/internal/crosscompile/c/target.go @@ -200,15 +200,10 @@ func (c *Target) pkgConfigPatch(commandEnv []string) build.Patch { return build.Patch{} } env := append([]string(nil), commandEnv...) - env = setMissingEnv(env, "PKG_CONFIG_SYSROOT_DIR", c.sysroot) libDirs, _ := envValue(env, "PKG_CONFIG_PATH") - paths := filepath.SplitList(libDirs) - paths = append(paths, - filepath.Join(c.sysroot, "usr", "lib64", "pkgconfig"), - filepath.Join(c.sysroot, "usr", "lib", "pkgconfig"), - filepath.Join(c.sysroot, "usr", "share", "pkgconfig"), - ) - env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) + // Use stores LLAR dependency .pc directories in PKG_CONFIG_PATH. Restrict + // lookup to them without rewriting their absolute installation prefixes. + env = setMissingEnv(env, "PKG_CONFIG_LIBDIR", libDirs) return build.Patch{Env: env} } diff --git a/internal/crosscompile/c/target_test.go b/internal/crosscompile/c/target_test.go index 35af076..0f70541 100644 --- a/internal/crosscompile/c/target_test.go +++ b/internal/crosscompile/c/target_test.go @@ -194,14 +194,11 @@ func TestUseCMake(t *testing.T) { if got, want := patch.AppendArg, []string{"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=" + c.toolchainFile}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } - if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { - t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + if got, ok := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); ok { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want unset", got) } - libDirs, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") - for _, want := range []string{"/deps/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { - if !slices.Contains(filepath.SplitList(libDirs), want) { - t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) - } + if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/deps/lib/pkgconfig" { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want /deps/lib/pkgconfig", got) } patch = c.Use(build.Command{Name: "cmake", Args: []string{"--build", "build"}}) if len(patch.AppendArg) != 0 { @@ -263,14 +260,11 @@ func TestUseAutotools(t *testing.T) { if got, want := patch.AppendArg, []string{"--host=aarch64-linux-gnu"}; !reflect.DeepEqual(got, want) { t.Fatalf("AppendArg = %q, want %q", got, want) } - if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { - t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + if got, ok := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); ok { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want unset", got) } - libDirs, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") - for _, want := range []string{"/deps/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { - if !slices.Contains(filepath.SplitList(libDirs), want) { - t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) - } + if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/deps/lib/pkgconfig" { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want /deps/lib/pkgconfig", got) } patch = c.Use(build.Command{Name: configure, Args: []string{"--host=custom-linux"}}) @@ -283,14 +277,12 @@ func TestUsePkgConfig(t *testing.T) { c := newTestTarget(t) depPaths := strings.Join([]string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig"}, string(os.PathListSeparator)) patch := c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_PATH=" + depPaths}}) - if got, _ := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { - t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q", got) + if got, ok := envValue(patch.Env, "PKG_CONFIG_SYSROOT_DIR"); ok { + t.Fatalf("PKG_CONFIG_SYSROOT_DIR = %q, want unset", got) } got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR") - for _, want := range []string{"/deps/a/lib/pkgconfig", "/deps/b/lib/pkgconfig", filepath.Join("/sdk", "usr", "lib", "pkgconfig")} { - if !slices.Contains(filepath.SplitList(got), want) { - t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, want) - } + if got != depPaths { + t.Fatalf("PKG_CONFIG_LIBDIR = %q, want %q", got, depPaths) } patch = c.Use(build.Command{Name: "pkg-config", Env: []string{"PKG_CONFIG_LIBDIR=/custom"}}) if got, _ := envValue(patch.Env, "PKG_CONFIG_LIBDIR"); got != "/custom" { diff --git a/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/v1.0.0/Consumer_llar.gox b/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/v1.0.0/Consumer_llar.gox new file mode 100644 index 0000000..6d4838c --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/v1.0.0/Consumer_llar.gox @@ -0,0 +1,18 @@ +id "llar/pkgconfig-consumer" + +fromVer "v1.0.0" + +onBuild ctx => { + installDir := ctx.outputDir + + c := cmake.new(ctx.SourceDir, ctx.SourceDir+"/_build", installDir) + c.buildType "Release" + for _, dep := range ctx.Proj.Deps { + c.use ctx.outputDir(dep) + } + c.configure + c.build + c.install + + ctx.setMetadata "" +} diff --git a/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/versions.json b/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/versions.json new file mode 100644 index 0000000..66467b3 --- /dev/null +++ b/testdata/crosscompile-e2e/formulas/llar/pkgconfig-consumer/versions.json @@ -0,0 +1,8 @@ +{ + "path": "llar/pkgconfig-consumer", + "deps": { + "v1.0.0": [ + {"path": "madler/zlib", "version": "v1.3.1"} + ] + } +} diff --git a/testdata/crosscompile-e2e/pkgconfig-consumer/project.cmake b/testdata/crosscompile-e2e/pkgconfig-consumer/project.cmake new file mode 100644 index 0000000..01d6ab1 --- /dev/null +++ b/testdata/crosscompile-e2e/pkgconfig-consumer/project.cmake @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.16) +project(llar_pkgconfig_consumer C) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(ZLIB REQUIRED IMPORTED_TARGET zlib) + +# zlib.pc already names its LLAR output root. A target sysroot prepended to +# that absolute path would make both directories below nonexistent. +foreach(include_dir IN LISTS ZLIB_INCLUDE_DIRS) + if(NOT EXISTS "${include_dir}/zlib.h") + message(FATAL_ERROR "pkg-config returned missing zlib include directory: ${include_dir}") + endif() +endforeach() +foreach(library_dir IN LISTS ZLIB_LIBRARY_DIRS) + if(NOT EXISTS "${library_dir}/libz.a") + message(FATAL_ERROR "pkg-config returned missing zlib library directory: ${library_dir}") + endif() +endforeach() + +add_executable(llar-pkgconfig-consumer main.c) +target_link_libraries(llar-pkgconfig-consumer PRIVATE PkgConfig::ZLIB) + +install(TARGETS llar-pkgconfig-consumer RUNTIME DESTINATION bin) diff --git a/x/autotools/autotools.go b/x/autotools/autotools.go index c2b6001..8356ba6 100644 --- a/x/autotools/autotools.go +++ b/x/autotools/autotools.go @@ -38,17 +38,10 @@ func (a *AutoTools) Sysroot(root string) { appendFlag(key, "--sysroot="+root) appendFlag(key, "-isysroot"+root) } - if _, ok := os.LookupEnv("PKG_CONFIG_SYSROOT_DIR"); !ok { - os.Setenv("PKG_CONFIG_SYSROOT_DIR", root) - } if _, ok := os.LookupEnv("PKG_CONFIG_LIBDIR"); !ok { - paths := filepath.SplitList(os.Getenv("PKG_CONFIG_PATH")) - paths = append(paths, - filepath.Join(root, "usr", "lib64", "pkgconfig"), - filepath.Join(root, "usr", "lib", "pkgconfig"), - filepath.Join(root, "usr", "share", "pkgconfig"), - ) - os.Setenv("PKG_CONFIG_LIBDIR", strings.Join(paths, string(os.PathListSeparator))) + // Use stores LLAR dependency .pc directories in PKG_CONFIG_PATH. Restrict + // lookup to them without rewriting their absolute installation prefixes. + os.Setenv("PKG_CONFIG_LIBDIR", os.Getenv("PKG_CONFIG_PATH")) } } diff --git a/x/autotools/autotools_test.go b/x/autotools/autotools_test.go index b94abe6..ecebb3e 100644 --- a/x/autotools/autotools_test.go +++ b/x/autotools/autotools_test.go @@ -5,7 +5,6 @@ import ( "os/exec" "path/filepath" "runtime" - "slices" "strings" "testing" ) @@ -143,19 +142,11 @@ func TestSysroot(t *testing.T) { t.Errorf("%s = %q, want %q", key, got, want) } } - if got := os.Getenv("PKG_CONFIG_SYSROOT_DIR"); got != "/sdk" { - t.Errorf("PKG_CONFIG_SYSROOT_DIR = %q, want /sdk", got) + if got, ok := os.LookupEnv("PKG_CONFIG_SYSROOT_DIR"); ok { + t.Errorf("PKG_CONFIG_SYSROOT_DIR = %q, want unset", got) } - libDirs := filepath.SplitList(os.Getenv("PKG_CONFIG_LIBDIR")) - for _, want := range []string{ - "/deps/lib/pkgconfig", - filepath.Join("/sdk", "usr", "lib64", "pkgconfig"), - filepath.Join("/sdk", "usr", "lib", "pkgconfig"), - filepath.Join("/sdk", "usr", "share", "pkgconfig"), - } { - if !slices.Contains(libDirs, want) { - t.Errorf("PKG_CONFIG_LIBDIR = %q, want %q", libDirs, want) - } + if got := os.Getenv("PKG_CONFIG_LIBDIR"); got != "/deps/lib/pkgconfig" { + t.Errorf("PKG_CONFIG_LIBDIR = %q, want /deps/lib/pkgconfig", got) } } From 9dd851d4522126737318d9cf5e401b5b56358999 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 19:58:03 +0800 Subject: [PATCH 38/39] fix(autotools): remove unused strings import --- x/autotools/autotools.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x/autotools/autotools.go b/x/autotools/autotools.go index 8356ba6..603bc44 100644 --- a/x/autotools/autotools.go +++ b/x/autotools/autotools.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "github.com/goplus/llar/internal/execbroker" "github.com/goplus/llar/x/pkgconfig" From d7738d32e944525842a242e19659dc49c36b2f83 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Tue, 11 Aug 2026 20:35:41 +0800 Subject: [PATCH 39/39] fix(autotools): defer pkg-config environment to target --- x/autotools/autotools.go | 11 +++-------- x/autotools/autotools_test.go | 11 ++++------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/x/autotools/autotools.go b/x/autotools/autotools.go index 603bc44..8bc45bd 100644 --- a/x/autotools/autotools.go +++ b/x/autotools/autotools.go @@ -29,19 +29,14 @@ func New(sourceDir, buildDir, installDir string) *AutoTools { // Source overrides the source directory. func (a *AutoTools) Source(dir string) { a.sourceDir = dir } -// Sysroot sets the target system root for compiler, linker, and pkg-config -// lookups. Both compiler spellings are supplied so the same Formula works for -// generic and Apple targets. +// Sysroot sets the target system root for the compiler and linker. Both +// compiler spellings are supplied so the same Formula works for generic and +// Apple targets. func (a *AutoTools) Sysroot(root string) { for _, key := range []string{"CPPFLAGS", "CFLAGS", "CXXFLAGS", "LDFLAGS"} { appendFlag(key, "--sysroot="+root) appendFlag(key, "-isysroot"+root) } - if _, ok := os.LookupEnv("PKG_CONFIG_LIBDIR"); !ok { - // Use stores LLAR dependency .pc directories in PKG_CONFIG_PATH. Restrict - // lookup to them without rewriting their absolute installation prefixes. - os.Setenv("PKG_CONFIG_LIBDIR", os.Getenv("PKG_CONFIG_PATH")) - } } // Use configures the process environment so that Autotools, compilers, and diff --git a/x/autotools/autotools_test.go b/x/autotools/autotools_test.go index ecebb3e..0d275b5 100644 --- a/x/autotools/autotools_test.go +++ b/x/autotools/autotools_test.go @@ -132,8 +132,6 @@ func TestSysroot(t *testing.T) { } }) } - t.Setenv("PKG_CONFIG_PATH", "/deps/lib/pkgconfig") - a := New("", "", "") a.Sysroot("/sdk") @@ -142,11 +140,10 @@ func TestSysroot(t *testing.T) { t.Errorf("%s = %q, want %q", key, got, want) } } - if got, ok := os.LookupEnv("PKG_CONFIG_SYSROOT_DIR"); ok { - t.Errorf("PKG_CONFIG_SYSROOT_DIR = %q, want unset", got) - } - if got := os.Getenv("PKG_CONFIG_LIBDIR"); got != "/deps/lib/pkgconfig" { - t.Errorf("PKG_CONFIG_LIBDIR = %q, want /deps/lib/pkgconfig", got) + for _, key := range []string{"PKG_CONFIG_SYSROOT_DIR", "PKG_CONFIG_LIBDIR"} { + if got, ok := os.LookupEnv(key); ok { + t.Errorf("%s = %q, want unset", key, got) + } } }