diff --git a/conf/app.conf b/conf/app.conf index d146190..4f2068c 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -19,6 +19,13 @@ casdoorApplication = app-casibase ; -- Outbound proxy (optional) ---- socks5Proxy = 127.0.0.1:10808 +; -- Built-in image registry mirror ----------------------------------------- +; auto = probe registry-1.docker.io at startup; enable mirrors only when the +; canonical registry is unreachable (default) +; always = always route built-in image pulls through the mirrors +; never = always pull from the canonical registries +imageRegistryMirror = auto + ; -- Helm images (optional) ------------------------------------------------- ; Set true only when implicit/latest chart images must use IfNotPresent. helmImplicitLatestPullPolicy = false diff --git a/controllers/node.go b/controllers/node.go index c3bee89..f91290d 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -184,9 +184,16 @@ func (c *ApiController) GetWorkerKubeconfig() { c.ResponseError("generate worker kubeconfig: " + err.Error()) return } - c.ResponseOk(map[string]string{ + resp := map[string]string{ "nodeName": wk.NodeName, "kubeconfig": wk.Kubeconfig, - "containerdConfig": deploy.GenerateContainerdConfig(cfg.SandboxImage, cfg.Socks5Proxy), - }) + "containerdConfig": deploy.GenerateContainerdConfig(cfg.SandboxImage), + } + if cfg.UseRegistryMirror { + // Manually joined workers need the per-registry mirror files too; + // containerdConfig alone only points config_path at an empty certs.d. + resp["dockerHubHostsToml"] = deploy.GenerateDockerHubHostsToml() + resp["k8sRegistryHostsToml"] = deploy.GenerateK8sRegistryHostsToml() + } + c.ResponseOk(resp) } diff --git a/deploy/containerd_config.go b/deploy/containerd_config.go index fbbb1fd..5864711 100644 --- a/deploy/containerd_config.go +++ b/deploy/containerd_config.go @@ -2,11 +2,13 @@ package deploy import "fmt" +const generatedRegistryHostsMarker = "# Generated by CasOS" + // GenerateContainerdConfig returns the content for /etc/containerd/config.toml. // For containerd 2.x the registry mirrors are configured via a hosts-dir; // use GenerateDockerHubHostsToml / GenerateK8sRegistryHostsToml for those files. -func GenerateContainerdConfig(sandboxImage, socks5Proxy string) string { - base := fmt.Sprintf(`# Generated by CasOS +func GenerateContainerdConfig(sandboxImage string) string { + return fmt.Sprintf(`# Generated by CasOS version = 2 [plugins.'io.containerd.cri.v1.images'] @@ -15,24 +17,18 @@ version = 2 [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options] SystemdCgroup = true -`, sandboxImage) - - if socks5Proxy == "" { - return base - } - // In restricted areas: point the CRI image plugin at the hosts-dir so - // per-registry mirrors (hosts.toml files) are picked up automatically. - return base + ` [plugins.'io.containerd.cri.v1.images'.registry] config_path = '/etc/containerd/certs.d' -` +`, sandboxImage) } // GenerateDockerHubHostsToml returns the content for -// /etc/containerd/certs.d/docker.io/hosts.toml in restricted areas. +// /etc/containerd/certs.d/docker.io/hosts.toml. The canonical server remains +// the fallback when the mirror is unavailable. func GenerateDockerHubHostsToml() string { - return `server = "https://registry-1.docker.io" + return generatedRegistryHostsMarker + ` +server = "https://registry-1.docker.io" [host."https://docker.1ms.run"] capabilities = ["pull", "resolve"] @@ -40,9 +36,11 @@ func GenerateDockerHubHostsToml() string { } // GenerateK8sRegistryHostsToml returns the content for -// /etc/containerd/certs.d/registry.k8s.io/hosts.toml in restricted areas. +// /etc/containerd/certs.d/registry.k8s.io/hosts.toml. The canonical server +// remains the fallback when the mirror is unavailable. func GenerateK8sRegistryHostsToml() string { - return `server = "https://registry.k8s.io" + return generatedRegistryHostsMarker + ` +server = "https://registry.k8s.io" [host."https://registry.aliyuncs.com/google_containers"] capabilities = ["pull", "resolve"] diff --git a/deploy/installer.go b/deploy/installer.go index 48c5991..eefe683 100644 --- a/deploy/installer.go +++ b/deploy/installer.go @@ -7,6 +7,16 @@ import ( const nodeDeployResolverPath = "/etc/casos-resolv.conf" +const ( + dockerHubHostsPath = "/etc/containerd/certs.d/docker.io/hosts.toml" + k8sRegistryHostsPath = "/etc/containerd/certs.d/registry.k8s.io/hosts.toml" +) + +type registryMirrorFileRunner interface { + RunRootContext(ctx context.Context, command string) (string, error) + WriteFileContext(ctx context.Context, path, content, mode string) error +} + func (d *NodeDeployer) installNodeBinaries(ctx context.Context, runner *NodeDeploySSHRunner, arch, k8sVersion string) error { version := k8sVersion cniVersion := defaultNodeDeployCNIVersion @@ -47,16 +57,11 @@ test -f %[1]s`, nodeDeployResolverPath)); err != nil { } d.logStep(nodeDeployPhaseConfiguring, "Configuring containerd") - if err := runner.WriteFileContext(ctx, "/etc/containerd/config.toml", GenerateContainerdConfig(d.config.SandboxImage, d.config.Socks5Proxy), "0644"); err != nil { + if err := runner.WriteFileContext(ctx, "/etc/containerd/config.toml", GenerateContainerdConfig(d.config.SandboxImage), "0644"); err != nil { return fmt.Errorf("write /etc/containerd/config.toml: %w", err) } - if d.config.Socks5Proxy != "" { - if err := runner.WriteFileContext(ctx, "/etc/containerd/certs.d/docker.io/hosts.toml", GenerateDockerHubHostsToml(), "0644"); err != nil { - return fmt.Errorf("write /etc/containerd/certs.d/docker.io/hosts.toml: %w", err) - } - if err := runner.WriteFileContext(ctx, "/etc/containerd/certs.d/registry.k8s.io/hosts.toml", GenerateK8sRegistryHostsToml(), "0644"); err != nil { - return fmt.Errorf("write /etc/containerd/certs.d/registry.k8s.io/hosts.toml: %w", err) - } + if err := reconcileRegistryMirrorFiles(ctx, runner, d.config.UseRegistryMirror); err != nil { + return err } if _, err := runner.RunRootContext(ctx, "systemctl enable --now containerd && systemctl restart containerd"); err != nil { return fmt.Errorf("start containerd: %w", err) @@ -95,6 +100,29 @@ fi`, version, version, arch, version, arch, cniVersion, arch, cniVersion) return nil } +func reconcileRegistryMirrorFiles(ctx context.Context, runner registryMirrorFileRunner, enabled bool) error { + if enabled { + if err := runner.WriteFileContext(ctx, dockerHubHostsPath, GenerateDockerHubHostsToml(), "0644"); err != nil { + return fmt.Errorf("write %s: %w", dockerHubHostsPath, err) + } + if err := runner.WriteFileContext(ctx, k8sRegistryHostsPath, GenerateK8sRegistryHostsToml(), "0644"); err != nil { + return fmt.Errorf("write %s: %w", k8sRegistryHostsPath, err) + } + return nil + } + + cleanupCommand := fmt.Sprintf(`set -e +for path in %s %s; do + if [ -f "$path" ] && [ "$(sed -n '1p' "$path")" = %s ]; then + rm -f -- "$path" + fi +done`, shellSingleQuote(dockerHubHostsPath), shellSingleQuote(k8sRegistryHostsPath), shellSingleQuote(generatedRegistryHostsMarker)) + if _, err := runner.RunRootContext(ctx, cleanupCommand); err != nil { + return fmt.Errorf("remove managed containerd registry hosts: %w", err) + } + return nil +} + func (d *NodeDeployer) writeNodeFiles(ctx context.Context, runner *NodeDeploySSHRunner, nodeName, kubeconfig string) error { ca, err := extractCertificateAuthority(kubeconfig) if err != nil { diff --git a/deploy/types.go b/deploy/types.go index 1026fc8..4ae379a 100644 --- a/deploy/types.go +++ b/deploy/types.go @@ -117,17 +117,17 @@ type Config struct { ApiserverBind string ApiserverPort int SandboxImage string - Socks5Proxy string + UseRegistryMirror bool GenerateKubeconfig KubeconfigGenerator } func ConfigFromServerConfig(cfg server.Config) Config { return Config{ - AdvertiseAddress: cfg.AdvertiseAddress, - ApiserverBind: cfg.ApiserverBind, - ApiserverPort: cfg.ApiserverPort, - SandboxImage: cfg.SandboxImage, - Socks5Proxy: cfg.Socks5Proxy, + AdvertiseAddress: cfg.AdvertiseAddress, + ApiserverBind: cfg.ApiserverBind, + ApiserverPort: cfg.ApiserverPort, + SandboxImage: cfg.SandboxImage, + UseRegistryMirror: cfg.UseRegistryMirror, GenerateKubeconfig: func(nodeName, apiserverURL string) (*NodeKubeconfig, error) { wk, err := server.GenerateWorkerKubeconfigForServer(cfg, nodeName, apiserverURL) if err != nil { diff --git a/server/config.go b/server/config.go index ad819ab..f430b40 100644 --- a/server/config.go +++ b/server/config.go @@ -17,13 +17,13 @@ type Config struct { WebhookPort int // HTTPS port for the Casbin admission webhook server DSN string // MySQL DSN forwarded to kine SandboxImage string // containerd sandbox (pause) image, empty = upstream default - Socks5Proxy string // outbound socks5 proxy, e.g. 127.0.0.1:10808 CoreDNSImage string // CoreDNS image used by the built-in DNS bootstrap LocalPathProvisionerImage string // local-path-provisioner controller image LocalPathHelperImage string // helper pod image used by local-path-provisioner FlannelImage string // Flannel daemon image used by the built-in network bootstrap FlannelCNIPluginImage string // Flannel CNI plugin image installed on worker hosts StorageProvisionerEnabled bool // install the built-in local-path provisioner for local clusters + UseRegistryMirror bool // route built-in image pulls through the registry mirrors (resolved from imageRegistryMirror) } // ConfigFromAppConf reads server config from the beego app.conf. @@ -48,24 +48,20 @@ func ConfigFromAppConf() (Config, error) { webhookPort := conf.GetConfigIntDefault("webhookPort", 9443) - socks5Proxy := conf.GetConfigString("socks5Proxy") - - sandboxImage := conf.GetConfigString("sandboxImage") - if sandboxImage == "" { - if socks5Proxy != "" { - sandboxImage = "registry.aliyuncs.com/google_containers/pause:3.10.1" - } else { - sandboxImage = "registry.k8s.io/pause:3.10.1" - } - } + sandboxImage := conf.GetConfigStringDefault("sandboxImage", "registry.k8s.io/pause:3.10.1") storageProvisionerEnabled := conf.GetConfigBoolDefault("storageProvisionerEnabled", true) - coreDNSImage := conf.GetConfigStringDefault("coreDNSImage", "docker.1ms.run/coredns/coredns:1.12.4") - localPathProvisionerImage := conf.GetConfigStringDefault("localPathProvisionerImage", "docker.1ms.run/rancher/local-path-provisioner:v0.0.32") - localPathHelperImage := conf.GetConfigStringDefault("localPathHelperImage", "docker.1ms.run/library/busybox:1.37.0") + coreDNSImage := conf.GetConfigStringDefault("coreDNSImage", "docker.io/coredns/coredns:1.12.4") + localPathProvisionerImage := conf.GetConfigStringDefault("localPathProvisionerImage", "docker.io/rancher/local-path-provisioner:v0.0.32") + localPathHelperImage := conf.GetConfigStringDefault("localPathHelperImage", "docker.io/library/busybox:1.37.0") flannelImage := conf.GetConfigStringDefault("flannelImage", defaultFlannelImage) flannelCNIPluginImage := conf.GetConfigStringDefault("flannelCNIPluginImage", defaultFlannelCNIPluginImage) + useRegistryMirror, err := resolveRegistryMirror() + if err != nil { + return Config{}, err + } + return Config{ DataDir: dataDir, ApiserverBind: bind, @@ -74,13 +70,13 @@ func ConfigFromAppConf() (Config, error) { WebhookPort: webhookPort, DSN: dsn, SandboxImage: sandboxImage, - Socks5Proxy: socks5Proxy, CoreDNSImage: coreDNSImage, LocalPathProvisionerImage: localPathProvisionerImage, LocalPathHelperImage: localPathHelperImage, FlannelImage: flannelImage, FlannelCNIPluginImage: flannelCNIPluginImage, StorageProvisionerEnabled: storageProvisionerEnabled, + UseRegistryMirror: useRegistryMirror, }, nil } diff --git a/server/flannel_bootstrap.go b/server/flannel_bootstrap.go index 5e1fe7e..97d0285 100644 --- a/server/flannel_bootstrap.go +++ b/server/flannel_bootstrap.go @@ -19,8 +19,8 @@ import ( ) const ( - defaultFlannelImage = "docker.1ms.run/flannel/flannel:v0.27.4" - defaultFlannelCNIPluginImage = "docker.1ms.run/flannel/flannel-cni-plugin:v1.8.0-flannel1" + defaultFlannelImage = "docker.io/flannel/flannel:v0.27.4" + defaultFlannelCNIPluginImage = "docker.io/flannel/flannel-cni-plugin:v1.8.0-flannel1" flannelNamespace = "kube-flannel" flannelServiceAccount = "flannel" flannelConfigMap = "kube-flannel-cfg" diff --git a/server/registry_mirror.go b/server/registry_mirror.go new file mode 100644 index 0000000..31ec335 --- /dev/null +++ b/server/registry_mirror.go @@ -0,0 +1,75 @@ +package server + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/casosorg/casos/conf" +) + +// imageRegistryMirror modes accepted in app.conf. +const ( + registryMirrorModeAuto = "auto" + registryMirrorModeAlways = "always" + registryMirrorModeNever = "never" +) + +// registryProbeURL is probed once (mode "auto") to decide whether the canonical +// registries are reachable. Any HTTP response, including the 401 that Docker +// Hub returns for anonymous requests, counts as reachable; only a connection +// failure or timeout marks the environment as restricted. +const ( + registryProbeURL = "https://registry-1.docker.io/v2/" + registryProbeTimeout = 4 * time.Second +) + +var ( + registryProbeOnce sync.Once + registryProbeRestricted bool +) + +// resolveRegistryMirror reads imageRegistryMirror from app.conf and decides +// whether built-in image pulls should be routed through the configured +// registry mirrors. Mode "auto" (the default) probes the canonical registry +// directly instead of guessing from timezone, locale, or IP geolocation; the +// result is cached for the lifetime of the process. An explicit +// "always"/"never" skips the probe entirely. +func resolveRegistryMirror() (bool, error) { + mode := strings.ToLower(strings.TrimSpace(conf.GetConfigStringDefault("imageRegistryMirror", registryMirrorModeAuto))) + switch mode { + case registryMirrorModeAlways: + return true, nil + case registryMirrorModeNever: + return false, nil + case registryMirrorModeAuto: + return canonicalRegistryRestricted(), nil + default: + return false, fmt.Errorf("invalid imageRegistryMirror %q in app.conf: expected auto, always, or never", mode) + } +} + +// canonicalRegistryRestricted reports whether the canonical Docker registry is +// unreachable from this host. The probe runs at most once per process. +func canonicalRegistryRestricted() bool { + registryProbeOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), registryProbeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodHead, registryProbeURL, nil) + if err != nil { + registryProbeRestricted = false + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + registryProbeRestricted = true + return + } + resp.Body.Close() + registryProbeRestricted = false + }) + return registryProbeRestricted +}