Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions conf/app.conf
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ casdoorApplication = app-casibase
; -- Outbound proxy (optional) ----
socks5Proxy = 127.0.0.1:10808

; -- Built-in image registry mirror -----------------------------------------
; auto = probe registry-1.docker.io at startup; enable mirrors only when the
; canonical registry is unreachable (default)
; always = always route built-in image pulls through the mirrors
; never = always pull from the canonical registries
imageRegistryMirror = auto

; -- Helm images (optional) -------------------------------------------------
; Set true only when implicit/latest chart images must use IfNotPresent.
helmImplicitLatestPullPolicy = false
Expand All @@ -27,3 +34,5 @@ helmImplicitLatestPullPolicy = false
apiserverPort = 6443
apiserverBind = 127.0.0.1
dataDir = /var/lib/casos
ingressControllerEnabled = true
serviceLBEnabled = true
42 changes: 26 additions & 16 deletions controllers/ingress.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,15 @@ type ingressRule struct {
}

type ingressSummary struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
IngressClass string `json:"ingressClass"`
Rules []ingressRule `json:"rules"`
TLSEnabled bool `json:"tlsEnabled"`
TLSSecretName string `json:"tlsSecretName"`
CreatedAt string `json:"createdAt"`
ResourceVersion string `json:"resourceVersion"`
Namespace string `json:"namespace"`
Name string `json:"name"`
IngressClass string `json:"ingressClass"`
Rules []ingressRule `json:"rules"`
TLSEnabled bool `json:"tlsEnabled"`
TLSSecretName string `json:"tlsSecretName"`
LoadBalancerAddresses []string `json:"loadBalancerAddresses,omitempty"`
CreatedAt string `json:"createdAt"`
ResourceVersion string `json:"resourceVersion"`
}

func toIngressSummary(ing networkingv1.Ingress) ingressSummary {
Expand Down Expand Up @@ -65,15 +66,24 @@ func toIngressSummary(ing networkingv1.Ingress) ingressSummary {
if tlsEnabled {
tlsSecretName = ing.Spec.TLS[0].SecretName
}
loadBalancerAddresses := make([]string, 0, len(ing.Status.LoadBalancer.Ingress))
for _, address := range ing.Status.LoadBalancer.Ingress {
if address.IP != "" {
loadBalancerAddresses = append(loadBalancerAddresses, address.IP)
} else if address.Hostname != "" {
loadBalancerAddresses = append(loadBalancerAddresses, address.Hostname)
}
}
return ingressSummary{
Namespace: ing.Namespace,
Name: ing.Name,
IngressClass: cls,
Rules: rules,
TLSEnabled: tlsEnabled,
TLSSecretName: tlsSecretName,
CreatedAt: ing.CreationTimestamp.UTC().Format("2006-01-02 15:04:05"),
ResourceVersion: ing.ResourceVersion,
Namespace: ing.Namespace,
Name: ing.Name,
IngressClass: cls,
Rules: rules,
TLSEnabled: tlsEnabled,
TLSSecretName: tlsSecretName,
LoadBalancerAddresses: loadBalancerAddresses,
CreatedAt: ing.CreationTimestamp.UTC().Format("2006-01-02 15:04:05"),
ResourceVersion: ing.ResourceVersion,
}
}

Expand Down
13 changes: 10 additions & 3 deletions controllers/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,16 @@ func (c *ApiController) GetWorkerKubeconfig() {
c.ResponseError("generate worker kubeconfig: " + err.Error())
return
}
c.ResponseOk(map[string]string{
resp := map[string]string{
"nodeName": wk.NodeName,
"kubeconfig": wk.Kubeconfig,
"containerdConfig": deploy.GenerateContainerdConfig(cfg.SandboxImage, cfg.Socks5Proxy),
})
"containerdConfig": deploy.GenerateContainerdConfig(cfg.SandboxImage),
}
if cfg.UseRegistryMirror {
// Manually joined workers need the per-registry mirror files too;
// containerdConfig alone only points config_path at an empty certs.d.
resp["dockerHubHostsToml"] = deploy.GenerateDockerHubHostsToml()
resp["k8sRegistryHostsToml"] = deploy.GenerateK8sRegistryHostsToml()
}
c.ResponseOk(resp)
}
42 changes: 26 additions & 16 deletions controllers/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@ type portSummary struct {
}

type serviceSummary struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Type string `json:"type"`
ClusterIP string `json:"clusterIP"`
Selector map[string]string `json:"selector"`
Ports []portSummary `json:"ports"`
CreatedAt string `json:"createdAt"`
ResourceVersion string `json:"resourceVersion"`
Namespace string `json:"namespace"`
Name string `json:"name"`
Type string `json:"type"`
ClusterIP string `json:"clusterIP"`
Selector map[string]string `json:"selector"`
Ports []portSummary `json:"ports"`
LoadBalancerAddresses []string `json:"loadBalancerAddresses,omitempty"`
CreatedAt string `json:"createdAt"`
ResourceVersion string `json:"resourceVersion"`
}

func toSvcSummary(svc corev1.Service) serviceSummary {
Expand All @@ -41,15 +42,24 @@ func toSvcSummary(svc corev1.Service) serviceSummary {
NodePort: p.NodePort,
})
}
loadBalancerAddresses := make([]string, 0, len(svc.Status.LoadBalancer.Ingress))
for _, ingress := range svc.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
loadBalancerAddresses = append(loadBalancerAddresses, ingress.IP)
} else if ingress.Hostname != "" {
loadBalancerAddresses = append(loadBalancerAddresses, ingress.Hostname)
}
}
return serviceSummary{
Namespace: svc.Namespace,
Name: svc.Name,
Type: string(svc.Spec.Type),
ClusterIP: svc.Spec.ClusterIP,
Selector: svc.Spec.Selector,
Ports: ports,
CreatedAt: svc.CreationTimestamp.UTC().Format("2006-01-02 15:04:05"),
ResourceVersion: svc.ResourceVersion,
Namespace: svc.Namespace,
Name: svc.Name,
Type: string(svc.Spec.Type),
ClusterIP: svc.Spec.ClusterIP,
Selector: svc.Spec.Selector,
Ports: ports,
LoadBalancerAddresses: loadBalancerAddresses,
CreatedAt: svc.CreationTimestamp.UTC().Format("2006-01-02 15:04:05"),
ResourceVersion: svc.ResourceVersion,
}
}

Expand Down
28 changes: 13 additions & 15 deletions deploy/containerd_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package deploy

import "fmt"

const generatedRegistryHostsMarker = "# Generated by CasOS"

// GenerateContainerdConfig returns the content for /etc/containerd/config.toml.
// For containerd 2.x the registry mirrors are configured via a hosts-dir;
// use GenerateDockerHubHostsToml / GenerateK8sRegistryHostsToml for those files.
func GenerateContainerdConfig(sandboxImage, socks5Proxy string) string {
base := fmt.Sprintf(`# Generated by CasOS
func GenerateContainerdConfig(sandboxImage string) string {
return fmt.Sprintf(`# Generated by CasOS
version = 2

[plugins.'io.containerd.cri.v1.images']
Expand All @@ -15,34 +17,30 @@ version = 2

[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options]
SystemdCgroup = true
`, sandboxImage)

if socks5Proxy == "" {
return base
}

// In restricted areas: point the CRI image plugin at the hosts-dir so
// per-registry mirrors (hosts.toml files) are picked up automatically.
return base + `
[plugins.'io.containerd.cri.v1.images'.registry]
config_path = '/etc/containerd/certs.d'
`
`, sandboxImage)
}

// GenerateDockerHubHostsToml returns the content for
// /etc/containerd/certs.d/docker.io/hosts.toml in restricted areas.
// /etc/containerd/certs.d/docker.io/hosts.toml. The canonical server remains
// the fallback when the mirror is unavailable.
func GenerateDockerHubHostsToml() string {
return `server = "https://registry-1.docker.io"
return generatedRegistryHostsMarker + `
server = "https://registry-1.docker.io"

[host."https://docker.1ms.run"]
capabilities = ["pull", "resolve"]
`
}

// GenerateK8sRegistryHostsToml returns the content for
// /etc/containerd/certs.d/registry.k8s.io/hosts.toml in restricted areas.
// /etc/containerd/certs.d/registry.k8s.io/hosts.toml. The canonical server
// remains the fallback when the mirror is unavailable.
func GenerateK8sRegistryHostsToml() string {
return `server = "https://registry.k8s.io"
return generatedRegistryHostsMarker + `
server = "https://registry.k8s.io"

[host."https://registry.aliyuncs.com/google_containers"]
capabilities = ["pull", "resolve"]
Expand Down
44 changes: 36 additions & 8 deletions deploy/installer.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ import (

const nodeDeployResolverPath = "/etc/casos-resolv.conf"

const (
dockerHubHostsPath = "/etc/containerd/certs.d/docker.io/hosts.toml"
k8sRegistryHostsPath = "/etc/containerd/certs.d/registry.k8s.io/hosts.toml"
)

type registryMirrorFileRunner interface {
RunRootContext(ctx context.Context, command string) (string, error)
WriteFileContext(ctx context.Context, path, content, mode string) error
}

func (d *NodeDeployer) installNodeBinaries(ctx context.Context, runner *NodeDeploySSHRunner, arch, k8sVersion string) error {
version := k8sVersion
cniVersion := defaultNodeDeployCNIVersion
Expand Down Expand Up @@ -47,16 +57,11 @@ test -f %[1]s`, nodeDeployResolverPath)); err != nil {
}

d.logStep(nodeDeployPhaseConfiguring, "Configuring containerd")
if err := runner.WriteFileContext(ctx, "/etc/containerd/config.toml", GenerateContainerdConfig(d.config.SandboxImage, d.config.Socks5Proxy), "0644"); err != nil {
if err := runner.WriteFileContext(ctx, "/etc/containerd/config.toml", GenerateContainerdConfig(d.config.SandboxImage), "0644"); err != nil {
return fmt.Errorf("write /etc/containerd/config.toml: %w", err)
}
if d.config.Socks5Proxy != "" {
if err := runner.WriteFileContext(ctx, "/etc/containerd/certs.d/docker.io/hosts.toml", GenerateDockerHubHostsToml(), "0644"); err != nil {
return fmt.Errorf("write /etc/containerd/certs.d/docker.io/hosts.toml: %w", err)
}
if err := runner.WriteFileContext(ctx, "/etc/containerd/certs.d/registry.k8s.io/hosts.toml", GenerateK8sRegistryHostsToml(), "0644"); err != nil {
return fmt.Errorf("write /etc/containerd/certs.d/registry.k8s.io/hosts.toml: %w", err)
}
if err := reconcileRegistryMirrorFiles(ctx, runner, d.config.UseRegistryMirror); err != nil {
return err
}
if _, err := runner.RunRootContext(ctx, "systemctl enable --now containerd && systemctl restart containerd"); err != nil {
return fmt.Errorf("start containerd: %w", err)
Expand Down Expand Up @@ -95,6 +100,29 @@ fi`, version, version, arch, version, arch, cniVersion, arch, cniVersion)
return nil
}

func reconcileRegistryMirrorFiles(ctx context.Context, runner registryMirrorFileRunner, enabled bool) error {
if enabled {
if err := runner.WriteFileContext(ctx, dockerHubHostsPath, GenerateDockerHubHostsToml(), "0644"); err != nil {
return fmt.Errorf("write %s: %w", dockerHubHostsPath, err)
}
if err := runner.WriteFileContext(ctx, k8sRegistryHostsPath, GenerateK8sRegistryHostsToml(), "0644"); err != nil {
return fmt.Errorf("write %s: %w", k8sRegistryHostsPath, err)
}
return nil
}

cleanupCommand := fmt.Sprintf(`set -e
for path in %s %s; do
if [ -f "$path" ] && [ "$(sed -n '1p' "$path")" = %s ]; then
rm -f -- "$path"
fi
done`, shellSingleQuote(dockerHubHostsPath), shellSingleQuote(k8sRegistryHostsPath), shellSingleQuote(generatedRegistryHostsMarker))
if _, err := runner.RunRootContext(ctx, cleanupCommand); err != nil {
return fmt.Errorf("remove managed containerd registry hosts: %w", err)
}
return nil
}

func (d *NodeDeployer) writeNodeFiles(ctx context.Context, runner *NodeDeploySSHRunner, nodeName, kubeconfig string) error {
ca, err := extractCertificateAuthority(kubeconfig)
if err != nil {
Expand Down
12 changes: 6 additions & 6 deletions deploy/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,17 +117,17 @@ type Config struct {
ApiserverBind string
ApiserverPort int
SandboxImage string
Socks5Proxy string
UseRegistryMirror bool
GenerateKubeconfig KubeconfigGenerator
}

func ConfigFromServerConfig(cfg server.Config) Config {
return Config{
AdvertiseAddress: cfg.AdvertiseAddress,
ApiserverBind: cfg.ApiserverBind,
ApiserverPort: cfg.ApiserverPort,
SandboxImage: cfg.SandboxImage,
Socks5Proxy: cfg.Socks5Proxy,
AdvertiseAddress: cfg.AdvertiseAddress,
ApiserverBind: cfg.ApiserverBind,
ApiserverPort: cfg.ApiserverPort,
SandboxImage: cfg.SandboxImage,
UseRegistryMirror: cfg.UseRegistryMirror,
GenerateKubeconfig: func(nodeName, apiserverURL string) (*NodeKubeconfig, error) {
wk, err := server.GenerateWorkerKubeconfigForServer(cfg, nodeName, apiserverURL)
if err != nil {
Expand Down
5 changes: 5 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ func main() {
if err := server.Bootstrap(ctx, adminCfg, srvCfg); err != nil {
logs.Warning("bootstrap: %v", err)
}
if srvCfg.ServiceLBEnabled {
if err := server.StartServiceLB(ctx, adminCfg, srvCfg); err != nil {
logs.Warning("start service load balancer: %v", err)
}
}
if err := server.StartScheduler(ctx, srvCfg); err != nil {
logs.Warning("start scheduler: %v", err)
}
Expand Down
12 changes: 10 additions & 2 deletions server/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
"k8s.io/client-go/rest"
)

// Bootstrap creates cluster-wide resources required for worker-node components
// to function correctly. It is idempotent — safe to call on every startup.
// Bootstrap creates CasOS-managed cluster add-ons. It is idempotent and safe
// to call on every startup; individual add-ons can be disabled in config.
func Bootstrap(ctx context.Context, cfg *rest.Config, srvCfg Config) error {
client, err := kubernetes.NewForConfig(cfg)
if err != nil {
Expand All @@ -32,6 +32,14 @@ func Bootstrap(ctx context.Context, cfg *rest.Config, srvCfg Config) error {
errs = append(errs, ensureNodeProxierBinding(ctx, client))
errs = append(errs, ensureFlannel(ctx, client, srvCfg))
errs = append(errs, ensureClusterDNS(ctx, client, srvCfg))
if srvCfg.IngressControllerEnabled {
errs = append(errs, ensureIngressController(ctx, client, srvCfg))
} else {
errs = append(errs, cleanupIngressController(ctx, client))
}
if !srvCfg.ServiceLBEnabled {
errs = append(errs, cleanupServiceLB(ctx, client))
}
if srvCfg.StorageProvisionerEnabled {
errs = append(errs, ensureDefaultStorageProvisioner(ctx, client, srvCfg))
}
Expand Down
Loading
Loading