diff --git a/README.adoc b/README.adoc index 7e1117d7..b8b24e87 100644 --- a/README.adoc +++ b/README.adoc @@ -219,6 +219,33 @@ kubectl logs -n project-controller deploy/project-controller -f --tail=200 \ 참고: 컨트롤러가 재기동하면 초기 동기화 과정에서 기존 소유 오브젝트 전체가 `Owned object created` 로 한 번씩 다시 출력됩니다(재기동 시점의 인벤토리이며, 오브젝트가 실제로 새로 생성된 것은 아닙니다). +=== 네임스페이스 allowlist + +클러스터 애드온이나 인프라 오퍼레이터처럼 프로젝트 밖에서 동작하면서도 모든 노드(project managed 노드 포함)에 파드를 실행해야 하는 시스템 컴포넌트를 위해, 네임스페이스 단위 allowlist를 제공합니다. +네임스페이스에 라벨을 붙이는 것만으로 동작하며, 컨트롤러 재시작 없이 즉시 반영됩니다. + +---- +# allowlist 지정 +kubectl label namespace project.ten1010.io/allowlisted=true + +# allowlist 해제 +kubectl label namespace project.ten1010.io/allowlisted- +---- + +allowlist 네임스페이스는 다음과 같이 처리됩니다. + +* *reconcile 제외* — 네임스페이스 라벨/ownerReference 정리, ResourceQuota, 이미지 레지스트리 Secret, RBAC 등 프로젝트 관리 동작을 수행하지 않습니다. +* *eviction 제외* — strict isolation 노드 위에 있어도 해당 네임스페이스의 파드를 삭제하지 않습니다. +* *toleration 주입* — 파드와 워크로드(Deployment 등)에 project managed taint를 허용하는 `Exists` toleration 쌍이 자동 주입되어, 어느 project managed 노드에나 스케줄링될 수 있습니다. +* *webhook 제외* — 워크로드 라벨/사용자 소유권 주입 webhook이 해당 네임스페이스를 건드리지 않습니다. + +주의 사항: + +* 워크로드가 이미 배포된 네임스페이스를 allowlist하면 toleration 주입으로 인해 해당 워크로드들이 일괄 롤아웃됩니다. 가급적 컴포넌트 설치 전에 라벨을 먼저 붙이세요. +* 파드의 toleration은 생성 후 변경할 수 없으므로, allowlist 이전에 생성된 파드는 재생성되어야 toleration을 받습니다. +* allowlist를 해제하면 해당 네임스페이스는 즉시 일반 정책으로 돌아가며, strict isolation 노드 위의 파드는 eviction 대상이 됩니다. +* 오브젝트 라벨 기반 제외(`app.aipub.reconcile-excluded-label-selectors`)는 "컨트롤러가 건드리지 않는" 용도이고, 네임스페이스 allowlist는 여기에 더해 "project managed 노드 접근을 허용하는" 용도입니다. 둘은 함께 사용할 수 있습니다. + === 버그 리포팅 및 개선 사항, 질의 버그를 발견하시거나 개선 사항, 질의가 있다면 link:https://github.com/ten1010-io/project-controller/issues[Github Issue]를 열어주세요. diff --git a/aip-2389-allowlist-scheduling.html b/aip-2389-allowlist-scheduling.html new file mode 100644 index 00000000..98dcfd21 --- /dev/null +++ b/aip-2389-allowlist-scheduling.html @@ -0,0 +1,1170 @@ +AIP-2389 · allowlist 네임스페이스 파드 스케줄 실패 원인 분석 + + + +
+ +
+

AIP-2389 · 원인 분석 · 논의 요청

+

allowlist 네임스페이스 파드가 스케줄되지 않는 원인

+

+ project-controller의 toleration 주입은 정상 동작했다. 파드를 붙잡고 있는 것은 + coaster 스케줄러의 admission webhook이며, 이는 우리 팀 소관 밖이다. +

+
    +
  • project-controller 정상
    • +
  • e2e 차단됨
    • +
  • coaster 팀 논의 필요
    • +
  • 리소스 회계 정책 미정
    • +
+
    +
  • 클러스터 cluster10 / c10-m1
    • +
  • 브랜치 feat/AIP-2389 @ 2a4b3bd
    • +
  • 확인일 2026-08-06
    • +
+
+ + +
+
+

요약

+

세 줄 요약

+
+
    +
  • + allowlist 라벨을 붙인 뒤 DaemonSet 파드 7개가 전부 Pending 상태로 남았다. + 노드 배정이 없고 스케줄러 이벤트조차 없다. +
    • +
  • + 원인은 project-controller가 아니다. toleration 주입·affinity 보존 모두 의도대로 동작했고, + 오히려 주입이 성공했기 때문에 파드가 7개 전 노드에 대해 생성되었다. +
    • +
  • + coaster 스케줄러의 mutating webhook이 schedulerName: coaster를 주입했으나, + 해당 파드는 coaster의 스케줄링 큐에 등록되지 않아 어느 스케줄러도 처리하지 않는 고아 상태가 되었다. +
    • +
+
+ 실사용 영향 +

+ 이 문제는 테스트 환경 특수사항이 아니다. KubeVirt를 설치하면 생성되는 kube-virt 네임스페이스도 + 똑같이 걸린다. coaster webhook의 제외 목록이 하드코딩이라, 앞으로 추가되는 모든 addon 네임스페이스가 + 동일하게 실패한다. AIP-2389 기능은 coaster 쪽 대응 없이는 실제로 동작하지 않는다. +

+
+
+ +
+ + +
+
+

현상

+

관측된 상태

+
+

+ 네임스페이스 allowlist-test에 project.aipub.ten1010.io/allowlisted=true 라벨을 부착하고 + 테스트용 DaemonSet allowlist-probe를 배포한 상태다. +

+ +
+
kubectl -n allowlist-test get ds,pods
+
NAME              DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE
+allowlist-probe   7         7         0       7            0
+
+NAME                    READY   STATUS    RESTARTS   AGE   NODE
+allowlist-probe-2q5cs   0/1     Pending   0          2m19s <none>
+allowlist-probe-dcxjv   0/1     Pending   0          2m19s <none>
+allowlist-probe-f6vhj   0/1     Pending   0          2m20s <none>
+... 7개 전부 동일
+
+ +

+ 결정적 단서는 파드의 status다. PodScheduled condition이 존재하지 않고, + Events도 비어 있다. +

+ +
+
pod status · events
+
$ kubectl -n allowlist-test get pod allowlist-probe-2q5cs -o jsonpath='{.status}'
+{"phase":"Pending","qosClass":"Burstable"}   ← conditions 자체가 없음
+
+$ kubectl -n allowlist-test describe pod allowlist-probe-2q5cs | tail -1
+Events:  <none>
+
+ +

+ 스케줄 실패라면 PodScheduled=False와 FailedScheduling 이벤트가 남는다. + 둘 다 없다는 것은 어떤 스케줄러도 이 파드를 집어가지 않았다는 뜻이다. +

+
+ +
+ + +
+
+

책임 구분

+

어느 쪽이 무엇을 했는가

+
+ +
+
+ project-controller · 우리 소관 +

의도대로 동작했다

+

+ DaemonSet 템플릿과 파드 양쪽에 project-managed taint에 대한 + Exists toleration 쌍이 주입되었고, DaemonSet의 + nodeAffinity(matchFields)와 직접 지정한 control-plane toleration도 보존되었다. +

+
+
+ coaster-scheduler · 타 팀 소관 +

파드를 고아로 만들었다

+

+ webhook이 schedulerName을 coaster로 바꿨지만, + coaster가 파드를 처리하는 데 필요한 큐 라벨은 아무도 붙여주지 않았다. + default-scheduler는 손댈 수 없고 coaster는 무시한다. +

+
+
+ +

project-controller 정상 동작 근거

+ +
+
DaemonSet 템플릿 tolerations — 컨트롤러가 주입한 2개 포함
+
[
+  {"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoSchedule"},  ← 직접 지정, 보존됨
+  {"key":"node-role.kubernetes.io/master","operator":"Exists","effect":"NoSchedule"},         ← 직접 지정, 보존됨
+  {"key":"project.aipub.ten1010.io/project-managed","operator":"Exists","effect":"NoSchedule"},  ← 주입
+  {"key":"project.aipub.ten1010.io/project-managed","operator":"Exists","effect":"NoExecute"}    ← 주입
+]
+
+ +

+ DESIRED=7이라는 숫자 자체가 증거다. 클러스터에는 project-managed 노드 4개와 + control-plane 3개가 있다. 라벨 부착 전이라면 DaemonSet 컨트롤러가 taint를 견디지 못하는 + managed 노드 4개를 건너뛰어 DESIRED=3이어야 한다. + 7로 늘어났다는 것은 toleration 주입이 실제로 반영되었다는 뜻이다. +

+ +
+ + + + + + + + + + + + + + + + + + + +
클러스터 노드 구성
노드역할project-managedisolation-mode
vnode1.pnode12workertruelenient
vnode1.pnode2workertruelenient
vnode4.pnode10workertruelenient
vnode4.pnode7workertruelenient
vnode2.pnode3control-planefalse—
vnode4.pnode12control-planefalse—
vnode6.pnode10control-planefalse—
+
+
+ +
+ + +
+
+

근본 원인

+

실패에 이르는 네 단계

+
+

+ 각 단계는 개별적으로는 정상 동작이다. 문제는 두 시스템이 서로의 존재를 모른 채 + 맞물린 지점에서 발생한다. +

+ +
    +
  1. +

    coaster webhook이 하드코딩된 네임스페이스 목록으로 대상을 가린다

    +

    + MutatingWebhookConfiguration/coaster-scheduler의 namespaceSelector는 + 인프라 네임스페이스 16개를 이름으로 나열해 제외한다. 라벨 기반이 아니라 이름 기반이다. +

    +
    +
    scheduler chart · values.yaml:75-94
    +
    webhook:
+  namespaceSelector:
+    matchExpressions:
+      - key: kubernetes.io/metadata.name
+        operator: NotIn
+        values:
+          - kube-system
+          - kube-public
+          - kube-node-lease
+          - kube-flannel
+          - aipub
+          - coaster
+          - ingress-nginx
+          - metallb
+          - aipub-efk
+          - aipub-linkerd
+          - aipub-monitoring
+          - aipub-promstack
+          - network-operator
+          - project-controller
+          - trident
    +
    +
    2. + +
  2. +

    allowlist-test는 목록에 없으므로 webhook이 걸린다

    +

    + 결과적으로 파드에 schedulerName: coaster가 주입된다. + 동시에 GPU 회피용 preferredDuringScheduling affinity도 함께 주입된다. +

    +
    +
    주입된 파드 spec (발췌)
    +
    schedulerName: coaster
+affinity:
+  nodeAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:      ← coaster 주입
+      - preference:
+          matchExpressions:
+            - key: node-type.coaster.ten1010.io/gpu
+              operator: DoesNotExist
+        weight: 1
+    requiredDuringSchedulingIgnoredDuringExecution:       ← DaemonSet 컨트롤러의 노드 핀, 보존됨
+      nodeSelectorTerms:
+        - matchFields:
+            - key: metadata.name
+              operator: In
+              values: ["vnode4.pnode10.idc1.ten1010.io"]
    +
    +
    3. + +
  3. +

    coaster는 스케줄링 큐에 등록된 파드만 처리한다

    +

    + 큐 등록에 필요한 scheduler.coaster.ten1010.io/queue 라벨은 + aipub-queue-controller가 붙인다. 이 컨트롤러는 AIPub 워크로드 + (Operation / Workspace 등)만 대상으로 하므로, 일반 DaemonSet에는 라벨을 붙이지 않는다. +

    +
    4. + +
  4. +

    큐 라벨이 없는 파드는 영구 방치된다

    +

    + SchedulingQueue/cluster-495ff7d의 status.pods에는 + 프로젝트 네임스페이스(abc)의 파드만 등재되어 있고 우리 파드는 없다. + default-scheduler는 schedulerName이 자기 것이 아니므로 무시하고, + coaster는 큐에 없으므로 무시한다. 이벤트조차 남지 않는 이유가 이것이다. +

    +
    5. +
+
+ +
+ + +
+
+

증거

+

정상 파드와의 차이

+
+

+ 동일하게 schedulerName: coaster를 부여받았지만 정상 스케줄되는 파드 + (abc 네임스페이스, AIPub Operation 워크로드)와 비교하면 차이가 명확하다. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
파드 메타데이터 비교
키정상 파드 (abc)우리 파드 (allowlist-test)
scheduler.coaster.ten1010.io/queuecluster-495ff7d없음
scheduler.coaster.ten1010.io/priority0없음
aipub-queue-controller.aipub.ten1010.io/queuecluster없음
spec.schedulerNamecoastercoaster
SchedulingQueue.status.pods 등재등재됨미등재
네임스페이스 성격프로젝트 (project.aipub.ten1010.io/project=abc)allowlist (프로젝트 아님)
+
+ +
+ 구조적 함의 +

+ coaster의 스케줄링 모델은 "모든 파드는 어떤 프로젝트/큐에 속한다"를 전제한다. + allowlist 네임스페이스는 정의상 프로젝트가 아닌 네임스페이스이므로 이 전제를 위반한다. + 즉 단순한 설정 누락이 아니라 두 모델 간의 개념 충돌이다. +

+
+
+ +
+ + +
+
+

해결 방안

+

선택지와 소관

+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
대응 옵션 비교
방안내용소관평가
A. coaster webhook에 라벨 기반 제외 추가 + namespaceSelector에 + allowlisted DoesNotExist 조건을 추가. + 하드코딩 목록을 건드리지 않고 allowlist 네임스페이스가 자동으로 default-scheduler를 쓰게 된다. + coaster권장
B. 제외 목록에 이름 추가 + addon 네임스페이스를 values.yaml 목록에 하나씩 추가. + coaster확장성 없음. 새 addon마다 차트 수정·재배포 필요
C. project-controller가 schedulerName 강제 + allowlist 파드에 schedulerName: default-scheduler를 주입. + project-controller비권장. coaster webhook이 reinvocationPolicy: IfNeeded라 순서 경쟁 발생, 신뢰 불가
+
+ +

방안 A 구체안

+

+ namespaceSelector는 이미 차트의 values 파라미터로 노출되어 있다. + 차트 템플릿 수정 없이 values 오버라이드만으로 적용 가능하다. +

+ +
+
values 오버라이드 제안
+
webhook:
+  namespaceSelector:
+    matchExpressions:
+      - key: kubernetes.io/metadata.name
+        operator: NotIn
+        values: [ ... 기존 16개 그대로 ... ]
+      - key: project.aipub.ten1010.io/allowlisted
+        operator: DoesNotExist
+
+ +
+ 확인 필요 · 버전 불일치 +

+ 클러스터에 배포된 차트는 scheduler-0.1.1인데, + aipub-installer의 로컬 패키지는 scheduler-0.1.0이다. + 위 values.yaml 내용은 0.1.0 기준이므로, 0.1.1에서 + webhook.namespaceSelector 파라미터가 동일하게 유지되는지 coaster 측 확인이 필요하다. +

+
+ +
+ 참조 위치 — coaster webhook 설정은 어디에 있는가 +
+
세 곳에서 확인 가능
+
# 1. 클러스터 (실제 배포값)
+kubectl get mutatingwebhookconfiguration coaster-scheduler -o yaml
+helm get values coaster-scheduler -n coaster
+   → namespaceSelector 없음 = 차트 기본값 그대로 사용 중
+
+# 2. 차트 패키지 (근본 소스)
+aipub-installer/scripts/ki-cp-node/services/python/5_0_x/
+  stages/01-coaster/packages/scheduler-0.1.0.tgz
+    scheduler/values.yaml:75-94                        ← 하드코딩 목록의 실체
+    scheduler/templates/mutating-webhook-config.yaml   ← 값만 참조
+
+# 3. 아카이브된 원본 YAML (참고)
+aipub-installer/scripts/ki-cp-node/services/python/5_0_x/
+  controllers-archive/coaster-scheduler/yamls/scheduler/mutating-webhook-config.yaml
+
+
+
+ +
+ + +
+
+

별도 논의 · 정책 결정 필요

+

helm으로 배포되는 워크로드의 리소스는 누구 몫인가

+
+ +

+ 스케줄링 문제와 별개로, 더 근본적인 정책 질문이 남는다. + AIPub은 현재 워크로드의 리소스 사용량을 프로젝트 단위로 관리한다. + allowlist 네임스페이스의 워크로드는 실제 노드 자원을 소비하면서도 + 두 개의 회계 레이어를 모두 빠져나간다. +

+ +

현재 구현 상태

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
allowlist 네임스페이스가 빠져나가는 회계 레이어
레이어정상 프로젝트allowlist 네임스페이스근거
AIPub 프로젝트 쿼터
(ResourceQuota)		적용됨미적용ResourceQuotaReconciler
allowlist면 즉시 return
프로젝트 상태 리포팅
(Project.status.quota)		집계됨집계 대상 없음프로젝트 자체가 존재하지 않음
coaster 노드 자원 할당
(NodeResource / TPC)		할당기 경유우회 (방안 A 적용 시)default-scheduler로 직행
+
+ +

+ 즉 allowlist 워크로드는 project-managed 노드의 CPU·메모리를 실제로 점유하지만, + 어느 프로젝트의 쿼터에도 잡히지 않고 coaster의 노드 자원 장부에도 남지 않을 수 있다. +

+ +

두 가지 입장

+ +
+
+ 관점 A +

시스템 리소스로 본다

+

+ KubeVirt·모니터링 에이전트 같은 addon은 클러스터 운영 기반이므로, + 사용자에게 청구하지 않는 인프라 오버헤드로 취급한다. +

+
    +
  • 쿼터 부과 없음 — 현재 구현과 일치
    • +
  • 단, 할당 가능 용량에서 선차감해야 한다. + 그러지 않으면 프로젝트 쿼터 총합이 실제 가용 자원을 초과해 + 과다 약정(oversubscription)이 발생한다
    • +
  • addon이 늘어날수록 사용자에게 보이지 않는 잠식이 커진다 — + 가시성 확보 수단이 별도로 필요
    • +
+
+ +
+ 관점 B +

사용자 워크로드로 본다

+

+ 누군가 helm으로 배포한 것이므로 소유 주체가 있고, 그 주체의 쿼터에서 차감한다. +

+
    +
  • 쿼터·큐·우선순위가 그대로 적용되어 회계 일관성이 유지된다
    • +
  • 단, 그렇다면 allowlist 기능 자체가 불필요하다. + 일반 프로젝트로 만들면 되기 때문 — 기능의 존재 이유와 충돌한다
    • +
  • addon이 쿼터 부족으로 스케줄 실패하면 클러스터 기능 자체가 마비될 수 있다
    • +
+
+
+ +
+ 우선 확인이 필요한 리스크 · GPU +

+ allowlist 워크로드가 GPU를 요청하는 경우가 특히 위험하다. + coaster를 우회하면 GPU 파티셔닝·라이선스 할당기를 거치지 않으므로 + 이중 할당 가능성이 있다. 현재 테스트한 DaemonSet은 CPU/메모리만 요청했기에 + 확인되지 않았다. GPU를 요구하는 addon을 allowlist에 넣을 것인지부터 정해야 한다. +

+
+ +

결정이 필요한 항목

+ +
    +
  1. +
    +

    allowlist 네임스페이스의 자원 소비를 프로젝트 쿼터 산정 시 클러스터 가용 용량에서 차감할 것인가?

    +

    차감하지 않으면 프로젝트에 약속한 쿼터가 실제로는 보장되지 않는다.

    +
    +
    2. +
  2. +
    +

    차감한다면 그 값은 어디서 산출하는가 — 실측 사용량인가, addon 설치 시 선언한 예약량인가?

    +

    실측은 변동성이 크고, 선언값은 관리 주체와 갱신 절차가 필요하다.

    +
    +
    3. +
  3. +
    +

    allowlist 네임스페이스에 GPU 요청을 허용할 것인가?

    +

    허용한다면 coaster 할당기 우회 문제를 반드시 먼저 해결해야 한다.

    +
    +
    4. +
  4. +
    +

    allowlist 라벨을 붙일 수 있는 권한 주체를 어디까지로 제한하는가?

    +

    네임스페이스 update 권한만 있으면 라벨 하나로 노드 격리를 우회할 수 있다. 사실상 클러스터 수준 권한이다.

    +
    +
    5. +
  5. +
    +

    allowlist 워크로드의 자원 사용량을 관리자에게 노출할 필요가 있는가?

    +

    어느 쪽 관점을 택하든, 보이지 않는 소비가 쌓이면 용량 계획이 불가능해진다.

    +
    +
    6. +
+ +
+ 확인 필요 +

+ coaster의 NodeResource가 자신이 스케줄하지 않은 파드 + (default-scheduler가 배치한 파드)의 점유량을 장부에 반영하는지는 이번 조사에서 확인하지 못했다. + 반영한다면 관점 A의 "선차감"이 자동으로 해결될 수 있으므로, coaster 팀에 함께 문의할 항목이다. +

+
+
+ +
+ + +
+
+

부록

+

재현 및 확인

+
+ +
+ 문서 오류 · 라벨 키 +

+ README.adoc과 커밋 메시지에 라벨 키가 + project.ten1010.io/allowlisted로 적혀 있으나, + 실제 코드값은 project.aipub.ten1010.io/allowlisted다 + (LabelConstants.ALLOWLISTED_KEY). + README를 그대로 따라하면 재현되지 않는다. 별도 수정 필요. +

+
+ +
+ 테스트 DaemonSet +
+
allowlist-probe.yaml
+
apiVersion: v1
+kind: Namespace
+metadata:
+  name: allowlist-test
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: allowlist-probe
+  namespace: allowlist-test
+spec:
+  selector:
+    matchLabels:
+      app: allowlist-probe
+  template:
+    metadata:
+      labels:
+        app: allowlist-probe
+    spec:
+      terminationGracePeriodSeconds: 0
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: probe
+          image: busybox:1.36
+          command: ["sh", "-c", "sleep infinity"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+
+
+ +
+ 진단 명령어 +
+
c10-m1 · kubeconfig는 /etc/kubernetes/admin.conf
+
# 비대화형 ssh에서는 kubeconfig가 잡히지 않으므로 명시 필요
+K="sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf"
+
+# allowlist 라벨 부착 (키 주의)
+$K label namespace allowlist-test project.aipub.ten1010.io/allowlisted=true
+
+# 컨트롤러가 toleration을 주입했는지
+$K -n allowlist-test get ds allowlist-probe \
+  -o jsonpath='{.spec.template.spec.tolerations}' | jq
+
+# 파드를 어느 스케줄러가 맡았는지 — 이번 문제의 핵심 지표
+$K -n allowlist-test get pods \
+  -o custom-columns=NAME:.metadata.name,SCHED:.spec.schedulerName,NODE:.spec.nodeName
+
+# coaster 큐 등재 여부
+$K get schedulingqueue cluster-495ff7d -o jsonpath='{.status.pods}' | jq
+
+# coaster webhook 대상 범위
+$K get mutatingwebhookconfiguration coaster-scheduler \
+  -o jsonpath='{.webhooks[0].namespaceSelector}' | jq
+
+
+
+ +
+ AIP-2389 + project-controller · feat/AIP-2389 @ 2a4b3bd + cluster10 + 2026-08-06 +
+ +
diff --git a/kubernetes/controller/project-controller/templates/mutating-webhook-configuration.yaml b/kubernetes/controller/project-controller/templates/mutating-webhook-configuration.yaml index ca9a0070..d85a6e57 100644 --- a/kubernetes/controller/project-controller/templates/mutating-webhook-configuration.yaml +++ b/kubernetes/controller/project-controller/templates/mutating-webhook-configuration.yaml @@ -64,7 +64,7 @@ webhooks: rules: - apiGroups: [ "" ] apiVersions: [ "v1" ] - operations: [ "DELETE" ] + operations: [ "CREATE", "UPDATE", "DELETE" ] resources: [ "namespaces" ] scope: "*" sideEffects: None diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/AipubProperties.java b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/AipubProperties.java index 1b4f7bca..9e7ebbc5 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/AipubProperties.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/AipubProperties.java @@ -28,8 +28,12 @@ public class AipubProperties { /** * project controller가 reconcile/mutating 대상에서 제외할 워크로드의 라벨 셀렉터 목록. * {@code "key=value"}(값 일치) 또는 {@code "key"}(존재만 확인) 형태를 지원하며, 하나라도 - * 매칭되면 제외한다. virt-operator처럼 자체 워크로드를 직접 소유하는 인프라 오퍼레이터와의 - * 소유권 충돌을 막기 위한 용도다. + * 매칭되면 제외한다. 자체 워크로드를 직접 소유하는 인프라 오퍼레이터와의 소유권 충돌을 막기 + * 위한 용도다. + * + *

네임스페이스 전체를 제외하면서 그 파드를 project-managed 노드에도 올리려면, 이 셀렉터 대신 + * 네임스페이스 라벨 기반 allowlist를 쓴다. + * {@link io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver} 참고. */ private List reconcileExcludedLabelSelectors = new ArrayList<>(); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/DomainConfiguration.java b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/DomainConfiguration.java index a661d130..b83aa45d 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/DomainConfiguration.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/DomainConfiguration.java @@ -1,10 +1,13 @@ package io.ten1010.aipub.projectcontroller.configuration; +import io.kubernetes.client.informer.SharedInformerFactory; import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.util.ClientBuilder; import io.kubernetes.client.util.KubeConfig; import io.ten1010.aipub.projectcontroller.domain.k8s.DockerConfigJsonResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; import io.ten1010.aipub.projectcontroller.domain.k8s.util.WorkloadExclusionResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; @@ -46,12 +49,22 @@ public K8sApiProvider k8sApiProvider(ApiClient apiClient) { return new K8sApiProvider(apiClient); } + @Bean + public NamespaceAllowlistResolver namespaceAllowlistResolver( + SharedInformerFactory sharedInformerFactory) { + return new NamespaceAllowlistResolver(sharedInformerFactory + .getExistingSharedIndexInformer(V1Namespace.class) + .getIndexer()); + } + @Bean public ReconciliationService reconciliationService(SubjectResolver subjectResolver, - DockerConfigJsonResolver dockerConfigJsonResolver, AipubProperties aipubProperties) { + DockerConfigJsonResolver dockerConfigJsonResolver, AipubProperties aipubProperties, + NamespaceAllowlistResolver namespaceAllowlistResolver) { return new ReconciliationService(subjectResolver, dockerConfigJsonResolver, aipubProperties.getReservedNamespace(), - new WorkloadExclusionResolver(aipubProperties.getReconcileExcludedLabelSelectors())); + new WorkloadExclusionResolver(aipubProperties.getReconcileExcludedLabelSelectors()), + namespaceAllowlistResolver); } } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java index 33b088c8..c1c4e4a2 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java @@ -7,6 +7,7 @@ import io.ten1010.aipub.projectcontroller.domain.aipubbackend.ArtifactService; import io.ten1010.aipub.projectcontroller.domain.aipubbackend.ImageHubService; import io.ten1010.aipub.projectcontroller.domain.aipubbackend.RepositoryService; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; import io.ten1010.aipub.projectcontroller.mutating.AdmissionReviewController; @@ -89,8 +90,10 @@ public NamespaceReviewHandler namespaceReviewHandler(AipubProperties aipubProper @Bean public ProjectReviewHandler projectReviewHandler(AipubProperties aipubProperties, - SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory) { - return new ProjectReviewHandler(aipubProperties, subjectResolver, sharedInformerFactory); + SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory, + NamespaceAllowlistResolver namespaceAllowlistResolver) { + return new ProjectReviewHandler(aipubProperties, subjectResolver, sharedInformerFactory, + namespaceAllowlistResolver); } @Bean @@ -115,15 +118,16 @@ public UserInfoAnalyzer userInfoAnalyzer(SharedInformerFactory sharedInformerFac @Qualifier("aipubReviewHandlers") public List aipubReviewHandlers( UserInfoAnalyzer userInfoAnalyzer, AipubProperties aipubProperties, - ApiResourceDiscovery apiResourceDiscovery, ApiClient apiClient) { + ApiResourceDiscovery apiResourceDiscovery, ApiClient apiClient, + NamespaceAllowlistResolver namespaceAllowlistResolver) { Set exceptGvkSet = aipubProperties.getAddOwnerExceptGvkList().stream() .map(String::trim) .filter(s -> !s.isEmpty()) .collect(Collectors.toSet()); UserOwnerReviewHandler userOwnerReviewHandler = new UserOwnerReviewHandler( - userInfoAnalyzer, exceptGvkSet); + userInfoAnalyzer, exceptGvkSet, namespaceAllowlistResolver); UserLabelReviewHandler userLabelReviewHandler = new UserLabelReviewHandler( - userInfoAnalyzer, apiResourceDiscovery, apiClient); + userInfoAnalyzer, apiResourceDiscovery, apiClient, namespaceAllowlistResolver); return List.of(userOwnerReviewHandler, userLabelReviewHandler); } @@ -135,8 +139,10 @@ public UserLabelSynchronizer userLabelSynchronizer( @Bean public WorkloadLabelReviewHandler workloadLabelReviewHandler( - ApiResourceDiscovery apiResourceDiscovery, ApiClient apiClient) { - return new WorkloadLabelReviewHandler(apiResourceDiscovery, apiClient); + ApiResourceDiscovery apiResourceDiscovery, ApiClient apiClient, + NamespaceAllowlistResolver namespaceAllowlistResolver) { + return new WorkloadLabelReviewHandler(apiResourceDiscovery, apiClient, + namespaceAllowlistResolver); } @Bean diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NamespaceReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NamespaceReconciler.java index 18371b48..7ae4a866 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NamespaceReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NamespaceReconciler.java @@ -13,6 +13,7 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceNameResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; @@ -20,7 +21,9 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class NamespaceReconciler extends AbstractReconciler { private final KeyResolver keyResolver; @@ -58,6 +61,16 @@ protected Result reconcileInternal(Request request) throws ApiException { Optional projectOpt = Optional.ofNullable( this.projectIndexer.getByKey(projKey)); + // allowlist 네임스페이스는 라벨/ownerReference 정리 대상이 아니다. 동명의 project가 함께 있으면 + // allowlist를 우선하고 경고 로그만 남긴다. + if (namespaceOpt.isPresent() && NamespaceAllowlistResolver.isAllowlisted(namespaceOpt.get())) { + if (projectOpt.isPresent()) { + log.warn("Namespace {} is allowlisted but a project with the same name exists;" + + " skipping reconciliation (allowlist takes precedence)", nsName); + } + return new Result(false); + } + Map reconciledLabels = this.reconciliationService.reconcileNamespaceLabels( namespaceOpt.orElse(null), projectOpt.orElse(null)); List reconciledReferences = this.reconciliationService.reconcileOwnerReferences( diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ImageRegistrySecretReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ImageRegistrySecretReconciler.java index 77ce4c11..76a59e7c 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ImageRegistrySecretReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ImageRegistrySecretReconciler.java @@ -63,6 +63,9 @@ public ImageRegistrySecretReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } Optional projNameOpt = this.secretNameResolver.resolveProjectName(request.getName()); if (projNameOpt.isEmpty()) { return new Result(false); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ResourceQuotaReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ResourceQuotaReconciler.java index 49e9c84d..1e29fdae 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ResourceQuotaReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/namespaced/ResourceQuotaReconciler.java @@ -53,6 +53,9 @@ public ResourceQuotaReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } Optional projNameOpt = this.quotaNameResolver.resolveProjectName(request.getName()); if (projNameOpt.isEmpty()) { return new Result(false); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleBindingReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleBindingReconciler.java index e38f2d75..8fef4efd 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleBindingReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleBindingReconciler.java @@ -64,6 +64,9 @@ public AipubUserRoleBindingReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } String roleKey = new RequestHelper(this.keyResolver).resolveKey(request); Optional roleBindingOpt = Optional.ofNullable( this.RoleBindingIndexer.getByKey(roleKey)); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleReconciler.java index ee3d1923..7df26221 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/aipub/AipubUserRoleReconciler.java @@ -115,6 +115,9 @@ public AipubUserRoleReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } String roleKey = new RequestHelper(this.keyResolver).resolveKey(request); Optional roleOpt = Optional.ofNullable(this.roleIndexer.getByKey(roleKey)); String projKey = this.keyResolver.resolveKey( diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleBindingReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleBindingReconciler.java index f857ae02..2bf98964 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleBindingReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleBindingReconciler.java @@ -14,6 +14,7 @@ import io.ten1010.aipub.projectcontroller.controller.AbstractReconciler; import io.ten1010.aipub.projectcontroller.controller.RequestHelper; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceNameResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectNameAndRole; import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectRoleEnum; @@ -61,6 +62,12 @@ protected Result reconcileInternal(Request request) throws ApiException { String projName = projNameOpt.get().projectName(); ProjectRoleEnum projRoleEnum = projNameOpt.get().projectRoleEnum(); + // allowlist 네임스페이스와 이름이 같은 project는 관리하지 않는다(allowlist 우선). + if (this.reconciliationService.isNamespaceAllowlisted( + new NamespaceNameResolver().resolveNamespaceName(projName))) { + return new Result(false); + } + String roleBindingKey = new RequestHelper(this.keyResolver).resolveKey(request); Optional roleBindingOpt = Optional.ofNullable( this.clusterRoleBindingIndexer.getByKey(roleBindingKey)); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleReconciler.java index f1b8154f..bae121f6 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/ClusterRoleReconciler.java @@ -17,6 +17,7 @@ import io.ten1010.aipub.projectcontroller.controller.RequestHelper; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceNameResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectNameAndRole; import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectRoleEnum; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; @@ -71,6 +72,12 @@ protected Result reconcileInternal(Request request) throws ApiException { String projName = projNameOpt.get().projectName(); ProjectRoleEnum projRoleEnum = projNameOpt.get().projectRoleEnum(); + // allowlist 네임스페이스와 이름이 같은 project는 관리하지 않는다(allowlist 우선). + if (this.reconciliationService.isNamespaceAllowlisted( + new NamespaceNameResolver().resolveNamespaceName(projName))) { + return new Result(false); + } + String roleKey = new RequestHelper(this.keyResolver).resolveKey(request); Optional roleOpt = Optional.ofNullable( this.clusterRoleIndexer.getByKey(roleKey)); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleBindingReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleBindingReconciler.java index cbdb59e0..1652b381 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleBindingReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleBindingReconciler.java @@ -56,6 +56,9 @@ public RoleBindingReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } Optional projNameOpt = this.roleNameResolver.resolveProjectName( request.getName()); if (projNameOpt.isEmpty()) { diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleReconciler.java index 4463a981..d82fdba1 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/rbac/member/RoleReconciler.java @@ -55,6 +55,9 @@ public RoleReconciler( @Override protected Result reconcileInternal(Request request) throws ApiException { + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return new Result(false); + } Optional projNameOpt = this.roleNameResolver.resolveProjectName( request.getName()); if (projNameOpt.isEmpty()) { diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java index 4c40ae16..189c680c 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java @@ -7,6 +7,7 @@ import io.kubernetes.client.openapi.models.V1DaemonSet; import io.kubernetes.client.openapi.models.V1Deployment; import io.kubernetes.client.openapi.models.V1Job; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Node; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1ReplicaSet; @@ -16,6 +17,7 @@ import io.kubernetes.client.openapi.models.V1Secret; import io.kubernetes.client.openapi.models.V1StatefulSet; import io.ten1010.aipub.projectcontroller.domain.k8s.AipubUserRoleNameResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.RoleNameResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1beta1Workspace; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUser; @@ -54,6 +56,12 @@ public BiPredicate projectNamespaceFilter() { || !K8sObjectUtils.getLabels(oldObj).equals(K8sObjectUtils.getLabels(newObj)); } + public BiPredicate namespaceAllowlistLabelFilter() { + return (oldObj, newObj) -> !Objects.equals( + K8sObjectUtils.getLabels(oldObj).get(LabelConstants.ALLOWLISTED_KEY), + K8sObjectUtils.getLabels(newObj).get(LabelConstants.ALLOWLISTED_KEY)); + } + public BiPredicate projectSpecFieldFilter() { return (oldObj, newObj) -> !Objects.equals(oldObj.getSpec(), newObj.getSpec()); } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java index bd7a17d1..afb09a7d 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java @@ -177,6 +177,20 @@ public Function> projectToNamespacedObjects( }; } + public Function> namespaceToNamespacedObjects( + Class objectClass) { + Indexer objectIndexer = this.sharedInformerFactory.getExistingSharedIndexInformer( + objectClass).getIndexer(); + return namespace -> { + List objects = objectIndexer.byIndex( + IndexerConstants.NAMESPACE_TO_OBJECTS_INDEXER_NAME, + K8sObjectUtils.getName(namespace)); + return objects.stream() + .map(e -> new Request(K8sObjectUtils.getNamespace(e), K8sObjectUtils.getName(e))) + .toList(); + }; + } + public Function> aipubUserToClusterRoles() { return user -> { String userName = K8sObjectUtils.getName(user); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java index 9762d66e..1ac10c42 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java @@ -6,6 +6,7 @@ import io.kubernetes.client.extended.controller.reconciler.Request; import io.kubernetes.client.extended.workqueue.WorkQueue; import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Node; import io.kubernetes.client.openapi.models.V1Pod; import io.ten1010.aipub.projectcontroller.controller.ControllerFactory; @@ -48,11 +49,14 @@ public Controller createController() { V1alpha1NodeGroup.class)::hasSynced) .withReadyFunc( this.sharedInformerFactory.getExistingSharedIndexInformer(V1Node.class)::hasSynced) + .withReadyFunc( + this.sharedInformerFactory.getExistingSharedIndexInformer(V1Namespace.class)::hasSynced) .watch(this::createPodWatch) .watch(this::createProjectWatch) .watch(this::createNodeGroupWatch) .watch(this::createNodeWatch) .watch(this::createBoundPodNodeWatch) + .watch(this::createNamespaceWatch) .withReconciler(new PodReconciler( this.sharedInformerFactory, this.k8sApiProvider, @@ -96,4 +100,14 @@ private ControllerWatch createBoundPodNodeWatch(WorkQueue workQ return watch; } + // allowlist 라벨이 제거되면 해당 네임스페이스의 파드를 다시 평가해, 재시작 없이 런타임에 + // eviction 정책이 반영되게 한다. + private ControllerWatch createNamespaceWatch(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, + V1Namespace.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.namespaceAllowlistLabelFilter()); + watch.setRequestBuilder(this.requestBuilderFactory.namespaceToNamespacedObjects(V1Pod.class)); + return watch; + } + } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java index 10ad2b80..a2df8c0e 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java @@ -6,6 +6,7 @@ import io.kubernetes.client.informer.cache.Indexer; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1Node; import io.kubernetes.client.openapi.models.V1Pod; import io.ten1010.aipub.projectcontroller.controller.AbstractReconciler; @@ -13,6 +14,7 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceNameResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; import io.ten1010.aipub.projectcontroller.domain.k8s.util.NodeUtils; @@ -31,6 +33,7 @@ public class PodReconciler extends AbstractReconciler { private final Indexer projectIndexer; private final CoreV1Api coreV1Api; private final PodNodesResolver podNodesResolver; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; public PodReconciler( SharedInformerFactory sharedInformerFactory, @@ -49,6 +52,9 @@ public PodReconciler( .getIndexer(); this.coreV1Api = new CoreV1Api(k8sApiProvider.getApiClient()); this.podNodesResolver = podNodesResolver; + this.namespaceAllowlistResolver = new NamespaceAllowlistResolver(sharedInformerFactory + .getExistingSharedIndexInformer(V1Namespace.class) + .getIndexer()); } @Override @@ -60,6 +66,13 @@ protected Result reconcileInternal(Request request) throws ApiException { } V1Pod pod = podOpt.get(); + // allowlist 네임스페이스의 파드는 project-managed 노드에 있어도 절대 삭제하지 않는다. owner + // kind가 등록된 워크로드 타입이 아닐 때 파드를 삭제하는 UnsupportedControllerException 분기보다 + // 먼저 검사해야 한다. + if (this.namespaceAllowlistResolver.isAllowlisted(K8sObjectUtils.getNamespace(pod))) { + return new Result(false); + } + Optional nodeNameOpt = WorkloadUtils.getNodeName(pod); if (nodeNameOpt.isEmpty()) { return new Result(false); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerFactory.java index e9a48a12..7de97747 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerFactory.java @@ -2,12 +2,19 @@ import io.kubernetes.client.common.KubernetesObject; import io.kubernetes.client.extended.controller.Controller; +import io.kubernetes.client.extended.controller.ControllerWatch; import io.kubernetes.client.extended.controller.builder.ControllerBuilder; import io.kubernetes.client.extended.controller.builder.DefaultControllerBuilder; import io.kubernetes.client.extended.controller.reconciler.Reconciler; +import io.kubernetes.client.extended.controller.reconciler.Request; +import io.kubernetes.client.extended.workqueue.WorkQueue; import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1PodTemplateSpec; import io.ten1010.aipub.projectcontroller.controller.ControllerFactory; +import io.ten1010.aipub.projectcontroller.controller.watch.DefaultControllerWatch; +import io.ten1010.aipub.projectcontroller.controller.watch.OnUpdateFilterFactory; +import io.ten1010.aipub.projectcontroller.controller.watch.RequestBuilderFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sObjectType; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; import java.util.function.Function; @@ -32,9 +39,9 @@ public Controller createController() { configureControllerName(); configureReadyFunc(); configureWatch(); + configureNamespaceAllowlistWatch(); this.builder.withWorkerCount(1); this.builder.withReconciler(createReconciler()); - this.builder.withReconciler(createReconciler()); return this.builder.build(); } @@ -53,6 +60,25 @@ public Controller createController() { protected abstract ControllerObjectReconciler getObjectReconciler(); + /** + * 네임스페이스의 allowlist 라벨이 바뀌면 해당 네임스페이스의 워크로드를 다시 reconcile해, 재시작 + * 없이 런타임에 toleration 주입/제거가 반영되게 한다. + */ + private void configureNamespaceAllowlistWatch() { + this.builder.withReadyFunc(this.sharedInformerFactory + .getExistingSharedIndexInformer(V1Namespace.class)::hasSynced); + this.builder.watch(this::createNamespaceWatch); + } + + private ControllerWatch createNamespaceWatch(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, + V1Namespace.class); + watch.setOnUpdateFilter(new OnUpdateFilterFactory().namespaceAllowlistLabelFilter()); + watch.setRequestBuilder(new RequestBuilderFactory(this.sharedInformerFactory) + .namespaceToNamespacedObjects(getObjectType().objClass())); + return watch; + } + private Reconciler createReconciler() { return new WorkloadControllerReconciler( this.sharedInformerFactory, diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerReconciler.java index 715818bc..1884a02a 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/WorkloadControllerReconciler.java @@ -66,16 +66,30 @@ protected Result reconcileInternal(Request request) throws ApiException { return new Result(false); } KubernetesObject controller = controllerOpt.get(); - if (this.reconciliationService.isExcludedFromReconciliation(controller)) { + if (K8sObjectUtils.findControllerOwnerReference(controller).isPresent()) { return new Result(false); } - if (K8sObjectUtils.findControllerOwnerReference(controller).isPresent()) { + + V1PodTemplateSpec templateSpec = this.podTemplateSpecResolver.apply(controller); + + // allowlist 네임스페이스는 라벨 제외 검사보다 먼저 처리한다. 제외 라벨이 붙은 워크로드라도 + // project-managed 노드에 스케줄되려면 toleration 주입이 필요하기 때문이다(PodReviewHandler와 + // 동일한 순서). project 소속과 무관하게 Exists toleration 쌍을 주입한다(project=null 경로는 + // affinity를 걷어내고 imagePullSecrets는 그대로 둔다). + if (this.reconciliationService.isNamespaceAllowlisted(request.getNamespace())) { + return this.controllerObjectReconciler.reconcileController(controller, + this.reconciliationService.reconcileTolerationsForAllowlistedNamespace(templateSpec), + this.reconciliationService.reconcileNodeSelectorTerms(templateSpec, null), + this.reconciliationService.reconcileImageRegistrySecrets(templateSpec, null)); + } + + if (this.reconciliationService.isExcludedFromReconciliation(controller)) { return new Result(false); } + String projKey = this.keyResolver.resolveKey(request.getNamespace()); V1alpha1Project project = this.projectIndexer.getByKey(projKey); - V1PodTemplateSpec templateSpec = this.podTemplateSpecResolver.apply(controller); List nodeObjects = this.workloadControllerNodesResolver.getNodes(controller); List reconciledTolerations = this.reconciliationService.reconcileTolerations( templateSpec, nodeObjects); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/LabelConstants.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/LabelConstants.java index cde63ae4..51433597 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/LabelConstants.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/LabelConstants.java @@ -12,6 +12,8 @@ public final class LabelConstants { ProjectApiConstants.AIPUB_GROUP + "/" + "userid"; public static final String PROJECT_LABEL_KEY = ProjectApiConstants.PROJECT_GROUP + "/" + "project"; + public static final String ALLOWLISTED_KEY = + ProjectApiConstants.PROJECT_GROUP + "/" + "allowlisted"; public static final String WORKLOAD_NAME_KEY = ProjectApiConstants.AIPUB_GROUP + "/" + "workload-name"; public static final String WORKLOAD_KIND_KEY = diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolver.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolver.java new file mode 100644 index 00000000..0eddaca5 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolver.java @@ -0,0 +1,51 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s; + +import io.kubernetes.client.informer.cache.Indexer; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; +import java.util.Map; +import org.jspecify.annotations.Nullable; + +/** + * 네임스페이스가 project controller 관리에서 allowlist되었는지 판정한다. + * + *

{@code project.ten1010.io/allowlisted: "true"} 라벨이 붙은 네임스페이스를 allowlist로 본다. + * project 네임스페이스 밖에서 동작하면서도 project-managed 노드에 파드를 올려야 하는 시스템 + * 컴포넌트를 위한 것이다. allowlist 네임스페이스는 reconcile과 eviction에서 빠지고, 그 파드에는 + * 어느 노드의 project-managed taint든 견디는 {@code Exists} toleration이 붙는다. + * + *

판정은 네임스페이스 informer 캐시를 직접 읽으므로 라벨 변경이 재시작 없이 런타임에 반영된다. + * 캐시에 없는 네임스페이스는 allowlist가 아닌 것으로 본다(fail-closed). + */ +public class NamespaceAllowlistResolver { + + private final KeyResolver keyResolver; + private final Indexer namespaceIndexer; + + public NamespaceAllowlistResolver(Indexer namespaceIndexer) { + this.keyResolver = new KeyResolver(); + this.namespaceIndexer = namespaceIndexer; + } + + public static boolean isAllowlisted(V1Namespace namespace) { + Map labels = K8sObjectUtils.getLabels(namespace); + String value = labels.get(LabelConstants.ALLOWLISTED_KEY); + if (value == null) { + return false; + } + return value.equalsIgnoreCase(NamespaceAllowlistValueEnum.TRUE.getStr()); + } + + public boolean isAllowlisted(@Nullable String namespaceName) { + if (namespaceName == null || namespaceName.isBlank()) { + return false; + } + V1Namespace namespace = this.namespaceIndexer.getByKey( + this.keyResolver.resolveKey(namespaceName)); + if (namespace == null) { + return false; + } + return isAllowlisted(namespace); + } + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistValueEnum.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistValueEnum.java new file mode 100644 index 00000000..73b42434 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistValueEnum.java @@ -0,0 +1,31 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public enum NamespaceAllowlistValueEnum { + + TRUE("true"), FALSE("false"); + + private static final Map STR_TO_ENUM; + + static { + STR_TO_ENUM = new HashMap<>(); + for (NamespaceAllowlistValueEnum e : NamespaceAllowlistValueEnum.values()) { + STR_TO_ENUM.put(e.getStr(), e); + } + } + + private final String str; + + public static Optional getEnum(String str) { + NamespaceAllowlistValueEnum parsed = STR_TO_ENUM.get(str.toLowerCase()); + return Optional.ofNullable(parsed); + } + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationService.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationService.java index acc90754..c2944936 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationService.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationService.java @@ -71,10 +71,12 @@ public class ReconciliationService { private final ObjectMapper mapper; private final List reservedNamespaces; private final WorkloadExclusionResolver workloadExclusionResolver; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; public ReconciliationService(SubjectResolver subjectResolver, DockerConfigJsonResolver dockerConfigJsonResolver, List reservedNamespaces, - WorkloadExclusionResolver workloadExclusionResolver) { + WorkloadExclusionResolver workloadExclusionResolver, + NamespaceAllowlistResolver namespaceAllowlistResolver) { this.subjectResolver = subjectResolver; this.dockerConfigJsonResolver = dockerConfigJsonResolver; this.roleNameResolver = new RoleNameResolver(); @@ -84,17 +86,27 @@ public ReconciliationService(SubjectResolver subjectResolver, this.mapper = new ObjectMapperFactory().createObjectMapper(); this.reservedNamespaces = reservedNamespaces; this.workloadExclusionResolver = workloadExclusionResolver; + this.namespaceAllowlistResolver = namespaceAllowlistResolver; } /** - * 주어진 워크로드가 reconcile/mutating 대상에서 제외되어야 하는지 판정한다. virt-operator처럼 - * 자체 워크로드를 직접 소유하는 인프라 오퍼레이터의 객체를 project controller가 건드리지 않도록 + * 주어진 워크로드가 reconcile/mutating 대상에서 제외되어야 하는지 판정한다. 자체 워크로드를 직접 + * 소유하는 인프라 오퍼레이터의 객체를 project controller가 건드리지 않도록 * {@code app.aipub.reconcile-excluded-label-selectors} 설정으로 지정한다. */ public boolean isExcludedFromReconciliation(KubernetesObject object) { return this.workloadExclusionResolver.isExcluded(object); } + /** + * 주어진 네임스페이스가 allowlist인지 반환한다. allowlist 네임스페이스는 reconcile과 eviction에서 + * 빠지고, 그 워크로드에는 project-managed 노드에 스케줄될 수 있도록 {@code Exists} toleration이 + * 붙는다. {@link NamespaceAllowlistResolver} 참고. + */ + public boolean isNamespaceAllowlisted(@Nullable String namespaceName) { + return this.namespaceAllowlistResolver.isAllowlisted(namespaceName); + } + private static List removeOwnerReferencesThatReferToProjectKind( List references) { return references.stream() @@ -274,6 +286,38 @@ private static List buildProjectManagedTolerations(String nodeName return List.of(noSchedule, noExecute); } + /** + * allowlist 네임스페이스의 워크로드에 주입할 toleration 쌍을 만든다. project-managed taint의 값은 + * 노드 이름이므로, 모든 project-managed 노드에서 견딜 수 있도록 {@code Exists} operator를 쓴다. + */ + private static List buildAllowlistedNamespaceTolerations() { + V1Toleration noSchedule = new V1TolerationBuilder() + .withKey(TaintConstants.PROJECT_MANAGED_KEY) + .withEffect(TaintConstants.NO_SCHEDULE_EFFECT) + .withOperator("Exists") + .build(); + V1Toleration noExecute = new V1TolerationBuilder() + .withKey(TaintConstants.PROJECT_MANAGED_KEY) + .withEffect(TaintConstants.NO_EXECUTE_EFFECT) + .withOperator("Exists") + .build(); + return List.of(noSchedule, noExecute); + } + + /** + * allowlist 네임스페이스의 toleration을 reconcile한다. 일반 경로({@code reconcileTolerations})와 + * 달리 {@code replaceAllKey*} 재작성은 적용하지 않는다. 워크로드가 스스로 붙인 catch-all + * toleration을 보존해야 하기 때문이다. project-managed toleration만 제거하고 {@code Exists} 쌍을 + * 덧붙이므로 멱등하다. + */ + private static List reconcileTolerationsForAllowlistedNamespace( + List existing) { + List reconciled = new ArrayList<>(removeProjectManagedTolerations(existing)); + reconciled.addAll(buildAllowlistedNamespaceTolerations()); + + return reconciled; + } + private static V1NodeSelectorTerm buildProjectManagedNodeSelectorTerm() { return new V1NodeSelectorTermBuilder() .withMatchExpressions(buildProjectManagedNodeSelectorRequirement()) @@ -1076,6 +1120,17 @@ public List reconcileTolerations(V1PodTemplateSpec existing, return reconcileTolerations(existingTolerations, allowedProjectNodes); } + public List reconcileTolerationsForAllowlistedNamespace(V1Pod existing) { + List existingTolerations = WorkloadUtils.getTolerations(existing); + return reconcileTolerationsForAllowlistedNamespace(existingTolerations); + } + + public List reconcileTolerationsForAllowlistedNamespace( + V1PodTemplateSpec existing) { + List existingTolerations = WorkloadUtils.getTolerations(existing); + return reconcileTolerationsForAllowlistedNamespace(existingTolerations); + } + public List reconcileNodeSelectorTerms(V1Pod existing, @Nullable V1alpha1Project project) { List existingNodeSelectorTerms = WorkloadUtils.getNodeSelectorTerms( diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/util/WorkloadExclusionResolver.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/util/WorkloadExclusionResolver.java index e03645ab..b23ebb83 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/util/WorkloadExclusionResolver.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/util/WorkloadExclusionResolver.java @@ -14,10 +14,10 @@ *

  • {@code "key"} : 라벨 {@code key} 가 존재하면(값 무관) 매칭
    • * * - *

    여러 셀렉터 중 하나라도 매칭되면 제외 대상으로 본다(OR). KubeVirt(virt-operator)처럼 - * 자체 컴포넌트를 직접 소유하는 인프라 오퍼레이터의 워크로드를 project controller가 건드리지 - * 않도록 하기 위한 용도다. 제외 대상 목록은 {@code app.aipub.reconcile-excluded-label-selectors} - * 설정으로 주입되며, 책임과 관리 주체를 project controller로 일원화한다. + *

    여러 셀렉터 중 하나라도 매칭되면 제외 대상으로 본다(OR). 자체 컴포넌트를 직접 소유하는 + * 인프라 오퍼레이터의 워크로드를 project controller가 건드리지 않도록 하기 위한 용도다. 제외 대상 + * 목록은 {@code app.aipub.reconcile-excluded-label-selectors} 설정으로 주입되며, 책임과 관리 주체를 + * project controller로 일원화한다. */ public class WorkloadExclusionResolver { diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/DeploymentReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/DeploymentReviewHandler.java index 6c88e786..5fa1cd6d 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/DeploymentReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/DeploymentReviewHandler.java @@ -47,6 +47,23 @@ public void handle(V1AdmissionReview review) { V1Deployment deployment = getRequestObject(review); + // allowlist 네임스페이스는 라벨 제외 검사보다 먼저 처리한다. 제외 라벨이 붙은 워크로드라도 + // project-managed 노드에 스케줄되려면 toleration 주입이 필요하기 때문이다(PodReviewHandler와 + // 동일한 순서). + if (this.reconciliationService.isNamespaceAllowlisted(review.getRequest().getNamespace())) { + List reconciledTolerations = + this.reconciliationService.reconcileTolerationsForAllowlistedNamespace( + WorkloadUtils.getPodTemplateSpec(deployment)); + JsonPatchBuilder allowlistPatchBuilder = new JsonPatchBuilder(); + allowlistPatchBuilder.addToOperations(new JsonPatchOperationBuilder() + .replace() + .setPath("/spec/template/spec/tolerations") + .setValue(createJsonNode(reconciledTolerations)) + .build()); + V1AdmissionReviewUtils.allow(review, allowlistPatchBuilder.build()); + return; + } + if (this.reconciliationService.isExcludedFromReconciliation(deployment)) { V1AdmissionReviewUtils.allow(review); return; diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandler.java index b3fa27b1..f5619400 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandler.java @@ -7,22 +7,28 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.K8sGroupConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sObjectTypeConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; import io.ten1010.aipub.projectcontroller.mutating.V1AdmissionReviewUtils; import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; import io.ten1010.aipub.projectcontroller.mutating.dto.V1UserInfo; import java.util.Objects; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @Slf4j public class NamespaceReviewHandler extends AbstractReviewHandler { + private static final String OPERATION_CREATE = "CREATE"; + private static final String OPERATION_UPDATE = "UPDATE"; + private final AipubProperties aipubProperties; private final KeyResolver keyResolver; private final SubjectResolver subjectResolver; - private final Indexer namespaceIndexer; + private final Indexer projectIndexer; public NamespaceReviewHandler(AipubProperties aipubProperties, SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory) { @@ -30,8 +36,8 @@ public NamespaceReviewHandler(AipubProperties aipubProperties, SubjectResolver s this.aipubProperties = aipubProperties; this.keyResolver = new KeyResolver(); this.subjectResolver = subjectResolver; - this.namespaceIndexer = sharedInformerFactory - .getExistingSharedIndexInformer(V1Namespace.class) + this.projectIndexer = sharedInformerFactory + .getExistingSharedIndexInformer(V1alpha1Project.class) .getIndexer(); } @@ -40,9 +46,55 @@ public void handle(V1AdmissionReview review) { Objects.requireNonNull(review.getRequest()); Objects.requireNonNull(review.getRequest().getUserInfo()); - V1Namespace namespace = this.namespaceIndexer.getByKey( - this.keyResolver.resolveKey(getNamespaceName(review))); - String namespaceName = K8sObjectUtils.getName(namespace); + String operation = review.getRequest().getOperation(); + if (OPERATION_CREATE.equals(operation) || OPERATION_UPDATE.equals(operation)) { + handleAllowlistLabeling(review); + return; + } + + handleReservedDeletion(review); + } + + /** + * 동명의 project가 아직 있는(종료 중인 경우 포함) 네임스페이스에 allowlist 라벨을 붙이는 것을 + * 거부한다. project 격리가 먼저 부여된 의미라 우선하며, allowlist 네임스페이스 이름으로 project를 + * 만드는 것을 거부하는 {@code ProjectReviewHandler}와 대칭이다. system-admin 예외 없는 hard + * block이다. 종료 중인 project도 거부하는데, 그 네임스페이스는 project 소유(ownerReference)라 + * project와 함께 garbage collection으로 지워지므로, allowlist를 붙여봐야 곧 사라질 네임스페이스에 + * 라벨만 남기기 때문이다. + * + *

    원하는 상태는 informer 캐시가 아니라 요청 객체에서 읽는다. UPDATE 시 캐시에는 patch 이전 + * 네임스페이스가 남아 있어 새로 붙는 라벨을 놓치기 때문이다. + */ + private void handleAllowlistLabeling(V1AdmissionReview review) { + V1Namespace desired = getRequestObject(review); + if (!NamespaceAllowlistResolver.isAllowlisted(desired)) { + V1AdmissionReviewUtils.allow(review); + return; + } + + String namespaceName = K8sObjectUtils.getName(desired); + Optional projectOpt = Optional.ofNullable( + this.projectIndexer.getByKey(this.keyResolver.resolveKey(namespaceName))); + if (projectOpt.isPresent()) { + log.debug("Namespace {} cannot be allowlisted while a project with the same name exists", + namespaceName); + V1AdmissionReviewUtils.reject(review, HttpStatus.CONFLICT.value(), + String.format("%s still has a project of the same name; allowlisting is allowed only" + + " after that project is deleted and the namespace is released", namespaceName)); + return; + } + + V1AdmissionReviewUtils.allow(review); + } + + /** + * reserved 네임스페이스 판정에는 이름만 필요하므로 요청에서 직접 읽는다. informer 캐시를 거치면 + * 캐시에 아직 없는 네임스페이스의 DELETE 요청에서 NPE가 나고, failurePolicy가 Ignore라 가드가 + * 조용히 무력화된다. + */ + private void handleReservedDeletion(V1AdmissionReview review) { + String namespaceName = getNamespaceName(review); if (isReservedName(namespaceName)) { V1UserInfo userInfo = review.getRequest().getUserInfo(); if (userInfo.getGroups() != null && diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/PodReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/PodReviewHandler.java index eb7b24ac..72f2f10d 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/PodReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/PodReviewHandler.java @@ -47,6 +47,22 @@ public void handle(V1AdmissionReview review) { V1Pod pod = getRequestObject(review); + // allowlist 네임스페이스는 라벨 제외 검사보다 먼저 처리한다. 파드의 toleration은 생성 후 바꿀 수 + // 없으므로, 제외 라벨이 붙은 파드라도 project-managed 노드에 스케줄되려면 여기서 toleration을 + // 주입해야 한다. + if (this.reconciliationService.isNamespaceAllowlisted(review.getRequest().getNamespace())) { + List reconciledTolerations = + this.reconciliationService.reconcileTolerationsForAllowlistedNamespace(pod); + JsonPatchBuilder allowlistPatchBuilder = new JsonPatchBuilder(); + allowlistPatchBuilder.addToOperations(new JsonPatchOperationBuilder() + .replace() + .setPath("/spec/tolerations") + .setValue(createJsonNode(reconciledTolerations)) + .build()); + V1AdmissionReviewUtils.allow(review, allowlistPatchBuilder.build()); + return; + } + if (this.reconciliationService.isExcludedFromReconciliation(pod)) { V1AdmissionReviewUtils.allow(review); return; diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandler.java index 6e82b0e5..0d5f4b0f 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandler.java @@ -7,6 +7,7 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.K8sGroupConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sObjectTypeConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectRoleEnum; import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; @@ -30,9 +31,11 @@ public class ProjectReviewHandler extends AbstractReviewHandler private final SubjectResolver subjectResolver; private final KeyResolver keyResolver; private final Indexer projectIndexer; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; public ProjectReviewHandler(AipubProperties aipubProperties, SubjectResolver subjectResolver, - SharedInformerFactory sharedInformerFactory) { + SharedInformerFactory sharedInformerFactory, + NamespaceAllowlistResolver namespaceAllowlistResolver) { super(K8sObjectTypeConstants.PROJECT_V1ALPHA1); this.aipubProperties = aipubProperties; this.keyResolver = new KeyResolver(); @@ -40,6 +43,7 @@ public ProjectReviewHandler(AipubProperties aipubProperties, SubjectResolver sub this.projectIndexer = sharedInformerFactory .getExistingSharedIndexInformer(V1alpha1Project.class) .getIndexer(); + this.namespaceAllowlistResolver = namespaceAllowlistResolver; } @Override @@ -50,21 +54,26 @@ public void handle(V1AdmissionReview review) { V1UserInfo userInfo = review.getRequest().getUserInfo(); V1alpha1Project proj = getRequestObject(review); String projName = K8sObjectUtils.getName(proj); + // reserved 네임스페이스 이름으로 project를 만들면 project 관리(quota, RBAC, secret)가 인프라 + // 네임스페이스로 새어 나간다. 그래서 아래에서 임의의 project 생성이 허용되는 system admin에게도 + // 예외 없이 막는 hard block이다. if (isReservedName(projName)) { - if (userInfo.getGroups() != null && - (userInfo.getGroups().contains(K8sGroupConstants.SYSTEM_MASTERS_GROUP_NAME) || - userInfo.getGroups().contains(K8sGroupConstants.CLUSTER_ADMINS_GROUP_NAME)) && - !userInfo.getGroups().contains(K8sGroupConstants.AIPUB_ADMIN_GROUP_NAME)) { - log.debug("Project name {} is reserved, but allowed for system admin", projName); - V1AdmissionReviewUtils.allow(review); - return; - } log.debug("Project name {} is reserved", projName); V1AdmissionReviewUtils.reject(review, HttpStatus.CONFLICT.value(), String.format("%s is reserved name", projName)); return; } + // allowlist 네임스페이스 이름의 project는 관리하지 않으므로(allowlist 우선), 애매한 상태를 막기 + // 위해 생성을 거부한다. allowlist가 먼저 부여된 의미라 우선하며, system-admin 예외 없는 hard + // block이다. + if (this.namespaceAllowlistResolver.isAllowlisted(projName)) { + log.debug("Project name {} matches allowlisted namespace", projName); + V1AdmissionReviewUtils.reject(review, HttpStatus.CONFLICT.value(), + String.format("%s is allowlisted namespace", projName)); + return; + } + if (userInfo.getGroups() != null && userInfo.getGroups().contains(K8sGroupConstants.SYSTEM_MASTERS_GROUP_NAME) || userInfo.getGroups().contains(K8sGroupConstants.CLUSTER_ADMINS_GROUP_NAME) || diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandler.java index 5c49436d..dfceb5aa 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandler.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.kubernetes.client.openapi.ApiClient; import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUser; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; @@ -35,13 +36,16 @@ public class UserLabelReviewHandler implements ReviewHandler { private final UserInfoAnalyzer userInfoAnalyzer; private final ApiResourceDiscovery apiResourceDiscovery; private final ApiClient k8sApiClient; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; private final ObjectMapper mapper; public UserLabelReviewHandler(UserInfoAnalyzer userInfoAnalyzer, - ApiResourceDiscovery apiResourceDiscovery, ApiClient k8sApiClient) { + ApiResourceDiscovery apiResourceDiscovery, ApiClient k8sApiClient, + NamespaceAllowlistResolver namespaceAllowlistResolver) { this.userInfoAnalyzer = userInfoAnalyzer; this.apiResourceDiscovery = apiResourceDiscovery; this.k8sApiClient = k8sApiClient; + this.namespaceAllowlistResolver = namespaceAllowlistResolver; this.mapper = new ObjectMapperFactory().createObjectMapper(); } @@ -66,6 +70,11 @@ public void handle(V1AdmissionReview review) { Objects.requireNonNull(request.getObject()); Objects.requireNonNull(request.getNamespace()); + if (this.namespaceAllowlistResolver.isAllowlisted(request.getNamespace())) { + V1AdmissionReviewUtils.allowMerging(review); + return; + } + log.debug("UserLabel handle: user={}, namespace={}, operation={}", request.getUserInfo().getUsername(), request.getNamespace(), request.getOperation()); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandler.java index 3624ee22..f2566732 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandler.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.kubernetes.client.openapi.models.V1OwnerReference; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUser; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; @@ -23,11 +24,14 @@ public class UserOwnerReviewHandler implements ReviewHandler { private final UserInfoAnalyzer userInfoAnalyzer; private final Set exceptGvkSet; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; private final ObjectMapper mapper; - public UserOwnerReviewHandler(UserInfoAnalyzer userInfoAnalyzer, Set exceptGvkSet) { + public UserOwnerReviewHandler(UserInfoAnalyzer userInfoAnalyzer, Set exceptGvkSet, + NamespaceAllowlistResolver namespaceAllowlistResolver) { this.userInfoAnalyzer = userInfoAnalyzer; this.exceptGvkSet = exceptGvkSet; + this.namespaceAllowlistResolver = namespaceAllowlistResolver; this.mapper = new ObjectMapperFactory().createObjectMapper(); } @@ -53,6 +57,11 @@ public void handle(V1AdmissionReview review) { Objects.requireNonNull(request.getUserInfo()); Objects.requireNonNull(request.getObject()); + if (this.namespaceAllowlistResolver.isAllowlisted(request.getNamespace())) { + V1AdmissionReviewUtils.allowMerging(review); + return; + } + String group = request.getKind().getGroup() != null ? request.getKind().getGroup() : ""; String gvk = group + "/" + request.getKind().getVersion() + "/" diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandler.java index 2e5ce761..d6480ae9 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandler.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.kubernetes.client.openapi.ApiClient; import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; import io.ten1010.aipub.projectcontroller.mutating.V1AdmissionReviewUtils; import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; @@ -36,13 +37,15 @@ public class WorkloadLabelReviewHandler implements ReviewHandler { private final ApiResourceDiscovery apiResourceDiscovery; private final ApiClient k8sApiClient; + private final NamespaceAllowlistResolver namespaceAllowlistResolver; private final ObjectMapper mapper; // Port of Python: __init__(self, owner_service) public WorkloadLabelReviewHandler(ApiResourceDiscovery apiResourceDiscovery, - ApiClient k8sApiClient) { + ApiClient k8sApiClient, NamespaceAllowlistResolver namespaceAllowlistResolver) { this.apiResourceDiscovery = apiResourceDiscovery; this.k8sApiClient = k8sApiClient; + this.namespaceAllowlistResolver = namespaceAllowlistResolver; this.mapper = new ObjectMapperFactory().createObjectMapper(); } @@ -67,6 +70,11 @@ public void handle(V1AdmissionReview review) { Objects.requireNonNull(request.getObject()); Objects.requireNonNull(request.getNamespace()); + if (this.namespaceAllowlistResolver.isAllowlisted(request.getNamespace())) { + V1AdmissionReviewUtils.allowMerging(review); + return; + } + log.debug("WorkloadLabel handle: namespace={}, operation={}", request.getNamespace(), request.getOperation()); diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolverTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolverTest.java new file mode 100644 index 00000000..ca8fbe77 --- /dev/null +++ b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/NamespaceAllowlistResolverTest.java @@ -0,0 +1,85 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.kubernetes.client.informer.cache.Cache; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class NamespaceAllowlistResolverTest { + + private static V1Namespace namespaceWithLabels(String name, Map labels) { + return new V1Namespace().metadata(new V1ObjectMeta().name(name).labels(labels)); + } + + private static NamespaceAllowlistResolver resolverWith(V1Namespace... namespaces) { + Cache cache = new Cache<>(); + for (V1Namespace namespace : namespaces) { + cache.add(namespace); + } + return new NamespaceAllowlistResolver(cache); + } + + @Test + @DisplayName("allowlisted 라벨 값이 true면 allowlist로 판정한다(대소문자 무관)") + void givenAllowlistedLabelTrue_thenAllowlisted() { + NamespaceAllowlistResolver resolver = resolverWith( + namespaceWithLabels("kubevirt", Map.of(LabelConstants.ALLOWLISTED_KEY, "true")), + namespaceWithLabels("monitoring", Map.of(LabelConstants.ALLOWLISTED_KEY, "TRUE"))); + + assertThat(resolver.isAllowlisted("kubevirt")).isTrue(); + assertThat(resolver.isAllowlisted("monitoring")).isTrue(); + } + + @Test + @DisplayName("allowlisted 라벨 값이 true가 아니면 allowlist가 아니다") + void givenAllowlistedLabelNotTrue_thenNotAllowlisted() { + NamespaceAllowlistResolver resolver = resolverWith( + namespaceWithLabels("ns-false", Map.of(LabelConstants.ALLOWLISTED_KEY, "false")), + namespaceWithLabels("ns-junk", Map.of(LabelConstants.ALLOWLISTED_KEY, "yes"))); + + assertThat(resolver.isAllowlisted("ns-false")).isFalse(); + assertThat(resolver.isAllowlisted("ns-junk")).isFalse(); + } + + @Test + @DisplayName("allowlisted 라벨이 없으면 allowlist가 아니다") + void givenNoAllowlistedLabel_thenNotAllowlisted() { + NamespaceAllowlistResolver resolver = resolverWith( + namespaceWithLabels("plain", Map.of("other-label", "true"))); + + assertThat(resolver.isAllowlisted("plain")).isFalse(); + } + + @Test + @DisplayName("캐시에 없는 네임스페이스는 allowlist가 아니다(fail-closed)") + void givenNamespaceNotInCache_thenNotAllowlisted() { + NamespaceAllowlistResolver resolver = resolverWith(); + + assertThat(resolver.isAllowlisted("unknown")).isFalse(); + } + + @Test + @DisplayName("null이나 빈 이름은 allowlist가 아니다") + void givenNullOrBlankName_thenNotAllowlisted() { + NamespaceAllowlistResolver resolver = resolverWith( + namespaceWithLabels("kubevirt", Map.of(LabelConstants.ALLOWLISTED_KEY, "true"))); + + assertThat(resolver.isAllowlisted((String) null)).isFalse(); + assertThat(resolver.isAllowlisted("")).isFalse(); + assertThat(resolver.isAllowlisted(" ")).isFalse(); + } + + @Test + @DisplayName("static 오버로드는 네임스페이스 오브젝트의 라벨만으로 판정한다") + void givenNamespaceObject_thenStaticCheckWorks() { + assertThat(NamespaceAllowlistResolver.isAllowlisted( + namespaceWithLabels("a", Map.of(LabelConstants.ALLOWLISTED_KEY, "true")))).isTrue(); + assertThat(NamespaceAllowlistResolver.isAllowlisted( + namespaceWithLabels("b", Map.of()))).isFalse(); + } + +} diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceOwnedRulesTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceOwnedRulesTest.java index b6ce07c2..f24213fb 100644 --- a/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceOwnedRulesTest.java +++ b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceOwnedRulesTest.java @@ -39,7 +39,8 @@ public Optional resolve( subjectResolver, project -> java.util.Map.of(), List.of(), - new WorkloadExclusionResolver(List.of())); + new WorkloadExclusionResolver(List.of()), + new NamespaceAllowlistResolver(new io.kubernetes.client.informer.cache.Cache<>())); this.user = new V1alpha1AipubUser(); V1ObjectMeta userMeta = new V1ObjectMeta(); diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceTest.java new file mode 100644 index 00000000..e6bbbcc6 --- /dev/null +++ b/src/test/java/io/ten1010/aipub/projectcontroller/domain/k8s/ReconciliationServiceTest.java @@ -0,0 +1,103 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import io.kubernetes.client.informer.cache.Cache; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1PodTemplateSpec; +import io.kubernetes.client.openapi.models.V1Toleration; +import io.ten1010.aipub.projectcontroller.domain.k8s.util.WorkloadExclusionResolver; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class ReconciliationServiceTest { + + private static ReconciliationService createService() { + return new ReconciliationService( + mock(SubjectResolver.class), + mock(DockerConfigJsonResolver.class), + List.of(), + new WorkloadExclusionResolver(List.of()), + new NamespaceAllowlistResolver(new Cache<>())); + } + + private static V1Pod podWithTolerations(List tolerations) { + return new V1Pod().spec(new V1PodSpec().tolerations(tolerations)); + } + + private static V1Toleration existsToleration(String effect) { + return new V1Toleration() + .key(TaintConstants.PROJECT_MANAGED_KEY) + .operator("Exists") + .effect(effect); + } + + @Test + @DisplayName("allowlist reconcile은 project managed 키의 Exists toleration 쌍을 추가한다") + void givenNoTolerations_whenAllowlistReconcile_thenExistsPairAdded() { + ReconciliationService service = createService(); + + List reconciled = service.reconcileTolerationsForAllowlistedNamespace( + podWithTolerations(null)); + + assertThat(reconciled).containsExactly( + existsToleration(TaintConstants.NO_SCHEDULE_EFFECT), + existsToleration(TaintConstants.NO_EXECUTE_EFFECT)); + } + + @Test + @DisplayName("allowlist reconcile은 기존 toleration을 재작성 없이 그대로 보존한다(catch-all 포함)") + void givenExistingTolerations_whenAllowlistReconcile_thenPreservedVerbatim() { + ReconciliationService service = createService(); + // 일반 경로의 replaceAllKeyAllEffectTolerations가 재작성해버릴 catch-all toleration(key/effect + // 없음)이, allowlist 경로에서는 그대로 보존되어야 한다. + V1Toleration catchAll = new V1Toleration().operator("Exists"); + V1Toleration custom = new V1Toleration() + .key("kubevirt.io/drain").operator("Exists").effect(TaintConstants.NO_SCHEDULE_EFFECT); + + List reconciled = service.reconcileTolerationsForAllowlistedNamespace( + podWithTolerations(new ArrayList<>(List.of(catchAll, custom)))); + + assertThat(reconciled).containsExactly( + catchAll, + custom, + existsToleration(TaintConstants.NO_SCHEDULE_EFFECT), + existsToleration(TaintConstants.NO_EXECUTE_EFFECT)); + } + + @Test + @DisplayName("allowlist reconcile은 기존 per-node Equal toleration을 제거하고 Exists 쌍으로 대체한다") + void givenPerNodeEqualTolerations_whenAllowlistReconcile_thenReplacedByExistsPair() { + ReconciliationService service = createService(); + V1Toleration perNodeEqual = new V1Toleration() + .key(TaintConstants.PROJECT_MANAGED_KEY) + .operator("Equal") + .value("node-1") + .effect(TaintConstants.NO_SCHEDULE_EFFECT); + + List reconciled = service.reconcileTolerationsForAllowlistedNamespace( + podWithTolerations(new ArrayList<>(List.of(perNodeEqual)))); + + assertThat(reconciled).containsExactly( + existsToleration(TaintConstants.NO_SCHEDULE_EFFECT), + existsToleration(TaintConstants.NO_EXECUTE_EFFECT)); + } + + @Test + @DisplayName("allowlist reconcile은 멱등이다(두 번 적용해도 쌍이 중복되지 않는다)") + void givenAlreadyReconciled_whenAllowlistReconcileAgain_thenIdempotent() { + ReconciliationService service = createService(); + V1PodTemplateSpec templateSpec = new V1PodTemplateSpec().spec(new V1PodSpec()); + + List once = service.reconcileTolerationsForAllowlistedNamespace(templateSpec); + templateSpec.getSpec().tolerations(once); + List twice = service.reconcileTolerationsForAllowlistedNamespace(templateSpec); + + assertThat(twice).isEqualTo(once); + } + +} diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandlerTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandlerTest.java new file mode 100644 index 00000000..00edac68 --- /dev/null +++ b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/NamespaceReviewHandlerTest.java @@ -0,0 +1,156 @@ +package io.ten1010.aipub.projectcontroller.mutating.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.common.KubernetesObject; +import io.kubernetes.client.informer.SharedIndexInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Indexer; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.ten1010.aipub.projectcontroller.configuration.AipubProperties; +import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; +import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReviewRequest; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1UserInfo; +import java.time.OffsetDateTime; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class NamespaceReviewHandlerTest { + + private NamespaceReviewHandler handler; + private Indexer mockNamespaceIndexer; + private Indexer mockProjectIndexer; + private ObjectMapper mapper; + + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() { + this.mockNamespaceIndexer = mock(Indexer.class); + this.mockProjectIndexer = mock(Indexer.class); + + SharedInformerFactory factory = mock(SharedInformerFactory.class); + mockInformer(factory, V1Namespace.class, this.mockNamespaceIndexer); + mockInformer(factory, V1alpha1Project.class, this.mockProjectIndexer); + + AipubProperties aipubProperties = new AipubProperties(); + this.handler = new NamespaceReviewHandler( + aipubProperties, mock(SubjectResolver.class), factory); + this.mapper = new ObjectMapperFactory().createObjectMapper(); + } + + @SuppressWarnings("unchecked") + private void mockInformer(SharedInformerFactory factory, + Class clazz, Indexer indexer) { + SharedIndexInformer informer = mock(SharedIndexInformer.class); + when(informer.getIndexer()).thenReturn(indexer); + when(factory.getExistingSharedIndexInformer(clazz)).thenReturn(informer); + } + + private V1Namespace namespace(String name, boolean allowlisted) { + V1ObjectMeta meta = new V1ObjectMeta().name(name); + if (allowlisted) { + meta.labels(Map.of(LabelConstants.ALLOWLISTED_KEY, "true")); + } + return new V1Namespace().metadata(meta); + } + + private V1alpha1Project project(String name, boolean terminating) { + V1ObjectMeta meta = new V1ObjectMeta().name(name); + if (terminating) { + meta.deletionTimestamp(OffsetDateTime.now()); + } + V1alpha1Project project = new V1alpha1Project(); + project.setMetadata(meta); + return project; + } + + private V1AdmissionReview createReview(String operation, V1Namespace desired) { + V1AdmissionReviewRequest request = new V1AdmissionReviewRequest(); + request.setUid("test-uid"); + request.setOperation(operation); + request.setName(desired.getMetadata().getName()); + request.setUserInfo(new V1UserInfo()); + request.setObject(this.mapper.valueToTree(desired)); + + V1AdmissionReview review = new V1AdmissionReview(); + review.setApiVersion("admission.k8s.io/v1"); + review.setKind("AdmissionReview"); + review.setRequest(request); + return review; + } + + @Test + @DisplayName("동명의 활성 project가 있으면 allowlist 라벨 부착(UPDATE)을 거부한다") + void update_allowlistLabelWithActiveProject_rejectsConflict() { + when(this.mockProjectIndexer.getByKey(anyString())).thenReturn(project("proj1", false)); + V1AdmissionReview review = createReview("UPDATE", namespace("proj1", true)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isFalse(); + assertThat(review.getResponse().getStatus().getCode()).isEqualTo(409); + } + + @Test + @DisplayName("동명 project가 없으면 allowlist 라벨 부착을 허용한다") + void update_allowlistLabelWithoutProject_allows() { + V1AdmissionReview review = createReview("UPDATE", namespace("kubevirt", true)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + assertThat(review.getResponse().getPatch()).isNull(); + } + + @Test + @DisplayName("동명 project가 종료 중이어도 allowlist 라벨 부착을 거부한다") + void update_allowlistLabelWithTerminatingProject_rejectsConflict() { + when(this.mockProjectIndexer.getByKey(anyString())).thenReturn(project("proj1", true)); + V1AdmissionReview review = createReview("UPDATE", namespace("proj1", true)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isFalse(); + assertThat(review.getResponse().getStatus().getCode()).isEqualTo(409); + } + + @Test + @DisplayName("allowlist 라벨이 없으면 동명 project가 있어도 허용한다") + void update_withoutAllowlistLabel_allows() { + when(this.mockProjectIndexer.getByKey(anyString())).thenReturn(project("proj1", false)); + V1AdmissionReview review = createReview("UPDATE", namespace("proj1", false)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + } + + @Test + @DisplayName("CREATE로 allowlist 라벨이 붙은 채 동명 활성 project가 있으면 거부한다") + void create_allowlistLabelWithActiveProject_rejectsConflict() { + when(this.mockProjectIndexer.getByKey(anyString())).thenReturn(project("proj1", false)); + V1AdmissionReview review = createReview("CREATE", namespace("proj1", true)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isFalse(); + assertThat(review.getResponse().getStatus().getCode()).isEqualTo(409); + } + +} diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandlerTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandlerTest.java new file mode 100644 index 00000000..f7733136 --- /dev/null +++ b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/ProjectReviewHandlerTest.java @@ -0,0 +1,124 @@ +package io.ten1010.aipub.projectcontroller.mutating.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.common.KubernetesObject; +import io.kubernetes.client.informer.SharedIndexInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Cache; +import io.kubernetes.client.informer.cache.Indexer; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.ten1010.aipub.projectcontroller.configuration.AipubProperties; +import io.ten1010.aipub.projectcontroller.domain.k8s.K8sGroupConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; +import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReviewRequest; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1UserInfo; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class ProjectReviewHandlerTest { + + private ProjectReviewHandler handler; + private ObjectMapper mapper; + + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() { + Indexer mockProjectIndexer = mock(Indexer.class); + SharedInformerFactory factory = mock(SharedInformerFactory.class); + mockInformer(factory, V1alpha1Project.class, mockProjectIndexer); + + Cache namespaceCache = new Cache<>(); + namespaceCache.add(new V1Namespace().metadata(new V1ObjectMeta() + .name("kubevirt") + .labels(Map.of(LabelConstants.ALLOWLISTED_KEY, "true")))); + + AipubProperties aipubProperties = new AipubProperties(); + aipubProperties.setReservedNamespace(List.of("aipub")); + + this.handler = new ProjectReviewHandler( + aipubProperties, mock(SubjectResolver.class), factory, + new NamespaceAllowlistResolver(namespaceCache)); + this.mapper = new ObjectMapperFactory().createObjectMapper(); + } + + @SuppressWarnings("unchecked") + private void mockInformer(SharedInformerFactory factory, + Class clazz, Indexer indexer) { + SharedIndexInformer informer = mock(SharedIndexInformer.class); + when(informer.getIndexer()).thenReturn(indexer); + when(factory.getExistingSharedIndexInformer(clazz)).thenReturn(informer); + } + + private V1AdmissionReview createReview(String projectName, List groups) { + V1alpha1Project project = new V1alpha1Project(); + project.setMetadata(new V1ObjectMeta().name(projectName)); + + V1UserInfo userInfo = new V1UserInfo(); + userInfo.setUsername("admin"); + userInfo.setGroups(groups); + + V1AdmissionReviewRequest request = new V1AdmissionReviewRequest(); + request.setUid("test-uid"); + request.setOperation("CREATE"); + request.setUserInfo(userInfo); + request.setObject(this.mapper.valueToTree(project)); + + V1AdmissionReview review = new V1AdmissionReview(); + review.setApiVersion("admission.k8s.io/v1"); + review.setKind("AdmissionReview"); + review.setRequest(request); + return review; + } + + @Test + @DisplayName("system admin이라도 reserved 이름 project 생성은 거부한다") + void reservedName_systemAdmin_rejectsConflict() { + V1AdmissionReview review = createReview("aipub", + List.of(K8sGroupConstants.SYSTEM_MASTERS_GROUP_NAME)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isFalse(); + assertThat(review.getResponse().getStatus().getCode()).isEqualTo(409); + } + + @Test + @DisplayName("system admin이라도 allowlist 네임스페이스 이름 project 생성은 거부한다") + void allowlistedName_systemAdmin_rejectsConflict() { + V1AdmissionReview review = createReview("kubevirt", + List.of(K8sGroupConstants.SYSTEM_MASTERS_GROUP_NAME)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isFalse(); + assertThat(review.getResponse().getStatus().getCode()).isEqualTo(409); + } + + @Test + @DisplayName("reserved/allowlist가 아닌 이름은 system admin에게 허용한다") + void ordinaryName_systemAdmin_allows() { + V1AdmissionReview review = createReview("team-alpha", + List.of(K8sGroupConstants.SYSTEM_MASTERS_GROUP_NAME)); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + } + +} diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandlerTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandlerTest.java index f1bdae5c..c9d27cbb 100644 --- a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandlerTest.java +++ b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserLabelReviewHandlerTest.java @@ -7,8 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.kubernetes.client.informer.cache.Cache; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUser; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUserSpec; @@ -31,7 +33,8 @@ void setUp() { this.mockAnalyzer = mock(UserInfoAnalyzer.class); ApiResourceDiscovery mockDiscovery = mock(ApiResourceDiscovery.class); ApiClient mockApiClient = mock(ApiClient.class); - this.handler = new UserLabelReviewHandler(this.mockAnalyzer, mockDiscovery, mockApiClient); + this.handler = new UserLabelReviewHandler(this.mockAnalyzer, mockDiscovery, mockApiClient, + new NamespaceAllowlistResolver(new Cache<>())); this.mapper = new ObjectMapperFactory().createObjectMapper(); } diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandlerTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandlerTest.java index ab8bbf9a..70011c00 100644 --- a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandlerTest.java +++ b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/UserOwnerReviewHandlerTest.java @@ -7,7 +7,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.informer.cache.Cache; +import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1AipubUser; import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; @@ -15,6 +19,7 @@ import io.ten1010.aipub.projectcontroller.mutating.dto.V1Kind; import io.ten1010.aipub.projectcontroller.mutating.dto.V1UserInfo; import java.util.List; +import java.util.Map; import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -30,7 +35,8 @@ void setUp() { this.mockAnalyzer = mock(UserInfoAnalyzer.class); this.handler = new UserOwnerReviewHandler( this.mockAnalyzer, - Set.of("aipub.ten1010.io/v1alpha1/Commit")); + Set.of("aipub.ten1010.io/v1alpha1/Commit"), + new NamespaceAllowlistResolver(new Cache<>())); this.mapper = new ObjectMapperFactory().createObjectMapper(); } @@ -155,6 +161,23 @@ void handle_memberUser_addsOwnerReference() { assertThat(review.getResponse().getPatchType()).isEqualTo("JSONPatch"); } + @Test + void handle_allowlistedNamespace_allowsWithoutPatch() { + Cache namespaceCache = new Cache<>(); + namespaceCache.add(new V1Namespace().metadata(new V1ObjectMeta() + .name("kubevirt") + .labels(Map.of(LabelConstants.ALLOWLISTED_KEY, "true")))); + UserOwnerReviewHandler allowlistAwareHandler = new UserOwnerReviewHandler( + this.mockAnalyzer, Set.of(), new NamespaceAllowlistResolver(namespaceCache)); + V1AdmissionReview review = createReview("CREATE", "kubevirt", "apps", "v1", "Deployment"); + + allowlistAwareHandler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + assertThat(review.getResponse().getPatch()).isNull(); + } + @Test void handle_analyzerThrows_rejectsWithServerError() { V1AdmissionReview review = createReview("CREATE", "default", "apps", "v1", "Deployment"); diff --git a/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandlerTest.java b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandlerTest.java new file mode 100644 index 00000000..b17fe8e4 --- /dev/null +++ b/src/test/java/io/ten1010/aipub/projectcontroller/mutating/service/WorkloadLabelReviewHandlerTest.java @@ -0,0 +1,82 @@ +package io.ten1010.aipub.projectcontroller.mutating.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.informer.cache.Cache; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.models.V1Namespace; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.ten1010.aipub.projectcontroller.domain.k8s.LabelConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceAllowlistResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.ObjectMapperFactory; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReviewRequest; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class WorkloadLabelReviewHandlerTest { + + private WorkloadLabelReviewHandler handler; + private ApiResourceDiscovery mockDiscovery; + private ApiClient mockApiClient; + private ObjectMapper mapper; + + @BeforeEach + void setUp() { + this.mockDiscovery = mock(ApiResourceDiscovery.class); + this.mockApiClient = mock(ApiClient.class); + Cache namespaceCache = new Cache<>(); + namespaceCache.add(new V1Namespace().metadata(new V1ObjectMeta() + .name("kubevirt") + .labels(Map.of(LabelConstants.ALLOWLISTED_KEY, "true")))); + this.handler = new WorkloadLabelReviewHandler(this.mockDiscovery, this.mockApiClient, + new NamespaceAllowlistResolver(namespaceCache)); + this.mapper = new ObjectMapperFactory().createObjectMapper(); + } + + private V1AdmissionReview createReview(String namespace) { + V1AdmissionReviewRequest request = new V1AdmissionReviewRequest(); + request.setUid("test-uid"); + request.setOperation("CREATE"); + request.setNamespace(namespace); + request.setObject(this.mapper.createObjectNode().putObject("metadata").objectNode()); + + V1AdmissionReview review = new V1AdmissionReview(); + review.setApiVersion("admission.k8s.io/v1"); + review.setKind("AdmissionReview"); + review.setRequest(request); + + return review; + } + + @Test + @DisplayName("allowlist 네임스페이스의 요청은 owner 조회 없이 무패치 허용한다") + void handle_allowlistedNamespace_allowsWithoutPatch() { + V1AdmissionReview review = createReview("kubevirt"); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + assertThat(review.getResponse().getPatch()).isNull(); + verifyNoInteractions(this.mockDiscovery, this.mockApiClient); + } + + @Test + @DisplayName("allowlist가 아닌 네임스페이스에서 owner가 없으면 무패치 허용한다") + void handle_nonAllowlistedNamespaceWithoutOwner_allowsWithoutPatch() { + V1AdmissionReview review = createReview("default"); + + this.handler.handle(review); + + assertThat(review.getResponse()).isNotNull(); + assertThat(review.getResponse().getAllowed()).isTrue(); + assertThat(review.getResponse().getPatch()).isNull(); + } + +}