diff --git a/pkg/admission/validatingadmissionpolicy/validating_admission_policy.go b/pkg/admission/validatingadmissionpolicy/validating_admission_policy.go index 65cbac6796c..ed1061aece6 100644 --- a/pkg/admission/validatingadmissionpolicy/validating_admission_policy.go +++ b/pkg/admission/validatingadmissionpolicy/validating_admission_policy.go @@ -18,6 +18,7 @@ package validatingadmissionpolicy import ( "context" + "fmt" "io" "sync" @@ -272,7 +273,7 @@ func (k *KubeValidatingAdmissionPolicy) getOrCreateDelegate(policyClusterName, t validating.NewValidatingAdmissionPolicyAccessor, validating.NewValidatingAdmissionPolicyBindingAccessor, validating.CompilePolicy, - nil, + &deferredInformerFactory{}, dynamicClient, restMapper, cn, @@ -315,3 +316,15 @@ type stoppableValidatingAdmissionPolicy struct { *validating.Plugin stop func() } + +// deferredInformerFactory is a helper object to minimize changes in our upstream kubernetes fork. +// By returning an error on ForResource, we trigger a fallback to dynamic informers in the generic policy source. +// We need dynamic informer creation to correctly scope the informer based on the target cluster, which can either +// be the policy cluster or the cluster where the APIExport resides. +type deferredInformerFactory struct { + informers.SharedInformerFactory +} + +func (d *deferredInformerFactory) ForResource(resource schema.GroupVersionResource) (informers.GenericInformer, error) { + return nil, fmt.Errorf("deferring creation to dynamic informer. This is expected") +} diff --git a/test/e2e/conformance/validatingadmissionpolicy_test.go b/test/e2e/conformance/validatingadmissionpolicy_test.go index 8d22590f47b..37d716e4318 100644 --- a/test/e2e/conformance/validatingadmissionpolicy_test.go +++ b/test/e2e/conformance/validatingadmissionpolicy_test.go @@ -27,6 +27,7 @@ import ( v1 "k8s.io/api/admission/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -413,3 +414,401 @@ func TestValidatingAdmissionPolicyCrossWorkspaceAPIBinding(t *testing.T) { _, err = cowbyClusterClient.Cluster(targetPath).WildwestV1alpha1().Cowboys("default").Create(ctx, newCowboy("good"), metav1.CreateOptions{}) require.NoError(t, err) } + +// TODO: Currently ValidatingAdmissionPolicy with Params only works with single shard. +// see https://github.com/kcp-dev/kcp/issues/4110 for more infos. Once VAPs with Params +// are supported in multi-shard setups, merge the two tests with the regular ones above; +// See 7f9ccf6217434531cf32438608be879391adfff8. +func TestValidatingAdmissionPolicyInWorkspaceWithParams(t *testing.T) { + t.Parallel() + framework.Suite(t, "control-plane") + + server := kcptesting.SharedKcpServer(t) + + if len(server.ShardNames()) > 1 { + t.Skip("VAP with Params is currently only supported with single shard setups, see code comments") + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + cfg := server.BaseConfig(t) + + orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path(), kcptesting.WithType(core.RootCluster.Path(), "organization")) + ws1Path, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + ws2Path, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + cowbyClusterClient, err := wildwestclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct cowboy client for server") + apiExtensionsClusterClient, err := kcpapiextensionsclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct apiextensions client for server") + + t.Logf("Install the Cowboy resources into logical clusters") + for _, wsPath := range []logicalcluster.Path{ws1Path, ws2Path} { + t.Logf("Bootstrapping Workspace CRDs in logical cluster %s", wsPath) + crdClient := apiExtensionsClusterClient.ApiextensionsV1().CustomResourceDefinitions() + wildwest.Create(t, wsPath, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"}) + } + + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct kcp cluster client for server") + cowboysGVR := schema.GroupVersionResource{Group: "wildwest.dev", Resource: "cowboys", Version: "v1alpha1"} + for _, wsPath := range []logicalcluster.Path{ws1Path, ws2Path} { + kcptesting.WaitForAPIReady(t, kcpClusterClient.Cluster(wsPath).Discovery(), cowboysGVR.GroupVersion()) + } + + t.Logf("Creating a ConfigMap to use as policy parameter in the first workspace") + paramConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "policy-params", + Namespace: "default", + }, + Data: map[string]string{ + "forbidden": "bad", + }, + } + _, err = kubeClusterClient.Cluster(ws1Path).CoreV1().ConfigMaps("default").Create(ctx, paramConfigMap, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create param ConfigMap") + + t.Logf("Installing validating admission policy with paramKind into the first workspace") + policy := &admissionregistrationv1.ValidatingAdmissionPolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "policy-", + }, + Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: ptr.To(admissionregistrationv1.Fail), + ParamKind: &admissionregistrationv1.ParamKind{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + MatchConstraints: &admissionregistrationv1.MatchResources{ + ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{wildwestv1alpha1.SchemeGroupVersion.Group}, + APIVersions: []string{wildwestv1alpha1.SchemeGroupVersion.Version}, + Resources: []string{"cowboys"}, + }, + }, + }, + }, + }, + Validations: []admissionregistrationv1.Validation{{ + Expression: "object.spec.intent != params.data.forbidden", + }}, + }, + } + policy, err = kubeClusterClient.Cluster(ws1Path).AdmissionregistrationV1().ValidatingAdmissionPolicies().Create(ctx, policy, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create ValidatingAdmissionPolicy") + require.Eventually(t, func() bool { + p, err := kubeClusterClient.Cluster(ws1Path).AdmissionregistrationV1().ValidatingAdmissionPolicies().Get(ctx, policy.Name, metav1.GetOptions{}) + if err != nil { + return false + } + + return p.Generation == p.Status.ObservedGeneration && p.Status.TypeChecking != nil && len(p.Status.TypeChecking.ExpressionWarnings) == 0 + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + t.Logf("Installing validating admission policy binding with paramRef into the first workspace") + binding := &admissionregistrationv1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "binding-", + }, + Spec: admissionregistrationv1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: policy.Name, + ParamRef: &admissionregistrationv1.ParamRef{ + Name: "policy-params", + Namespace: "default", + ParameterNotFoundAction: ptr.To(admissionregistrationv1.DenyAction), + }, + ValidationActions: []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}, + }, + } + + _, err = kubeClusterClient.Cluster(ws1Path).AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Create(ctx, binding, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create ValidatingAdmissionPolicyBinding") + + badCowboy := wildwestv1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cowboy-", + }, + Spec: wildwestv1alpha1.CowboySpec{ + Intent: "bad", + }, + } + + goodCowboy := wildwestv1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cowboy-", + }, + Spec: wildwestv1alpha1.CowboySpec{ + Intent: "good", + }, + } + + t.Logf("Verifying that creating bad cowboy resource in first logical cluster is rejected") + require.Eventually(t, func() bool { + _, err := cowbyClusterClient.Cluster(ws1Path).WildwestV1alpha1().Cowboys("default").Create(ctx, &badCowboy, metav1.CreateOptions{}) + if err != nil { + if errors.IsInvalid(err) { + if strings.Contains(err.Error(), "failed expression: object.spec.intent != params.data.forbidden") { + return true + } + } + t.Logf("Unexpected error when trying to create bad cowboy: %s", err) + } + return false + }, wait.ForeverTestTimeout, 1*time.Second) + + t.Logf("Verifying that creating good cowboy resource in first logical cluster succeeds") + _, err = cowbyClusterClient.Cluster(ws1Path).WildwestV1alpha1().Cowboys("default").Create(ctx, &goodCowboy, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Verifying that creating bad cowboy resource in second logical cluster succeeds (policy should not apply here)") + _, err = cowbyClusterClient.Cluster(ws2Path).WildwestV1alpha1().Cowboys("default").Create(ctx, &badCowboy, metav1.CreateOptions{}) + require.NoError(t, err) +} + +func TestValidatingAdmissionPolicyCrossWorkspaceAPIBindingWithParams(t *testing.T) { + t.Parallel() + framework.Suite(t, "control-plane") + + server := kcptesting.SharedKcpServer(t) + + if len(server.ShardNames()) > 1 { + t.Skip("VAP with Params is currently only supported with single shard setups, see code comments") + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + cfg := server.BaseConfig(t) + + orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path(), kcptesting.WithType(core.RootCluster.Path(), "organization")) + sourcePath, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + targetPath, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct kcp cluster client for server") + + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + + cowbyClusterClient, err := wildwestclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct cowboy client for server") + + t.Logf("Install a cowboys APIResourceSchema into workspace %q", sourcePath) + apiResourceSchema := &apisv1alpha1.APIResourceSchema{ + ObjectMeta: metav1.ObjectMeta{ + Name: "today.cowboys.wildwest.dev", + }, + Spec: apisv1alpha1.APIResourceSchemaSpec{ + Group: "wildwest.dev", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "Cowboy", + ListKind: "CowboyList", + Plural: "cowboys", + Singular: "cowboy", + }, + Scope: "Namespaced", + Versions: []apisv1alpha1.APIResourceVersion{ + { + Name: "v1alpha1", + Served: true, + Storage: true, + Schema: runtime.RawExtension{ + Raw: []byte(`{ + "description": "Cowboy is part of the wild west", + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "properties": { + "intent": {"type": "string"} + } + }, + "status": { + "type": "object", + "properties": { + "result": {"type": "string"} + } + } + }, + "type": "object" + }`), + }, + Subresources: apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + }, + }, + }, + } + _, err = kcpClusterClient.Cluster(sourcePath).ApisV1alpha1().APIResourceSchemas().Create(ctx, apiResourceSchema, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create an APIExport for it") + cowboysAPIExport := &apisv1alpha2.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cowboybebop", + }, + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Name: "cowboys", + Group: "wildwest.dev", + Schema: "today.cowboys.wildwest.dev", + Storage: apisv1alpha2.ResourceSchemaStorage{ + CRD: &apisv1alpha2.ResourceSchemaStorageCRD{}, + }, + }, + }, + }, + } + _, err = kcpClusterClient.Cluster(sourcePath).ApisV1alpha2().APIExports().Create(ctx, cowboysAPIExport, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create an APIBinding in workspace %q that points to the cowboybebop export", targetPath) + apiBinding := &apisv1alpha2.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cowboys", + }, + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: sourcePath.String(), + Name: cowboysAPIExport.Name, + }, + }, + }, + } + + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err := kcpClusterClient.Cluster(targetPath).ApisV1alpha2().APIBindings().Create(ctx, apiBinding, metav1.CreateOptions{}) + return err == nil, fmt.Sprintf("Error creating APIBinding: %v", err) + }, wait.ForeverTestTimeout, time.Millisecond*100) + + t.Logf("Ensure cowboys are served in target workspace") + require.Eventually(t, func() bool { + _, err := cowbyClusterClient.Cluster(targetPath).WildwestV1alpha1().Cowboys("default").List(ctx, metav1.ListOptions{}) + return err == nil + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + t.Logf("Creating a ConfigMap to use as policy parameter in the source workspace") + paramConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "policy-params", + Namespace: "default", + }, + Data: map[string]string{ + "forbidden": "bad", + }, + } + _, err = kubeClusterClient.Cluster(sourcePath).CoreV1().ConfigMaps("default").Create(ctx, paramConfigMap, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create param ConfigMap") + + t.Logf("Installing validating admission policy with paramKind into the source workspace") + policy := &admissionregistrationv1.ValidatingAdmissionPolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "policy-", + }, + Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: ptr.To(admissionregistrationv1.Fail), + ParamKind: &admissionregistrationv1.ParamKind{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + MatchConstraints: &admissionregistrationv1.MatchResources{ + ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{ + { + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{wildwestv1alpha1.SchemeGroupVersion.Group}, + APIVersions: []string{wildwestv1alpha1.SchemeGroupVersion.Version}, + Resources: []string{"cowboys"}, + }, + }, + }, + }, + }, + Validations: []admissionregistrationv1.Validation{{ + Expression: "object.spec.intent != params.data.forbidden", + }}, + }, + } + policy, err = kubeClusterClient.Cluster(sourcePath).AdmissionregistrationV1().ValidatingAdmissionPolicies().Create(ctx, policy, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create ValidatingAdmissionPolicy") + require.Eventually(t, func() bool { + p, err := kubeClusterClient.Cluster(sourcePath).AdmissionregistrationV1().ValidatingAdmissionPolicies().Get(ctx, policy.Name, metav1.GetOptions{}) + if err != nil { + return false + } + + return p.Generation == p.Status.ObservedGeneration && p.Status.TypeChecking != nil && len(p.Status.TypeChecking.ExpressionWarnings) == 0 + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + newCowboy := func(intent string) *wildwestv1alpha1.Cowboy { + return &wildwestv1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cowboy-", + }, + Spec: wildwestv1alpha1.CowboySpec{ + Intent: intent, + }, + } + } + + t.Logf("Verifying that creating bad cowboy resource in target workspace succeeds before binding (policy is inactive without binding)") + _, err = cowbyClusterClient.Cluster(targetPath).WildwestV1alpha1().Cowboys("default").Create(ctx, newCowboy("bad"), metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Installing validating admission policy binding with paramRef into the source workspace") + binding := &admissionregistrationv1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "binding-", + }, + Spec: admissionregistrationv1.ValidatingAdmissionPolicyBindingSpec{ + PolicyName: policy.Name, + ParamRef: &admissionregistrationv1.ParamRef{ + Name: "policy-params", + Namespace: "default", + ParameterNotFoundAction: ptr.To(admissionregistrationv1.DenyAction), + }, + ValidationActions: []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}, + }, + } + + _, err = kubeClusterClient.Cluster(sourcePath).AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Create(ctx, binding, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create ValidatingAdmissionPolicyBinding") + + t.Logf("Verifying that creating bad cowboy resource in target workspace is rejected by policy in source workspace") + require.Eventually(t, func() bool { + _, err := cowbyClusterClient.Cluster(targetPath).WildwestV1alpha1().Cowboys("default").Create(ctx, newCowboy("bad"), metav1.CreateOptions{}) + if err != nil { + if errors.IsInvalid(err) { + if strings.Contains(err.Error(), "failed expression: object.spec.intent != params.data.forbidden") { + return true + } + } + t.Logf("Unexpected error when trying to create bad cowboy: %s", err) + } + return false + }, wait.ForeverTestTimeout, 1*time.Second) + + t.Logf("Verifying that creating good cowboy resource in target workspace succeeds") + _, err = cowbyClusterClient.Cluster(targetPath).WildwestV1alpha1().Cowboys("default").Create(ctx, newCowboy("good"), metav1.CreateOptions{}) + require.NoError(t, err) +}