diff --git a/RELEASE.md b/RELEASE.md index fc8053cd87..04f6cc03ee 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -45,6 +45,7 @@ The following steps must be done by one of the [Gateway API maintainers][gateway in the upcoming steps. - Use `git` to cherry-pick all relevant PRs into your branch. - Update `pkg/consts/consts.go` with the new semver tag and any updates to the API review URL. +- Update regex spec.validations.expression in `config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml` to match older versions. - Run the following command `BASE_REF=vmajor.minor.patch make generate` which will update generated docs with the correct version info. (Note that you can't test with these YAMLs yet as they contain references to elements which wont @@ -63,6 +64,7 @@ The following steps must be done by one of the [Gateway API maintainers][gateway - Cut a `release-major.minor` branch that we can tag things in as needed. - Check out the `release-major.minor` release branch locally. - Update `pkg/consts/consts.go` with the new semver tag and any updates to the API review URL. +- Update regex spec.validations.expression in `config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml` to match older versions. - Run the following command `BASE_REF=vmajor.minor.patch make generate` which will update generated docs with the correct version info. (Note that you can't test with these YAMLs yet as they contain references to elements which wont diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c21d60cb74..4f840eae3c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,3 +5,4 @@ resources: - standard/gateway.networking.k8s.io_httproutes.yaml - standard/gateway.networking.k8s.io_referencegrants.yaml - standard/gateway.networking.k8s.io_backendtlspolicies.yaml +- standard/gateway.networking.k8s.io_vap_safeupgrades.yaml diff --git a/config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml b/config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml new file mode 100644 index 0000000000..90ead661b8 --- /dev/null +++ b/config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml @@ -0,0 +1,46 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + annotations: + gateway.networking.k8s.io/bundle-version: v1.4.0 + gateway.networking.k8s.io/channel: standard + name: "safe-upgrades.gateway.networking.k8s.io" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["apiextensions.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["*"] + validations: + - expression: "object.spec.group != 'gateway.networking.k8s.io' || ( + has(object.metadata.annotations) && object.metadata.annotations.exists(k, k == 'gateway.networking.k8s.io/channel') && + object.metadata.annotations['gateway.networking.k8s.io/channel'] == 'standard' )" + message: "Installing experimental CRDs on top of standard channel CRDs is prohibited by default. Uninstall ValidatingAdmissionPolicy gateway-api-safe-upgrades.gateway.networking.k8s.io to install experimental CRDs on top of standard channel CRDs." + reason: Invalid + - expression: "object.spec.group != 'gateway.networking.k8s.io' || + (has(object.metadata.annotations) && object.metadata.annotations.exists(k, k == 'gateway.networking.k8s.io/bundle-version') && + !matches(object.metadata.annotations['gateway.networking.k8s.io/bundle-version'], 'v1.[0-3].\\\\d+') && + !matches(object.metadata.annotations['gateway.networking.k8s.io/bundle-version'], 'v0'))" #TODO Kubernetes 1.37: Migrate to kubernetes semver library + message: "Installing CRDs with version before v1.4.0 is prohibited by default. Uninstall ValidatingAdmissionPolicy gateway-api-safe-upgrades.gateway.networking.k8s.io to install older versions." + reason: Invalid + +--- + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + annotations: + gateway.networking.k8s.io/bundle-version: v1.4.0 + gateway.networking.k8s.io/channel: standard + name: safe-upgrades.gateway.networking.k8s.io +spec: + policyName: safe-upgrades.gateway.networking.k8s.io + validationActions: [Deny] + matchResources: + resourceRules: + - apiGroups: ["apiextensions.k8s.io"] + apiVersions: ["v1"] + resources: ["customresourcedefinitions"] + operations: ["CREATE", "UPDATE"] diff --git a/hack/delete-crds.sh b/hack/delete-crds.sh index 098a6e8791..adb3cfabfa 100755 --- a/hack/delete-crds.sh +++ b/hack/delete-crds.sh @@ -26,3 +26,6 @@ for TYPE in ${RESOURCES}; do kubectl delete ${TYPE} --all kubectl delete crd/${TYPE} done + +kubectl delete ValidatingAdmissionPolicy/safe-upgrades.gateway.networking.k8s.io +kubectl delete ValidatingAdmissionPolicyBinding/safe-upgrades.gateway.networking.k8s.io diff --git a/pkg/test/crd/crd_test.go b/pkg/test/crd/crd_test.go index 9becc1aaf4..a1c9fed261 100644 --- a/pkg/test/crd/crd_test.go +++ b/pkg/test/crd/crd_test.go @@ -17,13 +17,17 @@ limitations under the License. package crd_test import ( + "bytes" "fmt" + "io" "io/fs" "os" "os/exec" "path/filepath" + "regexp" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -70,40 +74,105 @@ func TestCRDValidation(t *testing.T) { crdChannel = requestedCRDChannel } - t.Run("should be able to start test environment", func(_ *testing.T) { - testEnv = &envtest.Environment{ - Scheme: scheme, - ErrorIfCRDPathMissing: true, - DownloadBinaryAssets: true, - DownloadBinaryAssetsVersion: k8sVersion, - CRDInstallOptions: envtest.CRDInstallOptions{ - Paths: []string{ - filepath.Join("..", "..", "..", "config", "crd", crdChannel), - }, - CleanUpAfterUse: true, + testEnv = &envtest.Environment{ + Scheme: scheme, + ErrorIfCRDPathMissing: true, + DownloadBinaryAssets: true, + DownloadBinaryAssetsVersion: k8sVersion, + CRDInstallOptions: envtest.CRDInstallOptions{ + Paths: []string{ + filepath.Join("..", "..", "..", "config", "crd", crdChannel), }, - } - - _, err = testEnv.Start() - if err != nil { - panic(fmt.Sprintf("Error initializing test environment: %v", err)) - } - }) + CleanUpAfterUse: true, + }, + } + _, err = testEnv.Start() t.Cleanup(func() { require.NoError(t, testEnv.Stop()) }) + require.NoError(t, err, "Error initializing test environment") + + // Setup kubectl and kubeconfig + kubectlLocation = testEnv.ControlPlane.KubectlPath + require.NotEmpty(t, kubectlLocation, "Error initializing Kubectl") - t.Run("should be able to set kubectl and kubeconfig and connect to the cluster", func(t *testing.T) { - kubectlLocation = testEnv.ControlPlane.KubectlPath - require.NotEmpty(t, kubectlLocation) + kubeconfigLocation = fmt.Sprintf("%s/kubeconfig", filepath.Dir(kubectlLocation)) + err = os.WriteFile(kubeconfigLocation, testEnv.KubeConfig, 0o600) + require.NoError(t, err, "Error initializing kubeconfig") - kubeconfigLocation = fmt.Sprintf("%s/kubeconfig", filepath.Dir(kubectlLocation)) - require.NoError(t, os.WriteFile(kubeconfigLocation, testEnv.KubeConfig, 0o600)) + apiResources, err := executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, []string{"api-resources"}) + require.NoError(t, err) + require.Contains(t, apiResources, "gateway.networking.k8s.io/v1") + + t.Run("safeupgrades VAP should validate correctly", func(t *testing.T) { + if crdChannel == "experimental" { + t.Skipf("skipping safeupgrades VAP") + } - apiResources, err := executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, []string{"api-resources"}) + output, err := executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"apply", "--server-side", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "standard", "gateway.networking.k8s.io_vap_safeupgrades.yaml")}) require.NoError(t, err) - require.Contains(t, apiResources, "gateway.networking.k8s.io/v1") + + // Even though --wait is applied I noticed a race condition that causes tests to fail. + time.Sleep(time.Second) + + t.Run("should be able to install standard CRDs", func(t *testing.T) { + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"apply", "--server-side", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "standard")}) + require.NoError(t, err) + }) + + t.Run("should not be able to install k8s.io experimental CRDs", func(t *testing.T) { + t.Cleanup(func() { + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"delete", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "experimental", "*.k8s.*")}) + }) + + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"apply", "--server-side", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "experimental", "*.k8s.*")}) + require.Error(t, err) + assert.Contains(t, output, "Error from server (Invalid)") + assert.Contains(t, output, "ValidatingAdmissionPolicy 'safe-upgrades.gateway.networking.k8s.io' with binding 'safe-upgrades.gateway.networking.k8s.io' denied request") + + // Check that --api-group has no invalid crd's + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, []string{"describe", "CustomResourceDefinition"}) + require.NoError(t, err) + assert.NotContains(t, output, "gateway.networking.k8s.io/channel: experimental", "output contains 'gateway.networking.k8s.io/channel: experimental'") + }) + + t.Run("should be able to install x-k8s.io experimental CRDs", func(t *testing.T) { + t.Cleanup(func() { + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"delete", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "experimental", "*.x-k8s.*")}) + }) + + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"apply", "--server-side", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "experimental", "*.x-k8s.*")}) + require.NoError(t, err) + }) + + t.Run("should not be able to install CRDs with an older version", func(t *testing.T) { + t.Cleanup(func() { + output, err = executeKubectlCommand(t, kubectlLocation, kubeconfigLocation, + []string{"delete", "--wait", "-f", filepath.Join("..", "..", "..", "config", "crd", "standard", "gateway.networking.k8s.io_httproutes.yaml")}) + }) + + // Read test crd into []byte + httpCrd, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "crd", "standard", "gateway.networking.k8s.io_httproutes.yaml")) + require.NoError(t, err) + + // do replace on gateway.networking.k8s.io/bundle-version: v1.4.0 + re := regexp.MustCompile(`gateway\.networking\.k8s\.io\/bundle-version: \S*`) + sub := []byte("gateway.networking.k8s.io/bundle-version: v1.3.0") + oldCrd := re.ReplaceAll(httpCrd, sub) + + // supply crd to stdin of cmd and kubectl apply -f - + output, err = executeKubectlCommandStdin(t, kubectlLocation, kubeconfigLocation, bytes.NewReader(oldCrd), []string{"apply", "-f", "-"}) + + require.Error(t, err) + assert.Contains(t, output, "ValidatingAdmissionPolicy 'safe-upgrades.gateway.networking.k8s.io' with binding 'safe-upgrades.gateway.networking.k8s.io' denied request") + }) }) t.Run("should be able to install standard examples", func(t *testing.T) { @@ -155,6 +224,22 @@ func executeKubectlCommand(t *testing.T, kubectl, kubeconfig string, args []stri return string(output), err } +func executeKubectlCommandStdin(t *testing.T, kubectl, kubeconfig string, stdin io.Reader, args []string) (string, error) { + t.Helper() + + cacheDir := filepath.Dir(kubeconfig) + args = append([]string{"--cache-dir", cacheDir}, args...) + + cmd := exec.Command(kubectl, args...) + cmd.Env = []string{ + fmt.Sprintf("KUBECONFIG=%s", kubeconfig), + } + cmd.Stdin = stdin + + output, err := cmd.CombinedOutput() + return string(output), err +} + func getInvalidExamplesFiles(t *testing.T, crdChannel string) ([]string, error) { t.Helper()