From 576c73c60b91a36fa2ed51d4efe3d0cc7b7681c8 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Fri, 10 Apr 2026 22:54:13 +0200 Subject: [PATCH 1/7] Add e2e Signed-off-by: Chuan-Yen Chiang --- Makefile | 14 ++++-- build | 2 +- test/e2e/00-lifecycle.yaml | 16 ++++++ test/e2e/hooks/post-assert-lifecycle.sh | 65 +++++++++++++++++++++++++ test/e2e/hooks/pre-delete-lifecycle.sh | 25 ++++++++++ test/setup.sh | 62 +++++++++++++++++++++++ 6 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 test/e2e/00-lifecycle.yaml create mode 100755 test/e2e/hooks/post-assert-lifecycle.sh create mode 100755 test/e2e/hooks/pre-delete-lifecycle.sh create mode 100755 test/setup.sh diff --git a/Makefile b/Makefile index bdfce22..80c8ec5 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,17 @@ XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/crossplane XPKGS = provider-template -include build/makelib/xpkg.mk +# ==================================================================================== +# Setup Uptest + +CROSSPLANE_VERSION ?= 2.2.0 +-include build/makelib/local.xpkg.mk +-include build/makelib/controlplane.mk + +UPTEST_LOCAL_DEPLOY_TARGET = local.xpkg.deploy.provider.$(PROJECT_NAME) +UPTEST_INPUT_MANIFESTS = test/e2e/00-lifecycle.yaml +-include build/makelib/uptest.mk + # NOTE(hasheddan): we force image building to happen prior to xpkg build so that # we ensure image is present in daemon. xpkg.build.provider-template: do.build.images @@ -52,9 +63,6 @@ fallthrough: submodules @echo Initial setup complete. Running make again . . . @make -# integration tests -e2e.run: test-integration - # Run integration tests. test-integration: $(KIND) $(KUBECTL) $(CROSSPLANE_CLI) $(HELM3) @$(INFO) running integration tests using kind $(KIND_VERSION) diff --git a/build b/build index b964dbe..65819b0 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit b964dbe0ff0856a762f1a06fe554c647d22af7f0 +Subproject commit 65819b05239a0b22f1a98c3698a14acd22bc3cc0 diff --git a/test/e2e/00-lifecycle.yaml b/test/e2e/00-lifecycle.yaml new file mode 100644 index 0000000..404b2a6 --- /dev/null +++ b/test/e2e/00-lifecycle.yaml @@ -0,0 +1,16 @@ +apiVersion: sample.template.crossplane.io/v1alpha1 +kind: MyType +metadata: + name: e2e-lifecycle-test + namespace: default + annotations: + uptest.upbound.io/timeout: "120" + uptest.upbound.io/conditions: "Ready" + uptest.upbound.io/post-assert-hook: "hooks/post-assert-lifecycle.sh" + uptest.upbound.io/pre-delete-hook: "hooks/pre-delete-lifecycle.sh" +spec: + forProvider: + configurableField: "initial-value" + providerConfigRef: + name: example + kind: ProviderConfig diff --git a/test/e2e/hooks/post-assert-lifecycle.sh b/test/e2e/hooks/post-assert-lifecycle.sh new file mode 100755 index 0000000..a78bb78 --- /dev/null +++ b/test/e2e/hooks/post-assert-lifecycle.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +RESOURCE_NAME="e2e-lifecycle-test" +NAMESPACE="default" + +# Verify status and external-name are set +CONFIGURED=$(${KUBECTL} get mytype "${RESOURCE_NAME}" -n "${NAMESPACE}" \ + -o jsonpath='{.status.atProvider.configurableField}') + +if [[ -z "${CONFIGURED}" ]]; then + echo "FAIL: status.atProvider.configurableField is empty" + exit 1 +fi +echo "PASS: status.atProvider.configurableField = ${CONFIGURED}" + +EXTERNAL_NAME=$(${KUBECTL} get mytype "${RESOURCE_NAME}" -n "${NAMESPACE}" \ + -o jsonpath='{.metadata.annotations.crossplane\.io/external-name}') + +if [[ -z "${EXTERNAL_NAME}" ]]; then + echo "FAIL: external-name annotation is not set" + exit 1 +fi +echo "PASS: external-name = ${EXTERNAL_NAME}" + +# ---- Error case: MyType with non-existent ProviderConfig ---- +echo "" +echo "Testing error case: MyType with missing ProviderConfig..." + +ERROR_RESOURCE="e2e-error-no-config" + +# Ensure cleanup on any exit, preserving the original exit code +trap 'rc=$?; ${KUBECTL} delete mytype "${ERROR_RESOURCE}" -n "${NAMESPACE}" --ignore-not-found || true; exit $rc' EXIT + +cat < Date: Thu, 16 Jul 2026 07:19:54 +0200 Subject: [PATCH 2/7] Update build Signed-off-by: Chuan-Yen Chiang --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 65819b0..38cdd2d 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit 65819b05239a0b22f1a98c3698a14acd22bc3cc0 +Subproject commit 38cdd2d9558259446cdf476a769e4c462fbc308f From c5ab5af9ffdc2daf8835fd670521b2d643ff64a0 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Thu, 13 Aug 2026 23:17:37 +0200 Subject: [PATCH 3/7] Add test case examples. Signed-off-by: Chuan-Yen Chiang --- Makefile | 2 +- test/e2e/00-lifecycle.yaml | 2 +- test/e2e/hooks/cases/10-drift.sh | 44 +++++++++ test/e2e/hooks/cases/20-pause.sh | 65 +++++++++++++ .../hooks/cases/30-cluster-provider-config.sh | 94 +++++++++++++++++++ .../hooks/cases/40-unsupported-config-kind.sh | 42 +++++++++ test/e2e/hooks/post-assert-lifecycle.sh | 37 ++++++++ 7 files changed, 284 insertions(+), 2 deletions(-) create mode 100755 test/e2e/hooks/cases/10-drift.sh create mode 100755 test/e2e/hooks/cases/20-pause.sh create mode 100755 test/e2e/hooks/cases/30-cluster-provider-config.sh create mode 100755 test/e2e/hooks/cases/40-unsupported-config-kind.sh diff --git a/Makefile b/Makefile index 80c8ec5..d18bd1a 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ XPKGS = provider-template # ==================================================================================== # Setup Uptest -CROSSPLANE_VERSION ?= 2.2.0 +CROSSPLANE_VERSION ?= 2.3.4 -include build/makelib/local.xpkg.mk -include build/makelib/controlplane.mk diff --git a/test/e2e/00-lifecycle.yaml b/test/e2e/00-lifecycle.yaml index 404b2a6..1063fac 100644 --- a/test/e2e/00-lifecycle.yaml +++ b/test/e2e/00-lifecycle.yaml @@ -5,7 +5,7 @@ metadata: namespace: default annotations: uptest.upbound.io/timeout: "120" - uptest.upbound.io/conditions: "Ready" + uptest.upbound.io/conditions: "Ready,Synced" uptest.upbound.io/post-assert-hook: "hooks/post-assert-lifecycle.sh" uptest.upbound.io/pre-delete-hook: "hooks/pre-delete-lifecycle.sh" spec: diff --git a/test/e2e/hooks/cases/10-drift.sh b/test/e2e/hooks/cases/10-drift.sh new file mode 100755 index 0000000..4db0765 --- /dev/null +++ b/test/e2e/hooks/cases/10-drift.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Drift correction: the controller must repair external state that no longer +# matches the spec, without anyone touching the spec. This is the core +# Crossplane promise, and Observe() -> Update() is the loop that delivers it. +set -euo pipefail + +NAMESPACE="default" +RESOURCE="e2e-drift" + +trap 'rc=$?; ${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found >/dev/null 2>&1 || true; exit $rc' EXIT + +cat </dev/null 2>&1 || true; exit $rc' EXIT + +cat </dev/null 2>&1 || true + ${KUBECTL} delete clusterproviderconfig "${CPC}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + exit $rc +} +trap cleanup EXIT + +cat </dev/null 2>&1; then + echo "FAIL: in-use ClusterProviderConfig was deleted while a MyType still referenced it" + exit 1 +fi +DELETING=$(${KUBECTL} get clusterproviderconfig "${CPC}" -o jsonpath='{.metadata.deletionTimestamp}') +if [[ -z "${DELETING}" ]]; then + echo "FAIL: expected a deletionTimestamp on the ClusterProviderConfig" + exit 1 +fi +echo "PASS: deletion blocked while in use (deletionTimestamp=${DELETING})" + +# 4. Once the last user goes away the finalizer is released and it completes. +${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found +${KUBECTL} wait --for=delete clusterproviderconfig/"${CPC}" --timeout=120s +echo "PASS: deletion completed once the last user was removed" diff --git a/test/e2e/hooks/cases/40-unsupported-config-kind.sh b/test/e2e/hooks/cases/40-unsupported-config-kind.sh new file mode 100755 index 0000000..6eccec5 --- /dev/null +++ b/test/e2e/hooks/cases/40-unsupported-config-kind.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Connect() switches on providerConfigRef.kind and rejects anything that is +# neither ProviderConfig nor ClusterProviderConfig. Nothing validates the kind +# at admission, so this branch is reachable from a plain manifest and a real +# provider forked from this template will hit it via typos. +set -euo pipefail + +NAMESPACE="default" +RESOURCE="e2e-bad-config-kind" + +trap 'rc=$?; ${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found >/dev/null 2>&1 || true; exit $rc' EXIT + +cat < Date: Fri, 14 Aug 2026 00:33:13 +0200 Subject: [PATCH 4/7] Move scripts to chainsaw yaml files Signed-off-by: Chuan-Yen Chiang --- test/e2e/hooks/cases/10-drift.sh | 44 --------- test/e2e/hooks/cases/20-pause.sh | 65 ------------- .../hooks/cases/30-cluster-provider-config.sh | 94 ------------------- .../hooks/cases/40-unsupported-config-kind.sh | 42 --------- test/e2e/hooks/post-assert-lifecycle.sh | 19 ++-- 5 files changed, 7 insertions(+), 257 deletions(-) delete mode 100755 test/e2e/hooks/cases/10-drift.sh delete mode 100755 test/e2e/hooks/cases/20-pause.sh delete mode 100755 test/e2e/hooks/cases/30-cluster-provider-config.sh delete mode 100755 test/e2e/hooks/cases/40-unsupported-config-kind.sh diff --git a/test/e2e/hooks/cases/10-drift.sh b/test/e2e/hooks/cases/10-drift.sh deleted file mode 100755 index 4db0765..0000000 --- a/test/e2e/hooks/cases/10-drift.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Drift correction: the controller must repair external state that no longer -# matches the spec, without anyone touching the spec. This is the core -# Crossplane promise, and Observe() -> Update() is the loop that delivers it. -set -euo pipefail - -NAMESPACE="default" -RESOURCE="e2e-drift" - -trap 'rc=$?; ${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found >/dev/null 2>&1 || true; exit $rc' EXIT - -cat </dev/null 2>&1 || true; exit $rc' EXIT - -cat </dev/null 2>&1 || true - ${KUBECTL} delete clusterproviderconfig "${CPC}" --ignore-not-found --wait=false >/dev/null 2>&1 || true - exit $rc -} -trap cleanup EXIT - -cat </dev/null 2>&1; then - echo "FAIL: in-use ClusterProviderConfig was deleted while a MyType still referenced it" - exit 1 -fi -DELETING=$(${KUBECTL} get clusterproviderconfig "${CPC}" -o jsonpath='{.metadata.deletionTimestamp}') -if [[ -z "${DELETING}" ]]; then - echo "FAIL: expected a deletionTimestamp on the ClusterProviderConfig" - exit 1 -fi -echo "PASS: deletion blocked while in use (deletionTimestamp=${DELETING})" - -# 4. Once the last user goes away the finalizer is released and it completes. -${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found -${KUBECTL} wait --for=delete clusterproviderconfig/"${CPC}" --timeout=120s -echo "PASS: deletion completed once the last user was removed" diff --git a/test/e2e/hooks/cases/40-unsupported-config-kind.sh b/test/e2e/hooks/cases/40-unsupported-config-kind.sh deleted file mode 100755 index 6eccec5..0000000 --- a/test/e2e/hooks/cases/40-unsupported-config-kind.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Connect() switches on providerConfigRef.kind and rejects anything that is -# neither ProviderConfig nor ClusterProviderConfig. Nothing validates the kind -# at admission, so this branch is reachable from a plain manifest and a real -# provider forked from this template will hit it via typos. -set -euo pipefail - -NAMESPACE="default" -RESOURCE="e2e-bad-config-kind" - -trap 'rc=$?; ${KUBECTL} delete mytype "${RESOURCE}" -n "${NAMESPACE}" --ignore-not-found >/dev/null 2>&1 || true; exit $rc' EXIT - -cat < Date: Fri, 14 Aug 2026 01:14:59 +0200 Subject: [PATCH 5/7] Update tests Signed-off-by: Chuan-Yen Chiang --- .../chainsaw-test.yaml | 111 +++++++++++++++++ test/behavior/drift/chainsaw-test.yaml | 68 ++++++++++ test/behavior/pause/chainsaw-test.yaml | 116 ++++++++++++++++++ .../provider-config/chainsaw-test.yaml | 113 +++++++++++++++++ .../chainsaw-test.yaml | 56 +++++++++ test/e2e/00-lifecycle.yaml | 17 +++ 6 files changed, 481 insertions(+) create mode 100644 test/behavior/cluster-provider-config/chainsaw-test.yaml create mode 100644 test/behavior/drift/chainsaw-test.yaml create mode 100644 test/behavior/pause/chainsaw-test.yaml create mode 100644 test/behavior/provider-config/chainsaw-test.yaml create mode 100644 test/behavior/unsupported-config-kind/chainsaw-test.yaml diff --git a/test/behavior/cluster-provider-config/chainsaw-test.yaml b/test/behavior/cluster-provider-config/chainsaw-test.yaml new file mode 100644 index 0000000..6aed35d --- /dev/null +++ b/test/behavior/cluster-provider-config/chainsaw-test.yaml @@ -0,0 +1,111 @@ +# Connect()'s ClusterProviderConfig branch, and regression cover for b07ef7f: +# an in-use ClusterProviderConfig must not be deletable. Usage is recorded as a +# namespaced ProviderConfigUsage even though the config is cluster scoped -- +# there is deliberately no cluster scoped usage type. +# Pairs with test/behavior/provider-config, which covers the namespaced kind. +# Uses a dedicated config -- a deletionTimestamp cannot be undone. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: cluster-provider-config +spec: + timeouts: + apply: 1m + assert: 2m + delete: 2m + exec: 1m + steps: + - name: use a cluster scoped config + try: + - apply: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ClusterProviderConfig + metadata: + name: behaviour-cpc-inuse + spec: + credentials: + source: Secret + secretRef: + namespace: default + name: example-provider-secret + key: credentials + - apply: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-cpc-user + namespace: default + spec: + forProvider: + configurableField: cluster-scoped-creds + providerConfigRef: + name: behaviour-cpc-inuse + kind: ClusterProviderConfig + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-cpc-user + namespace: default + status: + atProvider: + configurableField: cluster-scoped-creds + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + + - name: usage is recorded and the config is held + try: + # Namespaced usage naming the cluster scoped kind -- what b07ef7f fixed. + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfigUsage + metadata: + namespace: default + providerConfigRef: + name: behaviour-cpc-inuse + kind: ClusterProviderConfig + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ClusterProviderConfig + metadata: + name: behaviour-cpc-inuse + (contains(finalizers, 'in-use.crossplane.io')): true + + - name: deleting it while in use must not remove it + try: + # A script because chainsaw's delete blocks until the object is gone. + - script: + content: | + ${KUBECTL:-kubectl} delete clusterproviderconfig behaviour-cpc-inuse --wait=false + - sleep: + duration: 10s + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ClusterProviderConfig + metadata: + name: behaviour-cpc-inuse + (deletionTimestamp != null): true + + - name: releasing the last user completes the deletion + try: + - delete: + ref: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + name: behaviour-cpc-user + namespace: default + - error: + timeout: 2m + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ClusterProviderConfig + metadata: + name: behaviour-cpc-inuse diff --git a/test/behavior/drift/chainsaw-test.yaml b/test/behavior/drift/chainsaw-test.yaml new file mode 100644 index 0000000..7329100 --- /dev/null +++ b/test/behavior/drift/chainsaw-test.yaml @@ -0,0 +1,68 @@ +# The controller repairs drifted external state without a spec change. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: drift-correction +spec: + timeouts: + apply: 1m + assert: 3m + delete: 2m + steps: + - name: create the resource + try: + - apply: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-drift + namespace: default + spec: + forProvider: + configurableField: drift-value + providerConfigRef: + name: example + kind: ProviderConfig + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-drift + namespace: default + status: + atProvider: + configurableField: drift-value + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + + - name: corrupt the observed state + try: + # A script because chainsaw 0.2.13's patch cannot target a subresource. + - script: + content: | + ${KUBECTL:-kubectl} patch mytype behaviour-drift -n default \ + --subresource=status --type=merge \ + -p '{"status":{"atProvider":{"configurableField":"drifted"}}}' + + - name: expect the controller to repair it + try: + # Status-only writes raise no event, so recovery waits for --poll (1m). + - assert: + timeout: 3m + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-drift + namespace: default + status: + atProvider: + configurableField: drift-value + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" diff --git a/test/behavior/pause/chainsaw-test.yaml b/test/behavior/pause/chainsaw-test.yaml new file mode 100644 index 0000000..05862c9 --- /dev/null +++ b/test/behavior/pause/chainsaw-test.yaml @@ -0,0 +1,116 @@ +# crossplane.io/paused must actually stop reconciliation, not just report it. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: pause-and-resume +spec: + timeouts: + apply: 1m + assert: 2m + delete: 2m + steps: + - name: create the resource + try: + - apply: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + spec: + forProvider: + configurableField: before-pause + providerConfigRef: + name: example + kind: ProviderConfig + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + status: + atProvider: + configurableField: before-pause + ((conditions[?type == 'Ready'])[0]): + status: "True" + + - name: pause it + try: + - patch: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + annotations: + crossplane.io/paused: "true" + # Ready stays True: conditions are sticky. + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + status: + ((conditions[?type == 'Synced'])[0]): + status: "False" + reason: ReconcilePaused + ((conditions[?type == 'Ready'])[0]): + status: "True" + + - name: a paused resource ignores spec changes + try: + - patch: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + spec: + forProvider: + configurableField: changed-while-paused + - sleep: + duration: 15s + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + status: + atProvider: + configurableField: before-pause + + - name: resume and catch up + try: + # A null value removes the annotation from the merge patch. + - patch: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + annotations: + crossplane.io/paused: null + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pause + namespace: default + status: + atProvider: + configurableField: changed-while-paused + ((conditions[?type == 'Synced'])[0]): + status: "True" + ((conditions[?type == 'Ready'])[0]): + status: "True" diff --git a/test/behavior/provider-config/chainsaw-test.yaml b/test/behavior/provider-config/chainsaw-test.yaml new file mode 100644 index 0000000..2b4a469 --- /dev/null +++ b/test/behavior/provider-config/chainsaw-test.yaml @@ -0,0 +1,113 @@ +# Connect()'s namespaced ProviderConfig branch, and its usage accounting: an +# in-use ProviderConfig must not be deletable. +# Pairs with test/behavior/cluster-provider-config, which covers the cluster +# scoped kind. Both record usage as a namespaced ProviderConfigUsage. +# Uses a dedicated config -- a deletionTimestamp cannot be undone. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: provider-config +spec: + timeouts: + apply: 1m + assert: 2m + delete: 2m + exec: 1m + steps: + - name: use a namespaced config + try: + - apply: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfig + metadata: + name: behaviour-pc-inuse + namespace: default + spec: + credentials: + source: Secret + secretRef: + namespace: default + name: example-provider-secret + key: credentials + - apply: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pc-user + namespace: default + spec: + forProvider: + configurableField: namespaced-creds + providerConfigRef: + name: behaviour-pc-inuse + kind: ProviderConfig + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-pc-user + namespace: default + status: + atProvider: + configurableField: namespaced-creds + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + + - name: usage is recorded and the config is held + try: + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfigUsage + metadata: + namespace: default + providerConfigRef: + name: behaviour-pc-inuse + kind: ProviderConfig + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfig + metadata: + name: behaviour-pc-inuse + namespace: default + (contains(finalizers, 'in-use.crossplane.io')): true + + - name: deleting it while in use must not remove it + try: + # A script because chainsaw's delete blocks until the object is gone. + - script: + content: | + ${KUBECTL:-kubectl} delete providerconfig behaviour-pc-inuse -n default --wait=false + - sleep: + duration: 10s + - assert: + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfig + metadata: + name: behaviour-pc-inuse + namespace: default + (deletionTimestamp != null): true + + - name: releasing the last user completes the deletion + try: + - delete: + ref: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + name: behaviour-pc-user + namespace: default + - error: + timeout: 2m + resource: + apiVersion: template.crossplane.io/v1alpha1 + kind: ProviderConfig + metadata: + name: behaviour-pc-inuse + namespace: default diff --git a/test/behavior/unsupported-config-kind/chainsaw-test.yaml b/test/behavior/unsupported-config-kind/chainsaw-test.yaml new file mode 100644 index 0000000..71ddae9 --- /dev/null +++ b/test/behavior/unsupported-config-kind/chainsaw-test.yaml @@ -0,0 +1,56 @@ +# Connect() rejects a providerConfigRef.kind it does not support. Nothing +# validates the kind at admission, so this is reachable from a plain manifest. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: unsupported-config-kind +spec: + timeouts: + apply: 1m + assert: 1m + delete: 2m + steps: + - name: an unsupported kind reports a useful error + try: + - apply: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-bad-config-kind + namespace: default + spec: + forProvider: + configurableField: unsupported-kind + providerConfigRef: + name: example + kind: NotAProviderConfigKind + - assert: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-bad-config-kind + namespace: default + status: + ((conditions[?type == 'Synced'])[0]): + status: "False" + reason: ReconcileError + (contains(message, 'unsupported provider config kind')): true + + - name: it stays deletable + try: + # Connect() can never succeed here; deletion must not depend on it. + - delete: + ref: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + name: behaviour-bad-config-kind + namespace: default + - error: + resource: + apiVersion: sample.template.crossplane.io/v1alpha1 + kind: MyType + metadata: + name: behaviour-bad-config-kind + namespace: default diff --git a/test/e2e/00-lifecycle.yaml b/test/e2e/00-lifecycle.yaml index 1063fac..013058e 100644 --- a/test/e2e/00-lifecycle.yaml +++ b/test/e2e/00-lifecycle.yaml @@ -14,3 +14,20 @@ spec: providerConfigRef: name: example kind: ProviderConfig +--- +# The same lifecycle, resolving credentials through the cluster scoped config. +# Both configs are created by test/setup.sh from examples/provider/config.yaml. +apiVersion: sample.template.crossplane.io/v1alpha1 +kind: MyType +metadata: + name: e2e-lifecycle-cluster-test + namespace: default + annotations: + uptest.upbound.io/timeout: "120" + uptest.upbound.io/conditions: "Ready,Synced" +spec: + forProvider: + configurableField: "initial-value" + providerConfigRef: + name: example + kind: ClusterProviderConfig From ef72f04582af7e101c8ebcb24b7ff96a2487378d Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Fri, 14 Aug 2026 01:28:25 +0200 Subject: [PATCH 6/7] Add README.md to test folder, and update README.md in project root for adding e2e tests. Signed-off-by: Chuan-Yen Chiang --- README.md | 41 +++++++++ test/README.md | 220 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 test/README.md diff --git a/README.md b/README.md index 53764b2..ab34bea 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,50 @@ with the following features that are meant to be refactored: 5. Run `make reviewable` to run code generation, linters, and tests. 5. Run `make build` to build the provider. +## Testing + +```shell +make test # unit tests +make e2e # end-to-end tests against a kind control plane +make uptest # only the e2e tests, reusing a control plane that is already up +``` + +`make e2e` runs two suites, and which one a new test belongs in depends on what +it checks: + +- **`test/e2e/`** — the managed resource lifecycle: create, observe, update, + import, delete. You do not write these by hand. [uptest] generates them from + the annotations on `test/e2e/00-lifecycle.yaml`, because that lifecycle is the + same for every managed resource. +- **`test/behavior/`** — everything that is not the lifecycle: drift correction, + the pause annotation, credential resolution, error paths. These are plain + [chainsaw] tests, one directory per behaviour. + +### Adding a test for your provider + +To cover a **new managed resource type**, add it to `test/e2e/00-lifecycle.yaml` +as another YAML document with a `uptest.upbound.io/conditions` annotation. uptest +will apply it, assert the conditions, and delete it. To assert more than +conditions, point `uptest.upbound.io/post-assert-hook` at a script. + +To cover a **controller behaviour**, create +`test/behavior//chainsaw-test.yaml`; it is picked up automatically. Prefer +declarative operations — `apply`, `assert`, `patch`, `delete`, and `error` +(which passes only when a resource is *absent*). Chainsaw removes whatever the +test applied, so no manual cleanup is needed. Validate before running: + +```shell +.cache/tools/*/chainsaw-* lint test -f test/behavior//chainsaw-test.yaml +``` + +`test/README.md` documents both suites in detail, including what every uptest +annotation means and which of them are documented only in uptest's source. + Refer to Crossplane's [CONTRIBUTING.md] file for more information on how the Crossplane community prefers to work. The [Provider Development][provider-dev] guide may also be of use. +[uptest]: https://github.com/crossplane/uptest +[chainsaw]: https://kyverno.github.io/chainsaw/ [CONTRIBUTING.md]: https://github.com/crossplane/crossplane/blob/master/CONTRIBUTING.md [provider-dev]: https://github.com/crossplane/crossplane/blob/master/contributing/guide-provider-development.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..23e6d29 --- /dev/null +++ b/test/README.md @@ -0,0 +1,220 @@ +# End-to-end tests + +These tests run the provider against a real Kubernetes cluster: a kind control plane +with Crossplane installed, the provider deployed from a locally built package, and +managed resources applied and asserted through their full lifecycle. + +They are driven by [uptest](https://github.com/crossplane/uptest), which reads a +manifest, generates [chainsaw](https://kyverno.github.io/chainsaw/) test steps from +its annotations, and runs them. + +## Layout + +There are two suites, and they exist for different reasons. + +``` +test/ +├── setup.sh # runs once before the tests; grants RBAC, applies ProviderConfig +├── e2e/ # resource lifecycle, generated by uptest +│ ├── 00-lifecycle.yaml # the test input manifest; its annotations configure uptest +│ └── hooks/ +│ ├── post-assert-lifecycle.sh # runs after the apply assertions +│ └── pre-delete-lifecycle.sh # runs before deletion +└── behavior/ # controller behaviour, plain chainsaw tests + ├── drift/chainsaw-test.yaml + ├── pause/chainsaw-test.yaml + ├── cluster-provider-config/chainsaw-test.yaml + └── unsupported-config-kind/chainsaw-test.yaml +``` + +**`e2e/` is the managed-resource lifecycle** — create, observe, update, import, +delete. uptest generates it, because that lifecycle is identical for every managed +resource and nobody wants to hand-write it per kind. + +**`behavior/` is everything that is not the lifecycle** — drift repair, the pause +annotation, credential resolution through a `ClusterProviderConfig`, unsupported +config kinds. uptest has no vocabulary for these, so they are written directly as +chainsaw tests: declarative assertions, one result per case, and automatic cleanup +of whatever the test applied. + +## Running them + +```bash +make e2e # build, recreate the control plane, deploy the provider, run the tests +make uptest # run only the tests, against a control plane that is already up +``` + +Both suites run from those two targets and **neither needs any Makefile wiring**. +`post-assert-lifecycle.sh` invokes chainsaw on `test/behavior` directly, which works +because the build submodule already provides everything: `uptest.mk:68` makes the +`uptest` target depend on `$(CHAINSAW)` so the binary is guaranteed present, and +`uptest.mk:54` exports `CHAINSAW` into the environment the hooks run in. + +To iterate on the behaviour suite alone, run chainsaw against a control plane that +already has the provider deployed: + +```bash +.cache/tools/*/chainsaw-* test test/behavior --parallel 1 +``` + +`make e2e` is not defined in this repo's `Makefile`. `build/makelib/uptest.mk:74` +declares it with prerequisites only; `build/makelib/common.mk:390` supplies the +recipe. Make merges the two into one rule — `make -p -n e2e` prints the result. + +Do not run it with `-j`: the prerequisites are order-dependent but declared as plain +prerequisites, so a parallel make will race. This is upstream `uptest.mk` behaviour, +shared by every provider that includes it. + +Useful while iterating: + +```bash +make e2e UPTEST_SKIP_DELETE=true # leave resources behind to inspect them +make uptest UPTEST_RENDER_ONLY=true # generate the chainsaw files without running +``` + +`UPTEST_RENDER_ONLY` is the fastest way to see what a manifest actually produces — +it writes the generated steps to a temp directory and prints the path. Reach for it +before debugging a test failure, because the generated step is often not what the +annotations suggest. + +## Fields in `00-lifecycle.yaml` + +Everything under `metadata.annotations` configures uptest. **The two namespaces are +documented in different places, and not all of them are documented at all:** + +| Annotation | Purpose | Where it is documented | +| --- | --- | --- | +| `uptest.upbound.io/timeout` | Per-resource test timeout, overriding `--default-timeout`. | uptest README | +| `uptest.upbound.io/conditions` | Comma-separated status conditions to assert, overriding `--default-conditions`. | uptest README | +| `uptest.upbound.io/pre-assert-hook` | Script to run after apply, before assertions. Path is relative to the manifest. | uptest README, *Hooks* section | +| `uptest.upbound.io/post-assert-hook` | Script to run after assertions. Path is relative to the manifest. | uptest README, *Hooks* section | +| `uptest.upbound.io/pre-delete-hook` | Script to run before deletion. Path is relative to the manifest. | uptest README, *Hooks* section | +| `uptest.upbound.io/post-delete-hook` | Script to run after deletion. Not used here. | uptest README, *Hooks* section | +| `uptest.upbound.io/update-parameter` | JSON merged into `spec.forProvider` for the update step. **Not in the README** — see the caveat below. | Source only: `internal/config/config.go`, `AnnotationKeyUpdateParameter` | +| `uptest.upbound.io/disable-import` | Skips the import step for this resource. **Not in the README.** | Source only: `internal/config/config.go`, `AnnotationKeyDisableImport` | +| `meta.upbound.io/example-id` | Marks the *root resource*. Format is `//`. **Not in the README.** | Source only: `internal/config/config.go`, `AnnotationKeyExampleID` | + +When an annotation is not in the README, the constants in +[`internal/config/config.go`](https://github.com/crossplane/uptest/blob/main/internal/config/config.go) +are the authority. The generated templates in +[`internal/templates/`](https://github.com/crossplane/uptest/tree/main/internal/templates) +show how each value is actually used. + +CLI flags — including the defaults these annotations override — come from: + +```bash +.cache/tools/*/uptest-* e2e --help +``` + +> `.cache/` is not part of this repo. The build submodule creates it as a local tool +> cache — `build/makelib/common.mk:191-195` builds the path +> `.cache/tools/_/`, and `build/makelib/k8s_tools.mk` downloads each pinned +> tool into it as `-` on first use. It is gitignored and disposable: +> delete it and the next `make` re-downloads everything. The glob above only resolves +> once a target has actually fetched uptest, so run `make uptest` at least once first. +> Tool versions are pinned in `k8s_tools.mk`, so bumping one means a change to the +> `crossplane/build` submodule rather than to this repo. + +Note `--default-conditions` defaults to **`Ready` only**. Asserting `Synced` as well +requires setting it explicitly, which is why this manifest uses +`conditions: "Ready,Synced"`. Without it, a resource that is `Ready=True` but +`Synced=False` passes the test — conditions are sticky, so `Ready` keeps its last +value even after reconciliation starts failing. + +### Make variables + +| Variable | Purpose | Defined in | +| --- | --- | --- | +| `UPTEST_INPUT_MANIFESTS` | Manifests to test. | This repo's `Makefile` | +| `UPTEST_LOCAL_DEPLOY_TARGET` | Target that deploys the package before testing. Required — uptest.mk raises `$(error)` without it. | `build/makelib/uptest.mk:18` | +| `UPTEST_SETUP_SCRIPT` | Setup script path; defaults to `test/setup.sh`. | `build/makelib/uptest.mk:67` | +| `UPTEST_SKIP_UPDATE` / `_IMPORT` / `_DELETE` | Skip individual steps. | `build/makelib/uptest.mk:27-40` | +| `UPTEST_RENDER_ONLY` | Generate the chainsaw files without running them. | `build/makelib/uptest.mk:47` | +| `UPTEST_DEFAULT_TIMEOUT` | Global timeout, overridable per resource. | `build/makelib/uptest.mk:42` | +| `CROSSPLANE_VERSION` | Crossplane chart version to install. Required — `controlplane.mk` defaults it to empty, which produces a `helm install --version` with no value. | `build/makelib/controlplane.mk:17` | +| `KIND_CLUSTER_NAME` | Control plane cluster name; defaults to `local-dev`. | `build/makelib/controlplane.mk:15` | + +## What each test covers + +| Test | Behaviour under test | +| --- | --- | +| `e2e/` generated steps | Apply, condition assertions, import (state wipe and rebuild), delete | +| `e2e/hooks/post-assert-lifecycle.sh` | `status.atProvider` populated, external-name set, spec update propagates, missing ProviderConfig reports `Synced=False` | +| `e2e/hooks/pre-delete-lifecycle.sh` | Resource is `Ready` and `spec == status` before deletion | +| `behavior/drift` | Controller repairs corrupted `status.atProvider` with no spec change | +| `behavior/pause` | `crossplane.io/paused` stops reconciliation; `Ready` stays sticky; spec changes are ignored until unpaused | +| `behavior/provider-config` | Namespaced `ProviderConfig` credential path; usage recorded; an in-use config cannot be deleted | +| `behavior/cluster-provider-config` | Same, for the cluster scoped `ClusterProviderConfig`. Its usage is still a *namespaced* `ProviderConfigUsage` — there is no cluster scoped usage type | +| `behavior/unsupported-config-kind` | An unsupported `providerConfigRef.kind` reports a useful error and the resource still deletes | + +`behavior/cluster-provider-config` is the regression test for commit `b07ef7f`, which +fixed an in-use `ClusterProviderConfig` being deletable out from under the resources +referencing it. + +### Writing a new behaviour test + +Add a directory under `test/behavior/` containing a `chainsaw-test.yaml`; it is +picked up automatically. Prefer declarative operations — `apply`, `assert`, +`patch`, `delete`, and `error` (which passes only when the resource does *not* +match, so it is how you assert absence). Chainsaw deletes whatever the test applied, +so no manual cleanup is needed — but a test that leaks a managed resource will hang +the lifecycle suite's `kubectl wait managed --all --for=delete` minutes later, far +from the actual cause. + +Two constraints from the pinned chainsaw `0.2.13`, both caught by `chainsaw lint`: + +- `description` is allowed on a step, but not on an individual operation. +- `patch` has no `subresource` field, so patching a status needs a `script` with + `kubectl --subresource=status`. That field exists in later chainsaw releases, but + the version is pinned in the build submodule. + +Lint before running — it is far faster than discovering a schema error mid-run: + +```bash +.cache/tools/*/chainsaw-* lint test -f test/behavior//chainsaw-test.yaml +``` + +## Caveats worth knowing before you edit these + +**The update step is skipped.** uptest only generates it for the resource it +considers the *root*, identified by `meta.upbound.io/example-id`. Without that +annotation it logs `Skipping update step because the root resource does not exist` +and produces no `01-update.yaml` — a silent gap, not an error. + +**Adding it back does not currently work.** `uptest.upbound.io/update-parameter` must +parse as valid JSON, but uptest v2.2.0 interpolates the value *raw* into a +double-quoted shell command, so the JSON's own quotes are stripped before `kubectl` +sees them (`error decoding patch: invalid character`). Escaping them instead breaks +the JSON unmarshal. v2.2.0 is the latest release and the one that introduced the +retry wrapper responsible. Update coverage therefore lives in +`post-assert-lifecycle.sh` instead. + +**`Create()` is never exercised.** crossplane-runtime installs +`NewNameAsExternalName` as the default initializer, which stamps the external-name +annotation before the first `Observe()`. The template's `Observe()` treats an empty +external-name as "does not exist", so that branch is unreachable and the reconciler +always routes a new resource through `Update()`. No test can close this gap; it needs +a change to `Observe()`. + +**`Ready` can lag `Synced` by a minute.** On creation `Synced` is set as soon as +`Update()` succeeds, but `Ready` is only set inside `Observe()` — which already ran. +The next `Observe()` arrives on the poll interval (`--poll`, default `1m` per +`cmd/provider/main.go:59`), because a status-only write raises no event the +controller acts on. A trace showing `Synced=True` a full minute before `Ready=True` +is expected. + +## Reference + +| Topic | Source | +| --- | --- | +| Annotations, hooks, data injection | [uptest README](https://github.com/crossplane/uptest) | +| Annotation keys not in the README | [`internal/config/config.go`](https://github.com/crossplane/uptest/blob/main/internal/config/config.go) | +| Generated step shapes | [`internal/templates/`](https://github.com/crossplane/uptest/tree/main/internal/templates) | +| CLI flags and defaults | `uptest e2e --help` | +| Make targets and variables | `build/makelib/uptest.mk`, `controlplane.mk`, `local.xpkg.mk` | +| `crossplane.io/paused`, `Ready` / `Synced` conditions | [Crossplane docs — Managed Resources](https://docs.crossplane.io/latest/managed-resources/managed-resources/) | +| Assertion syntax in generated steps | [chainsaw docs](https://kyverno.github.io/chainsaw/) | +| Reconciler and default initializers | `crossplane-runtime/v2` `pkg/reconciler/managed/reconciler.go` | + +A walkthrough of the reconcile timing and the full coverage matrix is in +[`docs/synced-before-ready.html`](../docs/synced-before-ready.html). From 84359d5fcaa99d0603b7b316de0626ce08de34d7 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Sat, 15 Aug 2026 22:04:08 +0200 Subject: [PATCH 7/7] - Update test cases - Update docs Signed-off-by: Chuan-Yen Chiang --- README.md | 33 +-- test/README.md | 266 ++++++------------ .../chainsaw-test.yaml | 12 +- test/e2e/hooks/post-assert-lifecycle.sh | 15 +- test/setup.sh | 48 ---- 5 files changed, 123 insertions(+), 251 deletions(-) diff --git a/README.md b/README.md index ab34bea..8662744 100644 --- a/README.md +++ b/README.md @@ -43,32 +43,25 @@ make uptest # only the e2e tests, reusing a control plane that is already up it checks: - **`test/e2e/`** — the managed resource lifecycle: create, observe, update, - import, delete. You do not write these by hand. [uptest] generates them from - the annotations on `test/e2e/00-lifecycle.yaml`, because that lifecycle is the - same for every managed resource. -- **`test/behavior/`** — everything that is not the lifecycle: drift correction, - the pause annotation, credential resolution, error paths. These are plain - [chainsaw] tests, one directory per behaviour. + import, delete. [uptest] generates these, so to cover a **new managed resource + type** you add it to `test/e2e/00-lifecycle.yaml` as another YAML document + with a `uptest.upbound.io/conditions` annotation. Point + `uptest.upbound.io/post-assert-hook` at a script to assert more than + conditions. +- **`test/behavior/`** — everything else: drift, the pause annotation, + credential resolution, error paths. To cover a **controller behaviour**, add + `test/behavior//chainsaw-test.yaml`; it is picked up automatically. + Prefer declarative [chainsaw] operations — `apply`, `assert`, `patch`, + `delete`, and `error` (which passes only when a resource is *absent*). -### Adding a test for your provider - -To cover a **new managed resource type**, add it to `test/e2e/00-lifecycle.yaml` -as another YAML document with a `uptest.upbound.io/conditions` annotation. uptest -will apply it, assert the conditions, and delete it. To assert more than -conditions, point `uptest.upbound.io/post-assert-hook` at a script. - -To cover a **controller behaviour**, create -`test/behavior//chainsaw-test.yaml`; it is picked up automatically. Prefer -declarative operations — `apply`, `assert`, `patch`, `delete`, and `error` -(which passes only when a resource is *absent*). Chainsaw removes whatever the -test applied, so no manual cleanup is needed. Validate before running: +Validate a new chainsaw test before running it: ```shell .cache/tools/*/chainsaw-* lint test -f test/behavior//chainsaw-test.yaml ``` -`test/README.md` documents both suites in detail, including what every uptest -annotation means and which of them are documented only in uptest's source. +`test/README.md` is the reference: every uptest annotation and where it is +documented, the make variables, what each test covers, and the known gotchas. Refer to Crossplane's [CONTRIBUTING.md] file for more information on how the Crossplane community prefers to work. The [Provider Development][provider-dev] diff --git a/test/README.md b/test/README.md index 23e6d29..32bbe1e 100644 --- a/test/README.md +++ b/test/README.md @@ -1,220 +1,140 @@ # End-to-end tests -These tests run the provider against a real Kubernetes cluster: a kind control plane -with Crossplane installed, the provider deployed from a locally built package, and -managed resources applied and asserted through their full lifecycle. - -They are driven by [uptest](https://github.com/crossplane/uptest), which reads a -manifest, generates [chainsaw](https://kyverno.github.io/chainsaw/) test steps from -its annotations, and runs them. - -## Layout - -There are two suites, and they exist for different reasons. +Reference for the two suites. The root `README.md` covers how to run them and +where a new test belongs. ``` test/ -├── setup.sh # runs once before the tests; grants RBAC, applies ProviderConfig +├── setup.sh # runs once before the tests; applies the ProviderConfigs ├── e2e/ # resource lifecycle, generated by uptest -│ ├── 00-lifecycle.yaml # the test input manifest; its annotations configure uptest +│ ├── 00-lifecycle.yaml # test input; its annotations configure uptest │ └── hooks/ │ ├── post-assert-lifecycle.sh # runs after the apply assertions │ └── pre-delete-lifecycle.sh # runs before deletion └── behavior/ # controller behaviour, plain chainsaw tests - ├── drift/chainsaw-test.yaml - ├── pause/chainsaw-test.yaml - ├── cluster-provider-config/chainsaw-test.yaml - └── unsupported-config-kind/chainsaw-test.yaml + ├── drift/ + ├── pause/ + ├── provider-config/ + ├── cluster-provider-config/ + └── unsupported-config-kind/ ``` -**`e2e/` is the managed-resource lifecycle** — create, observe, update, import, -delete. uptest generates it, because that lifecycle is identical for every managed -resource and nobody wants to hand-write it per kind. - -**`behavior/` is everything that is not the lifecycle** — drift repair, the pause -annotation, credential resolution through a `ClusterProviderConfig`, unsupported -config kinds. uptest has no vocabulary for these, so they are written directly as -chainsaw tests: declarative assertions, one result per case, and automatic cleanup -of whatever the test applied. - -## Running them - -```bash -make e2e # build, recreate the control plane, deploy the provider, run the tests -make uptest # run only the tests, against a control plane that is already up -``` - -Both suites run from those two targets and **neither needs any Makefile wiring**. -`post-assert-lifecycle.sh` invokes chainsaw on `test/behavior` directly, which works -because the build submodule already provides everything: `uptest.mk:68` makes the -`uptest` target depend on `$(CHAINSAW)` so the binary is guaranteed present, and -`uptest.mk:54` exports `CHAINSAW` into the environment the hooks run in. - -To iterate on the behaviour suite alone, run chainsaw against a control plane that -already has the provider deployed: - -```bash -.cache/tools/*/chainsaw-* test test/behavior --parallel 1 -``` - -`make e2e` is not defined in this repo's `Makefile`. `build/makelib/uptest.mk:74` -declares it with prerequisites only; `build/makelib/common.mk:390` supplies the -recipe. Make merges the two into one rule — `make -p -n e2e` prints the result. - -Do not run it with `-j`: the prerequisites are order-dependent but declared as plain -prerequisites, so a parallel make will race. This is upstream `uptest.mk` behaviour, -shared by every provider that includes it. +Neither suite needs Makefile wiring: `uptest.mk:68` already makes the `uptest` +target depend on `$(CHAINSAW)`, and `uptest.mk:54` exports `CHAINSAW` into the +environment the hooks run in, so `post-assert-lifecycle.sh` can invoke chainsaw +on `test/behavior` directly. -Useful while iterating: +While iterating: ```bash make e2e UPTEST_SKIP_DELETE=true # leave resources behind to inspect them -make uptest UPTEST_RENDER_ONLY=true # generate the chainsaw files without running +make uptest UPTEST_RENDER_ONLY=true # render the chainsaw files without running ``` -`UPTEST_RENDER_ONLY` is the fastest way to see what a manifest actually produces — -it writes the generated steps to a temp directory and prints the path. Reach for it -before debugging a test failure, because the generated step is often not what the -annotations suggest. +`UPTEST_RENDER_ONLY` prints where it wrote. Reach for it before debugging a +failure — the generated step is often not what the annotations suggest. ## Fields in `00-lifecycle.yaml` -Everything under `metadata.annotations` configures uptest. **The two namespaces are -documented in different places, and not all of them are documented at all:** +Everything under `metadata.annotations` configures uptest, and **not all of these +are documented — three exist only in uptest's source:** | Annotation | Purpose | Where it is documented | | --- | --- | --- | -| `uptest.upbound.io/timeout` | Per-resource test timeout, overriding `--default-timeout`. | uptest README | -| `uptest.upbound.io/conditions` | Comma-separated status conditions to assert, overriding `--default-conditions`. | uptest README | -| `uptest.upbound.io/pre-assert-hook` | Script to run after apply, before assertions. Path is relative to the manifest. | uptest README, *Hooks* section | -| `uptest.upbound.io/post-assert-hook` | Script to run after assertions. Path is relative to the manifest. | uptest README, *Hooks* section | -| `uptest.upbound.io/pre-delete-hook` | Script to run before deletion. Path is relative to the manifest. | uptest README, *Hooks* section | -| `uptest.upbound.io/post-delete-hook` | Script to run after deletion. Not used here. | uptest README, *Hooks* section | -| `uptest.upbound.io/update-parameter` | JSON merged into `spec.forProvider` for the update step. **Not in the README** — see the caveat below. | Source only: `internal/config/config.go`, `AnnotationKeyUpdateParameter` | -| `uptest.upbound.io/disable-import` | Skips the import step for this resource. **Not in the README.** | Source only: `internal/config/config.go`, `AnnotationKeyDisableImport` | -| `meta.upbound.io/example-id` | Marks the *root resource*. Format is `//`. **Not in the README.** | Source only: `internal/config/config.go`, `AnnotationKeyExampleID` | - -When an annotation is not in the README, the constants in -[`internal/config/config.go`](https://github.com/crossplane/uptest/blob/main/internal/config/config.go) -are the authority. The generated templates in -[`internal/templates/`](https://github.com/crossplane/uptest/tree/main/internal/templates) -show how each value is actually used. - -CLI flags — including the defaults these annotations override — come from: - -```bash -.cache/tools/*/uptest-* e2e --help -``` - -> `.cache/` is not part of this repo. The build submodule creates it as a local tool -> cache — `build/makelib/common.mk:191-195` builds the path -> `.cache/tools/_/`, and `build/makelib/k8s_tools.mk` downloads each pinned -> tool into it as `-` on first use. It is gitignored and disposable: -> delete it and the next `make` re-downloads everything. The glob above only resolves -> once a target has actually fetched uptest, so run `make uptest` at least once first. -> Tool versions are pinned in `k8s_tools.mk`, so bumping one means a change to the -> `crossplane/build` submodule rather than to this repo. - -Note `--default-conditions` defaults to **`Ready` only**. Asserting `Synced` as well -requires setting it explicitly, which is why this manifest uses -`conditions: "Ready,Synced"`. Without it, a resource that is `Ready=True` but -`Synced=False` passes the test — conditions are sticky, so `Ready` keeps its last -value even after reconciliation starts failing. +| `uptest.upbound.io/timeout` | Per-resource timeout, overriding `--default-timeout`. | uptest README | +| `uptest.upbound.io/conditions` | Comma-separated conditions to assert, overriding `--default-conditions`. | uptest README | +| `uptest.upbound.io/pre-assert-hook` | Script to run after apply, before assertions. Path relative to the manifest. | uptest README, *Hooks* | +| `uptest.upbound.io/post-assert-hook` | Script to run after assertions. | uptest README, *Hooks* | +| `uptest.upbound.io/pre-delete-hook` | Script to run before deletion. | uptest README, *Hooks* | +| `uptest.upbound.io/post-delete-hook` | Script to run after deletion. Not used here. | uptest README, *Hooks* | +| `uptest.upbound.io/update-parameter` | JSON merged into `spec.forProvider` for the update step. | Source only — `internal/config/config.go`, `AnnotationKeyUpdateParameter` | +| `uptest.upbound.io/disable-import` | Skips the import step for this resource. | Source only — `AnnotationKeyDisableImport` | +| `meta.upbound.io/example-id` | Marks the *root resource*: `//`. | Source only — `AnnotationKeyExampleID` | + +When an annotation is not in the README, [`internal/config/config.go`][cfg] is the +authority, and [`internal/templates/`][tmpl] shows how each value is actually used. +CLI flags and their defaults come from `.cache/tools/*/uptest-* e2e --help`. + +`.cache/` is not part of this repo — the build submodule creates it as a tool cache +(`common.mk:191-195` builds the path, `k8s_tools.mk` downloads into it). It is +gitignored and disposable, and tool versions are pinned in `k8s_tools.mk`, so +bumping one is a submodule change. ### Make variables | Variable | Purpose | Defined in | | --- | --- | --- | -| `UPTEST_INPUT_MANIFESTS` | Manifests to test. | This repo's `Makefile` | -| `UPTEST_LOCAL_DEPLOY_TARGET` | Target that deploys the package before testing. Required — uptest.mk raises `$(error)` without it. | `build/makelib/uptest.mk:18` | -| `UPTEST_SETUP_SCRIPT` | Setup script path; defaults to `test/setup.sh`. | `build/makelib/uptest.mk:67` | -| `UPTEST_SKIP_UPDATE` / `_IMPORT` / `_DELETE` | Skip individual steps. | `build/makelib/uptest.mk:27-40` | -| `UPTEST_RENDER_ONLY` | Generate the chainsaw files without running them. | `build/makelib/uptest.mk:47` | -| `UPTEST_DEFAULT_TIMEOUT` | Global timeout, overridable per resource. | `build/makelib/uptest.mk:42` | -| `CROSSPLANE_VERSION` | Crossplane chart version to install. Required — `controlplane.mk` defaults it to empty, which produces a `helm install --version` with no value. | `build/makelib/controlplane.mk:17` | -| `KIND_CLUSTER_NAME` | Control plane cluster name; defaults to `local-dev`. | `build/makelib/controlplane.mk:15` | +| `UPTEST_INPUT_MANIFESTS` | Manifests to test. | this repo's `Makefile` | +| `UPTEST_LOCAL_DEPLOY_TARGET` | Target that deploys the package first. Required — `uptest.mk` raises `$(error)` without it. | `uptest.mk:18` | +| `UPTEST_SETUP_SCRIPT` | Setup script; defaults to `test/setup.sh`. | `uptest.mk:67` | +| `UPTEST_SKIP_UPDATE` / `_IMPORT` / `_DELETE` | Skip individual steps. | `uptest.mk:27-40` | +| `UPTEST_RENDER_ONLY` | Generate the chainsaw files without running them. | `uptest.mk:47` | +| `CROSSPLANE_VERSION` | Chart version to install. Required — `controlplane.mk` defaults it to empty. | `controlplane.mk:17` | +| `KIND_CLUSTER_NAME` | Control plane name; defaults to `local-dev`. | `controlplane.mk:15` | -## What each test covers +## Coverage | Test | Behaviour under test | | --- | --- | -| `e2e/` generated steps | Apply, condition assertions, import (state wipe and rebuild), delete | +| `e2e/` generated steps | Apply, condition assertions, import, delete — for both a namespaced and a cluster scoped config | | `e2e/hooks/post-assert-lifecycle.sh` | `status.atProvider` populated, external-name set, spec update propagates, missing ProviderConfig reports `Synced=False` | | `e2e/hooks/pre-delete-lifecycle.sh` | Resource is `Ready` and `spec == status` before deletion | | `behavior/drift` | Controller repairs corrupted `status.atProvider` with no spec change | -| `behavior/pause` | `crossplane.io/paused` stops reconciliation; `Ready` stays sticky; spec changes are ignored until unpaused | +| `behavior/pause` | `crossplane.io/paused` stops reconciliation; `Ready` stays sticky; spec changes ignored until unpaused | | `behavior/provider-config` | Namespaced `ProviderConfig` credential path; usage recorded; an in-use config cannot be deleted | -| `behavior/cluster-provider-config` | Same, for the cluster scoped `ClusterProviderConfig`. Its usage is still a *namespaced* `ProviderConfigUsage` — there is no cluster scoped usage type | +| `behavior/cluster-provider-config` | Same for `ClusterProviderConfig`. Its usage is still a *namespaced* `ProviderConfigUsage` — there is no cluster scoped usage type | | `behavior/unsupported-config-kind` | An unsupported `providerConfigRef.kind` reports a useful error and the resource still deletes | -`behavior/cluster-provider-config` is the regression test for commit `b07ef7f`, which -fixed an in-use `ClusterProviderConfig` being deletable out from under the resources -referencing it. +## Why the tests are shaped this way + +Read these before "fixing" something here that looks wrong. + +**`--default-conditions` is `Ready` only,** so asserting `Synced` needs +`conditions: "Ready,Synced"` explicitly. Without it a resource that is `Ready=True` +but `Synced=False` passes — conditions are sticky, so `Ready` keeps its last value +after reconciliation starts failing. + +**uptest's update step is skipped,** because it is only generated for the resource +carrying `meta.upbound.io/example-id`. Adding that annotation does not help: +`uptest.upbound.io/update-parameter` must be valid JSON, but uptest v2.2.0 +interpolates it raw into a double-quoted shell command, so its quotes are stripped +before kubectl sees them. Update coverage lives in `post-assert-lifecycle.sh` +instead. + +**There is no `Create()` test,** because that branch is unreachable. +crossplane-runtime's default `NameAsExternalName` initializer stamps the +external-name annotation before the first `Observe()`, and `Observe()` treats an +empty external-name as "does not exist" — so new resources route through +`Update()`. -### Writing a new behaviour test +## When a run looks broken but isn't -Add a directory under `test/behavior/` containing a `chainsaw-test.yaml`; it is -picked up automatically. Prefer declarative operations — `apply`, `assert`, -`patch`, `delete`, and `error` (which passes only when the resource does *not* -match, so it is how you assert absence). Chainsaw deletes whatever the test applied, -so no manual cleanup is needed — but a test that leaks a managed resource will hang -the lifecycle suite's `kubectl wait managed --all --for=delete` minutes later, far -from the actual cause. +**`Ready` lagging `Synced` by a minute is expected.** `Synced` is set as soon as +`Update()` succeeds; `Ready` is only set inside `Observe()`, which already ran. The +next `Observe()` arrives on the poll interval (`--poll`, default `1m`), because a +status-only write raises no event the controller acts on. -Two constraints from the pinned chainsaw `0.2.13`, both caught by `chainsaw lint`: +**Do not run `make e2e -j`.** Its prerequisites are order-dependent but declared as +plain prerequisites, so a parallel make races. Upstream `uptest.mk` behaviour. + +**An interrupted run wedges the cluster,** and every later run then hangs or fails +with resources "not found". The import step pauses the resource and scales the +provider to 0; killed mid-window it leaves `replicas: 0` and a paused resource +whose finalizer can never be released. `make controlplane.down` is the reliable +fix; patching the `DeploymentRuntimeConfig` replicas back to 1 and removing the +`crossplane.io/paused` annotation recovers it in place. + +## chainsaw 0.2.13 constraints + +Two limits of the pinned version, both caught by `chainsaw lint`: - `description` is allowed on a step, but not on an individual operation. - `patch` has no `subresource` field, so patching a status needs a `script` with - `kubectl --subresource=status`. That field exists in later chainsaw releases, but - the version is pinned in the build submodule. + `kubectl --subresource=status`. -Lint before running — it is far faster than discovering a schema error mid-run: +A leaked managed resource is the other trap: chainsaw cleans up what the test +applied, but anything left behind hangs the lifecycle suite's +`kubectl wait managed --all --for=delete` minutes later, far from the cause. -```bash -.cache/tools/*/chainsaw-* lint test -f test/behavior//chainsaw-test.yaml -``` - -## Caveats worth knowing before you edit these - -**The update step is skipped.** uptest only generates it for the resource it -considers the *root*, identified by `meta.upbound.io/example-id`. Without that -annotation it logs `Skipping update step because the root resource does not exist` -and produces no `01-update.yaml` — a silent gap, not an error. - -**Adding it back does not currently work.** `uptest.upbound.io/update-parameter` must -parse as valid JSON, but uptest v2.2.0 interpolates the value *raw* into a -double-quoted shell command, so the JSON's own quotes are stripped before `kubectl` -sees them (`error decoding patch: invalid character`). Escaping them instead breaks -the JSON unmarshal. v2.2.0 is the latest release and the one that introduced the -retry wrapper responsible. Update coverage therefore lives in -`post-assert-lifecycle.sh` instead. - -**`Create()` is never exercised.** crossplane-runtime installs -`NewNameAsExternalName` as the default initializer, which stamps the external-name -annotation before the first `Observe()`. The template's `Observe()` treats an empty -external-name as "does not exist", so that branch is unreachable and the reconciler -always routes a new resource through `Update()`. No test can close this gap; it needs -a change to `Observe()`. - -**`Ready` can lag `Synced` by a minute.** On creation `Synced` is set as soon as -`Update()` succeeds, but `Ready` is only set inside `Observe()` — which already ran. -The next `Observe()` arrives on the poll interval (`--poll`, default `1m` per -`cmd/provider/main.go:59`), because a status-only write raises no event the -controller acts on. A trace showing `Synced=True` a full minute before `Ready=True` -is expected. - -## Reference - -| Topic | Source | -| --- | --- | -| Annotations, hooks, data injection | [uptest README](https://github.com/crossplane/uptest) | -| Annotation keys not in the README | [`internal/config/config.go`](https://github.com/crossplane/uptest/blob/main/internal/config/config.go) | -| Generated step shapes | [`internal/templates/`](https://github.com/crossplane/uptest/tree/main/internal/templates) | -| CLI flags and defaults | `uptest e2e --help` | -| Make targets and variables | `build/makelib/uptest.mk`, `controlplane.mk`, `local.xpkg.mk` | -| `crossplane.io/paused`, `Ready` / `Synced` conditions | [Crossplane docs — Managed Resources](https://docs.crossplane.io/latest/managed-resources/managed-resources/) | -| Assertion syntax in generated steps | [chainsaw docs](https://kyverno.github.io/chainsaw/) | -| Reconciler and default initializers | `crossplane-runtime/v2` `pkg/reconciler/managed/reconciler.go` | - -A walkthrough of the reconcile timing and the full coverage matrix is in -[`docs/synced-before-ready.html`](../docs/synced-before-ready.html). +[cfg]: https://github.com/crossplane/uptest/blob/main/internal/config/config.go +[tmpl]: https://github.com/crossplane/uptest/tree/main/internal/templates diff --git a/test/behavior/cluster-provider-config/chainsaw-test.yaml b/test/behavior/cluster-provider-config/chainsaw-test.yaml index 6aed35d..33d719c 100644 --- a/test/behavior/cluster-provider-config/chainsaw-test.yaml +++ b/test/behavior/cluster-provider-config/chainsaw-test.yaml @@ -1,9 +1,5 @@ -# Connect()'s ClusterProviderConfig branch, and regression cover for b07ef7f: -# an in-use ClusterProviderConfig must not be deletable. Usage is recorded as a -# namespaced ProviderConfigUsage even though the config is cluster scoped -- -# there is deliberately no cluster scoped usage type. -# Pairs with test/behavior/provider-config, which covers the namespaced kind. -# Uses a dedicated config -- a deletionTimestamp cannot be undone. +# ClusterProviderConfig credentials, and that an in-use config cannot be deleted. +# Uses a dedicated config -- deletion is one-way. apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -60,7 +56,7 @@ spec: - name: usage is recorded and the config is held try: - # Namespaced usage naming the cluster scoped kind -- what b07ef7f fixed. + # Usage is namespaced even for a cluster scoped config. - assert: resource: apiVersion: template.crossplane.io/v1alpha1 @@ -80,7 +76,7 @@ spec: - name: deleting it while in use must not remove it try: - # A script because chainsaw's delete blocks until the object is gone. + # Script: chainsaw's delete would block until the object is gone. - script: content: | ${KUBECTL:-kubectl} delete clusterproviderconfig behaviour-cpc-inuse --wait=false diff --git a/test/e2e/hooks/post-assert-lifecycle.sh b/test/e2e/hooks/post-assert-lifecycle.sh index ba6a625..c245fdb 100755 --- a/test/e2e/hooks/post-assert-lifecycle.sh +++ b/test/e2e/hooks/post-assert-lifecycle.sh @@ -90,8 +90,19 @@ echo "PASS: error message indicates config issue: ${MESSAGE}" # ---- Controller behaviours beyond the resource lifecycle ---- # CHAINSAW is exported by uptest.mk, so no Makefile wiring is needed. -BEHAVIOR_DIR="$(cd "$(dirname "$0")/../.." && pwd)/behavior" +PROJECT_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +BEHAVIOR_DIR="${PROJECT_ROOT}/test/behavior" echo "" echo "Running controller behaviour tests from ${BEHAVIOR_DIR}..." -"${CHAINSAW}" test "${BEHAVIOR_DIR}" --parallel 1 +# --quiet keeps per-operation logging out of the terminal; failures, errors and +# the summary are still printed. +# +# --report-path must be absolute. chainsaw runs a hook with its working +# directory set to the generated test directory, so a relative path would write +# the report into uptest's temp dir and it would be lost. +"${CHAINSAW}" test "${BEHAVIOR_DIR}" --parallel 1 --quiet \ + --report-format JUNIT-TEST \ + --report-path "${PROJECT_ROOT}" \ + --report-name junit \ + ${CHAINSAW_ARGS:-} diff --git a/test/setup.sh b/test/setup.sh index 5d024b9..c2d129d 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -1,59 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -echo "Granting provider CRD access for safe-start capability..." - -SA_NAME="" -for i in $(seq 1 60); do - SA_NAME=$(${KUBECTL} get sa -n crossplane-system -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep '^provider-template-' | head -1 || true) - if [[ -n "${SA_NAME}" ]]; then - break - fi - echo "Waiting for provider service account to be created... (attempt ${i}/60)" - sleep 2 -done - -if [[ -z "${SA_NAME}" ]]; then - echo "FAIL: could not find provider-template service account after 120s" - exit 1 -fi -echo "Found provider service account: ${SA_NAME}" - -# Required by safe-start gate: provider needs CRD list/watch access -cat <