feat: add e2e tests

Makefile

@@ -44,6 +44,17 @@ XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/crossplane
 XPKGS = provider-template
 -include build/makelib/xpkg.mk
 
+# ====================================================================================
+# Setup Uptest
+
+CROSSPLANE_VERSION ?= 2.3.4
+-include build/makelib/local.xpkg.mk
+-include build/makelib/controlplane.mk
+
+UPTEST_LOCAL_DEPLOY_TARGET = local.xpkg.deploy.provider.$(PROJECT_NAME)
+UPTEST_INPUT_MANIFESTS = test/e2e/00-lifecycle.yaml
+-include build/makelib/uptest.mk
+
 # NOTE(hasheddan): we force image building to happen prior to xpkg build so that
 # we ensure image is present in daemon.
 xpkg.build.provider-template: do.build.images
@@ -52,9 +63,6 @@ fallthrough: submodules
 	@echo Initial setup complete. Running make again . . .
 	@make
 
-# integration tests
-e2e.run: test-integration
-
 # Run integration tests.
 test-integration: $(KIND) $(KUBECTL) $(CROSSPLANE_CLI) $(HELM3)
 	@$(INFO) running integration tests using kind $(KIND_VERSION)

README.md

@@ -31,9 +31,43 @@ with the following features that are meant to be refactored:
 5. Run `make reviewable` to run code generation, linters, and tests.
 5. Run `make build` to build the provider.
 
+## Testing
+
+```shell
+make test      # unit tests
+make e2e       # end-to-end tests against a kind control plane
+make uptest    # only the e2e tests, reusing a control plane that is already up
+```
+
+`make e2e` runs two suites, and which one a new test belongs in depends on what
+it checks:
+
+- **`test/e2e/`** — the managed resource lifecycle: create, observe, update,
+  import, delete. [uptest] generates these, so to cover a **new managed resource
+  type** you add it to `test/e2e/00-lifecycle.yaml` as another YAML document
+  with a `uptest.upbound.io/conditions` annotation. Point
+  `uptest.upbound.io/post-assert-hook` at a script to assert more than
+  conditions.
+- **`test/behavior/`** — everything else: drift, the pause annotation,
+  credential resolution, error paths. To cover a **controller behaviour**, add
+  `test/behavior/<name>/chainsaw-test.yaml`; it is picked up automatically.
+  Prefer declarative [chainsaw] operations — `apply`, `assert`, `patch`,
+  `delete`, and `error` (which passes only when a resource is *absent*).
+
+Validate a new chainsaw test before running it:
+
+```shell
+.cache/tools/*/chainsaw-* lint test -f test/behavior/<name>/chainsaw-test.yaml
+```
+
+`test/README.md` is the reference: every uptest annotation and where it is
+documented, the make variables, what each test covers, and the known gotchas.
+
 Refer to Crossplane's [CONTRIBUTING.md] file for more information on how the
 Crossplane community prefers to work. The [Provider Development][provider-dev]
 guide may also be of use.
 
+[uptest]: https://github.com/crossplane/uptest
+[chainsaw]: https://kyverno.github.io/chainsaw/
 [CONTRIBUTING.md]: https://github.com/crossplane/crossplane/blob/master/CONTRIBUTING.md
 [provider-dev]: https://github.com/crossplane/crossplane/blob/master/contributing/guide-provider-development.md

test/README.md

@@ -0,0 +1,140 @@
+# End-to-end tests
+
+Reference for the two suites. The root `README.md` covers how to run them and
+where a new test belongs.
+
+```
+test/
+├── setup.sh                 # runs once before the tests; applies the ProviderConfigs
+├── e2e/                     # resource lifecycle, generated by uptest
+│   ├── 00-lifecycle.yaml    # test input; its annotations configure uptest
+│   └── hooks/
+│       ├── post-assert-lifecycle.sh   # runs after the apply assertions
+│       └── pre-delete-lifecycle.sh    # runs before deletion
+└── behavior/                # controller behaviour, plain chainsaw tests
+    ├── drift/
+    ├── pause/
+    ├── provider-config/
+    ├── cluster-provider-config/
+    └── unsupported-config-kind/
+```
+
+Neither suite needs Makefile wiring: `uptest.mk:68` already makes the `uptest`
+target depend on `$(CHAINSAW)`, and `uptest.mk:54` exports `CHAINSAW` into the
+environment the hooks run in, so `post-assert-lifecycle.sh` can invoke chainsaw
+on `test/behavior` directly.
+
+While iterating:
+
+```bash
+make e2e UPTEST_SKIP_DELETE=true      # leave resources behind to inspect them
+make uptest UPTEST_RENDER_ONLY=true   # render the chainsaw files without running
+```
+
+`UPTEST_RENDER_ONLY` prints where it wrote. Reach for it before debugging a
+failure — the generated step is often not what the annotations suggest.
+
+## Fields in `00-lifecycle.yaml`
+
+Everything under `metadata.annotations` configures uptest, and **not all of these
+are documented — three exist only in uptest's source:**
+
+| Annotation | Purpose | Where it is documented |
+| --- | --- | --- |
+| `uptest.upbound.io/timeout` | Per-resource timeout, overriding `--default-timeout`. | uptest README |
+| `uptest.upbound.io/conditions` | Comma-separated conditions to assert, overriding `--default-conditions`. | uptest README |
+| `uptest.upbound.io/pre-assert-hook` | Script to run after apply, before assertions. Path relative to the manifest. | uptest README, *Hooks* |
+| `uptest.upbound.io/post-assert-hook` | Script to run after assertions. | uptest README, *Hooks* |
+| `uptest.upbound.io/pre-delete-hook` | Script to run before deletion. | uptest README, *Hooks* |
+| `uptest.upbound.io/post-delete-hook` | Script to run after deletion. Not used here. | uptest README, *Hooks* |
+| `uptest.upbound.io/update-parameter` | JSON merged into `spec.forProvider` for the update step. | Source only — `internal/config/config.go`, `AnnotationKeyUpdateParameter` |
+| `uptest.upbound.io/disable-import` | Skips the import step for this resource. | Source only — `AnnotationKeyDisableImport` |
+| `meta.upbound.io/example-id` | Marks the *root resource*: `<first segment of API group>/<version>/<lowercased kind>`. | Source only — `AnnotationKeyExampleID` |
+
+When an annotation is not in the README, [`internal/config/config.go`][cfg] is the
+authority, and [`internal/templates/`][tmpl] shows how each value is actually used.
+CLI flags and their defaults come from `.cache/tools/*/uptest-* e2e --help`.
+
+`.cache/` is not part of this repo — the build submodule creates it as a tool cache
+(`common.mk:191-195` builds the path, `k8s_tools.mk` downloads into it). It is
+gitignored and disposable, and tool versions are pinned in `k8s_tools.mk`, so
+bumping one is a submodule change.
+
+### Make variables
+
+| Variable | Purpose | Defined in |
+| --- | --- | --- |
+| `UPTEST_INPUT_MANIFESTS` | Manifests to test. | this repo's `Makefile` |
+| `UPTEST_LOCAL_DEPLOY_TARGET` | Target that deploys the package first. Required — `uptest.mk` raises `$(error)` without it. | `uptest.mk:18` |
+| `UPTEST_SETUP_SCRIPT` | Setup script; defaults to `test/setup.sh`. | `uptest.mk:67` |
+| `UPTEST_SKIP_UPDATE` / `_IMPORT` / `_DELETE` | Skip individual steps. | `uptest.mk:27-40` |
+| `UPTEST_RENDER_ONLY` | Generate the chainsaw files without running them. | `uptest.mk:47` |
+| `CROSSPLANE_VERSION` | Chart version to install. Required — `controlplane.mk` defaults it to empty. | `controlplane.mk:17` |
+| `KIND_CLUSTER_NAME` | Control plane name; defaults to `local-dev`. | `controlplane.mk:15` |
+
+## Coverage
+
+| Test | Behaviour under test |
+| --- | --- |
+| `e2e/` generated steps | Apply, condition assertions, import, delete — for both a namespaced and a cluster scoped config |
+| `e2e/hooks/post-assert-lifecycle.sh` | `status.atProvider` populated, external-name set, spec update propagates, missing ProviderConfig reports `Synced=False` |
+| `e2e/hooks/pre-delete-lifecycle.sh` | Resource is `Ready` and `spec == status` before deletion |
+| `behavior/drift` | Controller repairs corrupted `status.atProvider` with no spec change |
+| `behavior/pause` | `crossplane.io/paused` stops reconciliation; `Ready` stays sticky; spec changes ignored until unpaused |
+| `behavior/provider-config` | Namespaced `ProviderConfig` credential path; usage recorded; an in-use config cannot be deleted |
+| `behavior/cluster-provider-config` | Same for `ClusterProviderConfig`. Its usage is still a *namespaced* `ProviderConfigUsage` — there is no cluster scoped usage type |
+| `behavior/unsupported-config-kind` | An unsupported `providerConfigRef.kind` reports a useful error and the resource still deletes |
+
+## Why the tests are shaped this way
+
+Read these before "fixing" something here that looks wrong.
+
+**`--default-conditions` is `Ready` only,** so asserting `Synced` needs
+`conditions: "Ready,Synced"` explicitly. Without it a resource that is `Ready=True`
+but `Synced=False` passes — conditions are sticky, so `Ready` keeps its last value
+after reconciliation starts failing.
+
+**uptest's update step is skipped,** because it is only generated for the resource
+carrying `meta.upbound.io/example-id`. Adding that annotation does not help:
+`uptest.upbound.io/update-parameter` must be valid JSON, but uptest v2.2.0
+interpolates it raw into a double-quoted shell command, so its quotes are stripped
+before kubectl sees them. Update coverage lives in `post-assert-lifecycle.sh`
+instead.
+
+**There is no `Create()` test,** because that branch is unreachable.
+crossplane-runtime's default `NameAsExternalName` initializer stamps the
+external-name annotation before the first `Observe()`, and `Observe()` treats an
+empty external-name as "does not exist" — so new resources route through
+`Update()`.
+
+## When a run looks broken but isn't
+
+**`Ready` lagging `Synced` by a minute is expected.** `Synced` is set as soon as
+`Update()` succeeds; `Ready` is only set inside `Observe()`, which already ran. The
+next `Observe()` arrives on the poll interval (`--poll`, default `1m`), because a
+status-only write raises no event the controller acts on.
+
+**Do not run `make e2e -j`.** Its prerequisites are order-dependent but declared as
+plain prerequisites, so a parallel make races. Upstream `uptest.mk` behaviour.
+
+**An interrupted run wedges the cluster,** and every later run then hangs or fails
+with resources "not found". The import step pauses the resource and scales the
+provider to 0; killed mid-window it leaves `replicas: 0` and a paused resource
+whose finalizer can never be released. `make controlplane.down` is the reliable
+fix; patching the `DeploymentRuntimeConfig` replicas back to 1 and removing the
+`crossplane.io/paused` annotation recovers it in place.
+
+## chainsaw 0.2.13 constraints
+
+Two limits of the pinned version, both caught by `chainsaw lint`:
+
+- `description` is allowed on a step, but not on an individual operation.
+- `patch` has no `subresource` field, so patching a status needs a `script` with
+  `kubectl --subresource=status`.
+
+A leaked managed resource is the other trap: chainsaw cleans up what the test
+applied, but anything left behind hangs the lifecycle suite's
+`kubectl wait managed --all --for=delete` minutes later, far from the cause.
+
+[cfg]: https://github.com/crossplane/uptest/blob/main/internal/config/config.go
+[tmpl]: https://github.com/crossplane/uptest/tree/main/internal/templates

test/behavior/cluster-provider-config/chainsaw-test.yaml

@@ -0,0 +1,107 @@
+# ClusterProviderConfig credentials, and that an in-use config cannot be deleted.
+# Uses a dedicated config -- deletion is one-way.
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: cluster-provider-config
+spec:
+  timeouts:
+    apply: 1m
+    assert: 2m
+    delete: 2m
+    exec: 1m
+  steps:
+    - name: use a cluster scoped config
+      try:
+        - apply:
+            resource:
+              apiVersion: template.crossplane.io/v1alpha1
+              kind: ClusterProviderConfig
+              metadata:
+                name: behaviour-cpc-inuse
+              spec:
+                credentials:
+                  source: Secret
+                  secretRef:
+                    namespace: default
+                    name: example-provider-secret
+                    key: credentials
+        - apply:
+            resource:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              metadata:
+                name: behaviour-cpc-user
+                namespace: default
+              spec:
+                forProvider:
+                  configurableField: cluster-scoped-creds
+                providerConfigRef:
+                  name: behaviour-cpc-inuse
+                  kind: ClusterProviderConfig
+        - assert:
+            resource:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              metadata:
+                name: behaviour-cpc-user
+                namespace: default
+              status:
+                atProvider:
+                  configurableField: cluster-scoped-creds
+                ((conditions[?type == 'Ready'])[0]):
+                  status: "True"
+                ((conditions[?type == 'Synced'])[0]):
+                  status: "True"
+
+    - name: usage is recorded and the config is held
+      try:
+        # Usage is namespaced even for a cluster scoped config.
+        - assert:
+            resource:
+              apiVersion: template.crossplane.io/v1alpha1
+              kind: ProviderConfigUsage
+              metadata:
+                namespace: default
+              providerConfigRef:
+                name: behaviour-cpc-inuse
+                kind: ClusterProviderConfig
+        - assert:
+            resource:
+              apiVersion: template.crossplane.io/v1alpha1
+              kind: ClusterProviderConfig
+              metadata:
+                name: behaviour-cpc-inuse
+                (contains(finalizers, 'in-use.crossplane.io')): true
+
+    - name: deleting it while in use must not remove it
+      try:
+        # Script: chainsaw's delete would block until the object is gone.
+        - script:
+            content: |
+              ${KUBECTL:-kubectl} delete clusterproviderconfig behaviour-cpc-inuse --wait=false
+        - sleep:
+            duration: 10s
+        - assert:
+            resource:
+              apiVersion: template.crossplane.io/v1alpha1
+              kind: ClusterProviderConfig
+              metadata:
+                name: behaviour-cpc-inuse
+                (deletionTimestamp != null): true
+
+    - name: releasing the last user completes the deletion
+      try:
+        - delete:
+            ref:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              name: behaviour-cpc-user
+              namespace: default
+        - error:
+            timeout: 2m
+            resource:
+              apiVersion: template.crossplane.io/v1alpha1
+              kind: ClusterProviderConfig
+              metadata:
+                name: behaviour-cpc-inuse

test/behavior/drift/chainsaw-test.yaml

@@ -0,0 +1,68 @@
+# The controller repairs drifted external state without a spec change.
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: drift-correction
+spec:
+  timeouts:
+    apply: 1m
+    assert: 3m
+    delete: 2m
+  steps:
+    - name: create the resource
+      try:
+        - apply:
+            resource:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              metadata:
+                name: behaviour-drift
+                namespace: default
+              spec:
+                forProvider:
+                  configurableField: drift-value
+                providerConfigRef:
+                  name: example
+                  kind: ProviderConfig
+        - assert:
+            resource:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              metadata:
+                name: behaviour-drift
+                namespace: default
+              status:
+                atProvider:
+                  configurableField: drift-value
+                ((conditions[?type == 'Ready'])[0]):
+                  status: "True"
+                ((conditions[?type == 'Synced'])[0]):
+                  status: "True"
+
+    - name: corrupt the observed state
+      try:
+        # A script because chainsaw 0.2.13's patch cannot target a subresource.
+        - script:
+            content: |
+              ${KUBECTL:-kubectl} patch mytype behaviour-drift -n default \
+                --subresource=status --type=merge \
+                -p '{"status":{"atProvider":{"configurableField":"drifted"}}}'
+
+    - name: expect the controller to repair it
+      try:
+        # Status-only writes raise no event, so recovery waits for --poll (1m).
+        - assert:
+            timeout: 3m
+            resource:
+              apiVersion: sample.template.crossplane.io/v1alpha1
+              kind: MyType
+              metadata:
+                name: behaviour-drift
+                namespace: default
+              status:
+                atProvider:
+                  configurableField: drift-value
+                ((conditions[?type == 'Ready'])[0]):
+                  status: "True"
+                ((conditions[?type == 'Synced'])[0]):
+                  status: "True"