Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 44 additions & 15 deletions .github/workflows/e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ on:
options: [aio, release-acceptance, fleet, fleet-cleanup, site-cleanup, windows-installer-preflight]
default: aio
aio-releases:
description: "AIO scenario only: comma-separated releases (e.g. 2607,2608). Fleet uses fixed 2607 and 2608 targets."
description: "AIO scenario only: comma-separated releases (e.g. 2608,2609). Fleet uses fixed 2607 and 2609 targets."
type: string
default: "2608"
default: "2609"
environment:
description: GitHub Environment whose secrets/approvers apply.
type: string
Expand Down Expand Up @@ -76,6 +76,10 @@ on:
description: "AIO scenario only: enabled,disabled. Disabled omits Secret Sync, OIDC issuer and workload identity. Fleet has Secret Sync disabled."
type: string
default: "enabled"
opcua-modes:
description: "Source AIO scenario only: Stable,Disabled. Disabled requires explicit tests limited to aio-install,enable-secretsync,sync-secrets,aio-upgrade."
type: string
default: "Stable"
tests:
description: "Subset of integration test files to run, comma-separated. Each value maps to one tests/integration/test_<value>_manifest.py file and narrows which manifest gets deployed onto the cluster. Empty = run all. Valid values: aio-install, enable-secretsync, sync-secrets, opc-ua-solution, dataflow-sample, aio-resources, resource-set-samples, aio-upgrade. Useful for demos and focused debugging when paired with keep-cluster-alive-minutes."
type: string
Expand Down Expand Up @@ -291,6 +295,7 @@ jobs:
pytest-paths: ${{ steps.v.outputs.pytest_paths }}
isolate-dataflow-sample: ${{ steps.v.outputs.isolate_dataflow_sample }}
secret-sync-modes: ${{ steps.v.outputs.secret_sync_modes }}
opcua-modes: ${{ steps.v.outputs.opcua_modes }}
max-parallel: ${{ steps.v.outputs.max_parallel }}
published-mode: ${{ steps.v.outputs.published_mode }}
published-release: ${{ steps.v.outputs.published_release }}
Expand Down Expand Up @@ -319,6 +324,7 @@ jobs:
INPUT_RELEASES: "${{ inputs.aio-releases }}"
INPUT_UPGRADE_TO: "${{ inputs.upgrade-to }}"
INPUT_SECRET_SYNC_MODES: "${{ inputs.secret-sync-modes }}"
INPUT_OPCUA_MODES: "${{ inputs.opcua-modes }}"
INPUT_TESTS: "${{ inputs.tests }}"
INPUT_PUBLISHED_RELEASE: "${{ inputs.published-release }}"
INPUT_PUBLISHED_SOURCE_SHA: "${{ inputs.published-source-sha }}"
Expand Down Expand Up @@ -417,11 +423,16 @@ jobs:
"secret-sync-modes entries not recognized: "
f"{invalid_secret_sync_modes}. Valid: ['disabled', 'enabled']."
)
opcua_modes = list(dict.fromkeys(
mode.strip() for mode in os.environ["INPUT_OPCUA_MODES"].split(",") if mode.strip()
))
if not opcua_modes or set(opcua_modes) - {"Stable", "Disabled"}:
sys.exit("opcua-modes must select Stable, Disabled, or both.")
persistent = bool(rg_in)
if cl_in and (len(rs) > 1 or len(secret_sync_modes) > 1):
if cl_in and (len(rs) > 1 or len(secret_sync_modes) > 1 or len(opcua_modes) > 1):
sys.exit(
"cluster-name cannot be combined with a matrix that has "
"multiple AIO releases or Secret Sync modes. Each matrix "
"multiple AIO releases or Secret Sync modes or OPC UA modes. Each matrix "
"cell registers its own cluster."
)
# Validate upgrade-to against shipped versions. Same-version cells
Expand Down Expand Up @@ -521,8 +532,10 @@ jobs:
if published_mode:
if len(rs) != 1:
sys.exit("Published E2E requires exactly one aio-releases entry.")
if published_journey == "guided" and rs != ["2608"]:
sys.exit("Published guided E2E currently requires aio-releases=2608.")
if published_journey == "guided" and rs != ["2609"]:
sys.exit("Published guided E2E currently requires aio-releases=2609.")
if opcua_modes != ["Stable"]:
sys.exit("Published E2E requires opcua-modes=Stable.")
if not persistent or cl_in:
sys.exit(
"Published E2E requires an existing resource group and an "
Expand All @@ -544,7 +557,8 @@ jobs:
if candidate_mode:
oid = target_inputs.get("custom-locations-oid", "")
if (
rs != ["2608"] or secret_sync_modes != ["enabled"] or tests_in
rs != ["2609"] or secret_sync_modes != ["enabled"] or tests_in
or opcua_modes != ["Stable"]
or published_mode or published_source_sha or published_journey != "configured"
or persistent or cl_in or upgrade_to or not isinstance(oid, str) or oid.strip()
or os.environ["INPUT_SKIP_TEARDOWN"].lower() == "true"
Expand All @@ -555,8 +569,17 @@ jobs:
"Leave the single Site options at their defaults."
)
secret_sync_modes = ["disabled", "enabled", "existing"]
if "Disabled" in opcua_modes and (
not tests_in
or set(tests_in) - {"aio-install", "enable-secretsync", "sync-secrets", "aio-upgrade"}
):
sys.exit(
"Disabled OPC UA requires explicit tests limited to "
"aio-install,enable-secretsync,sync-secrets,aio-upgrade."
)
print(f"versions={json.dumps(rs)}")
print(f"secret_sync_modes={json.dumps(secret_sync_modes)}")
print(f"opcua_modes={json.dumps(opcua_modes)}")
print(f"max_parallel={3 if candidate_mode else 1 if persistent or published_mode else 20}")
print(f"upgrade_to={upgrade_to}")
print(f"persistent={'true' if persistent else 'false'}")
Expand Down Expand Up @@ -622,7 +645,7 @@ jobs:
print(f"Site cases use {mode} resource groups.")

e2e:
name: ${{ needs.prep.outputs.candidate-mode == 'true' && format('Site case ({0})', fromJSON('{"disabled":"site-aio","enabled":"site-combined","existing":"site-existing-secretsync"}')[matrix.secret-sync-mode]) || format('e2e ({0}, {1})', matrix.aio-release, matrix.secret-sync-mode) }}
name: ${{ needs.prep.outputs.candidate-mode == 'true' && format('Site case ({0})', fromJSON('{"disabled":"site-aio","enabled":"site-combined","existing":"site-existing-secretsync"}')[matrix.secret-sync-mode]) || format('e2e ({0}, {1}{2})', matrix.aio-release, matrix.secret-sync-mode, matrix.opcua-mode == 'Disabled' && ', OPC UA Disabled' || '') }}
needs: [prep, site-groups]
if: ${{ !cancelled() && needs.prep.result == 'success' && (needs.prep.outputs.candidate-mode != 'true' || needs.site-groups.result == 'success') }}
runs-on: ubuntu-latest
Expand All @@ -642,18 +665,20 @@ jobs:
# Ephemeral: rg_key is per-run and the release suffix distinguishes
# cells, so matrix fan-out runs in parallel.
# Candidate cases take both values from site-groups.
group: e2e-${{ needs.site-groups.outputs.rg-key || needs.prep.outputs.rg-key }}${{ (needs.prep.outputs.persistent == 'true' || needs.site-groups.outputs.groups == 'persistent') && '-persistent' || format('-{0}-secretsync-{1}', matrix.aio-release, matrix.secret-sync-mode) }}
group: e2e-${{ needs.site-groups.outputs.rg-key || needs.prep.outputs.rg-key }}${{ (needs.prep.outputs.persistent == 'true' || needs.site-groups.outputs.groups == 'persistent') && '-persistent' || format('-{0}-secretsync-{1}-opcua-{2}', matrix.aio-release, matrix.secret-sync-mode, matrix.opcua-mode) }}
cancel-in-progress: false
strategy:
fail-fast: false
max-parallel: ${{ fromJSON(needs.site-groups.outputs.max-parallel || needs.prep.outputs.max-parallel) }}
matrix:
aio-release: ${{ fromJSON(needs.prep.outputs.versions) }}
secret-sync-mode: ${{ fromJSON(needs.prep.outputs.secret-sync-modes) }}
opcua-mode: ${{ fromJSON(needs.prep.outputs.opcua-modes) }}
env:
PERSISTENT_RG: ${{ needs.prep.outputs.persistent }}
PUBLISHED_JOURNEY: ${{ needs.prep.outputs.published-journey }}
CANDIDATE_MODE: ${{ needs.prep.outputs.candidate-mode }}
E2E_OPCUA_MODE: ${{ matrix.opcua-mode }}
DEBUG_USER_OID: ${{ secrets.DEBUG_USER_OID }}
FLEET_CANDIDATE: ${{ inputs.candidate }}
FLEET_RUN_ID: ${{ github.run_id }}
Expand Down Expand Up @@ -822,6 +847,10 @@ jobs:
LONG_SUFFIX="${RUN_ID}-${RUN_ATTEMPT}-${SAFE_VER}-secretsync-${SECRET_SYNC_MODE}"
SHORT_RUN="${RUN_ID: -8}-${RUN_ATTEMPT}"
RESOURCE_SUFFIX="${SHORT_RUN}-${SAFE_VER}-${SYNC_SUFFIX}"
if [[ "${E2E_OPCUA_MODE:-Stable}" == "Disabled" ]]; then
LONG_SUFFIX="${LONG_SUFFIX}-opcua-off"
RESOURCE_SUFFIX="${RESOURCE_SUFFIX}-opcua-off"
fi
# Candidate cases use the masked names owned by their run attempt.
RG="${RG_IN:-${FLEET_RESOURCE_GROUP:-rg-e2e-$LONG_SUFFIX}}"
CL="${CL_IN:-${FLEET_CLUSTER_NAME:-e2e-$RESOURCE_SUFFIX}}"
Expand Down Expand Up @@ -1743,7 +1772,7 @@ jobs:
run: |
set -euo pipefail
pytest $PYTEST_PATHS -v -m integration --tb=long \
--junitxml="e2e-results-${AIO_RELEASE}-secretsync-${SECRET_SYNC_MODE}.xml"
--junitxml="e2e-results-${AIO_RELEASE}-secretsync-${SECRET_SYNC_MODE}-opcua-${E2E_OPCUA_MODE}.xml"

- name: Deploy AIO through the published engine and package
id: deploy
Expand Down Expand Up @@ -2258,8 +2287,8 @@ jobs:
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always() && needs.prep.outputs.published-mode != 'true'
with:
name: e2e-results-${{ matrix.aio-release }}-secretsync-${{ matrix.secret-sync-mode }}
path: e2e-results-${{ matrix.aio-release }}-secretsync-${{ matrix.secret-sync-mode }}.xml
name: e2e-results-${{ matrix.aio-release }}-secretsync-${{ matrix.secret-sync-mode }}-opcua-${{ matrix.opcua-mode }}
path: e2e-results-${{ matrix.aio-release }}-secretsync-${{ matrix.secret-sync-mode }}-opcua-${{ matrix.opcua-mode }}.xml

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always() && needs.prep.outputs.published-mode == 'true' && env.CANDIDATE_MODE != 'true'
Expand Down Expand Up @@ -2316,13 +2345,13 @@ jobs:
run: |
set -euo pipefail
pytest $PYTEST_PATHS -v -m integration --tb=long \
--junitxml="e2e-results-${AIO_RELEASE_FROM}-to-${AIO_RELEASE_TO}-secretsync-${SECRET_SYNC_MODE}.xml"
--junitxml="e2e-results-${AIO_RELEASE_FROM}-to-${AIO_RELEASE_TO}-secretsync-${SECRET_SYNC_MODE}-opcua-${E2E_OPCUA_MODE}.xml"

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always() && needs.prep.outputs.upgrade-to != '' && needs.prep.outputs.upgrade-to != matrix.aio-release
with:
name: e2e-results-${{ matrix.aio-release }}-to-${{ needs.prep.outputs.upgrade-to }}-secretsync-${{ matrix.secret-sync-mode }}
path: e2e-results-${{ matrix.aio-release }}-to-${{ needs.prep.outputs.upgrade-to }}-secretsync-${{ matrix.secret-sync-mode }}.xml
name: e2e-results-${{ matrix.aio-release }}-to-${{ needs.prep.outputs.upgrade-to }}-secretsync-${{ matrix.secret-sync-mode }}-opcua-${{ matrix.opcua-mode }}
path: e2e-results-${{ matrix.aio-release }}-to-${{ needs.prep.outputs.upgrade-to }}-secretsync-${{ matrix.secret-sync-mode }}-opcua-${{ matrix.opcua-mode }}.xml

- name: Keep cluster alive (pre-teardown hold)
if: always() && inputs.keep-cluster-alive-minutes > 0
Expand Down
68 changes: 53 additions & 15 deletions docs/aio-releases.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ Azure IoT Operations (AIO) ships on a release cadence. Each AIO release pins spe
## How release selection works

```
site.properties.aioRelease: "2608"
site.properties.aioRelease: "2609"
│
▼
workspaces/iot-operations/parameters/aio-releases/2608.yaml
workspaces/iot-operations/parameters/aio-releases/2609.yaml
│
▼ (siteops auto-forwards matching params to Bicep)
templates/aio/enablement.bicep ──► cert-manager, secret store extensions
Expand All @@ -22,8 +22,8 @@ Each AIO release YAML has a stable schema at the top level.
differ even when two AIO releases use the same ARM API generation:

```yaml
# parameters/aio-releases/2608.yaml
aioVersion: "1.4.73" # AIO extension version pinned in Arc
# parameters/aio-releases/2609.yaml
aioVersion: "1.4.112" # AIO extension version pinned in Arc
aioTrain: stable # Extension release train
aioApiVersion: "2026-07-01" # Microsoft.IoTOperations/instances API version
adrApiVersion: "2026-04-01" # Microsoft.DeviceRegistry/namespaces API version
Expand All @@ -32,20 +32,20 @@ aioReleaseConfiguration:
wasmGraphControllerMqttTrust: true
resources:
opcUaConnector:
version: "1.4.10"
certManagerVersion: "1.0.0"
version: "1.4.14"
certManagerVersion: "1.1.2"
certManagerTrain: stable
certManagerConfigurationOverrides:
trust-manager.secretTargets.enabled: "false"
trust-manager.secretTargets.authorizedSecretsAll: "false"
secretStoreVersion: "1.5.2"
secretStoreVersion: "1.5.3"
secretStoreTrain: stable
```

The `aioApiVersion` and `adrApiVersion` values route CREATE and UPDATE operations through their matching versioned modules (for example `templates/aio/modules/instance-2026-07-01.bicep` and `templates/deps/modules/adr-ns-2026-04-01.bicep`). Configuration override objects specific to an AIO release are forwarded automatically to templates that declare them. Bicep cannot parameterize API version strings, so the dispatchers use `@allowed` + conditional modules. See [Adding a new AIO release](#adding-a-new-aio-release) below.

The AIO release 2608 configuration also deploys the OPC UA connector template
and supplies the MQTT trust settings required by its WebAssembly graph
The AIO release 2608 and 2609 configurations also deploy the OPC UA connector template
and supply the MQTT trust settings required by the WebAssembly graph
controller. Install and upgrade use the same typed connector template module.
Settings under `aioReleaseConfiguration.extension` belong to the AIO Arc
extension. Settings under `aioReleaseConfiguration.resources` describe child
Expand All @@ -60,6 +60,13 @@ AIO releases where it is not yet a default of the API generation.
`extension.wasmGraphControllerMqttTrust` enables the derived MQTT trust settings
required by the WebAssembly graph controller.

The connector template is deployed only when its version is configured and
the instance's effective OPC UA feature mode is not `Disabled`.
`features.opcua` takes precedence over the legacy `features.connectors` alias.
When `opcua` is absent, `connectors` supplies the feature. An omitted mode
uses `Stable`. Upgrade reads the existing instance's resolved features
rather than the Site's desired features.

For a Site override, keep the map under `parameters`:

```yaml
Expand All @@ -85,10 +92,10 @@ name: munich-prod
inherits: base-site.yaml

properties:
aioRelease: "2608" # must match parameters/aio-releases/2608.yaml
aioRelease: "2609" # must match parameters/aio-releases/2609.yaml
```

If not specified, the Site inherits the value from its parent template. Every committed Site in this workspace inherits `base-site.yaml`, directly or through a shared template, and `base-site.yaml` declares `"2608"`. A manifest that loads the release file fails preparation for a Site that carries no `aioRelease` at all, reporting that the Site does not carry the property the path selects on.
If not specified, the Site inherits the value from its parent template. Every committed Site in this workspace inherits `base-site.yaml`, directly or through a shared template, and `base-site.yaml` declares `"2609"`. A manifest that loads the release file fails preparation for a Site that carries no `aioRelease` at all, reporting that the Site does not carry the property the path selects on.

## Available AIO releases

Expand All @@ -103,17 +110,48 @@ Every file in `workspaces/iot-operations/parameters/aio-releases/` is a shipped
| `2605` | `2026-03-01` | `2026-04-01` | |
| `2606` | `2026-03-01` | `2026-04-01` | |
| `2607` | `2026-07-01` | `2026-04-01` | OPC connector template affected by [Microsoft known issue 1330](https://learn.microsoft.com/azure/iot-operations/troubleshoot/known-issues#opc-connector-template-missing) |
| `2608` | `2026-07-01` | `2026-04-01` | Default in `base-site.yaml` |
| `2608` | `2026-07-01` | `2026-04-01` | Selectable by an explicit pin |
| `2609` | `2026-07-01` | `2026-04-01` | Default in `base-site.yaml`. Pins from AIO CLI v2.10.0 |

Source of truth for every pinned version number is the YAML itself. Check it against the [IoT Operations release matrix](https://github.com/Azure/azure-iot-ops-cli-extension/wiki/IoT-Operations-versions) before shipping a new one.

### Select 2609 or 2608

2609 is the default for configured Sites and guided installation.
2608 remains selectable through a Site pin or the guided `aioRelease` input.
2609 uses the same AIO and ADR API versions as 2608.
The [AIO CLI v2.10.0 release notes](https://github.com/Azure/azure-iot-ops-cli-extension/releases/tag/v2.10.0)
record the fix that skips the OPC UA connector template when OPC UA is disabled.
Scale Kit applies that guard to both install and upgrade, for 2608 as well as
2609, because both releases deploy the same connector template.

For a configured Site, set:

```yaml
properties:
aioRelease: "2609"
```

For guided installation on an existing Arc cluster, select 2609 with an input:

```text
siteops deploy aio-install --source "official@<release>" --input "cluster=<Arc-cluster-resource-ID>" --input aioRelease=2609
```

Use a Scale Kit source release that includes `2609.yaml`.
To select 2608, set `properties.aioRelease: "2608"` on the Site or pass
`--input aioRelease=2608` to guided installation.
For an existing AIO installation, pin the Site to 2609 and follow the
[upgrade operation guide](../workspaces/iot-operations/manifests/aio-upgrade/README.md)
instead of running installation again.

## Upgrading an existing Site

Follow the [upgrade operation guide](../workspaces/iot-operations/manifests/aio-upgrade/README.md)
to prepare and deploy an AIO release change in place. The operation updates
extensions and the resources that the AIO release requires rather than reapplying the
greenfield instance configuration. An upgrade to 2608 adds the OPC UA
connector template.
greenfield instance configuration. Upgrades to 2608 or 2609 deploy the OPC UA
connector template when OPC UA is enabled.

### Catalog families and upgrade order

Expand All @@ -132,7 +170,7 @@ This applies to every resource area a Site selects through `properties.resourceS
Azure IoT Operations supports upgrade to any patch of the same minor version, or to the next minor version. Other transitions (downgrades, jumps across several minor versions, and moves between preview and GA) require uninstall and reinstall. See [Upgrade Azure IoT Operations](https://learn.microsoft.com/azure/iot-operations/deploy-iot-ops/howto-upgrade) for the authoritative rules.

The optional E2E workflow can exercise selected upgrades between adjacent AIO releases,
such as `2607` to `2608`. A passing dispatch establishes its selected
such as `2608` to `2609`. A passing dispatch establishes its selected
integration assertions. It is not a general readiness or support
certification for another cluster or upgrade path.

Expand Down
Loading
Loading