From 3463d78c6ef41e39a4b0cb28f6964a33c469acd5 Mon Sep 17 00:00:00 2001 From: castrojo <1264109+castrojo@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:48:43 +0000 Subject: [PATCH 1/3] chore: import reference architectures Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- data/architectures/catalog.json | 90 ++- data/architectures/records/adobe.json | 13 +- data/architectures/records/allianz.json | 19 +- .../records/cern-scientific-computing.json | 12 +- data/architectures/records/colopl.json | 15 +- .../records/flipkart-chaos-engineering.json | 6 +- .../records/swisscom-cloud-native-telco.json | 5 +- .../records/swisscom-kubernetes-service.json | 7 +- data/architectures/records/zeiss.json | 13 +- data/members.json | 69 ++- data/metrics.json | 12 +- docs/architectures/adobe.md | 561 +++++------------- docs/architectures/allianz.md | 193 +++--- .../cern-scientific-computing.md | 22 +- docs/architectures/colopl.md | 77 +-- .../flipkart-chaos-engineering.md | 479 ++++----------- .../swisscom-cloud-native-telco.md | 359 ++++------- .../swisscom-kubernetes-service.md | 18 +- docs/architectures/zeiss.md | 189 ++---- 19 files changed, 734 insertions(+), 1425 deletions(-) diff --git a/data/architectures/catalog.json b/data/architectures/catalog.json index 024a6bb7..0a8a5c40 100644 --- a/data/architectures/catalog.json +++ b/data/architectures/catalog.json @@ -4,7 +4,11 @@ "title": "Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture", "organization": "Adobe", "summary": "Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for both infrastructure and developer experience.", - "industries": ["Software", "Digital Media", "Digital Experience"], + "industries": [ + "Software", + "Digital Media", + "Digital Experience" + ], "tags": [ "adobe", "ci-cd", @@ -13,7 +17,12 @@ "developer-platforms", "developer-experience" ], - "projects": ["Kubernetes", "Helm", "Argo", "Backstage"], + "projects": [ + "Kubernetes", + "Helm", + "Argo", + "Backstage" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -30,9 +39,22 @@ "title": "Enabling Allianz Direct's scaling through Platform Engineering", "organization": "Allianz Direct", "summary": "The PE team is internally organised in DDOs - Distributed DevOps, where each ddo owns a Domain. This setup follows the reverse Conway manoeuver where the reference architecture structure dictates the teams setup. The PE team consist of 13 m", - "industries": ["Financial", "Insurance"], - "tags": ["finance", "insurance", "europe", "public_cloud", "aws"], - "projects": ["Kubernetes", "Helm", "Argo"], + "industries": [ + "Financial", + "Insurance" + ], + "tags": [ + "finance", + "insurance", + "europe", + "public_cloud", + "aws" + ], + "projects": [ + "Kubernetes", + "Helm", + "Argo" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -46,8 +68,16 @@ "title": "A Cloud Native Scientific Computing Platform for CERN NextGen AI", "organization": "CERN", "summary": "Some additional projects outside the CNCF are essential to this deployment. [MLFlow](https://mlflow.org/), a project under the Linux Foundation used for model management and versioning [Nvidia GPU operator](https://github.com/NVIDIA/gpu-ope", - "industries": ["Research", "Public Sector"], - "tags": ["cloud_native", "private_cloud", "kubernetes", "switzerland"], + "industries": [ + "Research", + "Public Sector" + ], + "tags": [ + "cloud_native", + "private_cloud", + "kubernetes", + "switzerland" + ], "projects": [ "Argo", "Bootc", @@ -75,9 +105,18 @@ "title": "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with Agones on Kubernetes", "organization": "Colopl Inc.", "summary": "At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud Spanner as the backend database. While that combination is somewhat distinctive, the overall structure is not far from a typical web service.", - "industries": ["gaming"], - "tags": ["agones", "gaming", "kubernetes"], - "projects": ["Kubernetes", "Agones"], + "industries": [ + "gaming" + ], + "tags": [ + "agones", + "gaming", + "kubernetes" + ], + "projects": [ + "Kubernetes", + "Agones" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -92,7 +131,11 @@ "title": "From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering Platform on LitmusChaos", "organization": "Flipkart Internet Pvt. Ltd.", "summary": "Flipkart is one of India’s leading digital commerce companies, running hundreds of tightly coupled microservices across Kubernetes and VM workloads that must withstand the traffic surges of Big Billion Days and other festive sales. The Cent", - "industries": ["E-commerce", "Retail", "Digital Commerce"], + "industries": [ + "E-commerce", + "Retail", + "Digital Commerce" + ], "tags": [ "Chaos engineering", "Resilience", @@ -123,7 +166,10 @@ "title": "End-to-End Cloud Native Telco Platform Automation at Swisscom", "organization": "Swisscom (Switzerland) Ltd", "summary": "Swisscom has built a cloud native telco platform for the end-to-end automation of its 5G standalone core network and cross-domain resource orchestration. The architecture replaces traditional imperative network management (Jenkins pipelines", - "industries": ["Communication", "Telecommunication"], + "industries": [ + "Communication", + "Telecommunication" + ], "tags": [ "cloud_native", "telco", @@ -160,7 +206,12 @@ "title": "A modern and sovereign Private Cloud «Kubernetes Service» for Swiss-based enterprises.", "organization": "Swisscom (Switzerland) Ltd", "summary": "Our next generation Private Cloud Container as a Service offering «Kubernetes Service» for the B2B market addresses customer’s need for scalable and highly available Kubernetes workload as a flexible and secure IT foundation. It is part of ", - "industries": ["Communication", "Telecommunication", "Software", "Cloud"], + "industries": [ + "Communication", + "Telecommunication", + "Software", + "Cloud" + ], "tags": [ "service-delivery", "cloud_native", @@ -194,7 +245,10 @@ "title": "ZEISS Vision Care - Order Fulfillment", "organization": "ZEISS", "summary": "Our Business Enablement & IT team develops, designs, and operates the order fulfillment platform that underpins this process.", - "industries": ["optics", "optical-manufacturing"], + "industries": [ + "optics", + "optical-manufacturing" + ], "tags": [ "public-cloud", "mass-manufacturing", @@ -202,7 +256,13 @@ "microservices", "platform engineering" ], - "projects": ["Kubernetes", "Dapr", "KEDA", "OpenTelemetry", "Helm"], + "projects": [ + "Kubernetes", + "Dapr", + "KEDA", + "OpenTelemetry", + "Helm" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/adobe.json b/data/architectures/records/adobe.json index c0d42fb3..da4b795e 100644 --- a/data/architectures/records/adobe.json +++ b/data/architectures/records/adobe.json @@ -3,7 +3,11 @@ "title": "Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture", "organization": "Adobe", "summary": "Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for both infrastructure and developer experience.", - "industries": ["Software", "Digital Media", "Digital Experience"], + "industries": [ + "Software", + "Digital Media", + "Digital Experience" + ], "tags": [ "adobe", "ci-cd", @@ -12,7 +16,12 @@ "developer-platforms", "developer-experience" ], - "projects": ["Kubernetes", "Helm", "Argo", "Backstage"], + "projects": [ + "Kubernetes", + "Helm", + "Argo", + "Backstage" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/allianz.json b/data/architectures/records/allianz.json index 904b6bd2..cf03b667 100644 --- a/data/architectures/records/allianz.json +++ b/data/architectures/records/allianz.json @@ -3,9 +3,22 @@ "title": "Enabling Allianz Direct's scaling through Platform Engineering", "organization": "Allianz Direct", "summary": "The PE team is internally organised in DDOs - Distributed DevOps, where each ddo owns a Domain. This setup follows the reverse Conway manoeuver where the reference architecture structure dictates the teams setup. The PE team consist of 13 m", - "industries": ["Financial", "Insurance"], - "tags": ["finance", "insurance", "europe", "public_cloud", "aws"], - "projects": ["Kubernetes", "Helm", "Argo"], + "industries": [ + "Financial", + "Insurance" + ], + "tags": [ + "finance", + "insurance", + "europe", + "public_cloud", + "aws" + ], + "projects": [ + "Kubernetes", + "Helm", + "Argo" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/cern-scientific-computing.json b/data/architectures/records/cern-scientific-computing.json index a522ed58..0c98478e 100644 --- a/data/architectures/records/cern-scientific-computing.json +++ b/data/architectures/records/cern-scientific-computing.json @@ -3,8 +3,16 @@ "title": "A Cloud Native Scientific Computing Platform for CERN NextGen AI", "organization": "CERN", "summary": "Some additional projects outside the CNCF are essential to this deployment. [MLFlow](https://mlflow.org/), a project under the Linux Foundation used for model management and versioning [Nvidia GPU operator](https://github.com/NVIDIA/gpu-ope", - "industries": ["Research", "Public Sector"], - "tags": ["cloud_native", "private_cloud", "kubernetes", "switzerland"], + "industries": [ + "Research", + "Public Sector" + ], + "tags": [ + "cloud_native", + "private_cloud", + "kubernetes", + "switzerland" + ], "projects": [ "Argo", "Bootc", diff --git a/data/architectures/records/colopl.json b/data/architectures/records/colopl.json index 65cf424b..9cfceddc 100644 --- a/data/architectures/records/colopl.json +++ b/data/architectures/records/colopl.json @@ -3,9 +3,18 @@ "title": "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with Agones on Kubernetes", "organization": "Colopl Inc.", "summary": "At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud Spanner as the backend database. While that combination is somewhat distinctive, the overall structure is not far from a typical web service.", - "industries": ["gaming"], - "tags": ["agones", "gaming", "kubernetes"], - "projects": ["Kubernetes", "Agones"], + "industries": [ + "gaming" + ], + "tags": [ + "agones", + "gaming", + "kubernetes" + ], + "projects": [ + "Kubernetes", + "Agones" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/flipkart-chaos-engineering.json b/data/architectures/records/flipkart-chaos-engineering.json index c86490c1..cfcb9596 100644 --- a/data/architectures/records/flipkart-chaos-engineering.json +++ b/data/architectures/records/flipkart-chaos-engineering.json @@ -3,7 +3,11 @@ "title": "From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering Platform on LitmusChaos", "organization": "Flipkart Internet Pvt. Ltd.", "summary": "Flipkart is one of India’s leading digital commerce companies, running hundreds of tightly coupled microservices across Kubernetes and VM workloads that must withstand the traffic surges of Big Billion Days and other festive sales. The Cent", - "industries": ["E-commerce", "Retail", "Digital Commerce"], + "industries": [ + "E-commerce", + "Retail", + "Digital Commerce" + ], "tags": [ "Chaos engineering", "Resilience", diff --git a/data/architectures/records/swisscom-cloud-native-telco.json b/data/architectures/records/swisscom-cloud-native-telco.json index 191cb09a..df76a0fb 100644 --- a/data/architectures/records/swisscom-cloud-native-telco.json +++ b/data/architectures/records/swisscom-cloud-native-telco.json @@ -3,7 +3,10 @@ "title": "End-to-End Cloud Native Telco Platform Automation at Swisscom", "organization": "Swisscom (Switzerland) Ltd", "summary": "Swisscom has built a cloud native telco platform for the end-to-end automation of its 5G standalone core network and cross-domain resource orchestration. The architecture replaces traditional imperative network management (Jenkins pipelines", - "industries": ["Communication", "Telecommunication"], + "industries": [ + "Communication", + "Telecommunication" + ], "tags": [ "cloud_native", "telco", diff --git a/data/architectures/records/swisscom-kubernetes-service.json b/data/architectures/records/swisscom-kubernetes-service.json index 14affdfb..11840f89 100644 --- a/data/architectures/records/swisscom-kubernetes-service.json +++ b/data/architectures/records/swisscom-kubernetes-service.json @@ -3,7 +3,12 @@ "title": "A modern and sovereign Private Cloud «Kubernetes Service» for Swiss-based enterprises.", "organization": "Swisscom (Switzerland) Ltd", "summary": "Our next generation Private Cloud Container as a Service offering «Kubernetes Service» for the B2B market addresses customer’s need for scalable and highly available Kubernetes workload as a flexible and secure IT foundation. It is part of ", - "industries": ["Communication", "Telecommunication", "Software", "Cloud"], + "industries": [ + "Communication", + "Telecommunication", + "Software", + "Cloud" + ], "tags": [ "service-delivery", "cloud_native", diff --git a/data/architectures/records/zeiss.json b/data/architectures/records/zeiss.json index de79e2cd..930ecb39 100644 --- a/data/architectures/records/zeiss.json +++ b/data/architectures/records/zeiss.json @@ -3,7 +3,10 @@ "title": "ZEISS Vision Care - Order Fulfillment", "organization": "ZEISS", "summary": "Our Business Enablement & IT team develops, designs, and operates the order fulfillment platform that underpins this process.", - "industries": ["optics", "optical-manufacturing"], + "industries": [ + "optics", + "optical-manufacturing" + ], "tags": [ "public-cloud", "mass-manufacturing", @@ -11,7 +14,13 @@ "microservices", "platform engineering" ], - "projects": ["Kubernetes", "Dapr", "KEDA", "OpenTelemetry", "Helm"], + "projects": [ + "Kubernetes", + "Dapr", + "KEDA", + "OpenTelemetry", + "Helm" + ], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/members.json b/data/members.json index a9e7075d..601d1cef 100644 --- a/data/members.json +++ b/data/members.json @@ -1,6 +1,9 @@ { "description": "CNCF End User Community member organisations derived from reference architectures and Top End User Award winners.", - "generatedFrom": ["data/architectures/catalog.json", "data/awards.json"], + "generatedFrom": [ + "data/architectures/catalog.json", + "data/awards.json" + ], "schema": { "id": "kebab-case identifier", "name": "display name", @@ -18,8 +21,17 @@ "name": "Adobe", "slug": "adobe", "logo": "/img/architectures/adobe/Adobe_Wordmark_RGB_Red.svg", - "industries": ["Digital Experience", "Digital Media", "Software"], - "projects": ["Argo", "Backstage", "Helm", "Kubernetes"], + "industries": [ + "Digital Experience", + "Digital Media", + "Software" + ], + "projects": [ + "Argo", + "Backstage", + "Helm", + "Kubernetes" + ], "architectures": [ { "id": "adobe", @@ -51,8 +63,15 @@ "name": "Allianz", "slug": "allianz", "logo": "/img/architectures/allianz/allianz.svg", - "industries": ["Financial", "Insurance"], - "projects": ["Argo", "Helm", "Kubernetes"], + "industries": [ + "Financial", + "Insurance" + ], + "projects": [ + "Argo", + "Helm", + "Kubernetes" + ], "architectures": [ { "id": "allianz", @@ -143,7 +162,10 @@ "name": "CERN", "slug": "cern", "logo": "/img/architectures/cern-scientific-computing/logo.svg", - "industries": ["Public Sector", "Research"], + "industries": [ + "Public Sector", + "Research" + ], "projects": [ "Argo", "Bootc", @@ -188,8 +210,13 @@ "name": "Colopl Inc.", "slug": "colopl", "logo": "/img/architectures/colopl/logo.png", - "industries": ["gaming"], - "projects": ["Agones", "Kubernetes"], + "industries": [ + "gaming" + ], + "projects": [ + "Agones", + "Kubernetes" + ], "architectures": [ { "id": "colopl", @@ -232,7 +259,11 @@ "name": "Flipkart", "slug": "flipkart", "logo": "/img/architectures/flipkart-chaos-engineering/logo.png", - "industries": ["Digital Commerce", "E-commerce", "Retail"], + "industries": [ + "Digital Commerce", + "E-commerce", + "Retail" + ], "projects": [ "Argo Workflows", "Helm", @@ -433,7 +464,12 @@ "name": "Swisscom", "slug": "swisscom", "logo": "/img/architectures/swisscom-cloud-native-telco/swisscom.png", - "industries": ["Cloud", "Communication", "Software", "Telecommunication"], + "industries": [ + "Cloud", + "Communication", + "Software", + "Telecommunication" + ], "projects": [ "ArgoCD", "CloudNativePG", @@ -505,8 +541,17 @@ "name": "ZEISS", "slug": "zeiss", "logo": "/img/architectures/zeiss/zeiss-logo-rgb.svg", - "industries": ["optical-manufacturing", "optics"], - "projects": ["Dapr", "Helm", "KEDA", "Kubernetes", "OpenTelemetry"], + "industries": [ + "optical-manufacturing", + "optics" + ], + "projects": [ + "Dapr", + "Helm", + "KEDA", + "Kubernetes", + "OpenTelemetry" + ], "architectures": [ { "id": "zeiss", diff --git a/data/metrics.json b/data/metrics.json index e067e176..ebef805d 100644 --- a/data/metrics.json +++ b/data/metrics.json @@ -1,10 +1,10 @@ { "generated": true, - "generatedAt": "2026-09-24T02:38:41.405Z", + "generatedAt": "2026-09-28T02:48:30.866Z", "sources": { "landscape": { "repository": "https://github.com/cncf/landscape", - "revision": "0448002c82defabcb100ba959475fbed3b91f3ab", + "revision": "97b2e16fdf4147166df9427299998a83991b0958", "sourceUrl": "https://github.com/cncf/landscape/blob/main/landscape.yml" }, "architectures": { @@ -20,7 +20,7 @@ "value": 255, "source": "landscape", "sourceUrl": "https://landscape.cncf.io/", - "collectedAt": "2026-09-24T02:38:41.405Z", + "collectedAt": "2026-09-28T02:48:30.866Z", "note": "Projects explicitly marked with a CNCF maturity level in landscape.yml." }, { @@ -29,7 +29,7 @@ "value": 8, "source": "architectures", "sourceUrl": "https://github.com/cncf/architecture/tree/main/content/en/architectures", - "collectedAt": "2026-09-24T02:38:41.405Z", + "collectedAt": "2026-09-28T02:48:30.866Z", "note": "Published architecture directories under content/en/architectures." } ], @@ -108,7 +108,7 @@ "sourceUrl": "https://landscape.cncf.io/", "values": [ { - "date": "2026-09-24", + "date": "2026-09-28", "value": 649 } ] @@ -153,7 +153,7 @@ "generated": true, "source": "cncf/tab and cncf/architecture GitHub APIs", "sourceUrl": "https://github.com/cncf/tab/issues?q=is%3Aissue+label%3Aarea%2Freference-architecture", - "observedAt": "2026-09-24T02:38:41.405Z", + "observedAt": "2026-09-28T02:48:30.865Z", "cards": [ { "id": "open-submissions", diff --git a/docs/architectures/adobe.md b/docs/architectures/adobe.md index b0ae7a95..69773ffb 100644 --- a/docs/architectures/adobe.md +++ b/docs/architectures/adobe.md @@ -1,78 +1,44 @@ --- -title: - 'Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture' -sidebar_label: 'Adobe' +title: "Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture" +sidebar_label: "Adobe" --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant CNCF projects - + - + - + - + ## Developer Platforms @ Adobe +Adobe has been working on building an Internal Developer Platform (IDP) for the past several years. We have a core infrastructure layer which offers a compute layer on top of cloud providers such as AWS and Azure. On top of the infrastructure layer, we provide an end-to-end developer experience which helps developers in different phases of SDLC. -Adobe has been working on building an Internal Developer Platform (IDP) for the -past several years. We have a core infrastructure layer which offers a compute -layer on top of cloud providers such as AWS and Azure. On top of the -infrastructure layer, we provide an end-to-end developer experience which helps -developers in different phases of SDLC. - -Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for -both infrastructure and developer experience. +Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for both infrastructure and developer experience. ### Developer Experience / Flex @ Adobe - -Adobe offers a rich developer experience to its developers to enable them to -write better software faster. This includes a seamless experience for all phases -of their application lifecycle: Concept → Code → Cloud → Customer. **Flex** is -the codename for Adobe’s **GitOps**-based Service Delivery foundation. - -- **Concept → Code**: Adobe offers standardized **bootstrappable golden - templates** via the unified developer portal (based on - [Backstage](https://backstage.io/)), which provides a guided path that - generates boilerplate code (with best practices built-in) for the app and - infrastructure. - - The promise is that you should be able to run and test the app locally right - after code generation. - - The generated infrastructure includes, among other things, the manifests for - the service’s CI/CD pipeline. - - _Toolchain_ - The developer portal is based on the industry-standard - open-source tool Backstage. -- **Code → Cloud**: Adobe offers customizable and flexible GitOps-based CI/CD - pipelines (based on Argo projects) to developers. - - The developers start with a paved path but Flex allows users to customize - their CI/CD pipeline if needed, including adding new steps or changing the - pipeline DAG. - - The CI/CD pipeline does just-in-time provisioning during deployments i.e. it - provisions necessary resources to make deployments successful. For example, - DNS endpoints, K8s namespaces, Argo CD apps etc. - - Advanced deployment strategies like Canary, Blue-Green etc. are supported - out of the box. - - _Toolchain_ - We use industry-standard open-source Argo projects: Argo CD, - Argo Workflows, Argo Events, and Argo Rollouts. -- **Cloud → Customer**: Adobe offers a “Single pane of glass” developer - experience (called Flexperience) in the developer portal for managing the - application. - - _Toolchain_ - The experience is provided as a Backstage plug-in in the - developer portal. -- **Control Plane** – The control plane for the service delivery workflow is - based on AWS EKS clusters. - -Flex has had rapid organic growth in the last 2 years since its launch. Here are -some numbers that represent the scale that we are handling (as of Oct 2024): - +Adobe offers a rich developer experience to its developers to enable them to write better software faster. This includes a seamless experience for all phases of their application lifecycle: Concept → Code → Cloud → Customer. **Flex** is the codename for Adobe’s **GitOps**-based Service Delivery foundation. + +- **Concept → Code**: Adobe offers standardized **bootstrappable golden templates** via the unified developer portal (based on [Backstage](https://backstage.io/)), which provides a guided path that generates boilerplate code (with best practices built-in) for the app and infrastructure. + - The promise is that you should be able to run and test the app locally right after code generation. + - The generated infrastructure includes, among other things, the manifests for the service’s CI/CD pipeline. + - *Toolchain* - The developer portal is based on the industry-standard open-source tool Backstage. +- **Code → Cloud**: Adobe offers customizable and flexible GitOps-based CI/CD pipelines (based on Argo projects) to developers. + - The developers start with a paved path but Flex allows users to customize their CI/CD pipeline if needed, including adding new steps or changing the pipeline DAG. + - The CI/CD pipeline does just-in-time provisioning during deployments i.e. it provisions necessary resources to make deployments successful. For example, DNS endpoints, K8s namespaces, Argo CD apps etc. + - Advanced deployment strategies like Canary, Blue-Green etc. are supported out of the box. + - *Toolchain* - We use industry-standard open-source Argo projects: Argo CD, Argo Workflows, Argo Events, and Argo Rollouts. +- **Cloud → Customer**: Adobe offers a “Single pane of glass” developer experience (called Flexperience) in the developer portal for managing the application. + - *Toolchain* - The experience is provided as a Backstage plug-in in the developer portal. +- **Control Plane** – The control plane for the service delivery workflow is based on AWS EKS clusters. + +Flex has had rapid organic growth in the last 2 years since its launch. Here are some numbers that represent the scale that we are handling (as of Oct 2024): - > 360 remote K8s clusters - > 22K Argo CD apps - > 30K deployments per month @@ -80,106 +46,47 @@ some numbers that represent the scale that we are handling (as of Oct 2024): And these numbers are increasing by the day. -In this article, we will cover Flex’s initial architecture, its challenges, and -the need for a re-architecture. Then, we will deep-dive into our new -Flex-in-a-box architecture, including the architecture requirements and an -introduction to cell-based architecture. We’ll conclude by discussing the -benefits, challenges, use cases, and future ideas. +In this article, we will cover Flex’s initial architecture, its challenges, and the need for a re-architecture. Then, we will deep-dive into our new Flex-in-a-box architecture, including the architecture requirements and an introduction to cell-based architecture. We’ll conclude by discussing the benefits, challenges, use cases, and future ideas. ## Initial Architecture - -Here is a simplified high-level diagram describing how Flex acts as the -foundation between the source (Github), destinations, and the developer -surfaces. +Here is a simplified high-level diagram describing how Flex acts as the foundation between the source (Github), destinations, and the developer surfaces. ![High Level Diagram](/img/architectures/adobe/FlexHighLevel.png) -As you can see in the diagram, the CI/CD foundation acts as the glue between the -source and the destinations. The developer surfaces, powered by the foundation, -offer paved roads and seamless workflows for various use cases. +As you can see in the diagram, the CI/CD foundation acts as the glue between the source and the destinations. The developer surfaces, powered by the foundation, offer paved roads and seamless workflows for various use cases. Here is the initial architecture that we started with. ![Initial Architecture Diagram](/img/architectures/adobe/InitialArchitecture.png) -We have a Hub-and-Spoke architecture with the Hub in the middle (aka Flex Hub -k8s cluster) connected to the various remote K8s clusters, which act as the -spokes. As you can see, we have Git as the source of truth on the left, and K8s -clusters as the remote clusters (destinations) on the right. - -We use Argo Workflows for creating the CI/CD pipeline, Argo CD for providing -GitOps capabilities, and Argo Events for catching events and triggering -workflows. The CI/CD pipeline runs on the Hub cluster in a per-service -namespace. Argo Rollouts is installed on remote K8s clusters and provides -advanced deployment capabilities with Argo CD. The service runtime runs on -namespaces provisioner on these remote K8s clusters. - -Provisioner is a homegrown component that does the just-in-time provisioning -during deployments. A couple of home-grown Observability components send events -and information to our homegrown Flex backend, where the data is aggregated and -exposed via a set of Flex APIs. The Flex developer experience (aka Flexperience) -in the Developer Portal / Backstage uses the Flex APIs provided by the Flex -backend to populate the service’s info. - -Each service has two repos associated with it: an app repo and a deploy repo. -App repo has the code for business logic, and the deploy repo has the configs -and manifests for CI/CD, K8s resources etc. +We have a Hub-and-Spoke architecture with the Hub in the middle (aka Flex Hub k8s cluster) connected to the various remote K8s clusters, which act as the spokes. As you can see, we have Git as the source of truth on the left, and K8s clusters as the remote clusters (destinations) on the right. + +We use Argo Workflows for creating the CI/CD pipeline, Argo CD for providing GitOps capabilities, and Argo Events for catching events and triggering workflows. The CI/CD pipeline runs on the Hub cluster in a per-service namespace. Argo Rollouts is installed on remote K8s clusters and provides advanced deployment capabilities with Argo CD. The service runtime runs on namespaces provisioner on these remote K8s clusters. + +Provisioner is a homegrown component that does the just-in-time provisioning during deployments. A couple of home-grown Observability components send events and information to our homegrown Flex backend, where the data is aggregated and exposed via a set of Flex APIs. The Flex developer experience (aka Flexperience) in the Developer Portal / Backstage uses the Flex APIs provided by the Flex backend to populate the service’s info. + +Each service has two repos associated with it: an app repo and a deploy repo. App repo has the code for business logic, and the deploy repo has the configs and manifests for CI/CD, K8s resources etc. ### Challenges with Initial Architecture +While the initial architecture provided a starting point, it had its challenges. We realized these challenges after 6 months when we crossed approximately 2.5K Argo CD applications. -While the initial architecture provided a starting point, it had its challenges. -We realized these challenges after 6 months when we crossed approximately 2.5K -Argo CD applications. - -**No tuning** - We used the Argo components with no/minimal tuning, and realized -that there were several knobs and controls available on the Argo components that -could be leveraged to improve the stability, performance, and scalability of the -components. - -**Heavy load on K8s control plane** - Flex Hub cluster is based on AWS EKS, and -we just had one Hub cluster. We realized that Argo CD and Argo Workflows -interact heavily with the EKS control plane, which includes the K8s API Server -and etcd. The CI/CD experience is severely impacted when the K8s control plane -is under heavy load. - -**Shared control plane** - Both Argo CD and Argo Workflows were running on the -same EKS cluster, which multiplied the overhead and load on the EKS control -plane. This impact was observed in terms of degraded latency of the K8s API -server, high etcd usage/churn etc., that caused a severe impact on the -performance of anything interacting with the K8s control plane, including the -Argo components. As a result, the user experience was getting impacted and -client deployments were either taking a long time or failing. - -**No automation to create Flex Hub cluster** - Flex Hub cluster was brought up -manually, over many quarters, by different individuals. We did not have a -concrete list of steps and automation to bring up a Flex Hub cluster, thereby -putting us at risk if the Hub cluster goes down. It was a huge risk for the -platform team and client teams because it would have taken us a long time to -create a new one, and client deployments would have been stuck during that -duration. - -As a result of these challenges, we experienced several outages for the -developer experience CI/CD workflows, including slow builds, hung workflows, and -deployments taking too long, among other things. +**No tuning** - We used the Argo components with no/minimal tuning, and realized that there were several knobs and controls available on the Argo components that could be leveraged to improve the stability, performance, and scalability of the components. -## Potential Solutions +**Heavy load on K8s control plane** - Flex Hub cluster is based on AWS EKS, and we just had one Hub cluster. We realized that Argo CD and Argo Workflows interact heavily with the EKS control plane, which includes the K8s API Server and etcd. The CI/CD experience is severely impacted when the K8s control plane is under heavy load. -Given the problems we faced, we realized quickly that one Flex Hub cluster would -not be enough. We invested heavily in vertically scaling Flex Hub cluster by -tuning and optimizing the Argo components, EKS control plane, and homegrown -tooling. This helped us get out of the woods and support >10K Argo CD apps. -While we will continue to scale Flex Hub cluster vertically to support more -services/Argo CD apps, we knew that the runway will be limited. +**Shared control plane** - Both Argo CD and Argo Workflows were running on the same EKS cluster, which multiplied the overhead and load on the EKS control plane. This impact was observed in terms of degraded latency of the K8s API server, high etcd usage/churn etc., that caused a severe impact on the performance of anything interacting with the K8s control plane, including the Argo components. As a result, the user experience was getting impacted and client deployments were either taking a long time or failing. -We were projecting 10X growth in the next two years, and one Flex Hub cluster -could not support all that scale alone. We needed more than one Flex Hub -cluster. +**No automation to create Flex Hub cluster** - Flex Hub cluster was brought up manually, over many quarters, by different individuals. We did not have a concrete list of steps and automation to bring up a Flex Hub cluster, thereby putting us at risk if the Hub cluster goes down. It was a huge risk for the platform team and client teams because it would have taken us a long time to create a new one, and client deployments would have been stuck during that duration. -### Scale Up +As a result of these challenges, we experienced several outages for the developer experience CI/CD workflows, including slow builds, hung workflows, and deployments taking too long, among other things. -There were a lot of configuration changes and optimizations we did to scale Flex -Hub cluster vertically. These included: +## Potential Solutions +Given the problems we faced, we realized quickly that one Flex Hub cluster would not be enough. We invested heavily in vertically scaling Flex Hub cluster by tuning and optimizing the Argo components, EKS control plane, and homegrown tooling. This helped us get out of the woods and support >10K Argo CD apps. While we will continue to scale Flex Hub cluster vertically to support more services/Argo CD apps, we knew that the runway will be limited. + +We were projecting 10X growth in the next two years, and one Flex Hub cluster could not support all that scale alone. We needed more than one Flex Hub cluster. +### Scale Up +There were a lot of configuration changes and optimizations we did to scale Flex Hub cluster vertically. These included: - Scale up replica count for Argo CD application controller and repo server - Ask Argo CD to exclude resources that we should not manage - Tune Argo CD self-heal and reconciliation timeouts @@ -188,365 +95,163 @@ Hub cluster vertically. These included: - Archive old Argo Workflows by setting a short TTL - Switch from K8s APIs to Argo CD APIs to leverage informer cache -This list is not exhaustive and there were many more improvements which helped -us tremendously. We have talked about some of these improvements in previous -talks: - +This list is not exhaustive and there were many more improvements which helped us tremendously. We have talked about some of these improvements in previous talks: - [Key Takeaways from Scaling Adobe's CI/CD Solution to Support 50K Argo CD Apps (KubeCon EU 2024)](https://www.youtube.com/watch?v=7yVXMCX62tY) - [Scaling a GitOps Platform at Adobe (GitOpsCon 2024)](https://www.youtube.com/watch?v=MDXrc_cLVns) ### Scale Out +While the scale-up improvements were helping, we decided to re-architect. Though we had a couple of options, we decided to scale horizontally by having more than one Flex Hub cluster because it satisfied many of the architecture requirements (listed below) and gave us the biggest bang for the buck. -While the scale-up improvements were helping, we decided to re-architect. Though -we had a couple of options, we decided to scale horizontally by having more than -one Flex Hub cluster because it satisfied many of the architecture requirements -(listed below) and gave us the biggest bang for the buck. - -This was not going to be easy, because we had hundreds of production services -running and we did not want to impact any of them. The rest of the article goes -into the details of how we approached the re-architecture. +This was not going to be easy, because we had hundreds of production services running and we did not want to impact any of them. The rest of the article goes into the details of how we approached the re-architecture. ### Flexbox concept +As a first step, we came up with the concept of a **Flexbox**. A Flexbox is a collection of Flex components that work together to provide the GitOps foundation for our CI/CD solution. We called the Flex Hub K8s cluster which formed the basis of our initial architecture, Flexbox 1. -As a first step, we came up with the concept of a **Flexbox**. A Flexbox is a -collection of Flex components that work together to provide the GitOps -foundation for our CI/CD solution. We called the Flex Hub K8s cluster which -formed the basis of our initial architecture, Flexbox 1. - -Keeping future extensibility in mind, we also decoupled the concept of Flexbox -from clusters. A Flexbox can be extended to support many physical and virtual -Hub clusters inside it. Because the initial architecture had just one physical -Flex Hub cluster, Flexbox 1 also had one Hub cluster. +Keeping future extensibility in mind, we also decoupled the concept of Flexbox from clusters. A Flexbox can be extended to support many physical and virtual Hub clusters inside it. Because the initial architecture had just one physical Flex Hub cluster, Flexbox 1 also had one Hub cluster. ## New Architecture Requirements - -We came up with the following key requirements for the new architecture. - -1. Ability to **predictably add scale** (horizontally and vertically) as needed, - for short-term and future needs. +We came up with the following key requirements for the new architecture. +1. Ability to **predictably add scale** (horizontally and vertically) as needed, for short-term and future needs. 2. Ability to **relocate services** out of Flexbox 1 and relieve pressure. 3. The services should not decide which Flexbox they are on. -4. **No impact of rearchitecture on any existing services** (runtime or CI/CD) - on Flexbox 1. +4. **No impact of rearchitecture on any existing services** (runtime or CI/CD) on Flexbox 1. 5. No impact on service runtime during box-2-box relocation. -6. A service (app and deploy repos) should be associated with one Flexbox at a - time. -7. Minimal impact/downtime during relocation to the service’s CI/CD pipeline, if - at all. -8. During relocation, the service owners should be aware of the relocation and - the downtime. -9. Ability to support more than one physical or virtual clusters inside a - Flexbox. -10. Ability to support non-container workflows as well e.g. Serverless, Cloud - Infrastructure etc. +6. A service (app and deploy repos) should be associated with one Flexbox at a time. +7. Minimal impact/downtime during relocation to the service’s CI/CD pipeline, if at all. +8. During relocation, the service owners should be aware of the relocation and the downtime. +9. Ability to support more than one physical or virtual clusters inside a Flexbox. +10. Ability to support non-container workflows as well e.g. Serverless, Cloud Infrastructure etc. ## What is Cell-Based architecture +Cell-based architecture is an architecture pattern that involves designing systems with multiple cells capable of executing the designated tasks independently. The basic idea is to have the ability to scale out by adding identical cells that handle the same kind of workload and isolate fault boundaries to limit the impact in case of failures. -Cell-based architecture is an architecture pattern that involves designing -systems with multiple cells capable of executing the designated tasks -independently. The basic idea is to have the ability to scale out by adding -identical cells that handle the same kind of workload and isolate fault -boundaries to limit the impact in case of failures. - -Each cell is independent, does not share any state with other cells, and handles -a subset of the overall workload requests. This reduces the potential impact of -a failure, such as a bad software update, on an individual cell and the requests -that it's processing. +Each cell is independent, does not share any state with other cells, and handles a subset of the overall workload requests. This reduces the potential impact of a failure, such as a bad software update, on an individual cell and the requests that it's processing. Here's a diagram that depicts the cell-based architecture at a high level: ![Cell-based Architecture Diagram](/img/architectures/adobe/CellBasedArchitecture.webp) -More details here: Diagram Source: -https://newsletter.systemdesign.one/p/cell-based-architecture \ +More details here: +Diagram Source: https://newsletter.systemdesign.one/p/cell-based-architecture \ Reference: https://docs.aws.amazon.com/wellarchitected/latest/reducing-scope-of-impact-with-cell-based-architecture/what-is-a-cell-based-architecture.html ## Flex-in-a-Box (FiaB) architecture +With the requirements above, we developed a **Flex-in-a-box (FiaB) architecture**. This architecture allows us to encapsulate Flex components in boxes and easily replicate them to add scale on demand. We started by adding a new Flexbox 2. Before FiaB, all services were associated with Flexbox 1. This means that the CI/CD pipelines for these services run on Flexbox1. With FiaB and newer Flexboxes, now services can be onboarded to Flexbox 2 also, and any of the future Flexboxes. -With the requirements above, we developed a **Flex-in-a-box (FiaB) -architecture**. This architecture allows us to encapsulate Flex components in -boxes and easily replicate them to add scale on demand. We started by adding a -new Flexbox 2. Before FiaB, all services were associated with Flexbox 1. This -means that the CI/CD pipelines for these services run on Flexbox1. With FiaB and -newer Flexboxes, now services can be onboarded to Flexbox 2 also, and any of the -future Flexboxes. - -To keep the impact of architectural changes to a minimum, we decided to have -just one physical Hub cluster in Flexbox 2 too, and make it very similar to the -Flex Hub cluster in Flexbox 1. - -Our FiaB architecture is based on three Rs: +To keep the impact of architectural changes to a minimum, we decided to have just one physical Hub cluster in Flexbox 2 too, and make it very similar to the Flex Hub cluster in Flexbox 1. -- **Recreation** – Ability to (re)create and upgrade more Flexboxes with - repeatability and predictability, with a GitOps-based CI/CD pipeline. -- **Redirection** – Ability for Flex admins to decide which service maps to - which Flexbox via declarative rules, and have services routed to the correct - boxes based on those rules. -- **Relocation** – Ability to relocate a service (with automation) from one - Flexbox to another with zero downtime for the service runtime, zero work for - the service team, and minimal downtime (< 15 mins) for the CI/CD pipeline. +Our FiaB architecture is based on three Rs: +- **Recreation** – Ability to (re)create and upgrade more Flexboxes with repeatability and predictability, with a GitOps-based CI/CD pipeline. +- **Redirection** – Ability for Flex admins to decide which service maps to which Flexbox via declarative rules, and have services routed to the correct boxes based on those rules. +- **Relocation** – Ability to relocate a service (with automation) from one Flexbox to another with zero downtime for the service runtime, zero work for the service team, and minimal downtime (< 15 mins) for the CI/CD pipeline. -Here’s a diagram that captures these three concepts. More details on these 3 Rs -below. +Here’s a diagram that captures these three concepts. More details on these 3 Rs below. ![3R diagram](/img/architectures/adobe/FiaBArchitecture.png) ### Recreation - -We created a GitOps-based pipeline for Flexbox lifecycle management. This means -that there is a Flexbox config in Git which defines what a Flexbox looks like. -Here is a diagram showing the workflow in action: +We created a GitOps-based pipeline for Flexbox lifecycle management. This means that there is a Flexbox config in Git which defines what a Flexbox looks like. Here is a diagram showing the workflow in action: ![Recreation diagram](/img/architectures/adobe/FiaBAutomation.png) -As can be seen from the diagram, any changes to the Flexbox config triggers a -Flexbox automation Argo workflow. The workflow has a series of steps that -automate the setup of a Flexbox on top of an EKS cluster. +As can be seen from the diagram, any changes to the Flexbox config triggers a Flexbox automation Argo workflow. The workflow has a series of steps that automate the setup of a Flexbox on top of an EKS cluster. The workflow has two key objectives: - -1. Creates the Helm charts for the various Flexbox components - This step - understands the Flexbox configs and creates the Helm charts for Flexbox - components, including Argo CD, Argo Workflows, Argo Events, Provisioner and - Monitoring. - - The workflow step then commits these Helm charts to Git, allowing Argo CD - apps to sync these changes to the destination Flexbox. -2. Provisioning resources for Flexbox components, including creating Argo CD - apps for each of the Flex components (Argo CD, Argo Workflows, Argo Events - etc.) that need to be installed in the Flexbox. It’s an Argo of Argos - architecture, where an Argo CD 0 (level zero) manages the Argo CD apps - (level 1) for the various Flex components to be deployed on the destination - Flexbox. +1. Creates the Helm charts for the various Flexbox components - This step understands the Flexbox configs and creates the Helm charts for Flexbox components, including Argo CD, Argo Workflows, Argo Events, Provisioner and Monitoring. + - The workflow step then commits these Helm charts to Git, allowing Argo CD apps to sync these changes to the destination Flexbox. +2. Provisioning resources for Flexbox components, including creating Argo CD apps for each of the Flex components (Argo CD, Argo Workflows, Argo Events etc.) that need to be installed in the Flexbox. It’s an Argo of Argos architecture, where an Argo CD 0 (level zero) manages the Argo CD apps (level 1) for the various Flex components to be deployed on the destination Flexbox. ### Redirection +Once we have a new Flexbox, we need the ability to map services to Flexboxes and redirect the Github events to the correct Flexbox. This function is performed by a new component called **Redirector**, which sits between Github and the Flexboxes (as can be seen in the new architecture diagram above). -Once we have a new Flexbox, we need the ability to map services to Flexboxes and -redirect the Github events to the correct Flexbox. This function is performed by -a new component called **Redirector**, which sits between Github and the -Flexboxes (as can be seen in the new architecture diagram above). - -Loosely speaking, Redirector acts as a poor man's load balancer for Flexboxes. -When the first event arrives for any service, Redirector evaluates the rules -defined by Flex admins and maps the service to a Flexbox based on the rules. -These rules are specified declaratively in a Github repo for Redirector by Flex -admins. Redirector stores the box<>service mapping in a DB for future -reference. For any subsequent GitHub events, it just picks up the mapping from -the DB and routes the events to the appropriate Flexbox. +Loosely speaking, Redirector acts as a poor man's load balancer for Flexboxes. When the first event arrives for any service, Redirector evaluates the rules defined by Flex admins and maps the service to a Flexbox based on the rules. These rules are specified declaratively in a Github repo for Redirector by Flex admins. Redirector stores the box<>service mapping in a DB for future reference. For any subsequent GitHub events, it just picks up the mapping from the DB and routes the events to the appropriate Flexbox. ### Relocation +While Flexbox 2 and other components in FiaB architecture were being built, there was constant organic adoption of Flex, Flexbox 1 was getting filled up and was under increasing pressure. -While Flexbox 2 and other components in FiaB architecture were being built, -there was constant organic adoption of Flex, Flexbox 1 was getting filled up and -was under increasing pressure. - -To relieve the pressure on Flexbox 1 and future use cases too, we needed a way -to cleanly relocate a service from a source Flexbox to a destination Flexbox, -without disrupting the service runtime. We created a new component called -**Relocator** which does just that. Here are the four broad phases in the -Relocator: - -1. **Stop the service’s CI/CD pipeline on source Flexbox** - This would include - pausing the Argo CD sync and stopping Redirector from sending the events for - this service to the source Flexbox. This step also flags the service status - (in Redirector) as “relocating” which helps us inform the developers that - their service’s deployment pipeline is “under maintenance”. -2. **Recreate the service’s CI/CD pipeline on the destination Flexbox** - This - includes recreating/copying the resources in the source Flexbox on the - destination Flexbox. -3. **Validate and start the service’s CI/CD on destination Flexbox** - This - includes starting the Argo CD sync and allowing Redirector to send the events - for this service to the destination Flexbox. -4. **Delete Flex pipeline on source Flexbox** - Once everything is working fine, - delete the resources on the source Flexbox to clean things up. This step also - removes the “relocating” service status in Redirector. +To relieve the pressure on Flexbox 1 and future use cases too, we needed a way to cleanly relocate a service from a source Flexbox to a destination Flexbox, without disrupting the service runtime. We created a new component called **Relocator** which does just that. Here are the four broad phases in the Relocator: +1. **Stop the service’s CI/CD pipeline on source Flexbox** - This would include pausing the Argo CD sync and stopping Redirector from sending the events for this service to the source Flexbox. This step also flags the service status (in Redirector) as “relocating” which helps us inform the developers that their service’s deployment pipeline is “under maintenance”. +2. **Recreate the service’s CI/CD pipeline on the destination Flexbox** - This includes recreating/copying the resources in the source Flexbox on the destination Flexbox. +3. **Validate and start the service’s CI/CD on destination Flexbox** - This includes starting the Argo CD sync and allowing Redirector to send the events for this service to the destination Flexbox. +4. **Delete Flex pipeline on source Flexbox** - Once everything is working fine, delete the resources on the source Flexbox to clean things up. This step also removes the “relocating” service status in Redirector. ### Flex-in-a-box (FiaB) vs Cell-based architecture +When we started on FiaB, we did not use cell-based architecture as a reference. We realized later when we had conceived the new architecture that it is conceptually very similar to cell-based architecture. -When we started on FiaB, we did not use cell-based architecture as a reference. -We realized later when we had conceived the new architecture that it is -conceptually very similar to cell-based architecture. - -In terms of comparison, a Cell in a cell-based architecture is loosely -equivalent to a Flexbox in Flex-in-a-Box (FiaB). +In terms of comparison, a Cell in a cell-based architecture is loosely equivalent to a Flexbox in Flex-in-a-Box (FiaB). ## Benefits / What worked well - The new Flex-in-a-box architecture offers various benefits: - 1. **“Stamp” setup** - Ability to predictably and reliably create new Flexboxes. -2. **Scale horizontally** - Industry-proven way to scale horizontally by adding - scale on demand. -3. **Reduce existing load** - Ability to relieve pressure on a Flexbox by - allowing us to relocate services across Flexboxes, without any downtime. -4. **Minimal relocation impact** - Minimal impact/downtime to the service’s - CI/CD pipeline for the duration of the relocation. -5. **Abstracted Flexbox mapping** - Service teams need not worry about which - Flexbox they are on. Flex team decides the service<>Flexbox mapping. -6. **Dedicated Flexboxes** - Ability to provide dedicated Flexboxes to teams if - need be. -7. **Enhanced testing ability** - Allow Flex team to test - features/rollouts/infrastructure better using multiple boxes. -8. **Configuration flexibility** - Different Flexboxes may have different - versions of Flex components installed if need be. In fact, different - Flexboxes may have different components altogether. For example, one may host - GitHub Action runners inside a Flexbox, rather than Argo Workflows. One - Flexbox may have two clusters in it. We need to be wary of snowflakes though. -9. **Better audit** - Ability to track, monitor and audit GitHub events coming - via Redirector. -10. **Efficiency** - Ability to maintain Flexboxes at optimum capacity rather - than stretching one Flexbox to the limit. -11. **Limit blast radius** - If one of the boxes goes down, only services mapped - to that Flexbox are affected, and can be relocated. -12. **Better disaster recovery** - Ability to recover quickly from disasters - because of relocation. +2. **Scale horizontally** - Industry-proven way to scale horizontally by adding scale on demand. +3. **Reduce existing load** - Ability to relieve pressure on a Flexbox by allowing us to relocate services across Flexboxes, without any downtime. +4. **Minimal relocation impact** - Minimal impact/downtime to the service’s CI/CD pipeline for the duration of the relocation. +5. **Abstracted Flexbox mapping** - Service teams need not worry about which Flexbox they are on. Flex team decides the service<>Flexbox mapping. +6. **Dedicated Flexboxes** - Ability to provide dedicated Flexboxes to teams if need be. +7. **Enhanced testing ability** - Allow Flex team to test features/rollouts/infrastructure better using multiple boxes. +8. **Configuration flexibility** - Different Flexboxes may have different versions of Flex components installed if need be. In fact, different Flexboxes may have different components altogether. For example, one may host GitHub Action runners inside a Flexbox, rather than Argo Workflows. One Flexbox may have two clusters in it. We need to be wary of snowflakes though. +9. **Better audit** - Ability to track, monitor and audit GitHub events coming via Redirector. +10. **Efficiency** - Ability to maintain Flexboxes at optimum capacity rather than stretching one Flexbox to the limit. +11. **Limit blast radius** - If one of the boxes goes down, only services mapped to that Flexbox are affected, and can be relocated. +12. **Better disaster recovery** - Ability to recover quickly from disasters because of relocation. ## Performance Tests / Benchmarking +We did several benchmarking and testing efforts to see the benefits of the various vertical and horizontal scaling improvements. -We did several benchmarking and testing efforts to see the benefits of the -various vertical and horizontal scaling improvements. - -We recently presented them with the details at KubeCon EU 2024. Here’s the link: -\ +We recently presented them with the details at KubeCon EU 2024. Here’s the link: \ [Key Takeaways from Scaling Adobe's CI/CD Solution to Support 50K Argo CD Apps](https://www.youtube.com/watch?v=7yVXMCX62tY) We also published a blog with focus on Argo Workflows scalability testing:\ [Argo Workflows Controller Scalability Testing on Amazon EKS](https://cnoe.io/blog/argo-workflow-scalability) ## Challenges / Risks - -While this architecture works well for us and provides us with a lot of -flexibility, every architecture has some challenges, risks and overheads. Here -are a few for FiaB architecture: - -1. **Infrastructure cost** - There is some extra cost of running each Flexbox - due to the additional control plane and the extra capacity buffer per - cluster. -2. **Complexity** - Architecture becomes more complex because of adding a new - Redirector component between GitHub and Flexbox. As a side effect, Redirector - becomes a single point of failure across Flexboxes. -3. **Thresholds/Limits** - Defining and managing limits for each Flexbox is a - challenge. -4. **Support cost** - The support team has to understand the role played by - Redirector and Relocator when troubleshooting issues. -5. **User experience** - If a service gets relocated, some links exposed to - clients may change e.g. Argo CD and Argo Workflows URLs, along with Hub - cluster information. +While this architecture works well for us and provides us with a lot of flexibility, every architecture has some challenges, risks and overheads. Here are a few for FiaB architecture: +1. **Infrastructure cost** - There is some extra cost of running each Flexbox due to the additional control plane and the extra capacity buffer per cluster. +2. **Complexity** - Architecture becomes more complex because of adding a new Redirector component between GitHub and Flexbox. As a side effect, Redirector becomes a single point of failure across Flexboxes. +3. **Thresholds/Limits** - Defining and managing limits for each Flexbox is a challenge. +4. **Support cost** - The support team has to understand the role played by Redirector and Relocator when troubleshooting issues. +5. **User experience** - If a service gets relocated, some links exposed to clients may change e.g. Argo CD and Argo Workflows URLs, along with Hub cluster information. ## Use cases - -The Flex-in-a-box architecture opens up a world of possibilities for us in the -future. Here are a few use cases: - -1. **Dedicated Flexboxes for specific orgs/teams** - Some teams have already - requested dedicated Flexboxes for their teams to effectively manage the noisy - neighbor problem, and also enable efficient chargeback. - - This also allows us to shard the clusters better across Flexboxes. -2. **Domain-specific Flexboxes** - Specific Flexboxes could be dedicated to - specific domains. For example: - 1. Windows - Windows builds are a different beast than builds for Linux. - Services workflows needing Windows builds may get routed to a Flexbox with - specific tools/components to do a Windows build. - 2. Desktop / Mobile - Builds for desktop and mobile apps may need a dedicated - build farm. One or more Flexboxes can be designated/dedicated for this. - 3. Cloud Infrastructure Provisioning - Cloud infrastructure provisioning - requires custom operators and may need a separate Flexbox to ensure that - specific custom operators do not adversely affect the performance of - existing workloads. We can provide a dedicated box for hosting such - workloads. - 4. FaaS/Wasm - Building and testing Wasm modules could be very different than - building containers. We can provide a dedicated box for hosting FaaS - workflows. -3. **Toolchain-specific Flexboxes** - Flexboxes could cater to specific - toolchains too. For example, if a service is using GitHub Actions (GHA) as - the technology for pipeline creation and executions, GHA runners for the same - could run in a specific Flexbox. +The Flex-in-a-box architecture opens up a world of possibilities for us in the future. Here are a few use cases: +1. **Dedicated Flexboxes for specific orgs/teams** - Some teams have already requested dedicated Flexboxes for their teams to effectively manage the noisy neighbor problem, and also enable efficient chargeback. + * This also allows us to shard the clusters better across Flexboxes. +2. **Domain-specific Flexboxes** - Specific Flexboxes could be dedicated to specific domains. For example: + 1. Windows - Windows builds are a different beast than builds for Linux. Services workflows needing Windows builds may get routed to a Flexbox with specific tools/components to do a Windows build. + 2. Desktop / Mobile - Builds for desktop and mobile apps may need a dedicated build farm. One or more Flexboxes can be designated/dedicated for this. + 3. Cloud Infrastructure Provisioning - Cloud infrastructure provisioning requires custom operators and may need a separate Flexbox to ensure that specific custom operators do not adversely affect the performance of existing workloads. We can provide a dedicated box for hosting such workloads. + 4. FaaS/Wasm - Building and testing Wasm modules could be very different than building containers. We can provide a dedicated box for hosting FaaS workflows. +3. **Toolchain-specific Flexboxes** - Flexboxes could cater to specific toolchains too. For example, if a service is using GitHub Actions (GHA) as the technology for pipeline creation and executions, GHA runners for the same could run in a specific Flexbox. ## What's Next / Future Exploration ### Flex-in-a-Box improvements - -While this architecture will serve us well in the future, here are some areas of -future exploration: - -- **Better sharding** - As of now, all remote clusters are registered with - Flexbox 1 and Flexbox 2. There is a hidden performance impact on the remote - cluster for each Argo CD instance monitoring it. We can improve the sharding - to have specific orgs/services on a specific Flexbox. This allows only - clusters associated with those services to be registered with Argo CD on that - box. -- **Split Argo CD and Argo Workflows across clusters** - One of the challenges - with current architecture is that both Argo CD and Argo Workflows are on the - same cluster. This can be improved by running Argo CD on a cluster other than - the one running Argo Workflows. This separate cluster could be another - physical cluster in the same Flexbox, or it could be a virtual cluster (e.g. - vCluster) running on the existing physical/host cluster. +While this architecture will serve us well in the future, here are some areas of future exploration: +- **Better sharding** - As of now, all remote clusters are registered with Flexbox 1 and Flexbox 2. There is a hidden performance impact on the remote cluster for each Argo CD instance monitoring it. We can improve the sharding to have specific orgs/services on a specific Flexbox. This allows only clusters associated with those services to be registered with Argo CD on that box. +- **Split Argo CD and Argo Workflows across clusters** - One of the challenges with current architecture is that both Argo CD and Argo Workflows are on the same cluster. This can be improved by running Argo CD on a cluster other than the one running Argo Workflows. This separate cluster could be another physical cluster in the same Flexbox, or it could be a virtual cluster (e.g. vCluster) running on the existing physical/host cluster. ### Open-sourcing - -Alongside the architecture improvements, Adobe is an integral part of CNOE -cohort and we are working with a few companies (inside and outside of CNOE) to -see how they can use what we have already built, and open-source Flex (or -perhaps parts of it to start with). +Alongside the architecture improvements, Adobe is an integral part of CNOE cohort and we are working with a few companies (inside and outside of CNOE) to see how they can use what we have already built, and open-source Flex (or perhaps parts of it to start with). ### Unified CI/CD - -We have started a new initiative called Unified CI/CD which has the following -goals: - -1. **Beyond containers** - We primarily cater to container based workloads and - this initiative helps us serve non-container use cases such as mobile, static - website, desktop, serverless etc. -2. **Secure by default** - We aim to incorporate all non-negotiable security - best practices by default in our pipelines and enforce them. -3. **Sealed Paved Roads** - We are working towards providing paved roads/paths - for the most common use cases and make them **"sealed"**, minimizing the need - (and making it hard) to deviate from the paved road while ensuring - standardization, configurability, security, compliance, and other best - practices. -4. **Simplified workflows** - Make the workflows simpler to ensure faster - onboarding, reduced time for "zero to first commit", faster “Concept -> Code - -> Running state”. +We have started a new initiative called Unified CI/CD which has the following goals: +1. **Beyond containers** - We primarily cater to container based workloads and this initiative helps us serve non-container use cases such as mobile, static website, desktop, serverless etc. +2. **Secure by default** - We aim to incorporate all non-negotiable security best practices by default in our pipelines and enforce them. +3. **Sealed Paved Roads** - We are working towards providing paved roads/paths for the most common use cases and make them **"sealed"**, minimizing the need (and making it hard) to deviate from the paved road while ensuring standardization, configurability, security, compliance, and other best practices. +4. **Simplified workflows** - Make the workflows simpler to ensure faster onboarding, reduced time for "zero to first commit", faster “Concept -> Code -> Running state”. ## Key Takeaways / Lessons Learnt - -There were many key takeaways in this journey and several lessons learnt that -can benefit the community. Here are a few: - -1. **Developer Experience as a Product** - Developer Experience should be - treated as a product and non-functional requirements (NFRs, such as - scalability, stability, performance etc.) should be given as much importance - as functionality. -2. **Plan for future scale** - Plan upfront about horizontal and vertical - scaling needs, and how a combination of the two can help you achieve your - scaling needs. The architecture should be able to evolve to handle future - requirements and scale. -3. **Sharding Strategy** - Think of a sharding strategy for your services - depending on your requirements. For Adobe, specific orgs/services on a - specific Flexbox works out well. -4. **Flexbox automation** - Automate the creation of “your Flexboxes” from the - beginning. This helps bring in predictability and reliability. -5. **Performance Monitoring** - Monitor key performance metrics from the - beginning to figure when your infrastructure is under stress. We figured it - too late and it led to clients facing issues before we could detect them. -6. **K8s Control Plane as the bottleneck** - We realized it pretty late that K8s - control plane can become a bottleneck and how Argo CD and Argo Workflows can - pound the K8s API server. Avoid having Argo CD and Argo Workflows on the same - cluster. -7. **Plan for Disasters / Relocation** - Disasters will happen. Clusters will go - down. Think of how you would deal with it. Can you relocate workloads to - another cluster? Another Flexbox? What would be the Developer Experience - before/during/after relocation? +There were many key takeaways in this journey and several lessons learnt that can benefit the community. Here are a few: +1. **Developer Experience as a Product** - Developer Experience should be treated as a product and non-functional requirements (NFRs, such as scalability, stability, performance etc.) should be given as much importance as functionality. +2. **Plan for future scale** - Plan upfront about horizontal and vertical scaling needs, and how a combination of the two can help you achieve your scaling needs. The architecture should be able to evolve to handle future requirements and scale. +3. **Sharding Strategy** - Think of a sharding strategy for your services depending on your requirements. For Adobe, specific orgs/services on a specific Flexbox works out well. +4. **Flexbox automation** - Automate the creation of “your Flexboxes” from the beginning. This helps bring in predictability and reliability. +5. **Performance Monitoring** - Monitor key performance metrics from the beginning to figure when your infrastructure is under stress. We figured it too late and it led to clients facing issues before we could detect them. +6. **K8s Control Plane as the bottleneck** - We realized it pretty late that K8s control plane can become a bottleneck and how Argo CD and Argo Workflows can pound the K8s API server. Avoid having Argo CD and Argo Workflows on the same cluster. +7. **Plan for Disasters / Relocation** - Disasters will happen. Clusters will go down. Think of how you would deal with it. Can you relocate workloads to another cluster? Another Flexbox? What would be the Developer Experience before/during/after relocation? ## Conclusion - -A rich and seamless developer experience acts as a developer productivity -multiplier and helps any enterprise compete better. Flex-in-a-box architecture -provides Adobe with a solid foundation which we can use to scale our current -developer experience foundation for future needs. This includes developer -experience needs for use cases such as Windows, Serverless, Cloud -Infrastructure, Desktop and Mobiles, Static Websites etc. +A rich and seamless developer experience acts as a developer productivity multiplier and helps any enterprise compete better. Flex-in-a-box architecture provides Adobe with a solid foundation which we can use to scale our current developer experience foundation for future needs. This includes developer experience needs for use cases such as Windows, Serverless, Cloud Infrastructure, Desktop and Mobiles, Static Websites etc. ## Discussion -End user members may participate in the -[discussion thread](https://github.com/cncf/tab/discussions/138) for this -architecture. +End user members may participate in the [discussion thread](https://github.com/cncf/tab/discussions/138) for this architecture. diff --git a/docs/architectures/allianz.md b/docs/architectures/allianz.md index 03de7481..8a408a36 100644 --- a/docs/architectures/allianz.md +++ b/docs/architectures/allianz.md @@ -1,169 +1,138 @@ --- title: "Enabling Allianz Direct's scaling through Platform Engineering" -sidebar_label: 'Allianz Direct' +sidebar_label: "Allianz Direct" --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant CNCF projects - + - + - + -## Platform Engineering @ Allianz Direct - -Allianz Direct has been working on building Platform Engineering for the past -several years. We have created a cloud-native future-proof platform which offers -an organisation-wide-devops(orgOps) experience on top of public cloud. Business, -Software Engineers, Security, Compliance, Architecture, Support and Cost -Efficiency are few of our stakeholders. +## Platform Engineering @ Allianz Direct +Allianz Direct has been working on building Platform Engineering for the past several years. We have created a cloud-native future-proof platform which offers an organisation-wide-devops(orgOps) experience on top of public cloud. Business, Software Engineers, Security, Compliance, Architecture, Support and Cost Efficiency are few of our stakeholders. ### Platform Engineering Team Topology - internal -The PE team is internally organised in DDOs - Distributed DevOps, where each ddo -owns a Domain. This setup follows the reverse Conway manoeuver where the -reference architecture structure dictates the teams setup. The PE team consist -of 13 members. All of them internal employees. +The PE team is internally organised in DDOs - Distributed DevOps, where each ddo owns a Domain. +This setup follows the reverse Conway manoeuver where the reference architecture structure dictates +the teams setup. +The PE team consist of 13 members. All of them internal employees. -The PE team is being supported by auxiliary teams: Developer Service Hub (DSH): -entry level engineers handling the low complexity tasks such as: access -management, 1st level technical support, SRE onboarding, ticket sorting and -filtering Cloud Auditing Hub (CAH): entry level engineers with a focus on -compliance and audit Production Management Hub (PMH): entry level engineers with -a focus on incident and problem management Technical Tribe Leads (TTL): -Technical leads on the Tribe (department) level that became our evangelist for -best practices and continuous innovation partners. +The PE team is being supported by auxiliary teams: +Developer Service Hub (DSH): entry level engineers handling the low complexity tasks such as: access management, +1st level technical support, SRE onboarding, ticket sorting and filtering +Cloud Auditing Hub (CAH): entry level engineers with a focus on compliance and audit +Production Management Hub (PMH): entry level engineers with a focus on incident and problem management +Technical Tribe Leads (TTL): Technical leads on the Tribe (department) level that became our evangelist for best practices and +continuous innovation partners. ### Platform Engineering Team Topology - external +Externally, the team has decided to scale by building on the concept of FDOs - Federated DevOps, where +each squad (developing team) has built internal devops expertise focusing a minimum amount of time on +Operations concerns. A FDO community has flourished and enabled successfully an extremely aggressive organisation scale up +The PE serves an approx. 500 engineers and 1500 employees. -Externally, the team has decided to scale by building on the concept of FDOs - -Federated DevOps, where each squad (developing team) has built internal devops -expertise focusing a minimum amount of time on Operations concerns. A FDO -community has flourished and enabled successfully an extremely aggressive -organisation scale up The PE serves an approx. 500 engineers and 1500 employees. - -## Platform Engineering Architecture +## Platform Engineering Architecture -The PE team has built a Cloud Native Platform that serves the core B2C business -of a fully digital insurance player acting on several european markets. +The PE team has built a Cloud Native Platform +that serves the core B2C business of a fully digital insurance player acting on +several european markets. ![High Level Diagram](/img/architectures/allianz/PEArchitecture.jpg) -It serves also all the newly created B2B business initiatives. The highly -aggressive scale-up of the both B2B and B2B businesses forced the PE team to -redesign the architecture towards a multi-tenant architecture with minimum -operational effort. On top, the Platform has become a Organisation Group -Standard and an example for Global initiatives. This forced the PE team to -choose a light architecture under a second Platform based on Fargate ECS, -reducing the operational effort and talent demand to minimum. +It serves also all the newly created B2B business initiatives. +The highly aggressive scale-up of the both B2B and B2B businesses forced the PE team to redesign the architecture towards +a multi-tenant architecture with minimum operational effort. +On top, the Platform has become a Organisation Group Standard and an example for Global initiatives. This forced the PE team to +choose a light architecture under a second Platform based on Fargate ECS, reducing the operational effort and talent demand to minimum. PE team delivered several cloud native best practices: - - GitOps (ORGops to be more precise) - Stateless clusters orchestrated by a management cluster -- Fargate ECS multitenancy +- Fargate ECS multitenancy - DevSecOps - SRE ### Special projects that served us well -- AWS obsession with customers served us well. The technical support and - continuous relation with AWS architects helped us maximise our effectiveness - and evolve our architecture as well as complementing our stack with SaaS - services such as EKS, Redis, MSK. +- AWS obsession with customers served us well. The technical support and continuous relation with AWS architects +helped us maximise our effectiveness and evolve our architecture as well as complementing our stack with SaaS services +such as EKS, Redis, MSK. -- AWS managed Kubernetes gave us the flexibility we needed to meet our users - needs with as little overhead as possible. Most of our application choices - come directly from our users, however our infrastructure choices were made to - balance our compliance needs, usability and supportability. +- AWS managed Kubernetes gave us the flexibility we needed to meet our users needs with as little overhead as +possible. Most of our application choices come directly from our users, however our +infrastructure choices were made to balance our compliance needs, usability and supportability. -- ArgoCD has become instrumental for us and the core of our internal innovation. - We are currently using it for spinning up and managing our multitenant - implementation whether that is on namespace level or Fargate ECS level. +- ArgoCD has become instrumental for us and the core of our internal innovation. We are currently using it for spinning +up and managing our multitenant implementation whether that is on namespace level or Fargate ECS level. -- PAGT(Prometheus, Alert manager, Grafana and Thanos) stack is the core of our - Observability. It is complemented by OTEL and ELK Stack and enabled us to - build a state of the art SRE journey for all our Stakeholders +- PAGT(Prometheus, Alert manager, Grafana and Thanos) stack is the core of our Observability. +It is complemented by OTEL and ELK Stack and enabled us to build a state of the art SRE journey for all our Stakeholders ### Things that worked well for us -EKS and now EKS + Fargate ECS works great for us. We can fully owns our core -domains and stay hardened with minimum effort. Complementing our Stack with AWS -services especially on Persistence, Data Streaming and Storage increase our -innovation speed. ArgoCD with helm, Atlantis with terraform and Crossplane are -working great for us, allowing us to follow the GitOps way, a great way of -working for heavily regulated industries. +EKS and now EKS + Fargate ECS works great for us. We can fully owns our core domains and stay hardened with minimum effort. +Complementing our Stack with AWS services especially on Persistence, Data Streaming and Storage increase our innovation speed. +ArgoCD with helm, Atlantis with terraform and Crossplane are working great for us, allowing us to follow the GitOps way, +a great way of working for heavily regulated industries. ## Stakeholders value streams -### **Business** - -- Cost savings of X% -- Business Delivery expedited by X% -- Security incidents reduced by X% -- Reliability incidents reduced by X% -- Organizational Performance increased by X% - -### **End-Customers** - -- User Experience improved by X% -- Fast Feedback loop increased by X% -- Reliability increased by X% - -### **Developers** +### **Business** + - Cost savings of X% + - Business Delivery expedited by X% + - Security incidents reduced by X% + - Reliability incidents reduced by X% + - Organizational Performance increased by X% -- DevX improved by X% -- Fast Software Delivery improved by X% -- Productivity increased by X% -- High Quality increased by X% -- Innovation increased by X% +### **End-Customers** + - User Experience improved by X% + - Fast Feedback loop increased by X% + - Reliability increased by X% -### **Security** +### **Developers** + - DevX improved by X% + - Fast Software Delivery improved by X% + - Productivity increased by X% + - High Quality increased by X% + - Innovation increased by X% -- Cloud native Security costs reduced by X% +### **Security** + - Cloud native Security costs reduced by X% -### **Compliance** +### **Compliance** + - Cloud compliance costs reduced by X% -- Cloud compliance costs reduced by X% +### **Legal** + - Cloud native legal cost reduced by X% -### **Legal** +### **QA** + - Automation increased by X% -- Cloud native legal cost reduced by X% - -### **QA** - -- Automation increased by X% - -### **FinOps/Sustainability** - -- Cloud Costs reduced by X% +### **FinOps/Sustainability** + - Cloud Costs reduced by X% ## Some things did not work as expected -Service Mesh did not work well for us. The Operational costs and lack of -integration on the CNI level made us revert the journey. Sidecar, eBPF Service -Mesh seems to be the way forward for us. - +Service Mesh did not work well for us. The Operational costs and lack of integration on the CNI level made us revert the journey. +Sidecar, eBPF Service Mesh seems to be the way forward for us. + + ## Architecture evolution -The Architecture was designed as a single tenant but due to its success it -needed to evolve to support multitenancy on AWS account level, next on namespace -level and now, due to the Global impact, on the Fargate ECS level. Inevitably we -ended with three Platforms(Architectures) that need to be simplify towards a -maximum two that will evolve rapidly. Migration work is inevitable. +The Architecture was designed as a single tenant but due to its success it needed to evolve to support multitenancy on +AWS account level, next on namespace level and now, due to the Global impact, on the Fargate ECS level. +Inevitably we ended with three Platforms(Architectures) that need to be simplify towards a maximum two that will evolve rapidly. +Migration work is inevitable. ## The Journey - -It took us 4 years to move towards the current Platform Engineering setup. There -is more to come. The full Journey as it happened till today 28.10.2024 can be -seen in the following diagram: +It took us 4 years to move towards the current Platform Engineering setup. There is more to come. The full Journey as it happened till today 28.10.2024 can be seen in the following diagram: ![High Level Diagram](/img/architectures/allianz/PEJourney.jpg) diff --git a/docs/architectures/cern-scientific-computing.md b/docs/architectures/cern-scientific-computing.md index 7f75b55f..a203b332 100644 --- a/docs/architectures/cern-scientific-computing.md +++ b/docs/architectures/cern-scientific-computing.md @@ -9,27 +9,27 @@ import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; ## Relevant Projects - + - + - + - + - + - + - + - + - + - + - + Some additional projects outside the CNCF are essential to this deployment. * [MLFlow](https://mlflow.org/), a project under the Linux Foundation used for model management and versioning diff --git a/docs/architectures/colopl.md b/docs/architectures/colopl.md index 1c093b8b..c1907586 100644 --- a/docs/architectures/colopl.md +++ b/docs/architectures/colopl.md @@ -1,73 +1,52 @@ --- -title: - "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with - Agones on Kubernetes" -sidebar_label: 'Colopl Inc.' +title: "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with Agones on Kubernetes" +sidebar_label: "Colopl Inc." --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant CNCF projects - + - + ## Architecture highlights ![](/img/architectures/colopl/festibattle-architecture.png) -At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud -Spanner as the backend database. While that combination is somewhat distinctive, -the overall structure is not far from a typical web service. - -What makes this a game architecture is the orchestration of dedicated game -servers with [Agones]. When allocating a dedicated game server through Agones, -we place an in-house component called the Allocator Service in front of the -allocation flow. It decides which game server each request should be assigned -to. Reducing the maintenance cost of this in-house Allocator Service, and -ultimately eliminating it, is one of our ongoing challenges. We develop and -operate multiple game titles concurrently, so custom components like this weigh -on every title. - -One of the reasons we built our own Allocator Service is that Agones originally -had no way to re-allocate a dedicated game server that had already been -allocated. Many of COLOPL's titles are MO (multiplayer online) games where a -small group of players plays together. Our dedicated game servers implement a -concept we call rooms: matched players are packed into the same room, which -enables communication between them. - -A single dedicated game server can host multiple rooms, so one process can -handle the traffic of many player groups at once. This lets us use server -resources very efficiently. The catch is that realizing the room concept -requires re-allocating game servers that have already been allocated. - -For [FESTIBATTLE], we implemented this re-allocation mechanism by adopting the -[High Density GameServer pattern]. This brings our allocation flow much closer -to the standard Agones Allocator Service and makes future migration easier. +At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud Spanner as the backend database. While that combination is somewhat distinctive, the overall structure is not far from a typical web service. + +What makes this a game architecture is the orchestration of dedicated game servers with [Agones]. +When allocating a dedicated game server through Agones, we place an in-house component called the Allocator Service in front of the allocation flow. +It decides which game server each request should be assigned to. +Reducing the maintenance cost of this in-house Allocator Service, and ultimately eliminating it, is one of our ongoing challenges. +We develop and operate multiple game titles concurrently, so custom components like this weigh on every title. + +One of the reasons we built our own Allocator Service is that Agones originally had no way to re-allocate a dedicated game server that had already been allocated. +Many of COLOPL's titles are MO (multiplayer online) games where a small group of players plays together. +Our dedicated game servers implement a concept we call rooms: matched players are packed into the same room, which enables communication between them. + +A single dedicated game server can host multiple rooms, so one process can handle the traffic of many player groups at once. +This lets us use server resources very efficiently. +The catch is that realizing the room concept requires re-allocating game servers that have already been allocated. + +For [FESTIBATTLE], we implemented this re-allocation mechanism by adopting the [High Density GameServer pattern]. +This brings our allocation flow much closer to the standard Agones Allocator Service and makes future migration easier. ## Current challenges and what's next -Agones now provides the [Counters and Lists] feature in Beta, which enables -dedicated game server orchestration that fits our use case even better. With -Counters and Lists, user-defined values (something like a room ID, for example) -can be stored on the GameServer resource and then used for allocation decisions -or as autoscaling thresholds. +Agones now provides the [Counters and Lists] feature in Beta, which enables dedicated game server orchestration that fits our use case even better. +With Counters and Lists, user-defined values (something like a room ID, for example) can be stored on the GameServer resource and then used for allocation decisions or as autoscaling thresholds. -Going forward, we will explore using this feature to migrate from our in-house -Allocator Service to the standard Agones one. +Going forward, we will explore using this feature to migrate from our in-house Allocator Service to the standard Agones one. -_NOTE: This article is an English translation adaptation of [our original -article] published in Japanese on Findy Tools._ +_NOTE: This article is an English translation adpation of [our original article] published in Japanese on Findy Tools._ [Agones]: https://agones.dev/site/ [FESTIBATTLE]: https://festibattle.jp/ -[High Density GameServer pattern]: - https://agones.dev/site/docs/integration-patterns/high-density-gameservers/ +[High Density GameServer pattern]: https://agones.dev/site/docs/integration-patterns/high-density-gameservers/ [Counters and Lists]: https://agones.dev/site/docs/guides/counters-and-lists/ [our original article]: https://findy-tools.io/companies/colopl/180/48 diff --git a/docs/architectures/flipkart-chaos-engineering.md b/docs/architectures/flipkart-chaos-engineering.md index 7bb9b17d..f516876a 100644 --- a/docs/architectures/flipkart-chaos-engineering.md +++ b/docs/architectures/flipkart-chaos-engineering.md @@ -1,154 +1,87 @@ --- -title: - 'From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering - Platform on LitmusChaos' -sidebar_label: 'Flipkart Internet Pvt. Ltd.' +title: "From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering Platform on LitmusChaos" +sidebar_label: "Flipkart Internet Pvt. Ltd." --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/flipkart-chaos-engineering). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/flipkart-chaos-engineering). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant CNCF projects - + - + - + - + - + - + ## Other Projects -| Project | Using Since | Notes | -| :-----------------------------------: | :---------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | -| Grafana | 2022 | Dashboarding layer for chaos experiment telemetry. Flipkart reuses Litmus’s out-of-the-box Grafana exporters to surface experiment run history, resilience scores, and infrastructure-level signals to both Kubernetes and VM-team consumers — giving both audiences the same observability surface. | -| MongoDB | 2023 | Backing store for the Litmus control plane. Flipkart’s probe-uniqueness contribution required reworking a global MongoDB index into a project-scoped one to make probes truly multi-tenant. | -| Linux stress tooling on Debian | 2023 | Internal stress packages (CPU hog, memory hog, network hog, disk-IO) targeted at the Debian host OS of Flipkart’s VM fleet, invoked by the Hybrid VM Chaos extension when the workload doesn’t live in Kubernetes. | -| Internal Flipkart Cloud Platform APIs | 2024 | Lifecycle APIs (start, stop, restart) for Flipkart’s VM fleet. The Hybrid VM Chaos extension calls these APIs to deliver VM-power experiments in parity with the Kubernetes-native experience. | -| Internal Chaos Hub | 2024 | Private, Flipkart-internal catalog of pre-curated chaos experiment templates. Built so that tenants can publish, version, and consume reusable experiments across teams — the chaos-engineering analogue of an internal package registry. | +| Project | Using Since | Notes | +|:-------------------------------------:|:-----------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| Grafana | 2022 | Dashboarding layer for chaos experiment telemetry. Flipkart reuses Litmus’s out-of-the-box Grafana exporters to surface experiment run history, resilience scores, and infrastructure-level signals to both Kubernetes and VM-team consumers — giving both audiences the same observability surface. | +| MongoDB | 2023 | Backing store for the Litmus control plane. Flipkart’s probe-uniqueness contribution required reworking a global MongoDB index into a project-scoped one to make probes truly multi-tenant. | +| Linux stress tooling on Debian | 2023 | Internal stress packages (CPU hog, memory hog, network hog, disk-IO) targeted at the Debian host OS of Flipkart’s VM fleet, invoked by the Hybrid VM Chaos extension when the workload doesn’t live in Kubernetes. | +| Internal Flipkart Cloud Platform APIs | 2024 | Lifecycle APIs (start, stop, restart) for Flipkart’s VM fleet. The Hybrid VM Chaos extension calls these APIs to deliver VM-power experiments in parity with the Kubernetes-native experience. | +| Internal Chaos Hub | 2024 | Private, Flipkart-internal catalog of pre-curated chaos experiment templates. Built so that tenants can publish, version, and consume reusable experiments across teams — the chaos-engineering analogue of an internal package registry. | ## Synopsis -Flipkart is one of India’s leading digital commerce companies, running hundreds -of tightly coupled microservices across Kubernetes and VM workloads that must -withstand the traffic surges of Big Billion Days and other festive sales. The -Central Reliability Engineering team built a centralized chaos engineering -platform on LitmusChaos to convert chaos engineering from an afterthought into a -continuous practice. - -On top of upstream Litmus, Flipkart engineered four high-leverage -customizations: (1) a hybrid multi-tenancy model that combines the operational -simplicity of a cluster-wide install with the isolation of a namespace-wide -install; (2) a DaemonSet-based high-availability layer for chaos injection that -runs one persistent injector per node and supports parallel sessions; (3) a -first-class “Script Runner” fault that allows dynamic target selection and -context chaining across steps in an experiment; and (4) a hybrid VM chaos -extension that reuses the Litmus experience for non-Kubernetes workloads via -Flipkart’s internal cloud APIs and Debian-host stress tooling. - -The result is operational readiness validated through live chaos drills ahead of -Big Billion Days, a measurable shift in engineering culture from “prevention” to -“practice,” and a steady stream of upstream contributions back to the -LitmusChaos project. +Flipkart is one of India’s leading digital commerce companies, running hundreds of tightly coupled microservices across Kubernetes and VM workloads that must withstand the traffic surges of Big Billion Days and other festive sales. The Central Reliability Engineering team built a centralized chaos engineering platform on LitmusChaos to convert chaos engineering from an afterthought into a continuous practice. + +On top of upstream Litmus, Flipkart engineered four high-leverage customizations: (1) a hybrid multi-tenancy model that combines the operational simplicity of a cluster-wide install with the isolation of a namespace-wide install; (2) a DaemonSet-based high-availability layer for chaos injection that runs one persistent injector per node and supports parallel sessions; (3) a first-class “Script Runner” fault that allows dynamic target selection and context chaining across steps in an experiment; and (4) a hybrid VM chaos extension that reuses the Litmus experience for non-Kubernetes workloads via Flipkart’s internal cloud APIs and Debian-host stress tooling. + +The result is operational readiness validated through live chaos drills ahead of Big Billion Days, a measurable shift in engineering culture from “prevention” to “practice,” and a steady stream of upstream contributions back to the LitmusChaos project. ## Organization -Flipkart is one of India’s leading digital commerce companies, serving hundreds -of millions of customers across an ecosystem that includes flagship apparel, -electronics, grocery, and fintech offerings. Its flagship event, Big Billion -Days, is the largest e-commerce sales event in India and drives massive -concurrent traffic across a tightly integrated microservices estate. Other -high-scale moments — the Independence Day sale and Diwali sale among them — -generate similar surges throughout the year. - -Flipkart’s engineering organization runs an extensive Kubernetes-as-a-Service -platform alongside a large VM fleet, with internal central platform teams -providing service mesh, networking, and database offerings to hundreds of -downstream application teams. Reliability at the scale of an Indian retail -tentpole event is therefore not a product-team concern — it is a horizontal -platform discipline. +Flipkart is one of India’s leading digital commerce companies, serving hundreds of millions of customers across an ecosystem that includes flagship apparel, electronics, grocery, and fintech offerings. Its flagship event, Big Billion Days, is the largest e-commerce sales event in India and drives massive concurrent traffic across a tightly integrated microservices estate. Other high-scale moments — the Independence Day sale and Diwali sale among them — generate similar surges throughout the year. + +Flipkart’s engineering organization runs an extensive Kubernetes-as-a-Service platform alongside a large VM fleet, with internal central platform teams providing service mesh, networking, and database offerings to hundreds of downstream application teams. Reliability at the scale of an Indian retail tentpole event is therefore not a product-team concern — it is a horizontal platform discipline. ## Teams Multiple teams collaborate to build, operate, and consume the chaos platform: -- **Central Reliability Engineering** — owns the chaos platform end-to-end: the - centralized LitmusChaos control plane, tenant onboarding, the DaemonSet - injection layer, the Script Runner fault, the hybrid VM chaos extension, the - internal Chaos Hub, and upstream contributions back to Litmus. +* **Central Reliability Engineering** — owns the chaos platform end-to-end: the centralized LitmusChaos control plane, tenant onboarding, the DaemonSet injection layer, the Script Runner fault, the hybrid VM chaos extension, the internal Chaos Hub, and upstream contributions back to Litmus. -- **Kubernetes-as-a-Service team** — first tenant of the chaos platform; runs - chaos drills against the Kubernetes control plane primitives consumed by every - Flipkart workload. +* **Kubernetes-as-a-Service team** — first tenant of the chaos platform; runs chaos drills against the Kubernetes control plane primitives consumed by every Flipkart workload. -- **Network Services and Service Mesh team** — uses chaos to validate Layer-4/7 - failure behavior, retry semantics, and circuit-breaker configurations across - the mesh. +* **Network Services and Service Mesh team** — uses chaos to validate Layer-4/7 failure behavior, retry semantics, and circuit-breaker configurations across the mesh. -- **Central Database team** — leverages the Script Runner fault to verify - leader-election behavior in leader-follower database topologies during chaos - drills. +* **Central Database team** — leverages the Script Runner fault to verify leader-election behavior in leader-follower database topologies during chaos drills. -- **Application teams** — onboarded progressively after the central platform - teams; consume curated experiments from the internal Chaos Hub. +* **Application teams** — onboarded progressively after the central platform teams; consume curated experiments from the internal Chaos Hub. -- **VM Infrastructure teams** — use the hybrid VM chaos extension to obtain the - same observability surface (Grafana, run history, resilience score) as - Kubernetes-native consumers, despite running outside Kubernetes. +* **VM Infrastructure teams** — use the hybrid VM chaos extension to obtain the same observability surface (Grafana, run history, resilience score) as Kubernetes-native consumers, despite running outside Kubernetes. ## Architecture overview & Goals ### Goals -1. **Make chaos engineering a continuous practice, not an afterthought.** Move - from runbooks-on-paper to regularly rehearsed failure scenarios that feed - back into incident response. +1. **Make chaos engineering a continuous practice, not an afterthought.** Move from runbooks-on-paper to regularly rehearsed failure scenarios that feed back into incident response. -2. **Operate one chaos platform for the whole company, safely.** Provide a - centralized control plane that hundreds of tenants can use without each - tenant carrying operational burden — and without any tenant being able to - touch another tenant’s workloads. +2. **Operate one chaos platform for the whole company, safely.** Provide a centralized control plane that hundreds of tenants can use without each tenant carrying operational burden — and without any tenant being able to touch another tenant’s workloads. -3. **Eliminate self-inflicted experiment failures.** Remove brittleness from the - chaos infrastructure itself — specifically, helper-pod scheduling failures - that were masking real reliability signal. +3. **Eliminate self-inflicted experiment failures.** Remove brittleness from the chaos infrastructure itself — specifically, helper-pod scheduling failures that were masking real reliability signal. -4. **Express realistic, real-world failure scenarios.** Support dynamic target - selection and context chaining inside experiments so a single chaos run can - express “find the leader, kill it, watch a new leader get elected” as one - declarative workflow. +4. **Express realistic, real-world failure scenarios.** Support dynamic target selection and context chaining inside experiments so a single chaos run can express “find the leader, kill it, watch a new leader get elected” as one declarative workflow. -5. **Treat VM workloads as first-class citizens.** Reuse the Litmus experience - for workloads that don’t run on Kubernetes — same UI, same probes, same - observability — instead of building a parallel tool. +5. **Treat VM workloads as first-class citizens.** Reuse the Litmus experience for workloads that don’t run on Kubernetes — same UI, same probes, same observability — instead of building a parallel tool. -6. **Validate operational readiness for Big Billion Days.** Convert chaos drills - into a measurable, repeatable pre-event sign-off rather than a manual - exercise. +6. **Validate operational readiness for Big Billion Days.** Convert chaos drills into a measurable, repeatable pre-event sign-off rather than a manual exercise. -7. **Contribute upstream.** Push fixes and architectural patterns back into - LitmusChaos so the broader CNCF community benefits and Flipkart avoids - long-term fork maintenance. +7. **Contribute upstream.** Push fixes and architectural patterns back into LitmusChaos so the broader CNCF community benefits and Flipkart avoids long-term fork maintenance. ### Architecture Overview -At a high level, the architecture consists of a centralised control plane that -is setup on a Kubernetes namespace or cluster that helps in designing and -coordinating the chaos experiments. The chaos experiments are designed, -implemented and certified and finally placed in a ChaosHub, which then are used -by various teams in the organization to run them on the target Kubernetes -clusters. Chaos experiment results are pushed back into the control plane as -resilience scores, which can be used to create resilience testing trends and -resilience posture reports for various teams’ consumption. +At a high level, the architecture consists of a centralised control plane that is setup on a Kubernetes namespace or cluster that helps in designing and coordinating the chaos experiments. The chaos experiments are designed, implemented and certified and finally placed in a ChaosHub, which then are used by various teams in the organization to run them on the target Kubernetes clusters. Chaos experiment results are pushed back into the control plane as resilience scores, which can be used to create resilience testing trends and resilience posture reports for various teams’ consumption. The platform is organized into four logical layers: @@ -158,361 +91,179 @@ The platform is organized into four logical layers: ![](/img/architectures/flipkart-chaos-engineering/layers.png) -**Control Plane Layer.** A single, centralized LitmusChaos installation — the -operator, workflow controller, MongoDB backing store, and Grafana dashboards — -lives in a dedicated central namespace and is operated by Central Reliability -Engineering. There is one control plane per cluster footprint (staging, -production), not one per tenant. - -**Tenant Subscriber Layer or a chaos agent.** For each tenant team, a dedicated -subscriber is deployed in the central namespace. The subscriber is scoped at -install time to the set of tenant-owned namespaces it is allowed to target. The -subscriber is the only Litmus identity that can dispatch chaos into a tenant’s -namespaces — giving namespace-grade isolation without forcing each tenant to -operate its own Litmus install. - -**Injection Layer.** Instead of spawning ephemeral helper pods on-demand per -experiment, Flipkart runs a DaemonSet (one replica per node, using the litmus-go -image) as a long-lived, highly privileged injection surface. Experiment pods now -offload target information to the node-local DaemonSet pod, which executes the -chaos in parallel shell sessions. This supports concurrent chaos injection on -the same node and eliminates a class of failures where the previous on-demand -helper pod couldn’t be scheduled in time. - -**Extension Layer.** Two domain-specific extensions plug into the platform: the -Script Runner fault (a user-defined container running a user-defined script, -whose stdout feeds the target list of any subsequent fault in the experiment), -and the Hybrid VM Chaos extension (which uses Flipkart’s internal cloud-platform -APIs for VM power operations and internal Debian stress packages for -resource-level chaos). An internal Chaos Hub layers on top, letting tenants -publish reusable experiment templates. - -Tenants interact with the platform almost entirely through the Litmus UI. They -author or pick a template from the internal Chaos Hub, attach probes (including -SSH-based probes for the VM surface), and run the experiment against namespaces -or VMs they own. Resilience scores, run history, and Grafana dashboards present -a single observability surface regardless of whether the target is a Kubernetes -workload or a VM. +**Control Plane Layer.** A single, centralized LitmusChaos installation — the operator, workflow controller, MongoDB backing store, and Grafana dashboards — lives in a dedicated central namespace and is operated by Central Reliability Engineering. There is one control plane per cluster footprint (staging, production), not one per tenant. + +**Tenant Subscriber Layer or a chaos agent.** For each tenant team, a dedicated subscriber is deployed in the central namespace. The subscriber is scoped at install time to the set of tenant-owned namespaces it is allowed to target. The subscriber is the only Litmus identity that can dispatch chaos into a tenant’s namespaces — giving namespace-grade isolation without forcing each tenant to operate its own Litmus install. + +**Injection Layer.** Instead of spawning ephemeral helper pods on-demand per experiment, Flipkart runs a DaemonSet (one replica per node, using the litmus-go image) as a long-lived, highly privileged injection surface. Experiment pods now offload target information to the node-local DaemonSet pod, which executes the chaos in parallel shell sessions. This supports concurrent chaos injection on the same node and eliminates a class of failures where the previous on-demand helper pod couldn’t be scheduled in time. + +**Extension Layer.** Two domain-specific extensions plug into the platform: the Script Runner fault (a user-defined container running a user-defined script, whose stdout feeds the target list of any subsequent fault in the experiment), and the Hybrid VM Chaos extension (which uses Flipkart’s internal cloud-platform APIs for VM power operations and internal Debian stress packages for resource-level chaos). An internal Chaos Hub layers on top, letting tenants publish reusable experiment templates. + +Tenants interact with the platform almost entirely through the Litmus UI. They author or pick a template from the internal Chaos Hub, attach probes (including SSH-based probes for the VM surface), and run the experiment against namespaces or VMs they own. Resilience scores, run history, and Grafana dashboards present a single observability surface regardless of whether the target is a Kubernetes workload or a VM. #### Key Design Principles -- **Centralized control plane, federated trust.** One operator and one workflow - controller for the whole organization; isolation is enforced at the subscriber - boundary, not by running N copies of Litmus. +* **Centralized control plane, federated trust.** One operator and one workflow controller for the whole organization; isolation is enforced at the subscriber boundary, not by running N copies of Litmus. -- **Persistent, node-local injection.** Trade ephemeral safety for scheduling - determinism by accepting a long-lived privileged DaemonSet — a deliberate - trade-off made only where the operational win (no scheduling failures during a - chaos drill) outweighs the cost (privileged residency). +* **Persistent, node-local injection.** Trade ephemeral safety for scheduling determinism by accepting a long-lived privileged DaemonSet — a deliberate trade-off made only where the operational win (no scheduling failures during a chaos drill) outweighs the cost (privileged residency). -- **Context chaining over static targeting.** Chaos experiments are workflows, - not single faults; the output of one step (“who is the leader?”) feeds the - inputs of the next (“kill that pod”). +* **Context chaining over static targeting.** Chaos experiments are workflows, not single faults; the output of one step (“who is the leader?”) feeds the inputs of the next (“kill that pod”). -- **One experience, two surfaces.** Kubernetes and VM chaos must look, feel, and - report the same to the user. The UI, probes, and observability layer are - surface-agnostic; the differences are pushed down into the extension layer. +* **One experience, two surfaces.** Kubernetes and VM chaos must look, feel, and report the same to the user. The UI, probes, and observability layer are surface-agnostic; the differences are pushed down into the extension layer. -- **Practice, then expand.** Start with central platform teams that already run - shared infrastructure; only then expand to application teams. Staging is the - primary chaos surface; production chaos is selective and gated. +* **Practice, then expand.** Start with central platform teams that already run shared infrastructure; only then expand to application teams. Staging is the primary chaos surface; production chaos is selective and gated. -- **Upstream where possible.** Local patches create long-term maintenance - burden. Bug fixes and reusable patterns go back to LitmusChaos. +* **Upstream where possible.** Local patches create long-term maintenance burden. Bug fixes and reusable patterns go back to LitmusChaos. ## Can you expand on why you are using those projects/services? ### Why LitmusChaos -Before standardizing on Litmus, the team ran a detailed proof-of-concept across -Chaos Monkey, Chaos Mesh, and LitmusChaos. Litmus won on four criteria that -mattered for Flipkart’s context: it is Kubernetes-native end-to-end (aligning -with the broader K8s-first direction), it ships a rich drag-and-drop UI that -drives adoption beyond SREs, it is deeply extensible (custom faults were a -non-negotiable requirement), and its resilience-probe model gave the team a path -toward a measurable resiliency score per service. +Before standardizing on Litmus, the team ran a detailed proof-of-concept across Chaos Monkey, Chaos Mesh, and LitmusChaos. Litmus won on four criteria that mattered for Flipkart’s context: it is Kubernetes-native end-to-end (aligning with the broader K8s-first direction), it ships a rich drag-and-drop UI that drives adoption beyond SREs, it is deeply extensible (custom faults were a non-negotiable requirement), and its resilience-probe model gave the team a path toward a measurable resiliency score per service. ### From single-tenant to hybrid multi-tenancy -The two install modes Litmus offers out of the box are cluster-wide (low -operational burden, but no isolation) and namespace-wide (strong isolation, but -every tenant has to operate its own Litmus install). At Flipkart’s scale — -hundreds of teams across central platform and application areas — neither -extreme works. The hybrid model takes the best of both: one centralized control -plane (so no team has to run Litmus) plus a per-tenant subscriber pinned to that -tenant’s namespaces (so isolation is enforced at the dispatch boundary). Litmus -maintainer Karthik Satchitanand described this pattern as “a very nice model… a -good balance” and has suggested it be documented upstream as a preferred mode of -operation. +The two install modes Litmus offers out of the box are cluster-wide (low operational burden, but no isolation) and namespace-wide (strong isolation, but every tenant has to operate its own Litmus install). At Flipkart’s scale — hundreds of teams across central platform and application areas — neither extreme works. The hybrid model takes the best of both: one centralized control plane (so no team has to run Litmus) plus a per-tenant subscriber pinned to that tenant’s namespaces (so isolation is enforced at the dispatch boundary). Litmus maintainer Karthik Satchitanand described this pattern as “a very nice model… a good balance” and has suggested it be documented upstream as a preferred mode of operation. ### Why a DaemonSet instead of helper pods -Litmus’s default execution model spins up an on-demand helper pod per -experiment. At scale, in busy staging clusters with active chaos drills, those -helper pods can fail to schedule — producing a chaos infrastructure failure that -masquerades as the chaos itself. The DaemonSet inverts this: one long-lived pod -per node, with the chaos image already present, executes parallel chaos sessions -invoked by the experiment pods. This makes injection deterministic at the cost -of a persistent privileged surface — a trade-off Flipkart accepts in staging, -where platform availability for chaos drills matters most. +Litmus’s default execution model spins up an on-demand helper pod per experiment. At scale, in busy staging clusters with active chaos drills, those helper pods can fail to schedule — producing a chaos infrastructure failure that masquerades as the chaos itself. The DaemonSet inverts this: one long-lived pod per node, with the chaos image already present, executes parallel chaos sessions invoked by the experiment pods. This makes injection deterministic at the cost of a persistent privileged surface — a trade-off Flipkart accepts in staging, where platform availability for chaos drills matters most. ### Why a Script Runner fault -Real-world failure scenarios are rarely “delete pod X.” They are “find the -current leader of a leader-follower system, delete it, and verify re-election.” -Litmus’s built-in faults could not express dynamic target selection or pass -state forward from one step to the next. The Script Runner is a first-class -fault type that executes a user-defined container running a user-defined script; -its stdout becomes the target list for any subsequent fault in the experiment, -not just the immediate next one. The database team’s leader-election -verification was the canonical use case that motivated this design. +Real-world failure scenarios are rarely “delete pod X.” They are “find the current leader of a leader-follower system, delete it, and verify re-election.” Litmus’s built-in faults could not express dynamic target selection or pass state forward from one step to the next. The Script Runner is a first-class fault type that executes a user-defined container running a user-defined script; its stdout becomes the target list for any subsequent fault in the experiment, not just the immediate next one. The database team’s leader-election verification was the canonical use case that motivated this design. ### **Why a Hybrid VM Chaos extension** -A meaningful slice of Flipkart’s estate runs on VMs, not Kubernetes. Building a -parallel chaos tool for that surface would have fragmented the user experience -and split the observability story. Instead, the team rebuilt the existing VM -Power experiment against Flipkart’s internal cloud platform APIs -(start/stop/restart) and added resource-level stress chaos (CPU hog, memory hog, -network hog, disk-level) using internal Debian stress packages. Because the -experiments are still driven by Litmus, VM teams get the same Grafana exporters, -the same experiment run history, the same resilience score tab, and the same -probe model (with custom SSH-based probes for pre- and post-experiment -infrastructure checks) that Kubernetes users get. +A meaningful slice of Flipkart’s estate runs on VMs, not Kubernetes. Building a parallel chaos tool for that surface would have fragmented the user experience and split the observability story. Instead, the team rebuilt the existing VM Power experiment against Flipkart’s internal cloud platform APIs (start/stop/restart) and added resource-level stress chaos (CPU hog, memory hog, network hog, disk-level) using internal Debian stress packages. Because the experiments are still driven by Litmus, VM teams get the same Grafana exporters, the same experiment run history, the same resilience score tab, and the same probe model (with custom SSH-based probes for pre- and post-experiment infrastructure checks) that Kubernetes users get. ## What has worked well? -- **Hybrid multi-tenancy.** The centralized-control-plane plus - per-tenant-subscriber pattern absorbed onboarding for many tenants with - effectively zero per-tenant infra burden, while still preventing cross-tenant - blast radius. Recognized by upstream Litmus maintainers as a candidate - preferred-mode pattern. +* **Hybrid multi-tenancy.** The centralized-control-plane plus per-tenant-subscriber pattern absorbed onboarding for many tenants with effectively zero per-tenant infra burden, while still preventing cross-tenant blast radius. Recognized by upstream Litmus maintainers as a candidate preferred-mode pattern. -- **DaemonSet injection.** Eliminated experiment failures caused by helper-pod - scheduling issues in staging. Chaos drills now fail because the system under - test failed — not because the chaos infrastructure couldn’t schedule a pod. +* **DaemonSet injection.** Eliminated experiment failures caused by helper-pod scheduling issues in staging. Chaos drills now fail because the system under test failed — not because the chaos infrastructure couldn’t schedule a pod. -- **Script Runner.** Unlocked previously inexpressible scenarios such as - leader-election verification. Context chaining turned chaos experiments into - proper workflows rather than single-shot fault injections. +* **Script Runner.** Unlocked previously inexpressible scenarios such as leader-election verification. Context chaining turned chaos experiments into proper workflows rather than single-shot fault injections. -- **Unified VM and Kubernetes experience.** Reusing Grafana exporters, run - history, and the resilience score tab gave VM teams a first-class user - experience and avoided fragmenting the chaos tooling landscape inside - Flipkart. +* **Unified VM and Kubernetes experience.** Reusing Grafana exporters, run history, and the resilience score tab gave VM teams a first-class user experience and avoided fragmenting the chaos tooling landscape inside Flipkart. -- **Pre-event readiness validation.** Chaos drills validated operational - readiness ahead of the recent Big Billion Days sale, confirmed that CPU - thresholds trigger the right alerts, that network failures surface correctly - as 5xx spikes, and that weak links in deployment specs and alert - configurations could be fixed before they became real incidents. +* **Pre-event readiness validation.** Chaos drills validated operational readiness ahead of the recent Big Billion Days sale, confirmed that CPU thresholds trigger the right alerts, that network failures surface correctly as 5xx spikes, and that weak links in deployment specs and alert configurations could be fixed before they became real incidents. -- **Cultural shift.** Engineering teams moved from “prevention” (try not to - fail) to “practice” (rehearse failure), and from “panic” to “procedure” during - incident response. Chaos scenarios now form the foundation of incident - runbooks. +* **Cultural shift.** Engineering teams moved from “prevention” (try not to fail) to “practice” (rehearse failure), and from “panic” to “procedure” during incident response. Chaos scenarios now form the foundation of incident runbooks. -- **Internal Chaos Hub.** Tenant-published experiment templates accelerate reuse - across teams — the chaos engineering analogue of an internal package registry. +* **Internal Chaos Hub.** Tenant-published experiment templates accelerate reuse across teams — the chaos engineering analogue of an internal package registry. ## What has not worked well? -- **Privileged DaemonSet residency.** Persistent, highly privileged pods on - every node are an accepted trade-off in staging but constrain how aggressively - the same pattern can be deployed in production. Reducing the blast radius of - the injection layer remains an open problem. +* **Privileged DaemonSet residency.** Persistent, highly privileged pods on every node are an accepted trade-off in staging but constrain how aggressively the same pattern can be deployed in production. Reducing the blast radius of the injection layer remains an open problem. -- **Multi-tenant gaps in upstream Litmus.** Several places in upstream Litmus - assumed single-tenant scope — most visibly, probe-name uniqueness enforced - globally instead of project-scoped (a MongoDB index issue). These needed - upstream fixes before the multi-tenant model could behave correctly. +* **Multi-tenant gaps in upstream Litmus.** Several places in upstream Litmus assumed single-tenant scope — most visibly, probe-name uniqueness enforced globally instead of project-scoped (a MongoDB index issue). These needed upstream fixes before the multi-tenant model could behave correctly. -- **Terminology drift across the project.** The renaming of “experiment” to - “fault” in parts of Litmus left behind frontend/backend mismatches — e.g., - experiment pod-block visibility — that required cleanup contributions. +* **Terminology drift across the project.** The renaming of “experiment” to “fault” in parts of Litmus left behind frontend/backend mismatches — e.g., experiment pod-block visibility — that required cleanup contributions. -- **Heterogeneous VM hosts.** The stress chaos extension is currently scoped to - Debian hosts; broadening to additional host OSes will mean factoring out the - OS-specific layer. +* **Heterogeneous VM hosts.** The stress chaos extension is currently scoped to Debian hosts; broadening to additional host OSes will mean factoring out the OS-specific layer. -- **Production chaos is still selective.** Most chaos still runs in staging. - Building enough confidence (and enough guardrails) to run the full chaos - surface in production is an ongoing journey. +* **Production chaos is still selective.** Most chaos still runs in staging. Building enough confidence (and enough guardrails) to run the full chaos surface in production is an ongoing journey. -- **Custom image registry support.** A long-standing (year-old) issue prevented - the custom image registry setting from flowing through to the generated - workflow YAML — a blocker for air-gapped and internal-registry deployments. - Required a Flipkart upstream fix to resolve. +* **Custom image registry support.** A long-standing (year-old) issue prevented the custom image registry setting from flowing through to the generated workflow YAML — a blocker for air-gapped and internal-registry deployments. Required a Flipkart upstream fix to resolve. ## What sort of "glue" have you had to develop? -- **Per-tenant subscriber onboarding pipeline.** Tooling and a defined - onboarding flow so that adding a new tenant means deploying a scoped - subscriber, registering its namespaces, and attaching it to the central - control plane — without any tenant operating its own Litmus install. +* **Per-tenant subscriber onboarding pipeline.** Tooling and a defined onboarding flow so that adding a new tenant means deploying a scoped subscriber, registering its namespaces, and attaching it to the central control plane — without any tenant operating its own Litmus install. -- **DaemonSet injection runtime.** Replaced the on-demand helper-pod model with - a long-lived, node-local DaemonSet (using the litmus-go image) that accepts - target information from experiment pods and executes parallel chaos sessions - in independent shells. +* **DaemonSet injection runtime.** Replaced the on-demand helper-pod model with a long-lived, node-local DaemonSet (using the litmus-go image) that accepts target information from experiment pods and executes parallel chaos sessions in independent shells. -- **Script Runner fault.** New first-class fault type backed by a user-defined - container image running a user-defined script, with stdout captured as - structured context that any subsequent fault in the experiment can consume — - not just the immediate next step. +* **Script Runner fault.** New first-class fault type backed by a user-defined container image running a user-defined script, with stdout captured as structured context that any subsequent fault in the experiment can consume — not just the immediate next step. -- **Hybrid VM Chaos extension.** A reimplementation of VM Power on top of - Flipkart’s internal cloud APIs, plus a stress-chaos suite (CPU, memory, - network, disk) built on internal Debian stress packages, plus custom SSH-based - probes that mirror the Kubernetes probe experience for VM workloads. +* **Hybrid VM Chaos extension.** A reimplementation of VM Power on top of Flipkart’s internal cloud APIs, plus a stress-chaos suite (CPU, memory, network, disk) built on internal Debian stress packages, plus custom SSH-based probes that mirror the Kubernetes probe experience for VM workloads. -- **Internal Chaos Hub publishing pipeline.** A workflow that lets tenants - publish reusable experiment templates to a private, Flipkart-internal catalog - and pull them down into their own experiments. +* **Internal Chaos Hub publishing pipeline.** A workflow that lets tenants publish reusable experiment templates to a private, Flipkart-internal catalog and pull them down into their own experiments. -- **Upstream fixes packaged as patches and PRs.** Probe-name uniqueness (MongoDB - index), probe-name reuse-after-deletion (filtration), experiment-update - duplicate-name validation, UI consistency (experiment-completion icon, - GCP/Azure fault logos), custom image registry, and pod-block visibility — each - shipped back to the LitmusChaos project. +* **Upstream fixes packaged as patches and PRs.** Probe-name uniqueness (MongoDB index), probe-name reuse-after-deletion (filtration), experiment-update duplicate-name validation, UI consistency (experiment-completion icon, GCP/Azure fault logos), custom image registry, and pod-block visibility — each shipped back to the LitmusChaos project. ## How did the Architecture Evolve** ### Journey -- **Phase 1 — Reactive runbooks.** Failure modes existed only on paper. Runbooks - were written but not practiced. Chaos engineering was an afterthought. +* **Phase 1 — Reactive runbooks.** Failure modes existed only on paper. Runbooks were written but not practiced. Chaos engineering was an afterthought. -- **Phase 2 — Tool selection.** Detailed PoC across Chaos Monkey, Chaos Mesh, - and LitmusChaos. Litmus selected on Kubernetes-native design, drag-and-drop - UI, extensibility, and resilience-probe model. +* **Phase 2 — Tool selection.** Detailed PoC across Chaos Monkey, Chaos Mesh, and LitmusChaos. Litmus selected on Kubernetes-native design, drag-and-drop UI, extensibility, and resilience-probe model. -- **Phase 3 — Centralization.** Stood up a single centralized Litmus control - plane and built the per-tenant subscriber model to hit multi-tenancy without - forcing tenants to run their own installs. +* **Phase 3 — Centralization.** Stood up a single centralized Litmus control plane and built the per-tenant subscriber model to hit multi-tenancy without forcing tenants to run their own installs. -- **Phase 4 — Hardening the injection layer.** Replaced ephemeral helper pods - with the DaemonSet HA model after observing scheduling-induced experiment - failures in staging. +* **Phase 4 — Hardening the injection layer.** Replaced ephemeral helper pods with the DaemonSet HA model after observing scheduling-induced experiment failures in staging. -- **Phase 5 — Expressing real scenarios.** Added the Script Runner fault to - unlock dynamic target selection and context chaining (e.g., leader-election - verification for the database team). +* **Phase 5 — Expressing real scenarios.** Added the Script Runner fault to unlock dynamic target selection and context chaining (e.g., leader-election verification for the database team). -- **Phase 6 — Beyond Kubernetes.** Built the Hybrid VM Chaos extension so VM - teams get the same Litmus experience and observability surface as Kubernetes - teams. +* **Phase 6 — Beyond Kubernetes.** Built the Hybrid VM Chaos extension so VM teams get the same Litmus experience and observability surface as Kubernetes teams. -- **Phase 7 — Internalize and contribute back.** Stood up the internal Chaos Hub - for cross-team reuse and began the upstream contribution cadence back to - LitmusChaos. +* **Phase 7 — Internalize and contribute back.** Stood up the internal Chaos Hub for cross-team reuse and began the upstream contribution cadence back to LitmusChaos. -- **Phase 8 — Pre-event validation.** Used the platform to validate operational - readiness ahead of Big Billion Days; chaos drills now feed incident runbooks - directly. +* **Phase 8 — Pre-event validation.** Used the platform to validate operational readiness ahead of Big Billion Days; chaos drills now feed incident runbooks directly. ### Key Lessons -- **Hybrid is the right answer for multi-tenancy at scale.** Neither - cluster-wide nor namespace-wide install models fit a hundreds-of-teams - organization on their own. A centralized control plane plus per-tenant - subscribers is the productive middle. +* **Hybrid is the right answer for multi-tenancy at scale.** Neither cluster-wide nor namespace-wide install models fit a hundreds-of-teams organization on their own. A centralized control plane plus per-tenant subscribers is the productive middle. -- **Chaos infrastructure must be more reliable than the systems it tests.** - Every false negative — a chaos experiment that failed for infrastructure - reasons — erodes trust in the practice. The DaemonSet HA model exists because - that trust was real and worth defending. +* **Chaos infrastructure must be more reliable than the systems it tests.** Every false negative — a chaos experiment that failed for infrastructure reasons — erodes trust in the practice. The DaemonSet HA model exists because that trust was real and worth defending. -- **Failure modes are workflows, not single faults.** Dynamic target selection - and context chaining (Script Runner) were not nice-to-have — they were the - difference between toy chaos and useful chaos. +* **Failure modes are workflows, not single faults.** Dynamic target selection and context chaining (Script Runner) were not nice-to-have — they were the difference between toy chaos and useful chaos. -- **Treat non-Kubernetes workloads as first-class.** Pushing VM chaos through - the same Litmus experience and observability surface kept the cultural and - tooling story unified. +* **Treat non-Kubernetes workloads as first-class.** Pushing VM chaos through the same Litmus experience and observability surface kept the cultural and tooling story unified. -- **Start with central platform teams.** Teams that already run shared - infrastructure (Kubernetes-as-a-Service, service mesh, central databases) had - the runbooks and the motivation to be the first chaos consumers. Application - teams followed. +* **Start with central platform teams.** Teams that already run shared infrastructure (Kubernetes-as-a-Service, service mesh, central databases) had the runbooks and the motivation to be the first chaos consumers. Application teams followed. -- **Documentation and discipline.** Clear hypotheses before the experiment, - clear results after; this is what turns a one-off chaos run into reusable - organizational knowledge. +* **Documentation and discipline.** Clear hypotheses before the experiment, clear results after; this is what turns a one-off chaos run into reusable organizational knowledge. -- **Don’t roll chaos out everywhere at once.** Pilot, learn, then expand. The - internal Chaos Hub exists because the expansion phase needed reusable - artifacts, not from-scratch experiments per team. +* **Don’t roll chaos out everywhere at once.** Pilot, learn, then expand. The internal Chaos Hub exists because the expansion phase needed reusable artifacts, not from-scratch experiments per team. -- **Recovery time matters as much as prevention.** You can’t prevent every - failure, but you can systematically improve how quickly you recover from it — - and chaos drills are the most honest measurement of that. +* **Recovery time matters as much as prevention.** You can’t prevent every failure, but you can systematically improve how quickly you recover from it — and chaos drills are the most honest measurement of that. -- **Contribute upstream.** Local patches become long-term debt; upstream - contributions become permanent improvements for everyone, including Flipkart’s - future self. +* **Contribute upstream.** Local patches become long-term debt; upstream contributions become permanent improvements for everyone, including Flipkart’s future self. ### What's next for your architecture? -- **Production chaos at scale.** Expand selective production chaos runs into a - more regular, more comprehensive practice — with new guardrails for the - injection layer to reduce the privileged surface area. +* **Production chaos at scale.** Expand selective production chaos runs into a more regular, more comprehensive practice — with new guardrails for the injection layer to reduce the privileged surface area. -- **Broader host OS support for VM chaos.** Factor out the OS-specific stress - layer so the hybrid VM chaos extension is not tied to Debian. +* **Broader host OS support for VM chaos.** Factor out the OS-specific stress layer so the hybrid VM chaos extension is not tied to Debian. -- **Deeper integration with incident runbooks.** Close the loop so that every - documented incident generates (or updates) a corresponding chaos scenario. +* **Deeper integration with incident runbooks.** Close the loop so that every documented incident generates (or updates) a corresponding chaos scenario. -- **Resiliency score as a service-level metric.** Promote the Litmus resilience - score from “informational” to a tracked, dashboarded, team-level reliability - KPI. +* **Resiliency score as a service-level metric.** Promote the Litmus resilience score from “informational” to a tracked, dashboarded, team-level reliability KPI. -- **Continued upstream contribution.** Push the hybrid multi-tenancy pattern, - the DaemonSet HA injection model, and the Script Runner fault toward - generalization in upstream LitmusChaos for the broader CNCF community. +* **Continued upstream contribution.** Push the hybrid multi-tenancy pattern, the DaemonSet HA injection model, and the Script Runner fault toward generalization in upstream LitmusChaos for the broader CNCF community. -- **Chaos in CI/CD.** Move from out-of-band chaos drills to chaos gates that run - as part of the deployment pipeline for high-tier services. +* **Chaos in CI/CD.** Move from out-of-band chaos drills to chaos gates that run as part of the deployment pipeline for high-tier services. -- **Observability and AIOps.** Use the rich telemetry produced by repeated chaos - runs to drive AI-assisted blast-radius prediction and pre-event readiness - scoring for events like Big Billion Days. +* **Observability and AIOps.** Use the rich telemetry produced by repeated chaos runs to drive AI-assisted blast-radius prediction and pre-event readiness scoring for events like Big Billion Days. ## Community Contributions -| Contribution | Details | -| :-------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Probe scoping fix** | Made probe-name uniqueness project-scoped instead of globally scoped (MongoDB index fix), enabling true multi-tenant use of probes in LitmusChaos. | -| **Probe name reuse** | Filtration fix that allows probe names to be reused after deletion — removed a stale-record blocker for teams iterating on probe definitions. | -| **Experiment update bug** | Fixed a duplicate-name validation that was incorrectly firing on tag/description-only edits when updating an experiment. | -| **UI consistency fixes** | Corrected the experiment-completion icon; fixed fault-card logos for GCP and Azure faults that were rendering the generic Litmus logo. | -| **Custom image registry** | Fixed long-standing (year-old) issue so the custom image registry setting correctly reflects in the generated workflow YAML — essential for air-gapped and internal-registry deployments. | -| **Pod-block visibility** | Fixed experiment pod-block visibility on the portal; resolved terminology drift between frontend and backend after the “experiment → fault” rename. | -| **Architecture pattern** | Hybrid multi-tenant architecture (centralized operator \+ per-tenant subscriber) recognized by Litmus maintainers as a candidate preferred mode of operation to be documented upstream. | -| **Conference contribution** | Case study presented at KubeCon India keynote (2026), sharing the multi-tenant chaos platform pattern with the broader CNCF community. | +| Contribution | Details | +| :---- | :---- | +| **Probe scoping fix** | Made probe-name uniqueness project-scoped instead of globally scoped (MongoDB index fix), enabling true multi-tenant use of probes in LitmusChaos. | +| **Probe name reuse** | Filtration fix that allows probe names to be reused after deletion — removed a stale-record blocker for teams iterating on probe definitions. | +| **Experiment update bug** | Fixed a duplicate-name validation that was incorrectly firing on tag/description-only edits when updating an experiment. | +| **UI consistency fixes** | Corrected the experiment-completion icon; fixed fault-card logos for GCP and Azure faults that were rendering the generic Litmus logo. | +| **Custom image registry** | Fixed long-standing (year-old) issue so the custom image registry setting correctly reflects in the generated workflow YAML — essential for air-gapped and internal-registry deployments. | +| **Pod-block visibility** | Fixed experiment pod-block visibility on the portal; resolved terminology drift between frontend and backend after the “experiment → fault” rename. | +| **Architecture pattern** | Hybrid multi-tenant architecture (centralized operator \+ per-tenant subscriber) recognized by Litmus maintainers as a candidate preferred mode of operation to be documented upstream. | +| **Conference contribution** | Case study presented at KubeCon India keynote (2026), sharing the multi-tenant chaos platform pattern with the broader CNCF community. | ## Voices from the team -_“We moved from prevention to practice; instead of just trying to prevent -issues, we actively practice handling them. And for incident response, we’ve -gone from panic to procedure.”_ +*“We moved from prevention to practice; instead of just trying to prevent issues, we actively practice handling them. And for incident response, we’ve gone from panic to procedure.”* — Khushi Tiwari, Software Developer, Central Reliability team -_“Implementing Litmus chaos at the kind of scale we operate at, and enabling the -platform teams that directly impact hundreds of development teams — that’s been -a very rewarding journey.”_ +*“Implementing Litmus chaos at the kind of scale we operate at, and enabling the platform teams that directly impact hundreds of development teams — that’s been a very rewarding journey.”* — Aditya, Software Developer, Central Reliability team -_“The scenarios we test become the foundation for our incident runbooks, which -makes the actual incident response much better.”_ +*“The scenarios we test become the foundation for our incident runbooks, which makes the actual incident response much better.”* — Hashith, Site Reliability Engineer ## Discussion -End user members may participate in the -[discussion thread](https://github.com/cncf/tab/discussions/135) for this -architecture. +End user members may participate in the [discussion thread](https://github.com/cncf/tab/discussions/135) for this architecture. diff --git a/docs/architectures/swisscom-cloud-native-telco.md b/docs/architectures/swisscom-cloud-native-telco.md index 8846fd15..1402de68 100644 --- a/docs/architectures/swisscom-cloud-native-telco.md +++ b/docs/architectures/swisscom-cloud-native-telco.md @@ -1,133 +1,80 @@ --- -title: 'End-to-End Cloud Native Telco Platform Automation at Swisscom' -sidebar_label: 'Swisscom (Switzerland) Ltd' +title: "End-to-End Cloud Native Telco Platform Automation at Swisscom" +sidebar_label: "Swisscom (Switzerland) Ltd" --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/swisscom-cloud-native-telco). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/swisscom-cloud-native-telco). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant Projects ### CNCF Projects - + - + - + - + - + - + - + - + - + ### Other Projects - + - + - + ## TL;DR or Synopsis -Swisscom has built a cloud native telco platform for the end-to-end automation -of its 5G standalone core network and cross-domain resource orchestration. The -architecture replaces traditional imperative network management (Jenkins -pipelines, Ansible playbooks) with a fully declarative, Kubernetes-native -automation model driven by GitOps and the Kubernetes Resource Model (KRM). +Swisscom has built a cloud native telco platform for the end-to-end automation of its 5G standalone core network and cross-domain resource orchestration. The architecture replaces traditional imperative network management (Jenkins pipelines, Ansible playbooks) with a fully declarative, Kubernetes-native automation model driven by GitOps and the Kubernetes Resource Model (KRM). -A mobile core development environment contains approximately 2,000 pods with -over 5,000 interdependent configuration parameters across Cloud-Native Network -Functions (CNFs) such as UPF, SMF, AMF, UDM, UDR, BSF, NRF, NSSF, and AUSF. -Engineers express high-level intents as Kubernetes Custom Resources; custom -operators dynamically assemble full configurations at runtime — fetching IP -addresses from IPAM, secrets from Vault, certificates from PKI, and -infrastructure details from the cluster. +A mobile core development environment contains approximately 2,000 pods with over 5,000 interdependent configuration parameters across Cloud-Native Network Functions (CNFs) such as UPF, SMF, AMF, UDM, UDR, BSF, NRF, NSSF, and AUSF. Engineers express high-level intents as Kubernetes Custom Resources; custom operators dynamically assemble full configurations at runtime — fetching IP addresses from IPAM, secrets from Vault, certificates from PKI, and infrastructure details from the cluster. -While the 5G core is the primary domain, the orchestration framework extends -across multiple network domains and infrastructure services, applying the same -intent-based automation patterns consistently. +While the 5G core is the primary domain, the orchestration framework extends across multiple network domains and infrastructure services, applying the same intent-based automation patterns consistently. ## Organisation -Swisscom is the leading Telecommunications/ISP and ICT company and offers -mobile, Internet and TV products, as well as comprehensive IT and digital -services to private and business customers. Swisscom's expertise in cloud native -technologies is well-established, as evidenced by its status as a former Gold -member and Management Board member of the Cloud Foundry Foundation, along with -its certification for Cloud Foundry. Additionally, Swisscom demonstrates a -strong commitment to the Open-Source community, having been a CNCF Silver Member -for several years and serving as a Kubernetes Certified Service Provider (KCSP) -partner. Our skilled employees have delivered numerous talks and presentations -at prestigious events such as KubeCon, Cloud Native Zürich, Swiss Cloud Native -Day, KCD Suisse Romande, ContainerDays. - -The company has embarked on a strategic transformation from a traditional -telecom operator ("Telco") to a technology company ("TechCo") with 5G as a -central driver. Swisscom operates an extensive 5G Non-Standalone (NSA) network -covering 99% of the Swiss population. The cloud native platform described here -powers the 5G Standalone (SA) core. +Swisscom is the leading Telecommunications/ISP and ICT company and offers mobile, Internet and TV products, as well as comprehensive IT and digital services to private and business customers. +Swisscom's expertise in cloud native technologies is well-established, as evidenced by its status as a former Gold member and Management Board member of the Cloud Foundry Foundation, along with its certification for Cloud Foundry. +Additionally, Swisscom demonstrates a strong commitment to the Open-Source community, having been a CNCF Silver Member for several years and serving as a Kubernetes Certified Service Provider (KCSP) partner. +Our skilled employees have delivered numerous talks and presentations at prestigious events such as KubeCon, Cloud Native Zürich, Swiss Cloud Native Day, KCD Suisse Romande, ContainerDays. + +The company has embarked on a strategic transformation from a traditional telecom operator ("Telco") to a technology company ("TechCo") with 5G as a central driver. +Swisscom operates an extensive 5G Non-Standalone (NSA) network covering 99% of the Swiss population. The cloud native platform described here powers the 5G Standalone (SA) core. ## Teams Multiple teams collaborate on this platform: -- **Cloud Native Resource Orchestration** — creates a robust framework for - orchestrating cloud-native resources such as CNFs, IPAM, Networks, DNS, - Kubernetes Clusters, and more. Designs and operates GitOps pipelines, builds - Kubernetes operators, and develops the management cluster UI/UX. -- **Mobile Cloud Native Engineering** — designs, implements, and operates the - cloud native 5G core platform, including GitOps pipelines, Kubernetes - operators, and network function lifecycle management. -- **DNS Engineering** — builds and operates the highly reliable cloud native DNS - service underpinning the 5G core and other infrastructures. +- **Cloud Native Resource Orchestration** — creates a robust framework for orchestrating cloud-native resources such as CNFs, IPAM, Networks, DNS, Kubernetes Clusters, and more. Designs and operates GitOps pipelines, builds Kubernetes operators, and develops the management cluster UI/UX. +- **Mobile Cloud Native Engineering** — designs, implements, and operates the cloud native 5G core platform, including GitOps pipelines, Kubernetes operators, and network function lifecycle management. +- **DNS Engineering** — builds and operates the highly reliable cloud native DNS service underpinning the 5G core and other infrastructures. - **Network Engineering** — provides IPAM and Network-as-a-Service. -- **Platform & Developer Experience** — manages Kubernetes clusters and builds - developer tooling. +- **Platform & Developer Experience** — manages Kubernetes clusters and builds developer tooling. ## Architecture overview & Goals ### Goals -1. **Full GitOps for the 5G Core** — Extend GitOps beyond CNF deployment to - include network function configuration, certificate management, DNS record - provisioning, IP address management, and testing — achieving continuous - reconciliation across all layers. -2. **Declarative, Intent-Based Configuration** — Replace static, low-level - configuration manifests with abstract, intent-driven Custom Resources. - Engineers specify _what_ they want using a high level intent (e.g., "this CNF - needs an IP address from a subnet in network zone A") rather than _how_ to - achieve it, with Kubernetes operators dynamically assembling configurations - at runtime. -3. **Automated CD** — High level of automation for telco deployment rollouts. - This includes rethinking Change Processes as well as building solid CI/CD/CT - pipelines to ensure a highly reliable network. -4. **In-Band with Kubernetes** — Bring all automation in-band with the - Kubernetes API, eliminating out-of-band tools like Jenkins pipelines and - Ansible playbooks. This ensures that the Kubernetes orchestrator has full - visibility and control over all resources, enabling self-healing and - reconciliation. -5. **Cloud Native DNS Service** — Operate a highly reliable, geo-redundant, - on-premises DNS service for the 5G core using open-source technologies - (CoreDNS, PowerDNS, ExternalDNS), fully automated via GitOps and Kubernetes - Custom Resources. -6. **Contribute to the Ecosystem and Shape the Industry Discussion** — - Open-source key components built or contributed to during this journey - (NetBox Operator, SDC, demo code) to enable other organizations to adopt - similar patterns. Contribute to Meetups and Conferences in order to achieve - broader success of Cloud Native adoption in the Telco Community. +1. **Full GitOps for the 5G Core** — Extend GitOps beyond CNF deployment to include network function configuration, certificate management, DNS record provisioning, IP address management, and testing — achieving continuous reconciliation across all layers. +2. **Declarative, Intent-Based Configuration** — Replace static, low-level configuration manifests with abstract, intent-driven Custom Resources. Engineers specify *what* they want using a high level intent (e.g., "this CNF needs an IP address from a subnet in network zone A") rather than *how* to achieve it, with Kubernetes operators dynamically assembling configurations at runtime. +3. **Automated CD** — High level of automation for telco deployment rollouts. This includes rethinking Change Processes as well as building solid CI/CD/CT pipelines to ensure a highly reliable network. +4. **In-Band with Kubernetes** — Bring all automation in-band with the Kubernetes API, eliminating out-of-band tools like Jenkins pipelines and Ansible playbooks. This ensures that the Kubernetes orchestrator has full visibility and control over all resources, enabling self-healing and reconciliation. +5. **Cloud Native DNS Service** — Operate a highly reliable, geo-redundant, on-premises DNS service for the 5G core using open-source technologies (CoreDNS, PowerDNS, ExternalDNS), fully automated via GitOps and Kubernetes Custom Resources. +6. **Contribute to the Ecosystem and Shape the Industry Discussion** — Open-source key components built or contributed to during this journey (NetBox Operator, SDC, demo code) to enable other organizations to adopt similar patterns. Contribute to Meetups and Conferences in order to achieve broader success of Cloud Native adoption in the Telco Community. ### Architecture overview @@ -135,54 +82,29 @@ The platform is organized in three layers: ![Cloud Native Telco Automation Layers](/img/architectures/swisscom-cloud-native-telco/swisscom-cloud-native-telco-automation-layers.svg) -The **Intent Layer** stores high-level desired state in Git — engineers define -_what_ they want using concise Custom Resources (e.g., a DNN configuration with -hostname, region, and SNSSAI). +The **Intent Layer** stores high-level desired state in Git — engineers define *what* they want using concise Custom Resources (e.g., a DNN configuration with hostname, region, and SNSSAI). -The **Automation Layer** runs on Kubernetes and continuously reconciles. Flux -pulls intents from Git. Custom operators dynamically assemble full -configurations by fetching IP addresses from NetBox (via the NetBox Operator) -and creating connectivity in a Network-as-a-Service platform. It then pushes the -KRM formatted Runtime Configuration to a Git repository for the Runtime layer to -consume. +The **Automation Layer** runs on Kubernetes and continuously reconciles. Flux pulls intents from Git. Custom operators dynamically assemble full configurations by fetching IP addresses from NetBox (via the NetBox Operator) and creating connectivity in a Network-as-a-Service platform. It then pushes the KRM formatted Runtime Configuration to a Git repository for the Runtime layer to consume. -The **Runtime Layer** hosts the 5G core CNFs and supporting services. Flux pulls -intents from Git. SDC pushes the assembled configuration to CNFs via -NETCONF/gNMI. MetalLB, cert-manager, External Secrets Operator, ExternalDNS are -used to configure the Workload cluster. +The **Runtime Layer** hosts the 5G core CNFs and supporting services. Flux pulls intents from Git. SDC pushes the assembled configuration to CNFs via NETCONF/gNMI. MetalLB, cert-manager, External Secrets Operator, ExternalDNS are used to configure the Workload cluster. ![Cloud Native Telco Automation Architecture Overview](/img/architectures/swisscom-cloud-native-telco/swisscom-cloud-native-telco-automation-architecture-overview.png) ### Key Design Principles -- **GitOps + KRM**: Git stores high-level intents; Kubernetes manages dynamic, - low-level configuration assembly. This is a shared source of truth across Git - and Kubernetes. -- **Continuous Reconciliation**: Every aspect of the system is continuously - reconciled against the desired state — following the four OpenGitOps - principles. -- **Abstraction over Complexity**: Engineers work with simple, high-level - intents; operators handle the complex assembly. +- **GitOps + KRM**: Git stores high-level intents; Kubernetes manages dynamic, low-level configuration assembly. This is a shared source of truth across Git and Kubernetes. +- **Continuous Reconciliation**: Every aspect of the system is continuously reconciled against the desired state — following the four OpenGitOps principles. +- **Abstraction over Complexity**: Engineers work with simple, high-level intents; operators handle the complex assembly. ## Can you expand on why you are using those projects/services? ### From GitOps to KRM -Standard GitOps tools (Flux, Argo CD) combined with Helm/Kustomize have -limitations for complex telco use cases: they cannot use live Kubernetes -resources as inputs during rendering, cannot invoke custom business logic during -template processing, and cannot dynamically assemble configurations from -multiple sources. By extending GitOps with the Kubernetes Resource Model (KRM) — -Custom Resource Definitions, Custom Resources, and custom Operators — the -platform achieves dynamic configuration assembly at runtime. +Standard GitOps tools (Flux, Argo CD) combined with Helm/Kustomize have limitations for complex telco use cases: they cannot use live Kubernetes resources as inputs during rendering, cannot invoke custom business logic during template processing, and cannot dynamically assemble configurations from multiple sources. By extending GitOps with the Kubernetes Resource Model (KRM) — Custom Resource Definitions, Custom Resources, and custom Operators — the platform achieves dynamic configuration assembly at runtime. ### Configuration Abstraction & Dynamic Assembly -A typical 5G core configuration includes IP addresses, VLAN IDs, DNS records, NF -variables, secret references, and certificate references. Previously, engineers -had to know all values upfront and embed them statically. The new intent-based -model inverts this. The following is an example of a DNN configuration in -pseudo-yaml-code. +A typical 5G core configuration includes IP addresses, VLAN IDs, DNS records, NF variables, secret references, and certificate references. Previously, engineers had to know all values upfront and embed them statically. The new intent-based model inverts this. The following is an example of a DNN configuration in pseudo-yaml-code. On the intent layer, only a very stripped-down KRM manifest exists: @@ -190,202 +112,125 @@ On the intent layer, only a very stripped-down KRM manifest exists: apiVersion: telco.swisscom.com/v1 kind: Dnn spec: - hostname: 'gprs.swisscom.com' + hostname: "gprs.swisscom.com" ipv4: true region: ch-east type: MobileInternet snssai: [1, 10] ``` -This KRM manifest is stored in git and synced to the Management cluster using -Flux. From this, the Operators in the automation layer create intermediate -resources, in this case an IP Address via NetBox: +This KRM manifest is stored in git and synced to the Management cluster using Flux. From this, the Operators in the automation layer create intermediate resources, in this case an IP Address via NetBox: ```yaml apiVersion: netbox.dev/v1 kind: IpAddressClaim spec: parentPrefixSelector: - region: 'ch-east' # from Dnn.spec.region - family: 'IPv4' # from Dnn.spec.ipv4 + region: "ch-east" # from Dnn.spec.region + family: "IPv4" # from Dnn.spec.ipv4 status: ipAddress: 1.2.3.4 ``` -The Automation Layer creates the following low level resources for the Runtime -Layer: +The Automation Layer creates the following low level resources for the Runtime Layer: ```yaml apiVersion: config.sdcio.dev/v1alpha1 kind: Config metadata: labels: - config.sdcio.dev/targetName: ch-east # from Dnn.spec.region + config.sdcio.dev/targetName: ch-east # from Dnn.spec.region config.sdcio.dev/targetNamespace: default spec: priority: 10 config: - - path: / - value: - dnn: - - name: 'gprs.swisscom.com' # from Dnn.spec.hostname - ip: 1.2.3.4 # from IpAddressClaim.status.ipAddress - snssai: [1, 10] # from Dnn.spec.snssai - type: MobileInternet # from Dnn.spec.type + - path: / + value: + dnn: + - name: "gprs.swisscom.com" # from Dnn.spec.hostname + ip: 1.2.3.4 # from IpAddressClaim.status.ipAddress + snssai: [1, 10] # from Dnn.spec.snssai + type: MobileInternet # from Dnn.spec.type --- apiVersion: externaldns.k8s.io/v1alpha1 kind: DNSEndpoint spec: endpoints: - - dnsName: 'gprs.swisscom.com' # from Dnn.spec.hostname + - dnsName: "gprs.swisscom.com" # from Dnn.spec.hostname recordType: A targets: - - 1.2.3.4 # from IpAddressClaim.status.ipAddress + - 1.2.3.4 # from IpAddressClaim.status.ipAddress ``` -SDC will now sync the configuration to the 5G CNF and ExternalDNS will create -the DNS records in the authoritative PowerDNS backend. +SDC will now sync the configuration to the 5G CNF and ExternalDNS will create the DNS records in the authoritative PowerDNS backend. ## What has worked well? -- **Intent-based configuration** dramatically reduced the complexity engineers - face. Instead of managing thousands of interdependent parameters, they work - with concise Custom Resources. -- **Full GitOps reconciliation** across all layers (deployment, configuration, - DNS, certificates, IPAM), configuration drift is detected and reverted to - ensure consistency. -- **Custom Kubernetes Operators** (built with Kubebuilder/controller-runtime) - proved to be the right pattern for telco domain-specific concerns, providing - full reconciliation support and native KRM integration. -- **The claim model for IPAM** (NetBox Operator) elegantly solved dynamic IP - allocation by following established Kubernetes patterns (PVC analogy). -- **Bringing all automation in-band with Kubernetes** gave the orchestrator full - visibility and control, enabling self-healing and eliminating the brittleness - of out-of-band tools. -- **DNS resilience engineering** — dedicated hackathons, chaos testing, and - disaster recovery playbooks significantly improved DNS service reliability. -- **Cross-team collaboration** on a shared platform and KRM patterns accelerated - adoption across multiple network domains. +- **Intent-based configuration** dramatically reduced the complexity engineers face. Instead of managing thousands of interdependent parameters, they work with concise Custom Resources. +- **Full GitOps reconciliation** across all layers (deployment, configuration, DNS, certificates, IPAM), configuration drift is detected and reverted to ensure consistency. +- **Custom Kubernetes Operators** (built with Kubebuilder/controller-runtime) proved to be the right pattern for telco domain-specific concerns, providing full reconciliation support and native KRM integration. +- **The claim model for IPAM** (NetBox Operator) elegantly solved dynamic IP allocation by following established Kubernetes patterns (PVC analogy). +- **Bringing all automation in-band with Kubernetes** gave the orchestrator full visibility and control, enabling self-healing and eliminating the brittleness of out-of-band tools. +- **DNS resilience engineering** — dedicated hackathons, chaos testing, and disaster recovery playbooks significantly improved DNS service reliability. +- **Cross-team collaboration** on a shared platform and KRM patterns accelerated adoption across multiple network domains. ## What has not worked well? -- **NETCONF as a configuration protocol** introduces complexity — it requires - SDC as an intermediary and prevents fully Kubernetes-native configuration. - Ideally, CNF vendors would support native K8s APIs using CRs/CRDs or - Secrets/ConfigMaps. -- **Tooling gap for KRM-based configuration assembly** — no mature, - community-standard Kubernetes-native tool exists for dynamic configuration - hydration. Swisscom had to build custom operators to fill this gap. -- **GitOps+KRM auditability trade-off** — with dynamically assembled - configurations, not all state is visible in Git history. The team continues to - explore automated intermediary Git layers. -- **Cumbersome vendor configuration manifests** — large, monolithic - configuration files from CNF vendors (with ~5,000 interdependent parameters) - required significant effort to decompose into intent-based abstractions. -- **Telco’s imperative model and Kubernetes’ declarative approach do not align - well** - SDC follows the declarative paradigm, where users define the desired - state and SDC determines the actions to achieve it. In contrast, NETCONF/gNMI - use an imperative model that requires explicit ordered steps (“do A, then B, - then C”). Translating declarative goals into imperative sequences is complex - when user‑defined ordering matters, such as for firewall rules where - evaluation order affects behaviour. Example: - [Issue "Support user sorted lists"](https://github.com/sdcio/data-server/issues/394) +- **NETCONF as a configuration protocol** introduces complexity — it requires SDC as an intermediary and prevents fully Kubernetes-native configuration. Ideally, CNF vendors would support native K8s APIs using CRs/CRDs or Secrets/ConfigMaps. +- **Tooling gap for KRM-based configuration assembly** — no mature, community-standard Kubernetes-native tool exists for dynamic configuration hydration. Swisscom had to build custom operators to fill this gap. +- **GitOps+KRM auditability trade-off** — with dynamically assembled configurations, not all state is visible in Git history. The team continues to explore automated intermediary Git layers. +- **Cumbersome vendor configuration manifests** — large, monolithic configuration files from CNF vendors (with ~5,000 interdependent parameters) required significant effort to decompose into intent-based abstractions. +- **Telco’s imperative model and Kubernetes’ declarative approach do not align well** - SDC follows the declarative paradigm, where users define the desired state and SDC determines the actions to achieve it. In contrast, NETCONF/gNMI use an imperative model that requires explicit ordered steps (“do A, then B, then C”). Translating declarative goals into imperative sequences is complex when user‑defined ordering matters, such as for firewall rules where evaluation order affects behaviour. Example: [Issue "Support user sorted lists"](https://github.com/sdcio/data-server/issues/394) ## What sort of "glue" have you had to develop? -- **Custom Kubernetes Operators** — domain-specific operators for CNF - configuration abstraction, config synchronization, IPAM integration, and DNS - automation, all scaffolded with Kubebuilder and controller-runtime. -- **Configuration hydration logic** — the CNF Config Operator dynamically - assembles full configurations from multiple sources (NetBox, Vault, cluster - environment) based on high-level intents. -- **SDC contributions** — - [significant development work on SDC](https://github.com/search?q=org%3Asdcio+author%3Aalexandernorth&type=pullrequests) - including early testing of CNF compatibility, configuration validation, - monitoring, config blame, drift detection, and NETCONF Actions support. -- **ExternalDNS NAPTR support** — contributed - [PR #4212](https://github.com/kubernetes-sigs/external-dns/pull/4212) to - enable NAPTR record support for SIP phone calls. -- **Resilient DNS architecture** — created a reference architecture and open - sourced on - [GitHub](https://github.com/swisscom/cloud-native-telco/tree/main/prototypes/dns). -- **Headlamp plugins** — - [Custom Resource plugin](https://github.com/kubernetes-sigs/headlamp/pulls?q=is%3Apr+author%3Afaebr+). +- **Custom Kubernetes Operators** — domain-specific operators for CNF configuration abstraction, config synchronization, IPAM integration, and DNS automation, all scaffolded with Kubebuilder and controller-runtime. +- **Configuration hydration logic** — the CNF Config Operator dynamically assembles full configurations from multiple sources (NetBox, Vault, cluster environment) based on high-level intents. +- **SDC contributions** — [significant development work on SDC](https://github.com/search?q=org%3Asdcio+author%3Aalexandernorth&type=pullrequests) including early testing of CNF compatibility, configuration validation, monitoring, config blame, drift detection, and NETCONF Actions support. +- **ExternalDNS NAPTR support** — contributed [PR #4212](https://github.com/kubernetes-sigs/external-dns/pull/4212) to enable NAPTR record support for SIP phone calls. +- **Resilient DNS architecture** — created a reference architecture and open sourced on [GitHub](https://github.com/swisscom/cloud-native-telco/tree/main/prototypes/dns). +- **Headlamp plugins** — [Custom Resource plugin](https://github.com/kubernetes-sigs/headlamp/pulls?q=is%3Apr+author%3Afaebr+). ## How did the Architecture Evolve ### Journey -The architecture evolved significantly from traditional imperative automation to -the current declarative, KRM-based model: +The architecture evolved significantly from traditional imperative automation to the current declarative, KRM-based model: -1. **Phase 1 — Ansible + Jenkins**: Initial automation used Ansible playbooks - triggered by Jenkins pipelines. Configuration was fire-and-forget with no - continuous reconciliation. -2. **Phase 2 — GitOps for deployment**: Introduced Flux for CNF deployment, but - configuration remained out-of-band via Ansible/NETCONF. -3. **Phase 3 — Full GitOps + KRM**: Extended GitOps to cover configuration, DNS, - IPAM, certificates, and testing. Built custom operators and adopted SDC for - config synchronization. Achieved continuous reconciliation across all layers. +1. **Phase 1 — Ansible + Jenkins**: Initial automation used Ansible playbooks triggered by Jenkins pipelines. Configuration was fire-and-forget with no continuous reconciliation. +2. **Phase 2 — GitOps for deployment**: Introduced Flux for CNF deployment, but configuration remained out-of-band via Ansible/NETCONF. +3. **Phase 3 — Full GitOps + KRM**: Extended GitOps to cover configuration, DNS, IPAM, certificates, and testing. Built custom operators and adopted SDC for config synchronization. Achieved continuous reconciliation across all layers. ### Key lessons -- **De facto GitOps for operators is not true GitOps** — creating Helm releases - in Git while configuring NFs via NETCONF out-of-band breaks the GitOps model. - Bringing configuration into the Kubernetes API was essential. -- **Bringing everything in-band with Kubernetes** enables self-healing, - reconciliation, and eliminates the brittleness of out-of-band tools. -- **Git as the _only_ source of truth is insufficient** — the shared source of - truth model (Git for intents, Kubernetes for dynamic state) was a deliberate - and necessary evolution. -- **Abstraction is critical** — engineers cannot effectively manage 5,000+ - parameters directly. Intent-based CRs with dynamic assembly significantly - reduced cognitive load and errors. -- **Custom Kubernetes Operators** are the right pattern for domain-specific - concerns that existing tools cannot address. -- **Contribute upstream** — local patches create long-term maintenance burden. - Swisscom prioritizes upstream contributions (ExternalDNS, SDC, NetBox - Operator) for sustainability. +- **De facto GitOps for operators is not true GitOps** — creating Helm releases in Git while configuring NFs via NETCONF out-of-band breaks the GitOps model. Bringing configuration into the Kubernetes API was essential. +- **Bringing everything in-band with Kubernetes** enables self-healing, reconciliation, and eliminates the brittleness of out-of-band tools. +- **Git as the *only* source of truth is insufficient** — the shared source of truth model (Git for intents, Kubernetes for dynamic state) was a deliberate and necessary evolution. +- **Abstraction is critical** — engineers cannot effectively manage 5,000+ parameters directly. Intent-based CRs with dynamic assembly significantly reduced cognitive load and errors. +- **Custom Kubernetes Operators** are the right pattern for domain-specific concerns that existing tools cannot address. +- **Contribute upstream** — local patches create long-term maintenance burden. Swisscom prioritizes upstream contributions (ExternalDNS, SDC, NetBox Operator) for sustainability. ### What's next for your architecture? -- **Mature SDC Integration** — Continue expanding SDC for full lifecycle - management with continuous reconciliation via gNMI and NETCONF, including - completion of NETCONF Actions support. -- **Eliminate NETCONF Dependency** — Work with CNF vendors to move toward fully - Kubernetes-native configuration APIs, reducing reliance on legacy telco - protocols. -- **Advanced Dynamic Configuration Assembly** — Develop more sophisticated - Kubernetes operators for multi-source configuration hydration, enabling even - more complex intent-based workflows across multiple network domains. -- **Multi-Cluster & Edge Expansion** — Scale the architecture to additional edge - locations and Kubernetes clusters while maintaining consistent GitOps-driven - automation. -- **Community Tooling for KRM** — Contribute toward a mature, Kubernetes-native - tool for dynamic configuration assembly that the wider cloud native community - can adopt, addressing the current gap in tooling identified during the - project. -- **Resilience & Reliability** — Ongoing improvements to cross-cluster - redundancy, disaster recovery playbooks, chaos testing framework, and enhanced - monitoring/alerting. -- **Observability & AIOps** — Integrate AI-driven operations capabilities - leveraging the rich telemetry data from the platform's monitoring stack. -- **Cross-Domain Expansion** — Extend the orchestration framework to additional - network domains and infrastructure services beyond the 5G core, applying the - same intent-based automation patterns consistently. +- **Mature SDC Integration** — Continue expanding SDC for full lifecycle management with continuous reconciliation via gNMI and NETCONF, including completion of NETCONF Actions support. +- **Eliminate NETCONF Dependency** — Work with CNF vendors to move toward fully Kubernetes-native configuration APIs, reducing reliance on legacy telco protocols. +- **Advanced Dynamic Configuration Assembly** — Develop more sophisticated Kubernetes operators for multi-source configuration hydration, enabling even more complex intent-based workflows across multiple network domains. +- **Multi-Cluster & Edge Expansion** — Scale the architecture to additional edge locations and Kubernetes clusters while maintaining consistent GitOps-driven automation. +- **Community Tooling for KRM** — Contribute toward a mature, Kubernetes-native tool for dynamic configuration assembly that the wider cloud native community can adopt, addressing the current gap in tooling identified during the project. +- **Resilience & Reliability** — Ongoing improvements to cross-cluster redundancy, disaster recovery playbooks, chaos testing framework, and enhanced monitoring/alerting. +- **Observability & AIOps** — Integrate AI-driven operations capabilities leveraging the rich telemetry data from the platform's monitoring stack. +- **Cross-Domain Expansion** — Extend the orchestration framework to additional network domains and infrastructure services beyond the 5G core, applying the same intent-based automation patterns consistently. ## Community Contributions -| Contribution | Details | -| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **NetBox Operator** | Open-sourced under [https://github.com/netbox-community/netbox-operator](https://github.com/netbox-community/netbox-operator) | -| **SDC Contributions** | Active contributor to the [SDC project](https://docs.sdcio.dev/) (on its path to CNCF incubation) | -| **KRM Demo Code** | [https://github.com/swisscom/containerdays-2024-krm](https://github.com/swisscom/containerdays-2024-krm) | -| **Conference Talks** | [KubeCon EU, ContainerDays, Open Source Summit EU](https://github.com/swisscom/cloud-native-telco/) | +| Contribution | Details | +| ------------ | ------- | +| **NetBox Operator** | Open-sourced under [https://github.com/netbox-community/netbox-operator](https://github.com/netbox-community/netbox-operator) | +| **SDC Contributions** | Active contributor to the [SDC project](https://docs.sdcio.dev/) (on its path to CNCF incubation) | +| **KRM Demo Code** | [https://github.com/swisscom/containerdays-2024-krm](https://github.com/swisscom/containerdays-2024-krm) | +| **Conference Talks** | [KubeCon EU, ContainerDays, Open Source Summit EU](https://github.com/swisscom/cloud-native-telco/) | | **CNCF/LFN Whitepaper** | Co-authored [Accelerating Cloud Native in Telco](https://github.com/lfn-cnti/bestpractices/blob/main/doc/whitepaper/Accelerating_Cloud_Native_in_Telco.md) | ## Discussion -End user members may participate in the -[discussion thread](https://github.com/cncf/tab/discussions/136) for this -architecture. +End user members may participate in the [discussion thread](https://github.com/cncf/tab/discussions/136) for this architecture. diff --git a/docs/architectures/swisscom-kubernetes-service.md b/docs/architectures/swisscom-kubernetes-service.md index 0d783c76..ac0a29dd 100644 --- a/docs/architectures/swisscom-kubernetes-service.md +++ b/docs/architectures/swisscom-kubernetes-service.md @@ -9,23 +9,23 @@ import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; ## Relevant CNCF projects - + - + - + - + - + - + - + - + - + ## Describe your organisation Swisscom is the leading ICT company in Switzerland and offers mobile, Internet and TV, as well as comprehensive IT and digital services to private and business customers. diff --git a/docs/architectures/zeiss.md b/docs/architectures/zeiss.md index 97d2d971..663f4a40 100644 --- a/docs/architectures/zeiss.md +++ b/docs/architectures/zeiss.md @@ -1,47 +1,33 @@ --- -title: 'ZEISS Vision Care - Order Fulfillment' -sidebar_label: 'ZEISS' +title: "ZEISS Vision Care - Order Fulfillment" +sidebar_label: "ZEISS" --- import CNCFProjectCard from '@site/src/components/CNCFProjectCard'; -> Imported from the -> [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss). -> Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is -> distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +> Imported from the [CNCF Cloud Native Reference Architecture project](https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss). Source revision: `3ddf917facd0aea9694cc9c12c52e583fd2f86cb`. Documentation is distributed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). ## Relevant CNCF projects - + 200 microservices supporting order fulfillment processes on managed Kubernetes. Provides the core compute platform for containerized services."} /> - + - + - + - + ## Organization +ZEISS Vision Care produces spectacle lenses, instruments for refraction, and glasses adjustment equipment. Often, lenses are manufactured to an individual consumer's prescription, effectively a batch size of one, which makes order fulfillment a multi-step coordination process. -ZEISS Vision Care produces spectacle lenses, instruments for refraction, and -glasses adjustment equipment. Often, lenses are manufactured to an individual -consumer's prescription, effectively a batch size of one, which makes order -fulfillment a multi-step coordination process. - -Our Business Enablement & IT team develops, designs, and operates the order -fulfillment platform that underpins this process. +Our Business Enablement & IT team develops, designs, and operates the order fulfillment platform that underpins this process. ## Synopsis - -We are reworking the order fulfillment process using a greenfield approach to -modernize core systems. Our goal is a reliable, scalable platform that can -evolve with business needs while remaining cost-efficient. The platform supports -several key capabilities: - +We are reworking the order fulfillment process using a greenfield approach to modernize core systems. Our goal is a reliable, scalable platform that can evolve with business needs while remaining cost-efficient. The platform supports several key capabilities: - Order routing: decide where the order should be produced. -- Order document generation: generate the necessary manufacturing and shipping - documents for the order. +- Order document generation: generate the necessary manufacturing and shipping documents for the order. - Logistics routing: decide how to ship the order to the customer. - Additional auxiliary domains as required. @@ -49,164 +35,73 @@ several key capabilities: ### Goals -The architecture is designed for future-proofing and scale. By leveraging -event-driven scaling with KEDA and decoupled microservices via Dapr, the system -is built to seamlessly absorb high-volume, global order loads as new processes -are continuously migrated to the new platform. +The architecture is designed for future-proofing and scale. By leveraging event-driven scaling with KEDA and decoupled microservices via Dapr, the system is built to seamlessly absorb high-volume, global order loads as new processes are continuously migrated to the new platform. **Key Requirements:** - - Modern cloud-based infrastructure - High reliability and scalability - Maintainable and extensible - Cost-effective operations ### Architecture Overview - ![Architecture](/img/architectures/zeiss/solutionArchitecture.png) #### Data & Storage Strategy - The platform uses Azure-managed data services as backing stores: - - **MSSQL**: Transactional order data and relational schemas - **Cosmos DB**: Scaled state management and cross-region replication - **Azure Blob Storage**: Order documents and manufacturing files - **Redis**: Caching operations to improve latency and throughput -Dapr's state abstraction layer is used for Cosmos DB, Azure Blob Storage, and -Redis, decoupling microservices from those storage backends and enabling -migrations without application-level changes. For Azure Blob Storage, we also -use Dapr's blob binding to interact with data, in addition to the state -abstraction layer. +Dapr's state abstraction layer is used for Cosmos DB, Azure Blob Storage, and Redis, decoupling microservices from those storage backends and enabling migrations without application-level changes. For Azure Blob Storage, we also use Dapr's blob binding to interact with data, in addition to the state abstraction layer. #### Messaging & Asynchronous Communication - -Azure Service Bus serves as the central message broker for asynchronous -processes. Services publish and subscribe to topics via Dapr's pub/sub building -block, enabling loose coupling between applications. This event-driven approach -provides resilience and allows each service to scale independently based on -demand. +Azure Service Bus serves as the central message broker for asynchronous processes. Services publish and subscribe to topics via Dapr's pub/sub building block, enabling loose coupling between applications. This event-driven approach provides resilience and allows each service to scale independently based on demand. #### Network & Service Discovery - -Services use Dapr's service invocation building block instead of hardcoded -endpoints, enabling resilience and the ability to replace or upgrade service -implementations without changing callers. External traffic is routed to services -via Ingress NGINX. +Services use Dapr's service invocation building block instead of hardcoded endpoints, enabling resilience and the ability to replace or upgrade service implementations without changing callers. External traffic is routed to services via Ingress NGINX. #### CI/CD & Deployment - -Azure DevOps Pipelines automate building, testing, and deploying applications. -We use a single, centralized Helm chart and store deployment configuration in -Git as the single source of truth; the pipeline generates environment- and -service-specific `values.yaml` files and deploys each service as its own Helm -release to Kubernetes. +Azure DevOps Pipelines automate building, testing, and deploying applications. We use a single, centralized Helm chart and store deployment configuration in Git as the single source of truth; the pipeline generates environment- and service-specific `values.yaml` files and deploys each service as its own Helm release to Kubernetes. ## Can you expand on why you are using those projects/services? +We rely heavily on CNCF projects and open-source tooling to form the backbone of our platform: -We rely heavily on CNCF projects and open-source tooling to form the backbone of -our platform: - -- **Kubernetes (AKS)** _(Using since 2020)_: Hosts > 200 microservices - supporting order fulfillment workflows. It provides the core compute platform - for containerized services, offering a flexible model that supports multiple - frameworks, programming languages, and dynamic scaling. -- **Dapr** _(Using since 2020)_: Provides common building blocks like service - invocation, pub/sub, and state management across microservices. Dapr enables - vendor-neutral capabilities, addressing service discovery and maintaining - portability. -- **KEDA** _(Using since 2021/2022)_: Event-driven scaling based on Azure - Service Bus queue depth and CPU/memory utilization. This allows the system to - scale aggressively to match real-time demand while running economically during - quieter periods. -- **Helm**: Package management and templating. Charts are stored in Git, - enabling reproducible deployments across environments since the project's - inception. -- **Ingress NGINX** _(currently in use, pending replacement – see Future - Outlook)_: External traffic routing and load balancing. -- **OpenTelemetry Collector** _(Using since 2020)_: Provides consistent - instrumentation and exports telemetry to multiple targets. It enables - distributed tracing and metrics collection across microservices, ensuring - observability and performance monitoring. +- **Kubernetes (AKS)** *(Using since 2020)*: Hosts > 200 microservices supporting order fulfillment workflows. It provides the core compute platform for containerized services, offering a flexible model that supports multiple frameworks, programming languages, and dynamic scaling. +- **Dapr** *(Using since 2020)*: Provides common building blocks like service invocation, pub/sub, and state management across microservices. Dapr enables vendor-neutral capabilities, addressing service discovery and maintaining portability. +- **KEDA** *(Using since 2021/2022)*: Event-driven scaling based on Azure Service Bus queue depth and CPU/memory utilization. This allows the system to scale aggressively to match real-time demand while running economically during quieter periods. +- **Helm**: Package management and templating. Charts are stored in Git, enabling reproducible deployments across environments since the project's inception. +- **Ingress NGINX** *(currently in use, pending replacement – see Future Outlook)*: External traffic routing and load balancing. +- **OpenTelemetry Collector** *(Using since 2020)*: Provides consistent instrumentation and exports telemetry to multiple targets. It enables distributed tracing and metrics collection across microservices, ensuring observability and performance monitoring. ## What has worked well? - Kubernetes, Helm, and KEDA have proven reliable and are widely used. -**Scaling to Zero and Back:** We initially attempted to scale services to 0 -replicas with KEDA to minimize costs, but this introduced operational -challenges: delayed data flows during testing when pods needed to start up, and -frequent scale churn during traffic pauses between messages. We learned that -setting a minimum replica count for baseline load while scaling out for peaks -was much more cost-effective and reliable for our specific workload patterns. - -**The Advantage of Abstraction:** A major architectural risk early on was -building the entire system on Dapr (starting at pre-1.0 release 0.7.0). Early -adoption carried organizational risk and we initially faced instability with -actors under heavy load. Over time, those issues were fully resolved. The -decision proved highly beneficial: Dapr’s maturity and vendor-neutral approach -validated the initial decision, giving the platform extreme portability, -flexibility, and saving custom boilerplate. - -**Platform Evolution:** Dapr is consistently evolving, allowing us to streamline -our platform by replacing specific SDKs with built-in functionalities over time. -For instance, we are transitioning from Azure App Configuration to Dapr's -Configuration Building Block. In hindsight, we would have standardized on CNCF -native components sooner (e.g., migrating earlier to OpenTelemetry Collector and -a CNCF ingress solution) to avoid replacing vendor-specific SDKs mid-project. - -**Data Ownership and Boundaries:** A critical lesson was the strict enforcement -of data ownership. In a microservices architecture with over 200 services, we -learned that clearly defining which service owns which data is non-negotiable. -We found that any direct data access between services, bypassing their dedicated -APIs, inevitably leads to tight coupling and maintenance challenges. Adhering to -Domain-Driven Design (DDD) principles, where each service exposes its data only -through a well-defined API, was essential for maintaining a scalable and -evolvable system. This prevented a "distributed monolith" and ensured long-term -architectural integrity. +**Scaling to Zero and Back:** +We initially attempted to scale services to 0 replicas with KEDA to minimize costs, but this introduced operational challenges: delayed data flows during testing when pods needed to start up, and frequent scale churn during traffic pauses between messages. We learned that setting a minimum replica count for baseline load while scaling out for peaks was much more cost-effective and reliable for our specific workload patterns. -## What sort of "glue" have you had to develop? +**The Advantage of Abstraction:** +A major architectural risk early on was building the entire system on Dapr (starting at pre-1.0 release 0.7.0). Early adoption carried organizational risk and we initially faced instability with actors under heavy load. Over time, those issues were fully resolved. The decision proved highly beneficial: Dapr’s maturity and vendor-neutral approach validated the initial decision, giving the platform extreme portability, flexibility, and saving custom boilerplate. -Since we had a greenfield start, we were not constrained by legacy endpoints or -migration paths. However, managing over 200 microservices required strict -boundaries and standardized communication patterns. We heavily utilized -Domain-Driven Design (DDD) principles—specifically **Bounded Contexts** to -ensure each microservice owns its data and domain logic. +**Platform Evolution:** +Dapr is consistently evolving, allowing us to streamline our platform by replacing specific SDKs with built-in functionalities over time. For instance, we are transitioning from Azure App Configuration to Dapr's Configuration Building Block. In hindsight, we would have standardized on CNCF native components sooner (e.g., migrating earlier to OpenTelemetry Collector and a CNCF ingress solution) to avoid replacing vendor-specific SDKs mid-project. -To enforce these boundaries and standardize the "glue" between services, we -developed: +**Data Ownership and Boundaries:** +A critical lesson was the strict enforcement of data ownership. In a microservices architecture with over 200 services, we learned that clearly defining which service owns which data is non-negotiable. We found that any direct data access between services, bypassing their dedicated APIs, inevitably leads to tight coupling and maintenance challenges. Adhering to Domain-Driven Design (DDD) principles, where each service exposes its data only through a well-defined API, was essential for maintaining a scalable and evolvable system. This prevented a "distributed monolith" and ensured long-term architectural integrity. -- **Standardized Helm Templates**: A unified set of Helm charts that abstract - away the complexity of Kubernetes manifests and Dapr sidecar configuration. - Developers provide application-specific KEDA `ScaledObject` definitions and - Dapr component definitions. -- **Common Libraries**: While Dapr abstracts away many infrastructure concerns, - we built thin, language-specific wrappers around the Dapr SDKs to enforce - internal logging and error-handling standards. +## What sort of "glue" have you had to develop? -## Impact & Results +Since we had a greenfield start, we were not constrained by legacy endpoints or migration paths. However, managing over 200 microservices required strict boundaries and standardized communication patterns. We heavily utilized Domain-Driven Design (DDD) principles—specifically **Bounded Contexts** to ensure each microservice owns its data and domain logic. -By adopting this cloud-native stack, we have built a highly scalable and -future-ready order fulfillment system. A key driver of this success has been our -extensive use of Dapr, which not only significantly reduced boilerplate code but -also provided us with a high degree of vendor neutrality. This abstraction layer -means we are not tightly coupled to specific cloud providers' SDKs, enabling an -architecture that is portable, flexible, and robust enough for long-term growth. +To enforce these boundaries and standardize the "glue" between services, we developed: +- **Standardized Helm Templates**: A unified set of Helm charts that abstract away the complexity of Kubernetes manifests and Dapr sidecar configuration. Developers provide application-specific KEDA `ScaledObject` definitions and Dapr component definitions. +- **Common Libraries**: While Dapr abstracts away many infrastructure concerns, we built thin, language-specific wrappers around the Dapr SDKs to enforce internal logging and error-handling standards. -## What's next for your architecture? +## Impact & Results +By adopting this cloud-native stack, we have built a highly scalable and future-ready order fulfillment system. A key driver of this success has been our extensive use of Dapr, which not only significantly reduced boilerplate code but also provided us with a high degree of vendor neutrality. This abstraction layer means we are not tightly coupled to specific cloud providers' SDKs, enabling an architecture that is portable, flexible, and robust enough for long-term growth. -We are constantly investigating how to run our services as efficiently and -economically as possible. A near-term priority is replacing our existing Ingress -NGINX setup. Because the Kubernetes project has -[officially announced the retirement of the ingress-nginx project](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/), -we are actively evaluating alternatives—including NGINX Gateway Fabric, Envoy -Gateway, and Traefik—while striving to preserve our routing behavior, TLS -automation, and operational stability. Transitioning to the Gateway API is a key -step in future-proofing our external traffic routing. +## What's next for your architecture? +We are constantly investigating how to run our services as efficiently and economically as possible. A near-term priority is replacing our existing Ingress NGINX setup. Because the Kubernetes project has [officially announced the retirement of the ingress-nginx project](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/), we are actively evaluating alternatives—including NGINX Gateway Fabric, Envoy Gateway, and Traefik—while striving to preserve our routing behavior, TLS automation, and operational stability. Transitioning to the Gateway API is a key step in future-proofing our external traffic routing. ## Discussion - -End user members may participate in the -[discussion thread](https://github.com/cncf/tab/discussions/135) for this -architecture. +End user members may participate in the [discussion thread](https://github.com/cncf/tab/discussions/135) for this architecture. From 8cf0bc0045874939bfd7a78ee331e998fc853836 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Mon, 28 Sep 2026 12:02:28 -0500 Subject: [PATCH 2/3] style: format generated architecture JSON with Prettier The automated import (npm run import:architectures) writes data/architectures/catalog.json, data/architectures/records/*.json and data/members.json with JSON.stringify(..., null, 2), which puts one array element per line even when Prettier would collapse the array onto a single line. That mismatch fails the Lint repository check on every automated import PR (this one included) and blocks it in the merge queue. Also exclude every generated docs/architectures/*.md page from Prettier in .prettierignore (previously only 2 of the 8 generated pages were excluded). These pages embed JSX component props (CNCFProjectCard description={"..."}) inline in prose; Prettier's markdown formatter rewraps that prose across lines, which breaks the JS string literal and fails the MDX build with 'Could not parse expression with acorn'. Discovered while attempting to format this PR: running 'npx prettier --write .' on the branch broke the production build for 6 of the imported pages before this exclusion was added. See #795 for the workflow-level fix so future import PRs don't need this by hand. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- .prettierignore | 13 ++- data/architectures/catalog.json | 90 ++++--------------- data/architectures/records/adobe.json | 13 +-- data/architectures/records/allianz.json | 19 +--- .../records/cern-scientific-computing.json | 12 +-- data/architectures/records/colopl.json | 15 +--- .../records/flipkart-chaos-engineering.json | 6 +- .../records/swisscom-cloud-native-telco.json | 5 +- .../records/swisscom-kubernetes-service.json | 7 +- data/architectures/records/zeiss.json | 13 +-- data/members.json | 69 +++----------- 11 files changed, 51 insertions(+), 211 deletions(-) diff --git a/.prettierignore b/.prettierignore index 6da4ecc3..b3f18d63 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,8 +1,13 @@ # Imported architecture pages are regenerated verbatim by -# `npm run import:architectures`; formatting them locally would be overwritten -# (and re-break the gate) on the next import. See issue #528. -docs/architectures/cern-scientific-computing.md -docs/architectures/swisscom-kubernetes-service.md +# `npm run import:architectures` and embed JSX component props (e.g. +# CNCFProjectCard `description={"..."}`) inline in prose. Prettier's markdown +# formatter rewraps that prose across lines, which breaks the JS string +# literal and fails the MDX build with "Could not parse expression with acorn" +# (see #681, #795); any local reformatting would also be overwritten by the +# next import anyway. Excludes every generated org page but keeps the +# hand-authored index. +docs/architectures/*.md +!docs/architectures/index.md # Workflow file: formatting it requires a credential with the Workflows permission: .github/workflows/create-milestones.yml # Blog posts contain MDX JSX comments ({/* truncate */}) that prettier's markdown diff --git a/data/architectures/catalog.json b/data/architectures/catalog.json index 0a8a5c40..024a6bb7 100644 --- a/data/architectures/catalog.json +++ b/data/architectures/catalog.json @@ -4,11 +4,7 @@ "title": "Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture", "organization": "Adobe", "summary": "Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for both infrastructure and developer experience.", - "industries": [ - "Software", - "Digital Media", - "Digital Experience" - ], + "industries": ["Software", "Digital Media", "Digital Experience"], "tags": [ "adobe", "ci-cd", @@ -17,12 +13,7 @@ "developer-platforms", "developer-experience" ], - "projects": [ - "Kubernetes", - "Helm", - "Argo", - "Backstage" - ], + "projects": ["Kubernetes", "Helm", "Argo", "Backstage"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -39,22 +30,9 @@ "title": "Enabling Allianz Direct's scaling through Platform Engineering", "organization": "Allianz Direct", "summary": "The PE team is internally organised in DDOs - Distributed DevOps, where each ddo owns a Domain. This setup follows the reverse Conway manoeuver where the reference architecture structure dictates the teams setup. The PE team consist of 13 m", - "industries": [ - "Financial", - "Insurance" - ], - "tags": [ - "finance", - "insurance", - "europe", - "public_cloud", - "aws" - ], - "projects": [ - "Kubernetes", - "Helm", - "Argo" - ], + "industries": ["Financial", "Insurance"], + "tags": ["finance", "insurance", "europe", "public_cloud", "aws"], + "projects": ["Kubernetes", "Helm", "Argo"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -68,16 +46,8 @@ "title": "A Cloud Native Scientific Computing Platform for CERN NextGen AI", "organization": "CERN", "summary": "Some additional projects outside the CNCF are essential to this deployment. [MLFlow](https://mlflow.org/), a project under the Linux Foundation used for model management and versioning [Nvidia GPU operator](https://github.com/NVIDIA/gpu-ope", - "industries": [ - "Research", - "Public Sector" - ], - "tags": [ - "cloud_native", - "private_cloud", - "kubernetes", - "switzerland" - ], + "industries": ["Research", "Public Sector"], + "tags": ["cloud_native", "private_cloud", "kubernetes", "switzerland"], "projects": [ "Argo", "Bootc", @@ -105,18 +75,9 @@ "title": "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with Agones on Kubernetes", "organization": "Colopl Inc.", "summary": "At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud Spanner as the backend database. While that combination is somewhat distinctive, the overall structure is not far from a typical web service.", - "industries": [ - "gaming" - ], - "tags": [ - "agones", - "gaming", - "kubernetes" - ], - "projects": [ - "Kubernetes", - "Agones" - ], + "industries": ["gaming"], + "tags": ["agones", "gaming", "kubernetes"], + "projects": ["Kubernetes", "Agones"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ @@ -131,11 +92,7 @@ "title": "From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering Platform on LitmusChaos", "organization": "Flipkart Internet Pvt. Ltd.", "summary": "Flipkart is one of India’s leading digital commerce companies, running hundreds of tightly coupled microservices across Kubernetes and VM workloads that must withstand the traffic surges of Big Billion Days and other festive sales. The Cent", - "industries": [ - "E-commerce", - "Retail", - "Digital Commerce" - ], + "industries": ["E-commerce", "Retail", "Digital Commerce"], "tags": [ "Chaos engineering", "Resilience", @@ -166,10 +123,7 @@ "title": "End-to-End Cloud Native Telco Platform Automation at Swisscom", "organization": "Swisscom (Switzerland) Ltd", "summary": "Swisscom has built a cloud native telco platform for the end-to-end automation of its 5G standalone core network and cross-domain resource orchestration. The architecture replaces traditional imperative network management (Jenkins pipelines", - "industries": [ - "Communication", - "Telecommunication" - ], + "industries": ["Communication", "Telecommunication"], "tags": [ "cloud_native", "telco", @@ -206,12 +160,7 @@ "title": "A modern and sovereign Private Cloud «Kubernetes Service» for Swiss-based enterprises.", "organization": "Swisscom (Switzerland) Ltd", "summary": "Our next generation Private Cloud Container as a Service offering «Kubernetes Service» for the B2B market addresses customer’s need for scalable and highly available Kubernetes workload as a flexible and secure IT foundation. It is part of ", - "industries": [ - "Communication", - "Telecommunication", - "Software", - "Cloud" - ], + "industries": ["Communication", "Telecommunication", "Software", "Cloud"], "tags": [ "service-delivery", "cloud_native", @@ -245,10 +194,7 @@ "title": "ZEISS Vision Care - Order Fulfillment", "organization": "ZEISS", "summary": "Our Business Enablement & IT team develops, designs, and operates the order fulfillment platform that underpins this process.", - "industries": [ - "optics", - "optical-manufacturing" - ], + "industries": ["optics", "optical-manufacturing"], "tags": [ "public-cloud", "mass-manufacturing", @@ -256,13 +202,7 @@ "microservices", "platform engineering" ], - "projects": [ - "Kubernetes", - "Dapr", - "KEDA", - "OpenTelemetry", - "Helm" - ], + "projects": ["Kubernetes", "Dapr", "KEDA", "OpenTelemetry", "Helm"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/adobe.json b/data/architectures/records/adobe.json index da4b795e..c0d42fb3 100644 --- a/data/architectures/records/adobe.json +++ b/data/architectures/records/adobe.json @@ -3,11 +3,7 @@ "title": "Scaling Adobe’s Service Delivery Foundation with a Cell-based Architecture", "organization": "Adobe", "summary": "Security, Compliance, Support and Cost Efficiency are cross-cutting concerns for both infrastructure and developer experience.", - "industries": [ - "Software", - "Digital Media", - "Digital Experience" - ], + "industries": ["Software", "Digital Media", "Digital Experience"], "tags": [ "adobe", "ci-cd", @@ -16,12 +12,7 @@ "developer-platforms", "developer-experience" ], - "projects": [ - "Kubernetes", - "Helm", - "Argo", - "Backstage" - ], + "projects": ["Kubernetes", "Helm", "Argo", "Backstage"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/adobe", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/allianz.json b/data/architectures/records/allianz.json index cf03b667..904b6bd2 100644 --- a/data/architectures/records/allianz.json +++ b/data/architectures/records/allianz.json @@ -3,22 +3,9 @@ "title": "Enabling Allianz Direct's scaling through Platform Engineering", "organization": "Allianz Direct", "summary": "The PE team is internally organised in DDOs - Distributed DevOps, where each ddo owns a Domain. This setup follows the reverse Conway manoeuver where the reference architecture structure dictates the teams setup. The PE team consist of 13 m", - "industries": [ - "Financial", - "Insurance" - ], - "tags": [ - "finance", - "insurance", - "europe", - "public_cloud", - "aws" - ], - "projects": [ - "Kubernetes", - "Helm", - "Argo" - ], + "industries": ["Financial", "Insurance"], + "tags": ["finance", "insurance", "europe", "public_cloud", "aws"], + "projects": ["Kubernetes", "Helm", "Argo"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/allianz", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/cern-scientific-computing.json b/data/architectures/records/cern-scientific-computing.json index 0c98478e..a522ed58 100644 --- a/data/architectures/records/cern-scientific-computing.json +++ b/data/architectures/records/cern-scientific-computing.json @@ -3,16 +3,8 @@ "title": "A Cloud Native Scientific Computing Platform for CERN NextGen AI", "organization": "CERN", "summary": "Some additional projects outside the CNCF are essential to this deployment. [MLFlow](https://mlflow.org/), a project under the Linux Foundation used for model management and versioning [Nvidia GPU operator](https://github.com/NVIDIA/gpu-ope", - "industries": [ - "Research", - "Public Sector" - ], - "tags": [ - "cloud_native", - "private_cloud", - "kubernetes", - "switzerland" - ], + "industries": ["Research", "Public Sector"], + "tags": ["cloud_native", "private_cloud", "kubernetes", "switzerland"], "projects": [ "Argo", "Bootc", diff --git a/data/architectures/records/colopl.json b/data/architectures/records/colopl.json index 9cfceddc..65cf424b 100644 --- a/data/architectures/records/colopl.json +++ b/data/architectures/records/colopl.json @@ -3,18 +3,9 @@ "title": "Colopl's FESTIBATTLE Infrastructure Architecture: Dedicated Game Servers with Agones on Kubernetes", "organization": "Colopl Inc.", "summary": "At COLOPL, our API servers run on Google Kubernetes Engine (GKE) with Cloud Spanner as the backend database. While that combination is somewhat distinctive, the overall structure is not far from a typical web service.", - "industries": [ - "gaming" - ], - "tags": [ - "agones", - "gaming", - "kubernetes" - ], - "projects": [ - "Kubernetes", - "Agones" - ], + "industries": ["gaming"], + "tags": ["agones", "gaming", "kubernetes"], + "projects": ["Kubernetes", "Agones"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/colopl", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/architectures/records/flipkart-chaos-engineering.json b/data/architectures/records/flipkart-chaos-engineering.json index cfcb9596..c86490c1 100644 --- a/data/architectures/records/flipkart-chaos-engineering.json +++ b/data/architectures/records/flipkart-chaos-engineering.json @@ -3,11 +3,7 @@ "title": "From Afterthought to Practice: Flipkart’s Multi-Tenant Chaos Engineering Platform on LitmusChaos", "organization": "Flipkart Internet Pvt. Ltd.", "summary": "Flipkart is one of India’s leading digital commerce companies, running hundreds of tightly coupled microservices across Kubernetes and VM workloads that must withstand the traffic surges of Big Billion Days and other festive sales. The Cent", - "industries": [ - "E-commerce", - "Retail", - "Digital Commerce" - ], + "industries": ["E-commerce", "Retail", "Digital Commerce"], "tags": [ "Chaos engineering", "Resilience", diff --git a/data/architectures/records/swisscom-cloud-native-telco.json b/data/architectures/records/swisscom-cloud-native-telco.json index df76a0fb..191cb09a 100644 --- a/data/architectures/records/swisscom-cloud-native-telco.json +++ b/data/architectures/records/swisscom-cloud-native-telco.json @@ -3,10 +3,7 @@ "title": "End-to-End Cloud Native Telco Platform Automation at Swisscom", "organization": "Swisscom (Switzerland) Ltd", "summary": "Swisscom has built a cloud native telco platform for the end-to-end automation of its 5G standalone core network and cross-domain resource orchestration. The architecture replaces traditional imperative network management (Jenkins pipelines", - "industries": [ - "Communication", - "Telecommunication" - ], + "industries": ["Communication", "Telecommunication"], "tags": [ "cloud_native", "telco", diff --git a/data/architectures/records/swisscom-kubernetes-service.json b/data/architectures/records/swisscom-kubernetes-service.json index 11840f89..14affdfb 100644 --- a/data/architectures/records/swisscom-kubernetes-service.json +++ b/data/architectures/records/swisscom-kubernetes-service.json @@ -3,12 +3,7 @@ "title": "A modern and sovereign Private Cloud «Kubernetes Service» for Swiss-based enterprises.", "organization": "Swisscom (Switzerland) Ltd", "summary": "Our next generation Private Cloud Container as a Service offering «Kubernetes Service» for the B2B market addresses customer’s need for scalable and highly available Kubernetes workload as a flexible and secure IT foundation. It is part of ", - "industries": [ - "Communication", - "Telecommunication", - "Software", - "Cloud" - ], + "industries": ["Communication", "Telecommunication", "Software", "Cloud"], "tags": [ "service-delivery", "cloud_native", diff --git a/data/architectures/records/zeiss.json b/data/architectures/records/zeiss.json index 930ecb39..de79e2cd 100644 --- a/data/architectures/records/zeiss.json +++ b/data/architectures/records/zeiss.json @@ -3,10 +3,7 @@ "title": "ZEISS Vision Care - Order Fulfillment", "organization": "ZEISS", "summary": "Our Business Enablement & IT team develops, designs, and operates the order fulfillment platform that underpins this process.", - "industries": [ - "optics", - "optical-manufacturing" - ], + "industries": ["optics", "optical-manufacturing"], "tags": [ "public-cloud", "mass-manufacturing", @@ -14,13 +11,7 @@ "microservices", "platform engineering" ], - "projects": [ - "Kubernetes", - "Dapr", - "KEDA", - "OpenTelemetry", - "Helm" - ], + "projects": ["Kubernetes", "Dapr", "KEDA", "OpenTelemetry", "Helm"], "sourceUrl": "https://github.com/cncf/architecture/tree/3ddf917facd0aea9694cc9c12c52e583fd2f86cb/content/en/architectures/zeiss", "sourceCommit": "3ddf917facd0aea9694cc9c12c52e583fd2f86cb", "assets": [ diff --git a/data/members.json b/data/members.json index 601d1cef..a9e7075d 100644 --- a/data/members.json +++ b/data/members.json @@ -1,9 +1,6 @@ { "description": "CNCF End User Community member organisations derived from reference architectures and Top End User Award winners.", - "generatedFrom": [ - "data/architectures/catalog.json", - "data/awards.json" - ], + "generatedFrom": ["data/architectures/catalog.json", "data/awards.json"], "schema": { "id": "kebab-case identifier", "name": "display name", @@ -21,17 +18,8 @@ "name": "Adobe", "slug": "adobe", "logo": "/img/architectures/adobe/Adobe_Wordmark_RGB_Red.svg", - "industries": [ - "Digital Experience", - "Digital Media", - "Software" - ], - "projects": [ - "Argo", - "Backstage", - "Helm", - "Kubernetes" - ], + "industries": ["Digital Experience", "Digital Media", "Software"], + "projects": ["Argo", "Backstage", "Helm", "Kubernetes"], "architectures": [ { "id": "adobe", @@ -63,15 +51,8 @@ "name": "Allianz", "slug": "allianz", "logo": "/img/architectures/allianz/allianz.svg", - "industries": [ - "Financial", - "Insurance" - ], - "projects": [ - "Argo", - "Helm", - "Kubernetes" - ], + "industries": ["Financial", "Insurance"], + "projects": ["Argo", "Helm", "Kubernetes"], "architectures": [ { "id": "allianz", @@ -162,10 +143,7 @@ "name": "CERN", "slug": "cern", "logo": "/img/architectures/cern-scientific-computing/logo.svg", - "industries": [ - "Public Sector", - "Research" - ], + "industries": ["Public Sector", "Research"], "projects": [ "Argo", "Bootc", @@ -210,13 +188,8 @@ "name": "Colopl Inc.", "slug": "colopl", "logo": "/img/architectures/colopl/logo.png", - "industries": [ - "gaming" - ], - "projects": [ - "Agones", - "Kubernetes" - ], + "industries": ["gaming"], + "projects": ["Agones", "Kubernetes"], "architectures": [ { "id": "colopl", @@ -259,11 +232,7 @@ "name": "Flipkart", "slug": "flipkart", "logo": "/img/architectures/flipkart-chaos-engineering/logo.png", - "industries": [ - "Digital Commerce", - "E-commerce", - "Retail" - ], + "industries": ["Digital Commerce", "E-commerce", "Retail"], "projects": [ "Argo Workflows", "Helm", @@ -464,12 +433,7 @@ "name": "Swisscom", "slug": "swisscom", "logo": "/img/architectures/swisscom-cloud-native-telco/swisscom.png", - "industries": [ - "Cloud", - "Communication", - "Software", - "Telecommunication" - ], + "industries": ["Cloud", "Communication", "Software", "Telecommunication"], "projects": [ "ArgoCD", "CloudNativePG", @@ -541,17 +505,8 @@ "name": "ZEISS", "slug": "zeiss", "logo": "/img/architectures/zeiss/zeiss-logo-rgb.svg", - "industries": [ - "optical-manufacturing", - "optics" - ], - "projects": [ - "Dapr", - "Helm", - "KEDA", - "Kubernetes", - "OpenTelemetry" - ], + "industries": ["optical-manufacturing", "optics"], + "projects": ["Dapr", "Helm", "KEDA", "Kubernetes", "OpenTelemetry"], "architectures": [ { "id": "zeiss", From fb81b349b61e39bc3ced8e8befc219e36b178420 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Mon, 28 Sep 2026 12:10:38 -0500 Subject: [PATCH 3/3] fix: correct 'adpation' typo in imported colopl.md Upstream cncf/architecture content has a typo (adpation -> adaptation) that npm run check:spelling now catches now that the format-check failure earlier in the same job no longer masks it. Fixing here rather than upstream so this automated import PR can merge; cncf/architecture should get the same fix separately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- docs/architectures/colopl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architectures/colopl.md b/docs/architectures/colopl.md index c1907586..573dbafb 100644 --- a/docs/architectures/colopl.md +++ b/docs/architectures/colopl.md @@ -43,7 +43,7 @@ With Counters and Lists, user-defined values (something like a room ID, for exam Going forward, we will explore using this feature to migrate from our in-house Allocator Service to the standard Agones one. -_NOTE: This article is an English translation adpation of [our original article] published in Japanese on Findy Tools._ +_NOTE: This article is an English translation adaptation of [our original article] published in Japanese on Findy Tools._ [Agones]: https://agones.dev/site/ [FESTIBATTLE]: https://festibattle.jp/