From ec2961a1dca4bed9499a6c211c789458dfcc1be9 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Fri, 11 Sep 2026 22:13:30 +0000 Subject: [PATCH 1/8] test(e2e): assert agent telemetry heartbeat per node in testkube A running ama-logs pod does not imply working telemetry: container logs reach the workspace over a local mdsd socket, so they keep flowing even when the agent's outbound telemetry path is entirely broken. Assert the ContainerLogDaemonSetHeartbeatEvent custom event arrives from every node running a DaemonSet pod. - Query the agent telemetry App Insights resource (heartbeats are agent self-telemetry and never land in the customer workspace), matching the cluster resource ID case-insensitively. - Only expect a heartbeat from pods older than the 300s publish interval so a rollout does not fail the assertion. - Generalize AssertContainerLogNodeCoverage into AssertNodeCoverage so both signals share the missing-node reporting. - Plumb AgentTelemetryResourceId through the testkube script and TestWorkflow as AGENT_TELEMETRY_RESOURCE_ID; the spec skips when unset. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../querylogs/querylogs_suite_test.go | 2 + test/ginkgo-e2e/querylogs/querylogs_test.go | 32 ++++++++++ test/ginkgo-e2e/utils/kubernetes_api_utils.go | 27 ++++++++ test/ginkgo-e2e/utils/query_logs_api_utils.go | 62 ++++++++++++++++--- .../install-and-execute-testkube-tests.sh | 3 + test/testkube/testkube-test-crs.yaml | 5 ++ 6 files changed, 124 insertions(+), 7 deletions(-) diff --git a/test/ginkgo-e2e/querylogs/querylogs_suite_test.go b/test/ginkgo-e2e/querylogs/querylogs_suite_test.go index 58b45c1515..715ad9fa59 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_suite_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_suite_test.go @@ -19,6 +19,7 @@ var AKSResourceId string var RetinaNetworkFlowLogsEnabled string var GenevaIntegrationEnabled string var PerNodeLogCoverageEnabled string +var AgentTelemetryResourceId string var Cfg *rest.Config func TestQuerylogs(t *testing.T) { @@ -36,6 +37,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) GenevaIntegrationEnabled = os.Getenv("GENEVA_INTEGRATION") PerNodeLogCoverageEnabled = os.Getenv("PER_NODE_LOG_COVERAGE") + AgentTelemetryResourceId = os.Getenv("AGENT_TELEMETRY_RESOURCE_ID") LogsClient, err = utils.SetupLogsClient() Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/ginkgo-e2e/querylogs/querylogs_test.go b/test/ginkgo-e2e/querylogs/querylogs_test.go index a215559234..fbba3278f0 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_test.go @@ -2,6 +2,7 @@ package querylogs_test import ( "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -9,6 +10,14 @@ import ( "docker-provider/test/utils" ) +const ( + // The Go output plugin publishes telemetry every defaultTelemetryPushIntervalSeconds + // (300s), so a pod younger than this has legitimately not reported a heartbeat yet. + agentTelemetryPublishInterval = 5 * time.Minute + // Span several publish intervals so one delayed batch does not fail the assertion. + agentTelemetryWindow = "20m" +) + var _ = Describe("When querying the logs for the table", func() { DescribeTable("All tables should have logs", func(table string) { @@ -86,3 +95,26 @@ var _ = Describe("When querying the number of resources of the cluster", func() Entry("Nodes", "KubeNodeInventory"), ) }) + +var _ = Describe("When querying the agent telemetry heartbeat", func() { + It("Every node running an ama-logs DaemonSet pod should report a telemetry heartbeat", func() { + if AgentTelemetryResourceId == "" { + Skip("Agent telemetry heartbeat check skipped because AGENT_TELEMETRY_RESOURCE_ID is not set") + } + + // A running agent does not imply working telemetry. Container logs reach the workspace + // over a local mdsd socket, so they keep flowing even when the agent's outbound + // telemetry path is entirely broken. Asserting that the heartbeat actually arrived is + // what distinguishes the two. + expectedNodes, err := utils.GetAgentNodesReadyLongerThan(K8sClient, "kube-system", "component", "ama-logs-agent", agentTelemetryPublishInterval) + Expect(err).NotTo(HaveOccurred()) + if len(expectedNodes) == 0 { + Skip("No ama-logs DaemonSet pod has been running long enough to have published telemetry") + } + + observed, err := utils.GetComputerFromAgentHeartbeat(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow) + Expect(err).NotTo(HaveOccurred()) + + Expect(utils.AssertNodeCoverage("agent telemetry heartbeat", expectedNodes, observed)).NotTo(HaveOccurred()) + }) +}) diff --git a/test/ginkgo-e2e/utils/kubernetes_api_utils.go b/test/ginkgo-e2e/utils/kubernetes_api_utils.go index 69206b4454..ab3fa66cba 100644 --- a/test/ginkgo-e2e/utils/kubernetes_api_utils.go +++ b/test/ginkgo-e2e/utils/kubernetes_api_utils.go @@ -600,3 +600,30 @@ func CheckFileForErrors(clientset *kubernetes.Clientset, Cfg *rest.Config, names return nil } + +// GetAgentNodesReadyLongerThan returns the names of the nodes running a pod with the given +// label that have been up for at least minAge. Younger pods are excluded because the agent +// publishes telemetry on a fixed interval, so a pod that has not yet reached its first publish +// has legitimately reported nothing and would otherwise fail the assertion during a rollout. +func GetAgentNodesReadyLongerThan(clientset *kubernetes.Clientset, namespace, labelName, labelValue string, minAge time.Duration) ([]string, error) { + pods, err := GetPodsWithLabel(clientset, namespace, labelName, labelValue) + if err != nil { + return nil, fmt.Errorf("failed to get pods with label %s=%s: %v", labelName, labelValue, err) + } + if len(pods) == 0 { + return nil, fmt.Errorf("no pods found with label %s=%s", labelName, labelValue) + } + + nodes := []string{} + for _, pod := range pods { + if pod.Spec.NodeName == "" || pod.Status.Phase != corev1.PodRunning { + continue + } + if pod.Status.StartTime == nil || time.Since(pod.Status.StartTime.Time) < minAge { + continue + } + nodes = append(nodes, pod.Spec.NodeName) + } + + return nodes, nil +} diff --git a/test/ginkgo-e2e/utils/query_logs_api_utils.go b/test/ginkgo-e2e/utils/query_logs_api_utils.go index 74b39d9e68..46ddfd7c7c 100644 --- a/test/ginkgo-e2e/utils/query_logs_api_utils.go +++ b/test/ginkgo-e2e/utils/query_logs_api_utils.go @@ -151,7 +151,6 @@ func CompareResourcesInLogsAndKubeAPI(K8sClient *kubernetes.Clientset, logsClien return CompareResourcesHelper(logsClient, resourceID, query, resources) } - func GetComputerFromContainerLog(logsClient *azquery.LogsClient, resourceID string, window string) (map[string]int64, error) { counts, v2Err := queryCountsByComputer(logsClient, resourceID, "ContainerLogV2", window) if v2Err == nil { @@ -189,12 +188,54 @@ func queryCountsByComputer(logsClient *azquery.LogsClient, resourceID string, ta return counts, nil } -// AssertContainerLogNodeCoverage returns nil if every expected node appears -// in the per-Computer count map with a positive row count (compared -// case-insensitively), or an error listing the missing nodes otherwise. -func AssertContainerLogNodeCoverage(expectedNodes []string, observedCountsByComputer map[string]int64) error { +// AgentTelemetryHeartbeatEvent is the App Insights custom event the Go output plugin emits once +// per publish interval from every DaemonSet pod. It is the signal that goes silent when the +// agent's outbound telemetry path breaks while container logs keep flowing over the local mdsd +// socket, so its arrival is asserted directly rather than inferred from the agent's own logs. +const AgentTelemetryHeartbeatEvent = "ContainerLogDaemonSetHeartbeatEvent" + +// GetComputerFromAgentHeartbeat returns the number of agent telemetry heartbeats received per +// node, keyed by lowercased node name. It queries the agent telemetry Application Insights +// resource rather than the cluster's workspace: the heartbeat is agent self-telemetry and is +// never ingested into the customer workspace. +func GetComputerFromAgentHeartbeat(logsClient *azquery.LogsClient, telemetryResourceID string, aksResourceID string, window string) (map[string]int64, error) { + // Resource IDs are matched case-insensitively because the agent reports both + // /resourcegroups/ and /resourceGroups/ spellings for the same cluster. + query := fmt.Sprintf(`customEvents +| where timestamp > ago(%s) +| where name == "%s" +| extend ClusterId = iff(isnotempty(tostring(customDimensions.ID)), tostring(customDimensions.ID), tostring(customDimensions.AKS_RESOURCE_ID)) +| where ClusterId =~ "%s" +| summarize count() by Computer = tostring(customDimensions.Computer)`, + window, AgentTelemetryHeartbeatEvent, aksResourceID) + + tables, err := QueryLogs(logsClient, telemetryResourceID, query) + if err != nil { + return nil, err + } + + counts := map[string]int64{} + for _, t := range tables { + for _, row := range t.Rows { + if len(row) < 2 { + continue + } + computer, ok := row[0].(string) + if !ok || computer == "" { + continue + } + count, _ := row[1].(float64) + counts[strings.ToLower(computer)] += int64(count) + } + } + return counts, nil +} + +// AssertNodeCoverage returns nil if every expected node appears in the per-Computer count map +// with a positive count (compared case-insensitively), or an error listing the missing nodes. +func AssertNodeCoverage(signal string, expectedNodes []string, observedCountsByComputer map[string]int64) error { if len(expectedNodes) == 0 { - return fmt.Errorf("no expected nodes provided; cannot verify ContainerLogV2 coverage") + return fmt.Errorf("no expected nodes provided; cannot verify %s coverage", signal) } var missing []string @@ -204,7 +245,14 @@ func AssertContainerLogNodeCoverage(expectedNodes []string, observedCountsByComp } } if len(missing) > 0 { - return fmt.Errorf("ContainerLogV2 ingestion is missing for %d/%d expected node(s): %s", len(missing), len(expectedNodes), strings.Join(missing, ", ")) + return fmt.Errorf("%s is missing for %d/%d expected node(s): %s", signal, len(missing), len(expectedNodes), strings.Join(missing, ", ")) } return nil } + +// AssertContainerLogNodeCoverage returns nil if every expected node appears +// in the per-Computer count map with a positive row count (compared +// case-insensitively), or an error listing the missing nodes otherwise. +func AssertContainerLogNodeCoverage(expectedNodes []string, observedCountsByComputer map[string]int64) error { + return AssertNodeCoverage("ContainerLogV2", expectedNodes, observedCountsByComputer) +} diff --git a/test/testkube/install-and-execute-testkube-tests.sh b/test/testkube/install-and-execute-testkube-tests.sh index f42a8d7f93..5c056a10c1 100644 --- a/test/testkube/install-and-execute-testkube-tests.sh +++ b/test/testkube/install-and-execute-testkube-tests.sh @@ -12,6 +12,7 @@ do LinuxTestsOnly) LinuxTestsOnly=$VALUE ;; GenevaIntegration) GenevaIntegration=$VALUE ;; PerNodeLogCoverage) PerNodeLogCoverage=$VALUE ;; + AgentTelemetryResourceId) AgentTelemetryResourceId=$VALUE ;; *) esac done @@ -68,6 +69,7 @@ export AZURE_TENANT_ID=$AzureTenantId export WEBHOOK_URI=$TeamsWebhookUri export GENEVA_INTEGRATION=$GenevaIntegration export PER_NODE_LOG_COVERAGE=$PerNodeLogCoverage +export AGENT_TELEMETRY_RESOURCE_ID=$AgentTelemetryResourceId kubectl apply -f ./api-server-permissions.yaml kubectl apply -f ./testkube-test-crs.yaml @@ -91,6 +93,7 @@ for wf in "${workflows[@]}"; do kubectl testkube run testworkflow "$wf" \ --config GENEVA_INTEGRATION="$GENEVA_INTEGRATION" \ --config PER_NODE_LOG_COVERAGE="$PER_NODE_LOG_COVERAGE" \ + --config AGENT_TELEMETRY_RESOURCE_ID="$AGENT_TELEMETRY_RESOURCE_ID" \ --config AZURE_TENANT_ID="$AZURE_TENANT_ID" \ --config AZURE_CLIENT_ID="$AZURE_CLIENT_ID" \ --config GOTOOLCHAIN="auto" \ diff --git a/test/testkube/testkube-test-crs.yaml b/test/testkube/testkube-test-crs.yaml index 54291bec9b..cebf362db0 100644 --- a/test/testkube/testkube-test-crs.yaml +++ b/test/testkube/testkube-test-crs.yaml @@ -133,6 +133,9 @@ spec: PER_NODE_LOG_COVERAGE: type: string default: "false" + AGENT_TELEMETRY_RESOURCE_ID: + type: string + default: "" GOTOOLCHAIN: type: string default: "" @@ -156,6 +159,8 @@ spec: value: "{{config.GENEVA_INTEGRATION}}" - name: PER_NODE_LOG_COVERAGE value: "{{config.PER_NODE_LOG_COVERAGE}}" + - name: AGENT_TELEMETRY_RESOURCE_ID + value: "{{config.AGENT_TELEMETRY_RESOURCE_ID}}" - name: GOTOOLCHAIN value: "{{config.GOTOOLCHAIN}}" shell: ginkgo ./querylogs From 59f2a2b8510342972b09cf647debd692e89fef81 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Fri, 11 Sep 2026 23:48:33 +0000 Subject: [PATCH 2/8] Fix testkube workflow failures being reported as successful runs The results check compared .result.status against "failed", but that status was never what it read. `kubectl testkube get testworkflowexecution --output json` returns an empty file when the execution has just finished, and an empty file satisfies `jq empty`, so it passed validation and .result.status then yielded an empty string. When the file was not empty the status was often still "running", because it is not final the moment `watch` returns. Neither value equals "failed", so failing workflows were recorded as successful and the pipeline job went green: build 126041 reported success while ginkgo reported "FAIL! -- 0 Passed | 1 Failed". Poll for a terminal status rather than reading whichever one happens to be available, and let it decide the outcome only once it is terminal. When it never becomes terminal, fall back to the exit code of `kubectl testkube watch`, which is non-zero for a failed execution and zero for a passing one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6b1dd36a-91a3-4eeb-9ffa-0f89b90b76bd --- .../install-and-execute-testkube-tests.sh | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/test/testkube/install-and-execute-testkube-tests.sh b/test/testkube/install-and-execute-testkube-tests.sh index 5c056a10c1..88ea760ec2 100644 --- a/test/testkube/install-and-execute-testkube-tests.sh +++ b/test/testkube/install-and-execute-testkube-tests.sh @@ -114,22 +114,39 @@ for wf in "${workflows[@]}"; do exit 1 fi - # Watch until the testworkflow finishes + # Watch until the testworkflow finishes. The exit code is the authoritative result: + # the CLI returns non-zero when the execution fails. kubectl testkube watch testworkflowexecution $execution_id + watch_rc=$? + + # Get the results as a formatted json file. + # The execution status is not necessarily final the moment `watch` returns, so poll briefly + # for a terminal one instead of reading a status that is still "running" and mistaking it + # for a result. An empty file satisfies `jq empty`, so the document is also confirmed to be + # an object before any field is read out of it. The poll is kept short because the status is + # only ever corroboration: it can add a failure, never clear one. + wf_status="" + for attempt in $(seq 1 10); do + kubectl testkube get testworkflowexecution $execution_id --output json > "testkube-results-${wf}.json" + if [[ -s "testkube-results-${wf}.json" ]] && jq -e 'type == "object"' "testkube-results-${wf}.json" >/dev/null 2>&1; then + wf_status=$(jq -r '.result.status // empty' "testkube-results-${wf}.json") + fi + case "$wf_status" in + passed|failed|aborted|canceled) break ;; + esac + sleep 1 + done + echo "TestWorkflow $wf finished with exit code $watch_rc and status '${wf_status:-unknown}'" + + # The status only decides the outcome once it is terminal. When it never became terminal, + # or no usable JSON was returned at all, the exit code of `watch` is the only signal left, + # and it is what stops a failing workflow from being reported as a successful one. + status_failed=0 + case "$wf_status" in + failed|aborted|canceled) status_failed=1 ;; + esac - # Get the results as a formatted json file - kubectl testkube get testworkflowexecution $execution_id --output json > "testkube-results-${wf}.json" - - # Verify the JSON is valid - if ! jq empty "testkube-results-${wf}.json" 2>/dev/null; then - echo "Error: Failed to get valid JSON results from testkube for $wf" - echo "Contents of testkube-results-${wf}.json:" - cat "testkube-results-${wf}.json" - exit 1 - fi - - # For any test that has failed, print out the logs - if [[ $(jq -r '.result.status' "testkube-results-${wf}.json") == "failed" ]]; then + if [[ $watch_rc -ne 0 || $status_failed -eq 1 ]]; then echo "TestWorkflow failed. Execution ID: $execution_id" From 641cee36e998eacb8b9abe717aad5a294a96a56e Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Sat, 12 Sep 2026 00:16:06 +0000 Subject: [PATCH 3/8] test(e2e): pin agent telemetry assertions to the deployed image and cover customMetrics The heartbeat assertion accepted any heartbeat for the cluster inside the query window, so telemetry published by the previous image satisfied it and a rollout that never reported would still pass. Resolve the agent version each node is expected to report and assert per node against it, reporting a node that reported nothing separately from one that reported only another version, which is what a stale image or a half-finished rollout looks like. The agent reports that version from AGENT_VERSION, which the build bakes in from its telemetry tag rather than the image tag; the two are equal except on release builds, where TELEMETRY_TAG overrides it. The pipeline therefore passes the telemetry tag explicitly and the image tag on the pod is used only as a fallback. Also assert customEvents and customMetrics per node. The heartbeat only exercises the custom event path, so a break confined to metrics stayed invisible. traces is queried and reported but not required: it carries only agent log lines above Information level, so a healthy agent emits none -- in a sampled 30 minute window just 41,491 of the 107,982 clusters reporting a heartbeat emitted a single trace -- and any trace that does arrive still has to come from the deployed image. Fixes an incorrect format verb that took the address of a string, which blocked go test from running in the utils package. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6b1dd36a-91a3-4eeb-9ffa-0f89b90b76bd --- .pipelines/azure_pipeline_mergedbranches.yaml | 9 ++ .../querylogs/querylogs_suite_test.go | 2 + test/ginkgo-e2e/querylogs/querylogs_test.go | 67 +++++++-- test/ginkgo-e2e/utils/image_tag_test.go | 31 ++++ test/ginkgo-e2e/utils/kubernetes_api_utils.go | 67 ++++++++- test/ginkgo-e2e/utils/query_logs_api_utils.go | 135 ++++++++++++++++-- .../install-and-execute-testkube-tests.sh | 3 + test/testkube/testkube-test-crs.yaml | 5 + 8 files changed, 291 insertions(+), 28 deletions(-) create mode 100644 test/ginkgo-e2e/utils/image_tag_test.go diff --git a/.pipelines/azure_pipeline_mergedbranches.yaml b/.pipelines/azure_pipeline_mergedbranches.yaml index 415c9075f7..7bbf729a2f 100644 --- a/.pipelines/azure_pipeline_mergedbranches.yaml +++ b/.pipelines/azure_pipeline_mergedbranches.yaml @@ -1041,6 +1041,10 @@ extends: # Use image tags built from the previous build stage linuxImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxImagetag']] windowsImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.windowsImageTag']] + # The agent reports this tag, not the image tag, as customDimensions.Version: it is + # baked into the image as AGENT_VERSION and the two differ on release builds, where + # TELEMETRY_TAG overrides it. The telemetry assertions match against it. + linuxTelemetryTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxTelemetryTag']] jobs: # ============================================================ # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Deploy via Helm @@ -1127,6 +1131,11 @@ extends: azureClientId: $(WcusFipsClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) + # Agent self-telemetry is published to the Container Insights agent Application + # Insights resource, never to the cluster's workspace, so it is queried separately. + # Only this cluster's identity is granted Monitoring Reader on that resource, so the + # telemetry assertions stay scoped here and self-skip everywhere else. + additionalTestParams: 'AgentTelemetryResourceId=/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsAgent-Prod/providers/microsoft.insights/components/ContainerInsightsAgent-Prod AgentTelemetryVersion=$(linuxTelemetryTagUnderTest)' # ============================================================ # Cluster: ci-logs-prod-aks-networkflowlogs — Deploy via Helm diff --git a/test/ginkgo-e2e/querylogs/querylogs_suite_test.go b/test/ginkgo-e2e/querylogs/querylogs_suite_test.go index 715ad9fa59..a308ebaaf0 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_suite_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_suite_test.go @@ -20,6 +20,7 @@ var RetinaNetworkFlowLogsEnabled string var GenevaIntegrationEnabled string var PerNodeLogCoverageEnabled string var AgentTelemetryResourceId string +var AgentTelemetryVersion string var Cfg *rest.Config func TestQuerylogs(t *testing.T) { @@ -38,6 +39,7 @@ var _ = BeforeSuite(func() { GenevaIntegrationEnabled = os.Getenv("GENEVA_INTEGRATION") PerNodeLogCoverageEnabled = os.Getenv("PER_NODE_LOG_COVERAGE") AgentTelemetryResourceId = os.Getenv("AGENT_TELEMETRY_RESOURCE_ID") + AgentTelemetryVersion = os.Getenv("AGENT_TELEMETRY_VERSION") LogsClient, err = utils.SetupLogsClient() Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/ginkgo-e2e/querylogs/querylogs_test.go b/test/ginkgo-e2e/querylogs/querylogs_test.go index fbba3278f0..e7f57e2f86 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_test.go @@ -1,6 +1,7 @@ package querylogs_test import ( + "fmt" "strings" "time" @@ -96,25 +97,73 @@ var _ = Describe("When querying the number of resources of the cluster", func() ) }) -var _ = Describe("When querying the agent telemetry heartbeat", func() { - It("Every node running an ama-logs DaemonSet pod should report a telemetry heartbeat", func() { +var _ = Describe("When querying the agent telemetry", func() { + // expectedVersionByNode maps each node to the image tag its agent is running, which the + // agent reports back as customDimensions.Version. Resolving it per node is what makes these + // assertions verify the image the deploy stage just rolled out, rather than accepting + // telemetry that its predecessor published earlier in the same query window. + var expectedVersionByNode map[string]string + + BeforeEach(func() { if AgentTelemetryResourceId == "" { - Skip("Agent telemetry heartbeat check skipped because AGENT_TELEMETRY_RESOURCE_ID is not set") + Skip("Agent telemetry checks skipped because AGENT_TELEMETRY_RESOURCE_ID is not set") + } + + var err error + expectedVersionByNode, err = utils.GetAgentImageTagsByNode(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs", agentTelemetryPublishInterval) + Expect(err).NotTo(HaveOccurred()) + if len(expectedVersionByNode) == 0 { + Skip("No ama-logs DaemonSet pod has been running long enough to have published telemetry") } + // The build bakes AGENT_VERSION in from its own telemetry tag rather than from the + // image tag, and the two diverge on release builds, where TELEMETRY_TAG overrides it. + // When the pipeline passes that tag it is authoritative, so prefer it over the tag + // read off the pod. + if AgentTelemetryVersion != "" { + for node := range expectedVersionByNode { + expectedVersionByNode[node] = AgentTelemetryVersion + } + } + }) + + It("Every node running an ama-logs DaemonSet pod should report a telemetry heartbeat from the deployed image", func() { // A running agent does not imply working telemetry. Container logs reach the workspace // over a local mdsd socket, so they keep flowing even when the agent's outbound // telemetry path is entirely broken. Asserting that the heartbeat actually arrived is // what distinguishes the two. - expectedNodes, err := utils.GetAgentNodesReadyLongerThan(K8sClient, "kube-system", "component", "ama-logs-agent", agentTelemetryPublishInterval) + observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, "customEvents", utils.AgentTelemetryHeartbeatEvent) Expect(err).NotTo(HaveOccurred()) - if len(expectedNodes) == 0 { - Skip("No ama-logs DaemonSet pod has been running long enough to have published telemetry") - } - observed, err := utils.GetComputerFromAgentHeartbeat(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow) + Expect(utils.AssertNodeVersionCoverage("agent telemetry heartbeat", expectedVersionByNode, observed)).NotTo(HaveOccurred()) + }) + + DescribeTable("Every node should publish agent telemetry from the deployed image to the table", + func(table string) { + // The heartbeat only proves the custom event path works. Metrics travel the same + // outbound connection but through a different SDK track call, so a break confined + // to one of them stays invisible until each table is asserted on its own. + observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, table, "") + Expect(err).NotTo(HaveOccurred()) + + Expect(utils.AssertNodeVersionCoverage(table, expectedVersionByNode, observed)).NotTo(HaveOccurred()) + }, + Entry("customEvents", "customEvents"), + Entry("customMetrics", "customMetrics"), + ) + + It("Agent log traces that arrive should come from the deployed image", func() { + // traces carries the agent's own log lines, and only those that are not "Information" + // level, so a healthy agent emits none: in a sampled 30 minute window only 41,491 of + // the 107,982 clusters reporting a heartbeat produced a single trace. Requiring traces + // would fail the majority of healthy clusters, so the query still has to succeed and + // any trace that does arrive still has to come from the deployed image, but an empty + // result is reported rather than failed. + observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, "traces", "") Expect(err).NotTo(HaveOccurred()) - Expect(utils.AssertNodeCoverage("agent telemetry heartbeat", expectedNodes, observed)).NotTo(HaveOccurred()) + AddReportEntry("agent log traces", fmt.Sprintf("%d trace(s) from %d/%d node(s) in the last %s", utils.TotalItems(observed), len(observed), len(expectedVersionByNode), agentTelemetryWindow)) + + Expect(utils.AssertReportedNodeVersions("agent log traces", expectedVersionByNode, observed)).NotTo(HaveOccurred()) }) }) diff --git a/test/ginkgo-e2e/utils/image_tag_test.go b/test/ginkgo-e2e/utils/image_tag_test.go new file mode 100644 index 0000000000..ebfbe04d66 --- /dev/null +++ b/test/ginkgo-e2e/utils/image_tag_test.go @@ -0,0 +1,31 @@ +package utils + +import "testing" + +func TestImageTag(t *testing.T) { + cases := []struct { + name string + imageRef string + want string + }{ + // The tag the agent reports back as customDimensions.Version. + {"linux agent", "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:3.8.0-ci-prod-09-06-2026-fd42f68c", "3.8.0-ci-prod-09-06-2026-fd42f68c"}, + {"windows agent", "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-3.8.0-ci-prod-09-06-2026-fd42f68c", "win-3.8.0-ci-prod-09-06-2026-fd42f68c"}, + // A digest is stripped first, so a reference pinning both still yields its tag. + {"tag and digest", "mcr.microsoft.com/geneva/mdsd:recommended@sha256:abc", "recommended"}, + // Nothing to match the reported version against. + {"digest only", "mcr.microsoft.com/oss/v2/calico/node@sha256:69124ac", ""}, + {"bare reference", "mcr.microsoft.com/oss/v2/kubernetes/pause", ""}, + // The port of a registry host must never be mistaken for a tag. + {"registry port, no tag", "localhost:5000/ciprod", ""}, + {"registry port and tag", "localhost:5000/ciprod:3.8.0", "3.8.0"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ImageTag(tc.imageRef); got != tc.want { + t.Fatalf("ImageTag(%q) = %q, want %q", tc.imageRef, got, tc.want) + } + }) + } +} diff --git a/test/ginkgo-e2e/utils/kubernetes_api_utils.go b/test/ginkgo-e2e/utils/kubernetes_api_utils.go index ab3fa66cba..fa193f3a0d 100644 --- a/test/ginkgo-e2e/utils/kubernetes_api_utils.go +++ b/test/ginkgo-e2e/utils/kubernetes_api_utils.go @@ -141,7 +141,7 @@ func GetContainerEnvVars(clientset *kubernetes.Clientset, namespace string, labe } } - return nil, fmt.Errorf("container %s not found in pod %s", containerName, &pods[0].Name) + return nil, fmt.Errorf("container %s not found in pod %s", containerName, pods[0].Name) } func GetAKSResourceID(clientset *kubernetes.Clientset, namespace string, labelKey string, labelValue string, containerName string) (string, error) { @@ -616,10 +616,7 @@ func GetAgentNodesReadyLongerThan(clientset *kubernetes.Clientset, namespace, la nodes := []string{} for _, pod := range pods { - if pod.Spec.NodeName == "" || pod.Status.Phase != corev1.PodRunning { - continue - } - if pod.Status.StartTime == nil || time.Since(pod.Status.StartTime.Time) < minAge { + if !podPublishedAtLeastOnce(pod, minAge) { continue } nodes = append(nodes, pod.Spec.NodeName) @@ -627,3 +624,63 @@ func GetAgentNodesReadyLongerThan(clientset *kubernetes.Clientset, namespace, la return nodes, nil } + +// podPublishedAtLeastOnce reports whether a pod is running on a node and has been up long +// enough to have reached at least one telemetry publish interval. +func podPublishedAtLeastOnce(pod corev1.Pod, minAge time.Duration) bool { + if pod.Spec.NodeName == "" || pod.Status.Phase != corev1.PodRunning { + return false + } + return pod.Status.StartTime != nil && time.Since(pod.Status.StartTime.Time) >= minAge +} + +// ImageTag returns the tag of a container image reference, or an empty string when the +// reference carries no tag. Any digest is stripped first so a reference that pins both a tag +// and a digest still yields its tag, and the tag is only looked for after the final "/" so +// that the port in a registry host is never mistaken for one. +func ImageTag(imageRef string) string { + if at := strings.Index(imageRef, "@"); at >= 0 { + imageRef = imageRef[:at] + } + colon := strings.LastIndex(imageRef, ":") + if colon <= strings.LastIndex(imageRef, "/") { + return "" + } + return imageRef[colon+1:] +} + +// GetAgentImageTagsByNode returns the image tag the agent container is running on each node, +// keyed by lowercased node name, for pods that have been up for at least minAge. +// +// The image build bakes this tag into the image as AGENT_VERSION (`ENV AGENT_VERSION +// ${IMAGE_TAG}`) and the agent reports it as customDimensions.Version on every telemetry item. +// Comparing the two is what ties an assertion to the image currently deployed, rather than to +// any agent that happens to have reported for the cluster inside the query window. +func GetAgentImageTagsByNode(clientset *kubernetes.Clientset, namespace, labelName, labelValue, containerName string, minAge time.Duration) (map[string]string, error) { + pods, err := GetPodsWithLabel(clientset, namespace, labelName, labelValue) + if err != nil { + return nil, fmt.Errorf("failed to get pods with label %s=%s: %v", labelName, labelValue, err) + } + if len(pods) == 0 { + return nil, fmt.Errorf("no pods found with label %s=%s", labelName, labelValue) + } + + tagsByNode := map[string]string{} + for _, pod := range pods { + if !podPublishedAtLeastOnce(pod, minAge) { + continue + } + for _, container := range pod.Spec.Containers { + if container.Name != containerName { + continue + } + tag := ImageTag(container.Image) + if tag == "" { + return nil, fmt.Errorf("container %s in pod %s runs image %q, which carries no tag to match the reported agent version against", containerName, pod.Name, container.Image) + } + tagsByNode[strings.ToLower(pod.Spec.NodeName)] = tag + } + } + + return tagsByNode, nil +} diff --git a/test/ginkgo-e2e/utils/query_logs_api_utils.go b/test/ginkgo-e2e/utils/query_logs_api_utils.go index 46ddfd7c7c..3c4785d9a6 100644 --- a/test/ginkgo-e2e/utils/query_logs_api_utils.go +++ b/test/ginkgo-e2e/utils/query_logs_api_utils.go @@ -2,7 +2,9 @@ package utils import ( "context" + "errors" "fmt" + "sort" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -194,43 +196,148 @@ func queryCountsByComputer(logsClient *azquery.LogsClient, resourceID string, ta // socket, so its arrival is asserted directly rather than inferred from the agent's own logs. const AgentTelemetryHeartbeatEvent = "ContainerLogDaemonSetHeartbeatEvent" -// GetComputerFromAgentHeartbeat returns the number of agent telemetry heartbeats received per -// node, keyed by lowercased node name. It queries the agent telemetry Application Insights -// resource rather than the cluster's workspace: the heartbeat is agent self-telemetry and is -// never ingested into the customer workspace. -func GetComputerFromAgentHeartbeat(logsClient *azquery.LogsClient, telemetryResourceID string, aksResourceID string, window string) (map[string]int64, error) { +// AgentTelemetryPeriodicTables are the Application Insights tables the agent writes to on a +// fixed interval, so every node running a healthy agent has to appear in all of them. +// +// `traces` is deliberately not in this list. It carries the agent's own log lines, and only +// those that are not "Information" level, so an agent with nothing to complain about emits +// none at all: in a sampled 30 minute window only 41,491 of the 107,982 clusters that reported +// a heartbeat produced a single trace. Requiring traces would therefore fail the majority of +// healthy clusters, so their arrival is reported rather than asserted. +var AgentTelemetryPeriodicTables = []string{"customEvents", "customMetrics"} + +// GetAgentTelemetryVersionsByNode returns the telemetry item count for one Application Insights +// table, keyed by lowercased node name and then by the agent version that reported it. It +// queries the agent telemetry Application Insights resource rather than the cluster's +// workspace: this is agent self-telemetry and is never ingested into the customer workspace. +// +// Keeping the version in the result is what allows a caller to tell a node that is reporting +// from the image under test apart from one that is only still reporting from its predecessor. +// Pass an empty eventName for tables such as `traces` that have no name column. +func GetAgentTelemetryVersionsByNode(logsClient *azquery.LogsClient, telemetryResourceID string, aksResourceID string, window string, table string, eventName string) (map[string]map[string]int64, error) { + nameFilter := "" + if eventName != "" { + nameFilter = fmt.Sprintf("\n| where name == \"%s\"", eventName) + } + // Resource IDs are matched case-insensitively because the agent reports both // /resourcegroups/ and /resourceGroups/ spellings for the same cluster. - query := fmt.Sprintf(`customEvents -| where timestamp > ago(%s) -| where name == "%s" + query := fmt.Sprintf(`%s +| where timestamp > ago(%s)%s | extend ClusterId = iff(isnotempty(tostring(customDimensions.ID)), tostring(customDimensions.ID), tostring(customDimensions.AKS_RESOURCE_ID)) | where ClusterId =~ "%s" -| summarize count() by Computer = tostring(customDimensions.Computer)`, - window, AgentTelemetryHeartbeatEvent, aksResourceID) +| summarize count() by Computer = tostring(customDimensions.Computer), Version = tostring(customDimensions.Version)`, + table, window, nameFilter, aksResourceID) tables, err := QueryLogs(logsClient, telemetryResourceID, query) if err != nil { return nil, err } - counts := map[string]int64{} + counts := map[string]map[string]int64{} for _, t := range tables { for _, row := range t.Rows { - if len(row) < 2 { + if len(row) < 3 { continue } computer, ok := row[0].(string) if !ok || computer == "" { continue } - count, _ := row[1].(float64) - counts[strings.ToLower(computer)] += int64(count) + version, _ := row[1].(string) + count, _ := row[2].(float64) + node := strings.ToLower(computer) + if counts[node] == nil { + counts[node] = map[string]int64{} + } + counts[node][version] += int64(count) } } return counts, nil } +// AssertNodeVersionCoverage returns nil when every node in expectedVersionByNode reported the +// signal under the agent version that node is actually running. +// +// A node that reported nothing and a node that reported only some other version are listed +// separately, because they are different faults: the first is a node whose telemetry is not +// arriving at all, while the second is telemetry arriving from an agent that is no longer the +// one deployed, which is what a stale image or a half-finished rollout looks like. +func AssertNodeVersionCoverage(signal string, expectedVersionByNode map[string]string, observed map[string]map[string]int64) error { + if len(expectedVersionByNode) == 0 { + return fmt.Errorf("no expected nodes provided; cannot verify %s coverage", signal) + } + + var missing, staleVersion []string + for node, expectedVersion := range expectedVersionByNode { + reported := observed[strings.ToLower(node)] + if len(reported) == 0 { + missing = append(missing, node) + continue + } + if reported[expectedVersion] <= 0 { + staleVersion = append(staleVersion, fmt.Sprintf("%s (expected %q, reported %s)", node, expectedVersion, strings.Join(sortedVersions(reported), ", "))) + } + } + + sort.Strings(missing) + sort.Strings(staleVersion) + + var problems []string + if len(missing) > 0 { + problems = append(problems, fmt.Sprintf("%s is missing for %d/%d expected node(s): %s", signal, len(missing), len(expectedVersionByNode), strings.Join(missing, ", "))) + } + if len(staleVersion) > 0 { + problems = append(problems, fmt.Sprintf("%s arrived for %d node(s) but not from the deployed agent version: %s", signal, len(staleVersion), strings.Join(staleVersion, "; "))) + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// AssertReportedNodeVersions returns nil when every node that did report the signal reported it +// under the version that node is running. Nodes that reported nothing are ignored, which makes +// it the right check for a signal whose absence is legitimate. +func AssertReportedNodeVersions(signal string, expectedVersionByNode map[string]string, observed map[string]map[string]int64) error { + reporting := map[string]string{} + for node, expectedVersion := range expectedVersionByNode { + if len(observed[strings.ToLower(node)]) > 0 { + reporting[node] = expectedVersion + } + } + if len(reporting) == 0 { + return nil + } + return AssertNodeVersionCoverage(signal, reporting, observed) +} + +// sortedVersions returns the versions in a per-version count map in a stable order so that +// failure messages do not change between runs for the same underlying data. +func sortedVersions(counts map[string]int64) []string { + versions := make([]string, 0, len(counts)) + for version := range counts { + if version == "" { + version = "" + } + versions = append(versions, version) + } + sort.Strings(versions) + return versions +} + +// TotalItems returns the number of telemetry items across every node and version in a result +// from GetAgentTelemetryVersionsByNode. +func TotalItems(observed map[string]map[string]int64) int64 { + var total int64 + for _, byVersion := range observed { + for _, count := range byVersion { + total += count + } + } + return total +} + // AssertNodeCoverage returns nil if every expected node appears in the per-Computer count map // with a positive count (compared case-insensitively), or an error listing the missing nodes. func AssertNodeCoverage(signal string, expectedNodes []string, observedCountsByComputer map[string]int64) error { diff --git a/test/testkube/install-and-execute-testkube-tests.sh b/test/testkube/install-and-execute-testkube-tests.sh index 88ea760ec2..4a38acc026 100644 --- a/test/testkube/install-and-execute-testkube-tests.sh +++ b/test/testkube/install-and-execute-testkube-tests.sh @@ -13,6 +13,7 @@ do GenevaIntegration) GenevaIntegration=$VALUE ;; PerNodeLogCoverage) PerNodeLogCoverage=$VALUE ;; AgentTelemetryResourceId) AgentTelemetryResourceId=$VALUE ;; + AgentTelemetryVersion) AgentTelemetryVersion=$VALUE ;; *) esac done @@ -70,6 +71,7 @@ export WEBHOOK_URI=$TeamsWebhookUri export GENEVA_INTEGRATION=$GenevaIntegration export PER_NODE_LOG_COVERAGE=$PerNodeLogCoverage export AGENT_TELEMETRY_RESOURCE_ID=$AgentTelemetryResourceId +export AGENT_TELEMETRY_VERSION=$AgentTelemetryVersion kubectl apply -f ./api-server-permissions.yaml kubectl apply -f ./testkube-test-crs.yaml @@ -94,6 +96,7 @@ for wf in "${workflows[@]}"; do --config GENEVA_INTEGRATION="$GENEVA_INTEGRATION" \ --config PER_NODE_LOG_COVERAGE="$PER_NODE_LOG_COVERAGE" \ --config AGENT_TELEMETRY_RESOURCE_ID="$AGENT_TELEMETRY_RESOURCE_ID" \ + --config AGENT_TELEMETRY_VERSION="$AGENT_TELEMETRY_VERSION" \ --config AZURE_TENANT_ID="$AZURE_TENANT_ID" \ --config AZURE_CLIENT_ID="$AZURE_CLIENT_ID" \ --config GOTOOLCHAIN="auto" \ diff --git a/test/testkube/testkube-test-crs.yaml b/test/testkube/testkube-test-crs.yaml index cebf362db0..674ef7797e 100644 --- a/test/testkube/testkube-test-crs.yaml +++ b/test/testkube/testkube-test-crs.yaml @@ -136,6 +136,9 @@ spec: AGENT_TELEMETRY_RESOURCE_ID: type: string default: "" + AGENT_TELEMETRY_VERSION: + type: string + default: "" GOTOOLCHAIN: type: string default: "" @@ -161,6 +164,8 @@ spec: value: "{{config.PER_NODE_LOG_COVERAGE}}" - name: AGENT_TELEMETRY_RESOURCE_ID value: "{{config.AGENT_TELEMETRY_RESOURCE_ID}}" + - name: AGENT_TELEMETRY_VERSION + value: "{{config.AGENT_TELEMETRY_VERSION}}" - name: GOTOOLCHAIN value: "{{config.GOTOOLCHAIN}}" shell: ginkgo ./querylogs From 94ca1d3b585b0c41ab36d112a7a89940a09f77a4 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Tue, 22 Sep 2026 23:32:08 +0000 Subject: [PATCH 4/8] simplify tests --- .pipelines/azure_pipeline_mergedbranches.yaml | 18 +-- test/ginkgo-e2e/querylogs/querylogs_test.go | 86 ++-------- test/ginkgo-e2e/utils/image_tag_test.go | 31 ---- test/ginkgo-e2e/utils/kubernetes_api_utils.go | 84 ---------- test/ginkgo-e2e/utils/query_logs_api_utils.go | 150 ------------------ 5 files changed, 22 insertions(+), 347 deletions(-) delete mode 100644 test/ginkgo-e2e/utils/image_tag_test.go diff --git a/.pipelines/azure_pipeline_mergedbranches.yaml b/.pipelines/azure_pipeline_mergedbranches.yaml index 7bbf729a2f..362d96e5ff 100644 --- a/.pipelines/azure_pipeline_mergedbranches.yaml +++ b/.pipelines/azure_pipeline_mergedbranches.yaml @@ -1041,10 +1041,6 @@ extends: # Use image tags built from the previous build stage linuxImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxImagetag']] windowsImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.windowsImageTag']] - # The agent reports this tag, not the image tag, as customDimensions.Version: it is - # baked into the image as AGENT_VERSION and the two differ on release builds, where - # TELEMETRY_TAG overrides it. The telemetry assertions match against it. - linuxTelemetryTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxTelemetryTag']] jobs: # ============================================================ # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Deploy via Helm @@ -1073,7 +1069,7 @@ extends: azureClientId: $(AksGenevaIntegrationMultiTenancyClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) - additionalTestParams: 'GenevaIntegration=true' + additionalTestParams: 'GenevaIntegration=true AgentTelemetryResourceId=$(AGENT_TELEMETRY_RESOURCE_ID) AgentTelemetryVersion=$(linuxImageTagUnderTest)' # ============================================================ # Cluster: ci-logs-prod-aks-work-load-identity — Deploy via Helm @@ -1102,7 +1098,7 @@ extends: azureClientId: $(AksWorkLoadIdentityClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) - additionalTestParams: 'LinuxTestsOnly=true' + additionalTestParams: 'LinuxTestsOnly=true AgentTelemetryResourceId=$(AGENT_TELEMETRY_RESOURCE_ID) AgentTelemetryVersion=$(linuxImageTagUnderTest)' # ============================================================ # Cluster: ci-logs-prod-wcus-fips — Deploy via Helm @@ -1131,11 +1127,7 @@ extends: azureClientId: $(WcusFipsClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) - # Agent self-telemetry is published to the Container Insights agent Application - # Insights resource, never to the cluster's workspace, so it is queried separately. - # Only this cluster's identity is granted Monitoring Reader on that resource, so the - # telemetry assertions stay scoped here and self-skip everywhere else. - additionalTestParams: 'AgentTelemetryResourceId=/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsAgent-Prod/providers/microsoft.insights/components/ContainerInsightsAgent-Prod AgentTelemetryVersion=$(linuxTelemetryTagUnderTest)' + additionalTestParams: 'AgentTelemetryResourceId=$(AGENT_TELEMETRY_RESOURCE_ID) AgentTelemetryVersion=$(linuxImageTagUnderTest)' # ============================================================ # Cluster: ci-logs-prod-aks-networkflowlogs — Deploy via Helm @@ -1165,7 +1157,7 @@ extends: azureClientId: $(NetworkFlowLogsClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) - additionalTestParams: 'LinuxTestsOnly=true' + additionalTestParams: 'LinuxTestsOnly=true AgentTelemetryResourceId=$(AGENT_TELEMETRY_RESOURCE_ID) AgentTelemetryVersion=$(linuxImageTagUnderTest)' # ============================================================ # Cluster: ci-logs-dev-aks-std-prof-config-test1 — Deploy via Helm @@ -1217,4 +1209,4 @@ extends: azureClientId: $(AllNodesClientId) azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) teamsWebhookUri: $(TeamsWebhookUri) - additionalTestParams: 'PerNodeLogCoverage=true' + additionalTestParams: 'PerNodeLogCoverage=true AgentTelemetryResourceId=$(AGENT_TELEMETRY_RESOURCE_ID) AgentTelemetryVersion=$(linuxImageTagUnderTest)' diff --git a/test/ginkgo-e2e/querylogs/querylogs_test.go b/test/ginkgo-e2e/querylogs/querylogs_test.go index e7f57e2f86..a8436969ea 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_test.go @@ -3,7 +3,6 @@ package querylogs_test import ( "fmt" "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -11,14 +10,6 @@ import ( "docker-provider/test/utils" ) -const ( - // The Go output plugin publishes telemetry every defaultTelemetryPushIntervalSeconds - // (300s), so a pod younger than this has legitimately not reported a heartbeat yet. - agentTelemetryPublishInterval = 5 * time.Minute - // Span several publish intervals so one delayed batch does not fail the assertion. - agentTelemetryWindow = "20m" -) - var _ = Describe("When querying the logs for the table", func() { DescribeTable("All tables should have logs", func(table string) { @@ -98,72 +89,29 @@ var _ = Describe("When querying the number of resources of the cluster", func() }) var _ = Describe("When querying the agent telemetry", func() { - // expectedVersionByNode maps each node to the image tag its agent is running, which the - // agent reports back as customDimensions.Version. Resolving it per node is what makes these - // assertions verify the image the deploy stage just rolled out, rather than accepting - // telemetry that its predecessor published earlier in the same query window. - var expectedVersionByNode map[string]string - - BeforeEach(func() { - if AgentTelemetryResourceId == "" { - Skip("Agent telemetry checks skipped because AGENT_TELEMETRY_RESOURCE_ID is not set") - } - - var err error - expectedVersionByNode, err = utils.GetAgentImageTagsByNode(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs", agentTelemetryPublishInterval) - Expect(err).NotTo(HaveOccurred()) - if len(expectedVersionByNode) == 0 { - Skip("No ama-logs DaemonSet pod has been running long enough to have published telemetry") - } - - // The build bakes AGENT_VERSION in from its own telemetry tag rather than from the - // image tag, and the two diverge on release builds, where TELEMETRY_TAG overrides it. - // When the pipeline passes that tag it is authoritative, so prefer it over the tag - // read off the pod. - if AgentTelemetryVersion != "" { - for node := range expectedVersionByNode { - expectedVersionByNode[node] = AgentTelemetryVersion + DescribeTable("Every node should report telemetry from the new agent version", + func(telemetrySource string) { + if AgentTelemetryResourceId == "" { + Skip("Agent telemetry checks skipped because AGENT_TELEMETRY_RESOURCE_ID is not set") } - } - }) - It("Every node running an ama-logs DaemonSet pod should report a telemetry heartbeat from the deployed image", func() { - // A running agent does not imply working telemetry. Container logs reach the workspace - // over a local mdsd socket, so they keep flowing even when the agent's outbound - // telemetry path is entirely broken. Asserting that the heartbeat actually arrived is - // what distinguishes the two. - observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, "customEvents", utils.AgentTelemetryHeartbeatEvent) - Expect(err).NotTo(HaveOccurred()) + Expect(AgentTelemetryVersion).NotTo(BeEmpty(), "AGENT_TELEMETRY_VERSION must be set to the newly deployed image tag") - Expect(utils.AssertNodeVersionCoverage("agent telemetry heartbeat", expectedVersionByNode, observed)).NotTo(HaveOccurred()) - }) - - DescribeTable("Every node should publish agent telemetry from the deployed image to the table", - func(table string) { - // The heartbeat only proves the custom event path works. Metrics travel the same - // outbound connection but through a different SDK track call, so a break confined - // to one of them stays invisible until each table is asserted on its own. - observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, table, "") + expectedNodes, err := utils.GetExpectedAmaLogsNodes(K8sClient) Expect(err).NotTo(HaveOccurred()) - Expect(utils.AssertNodeVersionCoverage(table, expectedVersionByNode, observed)).NotTo(HaveOccurred()) + query := fmt.Sprintf(`%s +| where timestamp > ago(15m) +| extend ClusterId = iff(isnotempty(tostring(customDimensions.ID)), tostring(customDimensions.ID), tostring(customDimensions.AKS_RESOURCE_ID)) +| where ClusterId =~ %q +| where tostring(customDimensions.Version) in (%q, %q) +| distinct Computer = tolower(tostring(customDimensions.Computer))`, telemetrySource, AKSResourceId, AgentTelemetryVersion, "win-"+AgentTelemetryVersion) + + err = utils.CompareResourcesHelper(LogsClient, AgentTelemetryResourceId, query, expectedNodes) + Expect(err).NotTo(HaveOccurred()) }, - Entry("customEvents", "customEvents"), Entry("customMetrics", "customMetrics"), + Entry("traces", "traces"), + Entry("heartbeat", `customEvents | where name == "ContainerLogDaemonSetHeartbeatEvent"`), ) - - It("Agent log traces that arrive should come from the deployed image", func() { - // traces carries the agent's own log lines, and only those that are not "Information" - // level, so a healthy agent emits none: in a sampled 30 minute window only 41,491 of - // the 107,982 clusters reporting a heartbeat produced a single trace. Requiring traces - // would fail the majority of healthy clusters, so the query still has to succeed and - // any trace that does arrive still has to come from the deployed image, but an empty - // result is reported rather than failed. - observed, err := utils.GetAgentTelemetryVersionsByNode(LogsClient, AgentTelemetryResourceId, AKSResourceId, agentTelemetryWindow, "traces", "") - Expect(err).NotTo(HaveOccurred()) - - AddReportEntry("agent log traces", fmt.Sprintf("%d trace(s) from %d/%d node(s) in the last %s", utils.TotalItems(observed), len(observed), len(expectedVersionByNode), agentTelemetryWindow)) - - Expect(utils.AssertReportedNodeVersions("agent log traces", expectedVersionByNode, observed)).NotTo(HaveOccurred()) - }) }) diff --git a/test/ginkgo-e2e/utils/image_tag_test.go b/test/ginkgo-e2e/utils/image_tag_test.go deleted file mode 100644 index ebfbe04d66..0000000000 --- a/test/ginkgo-e2e/utils/image_tag_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package utils - -import "testing" - -func TestImageTag(t *testing.T) { - cases := []struct { - name string - imageRef string - want string - }{ - // The tag the agent reports back as customDimensions.Version. - {"linux agent", "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:3.8.0-ci-prod-09-06-2026-fd42f68c", "3.8.0-ci-prod-09-06-2026-fd42f68c"}, - {"windows agent", "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-3.8.0-ci-prod-09-06-2026-fd42f68c", "win-3.8.0-ci-prod-09-06-2026-fd42f68c"}, - // A digest is stripped first, so a reference pinning both still yields its tag. - {"tag and digest", "mcr.microsoft.com/geneva/mdsd:recommended@sha256:abc", "recommended"}, - // Nothing to match the reported version against. - {"digest only", "mcr.microsoft.com/oss/v2/calico/node@sha256:69124ac", ""}, - {"bare reference", "mcr.microsoft.com/oss/v2/kubernetes/pause", ""}, - // The port of a registry host must never be mistaken for a tag. - {"registry port, no tag", "localhost:5000/ciprod", ""}, - {"registry port and tag", "localhost:5000/ciprod:3.8.0", "3.8.0"}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := ImageTag(tc.imageRef); got != tc.want { - t.Fatalf("ImageTag(%q) = %q, want %q", tc.imageRef, got, tc.want) - } - }) - } -} diff --git a/test/ginkgo-e2e/utils/kubernetes_api_utils.go b/test/ginkgo-e2e/utils/kubernetes_api_utils.go index fa193f3a0d..a1de5b2294 100644 --- a/test/ginkgo-e2e/utils/kubernetes_api_utils.go +++ b/test/ginkgo-e2e/utils/kubernetes_api_utils.go @@ -600,87 +600,3 @@ func CheckFileForErrors(clientset *kubernetes.Clientset, Cfg *rest.Config, names return nil } - -// GetAgentNodesReadyLongerThan returns the names of the nodes running a pod with the given -// label that have been up for at least minAge. Younger pods are excluded because the agent -// publishes telemetry on a fixed interval, so a pod that has not yet reached its first publish -// has legitimately reported nothing and would otherwise fail the assertion during a rollout. -func GetAgentNodesReadyLongerThan(clientset *kubernetes.Clientset, namespace, labelName, labelValue string, minAge time.Duration) ([]string, error) { - pods, err := GetPodsWithLabel(clientset, namespace, labelName, labelValue) - if err != nil { - return nil, fmt.Errorf("failed to get pods with label %s=%s: %v", labelName, labelValue, err) - } - if len(pods) == 0 { - return nil, fmt.Errorf("no pods found with label %s=%s", labelName, labelValue) - } - - nodes := []string{} - for _, pod := range pods { - if !podPublishedAtLeastOnce(pod, minAge) { - continue - } - nodes = append(nodes, pod.Spec.NodeName) - } - - return nodes, nil -} - -// podPublishedAtLeastOnce reports whether a pod is running on a node and has been up long -// enough to have reached at least one telemetry publish interval. -func podPublishedAtLeastOnce(pod corev1.Pod, minAge time.Duration) bool { - if pod.Spec.NodeName == "" || pod.Status.Phase != corev1.PodRunning { - return false - } - return pod.Status.StartTime != nil && time.Since(pod.Status.StartTime.Time) >= minAge -} - -// ImageTag returns the tag of a container image reference, or an empty string when the -// reference carries no tag. Any digest is stripped first so a reference that pins both a tag -// and a digest still yields its tag, and the tag is only looked for after the final "/" so -// that the port in a registry host is never mistaken for one. -func ImageTag(imageRef string) string { - if at := strings.Index(imageRef, "@"); at >= 0 { - imageRef = imageRef[:at] - } - colon := strings.LastIndex(imageRef, ":") - if colon <= strings.LastIndex(imageRef, "/") { - return "" - } - return imageRef[colon+1:] -} - -// GetAgentImageTagsByNode returns the image tag the agent container is running on each node, -// keyed by lowercased node name, for pods that have been up for at least minAge. -// -// The image build bakes this tag into the image as AGENT_VERSION (`ENV AGENT_VERSION -// ${IMAGE_TAG}`) and the agent reports it as customDimensions.Version on every telemetry item. -// Comparing the two is what ties an assertion to the image currently deployed, rather than to -// any agent that happens to have reported for the cluster inside the query window. -func GetAgentImageTagsByNode(clientset *kubernetes.Clientset, namespace, labelName, labelValue, containerName string, minAge time.Duration) (map[string]string, error) { - pods, err := GetPodsWithLabel(clientset, namespace, labelName, labelValue) - if err != nil { - return nil, fmt.Errorf("failed to get pods with label %s=%s: %v", labelName, labelValue, err) - } - if len(pods) == 0 { - return nil, fmt.Errorf("no pods found with label %s=%s", labelName, labelValue) - } - - tagsByNode := map[string]string{} - for _, pod := range pods { - if !podPublishedAtLeastOnce(pod, minAge) { - continue - } - for _, container := range pod.Spec.Containers { - if container.Name != containerName { - continue - } - tag := ImageTag(container.Image) - if tag == "" { - return nil, fmt.Errorf("container %s in pod %s runs image %q, which carries no tag to match the reported agent version against", containerName, pod.Name, container.Image) - } - tagsByNode[strings.ToLower(pod.Spec.NodeName)] = tag - } - } - - return tagsByNode, nil -} diff --git a/test/ginkgo-e2e/utils/query_logs_api_utils.go b/test/ginkgo-e2e/utils/query_logs_api_utils.go index 3c4785d9a6..a00a3e9fc9 100644 --- a/test/ginkgo-e2e/utils/query_logs_api_utils.go +++ b/test/ginkgo-e2e/utils/query_logs_api_utils.go @@ -2,9 +2,7 @@ package utils import ( "context" - "errors" "fmt" - "sort" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -190,154 +188,6 @@ func queryCountsByComputer(logsClient *azquery.LogsClient, resourceID string, ta return counts, nil } -// AgentTelemetryHeartbeatEvent is the App Insights custom event the Go output plugin emits once -// per publish interval from every DaemonSet pod. It is the signal that goes silent when the -// agent's outbound telemetry path breaks while container logs keep flowing over the local mdsd -// socket, so its arrival is asserted directly rather than inferred from the agent's own logs. -const AgentTelemetryHeartbeatEvent = "ContainerLogDaemonSetHeartbeatEvent" - -// AgentTelemetryPeriodicTables are the Application Insights tables the agent writes to on a -// fixed interval, so every node running a healthy agent has to appear in all of them. -// -// `traces` is deliberately not in this list. It carries the agent's own log lines, and only -// those that are not "Information" level, so an agent with nothing to complain about emits -// none at all: in a sampled 30 minute window only 41,491 of the 107,982 clusters that reported -// a heartbeat produced a single trace. Requiring traces would therefore fail the majority of -// healthy clusters, so their arrival is reported rather than asserted. -var AgentTelemetryPeriodicTables = []string{"customEvents", "customMetrics"} - -// GetAgentTelemetryVersionsByNode returns the telemetry item count for one Application Insights -// table, keyed by lowercased node name and then by the agent version that reported it. It -// queries the agent telemetry Application Insights resource rather than the cluster's -// workspace: this is agent self-telemetry and is never ingested into the customer workspace. -// -// Keeping the version in the result is what allows a caller to tell a node that is reporting -// from the image under test apart from one that is only still reporting from its predecessor. -// Pass an empty eventName for tables such as `traces` that have no name column. -func GetAgentTelemetryVersionsByNode(logsClient *azquery.LogsClient, telemetryResourceID string, aksResourceID string, window string, table string, eventName string) (map[string]map[string]int64, error) { - nameFilter := "" - if eventName != "" { - nameFilter = fmt.Sprintf("\n| where name == \"%s\"", eventName) - } - - // Resource IDs are matched case-insensitively because the agent reports both - // /resourcegroups/ and /resourceGroups/ spellings for the same cluster. - query := fmt.Sprintf(`%s -| where timestamp > ago(%s)%s -| extend ClusterId = iff(isnotempty(tostring(customDimensions.ID)), tostring(customDimensions.ID), tostring(customDimensions.AKS_RESOURCE_ID)) -| where ClusterId =~ "%s" -| summarize count() by Computer = tostring(customDimensions.Computer), Version = tostring(customDimensions.Version)`, - table, window, nameFilter, aksResourceID) - - tables, err := QueryLogs(logsClient, telemetryResourceID, query) - if err != nil { - return nil, err - } - - counts := map[string]map[string]int64{} - for _, t := range tables { - for _, row := range t.Rows { - if len(row) < 3 { - continue - } - computer, ok := row[0].(string) - if !ok || computer == "" { - continue - } - version, _ := row[1].(string) - count, _ := row[2].(float64) - node := strings.ToLower(computer) - if counts[node] == nil { - counts[node] = map[string]int64{} - } - counts[node][version] += int64(count) - } - } - return counts, nil -} - -// AssertNodeVersionCoverage returns nil when every node in expectedVersionByNode reported the -// signal under the agent version that node is actually running. -// -// A node that reported nothing and a node that reported only some other version are listed -// separately, because they are different faults: the first is a node whose telemetry is not -// arriving at all, while the second is telemetry arriving from an agent that is no longer the -// one deployed, which is what a stale image or a half-finished rollout looks like. -func AssertNodeVersionCoverage(signal string, expectedVersionByNode map[string]string, observed map[string]map[string]int64) error { - if len(expectedVersionByNode) == 0 { - return fmt.Errorf("no expected nodes provided; cannot verify %s coverage", signal) - } - - var missing, staleVersion []string - for node, expectedVersion := range expectedVersionByNode { - reported := observed[strings.ToLower(node)] - if len(reported) == 0 { - missing = append(missing, node) - continue - } - if reported[expectedVersion] <= 0 { - staleVersion = append(staleVersion, fmt.Sprintf("%s (expected %q, reported %s)", node, expectedVersion, strings.Join(sortedVersions(reported), ", "))) - } - } - - sort.Strings(missing) - sort.Strings(staleVersion) - - var problems []string - if len(missing) > 0 { - problems = append(problems, fmt.Sprintf("%s is missing for %d/%d expected node(s): %s", signal, len(missing), len(expectedVersionByNode), strings.Join(missing, ", "))) - } - if len(staleVersion) > 0 { - problems = append(problems, fmt.Sprintf("%s arrived for %d node(s) but not from the deployed agent version: %s", signal, len(staleVersion), strings.Join(staleVersion, "; "))) - } - if len(problems) > 0 { - return errors.New(strings.Join(problems, "; ")) - } - return nil -} - -// AssertReportedNodeVersions returns nil when every node that did report the signal reported it -// under the version that node is running. Nodes that reported nothing are ignored, which makes -// it the right check for a signal whose absence is legitimate. -func AssertReportedNodeVersions(signal string, expectedVersionByNode map[string]string, observed map[string]map[string]int64) error { - reporting := map[string]string{} - for node, expectedVersion := range expectedVersionByNode { - if len(observed[strings.ToLower(node)]) > 0 { - reporting[node] = expectedVersion - } - } - if len(reporting) == 0 { - return nil - } - return AssertNodeVersionCoverage(signal, reporting, observed) -} - -// sortedVersions returns the versions in a per-version count map in a stable order so that -// failure messages do not change between runs for the same underlying data. -func sortedVersions(counts map[string]int64) []string { - versions := make([]string, 0, len(counts)) - for version := range counts { - if version == "" { - version = "" - } - versions = append(versions, version) - } - sort.Strings(versions) - return versions -} - -// TotalItems returns the number of telemetry items across every node and version in a result -// from GetAgentTelemetryVersionsByNode. -func TotalItems(observed map[string]map[string]int64) int64 { - var total int64 - for _, byVersion := range observed { - for _, count := range byVersion { - total += count - } - } - return total -} - // AssertNodeCoverage returns nil if every expected node appears in the per-Computer count map // with a positive count (compared case-insensitively), or an error listing the missing nodes. func AssertNodeCoverage(signal string, expectedNodes []string, observedCountsByComputer map[string]int64) error { From 5462df9ff521894f62c13468143f1a06beb160ef Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Tue, 22 Sep 2026 23:48:57 +0000 Subject: [PATCH 5/8] test: temporarily run telemetry-only pipeline validation [skip ci] --- .pipelines/azure_pipeline_mergedbranches.yaml | 1098 +---------------- .../test-ci-image-in-aks-cluster.yml | 5 +- .../install-and-execute-testkube-tests.sh | 11 +- test/testkube/testkube-test-crs.yaml | 4 +- 4 files changed, 19 insertions(+), 1099 deletions(-) diff --git a/.pipelines/azure_pipeline_mergedbranches.yaml b/.pipelines/azure_pipeline_mergedbranches.yaml index 362d96e5ff..d7fcb6fcea 100644 --- a/.pipelines/azure_pipeline_mergedbranches.yaml +++ b/.pipelines/azure_pipeline_mergedbranches.yaml @@ -1,16 +1,9 @@ -trigger: - batch: true - branches: - include: - - ci_prod - - auto/upgrade-telegraf-* - - auto/upgrade-go-* - - dependabot/* -pr: - autoCancel: true - branches: - include: - - ci_prod +trigger: none +pr: none +parameters: +- name: telemetryValidationVersion + type: string + default: '3.8.0-ci-prod-09-20-2026-8e6bef56' variables: armServiceConnectionName: 'ci-1es-acr-connection' subscription: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' @@ -55,1010 +48,20 @@ extends: customBuildTags: - ES365AIMigrationTooling stages: - - stage: stage - displayName: 'Build and Publish Container Images' - jobs: - - job: common - pool: - name: Azure-Pipelines-CI-Test-EO - image: ci-1es-managed-ubuntu-2204 - os: linux - variables: - skipComponentGovernanceDetection: true - Codeql.SkipTaskAutoInjection: true - templateContext: - outputs: - - output: pipelineArtifact - targetPath: '$(Build.ArtifactStagingDirectory)' - artifactName: drop - steps: - - task: ComponentGovernanceComponentDetection@0 - - bash: | - # Derive SEMVER from the repo-root VERSION file instead of git tags. - # Format: --- - if [ "$(IS_PR)" == "True" ]; then - BRANCH_NAME=$(System.PullRequest.SourceBranch) - else - BRANCH_NAME=$(Build.SourceBranch) - BRANCH_NAME=${BRANCH_NAME#refs/heads/} - fi - BRANCH_NAME=$(echo "$BRANCH_NAME" | tr / - | tr . - | tr _ - | cut -c1-90) - COMMIT_SHA=$(echo "$(Build.SourceVersion)" | cut -b -8) - DATE=$(TZ=America/Los_Angeles date +%m-%d-%Y) - VERSION=$(cat $(Build.SourcesDirectory)/VERSION) - SEMVER=$VERSION-$BRANCH_NAME-$DATE-$COMMIT_SHA - linuxImagetag=$SEMVER - windowsImageTag=win-$SEMVER - telemetryTag=$linuxImagetag - if [ -z "$TELEMETRY_TAG" ] - then - echo "\$TELEMETRY_TAG variable is not set" - else - telemetryTag=$TELEMETRY_TAG - echo "\$TELEMETRY_TAG is $TELEMETRY_TAG" - fi - linuxTelemetryTag="$telemetryTag" - windowsTelemetryTag=win-"$telemetryTag" - echo "linuxImagetag is $linuxImagetag" - echo "windowsImageTag is $windowsImageTag" - echo "linuxTelemetryTag is $linuxTelemetryTag" - echo "windowsTelemetryTag is $windowsTelemetryTag" - echo "##vso[task.setvariable variable=linuxImagetag;isOutput=true]$linuxImagetag" - echo "##vso[task.setvariable variable=windowsImageTag;isOutput=true]$windowsImageTag" - echo "##vso[task.setvariable variable=linuxTelemetryTag;isOutput=true]$linuxTelemetryTag" - echo "##vso[task.setvariable variable=windowsTelemetryTag;isOutput=true]$windowsTelemetryTag" - echo "SEMVER is $SEMVER" - echo "##vso[task.setvariable variable=SEMVER;isOutput=true]$SEMVER" - # Set the pipeline run number to SEMVER so downstream release pipelines - # can consume this exact build version via - # resources.pipeline..runName (mirrors ama-metrics). A release then - # just picks a build run and uses whatever version is inside it, instead - # of hand-setting an image tag suffix. NOTE: the $(Build.BuildNumber) - # macros in the buildver.txt writes below are already expanded to the - # original unique run number at step start, so this override does not - # affect Ev2 artifact-version uniqueness. - echo "##vso[build.updatebuildnumber]$SEMVER" - echo "appRegistrationClientId is $APP_REGISTRATION_CLIENT_ID" - echo "appRegistrationTenantId is $APP_REGISTRATION_TENANT_ID" - echo "authAKVName is $AUTH_AKV_NAME" - echo "authCertName is $AUTH_CERT_NAME" - echo "authSignCertName is $AUTH_SIGN_CERT_NAME" - # Generate Chart.yaml/values.yaml from the committed *-template.yaml files - # so the Ev2 source tar + CopyFiles below include the SEMVER-stamped chart - # for the downstream release pipeline. Only templates are committed. - export HELM_SEMVER="$SEMVER" - export IMAGE_TAG="$linuxImagetag" - export IMAGE_TAG_WINDOWS="$windowsImageTag" - CI_CHART_DIR="$(Build.SourcesDirectory)/charts/azuremonitor-containerinsights" - envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < "$CI_CHART_DIR/Chart-template.yaml" > "$CI_CHART_DIR/Chart.yaml" - envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < "$CI_CHART_DIR/values-template.yaml" > "$CI_CHART_DIR/values.yaml" - cd $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment-Managed-SDP/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh pushChartToAcr.sh - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ ../../../../charts/azuremonitor-containerinsights/ pushChartToAcr.sh - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-Managed-SDP/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ ../../../../charts/azuremonitor-containerinsights/ pushChartToAcr.sh - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz arcExtensionRelease.sh - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz arcExtensionRelease.sh - # Stamp a unique Ev2 artifacts version per build. Ev2 dedups artifact - # registration on this versionFile; a static buildver.txt makes Ev2 skip - # uploading new artifacts ("already registered"), freezing the published - # chart at whatever was first registered. Writing $(Build.BuildNumber) - # forces fresh registration on every build. - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment/ServiceGroupRoot/buildver.txt - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment-Managed-SDP/ServiceGroupRoot/buildver.txt - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/buildver.txt - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-Managed-SDP/ServiceGroupRoot/buildver.txt - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2/ServiceGroupRoot/buildver.txt - echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot/buildver.txt - windowsAMAUrl="" - if [ -z "$WINDOWS_AMA_URL" ] - then - echo "\$WINDOWS_AMA_URL variable is not set" - else - windowsAMAUrl=$WINDOWS_AMA_URL - echo "\$WINDOWS_AMA_URL is $WINDOWS_AMA_URL" - fi - echo "##vso[task.setvariable variable=windowsAMAUrl;isOutput=true]$windowsAMAUrl" - name: setup - - task: CredScan@3 - displayName: "SDL : Run credscan" - - task: CopyFiles@2 - displayName: "Copy ev2 deployment artifacts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/deployment" - Contents: | - **/* - !**/ScanTelemetry_*.json - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - - task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/.pipelines" - Contents: | - **/*.sh - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - - task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/kubernetes" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - - task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/charts" - Contents: | - **/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - - task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/test/e2e" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - - task: Armory@2 - displayName: 'Run ARMory' - inputs: - toolVersion: Latest - targetDirectory: '$(Build.SourcesDirectory)' - - job: build_linux - # the emulated linux/arm64 leg is slow, and setup.sh retries native gem builds that - # segfault under emulation, so leave headroom rather than failing on the job timeout - timeoutInMinutes: 180 - dependsOn: common - variables: - linuxImagetag: $[ dependencies.common.outputs['setup.linuxImagetag'] ] - linuxTelemetryTag: $[ dependencies.common.outputs['setup.linuxTelemetryTag'] ] - Codeql.Enabled: true - Codeql.BuildIdentifier: 'linuxbuild' - DOCKER_BUILDKIT: 1 - templateContext: - outputs: - - output: pipelineArtifact - targetPath: '$(Build.ArtifactStagingDirectory)' - artifactName: linux-drop - steps: - - task: CodeQL3000Init@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - task: AzureCLI@2 - displayName: "Multi-arch Linux build" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - mkdir -p $(Build.ArtifactStagingDirectory)/linux - docker system prune --all -f - docker images -q --filter "dangling=true" | xargs docker rmi - # Register binfmt handlers for cross-arch (linux/arm64) emulation. - # NOTE: this previously used `multiarch/qemu-user-static --reset -p yes`, which pins the - # QEMU binary from that image into the kernel (-p yes => binfmt_misc 'F' flag). That image - # is abandoned (last push 2023-01, QEMU 7.2), and the apt qemu-user-static it sat on top of - # is frozen at QEMU 6.2 on ubuntu-22.04 -- apt only backports fixes, never new upstream - # versions. Emulated gcc segfaults at random on those, which broke the arm64 leg roughly - # half the time while compiling the ruby native gem extensions in kubernetes/linux/setup.sh. - # tonistiigi/binfmt tracks current QEMU releases and is what docker/setup-qemu-action uses. - # Pulled from the MCR mirror to avoid Docker Hub rate limiting. - docker run --rm --privileged $(QEMU_BINFMT_IMAGE) --uninstall 'qemu-*' || true - docker run --rm --privileged $(QEMU_BINFMT_IMAGE) --install all || exit 1 - # fail fast (instead of 30+ minutes into the arm64 build) if emulation is not usable. - # NOTE: this script does not run under 'set -e', so check explicitly. - docker run --rm --platform linux/arm64 mcr.microsoft.com/azurelinux/base/core:3.0 uname -m || exit 1 - docker buildx create --name testbuilder - docker buildx use testbuilder - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - # NOTE: Using the prometheus-collector team's cached buildx image since moby/buildkit:buildx-stable-1 getting throttled - docker pull mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1 - docker buildx create --name dockerbuilder --driver docker-container --driver-opt image=mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1 --use - docker buildx inspect --bootstrap - if [ "$(Build.Reason)" != "PullRequest" ]; then - docker buildx build --platform $(BUILD_PLATFORMS) --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --push --provenance=false . - echo "##vso[task.logissue type=warning]Linux image built with tag: ${{ variables.repoImageName }}:$(linuxImagetag)" - docker pull ${{ variables.repoImageName }}:$(linuxImagetag) - else - docker buildx build --platform $(BUILD_PLATFORMS) --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --provenance=false . - # load the multi-arch image to run tests - docker buildx build --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --load --provenance=false . - fi - - bash: | - curl -LO "https://github.com/oras-project/oras/releases/download/v1.0.0/oras_1.0.0_linux_amd64.tar.gz" - mkdir -p oras-install/ - tar -zxf oras_1.0.0_*.tar.gz -C oras-install/ - sudo mv oras-install/oras /usr/local/bin/ - rm -rf oras_1.0.0_*.tar.gz oras-install/ - TARGET_ARTIFACT=$(oras manifest fetch ${{ variables.repoImageName }}:$(linuxImagetag) --descriptor) - cat <>$(Build.ArtifactStagingDirectory)/linux/payload.json - {"targetArtifact":$TARGET_ARTIFACT} - EOF - cat $(Build.ArtifactStagingDirectory)/linux/payload.json - workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ - displayName: "Install oras and build the payload json file" - condition: eq(variables.IS_PR, false) - - task: EsrpCodeSigning@5 - condition: eq(variables.IS_PR, false) - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/linux' - Pattern: 'payload.json' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "keyCode": "CP-469451", - "operationSetCode": "NotaryCoseSign", - "parameters": [ - { - "parameterName": "CoseFlags", - "parameterValue": "chainunprotected" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - PendingAnalysisWaitTimeoutMinutes: '5' - displayName: 'Esrp Image Signing for linux image' - - bash: | - set -euxo pipefail - oras attach ${{ variables.repoImageName }}:$(linuxImagetag) \ - --artifact-type 'application/vnd.cncf.notary.signature' \ - ./payload.json:application/cose \ - -a "io.cncf.notary.x509chain.thumbprint#S256=[\"49D6CD5DB42623144D6990AA1669CE5D97F1F3D7\"]" - workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ - displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" - condition: eq(variables.IS_PR, false) - - bash: | - set -euxo pipefail - oras attach ${{ variables.repoImageName }}:$(linuxImagetag) \ - --artifact-type 'application/vnd.microsoft.artifact.lifecycle' \ - --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$(date -u -d '-1 hour' +"%Y-%m-%dT%H:%M:%SZ")" - workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ - displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" - condition: and(eq(variables.IS_RELEASE, false), eq(variables.IS_PR, false)) - - task: AzureCLI@2 - displayName: "Vulnerability Scan with Trivy" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - PRIMARY_TRIVY_DB_REPOSITORY="ghcr.io/aquasecurity/trivy-db" - SECONDARY_TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db" - PRIMARY_TRIVY_JAVA_DB_REPOSITORY="ghcr.io/aquasecurity/trivy-java-db" - SECONDARY_TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db" - # Set initial repositories to primary - export TRIVY_DB_REPOSITORY=$PRIMARY_TRIVY_DB_REPOSITORY - export TRIVY_JAVA_DB_REPOSITORY=$PRIMARY_TRIVY_JAVA_DB_REPOSITORY - # Function to run Trivy scan and handle output - run_trivy_scan() { - trivy image --exit-code 0 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM "${{ variables.repoImageName }}:$(linuxImagetag)" > trivy_output.log 2>&1 - return $? - } - # Attempt scan up to 5 times with repository fallback - for i in {1..5}; do - echo "Running Trivy scan attempt $i" - # Run the Trivy scan and capture exit code - run_trivy_scan - TRIVY_EXIT_CODE=$? - # Check if scan was successful - if [ $TRIVY_EXIT_CODE -eq 0 ]; then - echo "Trivy scan succeeded." - cat trivy_output.log - break - fi - # If the first attempt fails, switch to secondary repositories - if [ $i -eq 1 ]; then - echo "Primary repositories failed with an error. Switching to secondary repositories." - export TRIVY_DB_REPOSITORY=$SECONDARY_TRIVY_DB_REPOSITORY - export TRIVY_JAVA_DB_REPOSITORY=$SECONDARY_TRIVY_JAVA_DB_REPOSITORY - fi - # Log and wait before retrying if an error occurred - echo "Error: Trivy scan attempt $i failed. Retrying ($i/5)" - cat trivy_output.log - sleep 5 # Wait 5 seconds before retrying - done - # Final check: if still failing after 5 attempts, exit with error - if [ $TRIVY_EXIT_CODE -ne 0 ]; then - echo "Error: Trivy scan failed after 5 retries." - exit 1 - fi - - task: GoTool@0 - inputs: - version: '1.23.8' - - bash: | - cd $(System.DefaultWorkingDirectory)/build/linux/ - ls - make - displayName: 'Execute Makefile for Linux Build' - - task: CodeQL3000Finalize@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generation Task' - condition: eq(variables.IS_PR, false) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' - DockerImagesToScan: '$(GOLANG_BASE_IMAGE),$(CI_BASE_IMAGE),${{ variables.repoImageName }}:$(linuxImagetag)' - - bash: | - dockerImagesToScan='$(GOLANG_BASE_IMAGE),$(CI_BASE_IMAGE),${{ variables.repoImageName }}:$(linuxImagetag)' - echo "Docker images to scan: $dockerImagesToScan" - echo "##vso[task.setvariable variable=dockerImagesToScan]$dockerImagesToScan" - displayName: 'Set Docker images to scan' - - job: build_windows_2019 - pool: - name: Azure-Pipelines-CI-Test-EO - image: ci-1es-managed-windows-2022 - os: windows - timeoutInMinutes: 120 - dependsOn: - - common - variables: - windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] - windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] - windows2019BaseImageVersion: ltsc2019 - Codeql.Enabled: true - Codeql.BuildIdentifier: 'windowsbuild' - windowsAMAUrl: $[ dependencies.common.outputs['setup.windowsAMAUrl'] ] - steps: - - task: PowerShell@2 - inputs: - targetType: 'filePath' - filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 - displayName: 'install prereqs' - - task: CodeQL3000Init@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to CFS NuGet feed' - - script: | - setlocal enabledelayedexpansion - powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" - endlocal - exit /B %ERRORLEVEL% - displayName: 'build base' - - task: AzureCLI@2 - displayName: "Set up acr for windows ltsc2019 build" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: ps - scriptLocation: inlineScript - retryCountOnTaskFailure: 2 - inlineScript: | - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - # Check if directory exists and remove it before creating it to handle retry scenarios - if (Test-Path -Path "$(Build.ArtifactStagingDirectory)/windows") { - Remove-Item -Path "$(Build.ArtifactStagingDirectory)/windows" -Recurse -Force - } - New-Item -Path "$(Build.ArtifactStagingDirectory)/windows" -ItemType Directory -Force - cd kubernetes/windows - # Build the Docker image - docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned --build-arg WINDOWS_VERSION=$(windows2019BaseImageVersion) --build-arg IMAGE_TAG=$(windowsTelemetryTag) --build-arg WINDOWS_AMA_URL=$(windowsAMAUrl) . - displayName: "Docker windows build for ltsc2019" - retryCountOnTaskFailure: 2 - - task: PowerShell@2 - displayName: Extract files to sign - inputs: - targetType: 'inline' - script: | - echo "Creating docker container" - docker create --name signingContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned - echo "Creating fist party directory" - mkdir -p $(Build.ArtifactStagingDirectory)/fpSigning - cd $(Build.ArtifactStagingDirectory)/fpSigning - echo "Extract CertificateGenerator" - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe . - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll . - echo "Extract LivenessProbe" - docker cp signingContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe . - echo "Extract ps scripts" - docker cp signingContainer:C:\opt\amalogswindows\scripts\powershell . - echo "Creating OSS directory" - mkdir -p $(Build.ArtifactStagingDirectory)/ossSigning - cd $(Build.ArtifactStagingDirectory)/ossSigning - echo "Extract CertificateGenerator" - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll . - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll . - echo "Extract fluent-bit" - docker cp signingContainer:C:\opt\fluent-bit . - echo "Extract Ruby" - docker cp signingContainer:C:\ruby31 . - echo "Extract telegraf" - docker cp signingContainer:C:\opt\telegraf\telegraf.exe . - echo "Extract out oms" - docker cp signingContainer:C:\opt\amalogswindows\out_oms.so . - echo "Extract containerinventory.so" - docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so . - echo "Extract perf.so" - docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so . - echo "Removing container" - docker rm signingContainer - echo "List ArtifactStagingDirectory" - ls $(Build.ArtifactStagingDirectory) - ls . - - script: dir $(System.DefaultWorkingDirectory) - displayName: 'List files in DefaultWorking Directory' - - script: dir $(Build.ArtifactStagingDirectory) - displayName: 'List files in Staging Directory' - - task: EsrpCodeSigning@5 - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/fpSigning' - Pattern: '*.dll,*.exe,*.so,*.ps1' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "KeyCode" : "CP-230012", - "OperationCode" : "SigntoolSign", - "Parameters" : { - "OpusName" : "Microsoft", - "OpusInfo" : "http://www.microsoft.com", - "FileDigest" : "/fd \"SHA256\"", - "PageHash" : "/NPH", - "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName" : "sign", - "ToolVersion" : "1.0" - }, - { - "KeyCode" : "CP-230012", - "OperationCode" : "SigntoolVerify", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - displayName: 'EsrpCodeSigning for first party' - - task: EsrpCodeSigning@5 - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/ossSigning' - Pattern: '*.dll,*.exe,*.so' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "KeyCode" : "CP-231522", - "OperationCode" : "SigntoolSign", - "Parameters" : { - "OpusName" : "Microsoft", - "OpusInfo" : "http://www.microsoft.com", - "Append" : "/as", - "FileDigest" : "/fd \"SHA256\"", - "PageHash" : "/NPH", - "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName" : "sign", - "ToolVersion" : "1.0" - }, - { - "KeyCode" : "CP-231522", - "OperationCode" : "SigntoolVerify", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - displayName: 'EsrpCodeSigning for OSS' - - task: BinSkim@4 - displayName: 'SDL: run binskim' - inputs: - InputType: 'CommandLine' - arguments: 'analyze --rich-return-code $(Build.ArtifactStagingDirectory)\ossSigning\out_oms.so $(Build.ArtifactStagingDirectory)\ossSigning\perf.so $(Build.ArtifactStagingDirectory)\ossSigning\containerinventory.so $(Build.ArtifactStagingDirectory)\fpSigning\livenessprobe.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.dll' - retryCountOnTaskFailure: 1 - - task: PoliCheck@2 - displayName: "SDL : Run PoliCheck" - inputs: - targetType: 'F' - targetArgument: '$(Build.SourcesDirectory)' - - task: PowerShell@2 - displayName: Replace files in origin Image - inputs: - targetType: 'inline' - script: | - docker create --name pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned - echo "Copy Signed binaries/folders back to docker image" - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.exe pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.dll pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/livenessprobe.exe pushContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/powershell/. pushContainer:C:\opt\amalogswindows\scripts\powershell/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/BouncyCastle.Crypto.dll pushContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/Newtonsoft.Json.dll pushContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/fluent-bit/. pushContainer:C:\opt\fluent-bit/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/ruby31/. pushContainer:C:\ruby31/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/telegraf.exe pushContainer:C:\opt\telegraf\telegraf.exe - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/out_oms.so pushContainer:C:\opt\amalogswindows\out_oms.so - docker commit pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) - docker rm pushContainer - - task: PowerShell@2 - displayName: Push Docker Image - inputs: - targetType: 'inline' - script: | - if ("$(Build.Reason)" -ne "PullRequest") { - docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) - } - - task: CodeQL3000Finalize@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - job: build_windows_2022 - pool: - name: Azure-Pipelines-CI-Test-EO - image: ci-1es-managed-windows-2022 - os: windows - timeoutInMinutes: 120 - dependsOn: - - common - variables: - windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] - windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] - windows2022BaseImageVersion: ltsc2022 - Codeql.SkipTaskAutoInjection: true - windowsAMAUrl: $[ dependencies.common.outputs['setup.windowsAMAUrl'] ] - steps: - - task: PowerShell@2 - inputs: - targetType: 'filePath' - filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 - displayName: 'install prereqs' - - task: CodeQL3000Init@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to CFS NuGet feed' - - script: | - setlocal enabledelayedexpansion - powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" - endlocal - exit /B %ERRORLEVEL% - displayName: 'build base' - - task: AzureCLI@2 - displayName: "Docker windows build for ltsc2022" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: ps - scriptLocation: inlineScript - retryCountOnTaskFailure: 2 - inlineScript: | - mkdir -p $(Build.ArtifactStagingDirectory)/windows - cd kubernetes/windows - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned --build-arg WINDOWS_VERSION=$(windows2022BaseImageVersion) --build-arg IMAGE_TAG=$(windowsTelemetryTag) --build-arg WINDOWS_AMA_URL=$(windowsAMAUrl) . - - task: PowerShell@2 - displayName: Extract files to sign - inputs: - targetType: 'inline' - script: | - echo "Creating docker container" - docker create --name signingContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned - echo "Creating fist party directory" - mkdir -p $(Build.ArtifactStagingDirectory)/fpSigning - cd $(Build.ArtifactStagingDirectory)/fpSigning - echo "Extract CertificateGenerator" - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe . - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll . - echo "Extract LivenessProbe" - docker cp signingContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe . - echo "Extract ps scripts" - docker cp signingContainer:C:\opt\amalogswindows\scripts\powershell . - echo "Creating OSS directory" - mkdir -p $(Build.ArtifactStagingDirectory)/ossSigning - cd $(Build.ArtifactStagingDirectory)/ossSigning - echo "Extract CertificateGenerator" - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll . - docker cp signingContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll . - echo "Extract fluent-bit" - docker cp signingContainer:C:\opt\fluent-bit . - echo "Extract Ruby" - docker cp signingContainer:C:\ruby31 . - echo "Extract telegraf" - docker cp signingContainer:C:\opt\telegraf\telegraf.exe . - echo "Extract out oms" - docker cp signingContainer:C:\opt\amalogswindows\out_oms.so . - echo "Extract containerinventory.so" - docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so . - echo "Extract perf.so" - docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so . - echo "Removing container" - docker rm signingContainer - echo "List ArtifactStagingDirectory" - ls $(Build.ArtifactStagingDirectory) - ls . - - script: dir $(System.DefaultWorkingDirectory) - displayName: 'List files in DefaultWorking Directory' - - script: dir $(Build.ArtifactStagingDirectory) - displayName: 'List files in Staging Directory' - - task: EsrpCodeSigning@5 - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/fpSigning' - Pattern: '*.dll,*.exe,*.so,*.ps1' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "KeyCode" : "CP-230012", - "OperationCode" : "SigntoolSign", - "Parameters" : { - "OpusName" : "Microsoft", - "OpusInfo" : "http://www.microsoft.com", - "FileDigest" : "/fd \"SHA256\"", - "PageHash" : "/NPH", - "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName" : "sign", - "ToolVersion" : "1.0" - }, - { - "KeyCode" : "CP-230012", - "OperationCode" : "SigntoolVerify", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - displayName: 'EsrpCodeSigning for first party' - - task: EsrpCodeSigning@5 - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/ossSigning' - Pattern: '*.dll,*.exe,*.so' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "KeyCode" : "CP-231522", - "OperationCode" : "SigntoolSign", - "Parameters" : { - "OpusName" : "Microsoft", - "OpusInfo" : "http://www.microsoft.com", - "Append" : "/as", - "FileDigest" : "/fd \"SHA256\"", - "PageHash" : "/NPH", - "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName" : "sign", - "ToolVersion" : "1.0" - }, - { - "KeyCode" : "CP-231522", - "OperationCode" : "SigntoolVerify", - "Parameters" : {}, - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - displayName: 'EsrpCodeSigning for OSS' - - task: BinSkim@4 - displayName: 'SDL: run binskim' - inputs: - InputType: 'CommandLine' - arguments: 'analyze --rich-return-code $(Build.ArtifactStagingDirectory)\ossSigning\out_oms.so $(Build.ArtifactStagingDirectory)\ossSigning\perf.so $(Build.ArtifactStagingDirectory)\ossSigning\containerinventory.so $(Build.ArtifactStagingDirectory)\fpSigning\livenessprobe.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.dll' - retryCountOnTaskFailure: 1 - - task: PoliCheck@2 - displayName: "SDL : Run PoliCheck" - inputs: - targetType: 'F' - targetArgument: '$(Build.SourcesDirectory)' - - task: PowerShell@2 - displayName: Replace files in origin Image - inputs: - targetType: 'inline' - script: | - docker create --name pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned - echo "Copy Signed binaries/folders back to docker image" - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.exe pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.dll pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/livenessprobe.exe pushContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe - docker cp $(Build.ArtifactStagingDirectory)/fpSigning/powershell/. pushContainer:C:\opt\amalogswindows\scripts\powershell/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/BouncyCastle.Crypto.dll pushContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/Newtonsoft.Json.dll pushContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/fluent-bit/. pushContainer:C:\opt\fluent-bit/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/ruby31/. pushContainer:C:\ruby31/ - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/telegraf.exe pushContainer:C:\opt\telegraf\telegraf.exe - docker cp $(Build.ArtifactStagingDirectory)/ossSigning/out_oms.so pushContainer:C:\opt\amalogswindows\out_oms.so - docker commit pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) - docker rm pushContainer - - task: PowerShell@2 - displayName: Push Docker Image - inputs: - targetType: 'inline' - script: | - if ("$(Build.Reason)" -ne "PullRequest") { - docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) - } - - task: CodeQL3000Finalize@0 - condition: eq(variables.IS_MAIN_BRANCH, true) - - job: build_windows_multi_arc - pool: - name: Azure-Pipelines-CI-Test-EO - image: ci-1es-managed-windows-2022 - os: windows - timeoutInMinutes: 120 - dependsOn: - - common - - build_windows_2019 - - build_windows_2022 - variables: - windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] - windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] - windows2019BaseImageVersion: ltsc2019 - windows2022BaseImageVersion: ltsc2022 - Codeql.SkipTaskAutoInjection: true - templateContext: - outputs: - - output: pipelineArtifact - targetPath: '$(Build.ArtifactStagingDirectory)' - artifactName: windows-drop - steps: - - task: AzureCLI@2 - displayName: "Docker windows build for multi-arc image" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - mkdir -p $(Build.ArtifactStagingDirectory)/windows - cd kubernetes/windows - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - @{"image.name"="${{ variables.repoImageName }}:$(windowsImageTag)"} | ConvertTo-Json -Compress | Out-File -Encoding ascii $(Build.ArtifactStagingDirectory)/windows/metadata.json - if ("$(Build.Reason)" -ne "PullRequest") { - docker manifest create ${{ variables.repoImageName }}:$(windowsImageTag) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) - docker manifest push ${{ variables.repoImageName }}:$(windowsImageTag) - Write-Host "##vso[task.logissue type=warning]Windows image built with tag: ${{ variables.repoImageName }}:$(windowsImageTag)" - } - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generation Task' - condition: eq(variables.IS_PR, true) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' - DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019,mcr.microsoft.com/windows/servercore:ltsc2022' - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generation Task' - condition: eq(variables.IS_PR, false) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' - DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019,mcr.microsoft.com/windows/servercore:ltsc2022,${{ variables.repoImageName }}:$(windowsImageTag)' - - powershell: | - curl.exe -sLO "https://github.com/oras-project/oras/releases/download/v1.0.0/oras_1.0.0_windows_amd64.zip" - $currentDirectory = Get-Location - Expand-Archive -Path $currentDirectory\oras_1.0.0_windows_amd64.zip -DestinationPath . -Force - New-Item -ItemType Directory -Force -Path $env:USERPROFILE\bin - Copy-Item -Path $currentDirectory\oras.exe -Destination "$env:USERPROFILE\bin\" - $env:PATH = "$env:USERPROFILE\bin;$env:PATH" - $output = oras manifest fetch ${{ variables.repoImageName }}:$(windowsImageTag) --descriptor - $outputObject = $output | ConvertFrom-Json - $payload = @{ - targetArtifact = $outputObject - } | ConvertTo-Json -Depth 2 - $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false - [System.IO.File]::WriteAllLines("$(Build.ArtifactStagingDirectory)/windows/payload.json", $payload, $utf8NoBomEncoding) - Get-Content "$(Build.ArtifactStagingDirectory)/windows/payload.json" - workingDirectory: $(Build.ArtifactStagingDirectory)/windows - displayName: "Install oras and build the payload json file" - condition: eq(variables.IS_PR, false) - - task: EsrpCodeSigning@5 - condition: eq(variables.IS_PR, false) - inputs: - ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' - UseMSIAuthentication: true - AppRegistrationClientId: $(appRegistrationClientId) - AppRegistrationTenantId: $(appRegistrationTenantId) - EsrpClientId: $(esrpClientId) - AuthAKVName: $(authAKVName) - AuthCertName: $(authCertName) - AuthSignCertName: $(authSignCertName) - FolderPath: '$(Build.ArtifactStagingDirectory)/windows' - Pattern: 'payload.json' - signConfigType: 'inlineSignParams' - inlineOperation: | - [ - { - "keyCode": "CP-469451", - "operationSetCode": "NotaryCoseSign", - "parameters": [ - { - "parameterName": "CoseFlags", - "parameterValue": "chainunprotected" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: '60' - MaxConcurrency: '50' - MaxRetryAttempts: '5' - PendingAnalysisWaitTimeoutMinutes: '5' - displayName: 'Esrp Image Signing for windows image' - - powershell: | - $env:PATH = "$env:USERPROFILE\bin;$env:PATH" - oras attach ${{ variables.repoImageName }}:$(windowsImageTag) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\"49D6CD5DB42623144D6990AA1669CE5D97F1F3D7\"] - workingDirectory: $(Build.ArtifactStagingDirectory)/windows - displayName: "Run oras attach" - condition: eq(variables.IS_PR, false) - - powershell: | - $env:PATH = "$env:USERPROFILE\bin;$env:PATH" - oras attach ${{ variables.repoImageName }}:$(windowsImageTag) --artifact-type 'application/vnd.microsoft.artifact.lifecycle' --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$(powershell -Command "(Get-Date).AddHours(-1).ToString('yyyy-MM-ddTHH:mm:ssZ')")" - workingDirectory: $(Build.ArtifactStagingDirectory)/windows - displayName: "Run oras attach" - condition: and(eq(variables.IS_RELEASE, false), eq(variables.IS_PR, false)) - - task: AntiMalware@4 - displayName: 'Run MpCmdRun.exe' - inputs: - InputType: Basic - ScanType: CustomScan - FileDirPath: '$(Build.ArtifactStagingDirectory)' - DisableRemediation: false - AcceptableOutdatedSignatureInHours: 72 - - - job: helm_chart - displayName: "Package and Publish Helm Chart" - dependsOn: common - pool: - name: Azure-Pipelines-CI-Test-EO - image: ci-1es-managed-ubuntu-2204 - os: linux - variables: - HELM_SEMVER: $[ dependencies.common.outputs['setup.SEMVER'] ] - IMAGE_TAG: $[ dependencies.common.outputs['setup.linuxImagetag'] ] - IMAGE_TAG_WINDOWS: $[ dependencies.common.outputs['setup.windowsImageTag'] ] - skipComponentGovernanceDetection: true - Codeql.SkipTaskAutoInjection: true - templateContext: - outputs: - - output: pipelineArtifact - targetPath: '$(Build.ArtifactStagingDirectory)/helm-chart' - artifactName: helm-chart-drop - condition: and(succeeded(), eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) - steps: - - task: HelmInstaller@1 - displayName: "Install Helm" - inputs: - helmVersionToInstall: 'latest' - - bash: | - set -e - mkdir -p $(Build.ArtifactStagingDirectory)/helm-chart - # Generate the chart from the committed templates using the SEMVER-derived - # tags, then lint + package. The chart is pushed to the preview chart repo on - # every build (mirrors ama-metrics, whose chart push is ungated); only the ADO - # pipeline artifact publish is gated to non-PR ci_prod builds (see the gated - # output above). - export HELM_SEMVER="$(HELM_SEMVER)" - export IMAGE_TAG="$(IMAGE_TAG)" - export IMAGE_TAG_WINDOWS="$(IMAGE_TAG_WINDOWS)" - cd $(Build.SourcesDirectory)/charts/azuremonitor-containerinsights - envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < Chart-template.yaml > Chart.yaml - envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < values-template.yaml > values.yaml - echo "----- generated Chart.yaml -----" - cat Chart.yaml - helm lint . - helm package . --version "$(HELM_SEMVER)" --destination $(Build.ArtifactStagingDirectory)/helm-chart - # Package the same chart a second time under an -arc prerelease version so the - # build publishes both an AKS-extension tag () and an Arc tag - # (-arc) to the preview chart repo, mirroring the ama-metrics build. The - # prod release promotes both tags to the ciprod chart repo. - helm package . --version "$(HELM_SEMVER)-arc" --destination $(Build.ArtifactStagingDirectory)/helm-chart - ls -l $(Build.ArtifactStagingDirectory)/helm-chart - displayName: "Generate, lint and package Helm chart" - - task: AzureCLI@2 - displayName: "Push Helm chart to ACR" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - CHART_TGZ="$(Build.ArtifactStagingDirectory)/helm-chart/azuremonitor-containers-$(HELM_SEMVER).tgz" - CHART_TGZ_ARC="$(Build.ArtifactStagingDirectory)/helm-chart/azuremonitor-containers-$(HELM_SEMVER)-arc.tgz" - helm push "$CHART_TGZ" oci://${{ variables.chartRepoName }} - # Arc tag: same chart, pushed under -arc for the Arc K8s extension path. - helm push "$CHART_TGZ_ARC" oci://${{ variables.chartRepoName }} - echo "{\"helmChartName\":\"azuremonitor-containers\",\"helmChartVersion\":\"$(HELM_SEMVER)\",\"helmChartRef\":\"oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER)\",\"helmChartArcRef\":\"oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER)-arc\"}" > $(Build.ArtifactStagingDirectory)/helm-chart/metadata.json - cat $(Build.ArtifactStagingDirectory)/helm-chart/metadata.json - echo "##vso[task.logissue type=warning]Helm chart pushed: oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER) and :$(HELM_SEMVER)-arc" - - stage: Deploy_and_Test_Images_In_Dev_Clusters - displayName: Deploy and Test Images in Dev Clusters + displayName: Validate Existing Agent Telemetry lockBehavior: sequential - dependsOn: - - stage - condition: | - and(eq(dependencies.stage.result, 'Succeeded'), eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + dependsOn: [] + condition: and(eq(variables['Build.Reason'], 'Manual'), eq(variables['Build.SourceBranch'], 'refs/heads/suyadav/add-heartbeat-test')) variables: # Override the helm chart's default image repository (/azuremonitor/containerinsights/ciprod) to use cidev ImageRepositoryOverride: '/azuremonitor/containerinsights/cidev' - # Use image tags built from the previous build stage - linuxImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxImagetag']] - windowsImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.windowsImageTag']] + linuxImageTagUnderTest: ${{ parameters.telemetryValidationVersion }} + windowsImageTagUnderTest: win-${{ parameters.telemetryValidationVersion }} jobs: # ============================================================ # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Deploy via Helm # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-prod-aks-geneva-integration-multi-tenancy' - resourceGroup: 'ci-logs-prod-aks' - region: 'uksouth' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(GENEVA_INTEGRATION_MULTI_TENANCY_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-GenevaIntegrationMultiTenancy' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - - # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-geneva-integration-multi-tenancy' @@ -1074,20 +77,6 @@ extends: # ============================================================ # Cluster: ci-logs-prod-aks-work-load-identity — Deploy via Helm # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-prod-aks-work-load-identity' - resourceGroup: 'ci-logs-prod-aks' - region: 'eastus2' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(WORK_LOAD_IDENTITY_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-WorkLoadIdentity' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - - # Cluster: ci-logs-prod-aks-work-load-identity — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-work-load-identity' @@ -1103,20 +92,6 @@ extends: # ============================================================ # Cluster: ci-logs-prod-wcus-fips — Deploy via Helm # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-prod-wcus-fips' - resourceGroup: 'ci-logs-prod-aks' - region: 'westcentralus' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(WCUS_FIPS_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-WcusFips' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - - # Cluster: ci-logs-prod-wcus-fips — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-wcus-fips' @@ -1132,21 +107,6 @@ extends: # ============================================================ # Cluster: ci-logs-prod-aks-networkflowlogs — Deploy via Helm # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-prod-aks-networkflowlogs' - resourceGroup: 'ci-logs-prod-aks' - region: 'westus2' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(NETWORKFLOWLOGS_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-NetworkFlowLogs' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - additionalOverrides: 'OmsAgent.retinaFlowLogsEnabled=true' - - # Cluster: ci-logs-prod-aks-networkflowlogs — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-networkflowlogs' @@ -1163,42 +123,6 @@ extends: # Cluster: ci-logs-dev-aks-std-prof-config-test1 — Deploy via Helm # This cluster is used for ConfigMap testing # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-dev-aks-std-prof-config-test1' - resourceGroup: 'ci-logs-dev-aks-configmap-test' - region: 'westus2' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(CI-LOGS-DEV-AKS-CONFIGMAP-TEST_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-Dev-Aks-Configmap-Test1' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - - # ============================================================ - # Cluster: ci-logs-dev-aks-all-nodes — Deploy via Helm - # Matrix-coverage cluster with one node per OS / arch / FIPS combo - # (Ubuntu, AzureLinux, Windows2022; amd64 + arm64; non-FIPS + FIPS). - # Used to validate ama-logs works across different type of node variants. - # ============================================================ - - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self - parameters: - clusterName: 'ci-logs-dev-aks-all-nodes' - resourceGroup: 'ci-logs-dev-aks-eastus' - region: 'eastus' - subscriptionId: $(CI_BUILD_SUB_ID) - workspaceId: $(CI_LOGS_DEV_AKS_EASTUS_LAW_ID) - amalogsLinuxImage: $(linuxImageTagUnderTest) - amalogsWindowsImage: $(windowsImageTagUnderTest) - imageRepository: $(ImageRepositoryOverride) - environment: 'CI-Agent-Dev-Aks-All-Nodes' - azureSubscription: 'ContainerInsights_Build_Subscription_CI' - - # Cluster: ci-logs-dev-aks-all-nodes — Run E2E Tests - # PerNodeLogCoverage=true enables the per-node ContainerLogV2 ingestion - # check; this cluster has a log generator deployed on every node so - # every node is expected to ship logs. - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-dev-aks-all-nodes' diff --git a/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml b/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml index 39cfe0e663..ce357254f7 100644 --- a/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml +++ b/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml @@ -26,7 +26,7 @@ jobs: - deployment: Test_${{ replace(parameters.clusterName, '-', '_') }} displayName: 'Test: ${{ parameters.clusterName }}' environment: ${{ parameters.environmentName }} - dependsOn: ${{ parameters.dependsOnDeployJob }} + dependsOn: [] pool: name: Azure-Pipelines-CI-Test-EO image: ci-1es-managed-ubuntu-2204 @@ -61,6 +61,7 @@ jobs: - task: Bash@3 displayName: 'Wait for logs to be ingested into Log Analytics (20 min)' + condition: 'false' inputs: targetType: 'inline' script: | @@ -91,6 +92,8 @@ jobs: echo "========================================" - bash: | + set -euo pipefail + : "${AGENT_TELEMETRY_RESOURCE_ID:?AGENT_TELEMETRY_RESOURCE_ID must be configured}" echo "Running tests for cluster: ${{ parameters.clusterName }}" chmod +x ./install-and-execute-testkube-tests.sh diff --git a/test/testkube/install-and-execute-testkube-tests.sh b/test/testkube/install-and-execute-testkube-tests.sh index 4a38acc026..4c88f4ceb3 100644 --- a/test/testkube/install-and-execute-testkube-tests.sh +++ b/test/testkube/install-and-execute-testkube-tests.sh @@ -79,16 +79,9 @@ echo "Wait for cluster to be ready" sleep 300 echo "Run testkube testworkflows" -workflows=() +workflows=("querylogs") failed_workflows=() successful_workflows=() -if [[ $LinuxTestsOnly == "true" ]]; then - echo "Running Linux tests only" - workflows=("containerstatus-linux" "querylogs") -else - echo "Running all tests" - workflows=("containerstatus-linux" "containerstatus-windows" "querylogs") -fi for wf in "${workflows[@]}"; do echo "Running workflow: $wf" @@ -198,7 +191,7 @@ for wf in "${workflows[@]}"; do } EOF ) - curl -X POST -H "Content-Type: application/json" -d "$payload" $WEBHOOK_URI + # curl -X POST -H "Content-Type: application/json" -d "$payload" $WEBHOOK_URI # Track the failed workflow for summary reporting failed_workflows+=("${wf} (execution: ${execution_id})") diff --git a/test/testkube/testkube-test-crs.yaml b/test/testkube/testkube-test-crs.yaml index 674ef7797e..1f38059b0a 100644 --- a/test/testkube/testkube-test-crs.yaml +++ b/test/testkube/testkube-test-crs.yaml @@ -145,7 +145,7 @@ spec: content: git: uri: https://github.com/microsoft/Docker-Provider/ - revision: ci_prod + revision: suyadav/add-heartbeat-test paths: - test/ginkgo-e2e steps: @@ -168,7 +168,7 @@ spec: value: "{{config.AGENT_TELEMETRY_VERSION}}" - name: GOTOOLCHAIN value: "{{config.GOTOOLCHAIN}}" - shell: ginkgo ./querylogs + shell: ginkgo --focus="When querying the agent telemetry" ./querylogs pod: nodeSelector: kubernetes.io/os: linux From e8e5cdcb081655ede449d2c72a2b66299da9f926 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Wed, 23 Sep 2026 01:44:50 +0000 Subject: [PATCH 6/8] Revert temporary telemetry-only pipeline validation [skip ci] This reverts commit 5462df9ff521894f62c13468143f1a06beb160ef after live telemetry validation in pipeline 444. Normal build, deployment, and Testkube execution are restored; permanent telemetry checks remain. --- .pipelines/azure_pipeline_mergedbranches.yaml | 1098 ++++++++++++++++- .../test-ci-image-in-aks-cluster.yml | 5 +- .../install-and-execute-testkube-tests.sh | 11 +- test/testkube/testkube-test-crs.yaml | 4 +- 4 files changed, 1099 insertions(+), 19 deletions(-) diff --git a/.pipelines/azure_pipeline_mergedbranches.yaml b/.pipelines/azure_pipeline_mergedbranches.yaml index d7fcb6fcea..362d96e5ff 100644 --- a/.pipelines/azure_pipeline_mergedbranches.yaml +++ b/.pipelines/azure_pipeline_mergedbranches.yaml @@ -1,9 +1,16 @@ -trigger: none -pr: none -parameters: -- name: telemetryValidationVersion - type: string - default: '3.8.0-ci-prod-09-20-2026-8e6bef56' +trigger: + batch: true + branches: + include: + - ci_prod + - auto/upgrade-telegraf-* + - auto/upgrade-go-* + - dependabot/* +pr: + autoCancel: true + branches: + include: + - ci_prod variables: armServiceConnectionName: 'ci-1es-acr-connection' subscription: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' @@ -48,20 +55,1010 @@ extends: customBuildTags: - ES365AIMigrationTooling stages: + - stage: stage + displayName: 'Build and Publish Container Images' + jobs: + - job: common + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-ubuntu-2204 + os: linux + variables: + skipComponentGovernanceDetection: true + Codeql.SkipTaskAutoInjection: true + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: drop + steps: + - task: ComponentGovernanceComponentDetection@0 + - bash: | + # Derive SEMVER from the repo-root VERSION file instead of git tags. + # Format: --- + if [ "$(IS_PR)" == "True" ]; then + BRANCH_NAME=$(System.PullRequest.SourceBranch) + else + BRANCH_NAME=$(Build.SourceBranch) + BRANCH_NAME=${BRANCH_NAME#refs/heads/} + fi + BRANCH_NAME=$(echo "$BRANCH_NAME" | tr / - | tr . - | tr _ - | cut -c1-90) + COMMIT_SHA=$(echo "$(Build.SourceVersion)" | cut -b -8) + DATE=$(TZ=America/Los_Angeles date +%m-%d-%Y) + VERSION=$(cat $(Build.SourcesDirectory)/VERSION) + SEMVER=$VERSION-$BRANCH_NAME-$DATE-$COMMIT_SHA + linuxImagetag=$SEMVER + windowsImageTag=win-$SEMVER + telemetryTag=$linuxImagetag + if [ -z "$TELEMETRY_TAG" ] + then + echo "\$TELEMETRY_TAG variable is not set" + else + telemetryTag=$TELEMETRY_TAG + echo "\$TELEMETRY_TAG is $TELEMETRY_TAG" + fi + linuxTelemetryTag="$telemetryTag" + windowsTelemetryTag=win-"$telemetryTag" + echo "linuxImagetag is $linuxImagetag" + echo "windowsImageTag is $windowsImageTag" + echo "linuxTelemetryTag is $linuxTelemetryTag" + echo "windowsTelemetryTag is $windowsTelemetryTag" + echo "##vso[task.setvariable variable=linuxImagetag;isOutput=true]$linuxImagetag" + echo "##vso[task.setvariable variable=windowsImageTag;isOutput=true]$windowsImageTag" + echo "##vso[task.setvariable variable=linuxTelemetryTag;isOutput=true]$linuxTelemetryTag" + echo "##vso[task.setvariable variable=windowsTelemetryTag;isOutput=true]$windowsTelemetryTag" + echo "SEMVER is $SEMVER" + echo "##vso[task.setvariable variable=SEMVER;isOutput=true]$SEMVER" + # Set the pipeline run number to SEMVER so downstream release pipelines + # can consume this exact build version via + # resources.pipeline..runName (mirrors ama-metrics). A release then + # just picks a build run and uses whatever version is inside it, instead + # of hand-setting an image tag suffix. NOTE: the $(Build.BuildNumber) + # macros in the buildver.txt writes below are already expanded to the + # original unique run number at step start, so this override does not + # affect Ev2 artifact-version uniqueness. + echo "##vso[build.updatebuildnumber]$SEMVER" + echo "appRegistrationClientId is $APP_REGISTRATION_CLIENT_ID" + echo "appRegistrationTenantId is $APP_REGISTRATION_TENANT_ID" + echo "authAKVName is $AUTH_AKV_NAME" + echo "authCertName is $AUTH_CERT_NAME" + echo "authSignCertName is $AUTH_SIGN_CERT_NAME" + # Generate Chart.yaml/values.yaml from the committed *-template.yaml files + # so the Ev2 source tar + CopyFiles below include the SEMVER-stamped chart + # for the downstream release pipeline. Only templates are committed. + export HELM_SEMVER="$SEMVER" + export IMAGE_TAG="$linuxImagetag" + export IMAGE_TAG_WINDOWS="$windowsImageTag" + CI_CHART_DIR="$(Build.SourcesDirectory)/charts/azuremonitor-containerinsights" + envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < "$CI_CHART_DIR/Chart-template.yaml" > "$CI_CHART_DIR/Chart.yaml" + envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < "$CI_CHART_DIR/values-template.yaml" > "$CI_CHART_DIR/values.yaml" + cd $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment-Managed-SDP/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh pushChartToAcr.sh + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ ../../../../charts/azuremonitor-containerinsights/ pushChartToAcr.sh + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-Managed-SDP/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ ../../../../charts/azuremonitor-containerinsights/ pushChartToAcr.sh + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz arcExtensionRelease.sh + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz arcExtensionRelease.sh + # Stamp a unique Ev2 artifacts version per build. Ev2 dedups artifact + # registration on this versionFile; a static buildver.txt makes Ev2 skip + # uploading new artifacts ("already registered"), freezing the published + # chart at whatever was first registered. Writing $(Build.BuildNumber) + # forces fresh registration on every build. + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment/ServiceGroupRoot/buildver.txt + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/mergebranch-multiarch-agent-deployment-Managed-SDP/ServiceGroupRoot/buildver.txt + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/buildver.txt + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-Managed-SDP/ServiceGroupRoot/buildver.txt + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2/ServiceGroupRoot/buildver.txt + echo $(Build.BuildNumber) > $(Build.SourcesDirectory)/deployment/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot/buildver.txt + windowsAMAUrl="" + if [ -z "$WINDOWS_AMA_URL" ] + then + echo "\$WINDOWS_AMA_URL variable is not set" + else + windowsAMAUrl=$WINDOWS_AMA_URL + echo "\$WINDOWS_AMA_URL is $WINDOWS_AMA_URL" + fi + echo "##vso[task.setvariable variable=windowsAMAUrl;isOutput=true]$windowsAMAUrl" + name: setup + - task: CredScan@3 + displayName: "SDL : Run credscan" + - task: CopyFiles@2 + displayName: "Copy ev2 deployment artifacts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/deployment" + Contents: | + **/* + !**/ScanTelemetry_*.json + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/.pipelines" + Contents: | + **/*.sh + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/kubernetes" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/charts" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/test/e2e" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + - task: Armory@2 + displayName: 'Run ARMory' + inputs: + toolVersion: Latest + targetDirectory: '$(Build.SourcesDirectory)' + - job: build_linux + # the emulated linux/arm64 leg is slow, and setup.sh retries native gem builds that + # segfault under emulation, so leave headroom rather than failing on the job timeout + timeoutInMinutes: 180 + dependsOn: common + variables: + linuxImagetag: $[ dependencies.common.outputs['setup.linuxImagetag'] ] + linuxTelemetryTag: $[ dependencies.common.outputs['setup.linuxTelemetryTag'] ] + Codeql.Enabled: true + Codeql.BuildIdentifier: 'linuxbuild' + DOCKER_BUILDKIT: 1 + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: linux-drop + steps: + - task: CodeQL3000Init@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - task: AzureCLI@2 + displayName: "Multi-arch Linux build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/linux + docker system prune --all -f + docker images -q --filter "dangling=true" | xargs docker rmi + # Register binfmt handlers for cross-arch (linux/arm64) emulation. + # NOTE: this previously used `multiarch/qemu-user-static --reset -p yes`, which pins the + # QEMU binary from that image into the kernel (-p yes => binfmt_misc 'F' flag). That image + # is abandoned (last push 2023-01, QEMU 7.2), and the apt qemu-user-static it sat on top of + # is frozen at QEMU 6.2 on ubuntu-22.04 -- apt only backports fixes, never new upstream + # versions. Emulated gcc segfaults at random on those, which broke the arm64 leg roughly + # half the time while compiling the ruby native gem extensions in kubernetes/linux/setup.sh. + # tonistiigi/binfmt tracks current QEMU releases and is what docker/setup-qemu-action uses. + # Pulled from the MCR mirror to avoid Docker Hub rate limiting. + docker run --rm --privileged $(QEMU_BINFMT_IMAGE) --uninstall 'qemu-*' || true + docker run --rm --privileged $(QEMU_BINFMT_IMAGE) --install all || exit 1 + # fail fast (instead of 30+ minutes into the arm64 build) if emulation is not usable. + # NOTE: this script does not run under 'set -e', so check explicitly. + docker run --rm --platform linux/arm64 mcr.microsoft.com/azurelinux/base/core:3.0 uname -m || exit 1 + docker buildx create --name testbuilder + docker buildx use testbuilder + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + # NOTE: Using the prometheus-collector team's cached buildx image since moby/buildkit:buildx-stable-1 getting throttled + docker pull mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1 + docker buildx create --name dockerbuilder --driver docker-container --driver-opt image=mcr.microsoft.com/azuremonitor/containerinsights/cidev/prometheus-collector/images:buildx-stable-1 --use + docker buildx inspect --bootstrap + if [ "$(Build.Reason)" != "PullRequest" ]; then + docker buildx build --platform $(BUILD_PLATFORMS) --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --push --provenance=false . + echo "##vso[task.logissue type=warning]Linux image built with tag: ${{ variables.repoImageName }}:$(linuxImagetag)" + docker pull ${{ variables.repoImageName }}:$(linuxImagetag) + else + docker buildx build --platform $(BUILD_PLATFORMS) --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --provenance=false . + # load the multi-arch image to run tests + docker buildx build --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxTelemetryTag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --load --provenance=false . + fi + - bash: | + curl -LO "https://github.com/oras-project/oras/releases/download/v1.0.0/oras_1.0.0_linux_amd64.tar.gz" + mkdir -p oras-install/ + tar -zxf oras_1.0.0_*.tar.gz -C oras-install/ + sudo mv oras-install/oras /usr/local/bin/ + rm -rf oras_1.0.0_*.tar.gz oras-install/ + TARGET_ARTIFACT=$(oras manifest fetch ${{ variables.repoImageName }}:$(linuxImagetag) --descriptor) + cat <>$(Build.ArtifactStagingDirectory)/linux/payload.json + {"targetArtifact":$TARGET_ARTIFACT} + EOF + cat $(Build.ArtifactStagingDirectory)/linux/payload.json + workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ + displayName: "Install oras and build the payload json file" + condition: eq(variables.IS_PR, false) + - task: EsrpCodeSigning@5 + condition: eq(variables.IS_PR, false) + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/linux' + Pattern: 'payload.json' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "keyCode": "CP-469451", + "operationSetCode": "NotaryCoseSign", + "parameters": [ + { + "parameterName": "CoseFlags", + "parameterValue": "chainunprotected" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + PendingAnalysisWaitTimeoutMinutes: '5' + displayName: 'Esrp Image Signing for linux image' + - bash: | + set -euxo pipefail + oras attach ${{ variables.repoImageName }}:$(linuxImagetag) \ + --artifact-type 'application/vnd.cncf.notary.signature' \ + ./payload.json:application/cose \ + -a "io.cncf.notary.x509chain.thumbprint#S256=[\"49D6CD5DB42623144D6990AA1669CE5D97F1F3D7\"]" + workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ + displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" + condition: eq(variables.IS_PR, false) + - bash: | + set -euxo pipefail + oras attach ${{ variables.repoImageName }}:$(linuxImagetag) \ + --artifact-type 'application/vnd.microsoft.artifact.lifecycle' \ + --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$(date -u -d '-1 hour' +"%Y-%m-%dT%H:%M:%SZ")" + workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ + displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" + condition: and(eq(variables.IS_RELEASE, false), eq(variables.IS_PR, false)) + - task: AzureCLI@2 + displayName: "Vulnerability Scan with Trivy" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + PRIMARY_TRIVY_DB_REPOSITORY="ghcr.io/aquasecurity/trivy-db" + SECONDARY_TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db" + PRIMARY_TRIVY_JAVA_DB_REPOSITORY="ghcr.io/aquasecurity/trivy-java-db" + SECONDARY_TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db" + # Set initial repositories to primary + export TRIVY_DB_REPOSITORY=$PRIMARY_TRIVY_DB_REPOSITORY + export TRIVY_JAVA_DB_REPOSITORY=$PRIMARY_TRIVY_JAVA_DB_REPOSITORY + # Function to run Trivy scan and handle output + run_trivy_scan() { + trivy image --exit-code 0 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM "${{ variables.repoImageName }}:$(linuxImagetag)" > trivy_output.log 2>&1 + return $? + } + # Attempt scan up to 5 times with repository fallback + for i in {1..5}; do + echo "Running Trivy scan attempt $i" + # Run the Trivy scan and capture exit code + run_trivy_scan + TRIVY_EXIT_CODE=$? + # Check if scan was successful + if [ $TRIVY_EXIT_CODE -eq 0 ]; then + echo "Trivy scan succeeded." + cat trivy_output.log + break + fi + # If the first attempt fails, switch to secondary repositories + if [ $i -eq 1 ]; then + echo "Primary repositories failed with an error. Switching to secondary repositories." + export TRIVY_DB_REPOSITORY=$SECONDARY_TRIVY_DB_REPOSITORY + export TRIVY_JAVA_DB_REPOSITORY=$SECONDARY_TRIVY_JAVA_DB_REPOSITORY + fi + # Log and wait before retrying if an error occurred + echo "Error: Trivy scan attempt $i failed. Retrying ($i/5)" + cat trivy_output.log + sleep 5 # Wait 5 seconds before retrying + done + # Final check: if still failing after 5 attempts, exit with error + if [ $TRIVY_EXIT_CODE -ne 0 ]; then + echo "Error: Trivy scan failed after 5 retries." + exit 1 + fi + - task: GoTool@0 + inputs: + version: '1.23.8' + - bash: | + cd $(System.DefaultWorkingDirectory)/build/linux/ + ls + make + displayName: 'Execute Makefile for Linux Build' + - task: CodeQL3000Finalize@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' + DockerImagesToScan: '$(GOLANG_BASE_IMAGE),$(CI_BASE_IMAGE),${{ variables.repoImageName }}:$(linuxImagetag)' + - bash: | + dockerImagesToScan='$(GOLANG_BASE_IMAGE),$(CI_BASE_IMAGE),${{ variables.repoImageName }}:$(linuxImagetag)' + echo "Docker images to scan: $dockerImagesToScan" + echo "##vso[task.setvariable variable=dockerImagesToScan]$dockerImagesToScan" + displayName: 'Set Docker images to scan' + - job: build_windows_2019 + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-windows-2022 + os: windows + timeoutInMinutes: 120 + dependsOn: + - common + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] + windows2019BaseImageVersion: ltsc2019 + Codeql.Enabled: true + Codeql.BuildIdentifier: 'windowsbuild' + windowsAMAUrl: $[ dependencies.common.outputs['setup.windowsAMAUrl'] ] + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + - task: CodeQL3000Init@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to CFS NuGet feed' + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + - task: AzureCLI@2 + displayName: "Set up acr for windows ltsc2019 build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + retryCountOnTaskFailure: 2 + inlineScript: | + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + # Check if directory exists and remove it before creating it to handle retry scenarios + if (Test-Path -Path "$(Build.ArtifactStagingDirectory)/windows") { + Remove-Item -Path "$(Build.ArtifactStagingDirectory)/windows" -Recurse -Force + } + New-Item -Path "$(Build.ArtifactStagingDirectory)/windows" -ItemType Directory -Force + cd kubernetes/windows + # Build the Docker image + docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned --build-arg WINDOWS_VERSION=$(windows2019BaseImageVersion) --build-arg IMAGE_TAG=$(windowsTelemetryTag) --build-arg WINDOWS_AMA_URL=$(windowsAMAUrl) . + displayName: "Docker windows build for ltsc2019" + retryCountOnTaskFailure: 2 + - task: PowerShell@2 + displayName: Extract files to sign + inputs: + targetType: 'inline' + script: | + echo "Creating docker container" + docker create --name signingContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned + echo "Creating fist party directory" + mkdir -p $(Build.ArtifactStagingDirectory)/fpSigning + cd $(Build.ArtifactStagingDirectory)/fpSigning + echo "Extract CertificateGenerator" + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe . + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll . + echo "Extract LivenessProbe" + docker cp signingContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe . + echo "Extract ps scripts" + docker cp signingContainer:C:\opt\amalogswindows\scripts\powershell . + echo "Creating OSS directory" + mkdir -p $(Build.ArtifactStagingDirectory)/ossSigning + cd $(Build.ArtifactStagingDirectory)/ossSigning + echo "Extract CertificateGenerator" + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll . + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll . + echo "Extract fluent-bit" + docker cp signingContainer:C:\opt\fluent-bit . + echo "Extract Ruby" + docker cp signingContainer:C:\ruby31 . + echo "Extract telegraf" + docker cp signingContainer:C:\opt\telegraf\telegraf.exe . + echo "Extract out oms" + docker cp signingContainer:C:\opt\amalogswindows\out_oms.so . + echo "Extract containerinventory.so" + docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so . + echo "Extract perf.so" + docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so . + echo "Removing container" + docker rm signingContainer + echo "List ArtifactStagingDirectory" + ls $(Build.ArtifactStagingDirectory) + ls . + - script: dir $(System.DefaultWorkingDirectory) + displayName: 'List files in DefaultWorking Directory' + - script: dir $(Build.ArtifactStagingDirectory) + displayName: 'List files in Staging Directory' + - task: EsrpCodeSigning@5 + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/fpSigning' + Pattern: '*.dll,*.exe,*.so,*.ps1' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "KeyCode" : "CP-230012", + "OperationCode" : "SigntoolSign", + "Parameters" : { + "OpusName" : "Microsoft", + "OpusInfo" : "http://www.microsoft.com", + "FileDigest" : "/fd \"SHA256\"", + "PageHash" : "/NPH", + "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-230012", + "OperationCode" : "SigntoolVerify", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + displayName: 'EsrpCodeSigning for first party' + - task: EsrpCodeSigning@5 + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/ossSigning' + Pattern: '*.dll,*.exe,*.so' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "KeyCode" : "CP-231522", + "OperationCode" : "SigntoolSign", + "Parameters" : { + "OpusName" : "Microsoft", + "OpusInfo" : "http://www.microsoft.com", + "Append" : "/as", + "FileDigest" : "/fd \"SHA256\"", + "PageHash" : "/NPH", + "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-231522", + "OperationCode" : "SigntoolVerify", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + displayName: 'EsrpCodeSigning for OSS' + - task: BinSkim@4 + displayName: 'SDL: run binskim' + inputs: + InputType: 'CommandLine' + arguments: 'analyze --rich-return-code $(Build.ArtifactStagingDirectory)\ossSigning\out_oms.so $(Build.ArtifactStagingDirectory)\ossSigning\perf.so $(Build.ArtifactStagingDirectory)\ossSigning\containerinventory.so $(Build.ArtifactStagingDirectory)\fpSigning\livenessprobe.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.dll' + retryCountOnTaskFailure: 1 + - task: PoliCheck@2 + displayName: "SDL : Run PoliCheck" + inputs: + targetType: 'F' + targetArgument: '$(Build.SourcesDirectory)' + - task: PowerShell@2 + displayName: Replace files in origin Image + inputs: + targetType: 'inline' + script: | + docker create --name pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion)-unsigned + echo "Copy Signed binaries/folders back to docker image" + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.exe pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.dll pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/livenessprobe.exe pushContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/powershell/. pushContainer:C:\opt\amalogswindows\scripts\powershell/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/BouncyCastle.Crypto.dll pushContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/Newtonsoft.Json.dll pushContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/fluent-bit/. pushContainer:C:\opt\fluent-bit/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/ruby31/. pushContainer:C:\ruby31/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/telegraf.exe pushContainer:C:\opt\telegraf\telegraf.exe + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/out_oms.so pushContainer:C:\opt\amalogswindows\out_oms.so + docker commit pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) + docker rm pushContainer + - task: PowerShell@2 + displayName: Push Docker Image + inputs: + targetType: 'inline' + script: | + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) + } + - task: CodeQL3000Finalize@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - job: build_windows_2022 + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-windows-2022 + os: windows + timeoutInMinutes: 120 + dependsOn: + - common + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] + windows2022BaseImageVersion: ltsc2022 + Codeql.SkipTaskAutoInjection: true + windowsAMAUrl: $[ dependencies.common.outputs['setup.windowsAMAUrl'] ] + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + - task: CodeQL3000Init@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to CFS NuGet feed' + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2022" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + retryCountOnTaskFailure: 2 + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned --build-arg WINDOWS_VERSION=$(windows2022BaseImageVersion) --build-arg IMAGE_TAG=$(windowsTelemetryTag) --build-arg WINDOWS_AMA_URL=$(windowsAMAUrl) . + - task: PowerShell@2 + displayName: Extract files to sign + inputs: + targetType: 'inline' + script: | + echo "Creating docker container" + docker create --name signingContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned + echo "Creating fist party directory" + mkdir -p $(Build.ArtifactStagingDirectory)/fpSigning + cd $(Build.ArtifactStagingDirectory)/fpSigning + echo "Extract CertificateGenerator" + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe . + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll . + echo "Extract LivenessProbe" + docker cp signingContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe . + echo "Extract ps scripts" + docker cp signingContainer:C:\opt\amalogswindows\scripts\powershell . + echo "Creating OSS directory" + mkdir -p $(Build.ArtifactStagingDirectory)/ossSigning + cd $(Build.ArtifactStagingDirectory)/ossSigning + echo "Extract CertificateGenerator" + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll . + docker cp signingContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll . + echo "Extract fluent-bit" + docker cp signingContainer:C:\opt\fluent-bit . + echo "Extract Ruby" + docker cp signingContainer:C:\ruby31 . + echo "Extract telegraf" + docker cp signingContainer:C:\opt\telegraf\telegraf.exe . + echo "Extract out oms" + docker cp signingContainer:C:\opt\amalogswindows\out_oms.so . + echo "Extract containerinventory.so" + docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so . + echo "Extract perf.so" + docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so . + echo "Removing container" + docker rm signingContainer + echo "List ArtifactStagingDirectory" + ls $(Build.ArtifactStagingDirectory) + ls . + - script: dir $(System.DefaultWorkingDirectory) + displayName: 'List files in DefaultWorking Directory' + - script: dir $(Build.ArtifactStagingDirectory) + displayName: 'List files in Staging Directory' + - task: EsrpCodeSigning@5 + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/fpSigning' + Pattern: '*.dll,*.exe,*.so,*.ps1' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "KeyCode" : "CP-230012", + "OperationCode" : "SigntoolSign", + "Parameters" : { + "OpusName" : "Microsoft", + "OpusInfo" : "http://www.microsoft.com", + "FileDigest" : "/fd \"SHA256\"", + "PageHash" : "/NPH", + "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-230012", + "OperationCode" : "SigntoolVerify", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + displayName: 'EsrpCodeSigning for first party' + - task: EsrpCodeSigning@5 + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/ossSigning' + Pattern: '*.dll,*.exe,*.so' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "KeyCode" : "CP-231522", + "OperationCode" : "SigntoolSign", + "Parameters" : { + "OpusName" : "Microsoft", + "OpusInfo" : "http://www.microsoft.com", + "Append" : "/as", + "FileDigest" : "/fd \"SHA256\"", + "PageHash" : "/NPH", + "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-231522", + "OperationCode" : "SigntoolVerify", + "Parameters" : {}, + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + displayName: 'EsrpCodeSigning for OSS' + - task: BinSkim@4 + displayName: 'SDL: run binskim' + inputs: + InputType: 'CommandLine' + arguments: 'analyze --rich-return-code $(Build.ArtifactStagingDirectory)\ossSigning\out_oms.so $(Build.ArtifactStagingDirectory)\ossSigning\perf.so $(Build.ArtifactStagingDirectory)\ossSigning\containerinventory.so $(Build.ArtifactStagingDirectory)\fpSigning\livenessprobe.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.exe $(Build.ArtifactStagingDirectory)\fpSigning\CertificateGenerator.dll' + retryCountOnTaskFailure: 1 + - task: PoliCheck@2 + displayName: "SDL : Run PoliCheck" + inputs: + targetType: 'F' + targetArgument: '$(Build.SourcesDirectory)' + - task: PowerShell@2 + displayName: Replace files in origin Image + inputs: + targetType: 'inline' + script: | + docker create --name pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion)-unsigned + echo "Copy Signed binaries/folders back to docker image" + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.exe pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.exe + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/CertificateGenerator.dll pushContainer:C:\opt\amalogswindows\certgenerator\CertificateGenerator.dll + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/livenessprobe.exe pushContainer:C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe + docker cp $(Build.ArtifactStagingDirectory)/fpSigning/powershell/. pushContainer:C:\opt\amalogswindows\scripts\powershell/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/BouncyCastle.Crypto.dll pushContainer:C:\opt\amalogswindows\certgenerator\BouncyCastle.Crypto.dll + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/Newtonsoft.Json.dll pushContainer:C:\opt\amalogswindows\certgenerator\Newtonsoft.Json.dll + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/fluent-bit/. pushContainer:C:\opt\fluent-bit/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/ruby31/. pushContainer:C:\ruby31/ + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/telegraf.exe pushContainer:C:\opt\telegraf\telegraf.exe + docker cp $(Build.ArtifactStagingDirectory)/ossSigning/out_oms.so pushContainer:C:\opt\amalogswindows\out_oms.so + docker commit pushContainer ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + docker rm pushContainer + - task: PowerShell@2 + displayName: Push Docker Image + inputs: + targetType: 'inline' + script: | + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + } + - task: CodeQL3000Finalize@0 + condition: eq(variables.IS_MAIN_BRANCH, true) + - job: build_windows_multi_arc + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-windows-2022 + os: windows + timeoutInMinutes: 120 + dependsOn: + - common + - build_windows_2019 + - build_windows_2022 + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windowsTelemetryTag: $[ dependencies.common.outputs['setup.windowsTelemetryTag'] ] + windows2019BaseImageVersion: ltsc2019 + windows2022BaseImageVersion: ltsc2022 + Codeql.SkipTaskAutoInjection: true + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: windows-drop + steps: + - task: AzureCLI@2 + displayName: "Docker windows build for multi-arc image" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + @{"image.name"="${{ variables.repoImageName }}:$(windowsImageTag)"} | ConvertTo-Json -Compress | Out-File -Encoding ascii $(Build.ArtifactStagingDirectory)/windows/metadata.json + if ("$(Build.Reason)" -ne "PullRequest") { + docker manifest create ${{ variables.repoImageName }}:$(windowsImageTag) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + docker manifest push ${{ variables.repoImageName }}:$(windowsImageTag) + Write-Host "##vso[task.logissue type=warning]Windows image built with tag: ${{ variables.repoImageName }}:$(windowsImageTag)" + } + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, true) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019,mcr.microsoft.com/windows/servercore:ltsc2022' + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019,mcr.microsoft.com/windows/servercore:ltsc2022,${{ variables.repoImageName }}:$(windowsImageTag)' + - powershell: | + curl.exe -sLO "https://github.com/oras-project/oras/releases/download/v1.0.0/oras_1.0.0_windows_amd64.zip" + $currentDirectory = Get-Location + Expand-Archive -Path $currentDirectory\oras_1.0.0_windows_amd64.zip -DestinationPath . -Force + New-Item -ItemType Directory -Force -Path $env:USERPROFILE\bin + Copy-Item -Path $currentDirectory\oras.exe -Destination "$env:USERPROFILE\bin\" + $env:PATH = "$env:USERPROFILE\bin;$env:PATH" + $output = oras manifest fetch ${{ variables.repoImageName }}:$(windowsImageTag) --descriptor + $outputObject = $output | ConvertFrom-Json + $payload = @{ + targetArtifact = $outputObject + } | ConvertTo-Json -Depth 2 + $utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $false + [System.IO.File]::WriteAllLines("$(Build.ArtifactStagingDirectory)/windows/payload.json", $payload, $utf8NoBomEncoding) + Get-Content "$(Build.ArtifactStagingDirectory)/windows/payload.json" + workingDirectory: $(Build.ArtifactStagingDirectory)/windows + displayName: "Install oras and build the payload json file" + condition: eq(variables.IS_PR, false) + - task: EsrpCodeSigning@5 + condition: eq(variables.IS_PR, false) + inputs: + ConnectedServiceName: 'AME ESRPContainerInsights MSI FIC' + UseMSIAuthentication: true + AppRegistrationClientId: $(appRegistrationClientId) + AppRegistrationTenantId: $(appRegistrationTenantId) + EsrpClientId: $(esrpClientId) + AuthAKVName: $(authAKVName) + AuthCertName: $(authCertName) + AuthSignCertName: $(authSignCertName) + FolderPath: '$(Build.ArtifactStagingDirectory)/windows' + Pattern: 'payload.json' + signConfigType: 'inlineSignParams' + inlineOperation: | + [ + { + "keyCode": "CP-469451", + "operationSetCode": "NotaryCoseSign", + "parameters": [ + { + "parameterName": "CoseFlags", + "parameterValue": "chainunprotected" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: '60' + MaxConcurrency: '50' + MaxRetryAttempts: '5' + PendingAnalysisWaitTimeoutMinutes: '5' + displayName: 'Esrp Image Signing for windows image' + - powershell: | + $env:PATH = "$env:USERPROFILE\bin;$env:PATH" + oras attach ${{ variables.repoImageName }}:$(windowsImageTag) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\"49D6CD5DB42623144D6990AA1669CE5D97F1F3D7\"] + workingDirectory: $(Build.ArtifactStagingDirectory)/windows + displayName: "Run oras attach" + condition: eq(variables.IS_PR, false) + - powershell: | + $env:PATH = "$env:USERPROFILE\bin;$env:PATH" + oras attach ${{ variables.repoImageName }}:$(windowsImageTag) --artifact-type 'application/vnd.microsoft.artifact.lifecycle' --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$(powershell -Command "(Get-Date).AddHours(-1).ToString('yyyy-MM-ddTHH:mm:ssZ')")" + workingDirectory: $(Build.ArtifactStagingDirectory)/windows + displayName: "Run oras attach" + condition: and(eq(variables.IS_RELEASE, false), eq(variables.IS_PR, false)) + - task: AntiMalware@4 + displayName: 'Run MpCmdRun.exe' + inputs: + InputType: Basic + ScanType: CustomScan + FileDirPath: '$(Build.ArtifactStagingDirectory)' + DisableRemediation: false + AcceptableOutdatedSignatureInHours: 72 + + - job: helm_chart + displayName: "Package and Publish Helm Chart" + dependsOn: common + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-ubuntu-2204 + os: linux + variables: + HELM_SEMVER: $[ dependencies.common.outputs['setup.SEMVER'] ] + IMAGE_TAG: $[ dependencies.common.outputs['setup.linuxImagetag'] ] + IMAGE_TAG_WINDOWS: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + skipComponentGovernanceDetection: true + Codeql.SkipTaskAutoInjection: true + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/helm-chart' + artifactName: helm-chart-drop + condition: and(succeeded(), eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + steps: + - task: HelmInstaller@1 + displayName: "Install Helm" + inputs: + helmVersionToInstall: 'latest' + - bash: | + set -e + mkdir -p $(Build.ArtifactStagingDirectory)/helm-chart + # Generate the chart from the committed templates using the SEMVER-derived + # tags, then lint + package. The chart is pushed to the preview chart repo on + # every build (mirrors ama-metrics, whose chart push is ungated); only the ADO + # pipeline artifact publish is gated to non-PR ci_prod builds (see the gated + # output above). + export HELM_SEMVER="$(HELM_SEMVER)" + export IMAGE_TAG="$(IMAGE_TAG)" + export IMAGE_TAG_WINDOWS="$(IMAGE_TAG_WINDOWS)" + cd $(Build.SourcesDirectory)/charts/azuremonitor-containerinsights + envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < Chart-template.yaml > Chart.yaml + envsubst '${HELM_SEMVER} ${IMAGE_TAG} ${IMAGE_TAG_WINDOWS}' < values-template.yaml > values.yaml + echo "----- generated Chart.yaml -----" + cat Chart.yaml + helm lint . + helm package . --version "$(HELM_SEMVER)" --destination $(Build.ArtifactStagingDirectory)/helm-chart + # Package the same chart a second time under an -arc prerelease version so the + # build publishes both an AKS-extension tag () and an Arc tag + # (-arc) to the preview chart repo, mirroring the ama-metrics build. The + # prod release promotes both tags to the ciprod chart repo. + helm package . --version "$(HELM_SEMVER)-arc" --destination $(Build.ArtifactStagingDirectory)/helm-chart + ls -l $(Build.ArtifactStagingDirectory)/helm-chart + displayName: "Generate, lint and package Helm chart" + - task: AzureCLI@2 + displayName: "Push Helm chart to ACR" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + CHART_TGZ="$(Build.ArtifactStagingDirectory)/helm-chart/azuremonitor-containers-$(HELM_SEMVER).tgz" + CHART_TGZ_ARC="$(Build.ArtifactStagingDirectory)/helm-chart/azuremonitor-containers-$(HELM_SEMVER)-arc.tgz" + helm push "$CHART_TGZ" oci://${{ variables.chartRepoName }} + # Arc tag: same chart, pushed under -arc for the Arc K8s extension path. + helm push "$CHART_TGZ_ARC" oci://${{ variables.chartRepoName }} + echo "{\"helmChartName\":\"azuremonitor-containers\",\"helmChartVersion\":\"$(HELM_SEMVER)\",\"helmChartRef\":\"oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER)\",\"helmChartArcRef\":\"oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER)-arc\"}" > $(Build.ArtifactStagingDirectory)/helm-chart/metadata.json + cat $(Build.ArtifactStagingDirectory)/helm-chart/metadata.json + echo "##vso[task.logissue type=warning]Helm chart pushed: oci://${{ variables.chartRepoName }}/azuremonitor-containers:$(HELM_SEMVER) and :$(HELM_SEMVER)-arc" + - stage: Deploy_and_Test_Images_In_Dev_Clusters - displayName: Validate Existing Agent Telemetry + displayName: Deploy and Test Images in Dev Clusters lockBehavior: sequential - dependsOn: [] - condition: and(eq(variables['Build.Reason'], 'Manual'), eq(variables['Build.SourceBranch'], 'refs/heads/suyadav/add-heartbeat-test')) + dependsOn: + - stage + condition: | + and(eq(dependencies.stage.result, 'Succeeded'), eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) variables: # Override the helm chart's default image repository (/azuremonitor/containerinsights/ciprod) to use cidev ImageRepositoryOverride: '/azuremonitor/containerinsights/cidev' - linuxImageTagUnderTest: ${{ parameters.telemetryValidationVersion }} - windowsImageTagUnderTest: win-${{ parameters.telemetryValidationVersion }} + # Use image tags built from the previous build stage + linuxImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxImagetag']] + windowsImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.windowsImageTag']] jobs: # ============================================================ # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Deploy via Helm # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-prod-aks-geneva-integration-multi-tenancy' + resourceGroup: 'ci-logs-prod-aks' + region: 'uksouth' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(GENEVA_INTEGRATION_MULTI_TENANCY_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-GenevaIntegrationMultiTenancy' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + + # Cluster: ci-logs-prod-aks-geneva-integration-multi-tenancy — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-geneva-integration-multi-tenancy' @@ -77,6 +1074,20 @@ extends: # ============================================================ # Cluster: ci-logs-prod-aks-work-load-identity — Deploy via Helm # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-prod-aks-work-load-identity' + resourceGroup: 'ci-logs-prod-aks' + region: 'eastus2' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(WORK_LOAD_IDENTITY_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-WorkLoadIdentity' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + + # Cluster: ci-logs-prod-aks-work-load-identity — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-work-load-identity' @@ -92,6 +1103,20 @@ extends: # ============================================================ # Cluster: ci-logs-prod-wcus-fips — Deploy via Helm # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-prod-wcus-fips' + resourceGroup: 'ci-logs-prod-aks' + region: 'westcentralus' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(WCUS_FIPS_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-WcusFips' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + + # Cluster: ci-logs-prod-wcus-fips — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-wcus-fips' @@ -107,6 +1132,21 @@ extends: # ============================================================ # Cluster: ci-logs-prod-aks-networkflowlogs — Deploy via Helm # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-prod-aks-networkflowlogs' + resourceGroup: 'ci-logs-prod-aks' + region: 'westus2' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(NETWORKFLOWLOGS_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-NetworkFlowLogs' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + additionalOverrides: 'OmsAgent.retinaFlowLogsEnabled=true' + + # Cluster: ci-logs-prod-aks-networkflowlogs — Run E2E Tests - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-prod-aks-networkflowlogs' @@ -123,6 +1163,42 @@ extends: # Cluster: ci-logs-dev-aks-std-prof-config-test1 — Deploy via Helm # This cluster is used for ConfigMap testing # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-dev-aks-std-prof-config-test1' + resourceGroup: 'ci-logs-dev-aks-configmap-test' + region: 'westus2' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(CI-LOGS-DEV-AKS-CONFIGMAP-TEST_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-Dev-Aks-Configmap-Test1' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + + # ============================================================ + # Cluster: ci-logs-dev-aks-all-nodes — Deploy via Helm + # Matrix-coverage cluster with one node per OS / arch / FIPS combo + # (Ubuntu, AzureLinux, Windows2022; amd64 + arm64; non-FIPS + FIPS). + # Used to validate ama-logs works across different type of node variants. + # ============================================================ + - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'ci-logs-dev-aks-all-nodes' + resourceGroup: 'ci-logs-dev-aks-eastus' + region: 'eastus' + subscriptionId: $(CI_BUILD_SUB_ID) + workspaceId: $(CI_LOGS_DEV_AKS_EASTUS_LAW_ID) + amalogsLinuxImage: $(linuxImageTagUnderTest) + amalogsWindowsImage: $(windowsImageTagUnderTest) + imageRepository: $(ImageRepositoryOverride) + environment: 'CI-Agent-Dev-Aks-All-Nodes' + azureSubscription: 'ContainerInsights_Build_Subscription_CI' + + # Cluster: ci-logs-dev-aks-all-nodes — Run E2E Tests + # PerNodeLogCoverage=true enables the per-node ContainerLogV2 ingestion + # check; this cluster has a log generator deployed on every node so + # every node is expected to ship logs. - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self parameters: clusterName: 'ci-logs-dev-aks-all-nodes' diff --git a/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml b/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml index ce357254f7..39cfe0e663 100644 --- a/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml +++ b/.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml @@ -26,7 +26,7 @@ jobs: - deployment: Test_${{ replace(parameters.clusterName, '-', '_') }} displayName: 'Test: ${{ parameters.clusterName }}' environment: ${{ parameters.environmentName }} - dependsOn: [] + dependsOn: ${{ parameters.dependsOnDeployJob }} pool: name: Azure-Pipelines-CI-Test-EO image: ci-1es-managed-ubuntu-2204 @@ -61,7 +61,6 @@ jobs: - task: Bash@3 displayName: 'Wait for logs to be ingested into Log Analytics (20 min)' - condition: 'false' inputs: targetType: 'inline' script: | @@ -92,8 +91,6 @@ jobs: echo "========================================" - bash: | - set -euo pipefail - : "${AGENT_TELEMETRY_RESOURCE_ID:?AGENT_TELEMETRY_RESOURCE_ID must be configured}" echo "Running tests for cluster: ${{ parameters.clusterName }}" chmod +x ./install-and-execute-testkube-tests.sh diff --git a/test/testkube/install-and-execute-testkube-tests.sh b/test/testkube/install-and-execute-testkube-tests.sh index 4c88f4ceb3..4a38acc026 100644 --- a/test/testkube/install-and-execute-testkube-tests.sh +++ b/test/testkube/install-and-execute-testkube-tests.sh @@ -79,9 +79,16 @@ echo "Wait for cluster to be ready" sleep 300 echo "Run testkube testworkflows" -workflows=("querylogs") +workflows=() failed_workflows=() successful_workflows=() +if [[ $LinuxTestsOnly == "true" ]]; then + echo "Running Linux tests only" + workflows=("containerstatus-linux" "querylogs") +else + echo "Running all tests" + workflows=("containerstatus-linux" "containerstatus-windows" "querylogs") +fi for wf in "${workflows[@]}"; do echo "Running workflow: $wf" @@ -191,7 +198,7 @@ for wf in "${workflows[@]}"; do } EOF ) - # curl -X POST -H "Content-Type: application/json" -d "$payload" $WEBHOOK_URI + curl -X POST -H "Content-Type: application/json" -d "$payload" $WEBHOOK_URI # Track the failed workflow for summary reporting failed_workflows+=("${wf} (execution: ${execution_id})") diff --git a/test/testkube/testkube-test-crs.yaml b/test/testkube/testkube-test-crs.yaml index 1f38059b0a..674ef7797e 100644 --- a/test/testkube/testkube-test-crs.yaml +++ b/test/testkube/testkube-test-crs.yaml @@ -145,7 +145,7 @@ spec: content: git: uri: https://github.com/microsoft/Docker-Provider/ - revision: suyadav/add-heartbeat-test + revision: ci_prod paths: - test/ginkgo-e2e steps: @@ -168,7 +168,7 @@ spec: value: "{{config.AGENT_TELEMETRY_VERSION}}" - name: GOTOOLCHAIN value: "{{config.GOTOOLCHAIN}}" - shell: ginkgo --focus="When querying the agent telemetry" ./querylogs + shell: ginkgo ./querylogs pod: nodeSelector: kubernetes.io/os: linux From 9f9f54fdbc19410c9bb70532a36eb216f3cb2425 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Wed, 23 Sep 2026 05:17:14 +0000 Subject: [PATCH 7/8] test: tolerate transient deleted and missing-file errors --- test/ginkgo-e2e/utils/constants.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/ginkgo-e2e/utils/constants.go b/test/ginkgo-e2e/utils/constants.go index a5bb73f86a..e17f40309c 100644 --- a/test/ginkgo-e2e/utils/constants.go +++ b/test/ginkgo-e2e/utils/constants.go @@ -14,6 +14,8 @@ var ( "GetAgentConfigurations", "RefreshConfigurations", "canceled by user", + "(deleted)", + "errno=2] No such file or directory", } ) From decce9f0d4b04d0d843cbbb189dd5e329da19208 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Wed, 30 Sep 2026 21:44:42 +0000 Subject: [PATCH 8/8] test: widen agent telemetry query window to 30m The pipeline waits 10 minutes after deployment and the test runner waits another 5 before the suite starts, so agent startup traces can already be older than the 15-minute window by the time the query runs. Widen it to 30 minutes so the check no longer reports a false negative. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/ginkgo-e2e/querylogs/querylogs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ginkgo-e2e/querylogs/querylogs_test.go b/test/ginkgo-e2e/querylogs/querylogs_test.go index a8436969ea..4fb19f8854 100644 --- a/test/ginkgo-e2e/querylogs/querylogs_test.go +++ b/test/ginkgo-e2e/querylogs/querylogs_test.go @@ -101,7 +101,7 @@ var _ = Describe("When querying the agent telemetry", func() { Expect(err).NotTo(HaveOccurred()) query := fmt.Sprintf(`%s -| where timestamp > ago(15m) +| where timestamp > ago(30m) | extend ClusterId = iff(isnotempty(tostring(customDimensions.ID)), tostring(customDimensions.ID), tostring(customDimensions.AKS_RESOURCE_ID)) | where ClusterId =~ %q | where tostring(customDimensions.Version) in (%q, %q)