From 2cdc5af28184eee0f6ae70d4edbdba58a281aa00 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Mon, 17 Aug 2026 21:49:27 +0000 Subject: [PATCH 1/4] Fix telegraf deprecated options removed in 1.40 (linux) The linux agent ships telegraf 1.39.x and logs deprecation warnings for options that telegraf removes in 1.40.0. Telegraf treats an unrecognized option as a fatal config-load error (not a warning), and an empty value such as `fieldpass = []` fails identically, so on 1.40 every linux ama-logs telegraf process would refuse to start -- including on clusters using the default ConfigMap. Migrate the linux configs to the replacement option names: fieldpass -> fieldinclude fielddrop -> fieldexclude pid_tag -> tag_with (inputs.procstat, also removed in 1.40) Covers the daemonset, replicaset, prometheus sidecar and the separate ama-logs process-metrics telegraf instance, plus the prometheus stanzas generated at runtime by tomlparser-prom-customconfig.rb (linux branch) and tomlparser-osm-config.rb. Windows is intentionally left on the legacy option names: it is pinned to telegraf 1.24.2, which predates fieldinclude/fieldexclude/tag_with (added in 1.29.0). tomlparser-prom-customconfig.rb is shared by both platforms, so it now selects the option names via is_windows?, mirroring the existing timeout/response_timeout branch. The customer-facing ConfigMap keys (prometheus_data_collection_settings fieldpass/fielddrop) and the TELEMETRY_* env var names are unchanged; the new names are only used when rendering telegraf TOML. Also drop ignore_protocol_stats from inputs.net. Protocol stat collection was removed in telegraf 1.37.0 and the option is a no-op, so current behavior already matches the previous `true` setting. The startup validation used `--input-filter file -test`, which only loads the dummy file input and so returned success for configs that are fatally invalid, promoting a broken config that then killed telegraf silently at the real (backgrounded, unchecked) start. Replace it with `telegraf config check`, which validates every configured plugin without gathering, so it stays fast and performs no network scrapes. Verified against the telegraf 1.39.1 binary in a live cluster: daemonset config goes from 6 deprecation warnings to 0, process-metrics from 9 to 0, replicaset/sidecar/generated stanzas remain at 0, all with config check RC=0, and inputs.net emits the identical field set. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scripts/tomlparser-prom-customconfig.rb | 14 ++++++-- .../telegraf-ama-logs-process-metrics.conf | 36 +++++++++---------- .../conf/telegraf-prom-side-car.conf | 4 +-- build/linux/installer/conf/telegraf-rs.conf | 6 ++-- build/linux/installer/conf/telegraf.conf | 21 ++++++----- .../scripts/tomlparser-osm-config.rb | 2 +- kubernetes/linux/main.sh | 6 ++-- kubernetes/windows/setup.ps1 | 5 +++ 8 files changed, 54 insertions(+), 40 deletions(-) diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig.rb b/build/common/installer/scripts/tomlparser-prom-customconfig.rb index 8568850ec1..78edcb09a4 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig.rb @@ -119,6 +119,16 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu timeout_config_key = "response_timeout" end + # 'fieldpass'/'fielddrop' were deprecated in telegraf 1.29.0 and removed in 1.40.0. + # Windows still ships telegraf 1.24.2, which predates the replacement options, so it + # must keep emitting the legacy option names. + field_include_key = "fieldinclude" + field_exclude_key = "fieldexclude" + if is_windows? + field_include_key = "fieldpass" + field_exclude_key = "fielddrop" + end + pluginConfigsWithNamespaces = "" monitorKubernetesPodsNamespaces.each do |namespace| if !namespace.nil? @@ -133,8 +143,8 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu monitor_kubernetes_pods_namespace = \"#{namespace}\" kubernetes_label_selector = \"#{kubernetesLabelSelectors}\" kubernetes_field_selector = \"#{kubernetesFieldSelectors}\" - fieldpass = #{fieldPassSetting} - fielddrop = #{fieldDropSetting} + #{field_include_key} = #{fieldPassSetting} + #{field_exclude_key} = #{fieldDropSetting} metric_version = #{@metricVersion} url_tag = \"#{@urlTag}\" #{timeout_config_key} = \"#{@responseTimeout}\" diff --git a/build/linux/installer/conf/telegraf-ama-logs-process-metrics.conf b/build/linux/installer/conf/telegraf-ama-logs-process-metrics.conf index 02da703021..a22ec03d24 100644 --- a/build/linux/installer/conf/telegraf-ama-logs-process-metrics.conf +++ b/build/linux/installer/conf/telegraf-ama-logs-process-metrics.conf @@ -22,9 +22,9 @@ exe = "mdsd" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -38,9 +38,9 @@ exe = "fluent-bit" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -54,9 +54,9 @@ pattern = "telegraf(-rs|-prom-side-car)?.conf" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -70,9 +70,9 @@ pattern = "telegraf-ama-logs-process-metrics.conf" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -85,9 +85,9 @@ name_prefix = "t.azm.ms/" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -104,9 +104,9 @@ exe = "crond" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -120,9 +120,9 @@ exe = "inotifywait" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -135,9 +135,9 @@ name_prefix = "t.azm.ms/" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -154,9 +154,9 @@ pattern = "main.sh" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" diff --git a/build/linux/installer/conf/telegraf-prom-side-car.conf b/build/linux/installer/conf/telegraf-prom-side-car.conf index 5e1c99ee05..9a7bb0eb15 100644 --- a/build/linux/installer/conf/telegraf-prom-side-car.conf +++ b/build/linux/installer/conf/telegraf-prom-side-car.conf @@ -183,8 +183,8 @@ $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR - fieldpass = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS - fielddrop = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP + fieldinclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS + fieldexclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP metric_version = 2 url_tag = "scrapeUrl" diff --git a/build/linux/installer/conf/telegraf-rs.conf b/build/linux/installer/conf/telegraf-rs.conf index 887e660ccb..12cf533737 100644 --- a/build/linux/installer/conf/telegraf-rs.conf +++ b/build/linux/installer/conf/telegraf-rs.conf @@ -574,8 +574,8 @@ $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR - fieldpass = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS - fielddrop = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP + fieldinclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS + fieldexclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP metric_version = 2 url_tag = "scrapeUrl" @@ -641,7 +641,7 @@ $AZMON_TELEGRAF_OSM_PROM_PLUGINS #name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = $AZMON_INTEGRATION_NPM_METRICS_URL_LIST_CLUSTER - fielddrop = $AZMON_INTEGRATION_NPM_METRICS_DROP_LIST_CLUSTER + fieldexclude = $AZMON_INTEGRATION_NPM_METRICS_DROP_LIST_CLUSTER metric_version = 2 url_tag = "scrapeUrl" diff --git a/build/linux/installer/conf/telegraf.conf b/build/linux/installer/conf/telegraf.conf index 81925e0f0c..ed047284b8 100644 --- a/build/linux/installer/conf/telegraf.conf +++ b/build/linux/installer/conf/telegraf.conf @@ -448,7 +448,7 @@ ## Ignore mount points by filesystem type. ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs", "cifs", "fuse"] - fieldpass = ["free", "used", "used_percent"] + fieldinclude = ["free", "used", "used_percent"] taginclude = ["device","path","hostName"] # Below due to Bug - https://github.com/influxdata/telegraf/issues/5615 # ORDER matters here!! - i.e the below should be the LAST modifier @@ -489,7 +489,7 @@ ## The typical use case is for LVM volumes, to get the VG/LV name instead of ## the near-meaningless DM-0 name. # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"] - fieldpass = ["reads", "read_bytes", "read_time", "writes", "write_bytes", "write_time", "io_time", "iops_in_progress"] + fieldinclude = ["reads", "read_bytes", "read_time", "writes", "write_bytes", "write_time", "io_time", "iops_in_progress"] taginclude = ["name","hostName"] # Read metrics about network interface usage @@ -501,12 +501,11 @@ ## # interfaces = ["eth0"] ## - ## On linux systems telegraf also collects protocol stats. - ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics. + ## NOTE: protocol stats collection was removed in telegraf 1.37.0 and the + ## ignore_protocol_stats option became a no-op (removed in 1.45.0), so it is + ## no longer set here. Current behavior matches the previous "true" setting. ## - ignore_protocol_stats = true - ## - fieldpass = ["bytes_recv", "bytes_sent", "err_in", "err_out"] + fieldinclude = ["bytes_recv", "bytes_sent", "err_in", "err_out"] taginclude = ["interface","hostName"] # net plugin is collecting loopback interface metrics by default from telegraf 1.34.3 @@ -652,7 +651,7 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["$CADVISOR_METRICS_URL"] - fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC"] + fieldinclude = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC"] metric_version = 2 url_tag = "scrapeUrl" @@ -697,7 +696,7 @@ # <= 1.18: metric name is kubelet_running_pod_count # >= 1.19: metric name changed to kubelet_running_pods - fieldpass = ["kubelet_running_pod_count","kubelet_running_pods","volume_manager_total_volumes", "kubelet_node_config_error", "process_resident_memory_bytes", "process_cpu_seconds_total"] + fieldinclude = ["kubelet_running_pod_count","kubelet_running_pods","volume_manager_total_volumes", "kubelet_node_config_error", "process_resident_memory_bytes", "process_cpu_seconds_total"] metric_version = 2 url_tag = "scrapeUrl" @@ -721,9 +720,9 @@ ## An array of urls to scrape metrics from. urls = $AZMON_DS_PROM_URLS - fieldpass = $AZMON_DS_PROM_FIELDPASS + fieldinclude = $AZMON_DS_PROM_FIELDPASS - fielddrop = $AZMON_DS_PROM_FIELDDROP + fieldexclude = $AZMON_DS_PROM_FIELDDROP metric_version = 2 url_tag = "scrapeUrl" diff --git a/build/linux/installer/scripts/tomlparser-osm-config.rb b/build/linux/installer/scripts/tomlparser-osm-config.rb index e2fdcd289e..0215469149 100644 --- a/build/linux/installer/scripts/tomlparser-osm-config.rb +++ b/build/linux/installer/scripts/tomlparser-osm-config.rb @@ -108,7 +108,7 @@ def replaceOsmTelegrafConfigPlaceHolders monitor_kubernetes_pods = true pod_scrape_scope = \"#{(@controllerType.casecmp(@replicaset) == 0) ? "cluster" : "node"}\" monitor_kubernetes_pods_namespace = \"#{namespace}\" - fieldpass = #{@fieldPassSetting} + fieldinclude = #{@fieldPassSetting} metric_version = #{@metricVersion} url_tag = \"#{@urlTag}\" bearer_token = \"#{@bearerToken}\" diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 53aeb26d3a..a5b2b10854 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -1242,7 +1242,7 @@ else if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ] && [ -e "/opt/telegraf-test-prom-side-car.conf" ]; then if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then echo "****************Start Telegraf in Test Mode**************************" - /opt/telegraf --non-strict-env-handling --config /opt/telegraf-test-prom-side-car.conf --input-filter file -test + /opt/telegraf config check --non-strict-env-handling --config /opt/telegraf-test-prom-side-car.conf if [ $? -eq 0 ]; then mv "/opt/telegraf-test-prom-side-car.conf" "/etc/opt/microsoft/docker-cimprov/telegraf-prom-side-car.conf" echo "Moving test conf file to telegraf side-car conf since test run succeeded" @@ -1254,7 +1254,7 @@ else else if [ -e "/opt/telegraf-test.conf" ]; then echo "****************Start Telegraf in Test Mode**************************" - /opt/telegraf --non-strict-env-handling --config /opt/telegraf-test.conf --input-filter file -test + /opt/telegraf config check --non-strict-env-handling --config /opt/telegraf-test.conf if [ $? -eq 0 ]; then mv "/opt/telegraf-test.conf" "/etc/opt/microsoft/docker-cimprov/telegraf.conf" echo "Moving test conf file to telegraf daemonset conf since test run succeeded" @@ -1265,7 +1265,7 @@ else else if [ -e "/opt/telegraf-test-rs.conf" ]; then echo "****************Start Telegraf in Test Mode**************************" - /opt/telegraf --non-strict-env-handling --config /opt/telegraf-test-rs.conf --input-filter file -test + /opt/telegraf config check --non-strict-env-handling --config /opt/telegraf-test-rs.conf if [ $? -eq 0 ]; then mv "/opt/telegraf-test-rs.conf" "/etc/opt/microsoft/docker-cimprov/telegraf-rs.conf" echo "Moving test conf file to telegraf replicaset conf since test run succeeded" diff --git a/kubernetes/windows/setup.ps1 b/kubernetes/windows/setup.ps1 index ae544148d2..0b54deeb3a 100644 --- a/kubernetes/windows/setup.ps1 +++ b/kubernetes/windows/setup.ps1 @@ -42,6 +42,11 @@ Write-Host ('Finished Installing Fluentbit') Write-Host ('Installing Telegraf'); try { # For next telegraf update, make sure to update config changes in telegraf.conf, tomlparser-prom-customconfig.rb and tomlparser-osm-config.rb + # NOTE: Windows is intentionally pinned to an older telegraf than Linux. Linux uses the + # 'fieldinclude'/'fieldexclude' options (added in 1.29.0, replacing 'fieldpass'/'fielddrop' + # which were removed in 1.40.0), while the Windows configs below must keep the legacy + # 'fieldpass'/'fielddrop' names until telegraf here is upgraded to >= 1.29.0. + # tomlparser-prom-customconfig.rb is shared by both and branches on is_windows? for this. $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.24.2_windows_amd64.zip' Invoke-WebRequest -Uri $telegrafUri -OutFile /installation/telegraf.zip Expand-Archive -Path /installation/telegraf.zip -Destination /installation/telegraf From 5ba52e03dba421e7fdbb0ce19b1597efe120d94c Mon Sep 17 00:00:00 2001 From: zanejohnson-azure Date: Thu, 24 Sep 2026 15:07:35 -0700 Subject: [PATCH 2/4] Upgrade Windows Telegraf to 1.40.0 (#1785) * Upgrade Windows Telegraf to 1.40.0 Consume the official upstream Windows package with a pinned SHA256 and migrate timeout, field-filter, and procstat PID-tag configuration. Preserve ConfigMap keys and Linux behavior; document the deferred discovery cleanup and deadline gaps. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Fix Telegraf service startup in Windows containers Host the unchanged upstream executable with the existing win32-service dispatcher instead of relying on Telegraf's session-zero service detection. Preserve both service roles, bound shutdown and log growth, and contain the child in a kill-on-close job. Cover registration and lifecycle behavior without relaxing TLS, authentication, strict config parsing, or protected-memory defaults. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Close Telegraf service startup containment gap Enroll the service host in its kill-on-close job before creating the Telegraf child, retain the non-inheritable handle until host exit, and cover abrupt and normal startup-boundary exits with local Windows process tests. Wire Windows worker tests into the existing Ruby driver and narrowly suppress the public-checksum and test-loopback DevSkim false positives. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 08a78bbb-f751-41cb-87b6-96aaf4c1a1ae * Keep existing Telegraf field filters in Windows upgrade Remove the optional fieldinclude/fieldexclude migration and its OS-specific rendering/test helper. Preserve fieldpass/fielddrop and the original timeout variable while retaining the required overall-timeout and procstat PID-tag fixes. Add pinned upstream references for the compatibility rationale. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 08a78bbb-f751-41cb-87b6-96aaf4c1a1ae --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 08a78bbb-f751-41cb-87b6-96aaf4c1a1ae --- Dev Guide.md | 52 +++++ .../scripts/tomlparser-prom-customconfig.rb | 8 +- .../tomlparser-prom-customconfig_test.rb | 97 +++++++++- .../telegraf-ama-logs-process-metrics.conf | 22 +-- build/windows/installer/conf/telegraf.conf | 4 +- .../scripts/telegraf-windows-console.rb | 76 ++++++++ .../scripts/telegraf-windows-console_test.rb | 122 ++++++++++++ .../scripts/telegraf-windows-service.rb | 108 +++++++++++ .../scripts/telegraf-windows-service_test.rb | 177 ++++++++++++++++++ kubernetes/windows/main.ps1 | 38 +++- kubernetes/windows/setup.ps1 | 18 +- test/unit-tests/README.md | 18 ++ .../test_cases/Test-TelegrafPackage.ps1 | 91 +++++++++ .../test_cases/Test-TelegrafService.ps1 | 61 ++++++ test/unit-tests/test_driver.rb | 4 + test/unit-tests/test_main.ps1 | 4 +- 16 files changed, 865 insertions(+), 35 deletions(-) create mode 100644 build/windows/installer/scripts/telegraf-windows-console.rb create mode 100644 build/windows/installer/scripts/telegraf-windows-console_test.rb create mode 100644 build/windows/installer/scripts/telegraf-windows-service.rb create mode 100644 build/windows/installer/scripts/telegraf-windows-service_test.rb create mode 100644 test/unit-tests/test_cases/Test-TelegrafPackage.ps1 create mode 100644 test/unit-tests/test_cases/Test-TelegrafService.ps1 diff --git a/Dev Guide.md b/Dev Guide.md index 7057a4afef..8c8ac90c8c 100644 --- a/Dev Guide.md +++ b/Dev Guide.md @@ -4,6 +4,58 @@ More advanced information needed to develop or build the docker provider will li +## Windows Telegraf dependency + +`kubernetes/windows/setup.ps1` installs the official Telegraf 1.40.0 Windows AMD64 +ZIP and verifies its pinned SHA256 before extraction. The package corresponds to +upstream commit `e9017dc3266369d6fa185e0e130af1d1d4021ce9`. The existing Windows +pipeline continues to sign `C:\opt\telegraf\telegraf.exe` as an OSS dependency. + +The official binary is built with Go 1.27.0 for `windows/amd64`, `GOAMD64=v1`. +Go's [Windows OS floor](https://go.dev/wiki/MinimumRequirements#windows) is Windows +10 or Windows Server 2016 and newer. Both repository image targets, LTSC2019 and +LTSC2022, meet that floor; this does not replace validation inside those images +or in installed-service mode. + +The Windows entrypoint registers both Telegraf services through +`telegraf-windows-service.rb`, using the same already-installed `win32-service` +dispatcher as Fluentd. Telegraf 1.40's native service detection requires its +`services.exe` parent to be in session 0, which is not guaranteed in Windows containers. +The host runs the unchanged executable with `--console`, monitors child exit, and +forwards SCM stop through the child's private console/process group. Shutdown is +bounded; the host joins a kill-on-close job before spawning, so the child inherits +containment at creation, including if the host dies before `spawn` returns. The +non-inheritable job handle stays open until host process exit so console cleanup +does not kill the host before SCM shutdown completes. The service +PID is the Ruby host; the Telegraf PID is its child. Role-only host arguments keep +the existing procstat config-path filters selecting Telegraf rather than Ruby. +Per-role logs under `C:\opt\telegraf\logs` rotate at 5 MiB with two backups. +The native startup-boundary regression uses only local sleeping Ruby processes: +`ruby build/windows/installer/scripts/telegraf-windows-console_test.rb`. +It runs on Windows with the image's existing `ffi` dependency and skips on Linux. + +Upgrade the two configurations in `build/windows/installer/conf/` and +`tomlparser-prom-customconfig.rb` together with the binary. Windows uses `timeout` +to preserve the overall 15-second metric-scrape limit; upstream's +[1.40 documentation](https://github.com/influxdata/telegraf/blob/e9017dc3266369d6fa185e0e130af1d1d4021ce9/plugins/inputs/prometheus/README.md#L152-L157) +explains that `response_timeout` now covers headers only, unlike the +[1.24.2 client timeout](https://github.com/influxdata/telegraf/blob/9550e7a533dd00632e14435e87ed3eb4b04832c6/plugins/inputs/prometheus/prometheus.go#L254-L261). +Procstat uses `tag_with = ["pid"]` to retain PID tags. Existing `fieldpass`/`fielddrop` +names are unchanged on both OSes because +[1.40 still parses them](https://github.com/influxdata/telegraf/blob/e9017dc3266369d6fa185e0e130af1d1d4021ce9/config/config.go#L1649-L1687). +Linux rendering remains unchanged. Run `ruby build/common/installer/scripts/tomlparser-prom-customconfig_test.rb` +for rendering coverage with and without namespace filters. On Windows, set +`TELEGRAF_WINDOWS_BINARY` to the extracted `telegraf.exe` to also load the generated +configs with that binary in bounded `--test` mode, without Kubernetes access or +running output plugins. + +This stock upgrade is a partial mitigation: node discovery rereads the token file +on retries after a failed poll, without relying on file modification time. The +[1.40.0 discovery code](https://github.com/influxdata/telegraf/blob/e9017dc3266369d6fa185e0e130af1d1d4021ce9/plugins/inputs/prometheus/kubernetes.go) +still omits response cleanup on non-200 status codes and lacks an explicit +discovery request timeout. The metric-scrape `timeout` does not bound that path. +HTTP/2 negotiation is not a substitute for fixing those remaining issues. + ## Testing Last updated 8/18/2021 diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig.rb b/build/common/installer/scripts/tomlparser-prom-customconfig.rb index bbd313d991..3d7ddb07ad 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig.rb @@ -92,8 +92,8 @@ def checkForType(variable, varType) # Telegraf parses interval values with Go's time.ParseDuration, which accepts one or more # decimal numbers each followed by a unit suffix (for example "30s", "1.5s" or "1h30m"). -# The unit set below is the intersection of what the Linux and Windows agents accept; "d" -# is deliberately excluded because the older telegraf shipped on Windows rejects it. +# Keep the existing config-map duration contract on both Linux and Windows; "d" remains +# deliberately excluded even when newer telegraf versions accept days. # The anchors must be \A and \z (not ^ and $) so that a value such as "1m\n" # cannot pass validation by matching only its first line. TELEGRAF_DURATION_REGEX = /\A(?:(?:\d+(?:\.\d+)?|\.\d+)(?:ns|us|\u00B5s|\u03BCs|ms|s|m|h))+\z/ @@ -187,10 +187,6 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu new_contents = new_contents.gsub("$AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE") timeout_config_key = "timeout" - if is_windows? - # For windows, the timeout config key is different because of old version of telegraf - timeout_config_key = "response_timeout" - end # 'fieldpass'/'fielddrop' were deprecated in telegraf 1.29.0 and removed in 1.40.0. # Windows still ships telegraf 1.24.2, which predates the replacement options, so it diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb index f2b67bcb02..c30ede5526 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb @@ -91,7 +91,8 @@ def run_parser(scenario, configmap) "SIDECAR_SCRAPING_ENABLED" => nil, }.merge(spec[:env]) - stdout, stderr, = Open3.capture3(env, RbConfig.ruby, parser, chdir: sandbox) + stdout, stderr, status = Open3.capture3(env, RbConfig.ruby, parser, chdir: sandbox) + assert status.success?, "config parser failed: #{stdout}\n#{stderr}" telemetry_path = File.join(sandbox, "telemetry_prom_config_env_var") result = { @@ -170,7 +171,7 @@ def test_invalid_telegraf_durations_are_rejected "1m;id", # shell metacharacters "$(id)", "`id`", - "1d", # rejected: the telegraf shipped on windows does not support days + "1d", # rejected by the existing config-map duration contract "1", # missing unit "m", # missing number "", @@ -311,6 +312,98 @@ def test_namespace_plugins_are_generated_for_valid_namespaces assert_no_injected_plugins(:replicaset, result[:conf], "a benign namespace configuration") end + [:replicaset, :sidecar, :windows].each do |scenario| + { unfiltered: nil, empty_filter: [], namespace_filtered: ["default", "kube-system"] }.each do |name, namespaces| + define_method("test_prometheus_timeout_and_mapping_#{scenario}_#{name}") do + body = "monitor_kubernetes_pods = true\n" \ + "interval = \"45s\"\n" \ + "fieldpass = [\"requests_total\"]\n" \ + "fielddrop = [\"debug_total\"]\n" \ + "kubernetes_label_selector = \"app=metrics\"\n" \ + "kubernetes_field_selector = \"spec.nodeName=test-node\"" + body += "\nmonitor_kubernetes_pods_namespaces = #{namespaces.inspect}" unless namespaces.nil? + + result = run_parser(scenario, configmap_for(scenario, body)) + plugins = parse_generated_toml(result[:conf])["inputs"]["prometheus"] + monitored = plugins.select { |plugin| plugin["monitor_kubernetes_pods"] } + expected_namespaces = namespaces.nil? || namespaces.empty? ? [nil] : namespaces + assert_equal expected_namespaces, monitored.map { |plugin| plugin["monitor_kubernetes_pods_namespace"] } + + monitored.each do |plugin| + assert_equal "15s", plugin["timeout"], "the overall scrape timeout must remain 15s" + refute plugin.key?("response_timeout"), "response_timeout only bounds response headers in current Telegraf" + assert_equal "45s", plugin["interval"] + assert_equal 2, plugin["metric_version"] + assert_equal "scrapeUrl", plugin["url_tag"] + assert_equal "pod_namespace", plugin["pod_namespace_label_name"] + assert_equal (scenario == :replicaset ? "cluster" : "node"), plugin["pod_scrape_scope"] + assert_equal ["requests_total"], plugin["fieldpass"] + assert_equal ["debug_total"], plugin["fielddrop"] + assert_equal "app=metrics", plugin["kubernetes_label_selector"] + assert_equal "spec.nodeName=test-node", plugin["kubernetes_field_selector"] + end + + if scenario == :windows + plugins.each do |plugin| + assert_equal "15s", plugin["timeout"], "the base Windows plugin must also retain the overall timeout" + refute plugin.key?("response_timeout") + end + end + end + end + end + + def test_windows_rendered_configs_load_in_packaged_telegraf + binary = ENV["TELEGRAF_WINDOWS_BINARY"] + skip "Set TELEGRAF_WINDOWS_BINARY to run the Windows binary config smoke test" if binary.nil? || binary.empty? + assert File.file?(binary), "TELEGRAF_WINDOWS_BINARY must point to telegraf.exe" + + [nil, [], ["default", "kube-system"]].each do |namespaces| + body = "monitor_kubernetes_pods = true\nfieldpass = [\"requests_total\"]\nfielddrop = [\"debug_total\"]" + body += "\nmonitor_kubernetes_pods_namespaces = #{namespaces.inspect}" unless namespaces.nil? + conf = run_parser(:windows, configmap_for(:windows, body))[:conf] + + # This smoke test loads the actual generated options without contacting Kubernetes + # or requiring a mounted service-account CA. --test never runs output plugins. + conf = conf.gsub(/^(\s*monitor_kubernetes_pods\s*=\s*)true[ \t]*$/) { "#{Regexp.last_match(1)}false" } + conf = conf.gsub(/^(\s*tls_ca\s*=\s*)"[^"]*"[ \t]*$/) { "#{Regexp.last_match(1)}\"\"" } + + Dir.mktmpdir("windows-telegraf-config") do |dir| + path = File.join(dir, "telegraf.conf") + File.write(path, conf) + Open3.popen3({ "NODE_IP" => "127.0.0.1" }, binary, "--console", "--test", "--config", path) do |stdin, stdout, stderr, process| # DevSkim: ignore DS162092 -- Loopback-only config smoke test. + stdin.close + readers = [Thread.new { stdout.read }, Thread.new { stderr.read }] + completed = !process.join(20).nil? + Process.kill("KILL", process.pid) unless completed + output = readers.map(&:value).join("\n") + assert completed, "Telegraf config smoke test exceeded 20 seconds" + assert process.value.success?, "Telegraf rejected the rendered config for #{namespaces.inspect}: #{output}" + end + end + end + end + + def test_windows_process_metrics_config_preserves_pid_tags_and_fields + path = File.join(REPO_ROOT, "build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf") + config = Tomlrb.load_file(path) + assert_equal({ "telegraf_role" => "ama-logs-process-metrics" }, config["global_tags"]) + assert_equal 11, config["inputs"]["procstat"].length + config["inputs"]["procstat"].each do |plugin| + assert_equal ["pid"], plugin["tag_with"] + refute plugin.key?("pid_tag") + assert_equal ["cpu_usage", "memory_rss"], plugin["fieldpass"] + assert_equal "native", plugin["pid_finder"] + assert_equal "agent_telemetry", plugin["name_override"] + assert_equal "t.azm.ms/", plugin["name_prefix"] + assert_equal "DaemonSet-Windows", plugin["tags"]["ControllerType"] + end + assert_equal( + { "ai.cloud.role" => "ControllerType", "ai.cloud.roleInstance" => "PodName" }, + config["outputs"]["application_insights"].first["context_tag_sources"] + ) + end + def test_selectors_are_escaped_in_generated_namespace_plugins body = "monitor_kubernetes_pods = true\n" \ "monitor_kubernetes_pods_namespaces = [\"default\"]\n" \ diff --git a/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf b/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf index e499d3574d..9f004fda13 100644 --- a/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf +++ b/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf @@ -20,7 +20,7 @@ exe = "fluent-bit" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -36,7 +36,7 @@ pattern = "telegraf.conf" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -52,7 +52,7 @@ pattern = "telegraf-ama-logs-process-metrics.conf" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -68,7 +68,7 @@ exe = "MonAgentLauncher" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -84,7 +84,7 @@ exe = "MonAgentCore" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -100,7 +100,7 @@ exe = "MonAgentHost" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -116,7 +116,7 @@ exe = "MonAgentManager" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -132,7 +132,7 @@ exe = "AzurePerfCollectorExtension" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -148,7 +148,7 @@ exe = "AzureProfilerExtension" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -164,7 +164,7 @@ exe = "AggregatorHost" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] @@ -180,7 +180,7 @@ exe = "powershell" interval = "60s" pid_finder = "native" - pid_tag = true + tag_with = ["pid"] name_override = "agent_telemetry" fieldpass = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] diff --git a/build/windows/installer/conf/telegraf.conf b/build/windows/installer/conf/telegraf.conf index b2de52d8ed..8bf96e3f66 100644 --- a/build/windows/installer/conf/telegraf.conf +++ b/build/windows/installer/conf/telegraf.conf @@ -149,8 +149,8 @@ ## OR # bearer_token_string = "abc_123" - ## Specify timeout duration for slower prometheus clients (default is 3s) - response_timeout = "15s" + ## Overall HTTP timeout for metric scrapes, including the response body. + timeout = "15s" ## Optional TLS Config tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" diff --git a/build/windows/installer/scripts/telegraf-windows-console.rb b/build/windows/installer/scripts/telegraf-windows-console.rb new file mode 100644 index 0000000000..439e22301d --- /dev/null +++ b/build/windows/installer/scripts/telegraf-windows-console.rb @@ -0,0 +1,76 @@ +require "ffi" + +class TelegrafConsole + module Native + extend FFI::Library + ffi_lib "kernel32" + ffi_convention :stdcall + attach_function :GetConsoleProcessList, [:pointer, :ulong], :ulong + attach_function :AllocConsole, [], :bool + attach_function :FreeConsole, [], :bool + attach_function :GenerateConsoleCtrlEvent, [:ulong, :ulong], :bool + attach_function :GetLastError, [], :ulong + attach_function :GetCurrentProcess, [], :pointer + attach_function :CreateJobObjectW, [:pointer, :pointer], :pointer + attach_function :SetInformationJobObject, [:pointer, :int, :pointer, :ulong], :bool + attach_function :AssignProcessToJobObject, [:pointer, :pointer], :bool + attach_function :CloseHandle, [:pointer], :bool + + class BasicLimits < FFI::Struct + layout :process_time, :int64, :job_time, :int64, :flags, :uint32, + :min_working_set, :size_t, :max_working_set, :size_t, + :active_processes, :uint32, :affinity, :size_t, + :priority, :uint32, :scheduling, :uint32 + end + + class ExtendedLimits < FFI::Struct + layout :basic, BasicLimits, :io_counters, [:uint64, 6], + :process_memory, :size_t, :job_memory, :size_t, + :peak_process_memory, :size_t, :peak_job_memory, :size_t + end + end + + def prepare + buffer = FFI::MemoryPointer.new(:ulong, 1) + if Native.GetConsoleProcessList(buffer, 1) == 0 + unless Native.AllocConsole + raise SystemCallError.new("AllocConsole", Native.GetLastError) + end + @allocated = true + end + @job = Native.CreateJobObjectW(nil, nil) + raise SystemCallError.new("CreateJobObject", Native.GetLastError) if @job.null? + limits = Native::ExtendedLimits.new + limits[:basic][:flags] = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE. + unless Native.SetInformationJobObject(@job, 9, limits.pointer, limits.size) + raise SystemCallError.new("SetInformationJobObject", Native.GetLastError) + end + + # Enroll the host before spawning: children inherit membership atomically. + # The non-inheritable job handle remains owned only by this host. + unless Native.AssignProcessToJobObject(@job, Native.GetCurrentProcess) + raise SystemCallError.new("AssignProcessToJobObject(host)", Native.GetLastError) + end + @host_assigned = true + end + + def interrupt(pid) + Native.GenerateConsoleCtrlEvent(1, pid) # CTRL_BREAK_EVENT for this process group. + end + + def close + # Once enrolled, keep the handle until host exit. Closing it here would also + # kill the host before it can finish SCM stop notification and log cleanup. + if @job && !@job.null? && !@host_assigned + unless Native.CloseHandle(@job) + raise SystemCallError.new("CloseHandle(job)", Native.GetLastError) + end + @job = nil + end + return unless @allocated + unless Native.FreeConsole + raise SystemCallError.new("FreeConsole", Native.GetLastError) + end + @allocated = false + end +end diff --git a/build/windows/installer/scripts/telegraf-windows-console_test.rb b/build/windows/installer/scripts/telegraf-windows-console_test.rb new file mode 100644 index 0000000000..306845d385 --- /dev/null +++ b/build/windows/installer/scripts/telegraf-windows-console_test.rb @@ -0,0 +1,122 @@ +require "minitest/autorun" +require "rbconfig" +require "tmpdir" + +if Gem.win_platform? + require_relative "telegraf-windows-console" + + module TelegrafProcessObserver + extend FFI::Library + ffi_lib "kernel32" + ffi_convention :stdcall + attach_function :OpenProcess, [:ulong, :bool, :ulong], :pointer + attach_function :WaitForSingleObject, [:pointer, :ulong], :ulong + attach_function :TerminateProcess, [:pointer, :uint32], :bool + attach_function :CloseHandle, [:pointer], :bool + end +end + +class TelegrafConsoleTest < Minitest::Test + # Pause inside spawn, before the worker can perform any post-spawn action. + # Only a sleeping Ruby fixture is launched; no Telegraf, service or network. + HOST_SCRIPT = <<~'RUBY'.freeze + require ARGV.shift + require ARGV.shift + require "rbconfig" + + class StartupBoundary + def initialize(console, marker, close_console) + @console, @marker, @close_console = console, marker, close_console + end + + def spawn(*command, **options) + pid = Process.spawn(RbConfig.ruby, "-e", "sleep 60", **options) + @console.close if @close_console + File.write(@marker, pid.to_s) + $stdin.gets + exit 0 + end + end + + marker, close_console = ARGV + console = TelegrafConsole.new + processes = StartupBoundary.new(console, marker, close_console == "true") + TelegrafServiceWorker.new("prometheus", Logger.new(File::NULL), console, processes).run + RUBY + + def setup + skip "Native job containment requires Windows" unless Gem.win_platform? + end + + def with_startup_boundary(close_console: false) + Dir.mktmpdir("telegraf-job-test") do |directory| + marker = File.join(directory, "child.pid") + log_path = File.join(directory, "host.log") + input, command = IO.pipe + waiter = nil + child = nil + begin + File.open(log_path, "w") do |log| + host_pid = Process.spawn( + RbConfig.ruby, "-e", HOST_SCRIPT, + File.join(__dir__, "telegraf-windows-service.rb"), + File.join(__dir__, "telegraf-windows-console.rb"), + marker, close_console.to_s, in: input, out: log, err: log + ) + waiter = Process.detach(host_pid) + end + input.close + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 10 + until File.exist?(marker) && !File.zero?(marker) + break unless waiter.alive? && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline + sleep 0.02 + end + assert File.exist?(marker) && !File.zero?(marker), "Host did not reach the startup boundary: #{File.read(log_path)}" + child_pid = Integer(File.read(marker)) + child = TelegrafProcessObserver.OpenProcess(0x00100001, false, child_pid) # SYNCHRONIZE | TERMINATE. + refute child.null?, "Cannot observe the owned fixture child" + assert waiter.alive?, "Host exited before the startup boundary" + assert_equal 258, TelegrafProcessObserver.WaitForSingleObject(child, 0), "Child should initially be alive" + yield waiter, command + assert waiter.join(5), "Host did not exit" + assert_equal 0, TelegrafProcessObserver.WaitForSingleObject(child, 5000), "Child survived its host" + ensure + command.close unless command.closed? + input.close unless input.closed? + if waiter && waiter.alive? + Process.kill("KILL", waiter.pid) + waiter.join(5) + end + if child && !child.null? + if TelegrafProcessObserver.WaitForSingleObject(child, 0) == 258 + TelegrafProcessObserver.TerminateProcess(child, 1) + TelegrafProcessObserver.WaitForSingleObject(child, 5000) + end + TelegrafProcessObserver.CloseHandle(child) + end + end + end + end + + def test_host_killed_before_spawn_returns_cannot_orphan_child + with_startup_boundary do |host, _| + Process.kill("KILL", host.pid) + end + end + + def test_normal_host_exit_at_startup_boundary_cleans_up_child + with_startup_boundary do |host, command| + command.puts("exit") + assert host.join(5), "Host did not exit normally" + assert host.value.success?, "Normal host exit should complete its cleanup" + end + end + + def test_console_cleanup_keeps_job_alive_until_host_exit + with_startup_boundary(close_console: true) do |host, command| + command.puts("exit") + assert host.join(5), "Host did not exit after console cleanup" + assert host.value.success?, "Console cleanup must not kill the host" + end + end +end diff --git a/build/windows/installer/scripts/telegraf-windows-service.rb b/build/windows/installer/scripts/telegraf-windows-service.rb new file mode 100644 index 0000000000..aac1947ac0 --- /dev/null +++ b/build/windows/installer/scripts/telegraf-windows-service.rb @@ -0,0 +1,108 @@ +require "fileutils" +require "logger" +require "thread" + +class TelegrafServiceWorker + EXECUTABLE = 'C:\opt\telegraf\telegraf.exe'.freeze + CONFIGURATIONS = { + "prometheus" => 'C:\etc\telegraf\telegraf.conf', + "process-metrics" => 'C:\etc\telegraf\telegraf-ama-logs-process-metrics.conf', + }.freeze + STOP_TIMEOUT = 20 + + class UnexpectedExit < StandardError; end + + def initialize(role, logger, console, process_api = Process) + @configuration = CONFIGURATIONS.fetch(role) + @logger = logger + @console = console + @process_api = process_api + @mutex = Mutex.new + @stopping = false + end + + def run + reader = nil + @mutex.synchronize do + return if @stopping + @console.prepare + reader, writer = IO.pipe + begin + @pid = @process_api.spawn( + EXECUTABLE, "--console", "--config", @configuration, + in: File::NULL, out: writer, err: writer, new_pgroup: true + ) + ensure + writer.close + end + @waiter = @process_api.detach(@pid) + @logger.info("Started Telegraf PID #{@pid}") + end + + output = Thread.new do + reader.each_line { |line| @logger.info(line.chomp) } + end + output.abort_on_exception = true + status = @waiter.value + unless output.join(5) + raise IOError, "Telegraf output did not close after the process exited" + end + unless @mutex.synchronize { @stopping } + raise UnexpectedExit, "Telegraf PID #{@pid} exited unexpectedly: #{status}" + end + @logger.info("Telegraf PID #{@pid} stopped: #{status}") + ensure + reader.close if reader && !reader.closed? + @console.close + end + + def stop + pid, waiter = @mutex.synchronize do + @stopping = true + [@pid, @waiter] + end + return unless waiter && waiter.alive? + + # A private console and process group let SCM stop only this Telegraf child. + signaled = @console.interrupt(pid) + @logger.warn("Could not signal Telegraf PID #{pid}; terminating it") unless signaled + if !signaled || !waiter.join(STOP_TIMEOUT) + @logger.warn("Terminating Telegraf PID #{pid} after the shutdown deadline") if signaled + @process_api.kill("KILL", pid) if waiter.alive? + raise UnexpectedExit, "Telegraf PID #{pid} did not terminate" unless waiter.join(5) + end + end +end + +if $PROGRAM_NAME == __FILE__ + unless ARGV.length == 1 && TelegrafServiceWorker::CONFIGURATIONS.key?(ARGV[0]) + abort "Usage: telegraf-windows-service.rb prometheus|process-metrics" + end + + # The image already uses win32-service for Fluentd. Unlike Telegraf's native + # service detection, its dispatcher works in a container's nonzero session. + require "win32/daemon" + require_relative "telegraf-windows-console" + + class TelegrafWindowsService < Win32::Daemon + def initialize(role) + directory = 'C:\opt\telegraf\logs' + FileUtils.mkdir_p(directory) + @logger = Logger.new(File.join(directory, "#{role}-service.log"), 2, 5 * 1024 * 1024) + @worker = TelegrafServiceWorker.new(role, @logger, TelegrafConsole.new) + end + + def service_main + @worker.run + rescue TelegrafServiceWorker::UnexpectedExit, SystemCallError, IOError => error + @logger.fatal(error.message) + exit! 1 + end + + def service_stop + @worker.stop + end + end + + TelegrafWindowsService.new(ARGV[0]).mainloop +end diff --git a/build/windows/installer/scripts/telegraf-windows-service_test.rb b/build/windows/installer/scripts/telegraf-windows-service_test.rb new file mode 100644 index 0000000000..77f1d8db24 --- /dev/null +++ b/build/windows/installer/scripts/telegraf-windows-service_test.rb @@ -0,0 +1,177 @@ +require "minitest/autorun" +require_relative "telegraf-windows-service" + +class TelegrafServiceWorkerTest < Minitest::Test + class Waiter + attr_reader :waits + + def initialize + @mutex = Mutex.new + @condition = ConditionVariable.new + @finished = false + @waits = [] + end + + def finish + @mutex.synchronize do + @finished = true + @condition.broadcast + end + end + + def alive? + @mutex.synchronize { !@finished } + end + + def join(timeout) + @waits << timeout + alive? ? nil : self + end + + def value + @mutex.synchronize { @condition.wait(@mutex) until @finished } + "exit 0" + end + end + + class Processes + attr_reader :started, :waiter, :kills, :command, :options, :events + + def initialize + @started = Queue.new + @waiter = Waiter.new + @kills = [] + @events = [] + end + + def spawn(*command, **options) + @events << :spawn + @command, @options = command, options + options[:out].puts("fixture log") + @started << true + 123 + end + + def detach(pid) + raise "wrong PID" unless pid == 123 + @waiter + end + + def kill(signal, pid) + @kills << [signal, pid] + @waiter.finish + end + end + + class Console + attr_accessor :signal_result, :complete_on_interrupt, :fail_prepare + attr_reader :prepared, :closed, :interrupted + + def initialize(processes) + @processes = processes + @signal_result = true + @complete_on_interrupt = true + end + + def prepare + @processes.events << :prepare + raise Errno::EACCES, "job fixture" if @fail_prepare + @prepared = true + end + + def interrupt(pid) + @interrupted = pid + @processes.waiter.finish if @complete_on_interrupt + @signal_result + end + + def close + @closed = true + end + end + + def setup + @processes = Processes.new + @console = Console.new(@processes) + @logger = Logger.new(File::NULL) + @worker = TelegrafServiceWorker.new("prometheus", @logger, @console, @processes) + end + + def teardown + @processes.waiter.finish + @thread.join(1) if @thread && @thread.alive? + @logger.close + end + + def start_worker + @thread = Thread.new { @worker.run } + @thread.report_on_exception = false + @processes.started.pop + end + + def test_graceful_stop_keeps_stock_binary_and_configuration + start_worker + @worker.stop + @thread.value + assert_equal [TelegrafServiceWorker::EXECUTABLE, "--console", "--config", 'C:\etc\telegraf\telegraf.conf'], @processes.command + assert @processes.options[:new_pgroup] + assert @console.prepared + assert_equal [:prepare, :spawn], @processes.events + assert_equal 123, @console.interrupted + assert_empty @processes.kills + assert @console.closed + end + + def test_stop_deadline_kills_only_owned_child + @console.complete_on_interrupt = false + start_worker + @worker.stop + @thread.value + assert_equal [20, 5], @processes.waiter.waits + assert_equal [["KILL", 123]], @processes.kills + end + + def test_failed_signal_kills_only_owned_child + @console.signal_result = false + @console.complete_on_interrupt = false + start_worker + @worker.stop + @thread.value + assert_equal [["KILL", 123]], @processes.kills + assert_equal [5], @processes.waiter.waits + end + + def test_unexpected_clean_child_exit_is_a_service_failure + start_worker + @processes.waiter.finish + assert_raises(TelegrafServiceWorker::UnexpectedExit) { @thread.value } + assert @console.closed + end + + def test_job_preparation_failure_prevents_child_creation + @console.fail_prepare = true + assert_raises(Errno::EACCES) { @worker.run } + assert_nil @processes.command + assert_empty @processes.kills + assert_equal [:prepare], @processes.events + assert @console.closed + end + + def test_stop_before_start_does_not_launch_a_child + @worker.stop + @worker.run + assert_nil @processes.command + end + + def test_process_metrics_role_uses_its_existing_configuration + @worker = TelegrafServiceWorker.new("process-metrics", @logger, @console, @processes) + start_worker + @worker.stop + @thread.value + assert_equal 'C:\etc\telegraf\telegraf-ama-logs-process-metrics.conf', @processes.command.last + end + + def test_unknown_role_is_rejected + assert_raises(KeyError) { TelegrafServiceWorker.new("unexpected", @logger, @console) } + end +end diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 17c63d5c32..ff4c002844 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -892,8 +892,8 @@ function Start-Fluent-Telegraf { $appInsightsKey = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($appInsightsAuth)).Trim() (Get-Content $amaLogsProcessMetricsConfFile).replace('placeholder_appinsights_key', $appInsightsKey) | Set-Content $amaLogsProcessMetricsConfFile Write-Host "Starting telegraf for collecting process metrics inside ama-logs containers (Windows)" - C:\opt\telegraf\telegraf.exe --service install --service-name telegraf-ama-logs-process-metrics --config $amaLogsProcessMetricsConfFile - C:\opt\telegraf\telegraf.exe --service start --service-name telegraf-ama-logs-process-metrics + Install-TelegrafService -ServiceName telegraf-ama-logs-process-metrics + Start-Service -Name telegraf-ama-logs-process-metrics -ErrorAction Stop } else { Write-Host "APPLICATIONINSIGHTS_AUTH or AKS_RESOURCE_ID not set, skipping ama-logs process metrics monitoring" } @@ -907,6 +907,32 @@ function Start-Fluent-Telegraf { Notepad.exe | Out-Null } +function Install-TelegrafService { + param( + [Parameter(Mandatory = $true)] + [ValidateSet("telegraf", "telegraf-ama-logs-process-metrics")] + [string]$ServiceName + ) + + $serviceHost = "C:\opt\amalogswindows\scripts\ruby\telegraf-windows-service.rb" + if (!(Test-Path -LiteralPath $serviceHost -PathType Leaf)) { + throw "Telegraf Windows service host not found: $serviceHost" + } + $ruby = (Get-Command ruby.exe -ErrorAction Stop).Source + $role = "prometheus" + $displayName = "Telegraf Data Collector Service" + if ($ServiceName -eq "telegraf-ama-logs-process-metrics") { + $role = "process-metrics" + $displayName = "Telegraf AMA Logs Process Metrics" + } + + # The stock executable's service detection requires SCM in session 0, while + # Windows containers can run SCM in another session. Reuse the shipped Ruby + # service dispatcher and run the unchanged executable as its console child. + $binaryPath = "`"$ruby`" `"$serviceHost`" $role" + New-Service -Name $ServiceName -BinaryPathName $binaryPath -DisplayName $displayName -StartupType Automatic -ErrorAction Stop | Out-Null +} + function Start-Telegraf { # Set default telegraf environment variables for prometheus scraping Write-Host "**********Setting default environment variables for telegraf prometheus plugin..." @@ -949,7 +975,7 @@ function Start-Telegraf { (Get-Content "C:\etc\telegraf\telegraf.conf").replace('placeholder_hostname', $hostName) | Set-Content "C:\etc\telegraf\telegraf.conf" Write-Host "Installing telegraf service" - C:\opt\telegraf\telegraf.exe --service install --config "C:\etc\telegraf\telegraf.conf" + Install-TelegrafService -ServiceName telegraf if (Test-FluentbitTcpListener -port 25229) { Write-Host "Fluentbit tcp listener is running on port 25229" @@ -973,16 +999,16 @@ function Start-Telegraf { Write-Host "exception occured in delayed telegraf start.. continuing without exiting" } Write-Host "Running telegraf service in test mode" - C:\opt\telegraf\telegraf.exe --config "C:\etc\telegraf\telegraf.conf" --test + C:\opt\telegraf\telegraf.exe --console --config "C:\etc\telegraf\telegraf.conf" --test Write-Host "Starting telegraf service" - C:\opt\telegraf\telegraf.exe --service start + Start-Service -Name telegraf -ErrorAction Stop # Trying to start telegraf again if it did not start due to fluent bit not being ready at startup Get-Service telegraf | findstr Running if ($? -eq $false) { Write-Host "trying to start telegraf in again in 30 seconds, since fluentbit might not have been ready..." Start-Sleep -s 30 - C:\opt\telegraf\telegraf.exe --service start + Start-Service -Name telegraf -ErrorAction Stop Get-Service telegraf } } diff --git a/kubernetes/windows/setup.ps1 b/kubernetes/windows/setup.ps1 index ae544148d2..0353243410 100644 --- a/kubernetes/windows/setup.ps1 +++ b/kubernetes/windows/setup.ps1 @@ -41,19 +41,23 @@ Write-Host ('Finished Installing Fluentbit') Write-Host ('Installing Telegraf'); try { - # For next telegraf update, make sure to update config changes in telegraf.conf, tomlparser-prom-customconfig.rb and tomlparser-osm-config.rb - $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.24.2_windows_amd64.zip' - Invoke-WebRequest -Uri $telegrafUri -OutFile /installation/telegraf.zip - Expand-Archive -Path /installation/telegraf.zip -Destination /installation/telegraf - Move-Item -Path /installation/telegraf/*/* -Destination /opt/telegraf/ -ErrorAction SilentlyContinue + # Update the Windows Telegraf configs and tomlparser-prom-customconfig.rb together with this package. + $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.40.0_windows_amd64.zip' + $telegrafSha256='9d85e3fa89d99e4b0e53e4aa40f069e828204cf5548ede9b9b7c95c31fe869dd' # DevSkim: ignore DS173237 -- Public archive SHA256, not a credential. + Invoke-WebRequest -Uri $telegrafUri -OutFile \installation\telegraf.zip -ErrorAction Stop + if ((Get-FileHash -Path \installation\telegraf.zip -Algorithm SHA256 -ErrorAction Stop).Hash -ne $telegrafSha256) { + throw "SHA256 mismatch for Telegraf Windows package" + } + Expand-Archive -Path \installation\telegraf.zip -Destination \installation\telegraf -ErrorAction Stop + Move-Item -Path \installation\telegraf\telegraf-1.40.0\* -Destination \opt\telegraf\ -ErrorAction Stop } catch { $ex = $_.Exception - Write-Host "exception while downloading telegraf for windows" + Write-Host "exception while installing telegraf for windows" Write-Host $ex exit 1 } -Write-Host ('Finished downloading Telegraf') +Write-Host ('Finished Installing Telegraf') Write-Host ('Installing Visual C++ Redistributable Package') $vcRedistLocation = 'https://aka.ms/vs/16/release/vc_redist.x64.exe' diff --git a/test/unit-tests/README.md b/test/unit-tests/README.md index 1987e12970..b3671d79c1 100644 --- a/test/unit-tests/README.md +++ b/test/unit-tests/README.md @@ -54,6 +54,24 @@ To run a specific PowerShell test file: ## Available Tests +### Windows Telegraf Package +`test_cases/Test-TelegrafPackage.ps1` exercises the Telegraf installation block from +`kubernetes/windows/setup.ps1` with mocked package operations. It checks the pinned +URL, SHA256 verification before extraction, archive layout, and fail-fast behavior +for download, hash, extraction, and move failures. It does not download packages or +modify the host installation. + +### Windows Telegraf Service +`test_cases/Test-TelegrafService.ps1` checks registration of both Telegraf roles +through the bundled Windows service dispatcher. Run +`ruby build/windows/installer/scripts/telegraf-windows-service_test.rb` for child +process lifecycle, shutdown, pre-spawn containment-failure, and unexpected-exit coverage. +On Windows, `ruby build/windows/installer/scripts/telegraf-windows-console_test.rb` +also verifies native containment when the host dies before `spawn` returns and +when console cleanup precedes host exit. It uses only bounded local Ruby fixture +processes, not installed services, Telegraf, or cluster access. These tests are +included in the Ruby test driver; native cases skip on non-Windows platforms. + ### Cloud Environment Detection (Linux & Windows) Tests the cloud environment detection logic which determines the Azure cloud environment from either: - Environment variable (CLUSTER_CLOUD_ENVIRONMENT) diff --git a/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 b/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 new file mode 100644 index 0000000000..576a61ed73 --- /dev/null +++ b/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 @@ -0,0 +1,91 @@ +$ErrorActionPreference = 'Stop' +# Windows PowerShell 5.1 otherwise reads the UTF-8 framework using the ANSI code page. +$framework = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..\test_framework.ps1') -Raw -Encoding UTF8 +. ([scriptblock]::Create($framework)) + +$setupPath = Join-Path $PSScriptRoot '..\..\..\kubernetes\windows\setup.ps1' +$tokens = $null +$parseErrors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($setupPath, [ref]$tokens, [ref]$parseErrors) +if ($parseErrors.Count -ne 0) { + throw "Cannot parse Windows setup.ps1: $parseErrors" +} + +# Execute only the real Telegraf install body, never the other installers or their cleanup. +$assignment = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.AssignmentStatementAst] -and + $node.Left.Extent.Text -eq '$telegrafUri' +}, $true) +if ($null -eq $assignment -or $assignment.Parent.Parent -isnot [System.Management.Automation.Language.TryStatementAst]) { + throw 'Cannot locate the Telegraf install try block' +} +$body = $assignment.Parent.Extent.Text +$installTelegraf = [scriptblock]::Create($body.Substring(1, $body.Length - 2)) + +# Keep command mocks scoped to this invocation, including when the main runner calls this file. +& { + function Record-Operation($Name, $Action) { + $script:operations += $Name + Assert-Equals 'Stop' $Action "$Name must fail on errors" | Out-Null + if ($script:failureAt -eq $Name) { + throw "Simulated $Name failure" + } + } + + function Invoke-WebRequest($Uri, $OutFile, $ErrorAction) { + Record-Operation 'download' $ErrorAction + Assert-Equals 'https://dl.influxdata.com/telegraf/releases/telegraf-1.40.0_windows_amd64.zip' $Uri 'official versioned ZIP' | Out-Null + Assert-Equals '\installation\telegraf.zip' $OutFile 'download destination' | Out-Null + } + + function Get-FileHash($Path, $Algorithm, $ErrorAction) { + Record-Operation 'hash' $ErrorAction + Assert-Equals '\installation\telegraf.zip' $Path 'hash the downloaded ZIP' | Out-Null + Assert-Equals 'SHA256' $Algorithm 'hash algorithm' | Out-Null + [pscustomobject]@{ Hash = $script:archiveHash } + } + + function Expand-Archive($Path, $Destination, $ErrorAction) { + Record-Operation 'extract' $ErrorAction + Assert-Equals '\installation\telegraf.zip' $Path 'extract the verified ZIP' | Out-Null + Assert-Equals '\installation\telegraf' $Destination 'extraction directory' | Out-Null + } + + function Move-Item($Path, $Destination, $ErrorAction) { + Record-Operation 'move' $ErrorAction + Assert-Equals '\installation\telegraf\telegraf-1.40.0\*' $Path 'versioned archive layout' | Out-Null + Assert-Equals '\opt\telegraf\' $Destination 'existing runtime and signing path' | Out-Null + } + + $cases = @( + @{ Name = 'valid package'; FailureAt = ''; Error = ''; Operations = 'download,hash,extract,move' }, + @{ Name = 'hash mismatch'; FailureAt = ''; Error = 'SHA256 mismatch for Telegraf Windows package'; Operations = 'download,hash' }, + @{ Name = 'download failure'; FailureAt = 'download'; Error = 'Simulated download failure'; Operations = 'download' }, + @{ Name = 'hash failure'; FailureAt = 'hash'; Error = 'Simulated hash failure'; Operations = 'download,hash' }, + @{ Name = 'extraction failure'; FailureAt = 'extract'; Error = 'Simulated extract failure'; Operations = 'download,hash,extract' }, + @{ Name = 'move failure'; FailureAt = 'move'; Error = 'Simulated move failure'; Operations = 'download,hash,extract,move' } + ) + foreach ($case in $cases) { + $script:operations = @() + $script:failureAt = $case.FailureAt + $script:archiveHash = '9D85E3FA89D99E4B0E53E4AA40F069E828204CF5548EDE9B9B7C95C31FE869DD' + if ($case.Name -eq 'hash mismatch') { + $script:archiveHash = '0' * 64 + } + $errorMessage = '' + try { + & $installTelegraf + } + catch { + $errorMessage = $_.Exception.Message + } + Assert-Equals $case.Error $errorMessage "$($case.Name): expected failure behavior" | Out-Null + Assert-Equals $case.Operations ($script:operations -join ',') "$($case.Name): no operations after failure" | Out-Null + } +} + +if (Print-TestSummary) { + exit 0 +} +exit 1 diff --git a/test/unit-tests/test_cases/Test-TelegrafService.ps1 b/test/unit-tests/test_cases/Test-TelegrafService.ps1 new file mode 100644 index 0000000000..1f678e9e66 --- /dev/null +++ b/test/unit-tests/test_cases/Test-TelegrafService.ps1 @@ -0,0 +1,61 @@ +$ErrorActionPreference = 'Stop' +$framework = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..\test_framework.ps1') -Raw -Encoding UTF8 +. ([scriptblock]::Create($framework)) + +$tokens = $null +$errors = $null +$path = Join-Path $PSScriptRoot '..\..\..\kubernetes\windows\main.ps1' +$ast = [System.Management.Automation.Language.Parser]::ParseFile($path, [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw "Cannot parse main.ps1: $errors" } +$function = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq 'Install-TelegrafService' +}, $true) +if ($null -eq $function) { throw 'Cannot locate Install-TelegrafService' } +. ([scriptblock]::Create($function.Extent.Text)) + +& { + function Test-Path($LiteralPath, $PathType) { + Assert-Equals 'C:\opt\amalogswindows\scripts\ruby\telegraf-windows-service.rb' $LiteralPath 'packaged host path' | Out-Null + Assert-Equals 'Leaf' $PathType 'host must be a file' | Out-Null + return $script:hostPresent + } + + function Get-Command($Name, $ErrorAction) { + Assert-Equals 'ruby.exe' $Name 'existing runtime' | Out-Null + Assert-Equals 'Stop' $ErrorAction 'missing runtime is fatal' | Out-Null + [pscustomobject]@{ Source = 'C:\Program Files\Ruby31\ruby.exe' } + } + + function New-Service($Name, $BinaryPathName, $DisplayName, $StartupType, $ErrorAction) { + $script:installed = [pscustomobject]@{ + Name = $Name; BinaryPath = $BinaryPathName; DisplayName = $DisplayName + } + Assert-Equals 'Automatic' $StartupType 'preserve service startup mode' | Out-Null + Assert-Equals 'Stop' $ErrorAction 'registration errors are fatal' | Out-Null + } + + $script:hostPresent = $true + foreach ($case in @( + @{ Name = 'telegraf'; Role = 'prometheus'; DisplayName = 'Telegraf Data Collector Service' }, + @{ Name = 'telegraf-ama-logs-process-metrics'; Role = 'process-metrics'; DisplayName = 'Telegraf AMA Logs Process Metrics' } + )) { + $script:installed = $null + Install-TelegrafService -ServiceName $case.Name + Assert-Equals $case.Name $script:installed.Name 'preserve service name' | Out-Null + Assert-Equals $case.DisplayName $script:installed.DisplayName 'distinct display names' | Out-Null + $expected = '"C:\Program Files\Ruby31\ruby.exe" "C:\opt\amalogswindows\scripts\ruby\telegraf-windows-service.rb" ' + $case.Role + Assert-Equals $expected $script:installed.BinaryPath 'quoted host path; no config path in wrapper command line' | Out-Null + } + + $script:hostPresent = $false + $script:installed = $null + $message = '' + try { Install-TelegrafService -ServiceName telegraf } catch { $message = $_.Exception.Message } + Assert-Equals 'Telegraf Windows service host not found: C:\opt\amalogswindows\scripts\ruby\telegraf-windows-service.rb' $message 'missing host must not fall back to broken native detection' | Out-Null + Assert-Equals 'True' ($null -eq $script:installed).ToString() 'no service registered after missing host' | Out-Null +} + +if (Print-TestSummary) { exit 0 } +exit 1 diff --git a/test/unit-tests/test_driver.rb b/test/unit-tests/test_driver.rb index c10e3725b5..6989bda2e6 100644 --- a/test/unit-tests/test_driver.rb +++ b/test/unit-tests/test_driver.rb @@ -15,3 +15,7 @@ Dir.glob(File.join(script_path, "../../build/common/installer/scripts/*_test.rb")) do |filename| require_relative filename end + +Dir.glob(File.join(script_path, "../../build/windows/installer/scripts/*_test.rb")) do |filename| + require_relative filename +end diff --git a/test/unit-tests/test_main.ps1 b/test/unit-tests/test_main.ps1 index 761d5543ae..c671aa266b 100644 --- a/test/unit-tests/test_main.ps1 +++ b/test/unit-tests/test_main.ps1 @@ -23,7 +23,9 @@ $testFiles = @( "Test-GetLogAnalyticsWorkspaceDomain.ps1", "Test-GetMcsEndpoint.ps1", "Test-GetMcsGlobalEndpoint.ps1", - "Test-IsCanaryRegion.ps1" + "Test-IsCanaryRegion.ps1", + "Test-TelegrafPackage.ps1", + "Test-TelegrafService.ps1" ) foreach ($testFile in $testFiles) { From 16153dd1739789c0ac0312a92efbcbb11bd4e0b5 Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Thu, 24 Sep 2026 22:25:30 +0000 Subject: [PATCH 3/4] windows telefraf upgrade 1.40.1 and deprecated field changes --- Dev Guide.md | 15 ++++---- .../scripts/tomlparser-prom-customconfig.rb | 18 ++-------- .../tomlparser-prom-customconfig_test.rb | 35 ++++++++++--------- .../telegraf-ama-logs-process-metrics.conf | 22 ++++++------ build/windows/installer/conf/telegraf.conf | 4 +-- kubernetes/windows/setup.ps1 | 6 ++-- .../test_cases/Test-TelegrafPackage.ps1 | 6 ++-- 7 files changed, 49 insertions(+), 57 deletions(-) diff --git a/Dev Guide.md b/Dev Guide.md index 8c8ac90c8c..527d0cf1c5 100644 --- a/Dev Guide.md +++ b/Dev Guide.md @@ -6,12 +6,12 @@ More advanced information needed to develop or build the docker provider will li ## Windows Telegraf dependency -`kubernetes/windows/setup.ps1` installs the official Telegraf 1.40.0 Windows AMD64 +`kubernetes/windows/setup.ps1` installs the official Telegraf 1.40.1 Windows AMD64 ZIP and verifies its pinned SHA256 before extraction. The package corresponds to -upstream commit `e9017dc3266369d6fa185e0e130af1d1d4021ce9`. The existing Windows +upstream commit `26b8f4478b676f4f5e5d8ce1622cdf4f6c273bda`. The existing Windows pipeline continues to sign `C:\opt\telegraf\telegraf.exe` as an OSS dependency. -The official binary is built with Go 1.27.0 for `windows/amd64`, `GOAMD64=v1`. +The official binary is built with Go 1.27.1 for `windows/amd64`, `GOAMD64=v1`. Go's [Windows OS floor](https://go.dev/wiki/MinimumRequirements#windows) is Windows 10 or Windows Server 2016 and newer. Both repository image targets, LTSC2019 and LTSC2022, meet that floor; this does not replace validation inside those images @@ -40,10 +40,11 @@ to preserve the overall 15-second metric-scrape limit; upstream's [1.40 documentation](https://github.com/influxdata/telegraf/blob/e9017dc3266369d6fa185e0e130af1d1d4021ce9/plugins/inputs/prometheus/README.md#L152-L157) explains that `response_timeout` now covers headers only, unlike the [1.24.2 client timeout](https://github.com/influxdata/telegraf/blob/9550e7a533dd00632e14435e87ed3eb4b04832c6/plugins/inputs/prometheus/prometheus.go#L254-L261). -Procstat uses `tag_with = ["pid"]` to retain PID tags. Existing `fieldpass`/`fielddrop` -names are unchanged on both OSes because -[1.40 still parses them](https://github.com/influxdata/telegraf/blob/e9017dc3266369d6fa185e0e130af1d1d4021ce9/config/config.go#L1649-L1687). -Linux rendering remains unchanged. Run `ruby build/common/installer/scripts/tomlparser-prom-customconfig_test.rb` +Procstat uses `tag_with = ["pid"]` to retain PID tags. Both OSes render +`fieldinclude`/`fieldexclude` instead of the deprecated `fieldpass`/`fielddrop` +Telegraf options. The public ConfigMap keys remain `fieldpass`/`fielddrop` for +backward compatibility; the shared parser maps them to the current Telegraf names. +Run `ruby build/common/installer/scripts/tomlparser-prom-customconfig_test.rb` for rendering coverage with and without namespace filters. On Windows, set `TELEGRAF_WINDOWS_BINARY` to the extracted `telegraf.exe` to also load the generated configs with that binary in bounded `--test` mode, without Kubernetes access or diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig.rb b/build/common/installer/scripts/tomlparser-prom-customconfig.rb index 3d7ddb07ad..610d108908 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig.rb @@ -186,18 +186,6 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu new_contents = new_contents.gsub("$AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR") new_contents = new_contents.gsub("$AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE") - timeout_config_key = "timeout" - - # 'fieldpass'/'fielddrop' were deprecated in telegraf 1.29.0 and removed in 1.40.0. - # Windows still ships telegraf 1.24.2, which predates the replacement options, so it - # must keep emitting the legacy option names. - field_include_key = "fieldinclude" - field_exclude_key = "fieldexclude" - if is_windows? - field_include_key = "fieldpass" - field_exclude_key = "fielddrop" - end - pluginConfigsWithNamespaces = "" podScrapeScope = (@controller.casecmp(@replicaset) == 0) ? "cluster" : "node" monitorKubernetesPodsNamespaces.each do |namespace| @@ -218,11 +206,11 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu monitor_kubernetes_pods_namespace = #{toTomlBasicString(namespace)} kubernetes_label_selector = #{toTomlBasicString(kubernetesLabelSelectors)} kubernetes_field_selector = #{toTomlBasicString(kubernetesFieldSelectors)} - #{field_include_key} = #{fieldPassSetting} - #{field_exclude_key} = #{fieldDropSetting} + fieldinclude = #{fieldPassSetting} + fieldexclude = #{fieldDropSetting} metric_version = #{@metricVersion} url_tag = #{toTomlBasicString(@urlTag)} - #{timeout_config_key} = #{toTomlBasicString(@responseTimeout)} + timeout = #{toTomlBasicString(@responseTimeout)} tls_ca = #{toTomlBasicString(@tlsCa)} insecure_skip_verify = #{@insecureSkipVerify}\n" end diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb index c30ede5526..bb9da4b5b1 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb @@ -242,9 +242,6 @@ def test_kubernetes_namespace_validation # --- end to end generation ---------------------------------------------------------------- SCENARIOS.each_key do |scenario| - field_include_key = scenario == :windows ? "fieldpass" : "fieldinclude" - field_exclude_key = scenario == :windows ? "fielddrop" : "fieldexclude" - define_method("test_interval_breakout_is_neutralized_#{scenario}") do result = run_parser(scenario, configmap_for(scenario, "interval = '''#{BREAKOUT}'''")) assert_no_injected_plugins(scenario, result[:conf], "the #{scenario} interval") @@ -258,20 +255,24 @@ def test_kubernetes_namespace_validation result = run_parser(scenario, configmap_for(scenario, "fieldpass = ['''#{BREAKOUT}''']")) assert_no_injected_plugins(scenario, result[:conf], "the #{scenario} fieldpass array") # The value survives, but only as a single escaped string. - assert_includes collect_values(result[:conf], field_include_key), BREAKOUT + assert_includes collect_values(result[:conf], "fieldinclude"), BREAKOUT end define_method("test_fielddrop_breakout_is_neutralized_#{scenario}") do result = run_parser(scenario, configmap_for(scenario, "fielddrop = ['''#{BREAKOUT}''']")) assert_no_injected_plugins(scenario, result[:conf], "the #{scenario} fielddrop array") - assert_includes collect_values(result[:conf], field_exclude_key), BREAKOUT + assert_includes collect_values(result[:conf], "fieldexclude"), BREAKOUT end define_method("test_valid_settings_are_preserved_#{scenario}") do result = run_parser(scenario, configmap_for(scenario, "interval = \"45s\"\nfieldpass = [\"a\",\"b\"]\nfielddrop = [\"c\"]")) assert_includes result[:conf], "interval = \"45s\"", "a valid interval must be preserved" - assert_includes result[:conf], "#{field_include_key} = [\"a\",\"b\"]", "array formatting must be unchanged" - assert_includes result[:conf], "#{field_exclude_key} = [\"c\"]", "array formatting must be unchanged" + assert_includes result[:conf], "fieldinclude = [\"a\",\"b\"]", "array formatting must be unchanged" + assert_includes result[:conf], "fieldexclude = [\"c\"]", "array formatting must be unchanged" + parse_generated_toml(result[:conf])["inputs"]["prometheus"].each do |plugin| + refute plugin.key?("fieldpass") + refute plugin.key?("fielddrop") + end assert_no_injected_plugins(scenario, result[:conf], "a benign #{scenario} configuration") end end @@ -337,8 +338,10 @@ def test_namespace_plugins_are_generated_for_valid_namespaces assert_equal "scrapeUrl", plugin["url_tag"] assert_equal "pod_namespace", plugin["pod_namespace_label_name"] assert_equal (scenario == :replicaset ? "cluster" : "node"), plugin["pod_scrape_scope"] - assert_equal ["requests_total"], plugin["fieldpass"] - assert_equal ["debug_total"], plugin["fielddrop"] + assert_equal ["requests_total"], plugin["fieldinclude"] + assert_equal ["debug_total"], plugin["fieldexclude"] + refute plugin.key?("fieldpass") + refute plugin.key?("fielddrop") assert_equal "app=metrics", plugin["kubernetes_label_selector"] assert_equal "spec.nodeName=test-node", plugin["kubernetes_field_selector"] end @@ -379,6 +382,7 @@ def test_windows_rendered_configs_load_in_packaged_telegraf output = readers.map(&:value).join("\n") assert completed, "Telegraf config smoke test exceeded 20 seconds" assert process.value.success?, "Telegraf rejected the rendered config for #{namespaces.inspect}: #{output}" + refute_match(/Deprecation(?:Error|Warning).*"(?:fieldpass|fielddrop|pid_tag)"/, output) end end end @@ -392,7 +396,8 @@ def test_windows_process_metrics_config_preserves_pid_tags_and_fields config["inputs"]["procstat"].each do |plugin| assert_equal ["pid"], plugin["tag_with"] refute plugin.key?("pid_tag") - assert_equal ["cpu_usage", "memory_rss"], plugin["fieldpass"] + assert_equal ["cpu_usage", "memory_rss"], plugin["fieldinclude"] + refute plugin.key?("fieldpass") assert_equal "native", plugin["pid_finder"] assert_equal "agent_telemetry", plugin["name_override"] assert_equal "t.azm.ms/", plugin["name_prefix"] @@ -415,9 +420,8 @@ def test_selectors_are_escaped_in_generated_namespace_plugins assert_equal BREAKOUT, plugin["kubernetes_label_selector"] end - def test_namespace_filters_preserve_platform_keys_and_escaping + def test_namespace_filters_preserve_modern_keys_and_escaping [:replicaset, :sidecar, :windows].each do |scenario| - field_include_key, field_exclude_key = scenario == :windows ? ["fieldpass", "fielddrop"] : ["fieldinclude", "fieldexclude"] body = "monitor_kubernetes_pods = true\n" \ "monitor_kubernetes_pods_namespaces = [\"default\"]\n" \ "fieldpass = ['''#{BREAKOUT}''']\n" \ @@ -429,13 +433,12 @@ def test_namespace_filters_preserve_platform_keys_and_escaping plugin = parse_generated_toml(result[:conf])["inputs"]["prometheus"] .find { |entry| entry["monitor_kubernetes_pods_namespace"] == "default" } refute_nil plugin - assert_equal [BREAKOUT], plugin[field_include_key] - assert_equal [BREAKOUT], plugin[field_exclude_key] + assert_equal [BREAKOUT], plugin["fieldinclude"] + assert_equal [BREAKOUT], plugin["fieldexclude"] assert_equal BREAKOUT, plugin["kubernetes_label_selector"] assert_equal BREAKOUT, plugin["kubernetes_field_selector"] assert_equal(scenario == :replicaset ? "cluster" : "node", plugin["pod_scrape_scope"]) - unsupported_keys = scenario == :windows ? ["fieldinclude", "fieldexclude"] : ["fieldpass", "fielddrop"] - unsupported_keys.each { |key| refute plugin.key?(key), "#{scenario} must not emit #{key}" } + ["fieldpass", "fielddrop"].each { |key| refute plugin.key?(key), "#{scenario} must not emit #{key}" } end end diff --git a/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf b/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf index 9f004fda13..3a56e1d546 100644 --- a/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf +++ b/build/windows/installer/conf/telegraf-ama-logs-process-metrics.conf @@ -22,7 +22,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -38,7 +38,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -54,7 +54,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -70,7 +70,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -86,7 +86,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -102,7 +102,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -118,7 +118,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -134,7 +134,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -150,7 +150,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -166,7 +166,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" @@ -182,7 +182,7 @@ pid_finder = "native" tag_with = ["pid"] name_override = "agent_telemetry" - fieldpass = ["cpu_usage", "memory_rss"] + fieldinclude = ["cpu_usage", "memory_rss"] [inputs.procstat.tags] Computer = "placeholder_hostname" PodName = "placeholder_podname" diff --git a/build/windows/installer/conf/telegraf.conf b/build/windows/installer/conf/telegraf.conf index 8bf96e3f66..85c645c7d8 100644 --- a/build/windows/installer/conf/telegraf.conf +++ b/build/windows/installer/conf/telegraf.conf @@ -136,8 +136,8 @@ $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR - fieldpass = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS - fielddrop = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP + fieldinclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS + fieldexclude = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP metric_version = 2 url_tag = "scrapeUrl" diff --git a/kubernetes/windows/setup.ps1 b/kubernetes/windows/setup.ps1 index 0353243410..6ce8ec42bc 100644 --- a/kubernetes/windows/setup.ps1 +++ b/kubernetes/windows/setup.ps1 @@ -42,14 +42,14 @@ Write-Host ('Finished Installing Fluentbit') Write-Host ('Installing Telegraf'); try { # Update the Windows Telegraf configs and tomlparser-prom-customconfig.rb together with this package. - $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.40.0_windows_amd64.zip' - $telegrafSha256='9d85e3fa89d99e4b0e53e4aa40f069e828204cf5548ede9b9b7c95c31fe869dd' # DevSkim: ignore DS173237 -- Public archive SHA256, not a credential. + $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.40.1_windows_amd64.zip' + $telegrafSha256='cabe07907628afc17ce8c58a1c27b3a55a838b1fb9418b2c05e9342e6e8af8d9' # DevSkim: ignore DS173237 -- Public archive SHA256, not a credential. Invoke-WebRequest -Uri $telegrafUri -OutFile \installation\telegraf.zip -ErrorAction Stop if ((Get-FileHash -Path \installation\telegraf.zip -Algorithm SHA256 -ErrorAction Stop).Hash -ne $telegrafSha256) { throw "SHA256 mismatch for Telegraf Windows package" } Expand-Archive -Path \installation\telegraf.zip -Destination \installation\telegraf -ErrorAction Stop - Move-Item -Path \installation\telegraf\telegraf-1.40.0\* -Destination \opt\telegraf\ -ErrorAction Stop + Move-Item -Path \installation\telegraf\telegraf-1.40.1\* -Destination \opt\telegraf\ -ErrorAction Stop } catch { $ex = $_.Exception diff --git a/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 b/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 index 576a61ed73..026de0dcaf 100644 --- a/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 +++ b/test/unit-tests/test_cases/Test-TelegrafPackage.ps1 @@ -35,7 +35,7 @@ $installTelegraf = [scriptblock]::Create($body.Substring(1, $body.Length - 2)) function Invoke-WebRequest($Uri, $OutFile, $ErrorAction) { Record-Operation 'download' $ErrorAction - Assert-Equals 'https://dl.influxdata.com/telegraf/releases/telegraf-1.40.0_windows_amd64.zip' $Uri 'official versioned ZIP' | Out-Null + Assert-Equals 'https://dl.influxdata.com/telegraf/releases/telegraf-1.40.1_windows_amd64.zip' $Uri 'official versioned ZIP' | Out-Null Assert-Equals '\installation\telegraf.zip' $OutFile 'download destination' | Out-Null } @@ -54,7 +54,7 @@ $installTelegraf = [scriptblock]::Create($body.Substring(1, $body.Length - 2)) function Move-Item($Path, $Destination, $ErrorAction) { Record-Operation 'move' $ErrorAction - Assert-Equals '\installation\telegraf\telegraf-1.40.0\*' $Path 'versioned archive layout' | Out-Null + Assert-Equals '\installation\telegraf\telegraf-1.40.1\*' $Path 'versioned archive layout' | Out-Null Assert-Equals '\opt\telegraf\' $Destination 'existing runtime and signing path' | Out-Null } @@ -69,7 +69,7 @@ $installTelegraf = [scriptblock]::Create($body.Substring(1, $body.Length - 2)) foreach ($case in $cases) { $script:operations = @() $script:failureAt = $case.FailureAt - $script:archiveHash = '9D85E3FA89D99E4B0E53E4AA40F069E828204CF5548EDE9B9B7C95C31FE869DD' + $script:archiveHash = 'CABE07907628AFC17CE8C58A1C27B3A55A838B1FB9418B2C05E9342E6E8AF8D9' if ($case.Name -eq 'hash mismatch') { $script:archiveHash = '0' * 64 } From 8b5c889282ec3259c7a8367fb9bb7262b16ebb7a Mon Sep 17 00:00:00 2001 From: Sunil Yadav Date: Mon, 5 Oct 2026 18:08:16 +0000 Subject: [PATCH 4/4] tests --- .../tomlparser-prom-customconfig_test.rb | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb index bb9da4b5b1..f2b5b3df49 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig_test.rb @@ -409,6 +409,51 @@ def test_windows_process_metrics_config_preserves_pid_tags_and_fields ) end + def test_linux_process_metrics_config_preserves_pid_tags_and_fields + path = File.join(REPO_ROOT, "build/linux/installer/conf/telegraf-ama-logs-process-metrics.conf") + config = Tomlrb.load_file(path) + assert_equal({ "telegraf_role" => "ama-logs-process-metrics" }, config["global_tags"]) + assert_equal 9, config["inputs"]["procstat"].length + config["inputs"]["procstat"].each do |plugin| + assert_equal ["pid"], plugin["tag_with"] + refute plugin.key?("pid_tag") + assert_equal ["cpu_usage", "memory_rss"], plugin["fieldinclude"] + refute plugin.key?("fieldpass") + assert_equal "native", plugin["pid_finder"] + assert_equal "agent_telemetry", plugin["name_override"] + assert_equal "t.azm.ms/", plugin["name_prefix"] + assert_equal "$CONTROLLER_TYPE", plugin["tags"]["ControllerType"] + # ORDER matters: renamed keys must stay above [inputs.procstat.tags], otherwise + # TOML binds them to the nested table and telegraf never sees them. + ["tag_with", "fieldinclude"].each do |key| + refute plugin["tags"].key?(key), "#{key} must not be absorbed by [inputs.procstat.tags]" + end + end + assert_equal( + { "ai.cloud.role" => "ControllerType", "ai.cloud.roleInstance" => "PodName" }, + config["outputs"]["application_insights"].first["context_tag_sources"] + ) + end + + # Telegraf treats an unrecognized plugin option as a fatal config-load error, so a single + # reintroduced legacy name silently stops the agent from loading that configuration. + REMOVED_TELEGRAF_OPTIONS = %w[fieldpass fielddrop pid_tag ignore_protocol_stats].freeze + + def test_shipped_telegraf_templates_do_not_use_removed_options + templates = Dir.glob(File.join(REPO_ROOT, "build/{linux,windows}/installer/conf/telegraf*.conf")).sort + refute_empty templates, "expected to find shipped telegraf templates" + + templates.each do |path| + offenders = File.readlines(path).each_with_index.filter_map do |line, index| + stripped = line.strip + next if stripped.empty? || stripped.start_with?("#") + option = REMOVED_TELEGRAF_OPTIONS.find { |name| stripped =~ /\A#{name}\s*=/ } + "#{File.basename(path)}:#{index + 1}: #{stripped}" if option + end + assert_empty offenders, "removed telegraf options must not be used in shipped templates" + end + end + def test_selectors_are_escaped_in_generated_namespace_plugins body = "monitor_kubernetes_pods = true\n" \ "monitor_kubernetes_pods_namespaces = [\"default\"]\n" \