feat(metrics): add opt-in BEAM runtime metrics collector

lib/sentry/application.ex

@@ -86,18 +86,34 @@ defmodule Sentry.Application do
     with {:ok, pid} <-
            Supervisor.start_link(children, strategy: :one_for_one, name: Sentry.Supervisor) do
       start_integrations(integrations_config)
+      maybe_attach_runtime_metrics()
       maybe_add_logger_handler()
       {:ok, pid}
     end
   end
 
   @impl true
   def prep_stop(state) do
+    maybe_detach_runtime_metrics()
     # Flush while the telemetry processor and its HTTP client are still alive.
     Sentry.flush()
     state
   end
 
+  defp maybe_attach_runtime_metrics do
+    runtime_config = Config.metrics()[:runtime]
+
+    if runtime_config[:enabled] do
+      Sentry.Metrics.Runtime.attach(runtime_config)
+    end
+  end
+
+  defp maybe_detach_runtime_metrics do
+    if Config.metrics()[:runtime][:enabled] do
+      Sentry.Metrics.Runtime.detach()
+    end
+  end
+
   defp cache_loaded_applications do
     apps_with_vsns =
       if Config.report_deps?() do

lib/sentry/config.ex

@@ -504,6 +504,45 @@ defmodule Sentry.Config do
       default: [],
       keys: integrations_schema
     ],
+    metrics: [
+      type: :keyword_list,
+      default: [],
+      doc: """
+      Configuration for metrics the SDK collects on its own.
+      *Available since 14.0.0*.
+      """,
+      keys: [
+        runtime: [
+          type: :keyword_list,
+          default: [],
+          doc: """
+          Configuration for BEAM runtime metrics.
+          *Available since 14.0.0*.
+          """,
+          keys: [
+            enabled: [
+              type: :boolean,
+              default: false,
+              doc: """
+              Whether to report BEAM runtime metrics.
+              *Available since 14.0.0*.
+              """
+            ],
+            version_attributes: [
+              type: :boolean,
+              default: false,
+              doc: """
+              Whether to attach `elixir_version` and `otp_release` attributes to every
+              reported measurement, so metrics can be grouped by runtime version. Off by
+              default: the versions change only on upgrade, and attaching them to every
+              point starts a fresh series for each metric on every rolling deploy.
+              *Available since 14.0.0*.
+              """
+            ]
+          ]
+        ]
+      ]
+    ],
     logs: [
       type: {:or, [{:in, [nil]}, {:keyword_list, logs_schema}]},
       type_doc: "`t:keyword/0` or `nil`",
@@ -1127,6 +1166,9 @@ defmodule Sentry.Config do
   @spec integrations() :: keyword()
   def integrations, do: fetch!(:integrations)
 
+  @spec metrics() :: keyword()
+  def metrics, do: fetch!(:metrics)
+
   @spec tracing?() :: boolean()
   def tracing? do
     (Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() and

lib/sentry/metrics.ex

@@ -25,6 +25,43 @@ defmodule Sentry.Metrics do
       # Record a distribution
       Sentry.Metrics.distribution("response.time", 42.5, unit: "millisecond")
 
+  ## Automatically Collected Metrics
+
+  The SDK can also report BEAM runtime health on its own, without any calls to the
+  functions above. This is opt-in:
+
+      config :sentry,
+        metrics: [
+          runtime: [enabled: true]
+        ]
+
+  Once enabled, these gauges are reported:
+
+    * `elixir.runtime.mem.*` — `total`, `processes`, `processes_used`, `system`,
+      `atom`, `atom_used`, `binary`, `code` and `ets`, in bytes
+
+  ### Collection Frequency
+
+  The SDK does not collect these itself. It listens to the events that
+  [telemetry_poller](https://hexdocs.pm/telemetry_poller) already emits, so how often
+  they are reported is how often that library polls. Its default poller runs every
+  5 seconds, which is more often than most applications need for runtime health and
+  costs metric volume, so consider slowing it down:
+
+      config :telemetry_poller, default: [period: 30_000]
+
+  Add the dependency if you do not have it already:
+
+      {:telemetry_poller, "~> 1.0"}
+
+  ### Runtime Version Attributes
+
+  Setting `version_attributes: true` adds `elixir_version` and `otp_release`
+  attributes to every reported measurement, so metrics can be grouped by runtime
+  version. It is off by default, because those values change only on upgrade and
+  attaching them to every point starts a fresh series for each metric on every
+  rolling deploy.
+
   ## Configuration
 
   Metrics can be filtered using the `:before_send_metric` callback:

lib/sentry/metrics/runtime.ex

@@ -0,0 +1,79 @@
+defmodule Sentry.Metrics.Runtime do
+  @moduledoc false
+
+  alias Sentry.Metrics
+
+  @handler_id "sentry-runtime-metrics"
+  @origin "auto.elixir.runtime_metrics"
+
+  @memory_event [:vm, :memory]
+
+  @events [@memory_event]
+
+  @memory_keys [
+    :total,
+    :processes,
+    :processes_used,
+    :system,
+    :atom,
+    :atom_used,
+    :binary,
+    :code,
+    :ets
+  ]
+
+  @spec attach(keyword()) :: :ok
+  def attach(opts) when is_list(opts) do
+    :ok = detach()
+
+    _ = :telemetry.attach_many(@handler_id, @events, &__MODULE__.handle_event/4, config(opts))
+
+    :ok
+  end
+
+  @spec detach() :: :ok
+  def detach do
+    _ = :telemetry.detach(@handler_id)
+    :ok
+  end
+
+  @doc false
+  @spec handle_event(
+          :telemetry.event_name(),
+          :telemetry.event_measurements(),
+          :telemetry.event_metadata(),
+          :telemetry.handler_config()
+        ) :: :ok
+  def handle_event(@memory_event, measurements, _metadata, config) do
+    report_measured(config, measurements, @memory_keys, "elixir.runtime.mem", "byte")
+  end
+
+  defp report_measured(config, measurements, keys, prefix, unit) do
+    Enum.each(Map.take(measurements, keys), fn {key, value} ->
+      gauge(config, "#{prefix}.#{key}", value, unit)
+    end)
+  end
+
+  defp config(opts) do
+    attributes =
+      Map.merge(
+        %{"sentry.origin" => @origin},
+        version_attributes(Keyword.get(opts, :version_attributes, false))
+      )
+
+    %{attributes: attributes}
+  end
+
+  defp version_attributes(false), do: %{}
+
+  defp version_attributes(true) do
+    %{
+      "elixir_version" => System.version(),
+      "otp_release" => List.to_string(:erlang.system_info(:otp_release))
+    }
+  end
+
+  defp gauge(%{attributes: attributes}, name, value, unit) do
+    Metrics.gauge(name, value, unit: unit, attributes: attributes)
+  end
+end

mix-1.13-1.14.lock

@@ -70,6 +70,7 @@
   "spitfire": {:hex, :spitfire, "0.2.1", "29e154873f05444669c7453d3d931820822cbca5170e88f0f8faa1de74a79b47", [:mix], [], "hexpm", "6eeed75054a38341b2e1814d41bb0a250564092358de2669fdb57ff88141d91b"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
+  "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "telemetry_registry": {:hex, :telemetry_registry, "0.3.2", "701576890320be6428189bff963e865e8f23e0ff3615eade8f78662be0fc003c", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7ed191eb1d115a3034af8e1e35e4e63d5348851d556646d46ca3d1b4e16bab9"},
   "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},

mix-1.15-1.17.lock

@@ -73,6 +73,7 @@
   "spitfire": {:hex, :spitfire, "0.4.1", "69e90335d00ca328295e1e1e77cac5d7575aa6d34274e3467ebfc654b8858be3", [:mix], [], "hexpm", "27d86f67681179682b15c6758d64ac2eb2b3637ed8340800c8b885c69754cdcd"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
+  "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "telemetry_registry": {:hex, :telemetry_registry, "0.3.2", "701576890320be6428189bff963e865e8f23e0ff3615eade8f78662be0fc003c", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7ed191eb1d115a3034af8e1e35e4e63d5348851d556646d46ca3d1b4e16bab9"},
   "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},

mix-1.18.lock

@@ -74,6 +74,7 @@
   "spitfire": {:hex, :spitfire, "0.4.1", "69e90335d00ca328295e1e1e77cac5d7575aa6d34274e3467ebfc654b8858be3", [:mix], [], "hexpm", "27d86f67681179682b15c6758d64ac2eb2b3637ed8340800c8b885c69754cdcd"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
+  "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "telemetry_registry": {:hex, :telemetry_registry, "0.3.2", "701576890320be6428189bff963e865e8f23e0ff3615eade8f78662be0fc003c", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7ed191eb1d115a3034af8e1e35e4e63d5348851d556646d46ca3d1b4e16bab9"},
   "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},

mix.exs

@@ -140,6 +140,7 @@ defmodule Sentry.Mixfile do
       {:phoenix_live_view, "~> 0.20 or ~> 1.0", optional: true},
       {:plug, dep_version(:plug, current_elixir_version()), optional: true},
       {:telemetry, "~> 0.4 or ~> 1.0", optional: true},
+      {:telemetry_poller, "~> 1.0", optional: true},
 
       # Dev and test dependencies
       {:igniter, dep_version(:igniter, current_elixir_version()), only: :test},

mix.lock

@@ -72,6 +72,7 @@
   "spitfire": {:hex, :spitfire, "0.4.1", "69e90335d00ca328295e1e1e77cac5d7575aa6d34274e3467ebfc654b8858be3", [:mix], [], "hexpm", "27d86f67681179682b15c6758d64ac2eb2b3637ed8340800c8b885c69754cdcd"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
+  "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "telemetry_registry": {:hex, :telemetry_registry, "0.3.2", "701576890320be6428189bff963e865e8f23e0ff3615eade8f78662be0fc003c", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7ed191eb1d115a3034af8e1e35e4e63d5348851d556646d46ca3d1b4e16bab9"},
   "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},

test/sentry/application_test.exs

@@ -361,6 +361,35 @@ defmodule Sentry.ApplicationTest do
     end
   end
 
+  describe "runtime metrics" do
+    test "the handler is not attached by default" do
+      restart_sentry_with([])
+
+      refute runtime_metrics_attached?()
+    end
+
+    test "the handler is attached when runtime metrics are enabled" do
+      restart_sentry_with(metrics: [runtime: [enabled: true]])
+
+      assert runtime_metrics_attached?()
+    end
+
+    test "the handler is detached when the application stops" do
+      restart_sentry_with(metrics: [runtime: [enabled: true]])
+      assert runtime_metrics_attached?()
+
+      assert :ok = Application.stop(:sentry)
+
+      refute runtime_metrics_attached?()
+    end
+  end
+
+  defp runtime_metrics_attached? do
+    [:vm, :memory]
+    |> :telemetry.list_handlers()
+    |> Enum.any?(&(&1.id == "sentry-runtime-metrics"))
+  end
+
   defp hold_response(conn, owner) do
     send(owner, {:request_started, self()})
 

test/sentry/config_test.exs

@@ -487,6 +487,20 @@ defmodule Sentry.ConfigTest do
     end
   end
 
+  describe ":metrics" do
+    test "leaves runtime metrics disabled by default" do
+      runtime = Config.validate!([])[:metrics][:runtime]
+
+      assert runtime[:enabled] == false
+    end
+
+    test "does not attach runtime version attributes by default" do
+      runtime = Config.validate!([])[:metrics][:runtime]
+
+      assert runtime[:version_attributes] == false
+    end
+  end
+
   describe ":org_id" do
     test "defaults to nil" do
       assert Config.validate!([])[:org_id] == nil