From a85f64342275129f3e2767748f1489e118d676cc Mon Sep 17 00:00:00 2001 From: kadekillary Date: Sun, 16 Aug 2026 23:40:11 -0700 Subject: [PATCH 1/7] feat(config): add telemetry kill switch --- docs/CONFIGURATION.md | 18 ++++++ lib/langfuse.rb | 23 ++++++-- lib/langfuse/client.rb | 69 ++++++++++++++++------- lib/langfuse/config.rb | 45 +++++++++++++++ lib/langfuse/deferred_api_client.rb | 40 +++++++++++++ lib/langfuse/noop_score_client.rb | 42 ++++++++++++++ spec/langfuse/client_spec.rb | 44 +++++++++++++++ spec/langfuse/config_spec.rb | 55 ++++++++++++++++++ spec/langfuse/deferred_api_client_spec.rb | 36 ++++++++++++ spec/langfuse/noop_score_client_spec.rb | 14 +++++ spec/langfuse_spec.rb | 33 +++++++++++ spec/spec_helper.rb | 2 + 12 files changed, 396 insertions(+), 25 deletions(-) create mode 100644 lib/langfuse/deferred_api_client.rb create mode 100644 lib/langfuse/noop_score_client.rb create mode 100644 spec/langfuse/deferred_api_client_spec.rb create mode 100644 spec/langfuse/noop_score_client_spec.rb diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 18fbbd7..76f6ef7 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -294,6 +294,23 @@ Session-only and dataset-run-only scores are still sent because they are not tie For Ruby client instances, `sample_rate` is snapshotted when the client is built. Changing `config.sample_rate` later does not update that client's score sampler or the already-initialized trace sampler. Rebuild the client with `Langfuse.reset!` when changing sampling behavior. +#### `tracing_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Environment:** `LANGFUSE_TRACING_ENABLED` +- **Description:** Enables or disables Langfuse tracing and scoring + +```ruby +Langfuse.configure do |config| + config.tracing_enabled = false +end +``` + +Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. The standard `OTEL_SDK_DISABLED=true` setting always disables Langfuse telemetry. + +When disabled, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Call `Langfuse.reset!` after changing this setting on an initialized SDK. + #### `logger` - **Type:** Logger @@ -804,6 +821,7 @@ Client-readiness rules: - `base_url` must be an absolute HTTP or HTTPS URL - `batch_size` must be a positive Integer - `score_queue_capacity` must be a positive Integer +- `tracing_enabled` must be `true` or `false` - `flush_interval`, `timeout`, `cache_max_size`, `cache_lock_timeout`, and `cache_refresh_threads` must be positive numbers - `cache_ttl` must be a non-negative number - `cache_stale_ttl` must be a non-negative number or `:indefinite` diff --git a/lib/langfuse.rb b/lib/langfuse.rb index bb7eff0..a16eea1 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -81,6 +81,8 @@ class UnauthorizedError < ApiError; end require_relative "langfuse/trace_id" require_relative "langfuse/score_value" require_relative "langfuse/score_client" +require_relative "langfuse/noop_score_client" +require_relative "langfuse/deferred_api_client" require_relative "langfuse/prompt_renderer" require_relative "langfuse/text_prompt_client" require_relative "langfuse/chat_prompt_client" @@ -144,7 +146,11 @@ def client # # @return [Boolean] true when the local configuration is valid def configured? - configuration.valid? + config = configuration + return config.valid? if config.telemetry_enabled? + + config.validate_telemetry_disabled! + true rescue ConfigurationError # Reading `configuration` builds it from the environment, which can fail # on its own before there is anything to validate. @@ -153,7 +159,7 @@ def configured? # Return Langfuse's internal tracer provider for explicit global OpenTelemetry installation. # - # @return [OpenTelemetry::SDK::Trace::TracerProvider] + # @return [OpenTelemetry::SDK::Trace::TracerProvider, OpenTelemetry::Trace::TracerProvider] # @raise [ConfigurationError] if tracing configuration is invalid # # @example @@ -164,6 +170,8 @@ def configured? # # OpenTelemetry.tracer_provider = Langfuse.tracer_provider def tracer_provider + return noop_tracer_provider unless configuration.telemetry_enabled? + OtelSetup.setup(configuration) unless OtelSetup.initialized? OtelSetup.tracer_provider rescue ConfigurationError => e @@ -322,7 +330,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -441,12 +449,14 @@ def reset! @configuration = nil @client = nil @noop_tracer = nil + @noop_tracer_provider = nil @emitted_warnings = nil rescue StandardError # Ignore shutdown errors during reset (e.g., in tests) @configuration = nil @client = nil @noop_tracer = nil + @noop_tracer_provider = nil @emitted_warnings = nil end @@ -639,6 +649,7 @@ def wrap_otel_span(otel_span, type_str, otel_tracer, attributes: nil) end def ensure_tracing_started + return false unless configuration.telemetry_enabled? return true if OtelSetup.initialized? OtelSetup.setup(configuration) @@ -683,7 +694,11 @@ def warning_mutex end def noop_tracer - @noop_tracer ||= OpenTelemetry::Trace::TracerProvider.new.tracer(LANGFUSE_TRACER_NAME, Langfuse::VERSION) + @noop_tracer ||= noop_tracer_provider.tracer(LANGFUSE_TRACER_NAME, Langfuse::VERSION) + end + + def noop_tracer_provider + @noop_tracer_provider ||= OpenTelemetry::Trace::TracerProvider.new end end # rubocop:enable Metrics/ClassLength diff --git a/lib/langfuse/client.rb b/lib/langfuse/client.rb index 11098a9..4410a1c 100644 --- a/lib/langfuse/client.rb +++ b/lib/langfuse/client.rb @@ -29,7 +29,7 @@ class Client # @return [Config] The client configuration attr_reader :config - # @return [ApiClient] The underlying API client + # @return [ApiClient, DeferredApiClient] The underlying or deferred API client attr_reader :api_client # Pure pass-throughs to {ApiClient}. See {ApiClient} for parameter and @@ -73,29 +73,12 @@ class Client # @return [Client] def initialize(config) @config = config - @config.validate! - - # Create cache if enabled - cache = create_cache if cache_enabled? - - # Create API client with cache - @api_client = ApiClient.new( - public_key: config.public_key, - secret_key: config.secret_key, - base_url: config.base_url, - timeout: config.timeout, - logger: config.logger, - cache: cache, - cache_observer: config.prompt_cache_observer - ) - + @telemetry_enabled = config.telemetry_enabled? @project_id = nil # One-shot lookup: avoids repeated blocking API calls in URL helpers # (trace_url, dataset_url, dataset_run_url) when the project endpoint is down. @project_id_fetched = false - - # Initialize score client for batching score events - @score_client = ScoreClient.new(api_client: @api_client, config: config) + initialize_telemetry_clients end # Fetch a prompt and return the appropriate client @@ -385,6 +368,8 @@ def dataset_run_url(dataset_id:, dataset_run_id:) # rubocop:disable Metrics/ParameterLists def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless telemetry_enabled? + @score_client.create( name: name, value: value, @@ -416,7 +401,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -426,6 +411,8 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # rubocop:disable Metrics/ParameterLists def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless telemetry_enabled? + @score_client.create!( name: name, value: value, @@ -460,6 +447,8 @@ def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observ # client.score_active_observation(name: "accuracy", value: 0.92) # end def score_active_observation(name:, value:, comment: nil, metadata: nil, data_type: :numeric) + return unless telemetry_enabled? + @score_client.score_active_observation( name: name, value: value, @@ -486,6 +475,8 @@ def score_active_observation(name:, value:, comment: nil, metadata: nil, data_ty # client.score_active_trace(name: "overall_quality", value: 5) # end def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :numeric) + return unless telemetry_enabled? + @score_client.score_active_trace( name: name, value: value, @@ -504,6 +495,8 @@ def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :n # @example # client.flush_scores def flush_scores + return unless telemetry_enabled? + @score_client.flush end @@ -715,6 +708,40 @@ def run_experiment(name:, task:, data: nil, dataset_name: nil, description: nil, private + def initialize_telemetry_clients + if telemetry_enabled? + config.validate! + @api_client = build_api_client + @score_client = ScoreClient.new(api_client: @api_client, config: config) + else + config.validate_telemetry_disabled! + @api_client = DeferredApiClient.new { build_validated_api_client } + @score_client = NoopScoreClient.new + end + end + + def build_validated_api_client + config.validate! + build_api_client + end + + def build_api_client + cache = create_cache if cache_enabled? + ApiClient.new( + public_key: config.public_key, + secret_key: config.secret_key, + base_url: config.base_url, + timeout: config.timeout, + logger: config.logger, + cache: cache, + cache_observer: config.prompt_cache_observer + ) + end + + def telemetry_enabled? + @telemetry_enabled + end + attr_reader :score_client # Build a project-scoped URL, returning nil if project ID is unavailable diff --git a/lib/langfuse/config.rb b/lib/langfuse/config.rb index ef2a9be..07da6ab 100644 --- a/lib/langfuse/config.rb +++ b/lib/langfuse/config.rb @@ -64,6 +64,9 @@ class Config # @return [Boolean] Use OpenTelemetry batch scheduling for trace export attr_accessor :tracing_async + # @return [Boolean] Enable Langfuse tracing and scoring + attr_accessor :tracing_enabled + # @return [Integer] Number of events to batch before sending attr_accessor :batch_size @@ -146,6 +149,9 @@ class Config # @return [Boolean] Default async processing setting DEFAULT_TRACING_ASYNC = true + # @return [Boolean] Default telemetry setting + DEFAULT_TRACING_ENABLED = true + # @return [Integer] Default number of events to batch before sending DEFAULT_BATCH_SIZE = 50 @@ -218,6 +224,7 @@ def logger=(value) # @raise [ConfigurationError] if configuration is invalid # @return [void] def validate! + validate_tracing_enabled! validate_connection_settings! validate_batching_settings! validate_sample_rate! @@ -244,6 +251,7 @@ def valid? # @raise [ConfigurationError] if tracing configuration is invalid # @return [void] def validate_tracing! + validate_tracing_enabled! validate_connection_settings! validate_batching_settings! validate_sample_rate! @@ -255,6 +263,26 @@ def validate_tracing! validate_logger! end + # Validate settings needed while telemetry is disabled. + # + # @api private + # @raise [ConfigurationError] if disabled-client configuration is invalid + # @return [void] + def validate_telemetry_disabled! + validate_tracing_enabled! + validate_logger! + end + + # Check the effective tracing and scoring state. + # + # `OTEL_SDK_DISABLED=true` always disables telemetry. Otherwise, + # `tracing_enabled` controls the result. + # + # @return [Boolean] true when tracing and scoring are enabled + def telemetry_enabled? + tracing_enabled && !@otel_sdk_disabled + end + # Normalize stale_ttl value # # Converts :indefinite to 1000 years in seconds for practical "never expire" @@ -315,6 +343,8 @@ def default_logger end def initialize_tracing_defaults + @tracing_enabled = boolean_env("LANGFUSE_TRACING_ENABLED", default: DEFAULT_TRACING_ENABLED) + @otel_sdk_disabled = ENV.fetch("OTEL_SDK_DISABLED", nil) == "true" @environment = env_value("LANGFUSE_TRACING_ENVIRONMENT") @release = env_value("LANGFUSE_RELEASE") || detect_release_from_ci_env self.sample_rate = env_value("LANGFUSE_SAMPLE_RATE") || DEFAULT_SAMPLE_RATE @@ -331,6 +361,21 @@ def validate_connection_settings! validate_base_url! end + def validate_tracing_enabled! + return if [true, false].include?(tracing_enabled) + + raise ConfigurationError, "tracing_enabled must be true or false" + end + + def boolean_env(key, default:) + value = env_value(key) + return default if value.nil? + return true if value.casecmp("true").zero? + return false if value.casecmp("false").zero? + + raise ConfigurationError, "#{key} must be true or false" + end + def validate_batching_settings! unless batch_size.is_a?(Integer) && batch_size.positive? raise ConfigurationError, "batch_size must be a positive Integer" diff --git a/lib/langfuse/deferred_api_client.rb b/lib/langfuse/deferred_api_client.rb new file mode 100644 index 0000000..5e482ad --- /dev/null +++ b/lib/langfuse/deferred_api_client.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Langfuse + # Builds an API client only when a non-telemetry operation needs it. + # + # @api private + class DeferredApiClient + # @yield Builds and validates the real API client + # @return [DeferredApiClient] + def initialize(&factory) + @factory = factory + @mutex = Mutex.new + end + + # Avoid building an unused API client during disabled-client shutdown. + # + # @return [void] + def shutdown + @mutex.synchronize { @client }&.shutdown + end + + # @api private + def method_missing(name, ...) + return super unless ApiClient.public_instance_methods.include?(name) + + client.public_send(name, ...) + end + + # @api private + def respond_to_missing?(name, include_private = false) + ApiClient.public_instance_methods.include?(name) || super + end + + private + + def client + @mutex.synchronize { @client ||= @factory.call } + end + end +end diff --git a/lib/langfuse/noop_score_client.rb b/lib/langfuse/noop_score_client.rb new file mode 100644 index 0000000..b25b3e2 --- /dev/null +++ b/lib/langfuse/noop_score_client.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Langfuse + # Implements disabled score operations without validation or network access. + # + # @api private + class NoopScoreClient + # @param _options [Hash] Ignored score attributes + # @return [nil] + def create(**_options) + nil + end + + # @param _options [Hash] Ignored score attributes + # @return [nil] + def create!(**_options) + nil + end + + # @param _options [Hash] Ignored score attributes + # @return [nil] + def score_active_observation(**_options) + nil + end + + # @param _options [Hash] Ignored score attributes + # @return [nil] + def score_active_trace(**_options) + nil + end + + # @return [nil] + def flush + nil + end + + # @return [nil] + def shutdown + nil + end + end +end diff --git a/spec/langfuse/client_spec.rb b/spec/langfuse/client_spec.rb index 92ea5c6..69a63a0 100644 --- a/spec/langfuse/client_spec.rb +++ b/spec/langfuse/client_spec.rb @@ -42,6 +42,50 @@ ) end + context "when telemetry is disabled" do + let(:disabled_config) do + Langfuse::Config.new do |config| + config.public_key = nil + config.secret_key = nil + config.tracing_enabled = false + end + end + + it "constructs without credentials and makes score calls no-ops" do + client = described_class.new(disabled_config) + + expect(client.api_client).to be_a(Langfuse::DeferredApiClient) + expect(client.create_score(name: nil, value: nil)).to be_nil + expect(client.create_score!(name: nil, value: nil)).to be_nil + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "keeps prompt configuration validation deferred until prompt use" do + client = described_class.new(disabled_config) + + expect { client.get_prompt("greeting") }.to raise_error( + Langfuse::ConfigurationError, + "public_key is required" + ) + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "allows prompt access when the normal client configuration is valid" do + disabled_config.public_key = "pk_test" + disabled_config.secret_key = "sk_test" + stub_request(:get, "https://cloud.langfuse.com/api/public/v2/prompts/greeting") + .to_return( + status: 200, + body: { id: "prompt-1", name: "greeting", version: 1, type: "text", prompt: "Hello" }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + prompt = described_class.new(disabled_config).get_prompt("greeting") + + expect(prompt).to be_a(Langfuse::TextPromptClient) + end + end + context "with caching enabled" do let(:config_with_cache) do Langfuse::Config.new do |config| diff --git a/spec/langfuse/config_spec.rb b/spec/langfuse/config_spec.rb index b316e08..968827e 100644 --- a/spec/langfuse/config_spec.rb +++ b/spec/langfuse/config_spec.rb @@ -14,6 +14,8 @@ expect(config.cache_stale_ttl).to eq(0) # Defaults to 0 (SWR disabled) expect(config.cache_refresh_threads).to eq(5) expect(config.score_queue_capacity).to eq(100_000) + expect(config.tracing_enabled).to be true + expect(config.telemetry_enabled?).to be true expect(config.should_export_span).to be_nil expect(config.sample_rate).to eq(1.0) end @@ -65,6 +67,48 @@ ENV.delete("LANGFUSE_SAMPLE_RATE") end + it "reads LANGFUSE_TRACING_ENABLED without case sensitivity" do + ENV["LANGFUSE_TRACING_ENABLED"] = "FALSE" + + config = described_class.new + + expect(config.tracing_enabled).to be false + expect(config.telemetry_enabled?).to be false + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "lets explicit Ruby configuration override LANGFUSE_TRACING_ENABLED" do + ENV["LANGFUSE_TRACING_ENABLED"] = "false" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be true + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "honors OTEL_SDK_DISABLED even when Langfuse tracing is enabled" do + ENV["OTEL_SDK_DISABLED"] = "true" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be false + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + + it "raises ConfigurationError for an invalid LANGFUSE_TRACING_ENABLED value" do + ENV["LANGFUSE_TRACING_ENABLED"] = "sometimes" + + expect { described_class.new }.to raise_error( + Langfuse::ConfigurationError, + "LANGFUSE_TRACING_ENABLED must be true or false" + ) + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + it "falls back to CI release environment variables when LANGFUSE_RELEASE is not set" do release_envs = Langfuse::Config::COMMON_RELEASE_ENV_KEYS.to_h { |key| [key, ENV.fetch(key, nil)] } langfuse_release = ENV.fetch("LANGFUSE_RELEASE", nil) @@ -369,6 +413,17 @@ def string_like_key.to_str = "pk_test" end end + context "when tracing_enabled is invalid" do + it "raises ConfigurationError" do + config.tracing_enabled = "false" + + expect { config.validate! }.to raise_error( + Langfuse::ConfigurationError, + "tracing_enabled must be true or false" + ) + end + end + context "when cache_ttl is invalid" do it "raises ConfigurationError when nil" do config.cache_ttl = nil diff --git a/spec/langfuse/deferred_api_client_spec.rb b/spec/langfuse/deferred_api_client_spec.rb new file mode 100644 index 0000000..9ac4511 --- /dev/null +++ b/spec/langfuse/deferred_api_client_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::DeferredApiClient do + subject(:client) { described_class.new { api_client } } + + let(:api_client) { instance_double(Langfuse::ApiClient, list_prompts: ["prompt"], shutdown: nil) } + + it "does not build the API client for unused shutdown" do + factory_calls = 0 + deferred = described_class.new do + factory_calls += 1 + api_client + end + + deferred.shutdown + + expect(factory_calls).to eq(0) + end + + it "builds the API client once when an API method is used" do + expect(client.list_prompts).to eq(["prompt"]) + expect(client.list_prompts).to eq(["prompt"]) + end + + it "reports the API client method surface" do + expect(client).to respond_to(:list_prompts) + expect(client).not_to respond_to(:unknown_operation) + end + + it "shuts down an API client after it is built" do + client.list_prompts + + expect(api_client).to receive(:shutdown) + client.shutdown + end +end diff --git a/spec/langfuse/noop_score_client_spec.rb b/spec/langfuse/noop_score_client_spec.rb new file mode 100644 index 0000000..ceaa106 --- /dev/null +++ b/spec/langfuse/noop_score_client_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::NoopScoreClient do + subject(:client) { described_class.new } + + it "returns nil for every score lifecycle operation" do + expect(client.create(name: "quality", value: 1)).to be_nil + expect(client.create!(name: "quality", value: 1)).to be_nil + expect(client.score_active_observation(name: "quality", value: 1)).to be_nil + expect(client.score_active_trace(name: "quality", value: 1)).to be_nil + expect(client.flush).to be_nil + expect(client.shutdown).to be_nil + end +end diff --git a/spec/langfuse_spec.rb b/spec/langfuse_spec.rb index 3c8d4da..35fce12 100644 --- a/spec/langfuse_spec.rb +++ b/spec/langfuse_spec.rb @@ -277,6 +277,39 @@ end end + describe "disabled telemetry" do + before do + described_class.reset! + described_class.configure do |config| + config.public_key = nil + config.secret_key = nil + config.tracing_enabled = false + end + end + + it "constructs a client without credentials" do + expect(described_class.configured?).to be true + expect { described_class.client }.not_to raise_error + end + + it "uses non-recording spans and no-op scores without network requests" do + observation = described_class.observe("disabled-operation") + + expect(observation.otel_span).not_to be_recording + expect(described_class.create_score(name: nil, value: nil)).to be_nil + expect(described_class.create_score!(name: nil, value: nil)).to be_nil + expect(Langfuse::OtelSetup).not_to be_initialized + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "returns the no-op tracer provider" do + provider = described_class.tracer_provider + + expect(provider).to be_a(OpenTelemetry::Trace::TracerProvider) + expect(provider).not_to be_a(OpenTelemetry::SDK::Trace::TracerProvider) + end + end + describe ".propagate_attributes" do before do described_class.configure do |config| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ba33494..8496837 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -55,6 +55,8 @@ ENV.delete("LANGFUSE_FLUSH_AT") ENV.delete("LANGFUSE_FLUSH_INTERVAL") ENV.delete("LANGFUSE_DEBUG") + ENV.delete("LANGFUSE_TRACING_ENABLED") + ENV.delete("OTEL_SDK_DISABLED") # Stub OTLP endpoint BEFORE reset (which may flush traces) # (tests can override this with more specific stubs if needed) From 76c4d9876e1f4a7721b1a338269580337122dec3 Mon Sep 17 00:00:00 2001 From: kadekillary Date: Mon, 17 Aug 2026 07:40:08 -0700 Subject: [PATCH 2/7] fix(config): synchronize telemetry kill switch --- docs/CONFIGURATION.md | 2 +- lib/langfuse.rb | 1 - lib/langfuse/client.rb | 39 +++++++------ lib/langfuse/config.rb | 6 +- lib/langfuse/noop_score_client.rb | 42 -------------- lib/langfuse/score_client.rb | 8 ++- spec/langfuse/client_spec.rb | 44 --------------- spec/langfuse/client_telemetry_spec.rb | 63 +++++++++++++++++++++ spec/langfuse/config_spec.rb | 55 ------------------ spec/langfuse/config_telemetry_spec.rb | 74 +++++++++++++++++++++++++ spec/langfuse/noop_score_client_spec.rb | 14 ----- spec/langfuse_spec.rb | 18 ++++++ 12 files changed, 187 insertions(+), 179 deletions(-) delete mode 100644 lib/langfuse/noop_score_client.rb create mode 100644 spec/langfuse/client_telemetry_spec.rb create mode 100644 spec/langfuse/config_telemetry_spec.rb delete mode 100644 spec/langfuse/noop_score_client_spec.rb diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 76f6ef7..c775339 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -309,7 +309,7 @@ end Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. The standard `OTEL_SDK_DISABLED=true` setting always disables Langfuse telemetry. -When disabled, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Call `Langfuse.reset!` after changing this setting on an initialized SDK. +When disabled, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. #### `logger` diff --git a/lib/langfuse.rb b/lib/langfuse.rb index a16eea1..63557c8 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -81,7 +81,6 @@ class UnauthorizedError < ApiError; end require_relative "langfuse/trace_id" require_relative "langfuse/score_value" require_relative "langfuse/score_client" -require_relative "langfuse/noop_score_client" require_relative "langfuse/deferred_api_client" require_relative "langfuse/prompt_renderer" require_relative "langfuse/text_prompt_client" diff --git a/lib/langfuse/client.rb b/lib/langfuse/client.rb index 4410a1c..4e62723 100644 --- a/lib/langfuse/client.rb +++ b/lib/langfuse/client.rb @@ -73,7 +73,7 @@ class Client # @return [Client] def initialize(config) @config = config - @telemetry_enabled = config.telemetry_enabled? + @score_client_mutex = Mutex.new @project_id = nil # One-shot lookup: avoids repeated blocking API calls in URL helpers # (trace_url, dataset_url, dataset_run_url) when the project endpoint is down. @@ -368,9 +368,7 @@ def dataset_run_url(dataset_id:, dataset_run_id:) # rubocop:disable Metrics/ParameterLists def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) - return unless telemetry_enabled? - - @score_client.create( + active_score_client&.create( name: name, value: value, id: id, @@ -411,9 +409,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # rubocop:disable Metrics/ParameterLists def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) - return unless telemetry_enabled? - - @score_client.create!( + active_score_client&.create!( name: name, value: value, id: id, @@ -447,9 +443,7 @@ def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observ # client.score_active_observation(name: "accuracy", value: 0.92) # end def score_active_observation(name:, value:, comment: nil, metadata: nil, data_type: :numeric) - return unless telemetry_enabled? - - @score_client.score_active_observation( + active_score_client&.score_active_observation( name: name, value: value, comment: comment, @@ -475,9 +469,7 @@ def score_active_observation(name:, value:, comment: nil, metadata: nil, data_ty # client.score_active_trace(name: "overall_quality", value: 5) # end def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :numeric) - return unless telemetry_enabled? - - @score_client.score_active_trace( + active_score_client&.score_active_trace( name: name, value: value, comment: comment, @@ -495,9 +487,7 @@ def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :n # @example # client.flush_scores def flush_scores - return unless telemetry_enabled? - - @score_client.flush + active_score_client&.flush end # Shutdown the client and flush any pending scores @@ -506,7 +496,7 @@ def flush_scores # # @return [void] def shutdown - @score_client.shutdown + @score_client&.shutdown @api_client.shutdown end @@ -712,11 +702,10 @@ def initialize_telemetry_clients if telemetry_enabled? config.validate! @api_client = build_api_client - @score_client = ScoreClient.new(api_client: @api_client, config: config) + @score_client = build_score_client else config.validate_telemetry_disabled! @api_client = DeferredApiClient.new { build_validated_api_client } - @score_client = NoopScoreClient.new end end @@ -739,7 +728,17 @@ def build_api_client end def telemetry_enabled? - @telemetry_enabled + config.telemetry_enabled? + end + + def active_score_client + return unless telemetry_enabled? + + @score_client_mutex.synchronize { @score_client ||= build_score_client } + end + + def build_score_client + ScoreClient.new(api_client: @api_client, config: config) end attr_reader :score_client diff --git a/lib/langfuse/config.rb b/lib/langfuse/config.rb index 07da6ab..762ec95 100644 --- a/lib/langfuse/config.rb +++ b/lib/langfuse/config.rb @@ -344,7 +344,7 @@ def default_logger def initialize_tracing_defaults @tracing_enabled = boolean_env("LANGFUSE_TRACING_ENABLED", default: DEFAULT_TRACING_ENABLED) - @otel_sdk_disabled = ENV.fetch("OTEL_SDK_DISABLED", nil) == "true" + @otel_sdk_disabled = otel_sdk_disabled? @environment = env_value("LANGFUSE_TRACING_ENVIRONMENT") @release = env_value("LANGFUSE_RELEASE") || detect_release_from_ci_env self.sample_rate = env_value("LANGFUSE_SAMPLE_RATE") || DEFAULT_SAMPLE_RATE @@ -376,6 +376,10 @@ def boolean_env(key, default:) raise ConfigurationError, "#{key} must be true or false" end + def otel_sdk_disabled? + env_value("OTEL_SDK_DISABLED")&.casecmp?("true") || false + end + def validate_batching_settings! unless batch_size.is_a?(Integer) && batch_size.positive? raise ConfigurationError, "batch_size must be a positive Integer" diff --git a/lib/langfuse/noop_score_client.rb b/lib/langfuse/noop_score_client.rb deleted file mode 100644 index b25b3e2..0000000 --- a/lib/langfuse/noop_score_client.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Langfuse - # Implements disabled score operations without validation or network access. - # - # @api private - class NoopScoreClient - # @param _options [Hash] Ignored score attributes - # @return [nil] - def create(**_options) - nil - end - - # @param _options [Hash] Ignored score attributes - # @return [nil] - def create!(**_options) - nil - end - - # @param _options [Hash] Ignored score attributes - # @return [nil] - def score_active_observation(**_options) - nil - end - - # @param _options [Hash] Ignored score attributes - # @return [nil] - def score_active_trace(**_options) - nil - end - - # @return [nil] - def flush - nil - end - - # @return [nil] - def shutdown - nil - end - end -end diff --git a/lib/langfuse/score_client.rb b/lib/langfuse/score_client.rb index 95d21dc..69d5266 100644 --- a/lib/langfuse/score_client.rb +++ b/lib/langfuse/score_client.rb @@ -95,6 +95,8 @@ def initialize(api_client:, config:) # rubocop:disable Metrics/ParameterLists def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless config.telemetry_enabled? + score = build_score_body( name: name, value: value, @@ -141,7 +143,7 @@ def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_i # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -151,6 +153,8 @@ def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_i # rubocop:disable Metrics/ParameterLists def create!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless config.telemetry_enabled? + score = build_score_body( name: name, value: value, @@ -237,6 +241,8 @@ def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :n # # @return [void] def flush + return unless config.telemetry_enabled? + @flush_mutex.synchronize { flush_pending_batches } rescue StandardError => e logger.error("Langfuse score flush failed: #{e.message}") diff --git a/spec/langfuse/client_spec.rb b/spec/langfuse/client_spec.rb index 69a63a0..92ea5c6 100644 --- a/spec/langfuse/client_spec.rb +++ b/spec/langfuse/client_spec.rb @@ -42,50 +42,6 @@ ) end - context "when telemetry is disabled" do - let(:disabled_config) do - Langfuse::Config.new do |config| - config.public_key = nil - config.secret_key = nil - config.tracing_enabled = false - end - end - - it "constructs without credentials and makes score calls no-ops" do - client = described_class.new(disabled_config) - - expect(client.api_client).to be_a(Langfuse::DeferredApiClient) - expect(client.create_score(name: nil, value: nil)).to be_nil - expect(client.create_score!(name: nil, value: nil)).to be_nil - expect(a_request(:any, /.*/)).not_to have_been_made - end - - it "keeps prompt configuration validation deferred until prompt use" do - client = described_class.new(disabled_config) - - expect { client.get_prompt("greeting") }.to raise_error( - Langfuse::ConfigurationError, - "public_key is required" - ) - expect(a_request(:any, /.*/)).not_to have_been_made - end - - it "allows prompt access when the normal client configuration is valid" do - disabled_config.public_key = "pk_test" - disabled_config.secret_key = "sk_test" - stub_request(:get, "https://cloud.langfuse.com/api/public/v2/prompts/greeting") - .to_return( - status: 200, - body: { id: "prompt-1", name: "greeting", version: 1, type: "text", prompt: "Hello" }.to_json, - headers: { "Content-Type" => "application/json" } - ) - - prompt = described_class.new(disabled_config).get_prompt("greeting") - - expect(prompt).to be_a(Langfuse::TextPromptClient) - end - end - context "with caching enabled" do let(:config_with_cache) do Langfuse::Config.new do |config| diff --git a/spec/langfuse/client_telemetry_spec.rb b/spec/langfuse/client_telemetry_spec.rb new file mode 100644 index 0000000..288b86f --- /dev/null +++ b/spec/langfuse/client_telemetry_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::Client do + describe "telemetry state" do + let(:config) do + Langfuse::Config.new do |candidate| + candidate.public_key = nil + candidate.secret_key = nil + candidate.tracing_enabled = false + end + end + + it "constructs without credentials and makes score calls no-ops" do + client = described_class.new(config) + WebMock.reset_executed_requests! + + expect(client.api_client).to be_a(Langfuse::DeferredApiClient) + expect(client.create_score(name: nil, value: nil)).to be_nil + expect(client.create_score!(name: nil, value: nil)).to be_nil + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "defers prompt configuration validation until prompt use" do + client = described_class.new(config) + WebMock.reset_executed_requests! + + expect { client.get_prompt("greeting") }.to raise_error( + Langfuse::ConfigurationError, + "public_key is required" + ) + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "allows prompt access when normal client configuration is valid" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + stub_request(:get, "https://cloud.langfuse.com/api/public/v2/prompts/greeting") + .to_return( + status: 200, + body: { id: "prompt-1", name: "greeting", version: 1, type: "text", prompt: "Hello" }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + prompt = described_class.new(config).get_prompt("greeting") + + expect(prompt).to be_a(Langfuse::TextPromptClient) + end + + it "starts score delivery after telemetry is enabled" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + client = described_class.new(config) + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + + config.tracing_enabled = true + client.create_score(name: "quality", value: 1) + client.flush_scores + + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + end + end +end diff --git a/spec/langfuse/config_spec.rb b/spec/langfuse/config_spec.rb index 968827e..b316e08 100644 --- a/spec/langfuse/config_spec.rb +++ b/spec/langfuse/config_spec.rb @@ -14,8 +14,6 @@ expect(config.cache_stale_ttl).to eq(0) # Defaults to 0 (SWR disabled) expect(config.cache_refresh_threads).to eq(5) expect(config.score_queue_capacity).to eq(100_000) - expect(config.tracing_enabled).to be true - expect(config.telemetry_enabled?).to be true expect(config.should_export_span).to be_nil expect(config.sample_rate).to eq(1.0) end @@ -67,48 +65,6 @@ ENV.delete("LANGFUSE_SAMPLE_RATE") end - it "reads LANGFUSE_TRACING_ENABLED without case sensitivity" do - ENV["LANGFUSE_TRACING_ENABLED"] = "FALSE" - - config = described_class.new - - expect(config.tracing_enabled).to be false - expect(config.telemetry_enabled?).to be false - ensure - ENV.delete("LANGFUSE_TRACING_ENABLED") - end - - it "lets explicit Ruby configuration override LANGFUSE_TRACING_ENABLED" do - ENV["LANGFUSE_TRACING_ENABLED"] = "false" - - config = described_class.new { |candidate| candidate.tracing_enabled = true } - - expect(config.telemetry_enabled?).to be true - ensure - ENV.delete("LANGFUSE_TRACING_ENABLED") - end - - it "honors OTEL_SDK_DISABLED even when Langfuse tracing is enabled" do - ENV["OTEL_SDK_DISABLED"] = "true" - - config = described_class.new { |candidate| candidate.tracing_enabled = true } - - expect(config.telemetry_enabled?).to be false - ensure - ENV.delete("OTEL_SDK_DISABLED") - end - - it "raises ConfigurationError for an invalid LANGFUSE_TRACING_ENABLED value" do - ENV["LANGFUSE_TRACING_ENABLED"] = "sometimes" - - expect { described_class.new }.to raise_error( - Langfuse::ConfigurationError, - "LANGFUSE_TRACING_ENABLED must be true or false" - ) - ensure - ENV.delete("LANGFUSE_TRACING_ENABLED") - end - it "falls back to CI release environment variables when LANGFUSE_RELEASE is not set" do release_envs = Langfuse::Config::COMMON_RELEASE_ENV_KEYS.to_h { |key| [key, ENV.fetch(key, nil)] } langfuse_release = ENV.fetch("LANGFUSE_RELEASE", nil) @@ -413,17 +369,6 @@ def string_like_key.to_str = "pk_test" end end - context "when tracing_enabled is invalid" do - it "raises ConfigurationError" do - config.tracing_enabled = "false" - - expect { config.validate! }.to raise_error( - Langfuse::ConfigurationError, - "tracing_enabled must be true or false" - ) - end - end - context "when cache_ttl is invalid" do it "raises ConfigurationError when nil" do config.cache_ttl = nil diff --git a/spec/langfuse/config_telemetry_spec.rb b/spec/langfuse/config_telemetry_spec.rb new file mode 100644 index 0000000..0e7edad --- /dev/null +++ b/spec/langfuse/config_telemetry_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::Config do + describe "telemetry state" do + it "enables telemetry by default" do + config = described_class.new + + expect(config.tracing_enabled).to be true + expect(config.telemetry_enabled?).to be true + end + + it "reads LANGFUSE_TRACING_ENABLED without case sensitivity" do + ENV["LANGFUSE_TRACING_ENABLED"] = "FALSE" + + config = described_class.new + + expect(config.tracing_enabled).to be false + expect(config.telemetry_enabled?).to be false + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "lets Ruby configuration override LANGFUSE_TRACING_ENABLED" do + ENV["LANGFUSE_TRACING_ENABLED"] = "false" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be true + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "honors OTEL_SDK_DISABLED without case sensitivity" do + ENV["OTEL_SDK_DISABLED"] = "TRUE" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be false + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + + it "keeps telemetry enabled for other OTEL_SDK_DISABLED values" do + ENV["OTEL_SDK_DISABLED"] = "1" + + config = described_class.new + + expect(config.telemetry_enabled?).to be true + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + + it "rejects an invalid LANGFUSE_TRACING_ENABLED value" do + ENV["LANGFUSE_TRACING_ENABLED"] = "sometimes" + + expect { described_class.new }.to raise_error( + Langfuse::ConfigurationError, + "LANGFUSE_TRACING_ENABLED must be true or false" + ) + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "rejects a non-Boolean tracing_enabled value" do + config = described_class.new + config.tracing_enabled = "false" + + expect { config.validate! }.to raise_error( + Langfuse::ConfigurationError, + "tracing_enabled must be true or false" + ) + end + end +end diff --git a/spec/langfuse/noop_score_client_spec.rb b/spec/langfuse/noop_score_client_spec.rb deleted file mode 100644 index ceaa106..0000000 --- a/spec/langfuse/noop_score_client_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Langfuse::NoopScoreClient do - subject(:client) { described_class.new } - - it "returns nil for every score lifecycle operation" do - expect(client.create(name: "quality", value: 1)).to be_nil - expect(client.create!(name: "quality", value: 1)).to be_nil - expect(client.score_active_observation(name: "quality", value: 1)).to be_nil - expect(client.score_active_trace(name: "quality", value: 1)).to be_nil - expect(client.flush).to be_nil - expect(client.shutdown).to be_nil - end -end diff --git a/spec/langfuse_spec.rb b/spec/langfuse_spec.rb index 35fce12..44c5c00 100644 --- a/spec/langfuse_spec.rb +++ b/spec/langfuse_spec.rb @@ -308,6 +308,24 @@ expect(provider).to be_a(OpenTelemetry::Trace::TracerProvider) expect(provider).not_to be_a(OpenTelemetry::SDK::Trace::TracerProvider) end + + it "stops traces and queued score delivery on an initialized SDK" do + described_class.configure do |config| + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + end + initialized_client = described_class.client + described_class.create_score(name: "queued-before-disable", value: 1) + + described_class.configure { |config| config.tracing_enabled = false } + observation = described_class.observe("disabled-after-initialize") + described_class.flush_scores + + expect(described_class.client).to equal(initialized_client) + expect(observation.otel_span).not_to be_recording + expect(a_request(:any, /.*/)).not_to have_been_made + end end describe ".propagate_attributes" do From 70db6ee0d173abf03d6dc2ccda1edd073eccd6df Mon Sep 17 00:00:00 2001 From: kadekillary Date: Mon, 17 Aug 2026 08:31:09 -0700 Subject: [PATCH 3/7] refactor(client): remove deferred API proxy --- lib/langfuse.rb | 1 - lib/langfuse/client.rb | 33 ++++++++++--------- lib/langfuse/deferred_api_client.rb | 40 ----------------------- spec/langfuse/client_telemetry_spec.rb | 7 +++- spec/langfuse/deferred_api_client_spec.rb | 36 -------------------- 5 files changed, 23 insertions(+), 94 deletions(-) delete mode 100644 lib/langfuse/deferred_api_client.rb delete mode 100644 spec/langfuse/deferred_api_client_spec.rb diff --git a/lib/langfuse.rb b/lib/langfuse.rb index 63557c8..69a05a1 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -81,7 +81,6 @@ class UnauthorizedError < ApiError; end require_relative "langfuse/trace_id" require_relative "langfuse/score_value" require_relative "langfuse/score_client" -require_relative "langfuse/deferred_api_client" require_relative "langfuse/prompt_renderer" require_relative "langfuse/text_prompt_client" require_relative "langfuse/chat_prompt_client" diff --git a/lib/langfuse/client.rb b/lib/langfuse/client.rb index 4e62723..ebc823a 100644 --- a/lib/langfuse/client.rb +++ b/lib/langfuse/client.rb @@ -29,8 +29,13 @@ class Client # @return [Config] The client configuration attr_reader :config - # @return [ApiClient, DeferredApiClient] The underlying or deferred API client - attr_reader :api_client + # Return the underlying API client, building it on first non-telemetry use. + # + # @return [ApiClient] The underlying API client + # @raise [ConfigurationError] if the full client configuration is invalid + def api_client + @api_client || @api_client_mutex.synchronize { @api_client ||= build_validated_api_client } + end # Pure pass-throughs to {ApiClient}. See {ApiClient} for parameter and # return-value documentation; the public surface here is identical. @@ -73,12 +78,19 @@ class Client # @return [Client] def initialize(config) @config = config + @api_client_mutex = Mutex.new @score_client_mutex = Mutex.new @project_id = nil # One-shot lookup: avoids repeated blocking API calls in URL helpers # (trace_url, dataset_url, dataset_run_url) when the project endpoint is down. @project_id_fetched = false - initialize_telemetry_clients + if telemetry_enabled? + config.validate! + @api_client = build_api_client + @score_client = build_score_client + else + config.validate_telemetry_disabled! + end end # Fetch a prompt and return the appropriate client @@ -497,7 +509,7 @@ def flush_scores # @return [void] def shutdown @score_client&.shutdown - @api_client.shutdown + @api_client_mutex.synchronize { @api_client }&.shutdown end # Create a new dataset @@ -698,17 +710,6 @@ def run_experiment(name:, task:, data: nil, dataset_name: nil, description: nil, private - def initialize_telemetry_clients - if telemetry_enabled? - config.validate! - @api_client = build_api_client - @score_client = build_score_client - else - config.validate_telemetry_disabled! - @api_client = DeferredApiClient.new { build_validated_api_client } - end - end - def build_validated_api_client config.validate! build_api_client @@ -738,7 +739,7 @@ def active_score_client end def build_score_client - ScoreClient.new(api_client: @api_client, config: config) + ScoreClient.new(api_client: api_client, config: config) end attr_reader :score_client diff --git a/lib/langfuse/deferred_api_client.rb b/lib/langfuse/deferred_api_client.rb deleted file mode 100644 index 5e482ad..0000000 --- a/lib/langfuse/deferred_api_client.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Langfuse - # Builds an API client only when a non-telemetry operation needs it. - # - # @api private - class DeferredApiClient - # @yield Builds and validates the real API client - # @return [DeferredApiClient] - def initialize(&factory) - @factory = factory - @mutex = Mutex.new - end - - # Avoid building an unused API client during disabled-client shutdown. - # - # @return [void] - def shutdown - @mutex.synchronize { @client }&.shutdown - end - - # @api private - def method_missing(name, ...) - return super unless ApiClient.public_instance_methods.include?(name) - - client.public_send(name, ...) - end - - # @api private - def respond_to_missing?(name, include_private = false) - ApiClient.public_instance_methods.include?(name) || super - end - - private - - def client - @mutex.synchronize { @client ||= @factory.call } - end - end -end diff --git a/spec/langfuse/client_telemetry_spec.rb b/spec/langfuse/client_telemetry_spec.rb index 288b86f..88870f2 100644 --- a/spec/langfuse/client_telemetry_spec.rb +++ b/spec/langfuse/client_telemetry_spec.rb @@ -14,7 +14,6 @@ client = described_class.new(config) WebMock.reset_executed_requests! - expect(client.api_client).to be_a(Langfuse::DeferredApiClient) expect(client.create_score(name: nil, value: nil)).to be_nil expect(client.create_score!(name: nil, value: nil)).to be_nil expect(a_request(:any, /.*/)).not_to have_been_made @@ -31,6 +30,12 @@ expect(a_request(:any, /.*/)).not_to have_been_made end + it "does not validate unused API credentials during shutdown" do + client = described_class.new(config) + + expect { client.shutdown }.not_to raise_error + end + it "allows prompt access when normal client configuration is valid" do config.public_key = "pk_test" config.secret_key = "sk_test" diff --git a/spec/langfuse/deferred_api_client_spec.rb b/spec/langfuse/deferred_api_client_spec.rb deleted file mode 100644 index 9ac4511..0000000 --- a/spec/langfuse/deferred_api_client_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Langfuse::DeferredApiClient do - subject(:client) { described_class.new { api_client } } - - let(:api_client) { instance_double(Langfuse::ApiClient, list_prompts: ["prompt"], shutdown: nil) } - - it "does not build the API client for unused shutdown" do - factory_calls = 0 - deferred = described_class.new do - factory_calls += 1 - api_client - end - - deferred.shutdown - - expect(factory_calls).to eq(0) - end - - it "builds the API client once when an API method is used" do - expect(client.list_prompts).to eq(["prompt"]) - expect(client.list_prompts).to eq(["prompt"]) - end - - it "reports the API client method surface" do - expect(client).to respond_to(:list_prompts) - expect(client).not_to respond_to(:unknown_operation) - end - - it "shuts down an API client after it is built" do - client.list_prompts - - expect(api_client).to receive(:shutdown) - client.shutdown - end -end From 2330dbe9ffe11a03660a51308b94d28844992ff8 Mon Sep 17 00:00:00 2001 From: kadekillary Date: Mon, 17 Aug 2026 10:42:07 -0700 Subject: [PATCH 4/7] fix(telemetry): keep scores active when OpenTelemetry is disabled --- docs/CONFIGURATION.md | 4 ++-- lib/langfuse.rb | 4 ++-- lib/langfuse/config.rb | 16 ++++++++++------ spec/langfuse/client_telemetry_spec.rb | 17 +++++++++++++++++ spec/langfuse/config_telemetry_spec.rb | 8 ++++++-- spec/langfuse_spec.rb | 23 +++++++++++++++++++++++ 6 files changed, 60 insertions(+), 12 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index c775339..222a79b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -307,9 +307,9 @@ Langfuse.configure do |config| end ``` -Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. The standard `OTEL_SDK_DISABLED=true` setting always disables Langfuse telemetry. +Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. `OTEL_SDK_DISABLED=true` disables OpenTelemetry trace export. Direct score ingestion remains available. -When disabled, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. +When `tracing_enabled` is false, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. #### `logger` diff --git a/lib/langfuse.rb b/lib/langfuse.rb index 69a05a1..70b6f53 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -168,7 +168,7 @@ def configured? # # OpenTelemetry.tracer_provider = Langfuse.tracer_provider def tracer_provider - return noop_tracer_provider unless configuration.telemetry_enabled? + return noop_tracer_provider unless configuration.trace_export_enabled? OtelSetup.setup(configuration) unless OtelSetup.initialized? OtelSetup.tracer_provider @@ -647,7 +647,7 @@ def wrap_otel_span(otel_span, type_str, otel_tracer, attributes: nil) end def ensure_tracing_started - return false unless configuration.telemetry_enabled? + return false unless configuration.trace_export_enabled? return true if OtelSetup.initialized? OtelSetup.setup(configuration) diff --git a/lib/langfuse/config.rb b/lib/langfuse/config.rb index 762ec95..d28ff7b 100644 --- a/lib/langfuse/config.rb +++ b/lib/langfuse/config.rb @@ -273,14 +273,18 @@ def validate_telemetry_disabled! validate_logger! end - # Check the effective tracing and scoring state. + # Check whether Langfuse tracing and scoring are enabled. # - # `OTEL_SDK_DISABLED=true` always disables telemetry. Otherwise, - # `tracing_enabled` controls the result. - # - # @return [Boolean] true when tracing and scoring are enabled + # @return [Boolean] true when Langfuse telemetry is enabled def telemetry_enabled? - tracing_enabled && !@otel_sdk_disabled + tracing_enabled + end + + # Check whether OpenTelemetry trace export is enabled. + # + # @return [Boolean] true when Langfuse tracing is enabled and the OpenTelemetry SDK is active + def trace_export_enabled? + telemetry_enabled? && !@otel_sdk_disabled end # Normalize stale_ttl value diff --git a/spec/langfuse/client_telemetry_spec.rb b/spec/langfuse/client_telemetry_spec.rb index 88870f2..be43cb5 100644 --- a/spec/langfuse/client_telemetry_spec.rb +++ b/spec/langfuse/client_telemetry_spec.rb @@ -64,5 +64,22 @@ expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once end + + it "delivers scores when OTEL_SDK_DISABLED disables trace export" do + ENV["OTEL_SDK_DISABLED"] = "true" + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + client = described_class.new(config) + + client.create_score(name: "quality", value: 1) + client.flush_scores + + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + ensure + ENV.delete("OTEL_SDK_DISABLED") + end end end diff --git a/spec/langfuse/config_telemetry_spec.rb b/spec/langfuse/config_telemetry_spec.rb index 0e7edad..2cb2900 100644 --- a/spec/langfuse/config_telemetry_spec.rb +++ b/spec/langfuse/config_telemetry_spec.rb @@ -7,6 +7,7 @@ expect(config.tracing_enabled).to be true expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be true end it "reads LANGFUSE_TRACING_ENABLED without case sensitivity" do @@ -16,6 +17,7 @@ expect(config.tracing_enabled).to be false expect(config.telemetry_enabled?).to be false + expect(config.trace_export_enabled?).to be false ensure ENV.delete("LANGFUSE_TRACING_ENABLED") end @@ -30,12 +32,13 @@ ENV.delete("LANGFUSE_TRACING_ENABLED") end - it "honors OTEL_SDK_DISABLED without case sensitivity" do + it "disables only trace export when OTEL_SDK_DISABLED is true" do ENV["OTEL_SDK_DISABLED"] = "TRUE" config = described_class.new { |candidate| candidate.tracing_enabled = true } - expect(config.telemetry_enabled?).to be false + expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be false ensure ENV.delete("OTEL_SDK_DISABLED") end @@ -46,6 +49,7 @@ config = described_class.new expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be true ensure ENV.delete("OTEL_SDK_DISABLED") end diff --git a/spec/langfuse_spec.rb b/spec/langfuse_spec.rb index 44c5c00..bf99155 100644 --- a/spec/langfuse_spec.rb +++ b/spec/langfuse_spec.rb @@ -328,6 +328,29 @@ end end + describe "OpenTelemetry SDK disablement" do + it "keeps direct score ingestion active while trace export is disabled" do + ENV["OTEL_SDK_DISABLED"] = "true" + described_class.reset! + described_class.configure do |config| + config.public_key = "pk_test" + config.secret_key = "sk_test" + end + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + + observation = described_class.observe("otel-disabled") + described_class.create_score(name: "quality", value: 1) + described_class.flush_scores + + expect(observation.otel_span).not_to be_recording + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + ensure + ENV.delete("OTEL_SDK_DISABLED") + described_class.reset! + end + end + describe ".propagate_attributes" do before do described_class.configure do |config| From 95eb8fa3cce0e0b95a97d0588437618879e67b4f Mon Sep 17 00:00:00 2001 From: kadekillary Date: Mon, 17 Aug 2026 11:18:10 -0700 Subject: [PATCH 5/7] fix(telemetry): drop spans while disabled --- docs/CONFIGURATION.md | 2 +- lib/langfuse/otel_setup.rb | 6 +++- lib/langfuse/trace_export_guard.rb | 47 ++++++++++++++++++++++++++++++ spec/langfuse/otel_setup_spec.rb | 21 +++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 lib/langfuse/trace_export_guard.rb diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 222a79b..0f4676d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -309,7 +309,7 @@ end Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. `OTEL_SDK_DISABLED=true` disables OpenTelemetry trace export. Direct score ingestion remains available. -When `tracing_enabled` is false, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. +When `tracing_enabled` is false, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Buffered spans are discarded instead of exported while the setting is false. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. #### `logger` diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index b1e17ae..f82c7f7 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -4,6 +4,7 @@ require "opentelemetry/exporter/otlp" require "base64" require_relative "masking_exporter" +require_relative "trace_export_guard" module Langfuse # OpenTelemetry initialization and setup for Langfuse tracing. @@ -115,7 +116,10 @@ def build_tracer_provider(config) sampler: build_sampler(config.sample_rate) ) provider.add_span_processor( - SpanProcessor.new(config: config, exporter: build_exporter(config)) + SpanProcessor.new( + config: config, + exporter: TraceExportGuard.new(delegate: build_exporter(config), config: config) + ) ) provider end diff --git a/lib/langfuse/trace_export_guard.rb b/lib/langfuse/trace_export_guard.rb new file mode 100644 index 0000000..a817b3f --- /dev/null +++ b/lib/langfuse/trace_export_guard.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "opentelemetry/sdk" + +module Langfuse + # Stops trace export while the live telemetry kill switch is disabled. + # + # @api private + class TraceExportGuard + SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS + private_constant :SUCCESS + + # @param delegate [#export, #force_flush, #shutdown] OpenTelemetry span exporter + # @param config [Langfuse::Config] Live SDK configuration + def initialize(delegate:, config:) + @delegate = delegate + @config = config + end + + # Export spans only while trace export is enabled. + # + # @param span_data [Enumerable] + # @param timeout [Numeric, nil] + # @return [Integer] OpenTelemetry export result code + def export(span_data, timeout: nil) + return SUCCESS unless @config.trace_export_enabled? + + @delegate.export(span_data, timeout: timeout) + end + + # @param timeout [Numeric, nil] + # @return [Object] delegate result or OpenTelemetry success + def force_flush(timeout: nil) + return SUCCESS unless @config.trace_export_enabled? + + @delegate.force_flush(timeout: timeout) + end + + # Release exporter resources after the guarded processor drains its queue. + # + # @param timeout [Numeric, nil] + # @return [Object] delegate shutdown result + def shutdown(timeout: nil) + @delegate.shutdown(timeout: timeout) + end + end +end diff --git a/spec/langfuse/otel_setup_spec.rb b/spec/langfuse/otel_setup_spec.rb index 84a1d17..e7ff750 100644 --- a/spec/langfuse/otel_setup_spec.rb +++ b/spec/langfuse/otel_setup_spec.rb @@ -305,6 +305,27 @@ expect(exporter.finished_spans.map(&:name)).to eq(["langfuse-span"]) end + it "drops buffered spans when telemetry is disabled before export" do + Langfuse.observe("before-disable").end + + Langfuse.configure { |c| c.tracing_enabled = false } + Langfuse.force_flush(timeout: 1) + + expect(exporter.finished_spans).to be_empty + end + + it "exports new spans after telemetry is re-enabled" do + Langfuse.observe("before-disable").end + Langfuse.configure { |c| c.tracing_enabled = false } + Langfuse.force_flush(timeout: 1) + + Langfuse.configure { |c| c.tracing_enabled = true } + Langfuse.observe("after-enable").end + Langfuse.force_flush(timeout: 1) + + expect(exporter.finished_spans.map(&:name)).to eq(["after-enable"]) + end + it "exports each completed observation once" do Langfuse.observe("root") do |root| root.start_observation("generation", as_type: :generation) { |generation| generation.update(output: "ok") } From 918ca30d30453878c3c51b64ff7ed1209495e4e3 Mon Sep 17 00:00:00 2001 From: kadekillary Date: Mon, 17 Aug 2026 11:38:58 -0700 Subject: [PATCH 6/7] fix(telemetry): fail closed on invalid kill switch --- lib/langfuse/config.rb | 2 +- spec/langfuse/client_telemetry_spec.rb | 13 +++++++++++++ spec/langfuse/config_telemetry_spec.rb | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/langfuse/config.rb b/lib/langfuse/config.rb index d28ff7b..cd4a337 100644 --- a/lib/langfuse/config.rb +++ b/lib/langfuse/config.rb @@ -277,7 +277,7 @@ def validate_telemetry_disabled! # # @return [Boolean] true when Langfuse telemetry is enabled def telemetry_enabled? - tracing_enabled + tracing_enabled == true end # Check whether OpenTelemetry trace export is enabled. diff --git a/spec/langfuse/client_telemetry_spec.rb b/spec/langfuse/client_telemetry_spec.rb index be43cb5..0ce1f22 100644 --- a/spec/langfuse/client_telemetry_spec.rb +++ b/spec/langfuse/client_telemetry_spec.rb @@ -65,6 +65,19 @@ expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once end + it "fails closed when a live telemetry value is not Boolean" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + client = described_class.new(config) + + config.tracing_enabled = "false" + + expect(client.create_score(name: "quality", value: 1)).to be_nil + client.flush_scores + expect(a_request(:any, /.*/)).not_to have_been_made + end + it "delivers scores when OTEL_SDK_DISABLED disables trace export" do ENV["OTEL_SDK_DISABLED"] = "true" config.public_key = "pk_test" diff --git a/spec/langfuse/config_telemetry_spec.rb b/spec/langfuse/config_telemetry_spec.rb index 2cb2900..cb05042 100644 --- a/spec/langfuse/config_telemetry_spec.rb +++ b/spec/langfuse/config_telemetry_spec.rb @@ -69,6 +69,8 @@ config = described_class.new config.tracing_enabled = "false" + expect(config.telemetry_enabled?).to be false + expect(config.trace_export_enabled?).to be false expect { config.validate! }.to raise_error( Langfuse::ConfigurationError, "tracing_enabled must be true or false" From 6408b2ae91b50b032a90771c8226a4c2dad4bc3c Mon Sep 17 00:00:00 2001 From: kade Date: Mon, 17 Aug 2026 13:27:45 -0700 Subject: [PATCH 7/7] feat(prompts): expose template variables (#111) --- lib/langfuse.rb | 1 + lib/langfuse/chat_prompt_client.rb | 17 ++++++++ lib/langfuse/prompt_variables.rb | 54 ++++++++++++++++++++++++ lib/langfuse/text_prompt_client.rb | 12 ++++++ spec/langfuse/chat_prompt_client_spec.rb | 24 +++++++++++ spec/langfuse/text_prompt_client_spec.rb | 37 ++++++++++++++++ 6 files changed, 145 insertions(+) create mode 100644 lib/langfuse/prompt_variables.rb diff --git a/lib/langfuse.rb b/lib/langfuse.rb index 70b6f53..a586ab0 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -82,6 +82,7 @@ class UnauthorizedError < ApiError; end require_relative "langfuse/score_value" require_relative "langfuse/score_client" require_relative "langfuse/prompt_renderer" +require_relative "langfuse/prompt_variables" require_relative "langfuse/text_prompt_client" require_relative "langfuse/chat_prompt_client" require_relative "langfuse/timestamp_parser" diff --git a/lib/langfuse/chat_prompt_client.rb b/lib/langfuse/chat_prompt_client.rb index b516171..23dffe8 100644 --- a/lib/langfuse/chat_prompt_client.rb +++ b/lib/langfuse/chat_prompt_client.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "prompt_renderer" +require_relative "prompt_variables" module Langfuse # Chat prompt client for compiling chat prompts with variable substitution @@ -73,6 +74,22 @@ def type "chat" end + # Return the unique variables referenced by all message templates + # + # Section names are included because callers must provide their values. + # Message placeholder entries are not Mustache templates and are excluded. + # + # @return [Array] Referenced variable names in message and source order + # @raise [Mustache::Parser::SyntaxError] if a message contains invalid Mustache syntax + def variables + prompt.each_with_object([]) do |message, names| + normalized = symbolize_keys(message) + next if normalized[:type].to_s == PLACEHOLDER_TYPE + + names.concat(PromptVariables.extract(normalized[:content] || "")) + end.uniq + end + # Compile the chat prompt with variable substitution and message placeholders # # Returns an array of message hashes with roles and compiled content. diff --git a/lib/langfuse/prompt_variables.rb b/lib/langfuse/prompt_variables.rb new file mode 100644 index 0000000..d0fe4a4 --- /dev/null +++ b/lib/langfuse/prompt_variables.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "mustache" + +module Langfuse + # Extracts referenced variables from parsed Mustache templates. + # + # @api private + class PromptVariables + TAG_TYPES = %i[etag utag].freeze + SECTION_TYPES = %i[section inverted_section].freeze + + class << self + # @api private + def extract(template) + tokens = Mustache::Template.new(template).tokens + collect(tokens, []).reject(&:empty?).uniq + end + + private + + def collect(tokens, scope) + tokens.each_with_object([]) do |token, variables| + next unless token.is_a?(Array) + + variables.concat(token.first == :mustache ? from_tag(token, scope) : collect(token, scope)) + end + end + + def from_tag(token, scope) + return variable_path(token, scope) if TAG_TYPES.include?(token[1]) + return section_paths(token, scope) if SECTION_TYPES.include?(token[1]) + + [] + end + + def variable_path(token, scope) + path = scoped_path(token, scope) + path.empty? ? [] : [path.join(".")] + end + + def section_paths(token, scope) + section_path = scoped_path(token, scope) + body_scope = token[1] == :section ? section_path : scope + [section_path.join("."), *collect(token[4], body_scope)] + end + + def scoped_path(token, scope) + segments = token.dig(2, 2) + segments == ["."] ? scope : scope + segments + end + end + end +end diff --git a/lib/langfuse/text_prompt_client.rb b/lib/langfuse/text_prompt_client.rb index c3fd426..e9d1348 100644 --- a/lib/langfuse/text_prompt_client.rb +++ b/lib/langfuse/text_prompt_client.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "prompt_renderer" +require_relative "prompt_variables" module Langfuse # Text prompt client for compiling text prompts with variable substitution @@ -71,6 +72,17 @@ def type "text" end + # Return the unique variables referenced by the prompt template + # + # Section names are included because callers must provide their values. + # Variables inside sections include the full section path. + # + # @return [Array] Referenced variable names in source order + # @raise [Mustache::Parser::SyntaxError] if the prompt contains invalid Mustache syntax + def variables + PromptVariables.extract(prompt) + end + # Compile the prompt with variable substitution # # @param kwargs [Hash] Variables to substitute in the template (as keyword arguments) diff --git a/spec/langfuse/chat_prompt_client_spec.rb b/spec/langfuse/chat_prompt_client_spec.rb index 9c8b46d..9b240f8 100644 --- a/spec/langfuse/chat_prompt_client_spec.rb +++ b/spec/langfuse/chat_prompt_client_spec.rb @@ -152,6 +152,30 @@ end end + describe "#variables" do + it "returns unique variables across message templates" do + data = prompt_data.merge( + "prompt" => [ + { "role" => "system", "content" => "Hello {{user.name}} and {{shared}}" }, + { "role" => "user", "content" => "{{shared}} {{#details}}{{topic}}{{/details}}" } + ] + ) + + expect(described_class.new(data).variables).to eq(%w[user.name shared details details.topic]) + end + + it "excludes message placeholders" do + data = prompt_data.merge( + "prompt" => [ + { "type" => "placeholder", "name" => "history" }, + { type: "message", role: "user", content: "Question: {{{question}}}" } + ] + ) + + expect(described_class.new(data).variables).to eq(["question"]) + end + end + describe "#compile" do let(:client) { described_class.new(prompt_data) } diff --git a/spec/langfuse/text_prompt_client_spec.rb b/spec/langfuse/text_prompt_client_spec.rb index 57c34fc..11f370b 100644 --- a/spec/langfuse/text_prompt_client_spec.rb +++ b/spec/langfuse/text_prompt_client_spec.rb @@ -136,6 +136,43 @@ end end + describe "#variables" do + it "returns unique parsed variables in source order" do + data = prompt_data.merge( + "prompt" => "{{name}} {{name}} {{profile.email}} {{{raw_html}}} {{& plain_html}} {{! ignored }}" + ) + + expect(described_class.new(data).variables).to eq(%w[name profile.email raw_html plain_html]) + end + + it "includes sections and scopes variables inside nested sections" do + data = prompt_data.merge( + "prompt" => "{{#account}}{{#owner}}{{profile.email}}{{/owner}}{{/account}}" \ + "{{^items}}{{message}}{{/items}}" + ) + + expect(described_class.new(data).variables).to eq( + %w[account account.owner account.owner.profile.email items message] + ) + end + + it "keeps inverted-section variables in the enclosing scope" do + data = prompt_data.merge( + "prompt" => "{{#account}}{{^owner}}{{fallback.name}}{{/owner}}{{/account}}" + ) + + expect(described_class.new(data).variables).to eq( + %w[account account.owner account.fallback.name] + ) + end + + it "raises for invalid Mustache syntax" do + data = prompt_data.merge("prompt" => "{{#account}}{{name}}") + + expect { described_class.new(data).variables }.to raise_error(Mustache::Parser::SyntaxError) + end + end + describe "#compile" do let(:client) { described_class.new(prompt_data) }