diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 18fbbd7..0f4676d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -294,6 +294,23 @@ Session-only and dataset-run-only scores are still sent because they are not tie For Ruby client instances, `sample_rate` is snapshotted when the client is built. Changing `config.sample_rate` later does not update that client's score sampler or the already-initialized trace sampler. Rebuild the client with `Langfuse.reset!` when changing sampling behavior. +#### `tracing_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Environment:** `LANGFUSE_TRACING_ENABLED` +- **Description:** Enables or disables Langfuse tracing and scoring + +```ruby +Langfuse.configure do |config| + config.tracing_enabled = false +end +``` + +Explicit Ruby configuration overrides `LANGFUSE_TRACING_ENABLED`. The environment variable accepts `true` or `false` without case sensitivity. `OTEL_SDK_DISABLED=true` disables OpenTelemetry trace export. Direct score ingestion remains available. + +When `tracing_enabled` is false, trace and score calls are no-ops. They do not require Langfuse credentials and do not create network requests. Buffered spans are discarded instead of exported while the setting is false. Prompt and data API calls still validate the normal client configuration when used. Synchronous `create_score!` returns `nil` while telemetry is disabled. Changes to `tracing_enabled` apply to the initialized SDK. Environment variables are read when configuration is created. + #### `logger` - **Type:** Logger @@ -804,6 +821,7 @@ Client-readiness rules: - `base_url` must be an absolute HTTP or HTTPS URL - `batch_size` must be a positive Integer - `score_queue_capacity` must be a positive Integer +- `tracing_enabled` must be `true` or `false` - `flush_interval`, `timeout`, `cache_max_size`, `cache_lock_timeout`, and `cache_refresh_threads` must be positive numbers - `cache_ttl` must be a non-negative number - `cache_stale_ttl` must be a non-negative number or `:indefinite` diff --git a/lib/langfuse.rb b/lib/langfuse.rb index bb7eff0..a586ab0 100644 --- a/lib/langfuse.rb +++ b/lib/langfuse.rb @@ -82,6 +82,7 @@ class UnauthorizedError < ApiError; end require_relative "langfuse/score_value" require_relative "langfuse/score_client" require_relative "langfuse/prompt_renderer" +require_relative "langfuse/prompt_variables" require_relative "langfuse/text_prompt_client" require_relative "langfuse/chat_prompt_client" require_relative "langfuse/timestamp_parser" @@ -144,7 +145,11 @@ def client # # @return [Boolean] true when the local configuration is valid def configured? - configuration.valid? + config = configuration + return config.valid? if config.telemetry_enabled? + + config.validate_telemetry_disabled! + true rescue ConfigurationError # Reading `configuration` builds it from the environment, which can fail # on its own before there is anything to validate. @@ -153,7 +158,7 @@ def configured? # Return Langfuse's internal tracer provider for explicit global OpenTelemetry installation. # - # @return [OpenTelemetry::SDK::Trace::TracerProvider] + # @return [OpenTelemetry::SDK::Trace::TracerProvider, OpenTelemetry::Trace::TracerProvider] # @raise [ConfigurationError] if tracing configuration is invalid # # @example @@ -164,6 +169,8 @@ def configured? # # OpenTelemetry.tracer_provider = Langfuse.tracer_provider def tracer_provider + return noop_tracer_provider unless configuration.trace_export_enabled? + OtelSetup.setup(configuration) unless OtelSetup.initialized? OtelSetup.tracer_provider rescue ConfigurationError => e @@ -322,7 +329,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -441,12 +448,14 @@ def reset! @configuration = nil @client = nil @noop_tracer = nil + @noop_tracer_provider = nil @emitted_warnings = nil rescue StandardError # Ignore shutdown errors during reset (e.g., in tests) @configuration = nil @client = nil @noop_tracer = nil + @noop_tracer_provider = nil @emitted_warnings = nil end @@ -639,6 +648,7 @@ def wrap_otel_span(otel_span, type_str, otel_tracer, attributes: nil) end def ensure_tracing_started + return false unless configuration.trace_export_enabled? return true if OtelSetup.initialized? OtelSetup.setup(configuration) @@ -683,7 +693,11 @@ def warning_mutex end def noop_tracer - @noop_tracer ||= OpenTelemetry::Trace::TracerProvider.new.tracer(LANGFUSE_TRACER_NAME, Langfuse::VERSION) + @noop_tracer ||= noop_tracer_provider.tracer(LANGFUSE_TRACER_NAME, Langfuse::VERSION) + end + + def noop_tracer_provider + @noop_tracer_provider ||= OpenTelemetry::Trace::TracerProvider.new end end # rubocop:enable Metrics/ClassLength diff --git a/lib/langfuse/chat_prompt_client.rb b/lib/langfuse/chat_prompt_client.rb index b516171..23dffe8 100644 --- a/lib/langfuse/chat_prompt_client.rb +++ b/lib/langfuse/chat_prompt_client.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "prompt_renderer" +require_relative "prompt_variables" module Langfuse # Chat prompt client for compiling chat prompts with variable substitution @@ -73,6 +74,22 @@ def type "chat" end + # Return the unique variables referenced by all message templates + # + # Section names are included because callers must provide their values. + # Message placeholder entries are not Mustache templates and are excluded. + # + # @return [Array] Referenced variable names in message and source order + # @raise [Mustache::Parser::SyntaxError] if a message contains invalid Mustache syntax + def variables + prompt.each_with_object([]) do |message, names| + normalized = symbolize_keys(message) + next if normalized[:type].to_s == PLACEHOLDER_TYPE + + names.concat(PromptVariables.extract(normalized[:content] || "")) + end.uniq + end + # Compile the chat prompt with variable substitution and message placeholders # # Returns an array of message hashes with roles and compiled content. diff --git a/lib/langfuse/client.rb b/lib/langfuse/client.rb index 11098a9..ebc823a 100644 --- a/lib/langfuse/client.rb +++ b/lib/langfuse/client.rb @@ -29,8 +29,13 @@ class Client # @return [Config] The client configuration attr_reader :config + # Return the underlying API client, building it on first non-telemetry use. + # # @return [ApiClient] The underlying API client - attr_reader :api_client + # @raise [ConfigurationError] if the full client configuration is invalid + def api_client + @api_client || @api_client_mutex.synchronize { @api_client ||= build_validated_api_client } + end # Pure pass-throughs to {ApiClient}. See {ApiClient} for parameter and # return-value documentation; the public surface here is identical. @@ -73,29 +78,19 @@ class Client # @return [Client] def initialize(config) @config = config - @config.validate! - - # Create cache if enabled - cache = create_cache if cache_enabled? - - # Create API client with cache - @api_client = ApiClient.new( - public_key: config.public_key, - secret_key: config.secret_key, - base_url: config.base_url, - timeout: config.timeout, - logger: config.logger, - cache: cache, - cache_observer: config.prompt_cache_observer - ) - + @api_client_mutex = Mutex.new + @score_client_mutex = Mutex.new @project_id = nil # One-shot lookup: avoids repeated blocking API calls in URL helpers # (trace_url, dataset_url, dataset_run_url) when the project endpoint is down. @project_id_fetched = false - - # Initialize score client for batching score events - @score_client = ScoreClient.new(api_client: @api_client, config: config) + if telemetry_enabled? + config.validate! + @api_client = build_api_client + @score_client = build_score_client + else + config.validate_telemetry_disabled! + end end # Fetch a prompt and return the appropriate client @@ -385,7 +380,7 @@ def dataset_run_url(dataset_id:, dataset_run_id:) # rubocop:disable Metrics/ParameterLists def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) - @score_client.create( + active_score_client&.create( name: name, value: value, id: id, @@ -416,7 +411,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -426,7 +421,7 @@ def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observa # rubocop:disable Metrics/ParameterLists def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) - @score_client.create!( + active_score_client&.create!( name: name, value: value, id: id, @@ -460,7 +455,7 @@ def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observ # client.score_active_observation(name: "accuracy", value: 0.92) # end def score_active_observation(name:, value:, comment: nil, metadata: nil, data_type: :numeric) - @score_client.score_active_observation( + active_score_client&.score_active_observation( name: name, value: value, comment: comment, @@ -486,7 +481,7 @@ def score_active_observation(name:, value:, comment: nil, metadata: nil, data_ty # client.score_active_trace(name: "overall_quality", value: 5) # end def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :numeric) - @score_client.score_active_trace( + active_score_client&.score_active_trace( name: name, value: value, comment: comment, @@ -504,7 +499,7 @@ def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :n # @example # client.flush_scores def flush_scores - @score_client.flush + active_score_client&.flush end # Shutdown the client and flush any pending scores @@ -513,8 +508,8 @@ def flush_scores # # @return [void] def shutdown - @score_client.shutdown - @api_client.shutdown + @score_client&.shutdown + @api_client_mutex.synchronize { @api_client }&.shutdown end # Create a new dataset @@ -715,6 +710,38 @@ def run_experiment(name:, task:, data: nil, dataset_name: nil, description: nil, private + def build_validated_api_client + config.validate! + build_api_client + end + + def build_api_client + cache = create_cache if cache_enabled? + ApiClient.new( + public_key: config.public_key, + secret_key: config.secret_key, + base_url: config.base_url, + timeout: config.timeout, + logger: config.logger, + cache: cache, + cache_observer: config.prompt_cache_observer + ) + end + + def telemetry_enabled? + config.telemetry_enabled? + end + + def active_score_client + return unless telemetry_enabled? + + @score_client_mutex.synchronize { @score_client ||= build_score_client } + end + + def build_score_client + ScoreClient.new(api_client: api_client, config: config) + end + attr_reader :score_client # Build a project-scoped URL, returning nil if project ID is unavailable diff --git a/lib/langfuse/config.rb b/lib/langfuse/config.rb index ef2a9be..cd4a337 100644 --- a/lib/langfuse/config.rb +++ b/lib/langfuse/config.rb @@ -64,6 +64,9 @@ class Config # @return [Boolean] Use OpenTelemetry batch scheduling for trace export attr_accessor :tracing_async + # @return [Boolean] Enable Langfuse tracing and scoring + attr_accessor :tracing_enabled + # @return [Integer] Number of events to batch before sending attr_accessor :batch_size @@ -146,6 +149,9 @@ class Config # @return [Boolean] Default async processing setting DEFAULT_TRACING_ASYNC = true + # @return [Boolean] Default telemetry setting + DEFAULT_TRACING_ENABLED = true + # @return [Integer] Default number of events to batch before sending DEFAULT_BATCH_SIZE = 50 @@ -218,6 +224,7 @@ def logger=(value) # @raise [ConfigurationError] if configuration is invalid # @return [void] def validate! + validate_tracing_enabled! validate_connection_settings! validate_batching_settings! validate_sample_rate! @@ -244,6 +251,7 @@ def valid? # @raise [ConfigurationError] if tracing configuration is invalid # @return [void] def validate_tracing! + validate_tracing_enabled! validate_connection_settings! validate_batching_settings! validate_sample_rate! @@ -255,6 +263,30 @@ def validate_tracing! validate_logger! end + # Validate settings needed while telemetry is disabled. + # + # @api private + # @raise [ConfigurationError] if disabled-client configuration is invalid + # @return [void] + def validate_telemetry_disabled! + validate_tracing_enabled! + validate_logger! + end + + # Check whether Langfuse tracing and scoring are enabled. + # + # @return [Boolean] true when Langfuse telemetry is enabled + def telemetry_enabled? + tracing_enabled == true + end + + # Check whether OpenTelemetry trace export is enabled. + # + # @return [Boolean] true when Langfuse tracing is enabled and the OpenTelemetry SDK is active + def trace_export_enabled? + telemetry_enabled? && !@otel_sdk_disabled + end + # Normalize stale_ttl value # # Converts :indefinite to 1000 years in seconds for practical "never expire" @@ -315,6 +347,8 @@ def default_logger end def initialize_tracing_defaults + @tracing_enabled = boolean_env("LANGFUSE_TRACING_ENABLED", default: DEFAULT_TRACING_ENABLED) + @otel_sdk_disabled = otel_sdk_disabled? @environment = env_value("LANGFUSE_TRACING_ENVIRONMENT") @release = env_value("LANGFUSE_RELEASE") || detect_release_from_ci_env self.sample_rate = env_value("LANGFUSE_SAMPLE_RATE") || DEFAULT_SAMPLE_RATE @@ -331,6 +365,25 @@ def validate_connection_settings! validate_base_url! end + def validate_tracing_enabled! + return if [true, false].include?(tracing_enabled) + + raise ConfigurationError, "tracing_enabled must be true or false" + end + + def boolean_env(key, default:) + value = env_value(key) + return default if value.nil? + return true if value.casecmp("true").zero? + return false if value.casecmp("false").zero? + + raise ConfigurationError, "#{key} must be true or false" + end + + def otel_sdk_disabled? + env_value("OTEL_SDK_DISABLED")&.casecmp?("true") || false + end + def validate_batching_settings! unless batch_size.is_a?(Integer) && batch_size.positive? raise ConfigurationError, "batch_size must be a positive Integer" diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index b1e17ae..f82c7f7 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -4,6 +4,7 @@ require "opentelemetry/exporter/otlp" require "base64" require_relative "masking_exporter" +require_relative "trace_export_guard" module Langfuse # OpenTelemetry initialization and setup for Langfuse tracing. @@ -115,7 +116,10 @@ def build_tracer_provider(config) sampler: build_sampler(config.sample_rate) ) provider.add_span_processor( - SpanProcessor.new(config: config, exporter: build_exporter(config)) + SpanProcessor.new( + config: config, + exporter: TraceExportGuard.new(delegate: build_exporter(config), config: config) + ) ) provider end diff --git a/lib/langfuse/prompt_variables.rb b/lib/langfuse/prompt_variables.rb new file mode 100644 index 0000000..d0fe4a4 --- /dev/null +++ b/lib/langfuse/prompt_variables.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "mustache" + +module Langfuse + # Extracts referenced variables from parsed Mustache templates. + # + # @api private + class PromptVariables + TAG_TYPES = %i[etag utag].freeze + SECTION_TYPES = %i[section inverted_section].freeze + + class << self + # @api private + def extract(template) + tokens = Mustache::Template.new(template).tokens + collect(tokens, []).reject(&:empty?).uniq + end + + private + + def collect(tokens, scope) + tokens.each_with_object([]) do |token, variables| + next unless token.is_a?(Array) + + variables.concat(token.first == :mustache ? from_tag(token, scope) : collect(token, scope)) + end + end + + def from_tag(token, scope) + return variable_path(token, scope) if TAG_TYPES.include?(token[1]) + return section_paths(token, scope) if SECTION_TYPES.include?(token[1]) + + [] + end + + def variable_path(token, scope) + path = scoped_path(token, scope) + path.empty? ? [] : [path.join(".")] + end + + def section_paths(token, scope) + section_path = scoped_path(token, scope) + body_scope = token[1] == :section ? section_path : scope + [section_path.join("."), *collect(token[4], body_scope)] + end + + def scoped_path(token, scope) + segments = token.dig(2, 2) + segments == ["."] ? scope : scope + segments + end + end + end +end diff --git a/lib/langfuse/score_client.rb b/lib/langfuse/score_client.rb index 95d21dc..69d5266 100644 --- a/lib/langfuse/score_client.rb +++ b/lib/langfuse/score_client.rb @@ -95,6 +95,8 @@ def initialize(api_client:, config:) # rubocop:disable Metrics/ParameterLists def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless config.telemetry_enabled? + score = build_score_body( name: name, value: value, @@ -141,7 +143,7 @@ def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_i # @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction) # @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score # @param config_id [String, nil] Optional score config ID - # @return [String] ID of the created score + # @return [String, nil] ID of the created score, or nil when telemetry is disabled # @raise [ArgumentError] if validation fails # @raise [UnauthorizedError] if authentication fails # @raise [ApiError] if the API request fails @@ -151,6 +153,8 @@ def create(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_i # rubocop:disable Metrics/ParameterLists def create!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil, metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil) + return unless config.telemetry_enabled? + score = build_score_body( name: name, value: value, @@ -237,6 +241,8 @@ def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :n # # @return [void] def flush + return unless config.telemetry_enabled? + @flush_mutex.synchronize { flush_pending_batches } rescue StandardError => e logger.error("Langfuse score flush failed: #{e.message}") diff --git a/lib/langfuse/text_prompt_client.rb b/lib/langfuse/text_prompt_client.rb index c3fd426..e9d1348 100644 --- a/lib/langfuse/text_prompt_client.rb +++ b/lib/langfuse/text_prompt_client.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "prompt_renderer" +require_relative "prompt_variables" module Langfuse # Text prompt client for compiling text prompts with variable substitution @@ -71,6 +72,17 @@ def type "text" end + # Return the unique variables referenced by the prompt template + # + # Section names are included because callers must provide their values. + # Variables inside sections include the full section path. + # + # @return [Array] Referenced variable names in source order + # @raise [Mustache::Parser::SyntaxError] if the prompt contains invalid Mustache syntax + def variables + PromptVariables.extract(prompt) + end + # Compile the prompt with variable substitution # # @param kwargs [Hash] Variables to substitute in the template (as keyword arguments) diff --git a/lib/langfuse/trace_export_guard.rb b/lib/langfuse/trace_export_guard.rb new file mode 100644 index 0000000..a817b3f --- /dev/null +++ b/lib/langfuse/trace_export_guard.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "opentelemetry/sdk" + +module Langfuse + # Stops trace export while the live telemetry kill switch is disabled. + # + # @api private + class TraceExportGuard + SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS + private_constant :SUCCESS + + # @param delegate [#export, #force_flush, #shutdown] OpenTelemetry span exporter + # @param config [Langfuse::Config] Live SDK configuration + def initialize(delegate:, config:) + @delegate = delegate + @config = config + end + + # Export spans only while trace export is enabled. + # + # @param span_data [Enumerable] + # @param timeout [Numeric, nil] + # @return [Integer] OpenTelemetry export result code + def export(span_data, timeout: nil) + return SUCCESS unless @config.trace_export_enabled? + + @delegate.export(span_data, timeout: timeout) + end + + # @param timeout [Numeric, nil] + # @return [Object] delegate result or OpenTelemetry success + def force_flush(timeout: nil) + return SUCCESS unless @config.trace_export_enabled? + + @delegate.force_flush(timeout: timeout) + end + + # Release exporter resources after the guarded processor drains its queue. + # + # @param timeout [Numeric, nil] + # @return [Object] delegate shutdown result + def shutdown(timeout: nil) + @delegate.shutdown(timeout: timeout) + end + end +end diff --git a/spec/langfuse/chat_prompt_client_spec.rb b/spec/langfuse/chat_prompt_client_spec.rb index 9c8b46d..9b240f8 100644 --- a/spec/langfuse/chat_prompt_client_spec.rb +++ b/spec/langfuse/chat_prompt_client_spec.rb @@ -152,6 +152,30 @@ end end + describe "#variables" do + it "returns unique variables across message templates" do + data = prompt_data.merge( + "prompt" => [ + { "role" => "system", "content" => "Hello {{user.name}} and {{shared}}" }, + { "role" => "user", "content" => "{{shared}} {{#details}}{{topic}}{{/details}}" } + ] + ) + + expect(described_class.new(data).variables).to eq(%w[user.name shared details details.topic]) + end + + it "excludes message placeholders" do + data = prompt_data.merge( + "prompt" => [ + { "type" => "placeholder", "name" => "history" }, + { type: "message", role: "user", content: "Question: {{{question}}}" } + ] + ) + + expect(described_class.new(data).variables).to eq(["question"]) + end + end + describe "#compile" do let(:client) { described_class.new(prompt_data) } diff --git a/spec/langfuse/client_telemetry_spec.rb b/spec/langfuse/client_telemetry_spec.rb new file mode 100644 index 0000000..0ce1f22 --- /dev/null +++ b/spec/langfuse/client_telemetry_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::Client do + describe "telemetry state" do + let(:config) do + Langfuse::Config.new do |candidate| + candidate.public_key = nil + candidate.secret_key = nil + candidate.tracing_enabled = false + end + end + + it "constructs without credentials and makes score calls no-ops" do + client = described_class.new(config) + WebMock.reset_executed_requests! + + expect(client.create_score(name: nil, value: nil)).to be_nil + expect(client.create_score!(name: nil, value: nil)).to be_nil + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "defers prompt configuration validation until prompt use" do + client = described_class.new(config) + WebMock.reset_executed_requests! + + expect { client.get_prompt("greeting") }.to raise_error( + Langfuse::ConfigurationError, + "public_key is required" + ) + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "does not validate unused API credentials during shutdown" do + client = described_class.new(config) + + expect { client.shutdown }.not_to raise_error + end + + it "allows prompt access when normal client configuration is valid" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + stub_request(:get, "https://cloud.langfuse.com/api/public/v2/prompts/greeting") + .to_return( + status: 200, + body: { id: "prompt-1", name: "greeting", version: 1, type: "text", prompt: "Hello" }.to_json, + headers: { "Content-Type" => "application/json" } + ) + + prompt = described_class.new(config).get_prompt("greeting") + + expect(prompt).to be_a(Langfuse::TextPromptClient) + end + + it "starts score delivery after telemetry is enabled" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + client = described_class.new(config) + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + + config.tracing_enabled = true + client.create_score(name: "quality", value: 1) + client.flush_scores + + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + end + + it "fails closed when a live telemetry value is not Boolean" do + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + client = described_class.new(config) + + config.tracing_enabled = "false" + + expect(client.create_score(name: "quality", value: 1)).to be_nil + client.flush_scores + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "delivers scores when OTEL_SDK_DISABLED disables trace export" do + ENV["OTEL_SDK_DISABLED"] = "true" + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + client = described_class.new(config) + + client.create_score(name: "quality", value: 1) + client.flush_scores + + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + end +end diff --git a/spec/langfuse/config_telemetry_spec.rb b/spec/langfuse/config_telemetry_spec.rb new file mode 100644 index 0000000..cb05042 --- /dev/null +++ b/spec/langfuse/config_telemetry_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +RSpec.describe Langfuse::Config do + describe "telemetry state" do + it "enables telemetry by default" do + config = described_class.new + + expect(config.tracing_enabled).to be true + expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be true + end + + it "reads LANGFUSE_TRACING_ENABLED without case sensitivity" do + ENV["LANGFUSE_TRACING_ENABLED"] = "FALSE" + + config = described_class.new + + expect(config.tracing_enabled).to be false + expect(config.telemetry_enabled?).to be false + expect(config.trace_export_enabled?).to be false + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "lets Ruby configuration override LANGFUSE_TRACING_ENABLED" do + ENV["LANGFUSE_TRACING_ENABLED"] = "false" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be true + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "disables only trace export when OTEL_SDK_DISABLED is true" do + ENV["OTEL_SDK_DISABLED"] = "TRUE" + + config = described_class.new { |candidate| candidate.tracing_enabled = true } + + expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be false + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + + it "keeps telemetry enabled for other OTEL_SDK_DISABLED values" do + ENV["OTEL_SDK_DISABLED"] = "1" + + config = described_class.new + + expect(config.telemetry_enabled?).to be true + expect(config.trace_export_enabled?).to be true + ensure + ENV.delete("OTEL_SDK_DISABLED") + end + + it "rejects an invalid LANGFUSE_TRACING_ENABLED value" do + ENV["LANGFUSE_TRACING_ENABLED"] = "sometimes" + + expect { described_class.new }.to raise_error( + Langfuse::ConfigurationError, + "LANGFUSE_TRACING_ENABLED must be true or false" + ) + ensure + ENV.delete("LANGFUSE_TRACING_ENABLED") + end + + it "rejects a non-Boolean tracing_enabled value" do + config = described_class.new + config.tracing_enabled = "false" + + expect(config.telemetry_enabled?).to be false + expect(config.trace_export_enabled?).to be false + expect { config.validate! }.to raise_error( + Langfuse::ConfigurationError, + "tracing_enabled must be true or false" + ) + end + end +end diff --git a/spec/langfuse/otel_setup_spec.rb b/spec/langfuse/otel_setup_spec.rb index 84a1d17..e7ff750 100644 --- a/spec/langfuse/otel_setup_spec.rb +++ b/spec/langfuse/otel_setup_spec.rb @@ -305,6 +305,27 @@ expect(exporter.finished_spans.map(&:name)).to eq(["langfuse-span"]) end + it "drops buffered spans when telemetry is disabled before export" do + Langfuse.observe("before-disable").end + + Langfuse.configure { |c| c.tracing_enabled = false } + Langfuse.force_flush(timeout: 1) + + expect(exporter.finished_spans).to be_empty + end + + it "exports new spans after telemetry is re-enabled" do + Langfuse.observe("before-disable").end + Langfuse.configure { |c| c.tracing_enabled = false } + Langfuse.force_flush(timeout: 1) + + Langfuse.configure { |c| c.tracing_enabled = true } + Langfuse.observe("after-enable").end + Langfuse.force_flush(timeout: 1) + + expect(exporter.finished_spans.map(&:name)).to eq(["after-enable"]) + end + it "exports each completed observation once" do Langfuse.observe("root") do |root| root.start_observation("generation", as_type: :generation) { |generation| generation.update(output: "ok") } diff --git a/spec/langfuse/text_prompt_client_spec.rb b/spec/langfuse/text_prompt_client_spec.rb index 57c34fc..11f370b 100644 --- a/spec/langfuse/text_prompt_client_spec.rb +++ b/spec/langfuse/text_prompt_client_spec.rb @@ -136,6 +136,43 @@ end end + describe "#variables" do + it "returns unique parsed variables in source order" do + data = prompt_data.merge( + "prompt" => "{{name}} {{name}} {{profile.email}} {{{raw_html}}} {{& plain_html}} {{! ignored }}" + ) + + expect(described_class.new(data).variables).to eq(%w[name profile.email raw_html plain_html]) + end + + it "includes sections and scopes variables inside nested sections" do + data = prompt_data.merge( + "prompt" => "{{#account}}{{#owner}}{{profile.email}}{{/owner}}{{/account}}" \ + "{{^items}}{{message}}{{/items}}" + ) + + expect(described_class.new(data).variables).to eq( + %w[account account.owner account.owner.profile.email items message] + ) + end + + it "keeps inverted-section variables in the enclosing scope" do + data = prompt_data.merge( + "prompt" => "{{#account}}{{^owner}}{{fallback.name}}{{/owner}}{{/account}}" + ) + + expect(described_class.new(data).variables).to eq( + %w[account account.owner account.fallback.name] + ) + end + + it "raises for invalid Mustache syntax" do + data = prompt_data.merge("prompt" => "{{#account}}{{name}}") + + expect { described_class.new(data).variables }.to raise_error(Mustache::Parser::SyntaxError) + end + end + describe "#compile" do let(:client) { described_class.new(prompt_data) } diff --git a/spec/langfuse_spec.rb b/spec/langfuse_spec.rb index 3c8d4da..bf99155 100644 --- a/spec/langfuse_spec.rb +++ b/spec/langfuse_spec.rb @@ -277,6 +277,80 @@ end end + describe "disabled telemetry" do + before do + described_class.reset! + described_class.configure do |config| + config.public_key = nil + config.secret_key = nil + config.tracing_enabled = false + end + end + + it "constructs a client without credentials" do + expect(described_class.configured?).to be true + expect { described_class.client }.not_to raise_error + end + + it "uses non-recording spans and no-op scores without network requests" do + observation = described_class.observe("disabled-operation") + + expect(observation.otel_span).not_to be_recording + expect(described_class.create_score(name: nil, value: nil)).to be_nil + expect(described_class.create_score!(name: nil, value: nil)).to be_nil + expect(Langfuse::OtelSetup).not_to be_initialized + expect(a_request(:any, /.*/)).not_to have_been_made + end + + it "returns the no-op tracer provider" do + provider = described_class.tracer_provider + + expect(provider).to be_a(OpenTelemetry::Trace::TracerProvider) + expect(provider).not_to be_a(OpenTelemetry::SDK::Trace::TracerProvider) + end + + it "stops traces and queued score delivery on an initialized SDK" do + described_class.configure do |config| + config.public_key = "pk_test" + config.secret_key = "sk_test" + config.tracing_enabled = true + end + initialized_client = described_class.client + described_class.create_score(name: "queued-before-disable", value: 1) + + described_class.configure { |config| config.tracing_enabled = false } + observation = described_class.observe("disabled-after-initialize") + described_class.flush_scores + + expect(described_class.client).to equal(initialized_client) + expect(observation.otel_span).not_to be_recording + expect(a_request(:any, /.*/)).not_to have_been_made + end + end + + describe "OpenTelemetry SDK disablement" do + it "keeps direct score ingestion active while trace export is disabled" do + ENV["OTEL_SDK_DISABLED"] = "true" + described_class.reset! + described_class.configure do |config| + config.public_key = "pk_test" + config.secret_key = "sk_test" + end + stub_request(:post, "https://cloud.langfuse.com/api/public/ingestion") + .to_return(status: 200, body: { successes: [], errors: [] }.to_json) + + observation = described_class.observe("otel-disabled") + described_class.create_score(name: "quality", value: 1) + described_class.flush_scores + + expect(observation.otel_span).not_to be_recording + expect(a_request(:post, "https://cloud.langfuse.com/api/public/ingestion")).to have_been_made.once + ensure + ENV.delete("OTEL_SDK_DISABLED") + described_class.reset! + end + end + describe ".propagate_attributes" do before do described_class.configure do |config| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ba33494..8496837 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -55,6 +55,8 @@ ENV.delete("LANGFUSE_FLUSH_AT") ENV.delete("LANGFUSE_FLUSH_INTERVAL") ENV.delete("LANGFUSE_DEBUG") + ENV.delete("LANGFUSE_TRACING_ENABLED") + ENV.delete("OTEL_SDK_DISABLED") # Stub OTLP endpoint BEFORE reset (which may flush traces) # (tests can override this with more specific stubs if needed)