feat(telemetry): add OTel standard logs as third signal

src/sap_cloud_sdk/core/telemetry/_provider.py

@@ -1,16 +1,26 @@
-"""Internal module for setting up OpenTelemetry meter provider."""
+"""Internal module for setting up OpenTelemetry meter and logger providers."""
 
 import logging
 import os
 from typing import Optional
 
 from opentelemetry import metrics
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
+    OTLPLogExporter as GRPCLogExporter,
+)
 from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
     OTLPMetricExporter as GRPCMetricExporter,
 )
+from opentelemetry.exporter.otlp.proto.http._log_exporter import (
+    OTLPLogExporter as HTTPLogExporter,
+)
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
     OTLPMetricExporter as HTTPMetricExporter,
 )
+from opentelemetry.instrumentation.logging.handler import LoggingHandler
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.metrics import (
     MeterProvider,
     Counter,
@@ -40,6 +50,9 @@
 _meter_provider: Optional[MeterProvider] = None
 _meter: Optional[metrics.Meter] = None
 
+# Global logger provider
+_log_provider: Optional[LoggerProvider] = None
+
 
 def get_meter() -> metrics.Meter:
     """Get or create the global meter instance.
@@ -76,6 +89,54 @@ def shutdown() -> None:
         _meter_provider = None
 
 
+def setup_log_provider() -> Optional[LoggerProvider]:
+    """Set up the global OTel LoggerProvider using the shared resource attributes.
+
+    Installs a LoggingHandler on the root stdlib logger so all existing
+    logging.getLogger(...) calls in the app flow through OTel automatically.
+    No-op when telemetry is disabled.
+    """
+    global _log_provider
+
+    config = get_config()
+    if not config.enabled:
+        return None
+
+    try:
+        resource = Resource.create(create_resource_attributes_from_env())
+        exporter = _create_log_exporter()
+        provider = LoggerProvider(resource=resource)
+        provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+        set_logger_provider(provider)
+
+        handler = LoggingHandler(logger_provider=provider)
+        logging.getLogger().addHandler(handler)
+
+        _log_provider = provider
+        logger.info(
+            f"OpenTelemetry log provider initialized. "
+            f"Service: {config.service_name}, "
+            f"Endpoint: {config.otlp_endpoint}"
+        )
+        return provider
+
+    except Exception as e:
+        logger.error(f"Failed to initialize OpenTelemetry log provider: {e}")
+        return None
+
+
+def _create_log_exporter():
+    protocol = os.getenv(ENV_OTLP_PROTOCOL, "grpc").lower()
+    exporter_classes = {"grpc": GRPCLogExporter, "http/protobuf": HTTPLogExporter}
+
+    if protocol not in exporter_classes:
+        raise ValueError(
+            f"Unsupported OTEL_EXPORTER_OTLP_PROTOCOL: '{protocol}'. "
+            "Supported values are 'grpc' and 'http/protobuf'."
+        )
+    return exporter_classes[protocol]()
+
+
 def _create_metric_exporter():
     protocol = os.getenv(ENV_OTLP_PROTOCOL, "grpc").lower()
     exporter_classes = {"grpc": GRPCMetricExporter, "http/protobuf": HTTPMetricExporter}

src/sap_cloud_sdk/core/telemetry/auto_instrument.py

@@ -20,6 +20,7 @@
 from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SpanExporter
 from traceloop.sdk import Traceloop
 
+from sap_cloud_sdk.core.telemetry._provider import setup_log_provider
 from sap_cloud_sdk.core.telemetry.module import Module
 from sap_cloud_sdk.core.telemetry.operation import Operation
 from sap_cloud_sdk.core.telemetry.config import (
@@ -87,6 +88,8 @@ def auto_instrument(
     _set_baggage_processor()
     _set_propagated_attributes_processor()
 
+    setup_log_provider()
+
     if middlewares:
         _register_middleware_processors(middlewares)
 

src/sap_cloud_sdk/core/telemetry/user-guide.md

@@ -144,6 +144,54 @@ GenAIOperation.INVOKE_AGENT
 
 ---
 
+## Logging
+
+`auto_instrument()` sets up OTel logs alongside traces and metrics. It installs a handler on the root stdlib logger so all existing `logging.getLogger(...)` calls in your app automatically ship log records to the OTel backend with the same resource attributes (service name, region, subaccount, etc.).
+
+No changes to your logging code are needed:
+
+```python
+import logging
+
+logger = logging.getLogger(__name__)
+
+logger.info("Destination fetched")
+logger.warning("Retrying request, attempt %d", attempt)
+logger.error("Failed to connect", exc_info=True)
+```
+
+### Structured fields
+
+Use `extra={}` to attach structured attributes to a log record:
+
+```python
+logger.info("Request completed", extra={"tenant_id": tid, "duration_ms": 120})
+```
+
+### Log level filtering
+
+By default all levels (`DEBUG` and above) flow through OTel. To restrict what gets exported, set the level on the root logger or any specific logger:
+
+```python
+# Only WARNING and above to OTel
+logging.getLogger().setLevel(logging.WARNING)
+
+# Or scope it to your app's logger tree
+logging.getLogger("my_app").setLevel(logging.INFO)
+```
+
+### Correlation with traces
+
+OTel logs emitted inside an active span are automatically correlated — the `trace_id` and `span_id` are injected into the log record. No extra work needed.
+
+### Third-party logging libraries
+
+The OTel handler is installed on the root stdlib `logging` logger. Any library that propagates to stdlib works automatically.
+
+Libraries that bypass stdlib entirely need a custom sink that forwards records to `logging.getLogger(...).log(...)`. The OTel handler then picks them up from there.
+
+---
+
 ## Adding attributes
 
 ### To the current span
@@ -200,6 +248,7 @@ Propagation is scoped: once the parent span exits, its attributes stop propagati
 ## Complete example
 
 ```python
+import logging
 from sap_cloud_sdk.core.telemetry import (
     auto_instrument,
     invoke_agent_span,
@@ -212,9 +261,13 @@ auto_instrument()
 
 from litellm import completion
 
+logger = logging.getLogger(__name__)
+
 async def handle_request(query: str, user_id: str):
     set_tenant_id("bh7sjh...")
 
+    logger.info("Handling request", extra={"user_id": user_id})
+
     # Parent span carries business context for the whole agent turn.
     # Autoinstrumentation creates the child LLM span automatically.
     with invoke_agent_span(
@@ -224,6 +277,7 @@ async def handle_request(query: str, user_id: str):
     ):
         documents = await retrieve_knowledge_base(query)
         add_span_attribute("documents.retrieved", len(documents))
+        logger.debug("Retrieved %d documents", len(documents))
 
         response = completion(
             model="gpt-4",
@@ -306,7 +360,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector.example.com"
 
 ### Transport protocol
 
-Both traces and metrics use gRPC by default. Switch to HTTP/protobuf by setting:
+Traces, metrics, and logs all use gRPC by default. Switch to HTTP/protobuf by setting:
 
 ```bash
 export OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"

tests/core/unit/telemetry/test_auto_instrument.py

@@ -25,6 +25,7 @@ def mock_traceloop_components():
             'get_tracer_provider': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.trace.get_tracer_provider', return_value=create_autospec(SDKTracerProvider))),
             'create_resource': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.create_resource_attributes_from_env')),
             'get_app_name': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument._get_app_name')),
+            'setup_log_provider': stack.enter_context(patch('sap_cloud_sdk.core.telemetry.auto_instrument.setup_log_provider')),
         }
         yield mocks
 
@@ -405,3 +406,29 @@ def test_baggage_and_middleware_processors_both_added(self, mock_traceloop_compo
 
         # add_span_processor called 3 times: baggage, propagated attributes, middleware
         assert mock_traceloop_components['get_tracer_provider'].return_value.add_span_processor.call_count == 3
+
+
+class TestAutoInstrumentLogging:
+    def test_setup_log_provider_called_on_instrument(self, mock_traceloop_components):
+        mock_traceloop_components['get_app_name'].return_value = 'test-app'
+        mock_traceloop_components['create_resource'].return_value = {}
+
+        with patch.dict('os.environ', {'OTEL_EXPORTER_OTLP_ENDPOINT': 'http://localhost:4317'}, clear=True):
+            auto_instrument()
+
+            mock_traceloop_components['setup_log_provider'].assert_called_once()
+
+    def test_setup_log_provider_not_called_when_no_endpoint(self):
+        with patch.dict('os.environ', {}, clear=True):
+            with patch('sap_cloud_sdk.core.telemetry.auto_instrument.setup_log_provider') as mock_log:
+                auto_instrument()
+                mock_log.assert_not_called()
+
+    def test_setup_log_provider_called_with_console_exporter(self, mock_traceloop_components):
+        mock_traceloop_components['get_app_name'].return_value = 'test-app'
+        mock_traceloop_components['create_resource'].return_value = {}
+
+        with patch.dict('os.environ', {'OTEL_TRACES_EXPORTER': 'console'}, clear=True):
+            auto_instrument()
+
+            mock_traceloop_components['setup_log_provider'].assert_called_once()

tests/core/unit/telemetry/test_log_provider_e2e.py

@@ -0,0 +1,137 @@
+"""End-to-end tests for OTel log provider using a real in-memory exporter.
+
+These tests exercise the full log pipeline — LoggingHandler → LoggerProvider →
+processor → exporter — without mocking. They verify that logs emitted via the
+standard stdlib logging API arrive with the correct resource attributes and,
+when inside an active span, carry trace/span correlation IDs.
+"""
+
+import logging
+import pytest
+
+from opentelemetry import trace
+from opentelemetry._logs import _internal as _logs_internal
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter, SimpleLogRecordProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.instrumentation.logging.handler import LoggingHandler
+
+from sap_cloud_sdk.core.telemetry._provider import setup_log_provider
+from sap_cloud_sdk.core.telemetry.config import InstrumentationConfig
+
+
+@pytest.fixture()
+def log_exporter(monkeypatch):
+    """Set up a real LoggerProvider backed by an in-memory exporter.
+
+    Resets the OTel logger provider singleton before and after each test so
+    tests are fully isolated. Uses SimpleLogRecordProcessor so records flush
+    synchronously without needing a flush() call.
+    """
+    # Reset the OTel logger provider singleton so set_logger_provider() works
+    _logs_internal._LOGGER_PROVIDER_SET_ONCE._done = False
+    _logs_internal._LOGGER_PROVIDER = None
+
+    exporter = InMemoryLogRecordExporter()
+
+    monkeypatch.setattr(
+        "sap_cloud_sdk.core.telemetry._provider._create_log_exporter",
+        lambda: exporter,
+    )
+    monkeypatch.setattr(
+        "sap_cloud_sdk.core.telemetry._provider.BatchLogRecordProcessor",
+        SimpleLogRecordProcessor,
+    )
+    monkeypatch.setattr(
+        "sap_cloud_sdk.core.telemetry._provider.get_config",
+        lambda: InstrumentationConfig(
+            enabled=True,
+            service_name="test-svc",
+            otlp_endpoint="http://localhost:4317",
+        ),
+    )
+    monkeypatch.setenv("APPFND_CONHOS_APP_NAME", "test-svc")
+    monkeypatch.setenv("APPFND_CONHOS_REGION", "eu10")
+    monkeypatch.setenv("APPFND_CONHOS_SUBACCOUNTID", "sub-123")
+    monkeypatch.setenv("APPFND_CONHOS_SYSTEM_ROLE", "TEST")
+    monkeypatch.setenv("SAP_SOLUTION_AREA", "AFND")
+
+    provider = setup_log_provider()
+    assert provider is not None
+
+    root = logging.getLogger()
+    original_level = root.level
+    root.setLevel(logging.DEBUG)
+
+    yield exporter
+
+    root.setLevel(original_level)
+    for h in list(root.handlers):
+        if isinstance(h, LoggingHandler):
+            root.removeHandler(h)
+
+    # Reset singleton again so the next test starts clean
+    _logs_internal._LOGGER_PROVIDER_SET_ONCE._done = False
+    _logs_internal._LOGGER_PROVIDER = None
+
+
+@pytest.fixture()
+def tracer():
+    provider = TracerProvider()
+    trace.set_tracer_provider(provider)
+    return provider.get_tracer("test")
+
+
+class TestLogProviderEndToEnd:
+    def test_log_record_reaches_exporter(self, log_exporter):
+        logging.getLogger("test.basic").warning("hello from sdk")
+
+        records = log_exporter.get_finished_logs()
+        assert len(records) == 1
+        assert records[0].log_record.body == "hello from sdk"
+
+    def test_severity_mapped_correctly(self, log_exporter):
+        logger = logging.getLogger("test.severity")
+        logger.info("info msg")
+        logger.warning("warn msg")
+        logger.error("error msg")
+
+        records = log_exporter.get_finished_logs()
+        severities = [r.log_record.severity_text for r in records]
+        assert severities == ["INFO", "WARN", "ERROR"]
+
+    def test_resource_attributes_on_record(self, log_exporter):
+        logging.getLogger("test.resource").info("check resource")
+
+        r = log_exporter.get_finished_logs()[0]
+        attrs = r.resource.attributes
+        assert attrs.get("service.name") == "test-svc"
+        assert attrs.get("sap.cloud_sdk.language") == "python"
+        assert attrs.get("cloud.region") == "eu10"
+        assert attrs.get("sap.cld.subaccount_id") == "sub-123"
+
+    def test_extra_fields_become_log_attributes(self, log_exporter):
+        logging.getLogger("test.extra").warning(
+            "structured log", extra={"tenant_id": "t-abc", "duration_ms": 42}
+        )
+
+        r = log_exporter.get_finished_logs()[0]
+        assert r.log_record.attributes.get("tenant_id") == "t-abc"
+        assert r.log_record.attributes.get("duration_ms") == 42
+
+    def test_trace_correlation_inside_span(self, log_exporter, tracer):
+        with tracer.start_as_current_span("test-span") as span:
+            logging.getLogger("test.trace").info("inside span")
+            expected_trace_id = span.get_span_context().trace_id
+            expected_span_id = span.get_span_context().span_id
+
+        r = log_exporter.get_finished_logs()[0]
+        assert r.log_record.trace_id == expected_trace_id
+        assert r.log_record.span_id == expected_span_id
+
+    def test_no_trace_correlation_outside_span(self, log_exporter):
+        logging.getLogger("test.no_trace").info("outside span")
+
+        r = log_exporter.get_finished_logs()[0]
+        assert r.log_record.trace_id == 0
+        assert r.log_record.span_id == 0