diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 21d8174dc6d..a24fd4b355b 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -1762,6 +1762,11 @@ def submit_evaluation( "Failed to parse tags. Tags for evaluation metrics must be strings." ) + # Auto-add source:otel tag when OTel tracing is enabled + # This allows the backend to wait for OTel span conversion + if config._otel_trace_enabled: + evaluation_tags["source"] = "otel" + evaluation_metric: LLMObsEvaluationMetricEvent = { "join_on": join_on, "label": str(label), diff --git a/tests/llmobs/test_llmobs_otel_evaluation.py b/tests/llmobs/test_llmobs_otel_evaluation.py new file mode 100644 index 00000000000..da34a46b368 --- /dev/null +++ b/tests/llmobs/test_llmobs_otel_evaluation.py @@ -0,0 +1,45 @@ +""" +Tests for automatic source:otel tag on evaluations when OTel tracing is enabled. + +When DD_TRACE_OTEL_ENABLED=true, all evaluations should have `source:otel` tag +to allow the backend to wait for OTel span conversion (~3 minutes) before +discarding unmatched evaluations. +""" + +import mock + + +def test_submit_evaluation_adds_source_otel_when_otel_enabled(llmobs, mock_llmobs_eval_metric_writer): + """Verify source:otel tag is auto-added when DD_TRACE_OTEL_ENABLED=true.""" + with mock.patch("ddtrace.llmobs._llmobs.config._otel_trace_enabled", True): + llmobs.submit_evaluation( + span={"span_id": "123", "trace_id": "456"}, + label="quality", + metric_type="score", + value=0.9, + ml_app="test-app", + ) + + mock_llmobs_eval_metric_writer.enqueue.assert_called_once() + call_args = mock_llmobs_eval_metric_writer.enqueue.call_args[0][0] + + assert "tags" in call_args + assert "source:otel" in call_args["tags"] + + +def test_submit_evaluation_no_source_otel_when_otel_disabled(llmobs, mock_llmobs_eval_metric_writer): + """Verify source:otel tag is NOT added when DD_TRACE_OTEL_ENABLED=false (default).""" + with mock.patch("ddtrace.llmobs._llmobs.config._otel_trace_enabled", False): + llmobs.submit_evaluation( + span={"span_id": "123", "trace_id": "456"}, + label="quality", + metric_type="score", + value=0.9, + ml_app="test-app", + ) + + mock_llmobs_eval_metric_writer.enqueue.assert_called_once() + call_args = mock_llmobs_eval_metric_writer.enqueue.call_args[0][0] + + assert "tags" in call_args + assert "source:otel" not in call_args["tags"]