opentelemetry-exporter-otlp-proto-common: permit to serialize null values in logs (#4400)

* opentelemetry-exporter-otlp-proto-common: permit to serialize null values in logs

* Add CHANGELOG

* Update exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py

* Fix case where a list contains None values and make tests more clear

* Keep behavior the same for None values in list with allow_null=False

* fix typo

* Fix test failing on <= 3.8

---------

Co-authored-by: Aaron Abbott <[email protected]>

Author: xrmx

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#4371](https://github.com/open-telemetry/opentelemetry-python/pull/4371))
 - Fix span context manager typing by using ParamSpec from typing_extensions
   ([#4389](https://github.com/open-telemetry/opentelemetry-python/pull/4389))
+- Fix serialization of None values in logs body to match 1.31.0+ data model
+  ([#4400](https://github.com/open-telemetry/opentelemetry-python/pull/4400))
 - [BREAKING] semantic-conventions: Remove `opentelemetry.semconv.attributes.network_attributes.NETWORK_INTERFACE_NAME`
   introduced by mistake in the wrong module.
   ([#4391](https://github.com/open-telemetry/opentelemetry-python/pull/4391))

exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+from __future__ import annotations
+
 import logging
 from collections.abc import Sequence
 from itertools import count
@@ -66,7 +68,11 @@ def _encode_resource(resource: Resource) -> PB2Resource:
     return PB2Resource(attributes=_encode_attributes(resource.attributes))
 
 
-def _encode_value(value: Any) -> PB2AnyValue:
+def _encode_value(
+    value: Any, allow_null: bool = False
+) -> Optional[PB2AnyValue]:
+    if allow_null is True and value is None:
+        return None
     if isinstance(value, bool):
         return PB2AnyValue(bool_value=value)
     if isinstance(value, str):
@@ -79,19 +85,45 @@ def _encode_value(value: Any) -> PB2AnyValue:
         return PB2AnyValue(bytes_value=value)
     if isinstance(value, Sequence):
         return PB2AnyValue(
-            array_value=PB2ArrayValue(values=[_encode_value(v) for v in value])
+            array_value=PB2ArrayValue(
+                values=_encode_array(value, allow_null=allow_null)
+            )
         )
     elif isinstance(value, Mapping):
         return PB2AnyValue(
             kvlist_value=PB2KeyValueList(
-                values=[_encode_key_value(str(k), v) for k, v in value.items()]
+                values=[
+                    _encode_key_value(str(k), v, allow_null=allow_null)
+                    for k, v in value.items()
+                ]
             )
         )
     raise Exception(f"Invalid type {type(value)} of value {value}")
 
 
-def _encode_key_value(key: str, value: Any) -> PB2KeyValue:
-    return PB2KeyValue(key=key, value=_encode_value(value))
+def _encode_key_value(
+    key: str, value: Any, allow_null: bool = False
+) -> PB2KeyValue:
+    return PB2KeyValue(
+        key=key, value=_encode_value(value, allow_null=allow_null)
+    )
+
+
+def _encode_array(
+    array: Sequence[Any], allow_null: bool = False
+) -> Sequence[PB2AnyValue]:
+    if not allow_null:
+        # Let the exception get raised by _encode_value()
+        return [_encode_value(v, allow_null=allow_null) for v in array]
+
+    return [
+        _encode_value(v, allow_null=allow_null)
+        if v is not None
+        # Use an empty AnyValue to represent None in an array. Behavior may change pending
+        # https://github.com/open-telemetry/opentelemetry-specification/issues/4392
+        else PB2AnyValue()
+        for v in array
+    ]
 
 
 def _encode_span_id(span_id: int) -> bytes:

exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py

@@ -55,7 +55,7 @@ def _encode_log(log_data: LogData) -> PB2LogRecord:
         span_id=span_id,
         trace_id=trace_id,
         flags=int(log_data.log_record.trace_flags),
-        body=_encode_value(body) if body is not None else None,
+        body=_encode_value(body, allow_null=True),
         severity_text=log_data.log_record.severity_text,
         attributes=_encode_attributes(log_data.log_record.attributes),
         dropped_attributes_count=log_data.log_record.dropped_attributes,

exporter/opentelemetry-exporter-otlp-proto-common/tests/test_attribute_encoder.py

@@ -88,6 +88,24 @@ def test_encode_attributes_all_kinds(self):
             ],
         )
 
+    def test_encode_attributes_error_list_none(self):
+        with self.assertLogs(level=ERROR) as error:
+            result = _encode_attributes(
+                {"a": 1, "bad_key": ["test", None, "test"], "b": 2}
+            )
+
+        self.assertEqual(len(error.records), 1)
+        self.assertEqual(error.records[0].msg, "Failed to encode key %s: %s")
+        self.assertEqual(error.records[0].args[0], "bad_key")
+        self.assertIsInstance(error.records[0].args[1], Exception)
+        self.assertEqual(
+            result,
+            [
+                PB2KeyValue(key="a", value=PB2AnyValue(int_value=1)),
+                PB2KeyValue(key="b", value=PB2AnyValue(int_value=2)),
+            ],
+        )
+
     def test_encode_attributes_error_logs_key(self):
         with self.assertLogs(level=ERROR) as error:
             result = _encode_attributes({"a": 1, "bad_key": None, "b": 2})

exporter/opentelemetry-exporter-otlp-proto-common/tests/test_log_encoder.py

@@ -27,10 +27,16 @@
     ExportLogsServiceRequest,
 )
 from opentelemetry.proto.common.v1.common_pb2 import AnyValue as PB2AnyValue
+from opentelemetry.proto.common.v1.common_pb2 import (
+    ArrayValue as PB2ArrayValue,
+)
 from opentelemetry.proto.common.v1.common_pb2 import (
     InstrumentationScope as PB2InstrumentationScope,
 )
 from opentelemetry.proto.common.v1.common_pb2 import KeyValue as PB2KeyValue
+from opentelemetry.proto.common.v1.common_pb2 import (
+    KeyValueList as PB2KeyValueList,
+)
 from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord as PB2LogRecord
 from opentelemetry.proto.logs.v1.logs_pb2 import (
     ResourceLogs as PB2ResourceLogs,
@@ -154,7 +160,25 @@ def _get_sdk_log_data() -> List[LogData]:
             ),
         )
 
-        return [log1, log2, log3, log4]
+        log5 = LogData(
+            log_record=SDKLogRecord(
+                timestamp=1644650584292683009,
+                observed_timestamp=1644650584292683010,
+                trace_id=212592107417388365804938480559624925555,
+                span_id=6077757853989569445,
+                trace_flags=TraceFlags(0x01),
+                severity_text="INFO",
+                severity_number=SeverityNumber.INFO,
+                body={"error": None, "array_with_nones": [1, None, 2]},
+                resource=SDKResource({}),
+                attributes={},
+            ),
+            instrumentation_scope=InstrumentationScope(
+                "last_name", "last_version"
+            ),
+        )
+
+        return [log1, log2, log3, log4, log5]
 
     def get_test_logs(
         self,
@@ -287,6 +311,56 @@ def get_test_logs(
                         ),
                     ],
                 ),
+                PB2ResourceLogs(
+                    resource=PB2Resource(),
+                    scope_logs=[
+                        PB2ScopeLogs(
+                            scope=PB2InstrumentationScope(
+                                name="last_name",
+                                version="last_version",
+                            ),
+                            log_records=[
+                                PB2LogRecord(
+                                    time_unix_nano=1644650584292683009,
+                                    observed_time_unix_nano=1644650584292683010,
+                                    trace_id=_encode_trace_id(
+                                        212592107417388365804938480559624925555
+                                    ),
+                                    span_id=_encode_span_id(
+                                        6077757853989569445,
+                                    ),
+                                    flags=int(TraceFlags(0x01)),
+                                    severity_text="INFO",
+                                    severity_number=SeverityNumber.INFO.value,
+                                    body=PB2AnyValue(
+                                        kvlist_value=PB2KeyValueList(
+                                            values=[
+                                                PB2KeyValue(key="error"),
+                                                PB2KeyValue(
+                                                    key="array_with_nones",
+                                                    value=PB2AnyValue(
+                                                        array_value=PB2ArrayValue(
+                                                            values=[
+                                                                PB2AnyValue(
+                                                                    int_value=1
+                                                                ),
+                                                                PB2AnyValue(),
+                                                                PB2AnyValue(
+                                                                    int_value=2
+                                                                ),
+                                                            ]
+                                                        )
+                                                    ),
+                                                ),
+                                            ]
+                                        )
+                                    ),
+                                    attributes={},
+                                ),
+                            ],
+                        ),
+                    ],
+                ),
             ]
         )