Skip to content

Commit 6041aa7

Browse files
authored
Merge branch 'main' into test/langsmith-builtin-query-filtering
2 parents cf3c824 + b66cf29 commit 6041aa7

24 files changed

Lines changed: 770 additions & 205 deletions

.github/scripts/cloud_namespace.py

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
"""Create and delete an isolated Temporal Cloud namespace for CI."""
2+
3+
import asyncio
4+
import os
5+
import sys
6+
import time
7+
from pathlib import Path
8+
9+
from temporalio.api.cloud.cloudservice.v1 import (
10+
CreateNamespaceRequest,
11+
DeleteNamespaceRequest,
12+
GetAsyncOperationRequest,
13+
GetNamespaceRequest,
14+
)
15+
from temporalio.api.cloud.namespace.v1 import MtlsAuthSpec, NamespaceSpec
16+
from temporalio.api.cloud.operation.v1 import AsyncOperation
17+
from temporalio.client import CloudOperationsClient
18+
19+
20+
async def wait_for_operation(
21+
client: CloudOperationsClient, operation: AsyncOperation
22+
) -> None:
23+
deadline = time.monotonic() + 10 * 60
24+
while True:
25+
operation = (
26+
await client.cloud_service.get_async_operation(
27+
GetAsyncOperationRequest(async_operation_id=operation.id)
28+
)
29+
).async_operation
30+
if operation.state == AsyncOperation.STATE_FULFILLED:
31+
return
32+
if operation.state in {
33+
AsyncOperation.STATE_FAILED,
34+
AsyncOperation.STATE_CANCELLED,
35+
AsyncOperation.STATE_REJECTED,
36+
}:
37+
raise RuntimeError(
38+
"Cloud operation "
39+
f"{operation.id} {AsyncOperation.State.Name(operation.state).lower()}: "
40+
f"{operation.failure_reason}"
41+
)
42+
if time.monotonic() >= deadline:
43+
raise TimeoutError(f"Timed out waiting for Cloud operation {operation.id}")
44+
delay = max(
45+
operation.check_duration.seconds
46+
+ operation.check_duration.nanos / 1_000_000_000,
47+
1,
48+
)
49+
await asyncio.sleep(min(delay, deadline - time.monotonic()))
50+
51+
52+
async def create() -> None:
53+
client = await cloud_client()
54+
namespace_name = "sdk-python-ci-{}-{}".format(
55+
os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"]
56+
)
57+
result = await client.cloud_service.create_namespace(
58+
CreateNamespaceRequest(
59+
spec=NamespaceSpec(
60+
name=namespace_name,
61+
regions=["aws-ca-central-1"],
62+
retention_days=1,
63+
mtls_auth=MtlsAuthSpec(
64+
accepted_client_ca=Path(
65+
os.environ["TEMPORAL_CLOUD_CLIENT_CA_PATH"]
66+
).read_bytes(),
67+
enabled=True,
68+
),
69+
)
70+
)
71+
)
72+
# Make cleanup possible even if provisioning fails after Cloud accepts the request.
73+
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
74+
output.write(f"namespace={result.namespace}\n")
75+
await wait_for_operation(client, result.async_operation)
76+
77+
78+
async def delete(namespace: str) -> None:
79+
client = await cloud_client()
80+
existing = await client.cloud_service.get_namespace(
81+
GetNamespaceRequest(namespace=namespace)
82+
)
83+
result = await client.cloud_service.delete_namespace(
84+
DeleteNamespaceRequest(
85+
namespace=namespace,
86+
resource_version=existing.namespace.resource_version,
87+
)
88+
)
89+
await wait_for_operation(client, result.async_operation)
90+
91+
92+
async def cloud_client() -> CloudOperationsClient:
93+
return await CloudOperationsClient.connect(
94+
api_key=os.environ["TEMPORAL_CLIENT_CLOUD_API_KEY"],
95+
version=os.environ["TEMPORAL_CLIENT_CLOUD_API_VERSION"],
96+
)
97+
98+
99+
async def main() -> None:
100+
match sys.argv[1:]:
101+
case ["create"]:
102+
await create()
103+
case ["delete", namespace]:
104+
await delete(namespace)
105+
case _:
106+
raise ValueError("Usage: cloud_namespace.py create|delete <namespace>")
107+
108+
109+
if __name__ == "__main__":
110+
asyncio.run(main())

.github/workflows/ci.yml

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -230,19 +230,46 @@ jobs:
230230
- run: uv tool install poethepoet
231231
- run: uv sync --all-extras
232232
- run: poe build-develop
233+
- name: Generate Cloud test certificates
234+
run: |
235+
cert_dir="$RUNNER_TEMP/cloud-test-certs"
236+
mkdir "$cert_dir"
237+
openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
238+
-keyout "$cert_dir/ca.key" -out "$cert_dir/ca.pem" \
239+
-subj '/CN=Temporal Python SDK Cloud CI CA'
240+
openssl req -newkey rsa:2048 -nodes \
241+
-keyout "$cert_dir/client.key" -out "$cert_dir/client.csr" \
242+
-subj '/CN=Temporal Python SDK Cloud CI'
243+
openssl x509 -req -days 1 -in "$cert_dir/client.csr" \
244+
-CA "$cert_dir/ca.pem" -CAkey "$cert_dir/ca.key" -CAcreateserial \
245+
-out "$cert_dir/client.pem" -extfile <(printf 'extendedKeyUsage=clientAuth')
246+
{
247+
echo "TEMPORAL_CLOUD_CLIENT_CA_PATH=$cert_dir/ca.pem"
248+
echo "TEMPORAL_TLS_CLIENT_CERT_PATH=$cert_dir/client.pem"
249+
echo "TEMPORAL_TLS_CLIENT_KEY_PATH=$cert_dir/client.key"
250+
} >> "$GITHUB_ENV"
251+
- name: Create Cloud namespace
252+
id: create-cloud-namespace
253+
run: uv run python .github/scripts/cloud_namespace.py create
254+
env:
255+
TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
256+
TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
233257
- run: mkdir junit-xml
234258
- run: poe test -s --workflow-environment envconfig --junit-xml=junit-xml/cloud.xml
235259
timeout-minutes: 15
236260
env:
237-
TEMPORAL_ADDRESS: sdk-ci.a2dd6.tmprl.cloud:7233
238-
TEMPORAL_NAMESPACE: sdk-ci.a2dd6
239-
TEMPORAL_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
240-
TEMPORAL_TLS_CLIENT_CERT_DATA: ${{ secrets.TEMPORAL_CLIENT_CERT }}
241-
TEMPORAL_TLS_CLIENT_KEY_DATA: ${{ secrets.TEMPORAL_CLIENT_KEY }}
261+
TEMPORAL_ADDRESS: ${{ steps.create-cloud-namespace.outputs.namespace }}.tmprl.cloud:7233
262+
TEMPORAL_NAMESPACE: ${{ steps.create-cloud-namespace.outputs.namespace }}
242263
TEMPORAL_IS_CLOUD_TESTS: true
243264
TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
244-
TEMPORAL_CLIENT_CLOUD_API_VERSION: 2024-05-13-00
245-
TEMPORAL_CLIENT_CLOUD_NAMESPACE: sdk-ci.a2dd6
265+
TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
266+
TEMPORAL_CLIENT_CLOUD_NAMESPACE: ${{ steps.create-cloud-namespace.outputs.namespace }}
267+
- name: Delete Cloud namespace
268+
if: ${{ always() && steps.create-cloud-namespace.outputs.namespace != '' }}
269+
run: uv run python .github/scripts/cloud_namespace.py delete "${{ steps.create-cloud-namespace.outputs.namespace }}"
270+
env:
271+
TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
272+
TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
246273
- name: "Upload junit-xml artifacts"
247274
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
248275
if: always()

CHANGELOG.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ to include examples, links to docs, or any other relevant information.
2222

2323
### Changed
2424

25+
- `temporalio.contrib.pydantic` converters now reuse Pydantic type adapters
26+
for repeated type hints instead of rebuilding their schemas for every
27+
payload, greatly speeding up decode of non-model hints such as discriminated
28+
unions ([#1695](https://github.com/temporalio/sdk-python/issues/1695)). Up
29+
to 1024 type adapters are cached per converter instance by default, with
30+
least-recently-used eviction. To change the bound, pass
31+
``max_cached_type_adapters`` to ``PydanticPayloadConverter`` (or
32+
``PydanticJSONPlainPayloadConverter``) from a nullary subclass used as the
33+
``DataConverter.payload_converter_class``; ``None`` makes the cache
34+
unbounded and zero disables caching.
35+
2536
### Deprecated
2637

2738
### :boom: Breaking Changes

temporalio/contrib/pydantic.py

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
Pydantic v1 is not supported.
1414
"""
1515

16+
import functools
1617
from dataclasses import dataclass
1718
from typing import Any
1819

@@ -53,10 +54,28 @@ class PydanticJSONPlainPayloadConverter(EncodingPayloadConverter):
5354
See https://docs.pydantic.dev/latest/api/standard_library_types/
5455
"""
5556

56-
def __init__(self, to_json_options: ToJsonOptions | None = None):
57-
"""Create a new payload converter."""
57+
def __init__(
58+
self,
59+
to_json_options: ToJsonOptions | None = None,
60+
*,
61+
max_cached_type_adapters: int | None = 1024,
62+
) -> None:
63+
"""Create a new payload converter.
64+
65+
Args:
66+
to_json_options: Options for serializing values to JSON.
67+
max_cached_type_adapters: Maximum number of type adapters to
68+
cache, with least-recently-used eviction. Defaults to 1024.
69+
If ``None``, the cache is unbounded. If zero, caching is
70+
disabled.
71+
"""
72+
if max_cached_type_adapters is not None and max_cached_type_adapters < 0:
73+
raise ValueError("max_cached_type_adapters cannot be negative")
5874
self._schema_serializer = SchemaSerializer(any_schema())
5975
self._to_json_options = to_json_options
76+
self._type_adapter = functools.lru_cache(maxsize=max_cached_type_adapters)(
77+
TypeAdapter
78+
)
6079

6180
@property
6281
def encoding(self) -> str:
@@ -91,12 +110,26 @@ def from_payload(
91110
92111
Uses ``pydantic.TypeAdapter.validate_json`` to construct an
93112
instance of the type specified by ``type_hint`` from the JSON payload.
113+
Type adapters are cached per hashable type hint; see
114+
``max_cached_type_adapters`` on the constructor.
94115
95116
See
96117
https://docs.pydantic.dev/latest/api/type_adapter/#pydantic.type_adapter.TypeAdapter.validate_json.
97118
"""
98119
_type_hint = type_hint if type_hint is not None else Any
99-
return TypeAdapter(_type_hint).validate_json(payload.data)
120+
type_adapter: TypeAdapter[Any]
121+
try:
122+
type_adapter = self._type_adapter(_type_hint)
123+
except TypeError:
124+
# Distinguish an unhashable hint (bypass the cache) from a
125+
# TypeError raised while constructing the adapter (re-raise).
126+
try:
127+
hash(_type_hint)
128+
except TypeError:
129+
type_adapter = TypeAdapter(_type_hint)
130+
else:
131+
raise
132+
return type_adapter.validate_json(payload.data)
100133

101134

102135
class PydanticPayloadConverter(CompositePayloadConverter):
@@ -106,9 +139,36 @@ class PydanticPayloadConverter(CompositePayloadConverter):
106139
:py:class:`PydanticJSONPlainPayloadConverter`.
107140
"""
108141

109-
def __init__(self, to_json_options: ToJsonOptions | None = None) -> None:
110-
"""Initialize object"""
111-
json_payload_converter = PydanticJSONPlainPayloadConverter(to_json_options)
142+
def __init__(
143+
self,
144+
to_json_options: ToJsonOptions | None = None,
145+
*,
146+
max_cached_type_adapters: int | None = 1024,
147+
) -> None:
148+
"""Initialize object.
149+
150+
Args:
151+
to_json_options: Options for serializing values to JSON.
152+
max_cached_type_adapters: Maximum number of type adapters to
153+
cache, with least-recently-used eviction. Defaults to 1024.
154+
If ``None``, the cache is unbounded. If zero, caching is
155+
disabled.
156+
157+
To configure this through a :py:class:`DataConverter`, use a
158+
nullary subclass as the payload converter class::
159+
160+
class MyPayloadConverter(PydanticPayloadConverter):
161+
def __init__(self) -> None:
162+
super().__init__(max_cached_type_adapters=128)
163+
164+
my_data_converter = DataConverter(
165+
payload_converter_class=MyPayloadConverter
166+
)
167+
"""
168+
json_payload_converter = PydanticJSONPlainPayloadConverter(
169+
to_json_options,
170+
max_cached_type_adapters=max_cached_type_adapters,
171+
)
112172
super().__init__(
113173
*(
114174
c

temporalio/nexus/_operation_context.py

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -766,25 +766,20 @@ def _apply_nexus_context_to_start_activity_request( # pyright: ignore[reportUnu
766766
"""Apply the current Nexus operation context to an activity start request.
767767
768768
This is a no-op outside a Nexus operation context. Within one, it attaches
769-
the Nexus request ID and inbound links and configures conflict handling to
770-
preserve the Nexus metadata. Completion callbacks are added only when the
771-
activity is backing the Nexus operation.
769+
the Nexus request ID and configures conflict handling to preserve the Nexus
770+
metadata. Inbound links are attached to the completion callback when the
771+
activity backs the operation and to the request otherwise.
772772
"""
773773
nexus_ctx = _try_start_operation_context()
774774
if nexus_ctx is not None:
775775
req.on_conflict_options.attach_request_id = True
776776
req.on_conflict_options.attach_completion_callbacks = True
777777
req.on_conflict_options.attach_links = True
778778

779-
# Add request_id and all Nexus links if we're in a Nexus context, backing or otherwise
780779
req.request_id = nexus_ctx.nexus_context.request_id
781780
request_links = nexus_ctx._get_request_links()
782781

783-
# Links are duplicated on request for compatibility with older server versions.
784-
req.links.extend(request_links)
785-
786782
if _in_nexus_backing_start_context():
787-
# Add callbacks only if we're in a backing Nexus context
788783
callbacks = nexus_ctx._get_callbacks(
789784
OperationToken(
790785
type=OperationTokenType.ACTIVITY,
@@ -802,6 +797,8 @@ def _apply_nexus_context_to_start_activity_request( # pyright: ignore[reportUnu
802797
)
803798
for callback in callbacks
804799
)
800+
else:
801+
req.links.extend(request_links)
805802

806803

807804
def _apply_start_activity_response_to_nexus_context( # pyright: ignore[reportUnusedFunction]

tests/contrib/langsmith/conftest.py

Lines changed: 13 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
import pytest
1010

11+
from tests.helpers.trace import TraceNode
12+
1113

1214
@pytest.fixture(autouse=True)
1315
def _clear_langsmith_env_cache() -> Any: # pyright: ignore[reportUnusedFunction]
@@ -92,13 +94,8 @@ def clear(self) -> None:
9294
self._by_id.clear()
9395

9496

95-
def dump_traces(collector: InMemoryRunCollector) -> list[list[str]]:
96-
"""Reconstruct parent-child hierarchy grouped by root trace.
97-
98-
Returns a list of traces, where each trace is a list of indented
99-
strings (same format as dump_runs). Each trace starts from a
100-
different root run.
101-
"""
97+
def build_trace_trees(collector: InMemoryRunCollector) -> list[TraceNode]:
98+
"""Build trace trees from the collector's run parent relationships."""
10299
runs = collector.runs
103100
children: dict[str | None, list[_RunRecord]] = {}
104101
for r in runs:
@@ -113,30 +110,18 @@ def dump_traces(collector: InMemoryRunCollector) -> list[list[str]]:
113110
f"which is not in the collected runs — dangling parent reference"
114111
)
115112

116-
traces: list[list[str]] = []
117-
for root in children.get(None, []):
118-
trace: list[str] = []
119-
120-
def _walk(parent_id: str | None, depth: int) -> None:
121-
for child in children.get(parent_id, []):
122-
trace.append(" " * depth + child.name)
123-
_walk(child.id, depth + 1)
124-
125-
trace.append(root.name)
126-
_walk(root.id, 1)
127-
traces.append(trace)
128-
129-
return traces
130-
113+
def build_tree(run: _RunRecord) -> TraceNode:
114+
return TraceNode(
115+
run.name,
116+
[build_tree(child) for child in children.get(run.id, [])],
117+
)
131118

132-
def dump_runs(collector: InMemoryRunCollector) -> list[str]:
133-
"""Flat list of all runs across all traces."""
134-
return [run for trace in dump_traces(collector) for run in trace]
119+
return [build_tree(root) for root in children.get(None, [])]
135120

136121

137-
def find_traces(traces: list[list[str]], root_name: str) -> list[list[str]]:
138-
"""Filter traces by exact root name match."""
139-
return [t for t in traces if t[0] == root_name]
122+
def find_trace_trees(traces: list[TraceNode], root_name: str) -> list[TraceNode]:
123+
"""Filter trace trees by exact root run name."""
124+
return [trace for trace in traces if trace.name == root_name]
140125

141126

142127
def make_mock_ls_client(collector: InMemoryRunCollector) -> MagicMock:

0 commit comments

Comments
 (0)