Skip to content
Open
26 changes: 17 additions & 9 deletions agents/src/llm/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ function ensureRGBCompatible(frame: VideoFrame): VideoFrame {
return frame.convert(VideoBufferType.RGBA);
} catch (error) {
throw new Error(
`Failed to convert format ${frame.type} to RGB: ${error}. ` +
`Consider using RGB/RGBA formats or converting on the client side.`,
`Failed to convert format ${frame.type} to RGB: ${error}. Consider using RGB/RGBA formats or converting on the client side.`,
);
}
}
Expand Down Expand Up @@ -135,7 +134,7 @@ export async function serializeImage(image: ImageContent): Promise<SerializedIma
/** Raw OpenAI-adherent function parameters. */
export type OpenAIFunctionParameters = {
type: 'object';
properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any
properties: { [id: string]: unknown }; // Using unknown instead of any
required: string[];
additionalProperties?: boolean;
};
Expand All @@ -149,10 +148,15 @@ export const createToolOptions = <UserData extends UnknownUserData>(
};

/** @internal */
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {
//eslint-disable-next-line @typescript-eslint/no-explicit-any
export const oaiParams = (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
schema: any, //KEEP as'any'but disable the warning
isOpenai = true,
strict = true,
): OpenAIFunctionParameters => {
// Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts
const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);
const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai, strict);
const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;

return {
Expand All @@ -162,7 +166,6 @@ export const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunction
additionalProperties,
};
};

/** @internal */
export const oaiBuildFunctionInfo = (
toolCtx: ToolContext,
Expand Down Expand Up @@ -323,9 +326,14 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
};
}

export function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {
export function toJsonSchema(
schema: ToolInputSchema<unknown>,
isOpenai = true,
strict = true,
): JSONSchema7 {
if (isZodSchema(schema)) {
return zodSchemaToJsonSchema(schema, isOpenai);
return zodSchemaToJsonSchema(schema, isOpenai, strict);
}

return schema as JSONSchema7;
}
78 changes: 63 additions & 15 deletions agents/src/llm/zod-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,35 +91,83 @@ export function isZodObjectSchema(schema: ZodSchema): boolean {
}

/**
* Converts a Zod schema to JSON Schema format.
* Handles both Zod v3 and v4 schemas automatically.
* Ensures a JSON schema has strict validation enabled.
* Based on OpenAI's ensureStrictJsonSchema implementation.
*
* Adapted from Vercel AI SDK's zod3Schema and zod4Schema functions.
* Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L237-L269
* @param schema - The JSON schema to make strict
* @param strict - Whether to enable strict mode (default: true)
* @returns The strict JSON schema
*/
export function ensureStrictJsonSchema(schema: JSONSchema7, strict: boolean = true): JSONSchema7 {
if (!strict) {
return schema;
}

// Create a deep copy to avoid mutating the original
const strictSchema = JSON.parse(JSON.stringify(schema)) as JSONSchema7;

// Set strict mode for object schemas
if (strictSchema.type === 'object' && strictSchema.properties) {
strictSchema.additionalProperties = false;

// Ensure required array exists for object schemas
if (!strictSchema.required && strictSchema.properties) {
strictSchema.required = Object.keys(strictSchema.properties);
}
}

// Recursively apply strict mode to nested object schemas
if (strictSchema.properties) {
for (const [key, propSchema] of Object.entries(strictSchema.properties)) {
if (typeof propSchema === 'object' && propSchema !== null && !Array.isArray(propSchema)) {
strictSchema.properties[key] = ensureStrictJsonSchema(propSchema, strict);
}
}
}

// Handle array items (single schema, not array of schemas)
if (
strictSchema.items &&
typeof strictSchema.items === 'object' &&
!Array.isArray(strictSchema.items)
) {
strictSchema.items = ensureStrictJsonSchema(strictSchema.items, strict);
}

return strictSchema;
}

/**
* Converts a Zod schema to JSON Schema format with strict mode support.
* Handles both Zod v3 and v4 schemas automatically.
*
* @param schema - The Zod schema to convert
* @param isOpenai - Whether to use OpenAI-specific formatting (default: true)
* @param strict - Whether to enable strict validation (default: true)
* @returns A JSON Schema representation of the Zod schema
*/
export function zodSchemaToJsonSchema(schema: ZodSchema, isOpenai: boolean = true): JSONSchema7 {
export function zodSchemaToJsonSchema(
schema: ZodSchema,
isOpenai: boolean = true,
strict: boolean = true,
): JSONSchema7 {
let jsonSchema: JSONSchema7;

if (isZod4Schema(schema)) {
// Zod v4 has native toJSONSchema support
// Configuration adapted from Vercel AI SDK to support OpenAPI conversion for Google
// Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L255-L258
return z4.toJSONSchema(schema, {
jsonSchema = z4.toJSONSchema(schema, {
target: 'draft-7',
io: 'output',
reused: 'inline', // Don't use references by default (to support openapi conversion for google)
reused: 'inline',
}) as JSONSchema7;
} else {
// Zod v3 requires the zod-to-json-schema library
// Configuration adapted from Vercel AI SDK
// $refStrategy: 'none' is equivalent to v4's reused: 'inline'
return zodToJsonSchemaV3(schema, {
jsonSchema = zodToJsonSchemaV3(schema, {
target: isOpenai ? 'openAi' : 'jsonSchema7',
$refStrategy: 'none', // Don't use references by default (to support openapi conversion for google)
$refStrategy: 'none',
}) as JSONSchema7;
}

// Apply strict schema enforcement
return ensureStrictJsonSchema(jsonSchema, strict);
}

/**
Expand Down
98 changes: 63 additions & 35 deletions agents/src/tts/tts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,40 +157,54 @@ export abstract class SynthesizeStream
}

private async mainTask() {
let lastError: unknown;

for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
try {
return await this.run();
} catch (error) {
} catch (error: unknown) {
lastError = error;

if (error instanceof APIError) {
const retryInterval = this._connOptions._intervalForRetry(i);

if (this._connOptions.maxRetry === 0 || !error.retryable) {
this.emitError({ error, recoverable: false });
throw error;
} else if (i === this._connOptions.maxRetry) {
this.emitError({ error, recoverable: false });
throw new APIConnectionError({
message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
options: { retryable: false },
});
} else {
// Don't emit error event for recoverable errors during retry loop
// to avoid ERR_UNHANDLED_ERROR or premature session termination
// Non-retryable error or retries disabled - break immediately
break;
} else if (i < this._connOptions.maxRetry) {
// Retryable error with retries remaining - log and wait
this.logger.warn(
{ tts: this.#tts.label, attempt: i + 1, error },
`failed to synthesize speech, retrying in ${retryInterval}s`,
`failed to synthesize speech, retrying in ${retryInterval}s`,
);
}

if (retryInterval > 0) {
await delay(retryInterval);
if (retryInterval > 0) {
await delay(retryInterval);
}
}
// If i === maxRetry, we break and handle below
} else {
this.emitError({ error: toError(error), recoverable: false });
throw error;
// Non-APIError - break immediately
break;
}
}
}

// Only emit error after all retries are exhausted
if (lastError) {
const error = toError(lastError);
const recoverable = error instanceof APIError && error.retryable;
this.emitError({ error, recoverable });

if (error instanceof APIError && recoverable) {
throw new APIConnectionError({
message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
options: { retryable: false },
});
} else {
throw error;
}
}
}

private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
Expand Down Expand Up @@ -385,40 +399,54 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
}

private async mainTask() {
let lastError: unknown;

for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
try {
return await this.run();
} catch (error) {
} catch (error: unknown) {
lastError = error;

if (error instanceof APIError) {
const retryInterval = this._connOptions._intervalForRetry(i);

if (this._connOptions.maxRetry === 0 || !error.retryable) {
this.emitError({ error, recoverable: false });
throw error;
} else if (i === this._connOptions.maxRetry) {
this.emitError({ error, recoverable: false });
throw new APIConnectionError({
message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
options: { retryable: false },
});
} else {
// Don't emit error event for recoverable errors during retry loop
// to avoid ERR_UNHANDLED_ERROR or premature session termination
// Non-retryable error or retries disabled - break immediately
break;
} else if (i < this._connOptions.maxRetry) {
// Retryable error with retries remaining - log and wait
this.logger.warn(
{ tts: this.#tts.label, attempt: i + 1, error },
`failed to generate TTS completion, retrying in ${retryInterval}s`,
);
}

if (retryInterval > 0) {
await delay(retryInterval);
if (retryInterval > 0) {
await delay(retryInterval);
}
}
// If i === maxRetry, we break and handle below
} else {
this.emitError({ error: toError(error), recoverable: false });
throw error;
// Non-APIError - break immediately
break;
}
}
}

// Only emit error after all retries are exhausted
if (lastError) {
const error = toError(lastError);
const recoverable = error instanceof APIError && error.retryable;
this.emitError({ error, recoverable });

if (error instanceof APIError && recoverable) {
throw new APIConnectionError({
message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
options: { retryable: false },
});
} else {
throw error;
}
}
}

private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
Expand Down