Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
436 changes: 436 additions & 0 deletions python/agents/persona_ad_gen/README.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"eval_set_id": "persona_ad_gen_evalset",
"name": "persona_ad_gen_evalset",
"description": null,
"eval_cases": [
{
"eval_id": "hello",
"conversation": [
{
"invocation_id": "e-ea7feda9-2e21-43c8-9149-97d4c48db2a3",
"user_content": {
"parts": [
{
"video_metadata": null,
"thought": null,
"inline_data": null,
"code_execution_result": null,
"executable_code": null,
"file_data": null,
"function_call": null,
"function_response": null,
"text": "hello"
}
],
"role": "user"
},
"final_response": {
"parts": [
{
"video_metadata": null,
"thought": null,
"inline_data": null,
"code_execution_result": null,
"executable_code": null,
"file_data": null,
"function_call": null,
"function_response": null,
"text": "Great ads connect with a real person by solving a real problem. Instead of just filling out a form, we're going to build your ad's story step-by-step. First, let's get to know your ideal customer.\n\nDescribe the single person you want to reach. What is a key problem, need, or desire they are currently facing that your business can help with?"
}
],
"role": null
},
"intermediate_data": {
"tool_uses": [],
"intermediate_responses": []
},
"creation_timestamp": 1748011266.735581
}
],
"session_input": {
"app_name": "persona_ad_gen",
"user_id": "user",
"state": {}
},
"creation_timestamp": 1748011280.510737
},
{
"eval_id": "horse_helmet_safety",
"conversation": [
{
"invocation_id": "e-2d8c7b9f-3a1b-4e9f-9d3a-8c7b9f3a1b3e",
"user_content": {
"parts": [
{
"text": "I'm a 40 year old male, interested in horse back riding. I care about safety when riding and I want a new helmet that makes me feel safe. I live in paris"
}
],
"role": "user"
},
"final_response": {
"parts": [
{
"text": "Great ads connect with a real person by solving a real problem. Instead of just filling out a form, we're going to build your ad's story step-by-step. First, let's get to know your ideal customer.\n\nI understand you're looking for a safe riding helmet, but let me help you create a compelling ad for your business. Let's start by identifying your ideal customer.\n\nDescribe the single person you want to reach. What is a key problem, need, or desire they are currently facing that your business can help with?"
}
],
"role": null
},
"intermediate_data": {
"tool_uses": [],
"intermediate_responses": []
},
"creation_timestamp": 1748011300.123456
}
],
"session_input": {
"app_name": "persona_ad_gen",
"user_id": "user",
"state": {}
},
"creation_timestamp": 1748011310.987654
}
],
"creation_timestamp": 1748011130.5057902
}
66 changes: 66 additions & 0 deletions python/agents/persona_ad_gen/eval/test_eval.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import pathlib
import dotenv
import pytest
from google.adk.runners import Runner
from google.adk.sessions import InMemorySessionService
from google.genai import types

from persona_ad_gen import PersonaAdGenAgent

pytest_plugins = ("pytest_asyncio",)


@pytest.fixture(scope="session", autouse=True)
def load_env():
dotenv.load_dotenv()


@pytest.mark.asyncio
async def test_agent_introduction():
"""Test that the agent provides its correct introduction."""
agent = PersonaAdGenAgent()

session_service = InMemorySessionService()
runner = Runner(
agent=agent,
app_name="test_app",
session_service=session_service
)
session = await session_service.create_session(
app_name="test_app",
user_id="test_user",
session_id="test_session"
)

user_message = types.Content(role="user", parts=[types.Part(text="hello")])

response_text = ""
async for event in runner.run_async(user_id="test_user", session_id="test_session", new_message=user_message):
if event.is_final_response() and event.content and event.content.parts:
response_text = event.content.parts[0].text
break

# The expected introduction from the agent's instructions.
expected_introduction = (
"Great ads connect with a real person by solving a real problem. "
"Instead of just filling out a form, we're going to build your ad's story step-by-step. "
"First, let's get to know your ideal customer."
)

assert response_text is not None, "Agent response format is not as expected."
assert response_text.strip().startswith(expected_introduction.strip())
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"{\"eval_set_result_id\":\"persona_ad_gen_persona_ad_gen_evalset_1757136321.2961211\",\"eval_set_result_name\":\"persona_ad_gen_persona_ad_gen_evalset_1757136321.2961211\",\"eval_set_id\":\"persona_ad_gen_evalset\",\"eval_case_results\":[{\"eval_set_file\":\"persona_ad_gen_evalset\",\"eval_set_id\":\"persona_ad_gen_evalset\",\"eval_id\":\"horse_helmet_safety\",\"final_eval_status\":1,\"eval_metric_results\":null,\"overall_eval_metric_results\":[{\"metric_name\":\"tool_trajectory_avg_score\",\"threshold\":1.0,\"judge_model_options\":null,\"score\":1.0,\"eval_status\":1},{\"metric_name\":\"response_match_score\",\"threshold\":0.8,\"judge_model_options\":null,\"score\":0.8148148148148148,\"eval_status\":1}],\"eval_metric_result_per_invocation\":[{\"actual_invocation\":{\"invocation_id\":\"e-62bfd66c-238f-4bf3-abf5-b5173396c1b4\",\"user_content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"I'm a 40 year old male, interested in horse back riding. I care about safety when riding and I want a new helmet that makes me feel safe. I live in paris\"}],\"role\":\"user\"},\"final_response\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":\"CpMJAR_MhbZzPBTjbv5SBwJvuwJVh6UVc5QdJJ4CcEWOOHlc914yQMp0CQcac2sZD5U4R9EWPeXXlxc_mCgtt1_PtNRCLRkA9ly3fpH0kb-C4MolY6JCH5uUebpT1OVuLN1BXgarOmBDWSXPH091EhbyAbrHhSz-gbiVO9MKxlhsV0B6FTK-owEXpWmzMEvahYMEky6jcFf30YLFif9uaV4VFeDwJo8Yh5jjGb_piNsnDXx_yWnDqZTNxXgEDByaEJZZ7eqHY5VcyZgWofmNloVrD-QCQIiABJYV7qIgnsCPfVAp-G6G2KYLFS9fDlxgVa1JDarWtFfBmHpyfJbMBHSLj-NRMTnuKFdk_HLc6NyzvOIyyg4C5ec03fK3BATybvck7W0l2QD_goDgNrpU1qL67In0Ue2BaigAmJ7JX6Jw0CO8nfngJg74brFOnnUy2xr5RktsP8UTa2JdMVsY303o1CQTLEok3jGZDigMmxBo8px5RcyfumrfuIMsQGgHfpNdYF8-GyVadAVzzmUeHInxUMaB__n-PqiH_KyR66imn-ZqR7Kfar1d-fLeDDhGpPZ36issdDvbTHmiLf8nmz5ykYesDG89pqSU2Jh0I3QQ_wO0-XnvnurjgA2gvn--dq_2IusFehi55pg309EIKajUEVMcEjUqxHd7MWN1cxLrz-L1booPdoxvNhKJwBoZW4_fxv9W98hJrvfQuzUsOr1JvlEWcDYF4lDMFqVVWDzhkNuCuLFkyEpa57trEOtK7AUPDtlxdVx_0mcaE2wz7S7doUYdZTLudAK9wro-S-M-whJ6XWmx0iCX8A0kZuVmU1LNTpHNgiNBBHCfk0aSS9Y2zbBstPptO7QBSQyXbCcaAE7XGffjf2KtDMX9AwN5R1NkgHHOMao8LNSlhGImubNYdvILQkStsQx8F4CoKR3qbCmdEo7ycNqIzQ18NEr9Zv5LmQznNC7ZRl1ry5H6n5PJxjMYRbIu2DpVykw5qwfXgDbngz8NJuhu7yMVhxXM9ndVmJ_CqNcjs6en6NCbBcejeWluZuidTL_Aceg3LUZE_TBcQLJjNrVN-ogRvQJzgUOVaQxbYFmzim-TA9-KEIjt7mKdXMMMlDUhsJdTunIyyufekhg0rzS6xn8oTY6H7K0ll4fO2rUG7h3n-ABtBll-tpSNE74Qh2-356Fhc4pgeGQiliP7twHXGFX4pVINduvlzp_kD3RKDlPbgIuVer3WmCyKd0UgmluflZU4rvAE_sAf_LZu00b6XULn0PRf0uN0qUzg22-bn_JMVOQ935RW66lbNfi0mXYf03sshg1X4U2juLhXSd44OAUU246J_BF8pIBqz-5WGkq6df7gUTblMpNsaGWNfK_2HjtP_gSRKrW2e6Xmd7jvuBKoAPKws02a4EzJ9gqUynS50_WDStTRF6yZrSPMG_06gKtz9I6bELiigKeIdG_-mgw37FsEM3TgwFMh0BvQjBYTaMcr8UmhhAsmKoLN6JaGFVZ1C1VoBf7Rm76fngBDM7KtJrtjz8fQUA0xbzitufgISqj4OBa5I2NFxA==\",\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"Great ads connect with a real person by solving a real problem. Instead of just filling out a form, we're going to build your ad's story step-by-step. First, let's get to know your ideal customer.\\n\\nDescribe the single person you want to reach. What is a key problem, need, or desire they are currently facing that your business can help with?\"}],\"role\":\"model\"},\"intermediate_data\":{\"tool_uses\":[],\"intermediate_responses\":[]},\"creation_timestamp\":0.0},\"expected_invocation\":{\"invocation_id\":\"e-2d8c7b9f-3a1b-4e9f-9d3a-8c7b9f3a1b3e\",\"user_content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"I'm a 40 year old male, interested in horse back riding. I care about safety when riding and I want a new helmet that makes me feel safe. I live in paris\"}],\"role\":\"user\"},\"final_response\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"Great ads connect with a real person by solving a real problem. Instead of just filling out a form, we're going to build your ad's story step-by-step. First, let's get to know your ideal customer.\\n\\nI understand you're looking for a safe riding helmet, but let me help you create a compelling ad for your business. Let's start by identifying your ideal customer.\\n\\nDescribe the single person you want to reach. What is a key problem, need, or desire they are currently facing that your business can help with?\"}],\"role\":null},\"intermediate_data\":{\"tool_uses\":[],\"intermediate_responses\":[]},\"creation_timestamp\":1748011300.123456},\"eval_metric_results\":[{\"metric_name\":\"tool_trajectory_avg_score\",\"threshold\":1.0,\"judge_model_options\":null,\"score\":1.0,\"eval_status\":1},{\"metric_name\":\"response_match_score\",\"threshold\":0.8,\"judge_model_options\":null,\"score\":0.8148148148148148,\"eval_status\":1}]}],\"session_id\":\"___eval___session___4cc0bc82-5c58-4e47-a92a-c4d6e123ca43\",\"session_details\":{\"id\":\"___eval___session___4cc0bc82-5c58-4e47-a92a-c4d6e123ca43\",\"app_name\":\"persona_ad_gen\",\"user_id\":\"user\",\"state\":{},\"events\":[{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":null,\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"I'm a 40 year old male, interested in horse back riding. I care about safety when riding and I want a new helmet that makes me feel safe. I live in paris\"}],\"role\":\"user\"},\"grounding_metadata\":null,\"partial\":null,\"turn_complete\":null,\"finish_reason\":null,\"error_code\":null,\"error_message\":null,\"interrupted\":null,\"custom_metadata\":null,\"usage_metadata\":null,\"live_session_resumption_update\":null,\"input_transcription\":null,\"output_transcription\":null,\"invocation_id\":\"e-62bfd66c-238f-4bf3-abf5-b5173396c1b4\",\"author\":\"user\",\"actions\":{\"skip_summarization\":null,\"state_delta\":{},\"artifact_delta\":{},\"transfer_to_agent\":null,\"escalate\":null,\"requested_auth_configs\":{}},\"long_running_tool_ids\":null,\"branch\":null,\"id\":\"2a373660-3eab-478e-81aa-0b5bba94ecf6\",\"timestamp\":1757136318.472101},{\"content\":{\"parts\":[{\"video_metadata\":null,\"thought\":null,\"inline_data\":null,\"file_data\":null,\"thought_signature\":\"CpMJAR_MhbZzPBTjbv5SBwJvuwJVh6UVc5QdJJ4CcEWOOHlc914yQMp0CQcac2sZD5U4R9EWPeXXlxc_mCgtt1_PtNRCLRkA9ly3fpH0kb-C4MolY6JCH5uUebpT1OVuLN1BXgarOmBDWSXPH091EhbyAbrHhSz-gbiVO9MKxlhsV0B6FTK-owEXpWmzMEvahYMEky6jcFf30YLFif9uaV4VFeDwJo8Yh5jjGb_piNsnDXx_yWnDqZTNxXgEDByaEJZZ7eqHY5VcyZgWofmNloVrD-QCQIiABJYV7qIgnsCPfVAp-G6G2KYLFS9fDlxgVa1JDarWtFfBmHpyfJbMBHSLj-NRMTnuKFdk_HLc6NyzvOIyyg4C5ec03fK3BATybvck7W0l2QD_goDgNrpU1qL67In0Ue2BaigAmJ7JX6Jw0CO8nfngJg74brFOnnUy2xr5RktsP8UTa2JdMVsY303o1CQTLEok3jGZDigMmxBo8px5RcyfumrfuIMsQGgHfpNdYF8-GyVadAVzzmUeHInxUMaB__n-PqiH_KyR66imn-ZqR7Kfar1d-fLeDDhGpPZ36issdDvbTHmiLf8nmz5ykYesDG89pqSU2Jh0I3QQ_wO0-XnvnurjgA2gvn--dq_2IusFehi55pg309EIKajUEVMcEjUqxHd7MWN1cxLrz-L1booPdoxvNhKJwBoZW4_fxv9W98hJrvfQuzUsOr1JvlEWcDYF4lDMFqVVWDzhkNuCuLFkyEpa57trEOtK7AUPDtlxdVx_0mcaE2wz7S7doUYdZTLudAK9wro-S-M-whJ6XWmx0iCX8A0kZuVmU1LNTpHNgiNBBHCfk0aSS9Y2zbBstPptO7QBSQyXbCcaAE7XGffjf2KtDMX9AwN5R1NkgHHOMao8LNSlhGImubNYdvILQkStsQx8F4CoKR3qbCmdEo7ycNqIzQ18NEr9Zv5LmQznNC7ZRl1ry5H6n5PJxjMYRbIu2DpVykw5qwfXgDbngz8NJuhu7yMVhxXM9ndVmJ_CqNcjs6en6NCbBcejeWluZuidTL_Aceg3LUZE_TBcQLJjNrVN-ogRvQJzgUOVaQxbYFmzim-TA9-KEIjt7mKdXMMMlDUhsJdTunIyyufekhg0rzS6xn8oTY6H7K0ll4fO2rUG7h3n-ABtBll-tpSNE74Qh2-356Fhc4pgeGQiliP7twHXGFX4pVINduvlzp_kD3RKDlPbgIuVer3WmCyKd0UgmluflZU4rvAE_sAf_LZu00b6XULn0PRf0uN0qUzg22-bn_JMVOQ935RW66lbNfi0mXYf03sshg1X4U2juLhXSd44OAUU246J_BF8pIBqz-5WGkq6df7gUTblMpNsaGWNfK_2HjtP_gSRKrW2e6Xmd7jvuBKoAPKws02a4EzJ9gqUynS50_WDStTRF6yZrSPMG_06gKtz9I6bELiigKeIdG_-mgw37FsEM3TgwFMh0BvQjBYTaMcr8UmhhAsmKoLN6JaGFVZ1C1VoBf7Rm76fngBDM7KtJrtjz8fQUA0xbzitufgISqj4OBa5I2NFxA==\",\"code_execution_result\":null,\"executable_code\":null,\"function_call\":null,\"function_response\":null,\"text\":\"Great ads connect with a real person by solving a real problem. Instead of just filling out a form, we're going to build your ad's story step-by-step. First, let's get to know your ideal customer.\\n\\nDescribe the single person you want to reach. What is a key problem, need, or desire they are currently facing that your business can help with?\"}],\"role\":\"model\"},\"grounding_metadata\":null,\"partial\":null,\"turn_complete\":null,\"finish_reason\":\"STOP\",\"error_code\":null,\"error_message\":null,\"interrupted\":null,\"custom_metadata\":null,\"usage_metadata\":{\"cache_tokens_details\":null,\"cached_content_token_count\":null,\"candidates_token_count\":81,\"candidates_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":81}],\"prompt_token_count\":1082,\"prompt_tokens_details\":[{\"modality\":\"TEXT\",\"token_count\":1082}],\"thoughts_token_count\":257,\"tool_use_prompt_token_count\":null,\"tool_use_prompt_tokens_details\":null,\"total_token_count\":1420,\"traffic_type\":\"ON_DEMAND\"},\"live_session_resumption_update\":null,\"input_transcription\":null,\"output_transcription\":null,\"invocation_id\":\"e-62bfd66c-238f-4bf3-abf5-b5173396c1b4\",\"author\":\"persona_ad_gen\",\"actions\":{\"skip_summarization\":null,\"state_delta\":{},\"artifact_delta\":{},\"transfer_to_agent\":null,\"escalate\":null,\"requested_auth_configs\":{}},\"long_running_tool_ids\":null,\"branch\":null,\"id\":\"78066d69-37c2-4b8a-ba9a-ca1a8b33c12a\",\"timestamp\":1757136318.473299}],\"last_update_time\":1757136318.473299},\"user_id\":\"user\"}],\"creation_timestamp\":1757136321.2961211}"
Loading