Decouple Platform Token Usage handling from Agent by introducing new interface

Author: chr-hertel

examples/anthropic/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\Anthropic\PlatformFactory;
-use Symfony\AI\Platform\Bridge\Anthropic\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('ANTHROPIC_API_KEY'), http_client());
 
-$agent = new Agent($platform, 'claude-sonnet-4-5-20250929', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'claude-sonnet-4-5-20250929');
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),
     Message::ofUser('What is the Symfony framework?'),

examples/bootstrap.php

@@ -14,8 +14,8 @@
 use Symfony\AI\Agent\Exception\ExceptionInterface as AgentException;
 use Symfony\AI\Platform\Exception\ExceptionInterface as PlatformException;
 use Symfony\AI\Platform\Metadata\Metadata;
-use Symfony\AI\Platform\Metadata\TokenUsage;
 use Symfony\AI\Platform\Result\DeferredResult;
+use Symfony\AI\Platform\TokenUsage\TokenUsage;
 use Symfony\AI\Store\Exception\ExceptionInterface as StoreException;
 use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Logger\ConsoleLogger;

examples/deepseek/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\DeepSeek\PlatformFactory;
-use Symfony\AI\Platform\Bridge\DeepSeek\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('DEEPSEEK_API_KEY'), http_client());
 
-$agent = new Agent($platform, 'deepseek-chat', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'deepseek-chat');
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),
     Message::ofUser('What is the Symfony framework?'),

examples/gemini/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\Gemini\PlatformFactory;
-use Symfony\AI\Platform\Bridge\Gemini\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('GEMINI_API_KEY'), http_client());
 
-$agent = new Agent($platform, 'gemini-2.0-flash', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'gemini-2.0-flash');
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),
     Message::ofUser('What is the Symfony framework?'),

examples/mistral/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\Mistral\PlatformFactory;
-use Symfony\AI\Platform\Bridge\Mistral\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('MISTRAL_API_KEY'), http_client());
 
-$agent = new Agent($platform, 'mistral-large-latest', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'mistral-large-latest');
 
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),

examples/openai/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
-use Symfony\AI\Platform\Bridge\OpenAi\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 
-$agent = new Agent($platform, 'gpt-4o-mini', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'gpt-4o-mini');
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),
     Message::ofUser('What is the Symfony framework?'),

examples/perplexity/token-metadata.php

@@ -11,14 +11,13 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\Perplexity\PlatformFactory;
-use Symfony\AI\Platform\Bridge\Perplexity\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('PERPLEXITY_API_KEY'), http_client());
-$agent = new Agent($platform, 'sonar', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'sonar');
 
 $messages = new MessageBag(
     Message::forSystem('You are a pirate and you write funny.'),

examples/vertexai/token-metadata.php

@@ -11,15 +11,14 @@
 
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Platform\Bridge\VertexAi\PlatformFactory;
-use Symfony\AI\Platform\Bridge\VertexAi\TokenOutputProcessor;
 use Symfony\AI\Platform\Message\Message;
 use Symfony\AI\Platform\Message\MessageBag;
 
 require_once __DIR__.'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('GOOGLE_CLOUD_LOCATION'), env('GOOGLE_CLOUD_PROJECT'), adc_aware_http_client());
 
-$agent = new Agent($platform, 'gemini-2.0-flash-lite', outputProcessors: [new TokenOutputProcessor()]);
+$agent = new Agent($platform, 'gemini-2.0-flash-lite');
 $messages = new MessageBag(
     Message::forSystem('You are an expert assistant in animal study.'),
     Message::ofUser('What does a cat usually eat?'),

src/ai-bundle/config/services.php

@@ -28,7 +28,7 @@
 use Symfony\AI\Platform\Bridge\Albert\ModelCatalog as AlbertModelCatalog;
 use Symfony\AI\Platform\Bridge\Anthropic\Contract\AnthropicContract;
 use Symfony\AI\Platform\Bridge\Anthropic\ModelCatalog as AnthropicModelCatalog;
-use Symfony\AI\Platform\Bridge\Anthropic\TokenOutputProcessor as AnthropicTokenOutputProcessor;
+use Symfony\AI\Platform\Bridge\Anthropic\TokenUsageExtractor as AnthropicTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Azure\OpenAi\ModelCatalog as AzureOpenAiModelCatalog;
 use Symfony\AI\Platform\Bridge\Cartesia\ModelCatalog as CartesiaModelCatalog;
 use Symfony\AI\Platform\Bridge\Cerebras\ModelCatalog as CerebrasModelCatalog;
@@ -37,29 +37,29 @@
 use Symfony\AI\Platform\Bridge\DockerModelRunner\ModelCatalog as DockerModelRunnerModelCatalog;
 use Symfony\AI\Platform\Bridge\ElevenLabs\ModelCatalog as ElevenLabsModelCatalog;
 use Symfony\AI\Platform\Bridge\Gemini\Contract\GeminiContract;
+use Symfony\AI\Platform\Bridge\Gemini\Gemini\TokenUsageExtractor as GeminiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Gemini\ModelCatalog as GeminiModelCatalog;
-use Symfony\AI\Platform\Bridge\Gemini\TokenOutputProcessor as GeminiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\HuggingFace\Contract\HuggingFaceContract;
 use Symfony\AI\Platform\Bridge\HuggingFace\ModelCatalog as HuggingFaceModelCatalog;
 use Symfony\AI\Platform\Bridge\LmStudio\ModelCatalog as LmStudioModelCatalog;
 use Symfony\AI\Platform\Bridge\Meta\ModelCatalog as MetaModelCatalog;
+use Symfony\AI\Platform\Bridge\Mistral\Llm\TokenUsageExtractor as MistralTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Mistral\ModelCatalog as MistralModelCatalog;
-use Symfony\AI\Platform\Bridge\Mistral\TokenOutputProcessor as MistralTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Ollama\Contract\OllamaContract;
 use Symfony\AI\Platform\Bridge\Ollama\ModelCatalog as OllamaModelCatalog;
 use Symfony\AI\Platform\Bridge\OpenAi\Contract\OpenAiContract;
+use Symfony\AI\Platform\Bridge\OpenAi\Gpt\TokenUsageExtractor as OpenAiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\OpenAi\ModelCatalog as OpenAiModelCatalog;
-use Symfony\AI\Platform\Bridge\OpenAi\TokenOutputProcessor as OpenAiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\OpenRouter\ModelCatalog as OpenRouterModelCatalog;
 use Symfony\AI\Platform\Bridge\Perplexity\Contract\PerplexityContract;
 use Symfony\AI\Platform\Bridge\Perplexity\ModelCatalog as PerplexityModelCatalog;
 use Symfony\AI\Platform\Bridge\Perplexity\SearchResultProcessor as PerplexitySearchResultProcessor;
-use Symfony\AI\Platform\Bridge\Perplexity\TokenOutputProcessor as PerplexityTokenOutputProcessor;
+use Symfony\AI\Platform\Bridge\Perplexity\TokenUsageExtractor as PerplexityTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Replicate\ModelCatalog as ReplicateModelCatalog;
 use Symfony\AI\Platform\Bridge\Scaleway\ModelCatalog as ScalewayModelCatalog;
 use Symfony\AI\Platform\Bridge\VertexAi\Contract\GeminiContract as VertexAiGeminiContract;
+use Symfony\AI\Platform\Bridge\VertexAi\Gemini\TokenOutputProcessor as VertexAiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\VertexAi\ModelCatalog as VertexAiModelCatalog;
-use Symfony\AI\Platform\Bridge\VertexAi\TokenOutputProcessor as VertexAiTokenOutputProcessor;
 use Symfony\AI\Platform\Bridge\Voyage\ModelCatalog as VoyageModelCatalog;
 use Symfony\AI\Platform\Contract;
 use Symfony\AI\Platform\Contract\JsonSchema\DescriptionParser;

src/platform/src/Bridge/Anthropic/ResultConverter.php

@@ -77,6 +77,11 @@ public function convert(RawHttpResult|RawResultInterface $result, array $options
         return new TextResult($data['content'][0]['text']);
     }
 
+    public function getTokenUsageExtractor(): TokenUsageExtractor
+    {
+        return new TokenUsageExtractor();
+    }
+
     private function convertStream(RawResultInterface $result): \Generator
     {
         foreach ($result->getDataStream() as $data) {