[Agent] Enable remote MCP tools for agents

Author: camilleislasse

.claude/settings.local.json

@@ -0,0 +1,5 @@
+{
+  "disabledMcpjsonServers": [
+    "openapi-generator"
+  ]
+}

PR.md

@@ -0,0 +1,152 @@
+## [Agent] Enable remote MCP tools for agents
+
+This PR adds a **proof-of-concept** MCP client implementation allowing Symfony AI agents to connect to remote MCP servers and use their tools.
+
+> ⚠️ **This is a POC** - The implementation is functional but not optimized for production use. It serves as a testing ground while waiting for the official [PHP SDK](https://github.com/modelcontextprotocol/php-sdk) to implement its client component (currently on their roadmap). Once available, this implementation should be migrated to use it.
+
+
+### How it works
+
+#### 1. Transports
+
+Three transport implementations for different MCP server types:
+
+| Transport | Use case |
+|-----------|----------|
+| `SseTransport` | Gradio/HuggingFace Spaces (Server-Sent Events) |
+| `HttpTransport` | Streamable HTTP |
+| `StdioTransport` | Local process via stdin/stdout |
+
+#### 2. McpToolbox
+
+Adapts MCP tools to Symfony AI's `ToolboxInterface`:
+- Converts MCP tool definitions to `Tool` objects
+- Executes tools via `McpClient::callTool()`
+- Converts MCP content types to Platform types:
+
+| MCP Type | Platform Type |
+|----------|---------------|
+| `TextContent` | `Text` |
+| `ImageContent` | `Image` (base64 decoded) |
+| `AudioContent` | `Audio` (base64 decoded) |
+| `EmbeddedResource` | `Text` / `Image` / `Audio` / `File` |
+
+#### 3. ChainToolbox
+
+Combines multiple `ToolboxInterface` into one, enabling agents to use tools from different sources (local + multiple MCPs).
+
+#### 4. ToolCallMessage changes
+
+`ToolCallMessage` now supports multiple content types (not just string):
+```php
+// Before
+new ToolCallMessage($toolCall, 'text result');
+
+// After
+new ToolCallMessage($toolCall, new Text('...'), new Image($data, 'image/png'));
+```
+
+This enables MCP tools to return images/audio that get displayed in chat.
+
+#### 5. Bundle Integration
+
+```yaml
+ai:
+    mcp:
+        my_mcp:
+            transport: sse  # or http, stdio
+            url: 'https://example.com/mcp/sse'
+            tools:
+                - 'tool_name'  # Optional: filter exposed tools
+```
+
+Creates services:
+- `ai.mcp.client.{name}` - The MCP client
+- `ai.mcp.toolbox.{name}` - The toolbox (use this in agent config)
+
+Use in agent:
+```yaml
+ai:
+    agent:
+        my_agent:
+            tools:
+                - 'ai.mcp.toolbox.my_mcp'
+```
+
+---
+
+###  Breaking Changes
+
+This PR introduces breaking changes to enable multimodal tool results:
+
+#### 1. `ToolCallMessage::getContent()` return type changed
+
+```php
+// Before
+public function getContent(): string
+
+// After
+public function getContent(): array // ContentInterface[]
+```
+
+**Migration**: Use `$message->asText()` to get the text content as a string.
+
+#### 2. `ToolResultConverter::convert()` return type changed
+
+```php
+// Before
+public function convert(ToolResult $toolResult): ?string
+
+// After
+public function convert(ToolResult $toolResult): array // ContentInterface[]
+```
+
+#### 3. `ToolCallMessage` constructor signature changed
+
+```php
+// Before
+new ToolCallMessage($toolCall, 'content string')
+
+// After (variadic)
+new ToolCallMessage($toolCall, 'string', $image, $audio, ...)
+new ToolCallMessage($toolCall, new Text('...'), new Image(...))
+```
+
+Passing a single string still works but internally converts to `Text` content.
+
+---
+
+### Demo
+
+`Timeline` bot combining two MCP servers:
+1. **Live City MCP** → fetches news for a city
+2. **Graphify** → generates a timeline diagram from the news
+
+User asks about a city → Agent fetches news → Agent generates timeline → Image displayed in chat.
+
+```yaml
+ai:
+    mcp:
+        graphify:
+            transport: sse
+            url: 'https://agents-mcp-hackathon-graphify.hf.space/gradio_api/mcp/sse'
+            tools:
+                - 'Graphify_generate_timeline_diagram'
+        city:
+            transport: sse
+            url: 'https://kingabzpro-live-city-mcp.hf.space/gradio_api/mcp/sse'
+            tools:
+                - 'live_city_mcp_get_city_news'
+    agent:
+        timeline:
+            platform: 'ai.platform.openai'
+            model: 'gpt-4o-mini'
+            prompt: |
+                You are a news timeline generator. When the user asks about a city:
+                1) First use live_city_mcp_get_city_news to fetch news for that city
+                2) Then use Graphify_generate_timeline_diagram with this JSON format:
+                {"title": "News from [City]", "events_per_row": 3, "events": [{"id": "1", "label": "Short title", "date": "2024-12-13"}]}
+            tools:
+                - 'ai.mcp.toolbox.graphify'
+                - 'ai.mcp.toolbox.city'
+```

demo/assets/controllers/timeline_controller.js

@@ -0,0 +1,59 @@
+import { Controller } from '@hotwired/stimulus';
+import { getComponent } from '@symfony/ux-live-component';
+
+export default class extends Controller {
+    async initialize() {
+        this.component = await getComponent(this.element);
+        this.scrollToBottom();
+
+        const input = document.getElementById('chat-message');
+        input.addEventListener('keypress', (event) => {
+            if (event.key === 'Enter') {
+                this.submitMessage();
+            }
+        });
+        input.focus();
+
+        const resetButton = document.getElementById('chat-reset');
+        resetButton.addEventListener('click', (event) => {
+            this.component.action('reset');
+        });
+
+        const submitButton = document.getElementById('chat-submit');
+        submitButton.addEventListener('click', (event) => {
+            this.submitMessage();
+        });
+
+        this.component.on('loading.state:started', (e,r) => {
+            if (r.actions.includes('reset')) {
+                return;
+            }
+            document.getElementById('welcome')?.remove();
+            document.getElementById('loading-message').removeAttribute('class');
+            this.scrollToBottom();
+        });
+
+        this.component.on('loading.state:finished', () => {
+            document.getElementById('loading-message').setAttribute('class', 'd-none');
+        });
+
+        this.component.on('render:finished', () => {
+            this.scrollToBottom();
+        });
+    };
+
+    submitMessage() {
+        const input = document.getElementById('chat-message');
+        const message = input.value;
+        document
+            .getElementById('loading-message')
+            .getElementsByClassName('user-message')[0].innerHTML = message;
+        this.component.action('submit', { message });
+        input.value = '';
+    }
+
+    scrollToBottom() {
+        const chatBody = document.getElementById('chat-body');
+        chatBody.scrollTop = chatBody.scrollHeight;
+    }
+}

demo/assets/styles/app.css

@@ -63,3 +63,7 @@ body {
         }
     }
 }
+
+.timeline .bot-message img {
+    max-width: 500px;
+}

demo/config/packages/ai.yaml

@@ -4,6 +4,17 @@ ai:
             api_key: '%env(OPENAI_API_KEY)%'
         huggingface:
             api_key: '%env(HUGGINGFACE_API_KEY)%'
+    mcp:
+        graphify:
+            transport: sse
+            url: 'https://agents-mcp-hackathon-graphify.hf.space/gradio_api/mcp/sse'
+            tools:
+                - 'Graphify_generate_timeline_diagram'
+        city:
+            transport: sse
+            url: 'https://kingabzpro-live-city-mcp.hf.space/gradio_api/mcp/sse'
+            tools:
+                - 'live_city_mcp_get_city_news'
     agent:
         blog:
             platform: 'ai.platform.openai'
@@ -75,6 +86,17 @@ ai:
             model: 'gpt-4o-mini'
             prompt: 'You are a helpful general assistant. Assist users with any questions or tasks they may have.'
             tools: false
+        timeline:
+            platform: 'ai.platform.openai'
+            model: 'gpt-4o-mini'
+            prompt: |
+                You are a news timeline generator. When the user asks about a city:
+                1) First use live_city_mcp_get_city_news to fetch news for that city
+                2) Then use Graphify_generate_timeline_diagram with this JSON format:
+                {"title": "News from [City]", "events_per_row": 3, "events": [{"id": "1", "label": "Short title", "date": "2024-12-13"}]}
+            tools:
+                - 'ai.mcp.toolbox.graphify'
+                - 'ai.mcp.toolbox.city'
     multi_agent:
         support:
             orchestrator: 'orchestrator'

demo/config/routes.yaml

@@ -63,6 +63,13 @@ youtube:
         template:  'chat.html.twig'
         context: { chat: 'youtube' }
 
+timeline:
+    path: '/timeline'
+    controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'
+    defaults:
+        template:  'chat.html.twig'
+        context: { chat: 'timeline' }
+
 # Load MCP routes conditionally based on configuration
 _mcp:
     resource: .

demo/public/.user.ini

@@ -0,0 +1 @@
+max_execution_time = 120

demo/src/Timeline/Chat.php

@@ -0,0 +1,63 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace App\Timeline;
+
+use Symfony\AI\Agent\AgentInterface;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Platform\Result\TextResult;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Component\HttpFoundation\RequestStack;
+
+/**
+ * @author Camille Islasse <[email protected]>
+ */
+final class Chat
+{
+    private const SESSION_KEY = 'timeline-chat';
+
+    public function __construct(
+        private readonly RequestStack $requestStack,
+        #[Autowire(service: 'ai.agent.timeline')]
+        private readonly AgentInterface $agent,
+    ) {
+    }
+
+    public function loadMessages(): MessageBag
+    {
+        return $this->requestStack->getSession()->get(self::SESSION_KEY, new MessageBag());
+    }
+
+    public function submitMessage(string $message): void
+    {
+        $messages = $this->loadMessages();
+
+        $messages->add(Message::ofUser($message));
+        $result = $this->agent->call($messages);
+
+        \assert($result instanceof TextResult);
+
+        $messages->add(Message::ofAssistant($result->getContent()));
+
+        $this->saveMessages($messages);
+    }
+
+    public function reset(): void
+    {
+        $this->requestStack->getSession()->remove(self::SESSION_KEY);
+    }
+
+    private function saveMessages(MessageBag $messages): void
+    {
+        $this->requestStack->getSession()->set(self::SESSION_KEY, $messages);
+    }
+}

demo/src/Timeline/TwigComponent.php

@@ -0,0 +1,52 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace App\Timeline;
+
+use Symfony\AI\Platform\Message\MessageInterface;
+use Symfony\UX\LiveComponent\Attribute\AsLiveComponent;
+use Symfony\UX\LiveComponent\Attribute\LiveAction;
+use Symfony\UX\LiveComponent\Attribute\LiveArg;
+use Symfony\UX\LiveComponent\DefaultActionTrait;
+
+/**
+ * @author Camille Islasse <[email protected]>
+ */
+#[AsLiveComponent('timeline')]
+final class TwigComponent
+{
+    use DefaultActionTrait;
+
+    public function __construct(
+        private readonly Chat $timeline,
+    ) {
+    }
+
+    /**
+     * @return MessageInterface[]
+     */
+    public function getMessages(): array
+    {
+        return $this->timeline->loadMessages()->withoutSystemMessage()->getMessages();
+    }
+
+    #[LiveAction]
+    public function submit(#[LiveArg] string $message): void
+    {
+        $this->timeline->submitMessage($message);
+    }
+
+    #[LiveAction]
+    public function reset(): void
+    {
+        $this->timeline->reset();
+    }
+}