From d47de6ddc8c90ef98ea1b574cc456e8484232970 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 01:25:21 +0000 Subject: [PATCH 1/3] feat: add tools with instructions and examples to agent config types Co-Authored-By: Paul Loeb --- packages/sdk/server-ai/src/ldai/__init__.py | 5 +-- packages/sdk/server-ai/src/ldai/client.py | 20 +++++++++-- packages/sdk/server-ai/src/ldai/models.py | 38 +++++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/__init__.py b/packages/sdk/server-ai/src/ldai/__init__.py index 2868b57..24d597f 100644 --- a/packages/sdk/server-ai/src/ldai/__init__.py +++ b/packages/sdk/server-ai/src/ldai/__init__.py @@ -10,8 +10,8 @@ AIAgentConfig, AIAgentConfigDefault, AIAgentConfigRequest, AIAgentGraphConfig, AIAgents, AICompletionConfig, AICompletionConfigDefault, AIConfig, AIJudgeConfig, AIJudgeConfigDefault, - Edge, JudgeConfiguration, LDAIAgent, LDAIAgentConfig, LDAIAgentDefaults, - LDMessage, ModelConfig, ProviderConfig) + AITool, Edge, JudgeConfiguration, LDAIAgent, LDAIAgentConfig, + LDAIAgentDefaults, LDMessage, ModelConfig, ProviderConfig) from ldai.providers.types import EvalScore, JudgeResponse from ldai.tracker import AIGraphTracker @@ -23,6 +23,7 @@ 'AIAgents', 'AIAgentGraphConfig', 'AIGraphTracker', + 'AITool', 'Edge', 'AICompletionConfig', 'AICompletionConfigDefault', diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index 8289d06..251b9c5 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -11,7 +11,7 @@ from ldai.models import (AIAgentConfig, AIAgentConfigDefault, AIAgentConfigRequest, AIAgentGraphConfig, AIAgents, AICompletionConfig, AICompletionConfigDefault, - AIJudgeConfig, AIJudgeConfigDefault, Edge, + AIJudgeConfig, AIJudgeConfigDefault, AITool, Edge, JudgeConfiguration, LDMessage, ModelConfig, ProviderConfig) from ldai.providers.ai_provider_factory import AIProviderFactory @@ -706,19 +706,35 @@ def __evaluate_agent( :param variables: Variables for interpolation. :return: Configured AIAgentConfig instance. """ - model, provider, messages, instructions, tracker, enabled, judge_configuration, _ = self.__evaluate( + model, provider, messages, instructions, tracker, enabled, judge_configuration, variation = self.__evaluate( key, context, default.to_dict(), variables ) # For agents, prioritize instructions over messages final_instructions = instructions if instructions is not None else default.instructions + # Parse tools from variation data + tools = None + if 'tools' in variation and isinstance(variation['tools'], list): + tools = [ + AITool( + key=tool['key'], + version=tool.get('version', 0), + instructions=tool.get('instructions'), + examples=tool.get('examples'), + custom_parameters=tool.get('customParameters'), + ) + for tool in variation['tools'] + if isinstance(tool, dict) and 'key' in tool + ] or None + return AIAgentConfig( key=key, enabled=bool(enabled) if enabled is not None else (default.enabled or False), model=model or default.model, provider=provider or default.provider, instructions=final_instructions, + tools=tools if tools is not None else (default.tools if default.tools else None), tracker=tracker, judge_configuration=judge_configuration or default.judge_configuration, ) diff --git a/packages/sdk/server-ai/src/ldai/models.py b/packages/sdk/server-ai/src/ldai/models.py index 07b02c2..3040a22 100644 --- a/packages/sdk/server-ai/src/ldai/models.py +++ b/packages/sdk/server-ai/src/ldai/models.py @@ -139,6 +139,38 @@ def to_dict(self) -> dict: } +# ============================================================================ +# Tool Types +# ============================================================================ + +@dataclass(frozen=True) +class AITool: + """ + Configuration for an AI tool. + """ + key: str + version: int + instructions: Optional[str] = None + examples: Optional[str] = None + custom_parameters: Optional[Dict[str, Any]] = None + + def to_dict(self) -> dict: + """ + Render the tool as a dictionary object. + """ + result: Dict[str, Any] = { + 'key': self.key, + 'version': self.version, + } + if self.instructions is not None: + result['instructions'] = self.instructions + if self.examples is not None: + result['examples'] = self.examples + if self.custom_parameters is not None: + result['customParameters'] = self.custom_parameters + return result + + # ============================================================================ # Base AI Config Types # ============================================================================ @@ -249,6 +281,7 @@ class AIAgentConfigDefault(AIConfigDefault): Default Agent-specific AI Config with instructions. """ instructions: Optional[str] = None + tools: Optional[List[AITool]] = None judge_configuration: Optional[JudgeConfiguration] = None def to_dict(self) -> Dict[str, Any]: @@ -258,6 +291,8 @@ def to_dict(self) -> Dict[str, Any]: result = self._base_to_dict() if self.instructions is not None: result['instructions'] = self.instructions + if self.tools is not None: + result['tools'] = [tool.to_dict() for tool in self.tools] if self.judge_configuration is not None: result['judgeConfiguration'] = self.judge_configuration.to_dict() return result @@ -269,6 +304,7 @@ class AIAgentConfig(AIConfig): Agent-specific AI Config with instructions. """ instructions: Optional[str] = None + tools: Optional[List[AITool]] = None judge_configuration: Optional[JudgeConfiguration] = None def to_dict(self) -> Dict[str, Any]: @@ -278,6 +314,8 @@ def to_dict(self) -> Dict[str, Any]: result = self._base_to_dict() if self.instructions is not None: result['instructions'] = self.instructions + if self.tools is not None: + result['tools'] = [tool.to_dict() for tool in self.tools] if self.judge_configuration is not None: result['judgeConfiguration'] = self.judge_configuration.to_dict() return result From c01300ca28a2d32b1d8d0a82a3a1f51f0a337055 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 01:50:05 +0000 Subject: [PATCH 2/3] feat: add tools to completion configs with test coverage Co-Authored-By: Paul Loeb --- packages/sdk/server-ai/src/ldai/client.py | 18 +- packages/sdk/server-ai/src/ldai/models.py | 6 + packages/sdk/server-ai/tests/test_tools.py | 372 +++++++++++++++++++++ 3 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 packages/sdk/server-ai/tests/test_tools.py diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index 251b9c5..8b41ce7 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -52,15 +52,31 @@ def _completion_config( default: AICompletionConfigDefault, variables: Optional[Dict[str, Any]] = None, ) -> AICompletionConfig: - model, provider, messages, instructions, tracker, enabled, judge_configuration, _ = self.__evaluate( + model, provider, messages, instructions, tracker, enabled, judge_configuration, variation = self.__evaluate( key, context, default.to_dict(), variables ) + # Parse tools from variation data + tools = None + if 'tools' in variation and isinstance(variation['tools'], list): + tools = [ + AITool( + key=tool['key'], + version=tool.get('version', 0), + instructions=tool.get('instructions'), + examples=tool.get('examples'), + custom_parameters=tool.get('customParameters'), + ) + for tool in variation['tools'] + if isinstance(tool, dict) and 'key' in tool + ] or None + config = AICompletionConfig( key=key, enabled=bool(enabled), model=model, messages=messages, + tools=tools if tools is not None else (default.tools if default.tools else None), provider=provider, tracker=tracker, judge_configuration=judge_configuration, diff --git a/packages/sdk/server-ai/src/ldai/models.py b/packages/sdk/server-ai/src/ldai/models.py index 3040a22..40d91be 100644 --- a/packages/sdk/server-ai/src/ldai/models.py +++ b/packages/sdk/server-ai/src/ldai/models.py @@ -239,6 +239,7 @@ class AICompletionConfigDefault(AIConfigDefault): Default Completion AI Config (default mode). """ messages: Optional[List[LDMessage]] = None + tools: Optional[List[AITool]] = None judge_configuration: Optional[JudgeConfiguration] = None def to_dict(self) -> dict: @@ -247,6 +248,8 @@ def to_dict(self) -> dict: """ result = self._base_to_dict() result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + if self.tools is not None: + result['tools'] = [tool.to_dict() for tool in self.tools] if self.judge_configuration is not None: result['judgeConfiguration'] = self.judge_configuration.to_dict() return result @@ -258,6 +261,7 @@ class AICompletionConfig(AIConfig): Completion AI Config (default mode). """ messages: Optional[List[LDMessage]] = None + tools: Optional[List[AITool]] = None judge_configuration: Optional[JudgeConfiguration] = None def to_dict(self) -> dict: @@ -266,6 +270,8 @@ def to_dict(self) -> dict: """ result = self._base_to_dict() result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None + if self.tools is not None: + result['tools'] = [tool.to_dict() for tool in self.tools] if self.judge_configuration is not None: result['judgeConfiguration'] = self.judge_configuration.to_dict() return result diff --git a/packages/sdk/server-ai/tests/test_tools.py b/packages/sdk/server-ai/tests/test_tools.py new file mode 100644 index 0000000..6740594 --- /dev/null +++ b/packages/sdk/server-ai/tests/test_tools.py @@ -0,0 +1,372 @@ +import pytest +from ldclient import Config, Context, LDClient +from ldclient.integrations.test_data import TestData + +from ldai import LDAIClient, ModelConfig +from ldai.models import (AIAgentConfigDefault, AICompletionConfigDefault, + AITool) + + +@pytest.fixture +def td() -> TestData: + td = TestData.data_source() + + # Completion config with tools + td.update( + td.flag('completion-with-tools') + .variations( + { + 'model': {'name': 'gpt-4', 'parameters': {'temperature': 0.5}}, + 'provider': {'name': 'openai'}, + 'messages': [{'role': 'system', 'content': 'You are a helpful assistant.'}], + 'tools': [ + { + 'key': 'get-customer', + 'version': 1, + 'instructions': 'Use this tool to look up customer details by email.', + 'examples': 'Input: {"email": "jane@example.com"}\nOutput: {"name": "Jane Doe"}', + 'customParameters': {'endpoint': '/api/customers'}, + }, + { + 'key': 'search-orders', + 'version': 2, + 'instructions': 'Search for orders by customer ID.', + }, + ], + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + # Agent config with tools + td.update( + td.flag('agent-with-tools') + .variations( + { + 'model': {'name': 'gpt-4', 'parameters': {'temperature': 0.3}}, + 'provider': {'name': 'openai'}, + 'instructions': 'You are a customer support agent for {{company_name}}.', + 'tools': [ + { + 'key': 'crm-lookup', + 'version': 1, + 'instructions': 'Look up customer info in the CRM.', + 'examples': 'Input: {"id": "123"}\nOutput: {"name": "John", "plan": "Enterprise"}', + 'customParameters': {'timeout': 30}, + }, + ], + '_ldMeta': {'enabled': True, 'variationKey': 'agent-v1', 'version': 1, 'mode': 'agent'}, + } + ) + .variation_for_all(0) + ) + + # Config with no tools + td.update( + td.flag('no-tools-config') + .variations( + { + 'model': {'name': 'gpt-4'}, + 'messages': [{'role': 'system', 'content': 'Hello'}], + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + # Config with empty tools array + td.update( + td.flag('empty-tools-config') + .variations( + { + 'model': {'name': 'gpt-4'}, + 'messages': [{'role': 'system', 'content': 'Hello'}], + 'tools': [], + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + # Agent config with tools that have minimal fields + td.update( + td.flag('agent-minimal-tools') + .variations( + { + 'instructions': 'Minimal agent.', + 'tools': [ + {'key': 'basic-tool', 'version': 1}, + ], + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1, 'mode': 'agent'}, + } + ) + .variation_for_all(0) + ) + + return td + + +@pytest.fixture +def client(td: TestData) -> LDClient: + config = Config('sdk-key', update_processor_class=td, send_events=False) + return LDClient(config=config) + + +@pytest.fixture +def ldai_client(client: LDClient) -> LDAIClient: + return LDAIClient(client) + + +# ============================================================================ +# AITool dataclass tests +# ============================================================================ + +def test_ai_tool_to_dict_full(): + """Test AITool.to_dict() with all fields populated.""" + tool = AITool( + key='get-customer', + version=1, + instructions='Look up customer by email.', + examples='Input: {"email": "a@b.com"}\nOutput: {"name": "Alice"}', + custom_parameters={'endpoint': '/api/customers'}, + ) + + result = tool.to_dict() + + assert result == { + 'key': 'get-customer', + 'version': 1, + 'instructions': 'Look up customer by email.', + 'examples': 'Input: {"email": "a@b.com"}\nOutput: {"name": "Alice"}', + 'customParameters': {'endpoint': '/api/customers'}, + } + + +def test_ai_tool_to_dict_minimal(): + """Test AITool.to_dict() with only required fields.""" + tool = AITool(key='basic-tool', version=1) + + result = tool.to_dict() + + assert result == { + 'key': 'basic-tool', + 'version': 1, + } + assert 'instructions' not in result + assert 'examples' not in result + assert 'customParameters' not in result + + +def test_ai_tool_is_frozen(): + """Test that AITool is immutable.""" + tool = AITool(key='test', version=1) + with pytest.raises(AttributeError): + tool.key = 'changed' # type: ignore[misc] + + +# ============================================================================ +# Completion config with tools tests +# ============================================================================ + +def test_completion_config_with_tools(ldai_client: LDAIClient): + """Test that completion config correctly parses tools from variation.""" + context = Context.create('user-key') + default = AICompletionConfigDefault(enabled=False, model=ModelConfig('fallback')) + + config = ldai_client.completion_config('completion-with-tools', context, default) + + assert config.enabled is True + assert config.tools is not None + assert len(config.tools) == 2 + + tool1 = config.tools[0] + assert tool1.key == 'get-customer' + assert tool1.version == 1 + assert tool1.instructions == 'Use this tool to look up customer details by email.' + assert tool1.examples == 'Input: {"email": "jane@example.com"}\nOutput: {"name": "Jane Doe"}' + assert tool1.custom_parameters == {'endpoint': '/api/customers'} + + tool2 = config.tools[1] + assert tool2.key == 'search-orders' + assert tool2.version == 2 + assert tool2.instructions == 'Search for orders by customer ID.' + assert tool2.examples is None + assert tool2.custom_parameters is None + + +def test_completion_config_without_tools(ldai_client: LDAIClient): + """Test that completion config has no tools when variation has none.""" + context = Context.create('user-key') + default = AICompletionConfigDefault(enabled=False, model=ModelConfig('fallback')) + + config = ldai_client.completion_config('no-tools-config', context, default) + + assert config.enabled is True + assert config.tools is None + + +def test_completion_config_empty_tools(ldai_client: LDAIClient): + """Test that completion config handles empty tools array.""" + context = Context.create('user-key') + default = AICompletionConfigDefault(enabled=False, model=ModelConfig('fallback')) + + config = ldai_client.completion_config('empty-tools-config', context, default) + + assert config.enabled is True + assert config.tools is None + + +def test_completion_config_uses_default_tools(ldai_client: LDAIClient): + """Test that completion config falls back to default tools when variation has none.""" + context = Context.create('user-key') + default_tools = [AITool(key='default-tool', version=1, instructions='Default tool')] + default = AICompletionConfigDefault( + enabled=True, + model=ModelConfig('fallback'), + tools=default_tools, + ) + + config = ldai_client.completion_config('no-tools-config', context, default) + + assert config.tools is not None + assert len(config.tools) == 1 + assert config.tools[0].key == 'default-tool' + + +def test_completion_config_default_to_dict_with_tools(): + """Test AICompletionConfigDefault.to_dict() includes tools.""" + tools = [ + AITool(key='tool-a', version=1, instructions='Do A'), + AITool(key='tool-b', version=2), + ] + default = AICompletionConfigDefault( + enabled=True, + model=ModelConfig('gpt-4'), + tools=tools, + ) + + result = default.to_dict() + + assert 'tools' in result + assert len(result['tools']) == 2 + assert result['tools'][0]['key'] == 'tool-a' + assert result['tools'][0]['instructions'] == 'Do A' + assert result['tools'][1]['key'] == 'tool-b' + + +def test_completion_config_to_dict_with_tools(): + """Test AICompletionConfig.to_dict() includes tools.""" + from ldai.models import AICompletionConfig + + tools = [AITool(key='my-tool', version=3, examples='example text')] + config = AICompletionConfig( + key='test', + enabled=True, + tools=tools, + ) + + result = config.to_dict() + + assert 'tools' in result + assert len(result['tools']) == 1 + assert result['tools'][0]['key'] == 'my-tool' + assert result['tools'][0]['version'] == 3 + assert result['tools'][0]['examples'] == 'example text' + + +# ============================================================================ +# Agent config with tools tests +# ============================================================================ + +def test_agent_config_with_tools(ldai_client: LDAIClient): + """Test that agent config correctly parses tools from variation.""" + context = Context.create('user-key') + default = AIAgentConfigDefault(enabled=False, model=ModelConfig('fallback')) + + agent = ldai_client.agent_config( + 'agent-with-tools', context, default, {'company_name': 'Acme Corp'} + ) + + assert agent.enabled is True + assert agent.instructions == 'You are a customer support agent for Acme Corp.' + assert agent.tools is not None + assert len(agent.tools) == 1 + + tool = agent.tools[0] + assert tool.key == 'crm-lookup' + assert tool.version == 1 + assert tool.instructions == 'Look up customer info in the CRM.' + assert tool.examples == 'Input: {"id": "123"}\nOutput: {"name": "John", "plan": "Enterprise"}' + assert tool.custom_parameters == {'timeout': 30} + + +def test_agent_config_minimal_tools(ldai_client: LDAIClient): + """Test agent config with tools that have only required fields.""" + context = Context.create('user-key') + default = AIAgentConfigDefault(enabled=False) + + agent = ldai_client.agent_config('agent-minimal-tools', context, default) + + assert agent.enabled is True + assert agent.tools is not None + assert len(agent.tools) == 1 + assert agent.tools[0].key == 'basic-tool' + assert agent.tools[0].version == 1 + assert agent.tools[0].instructions is None + assert agent.tools[0].examples is None + assert agent.tools[0].custom_parameters is None + + +def test_agent_config_uses_default_tools(ldai_client: LDAIClient): + """Test that agent config falls back to default tools when variation has none.""" + context = Context.create('user-key') + default_tools = [AITool(key='default-agent-tool', version=1)] + default = AIAgentConfigDefault( + enabled=True, + model=ModelConfig('fallback'), + tools=default_tools, + instructions='Default instructions', + ) + + agent = ldai_client.agent_config('non-existent-agent', context, default) + + assert agent.tools is not None + assert len(agent.tools) == 1 + assert agent.tools[0].key == 'default-agent-tool' + + +def test_agent_config_default_to_dict_with_tools(): + """Test AIAgentConfigDefault.to_dict() includes tools.""" + tools = [AITool(key='agent-tool', version=1, instructions='Do something')] + default = AIAgentConfigDefault( + enabled=True, + instructions='Be helpful.', + tools=tools, + ) + + result = default.to_dict() + + assert 'tools' in result + assert len(result['tools']) == 1 + assert result['tools'][0]['key'] == 'agent-tool' + + +def test_agent_config_to_dict_with_tools(): + """Test AIAgentConfig.to_dict() includes tools.""" + from ldai.models import AIAgentConfig + + tools = [AITool(key='my-tool', version=2)] + config = AIAgentConfig( + key='test', + enabled=True, + instructions='Test instructions', + tools=tools, + ) + + result = config.to_dict() + + assert 'tools' in result + assert len(result['tools']) == 1 + assert result['tools'][0]['key'] == 'my-tool' + assert result['tools'][0]['version'] == 2 From 0996e4ecb0cd544d1d91f41f37676a5063b60cf5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 02:02:17 +0000 Subject: [PATCH 3/3] fix: preserve empty tools array from server instead of coercing to None Co-Authored-By: Paul Loeb --- packages/sdk/server-ai/src/ldai/client.py | 8 ++++---- packages/sdk/server-ai/tests/test_tools.py | 24 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index 8b41ce7..ec075c8 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -69,14 +69,14 @@ def _completion_config( ) for tool in variation['tools'] if isinstance(tool, dict) and 'key' in tool - ] or None + ] config = AICompletionConfig( key=key, enabled=bool(enabled), model=model, messages=messages, - tools=tools if tools is not None else (default.tools if default.tools else None), + tools=tools if tools is not None else default.tools, provider=provider, tracker=tracker, judge_configuration=judge_configuration, @@ -742,7 +742,7 @@ def __evaluate_agent( ) for tool in variation['tools'] if isinstance(tool, dict) and 'key' in tool - ] or None + ] return AIAgentConfig( key=key, @@ -750,7 +750,7 @@ def __evaluate_agent( model=model or default.model, provider=provider or default.provider, instructions=final_instructions, - tools=tools if tools is not None else (default.tools if default.tools else None), + tools=tools if tools is not None else default.tools, tracker=tracker, judge_configuration=judge_configuration or default.judge_configuration, ) diff --git a/packages/sdk/server-ai/tests/test_tools.py b/packages/sdk/server-ai/tests/test_tools.py index 6740594..272ea77 100644 --- a/packages/sdk/server-ai/tests/test_tools.py +++ b/packages/sdk/server-ai/tests/test_tools.py @@ -207,14 +207,34 @@ def test_completion_config_without_tools(ldai_client: LDAIClient): def test_completion_config_empty_tools(ldai_client: LDAIClient): - """Test that completion config handles empty tools array.""" + """Test that completion config preserves empty tools array from server.""" context = Context.create('user-key') default = AICompletionConfigDefault(enabled=False, model=ModelConfig('fallback')) config = ldai_client.completion_config('empty-tools-config', context, default) assert config.enabled is True - assert config.tools is None + # An explicit empty tools array from server should be preserved as empty list, + # NOT fall back to defaults + assert config.tools is not None + assert config.tools == [] + + +def test_completion_config_empty_tools_no_default_fallback(ldai_client: LDAIClient): + """Test that empty tools array from server does NOT fall back to defaults.""" + context = Context.create('user-key') + default_tools = [AITool(key='default-tool', version=1)] + default = AICompletionConfigDefault( + enabled=False, + model=ModelConfig('fallback'), + tools=default_tools, + ) + + config = ldai_client.completion_config('empty-tools-config', context, default) + + assert config.enabled is True + # Server sent empty tools=[], so we should honor that, not use defaults + assert config.tools == [] def test_completion_config_uses_default_tools(ldai_client: LDAIClient):