From b61875df69f07169c2f6eb8383c043ec2b860aa2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 17:27:00 +0000 Subject: [PATCH 1/3] [AIC-1636] fix: Improve usage reporting Co-Authored-By: jbailey@launchdarkly.com --- packages/sdk/server-ai/src/ldai/client.py | 33 +++++++++++++++---- .../sdk/server-ai/tests/test_model_config.py | 27 +++++++++++++-- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index a4605e3..543e151 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -4,6 +4,7 @@ from ldclient import Context from ldclient.client import LDClient +from ldai import __version__ as ai_sdk_version from ldai import log from ldai.agent_graph import AgentGraphDefinition from ldai.chat import Chat @@ -17,12 +18,32 @@ from ldai.providers.ai_provider_factory import AIProviderFactory from ldai.tracker import LDAIConfigTracker +_TRACK_SDK_INFO = '$ld:ai:sdk-info' +_TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config' +_TRACK_USAGE_CREATE_CHAT = '$ld:ai:usage:create-chat' +_TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config' +_TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge' +_TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config' +_TRACK_USAGE_AGENT_CONFIGS = '$ld:ai:usage:agent-configs' + +_INIT_TRACK_CONTEXT = Context.builder('ld-internal-tracking').anonymous(True).build() + class LDAIClient: """The LaunchDarkly AI SDK client object.""" def __init__(self, client: LDClient): self._client = client + self._client.track( + _TRACK_SDK_INFO, + _INIT_TRACK_CONTEXT, + { + 'aiSdkName': 'launchdarkly-server-sdk-ai', + 'aiSdkVersion': ai_sdk_version, + 'aiSdkLanguage': 'python', + }, + 1, + ) def completion_config( self, @@ -40,7 +61,7 @@ def completion_config( :param variables: Additional variables for the completion configuration. :return: The completion configuration with a tracker used for gathering metrics. """ - self._client.track('$ld:ai:config:function:single', context, key, 1) + self._client.track(_TRACK_USAGE_COMPLETION_CONFIG, context, key, 1) model, provider, messages, instructions, tracker, enabled, judge_configuration, _ = self.__evaluate( key, context, default_value.to_dict(), variables @@ -94,7 +115,7 @@ def judge_config( :param variables: Additional variables for the judge configuration. :return: The judge configuration with a tracker used for gathering metrics. """ - self._client.track('$ld:ai:judge:function:single', context, key, 1) + self._client.track(_TRACK_USAGE_JUDGE_CONFIG, context, key, 1) model, provider, messages, instructions, tracker, enabled, judge_configuration, variation = self.__evaluate( key, context, default_value.to_dict(), variables @@ -170,7 +191,7 @@ async def create_judge( if relevance_eval: print('Relevance score:', relevance_eval.score) """ - self._client.track('$ld:ai:judge:function:createJudge', context, key, 1) + self._client.track(_TRACK_USAGE_CREATE_JUDGE, context, key, 1) try: if variables: @@ -281,7 +302,7 @@ async def create_chat( messages = chat.get_messages() print(f"Conversation has {len(messages)} messages") """ - self._client.track('$ld:ai:config:function:createChat', context, key, 1) + self._client.track(_TRACK_USAGE_CREATE_CHAT, context, key, 1) log.debug(f"Creating chat for key: {key}") config = self.completion_config(key, context, default_value, variables) @@ -340,7 +361,7 @@ def agent_config( :return: Configured AIAgentConfig instance. """ self._client.track( - "$ld:ai:agent:function:single", + _TRACK_USAGE_AGENT_CONFIG, context, key, 1 @@ -406,7 +427,7 @@ def agent_configs( """ agent_count = len(agent_configs) self._client.track( - "$ld:ai:agent:function:multiple", + _TRACK_USAGE_AGENT_CONFIGS, context, agent_count, agent_count diff --git a/packages/sdk/server-ai/tests/test_model_config.py b/packages/sdk/server-ai/tests/test_model_config.py index 26a02c9..3442308 100644 --- a/packages/sdk/server-ai/tests/test_model_config.py +++ b/packages/sdk/server-ai/tests/test_model_config.py @@ -307,7 +307,7 @@ def test_model_initial_config_enabled(ldai_client: LDAIClient): def test_config_method_tracking(ldai_client: LDAIClient): - from unittest.mock import Mock + from unittest.mock import Mock, call mock_client = Mock() mock_client.variation.return_value = { @@ -323,9 +323,30 @@ def test_config_method_tracking(ldai_client: LDAIClient): config = client.config('test-config-key', context, default_value) - mock_client.track.assert_called_once_with( - '$ld:ai:config:function:single', + mock_client.track.assert_any_call( + '$ld:ai:usage:completion-config', context, 'test-config-key', 1 ) + + +def test_sdk_info_tracked_on_init(): + from unittest.mock import Mock, call + + from ldai.client import _INIT_TRACK_CONTEXT + + mock_client = Mock() + + client = LDAIClient(mock_client) + + mock_client.track.assert_called_once_with( + '$ld:ai:sdk-info', + _INIT_TRACK_CONTEXT, + { + 'aiSdkName': 'launchdarkly-server-sdk-ai', + 'aiSdkVersion': '0.14.0', + 'aiSdkLanguage': 'python', + }, + 1, + ) From 583883f0fa1b89681d2f9b239eefb96ddecf0df3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 17:31:57 +0000 Subject: [PATCH 2/3] [AIC-1636] fix: clean up unused import and use dynamic version in test Co-Authored-By: jbailey@launchdarkly.com --- packages/sdk/server-ai/tests/test_model_config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-ai/tests/test_model_config.py b/packages/sdk/server-ai/tests/test_model_config.py index 3442308..c6e6bcb 100644 --- a/packages/sdk/server-ai/tests/test_model_config.py +++ b/packages/sdk/server-ai/tests/test_model_config.py @@ -307,7 +307,7 @@ def test_model_initial_config_enabled(ldai_client: LDAIClient): def test_config_method_tracking(ldai_client: LDAIClient): - from unittest.mock import Mock, call + from unittest.mock import Mock mock_client = Mock() mock_client.variation.return_value = { @@ -332,8 +332,9 @@ def test_config_method_tracking(ldai_client: LDAIClient): def test_sdk_info_tracked_on_init(): - from unittest.mock import Mock, call + from unittest.mock import Mock + from ldai import __version__ as ai_sdk_version from ldai.client import _INIT_TRACK_CONTEXT mock_client = Mock() @@ -345,7 +346,7 @@ def test_sdk_info_tracked_on_init(): _INIT_TRACK_CONTEXT, { 'aiSdkName': 'launchdarkly-server-sdk-ai', - 'aiSdkVersion': '0.14.0', + 'aiSdkVersion': ai_sdk_version, 'aiSdkLanguage': 'python', }, 1, From 158b1103515b9fe7a8f206b457b6794f144fc29c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 17:45:50 +0000 Subject: [PATCH 3/3] [AIC-1636] refactor: extract SDK info into dedicated sdk_info module Co-Authored-By: jbailey@launchdarkly.com --- packages/sdk/server-ai/src/ldai/client.py | 8 ++++---- packages/sdk/server-ai/src/ldai/sdk_info.py | 7 +++++++ packages/sdk/server-ai/tests/test_model_config.py | 8 ++++---- 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 packages/sdk/server-ai/src/ldai/sdk_info.py diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index 543e151..c5fda5a 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -4,7 +4,6 @@ from ldclient import Context from ldclient.client import LDClient -from ldai import __version__ as ai_sdk_version from ldai import log from ldai.agent_graph import AgentGraphDefinition from ldai.chat import Chat @@ -16,6 +15,7 @@ JudgeConfiguration, LDMessage, ModelConfig, ProviderConfig) from ldai.providers.ai_provider_factory import AIProviderFactory +from ldai.sdk_info import AI_SDK_LANGUAGE, AI_SDK_NAME, AI_SDK_VERSION from ldai.tracker import LDAIConfigTracker _TRACK_SDK_INFO = '$ld:ai:sdk-info' @@ -38,9 +38,9 @@ def __init__(self, client: LDClient): _TRACK_SDK_INFO, _INIT_TRACK_CONTEXT, { - 'aiSdkName': 'launchdarkly-server-sdk-ai', - 'aiSdkVersion': ai_sdk_version, - 'aiSdkLanguage': 'python', + 'aiSdkName': AI_SDK_NAME, + 'aiSdkVersion': AI_SDK_VERSION, + 'aiSdkLanguage': AI_SDK_LANGUAGE, }, 1, ) diff --git a/packages/sdk/server-ai/src/ldai/sdk_info.py b/packages/sdk/server-ai/src/ldai/sdk_info.py new file mode 100644 index 0000000..743f729 --- /dev/null +++ b/packages/sdk/server-ai/src/ldai/sdk_info.py @@ -0,0 +1,7 @@ +from importlib.metadata import metadata + +_meta = metadata('launchdarkly-server-sdk-ai') + +AI_SDK_NAME: str = _meta['Name'] +AI_SDK_VERSION: str = _meta['Version'] +AI_SDK_LANGUAGE: str = 'python' diff --git a/packages/sdk/server-ai/tests/test_model_config.py b/packages/sdk/server-ai/tests/test_model_config.py index c6e6bcb..d0ae24d 100644 --- a/packages/sdk/server-ai/tests/test_model_config.py +++ b/packages/sdk/server-ai/tests/test_model_config.py @@ -334,8 +334,8 @@ def test_config_method_tracking(ldai_client: LDAIClient): def test_sdk_info_tracked_on_init(): from unittest.mock import Mock - from ldai import __version__ as ai_sdk_version from ldai.client import _INIT_TRACK_CONTEXT + from ldai.sdk_info import AI_SDK_LANGUAGE, AI_SDK_NAME, AI_SDK_VERSION mock_client = Mock() @@ -345,9 +345,9 @@ def test_sdk_info_tracked_on_init(): '$ld:ai:sdk-info', _INIT_TRACK_CONTEXT, { - 'aiSdkName': 'launchdarkly-server-sdk-ai', - 'aiSdkVersion': ai_sdk_version, - 'aiSdkLanguage': 'python', + 'aiSdkName': AI_SDK_NAME, + 'aiSdkVersion': AI_SDK_VERSION, + 'aiSdkLanguage': AI_SDK_LANGUAGE, }, 1, )