diff --git a/.size-limit.js b/.size-limit.js index bc3da7fd7eb0..1983b738cd3c 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -148,7 +148,7 @@ module.exports = [ import: createImport('init', 'ErrorBoundary', 'reactRouterV6BrowserTracingIntegration'), ignore: ['react/jsx-runtime'], gzip: true, - limit: '44.5 KB', + limit: '44.6 KB', }, // Vue SDK (ESM) { diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-message-truncation.mjs new file mode 100644 index 000000000000..6c3f7327e64b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-message-truncation.mjs @@ -0,0 +1,51 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const largeContent1 = 'A'.repeat(15000); // ~15KB + const largeContent2 = 'B'.repeat(15000); // ~15KB + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated) + + // Test 1: Messages array with large last message that gets truncated + // Only the last message should be kept, and it should be truncated to only Cs + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 5 }, + text: 'Response to truncated messages', + }), + }), + messages: [ + { role: 'user', content: largeContent1 }, + { role: 'assistant', content: largeContent2 }, + { role: 'user', content: largeContent3 }, + ], + }); + + // Test 2: Messages array where last message is small and kept intact + const smallContent = 'This is a small message that fits within the limit'; + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 5 }, + text: 'Response to small message', + }), + }), + messages: [ + { role: 'user', content: largeContent1 }, + { role: 'assistant', content: largeContent2 }, + { role: 'user', content: smallContent }, + ], + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index a98e7b97e919..0f1efb26d1f0 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -5,7 +5,6 @@ import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, - GEN_AI_PROMPT_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, @@ -90,7 +89,6 @@ describe('Vercel AI integration', () => { // Third span - explicit telemetry enabled, should record inputs/outputs regardless of sendDefaultPii expect.objectContaining({ data: { - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', @@ -105,7 +103,7 @@ describe('Vercel AI integration', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.response.finishReason': 'stop', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.settings.maxSteps': 1, @@ -230,7 +228,6 @@ describe('Vercel AI integration', () => { // First span - no telemetry config, should enable telemetry AND record inputs/outputs when sendDefaultPii: true expect.objectContaining({ data: { - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the first span?"}', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', @@ -245,7 +242,7 @@ describe('Vercel AI integration', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the first span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', 'vercel.ai.response.finishReason': 'stop', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.settings.maxSteps': 1, @@ -303,7 +300,6 @@ describe('Vercel AI integration', () => { // Third span - explicitly enabled telemetry, should record inputs/outputs regardless of sendDefaultPii expect.objectContaining({ data: { - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', @@ -318,7 +314,7 @@ describe('Vercel AI integration', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.response.finishReason': 'stop', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.settings.maxSteps': 1, @@ -375,7 +371,6 @@ describe('Vercel AI integration', () => { // Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true) expect.objectContaining({ data: { - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"What is the weather in San Francisco?"}', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', @@ -391,7 +386,7 @@ describe('Vercel AI integration', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"What is the weather in San Francisco?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', 'vercel.ai.response.finishReason': 'tool-calls', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.settings.maxSteps': 1, @@ -796,4 +791,43 @@ describe('Vercel AI integration', () => { }); }, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-message-truncation.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('truncates messages when they exceed byte limit', async () => { + await createRunner() + .ignore('event') + .expect({ + transaction: { + transaction: 'main', + spans: expect.arrayContaining([ + // First call: Last message truncated (only C's remain, D's are cropped) + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[.*"(?:text|content)":"C+".*\]$/), + [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Response to truncated messages', + }), + }), + // Second call: Last message is small and kept intact + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining( + 'This is a small message that fits within the limit', + ), + [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Response to small message', + }), + }), + ]), + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 332f84777264..eb42156920e9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -5,7 +5,6 @@ import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, - GEN_AI_PROMPT_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, @@ -92,12 +91,11 @@ describe('Vercel AI integration (V5)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', @@ -229,14 +227,13 @@ describe('Vercel AI integration (V5)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the first span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'First span here!', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the first span?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, @@ -290,14 +287,13 @@ describe('Vercel AI integration (V5)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, @@ -350,14 +346,13 @@ describe('Vercel AI integration (V5)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"What is the weather in San Francisco?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', 'vercel.ai.response.finishReason': 'tool-calls', [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"What is the weather in San Francisco?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts index f779eebdf0e3..2a75cfdfbfca 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts @@ -4,7 +4,6 @@ import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, - GEN_AI_PROMPT_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, @@ -93,13 +92,12 @@ describe('Vercel AI integration (V6)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.request.headers.user-agent': expect.any(String), 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, @@ -232,14 +230,13 @@ describe('Vercel AI integration (V6)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the first span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', 'vercel.ai.request.headers.user-agent': expect.any(String), [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'First span here!', 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the first span?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, @@ -293,14 +290,13 @@ describe('Vercel AI integration (V6)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.request.headers.user-agent': expect.any(String), [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', 'vercel.ai.response.finishReason': 'stop', [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"Where is the second span?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, @@ -353,14 +349,13 @@ describe('Vercel AI integration (V6)', () => { 'vercel.ai.model.provider': 'mock-provider', 'vercel.ai.operationId': 'ai.generateText', 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '{"prompt":"What is the weather in San Francisco?"}', + 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', 'vercel.ai.request.headers.user-agent': expect.any(String), [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', 'vercel.ai.response.finishReason': 'tool-calls', [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), 'vercel.ai.settings.maxRetries': 2, 'vercel.ai.streaming': false, - [GEN_AI_PROMPT_ATTRIBUTE]: '{"prompt":"What is the weather in San Francisco?"}', [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, diff --git a/packages/core/src/tracing/vercel-ai/utils.ts b/packages/core/src/tracing/vercel-ai/utils.ts index 2a0878f1e591..58a5653015d4 100644 --- a/packages/core/src/tracing/vercel-ai/utils.ts +++ b/packages/core/src/tracing/vercel-ai/utils.ts @@ -105,22 +105,61 @@ export function convertAvailableToolsToJsonString(tools: unknown[]): string { } /** - * Convert the prompt string to messages array + * Filter out invalid entries in messages array + * @param input - The input array to filter + * @returns The filtered array */ -export function convertPromptToMessages(prompt: string): { role: string; content: string }[] { +function filterMessagesArray(input: unknown[]): { role: string; content: string }[] { + return input.filter( + (m: unknown): m is { role: string; content: string } => + !!m && typeof m === 'object' && 'role' in m && 'content' in m, + ); +} + +/** + * Normalize the user input (stringified object with prompt, system, messages) to messages array + */ +export function convertUserInputToMessagesFormat(userInput: string): { role: string; content: string }[] { try { - const p = JSON.parse(prompt); + const p = JSON.parse(userInput); if (!!p && typeof p === 'object') { + let { messages } = p; const { prompt, system } = p; - if (typeof prompt === 'string' || typeof system === 'string') { - const messages: { role: string; content: string }[] = []; - if (typeof system === 'string') { - messages.push({ role: 'system', content: system }); - } - if (typeof prompt === 'string') { - messages.push({ role: 'user', content: prompt }); + const result: { role: string; content: string }[] = []; + + // prepend top-level system instruction if present + if (typeof system === 'string') { + result.push({ role: 'system', content: system }); + } + + // stringified messages array + if (typeof messages === 'string') { + try { + messages = JSON.parse(messages); + } catch { + // ignore parse errors } - return messages; + } + + // messages array format: { messages: [...] } + if (Array.isArray(messages)) { + result.push(...filterMessagesArray(messages)); + return result; + } + + // prompt array format: { prompt: [...] } + if (Array.isArray(prompt)) { + result.push(...filterMessagesArray(prompt)); + return result; + } + + // prompt string format: { prompt: "..." } + if (typeof prompt === 'string') { + result.push({ role: 'user', content: prompt }); + } + + if (result.length > 0) { + return result; } } // eslint-disable-next-line no-empty @@ -133,17 +172,17 @@ export function convertPromptToMessages(prompt: string): { role: string; content * invoke_agent op */ export function requestMessagesFromPrompt(span: Span, attributes: SpanAttributes): void { - if (attributes[AI_PROMPT_ATTRIBUTE]) { - const truncatedPrompt = getTruncatedJsonString(attributes[AI_PROMPT_ATTRIBUTE] as string | string[]); - span.setAttribute('gen_ai.prompt', truncatedPrompt); - } - const prompt = attributes[AI_PROMPT_ATTRIBUTE]; if ( - typeof prompt === 'string' && + typeof attributes[AI_PROMPT_ATTRIBUTE] === 'string' && !attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] && !attributes[AI_PROMPT_MESSAGES_ATTRIBUTE] ) { - const messages = convertPromptToMessages(prompt); + // No messages array is present, so we need to convert the prompt to the proper messages format + // This is the case for ai.generateText spans + // The ai.prompt attribute is a stringified object with prompt, system, messages attributes + // The format of these is described in the vercel docs, for instance: https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-object#parameters + const userInput = attributes[AI_PROMPT_ATTRIBUTE]; + const messages = convertUserInputToMessagesFormat(userInput); if (messages.length) { const { systemInstructions, filteredMessages } = extractSystemInstructions(messages); @@ -152,12 +191,17 @@ export function requestMessagesFromPrompt(span: Span, attributes: SpanAttributes } const filteredLength = Array.isArray(filteredMessages) ? filteredMessages.length : 0; + const truncatedMessages = getTruncatedJsonString(filteredMessages); + span.setAttributes({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: getTruncatedJsonString(filteredMessages), + [AI_PROMPT_ATTRIBUTE]: truncatedMessages, + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: truncatedMessages, [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: filteredLength, }); } } else if (typeof attributes[AI_PROMPT_MESSAGES_ATTRIBUTE] === 'string') { + // In this case we already get a properly formatted messages array, this is the preferred way to get the messages + // This is the case for ai.generateText.doGenerate spans try { const messages = JSON.parse(attributes[AI_PROMPT_MESSAGES_ATTRIBUTE]); if (Array.isArray(messages)) { @@ -168,9 +212,11 @@ export function requestMessagesFromPrompt(span: Span, attributes: SpanAttributes } const filteredLength = Array.isArray(filteredMessages) ? filteredMessages.length : 0; + const truncatedMessages = getTruncatedJsonString(filteredMessages); + span.setAttributes({ - [AI_PROMPT_MESSAGES_ATTRIBUTE]: undefined, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: getTruncatedJsonString(filteredMessages), + [AI_PROMPT_MESSAGES_ATTRIBUTE]: truncatedMessages, + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: truncatedMessages, [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: filteredLength, }); } diff --git a/packages/core/test/lib/utils/vercelai-utils.test.ts b/packages/core/test/lib/utils/vercelai-utils.test.ts index be329e6f5970..d30c3602ab1c 100644 --- a/packages/core/test/lib/utils/vercelai-utils.test.ts +++ b/packages/core/test/lib/utils/vercelai-utils.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { convertPromptToMessages } from '../../../src/tracing/vercel-ai/utils'; +import { convertUserInputToMessagesFormat } from '../../../src/tracing/vercel-ai/utils'; describe('vercel-ai-utils', () => { - describe('convertPromptToMessages', () => { + describe('convertUserInputToMessagesFormat', () => { it('should convert a prompt with system to a messages array', () => { expect( - convertPromptToMessages( + convertUserInputToMessagesFormat( JSON.stringify({ system: 'You are a friendly robot', prompt: 'Hello, robot', @@ -19,7 +19,7 @@ describe('vercel-ai-utils', () => { it('should convert a system prompt to a messages array', () => { expect( - convertPromptToMessages( + convertUserInputToMessagesFormat( JSON.stringify({ system: 'You are a friendly robot', }), @@ -29,7 +29,7 @@ describe('vercel-ai-utils', () => { it('should convert a user only prompt to a messages array', () => { expect( - convertPromptToMessages( + convertUserInputToMessagesFormat( JSON.stringify({ prompt: 'Hello, robot', }), @@ -37,9 +37,57 @@ describe('vercel-ai-utils', () => { ).toStrictEqual([{ role: 'user', content: 'Hello, robot' }]); }); + it('should convert a messages array with multiple messages', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + messages: [ + { role: 'user', content: 'What is the weather?' }, + { role: 'assistant', content: "I'll check." }, + { role: 'user', content: 'Also New York?' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'user', content: 'What is the weather?' }, + { role: 'assistant', content: "I'll check." }, + { role: 'user', content: 'Also New York?' }, + ]); + }); + + it('should convert a messages array with a single message', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + messages: [{ role: 'user', content: 'Hello' }], + }), + ), + ).toStrictEqual([{ role: 'user', content: 'Hello' }]); + }); + + it('should filter out invalid entries in messages array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + messages: [ + { role: 'user', content: 'Hello' }, + 'not an object', + null, + { role: 'user' }, + { content: 'missing role' }, + { role: 'assistant', content: 'Valid' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Valid' }, + ]); + }); + it('should ignore unexpected data', () => { expect( - convertPromptToMessages( + convertUserInputToMessagesFormat( JSON.stringify({ randomField: 'Hello, robot', nothing: 'that we know how to handle', @@ -48,8 +96,96 @@ describe('vercel-ai-utils', () => { ).toStrictEqual([]); }); + it('should prepend system instruction to messages array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + system: 'You are a friendly robot', + messages: [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'system', content: 'You are a friendly robot' }, + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]); + }); + + it('should handle double-encoded messages array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + messages: JSON.stringify([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]), + }), + ), + ).toStrictEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]); + }); + it('should not break on invalid json', () => { - expect(convertPromptToMessages('this is not json')).toStrictEqual([]); + expect(convertUserInputToMessagesFormat('this is not json')).toStrictEqual([]); + }); + + it('should convert a prompt array to a messages array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + prompt: [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]); + }); + + it('should prepend system instruction to prompt array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + system: 'You are a friendly robot', + prompt: [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'system', content: 'You are a friendly robot' }, + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]); + }); + + it('should filter out invalid entries in prompt array', () => { + expect( + convertUserInputToMessagesFormat( + JSON.stringify({ + prompt: [ + { role: 'user', content: 'Hello' }, + 'not an object', + null, + { role: 'user' }, + { content: 'missing role' }, + { role: 'assistant', content: 'Valid' }, + ], + }), + ), + ).toStrictEqual([ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Valid' }, + ]); }); }); });