From e2452f858dcb2401fe3b2c9e9f304b46d3a4d1a4 Mon Sep 17 00:00:00 2001 From: hoshinotsuyoshi Date: Fri, 26 Sep 2025 20:39:03 +0900 Subject: [PATCH 1/5] fix: add explicit Sentry error reporting for SQL syntax errors in saveTestcase tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, SQL syntax errors were not being sent to Sentry even though they were wrapped with withSentryCaptureException. This was because these errors are intentionally thrown to trigger LangGraph's retry mechanism. Now explicitly capturing SQL syntax errors in Sentry before re-throwing them, allowing us to monitor these errors while maintaining the retry behavior. - Added Sentry.captureException() call with appropriate tags and context - Tags include errorType: 'sql_syntax_error' and toolName: 'saveTestcase' - Extra context includes the actual SQL query and parse error message Fixes #5724 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .../src/qa-agent/tools/saveTestcaseTool.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts index 89957563ee..2d06dc8bd1 100644 --- a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts +++ b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts @@ -5,6 +5,7 @@ import { type StructuredTool, tool } from '@langchain/core/tools' import { Command } from '@langchain/langgraph' import { dmlOperationSchema } from '@liam-hq/artifact' import { type PgParseResult, pgParse } from '@liam-hq/schema/parser' +import * as Sentry from '@sentry/node' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' import { SSE_EVENTS } from '../../streaming/constants' @@ -44,11 +45,27 @@ const validateSqlSyntax = async (sql: string): Promise => { const parseResult: PgParseResult = await pgParse(sql) if (parseResult.error) { - // LangGraph tool nodes require throwing errors to trigger retry mechanism - // eslint-disable-next-line no-throw-error/no-throw-error - throw new Error( + const error = new Error( `SQL syntax error: ${parseResult.error.message}. Fix testcaseWithDml.dmlOperation.sql and retry.`, ) + + // Capture the SQL syntax error in Sentry for monitoring + // This is separate from the withSentryCaptureException wrapper + // because we want to track these errors even though they trigger retries + Sentry.captureException(error, { + tags: { + errorType: 'sql_syntax_error', + toolName: 'saveTestcase', + }, + extra: { + sql, + parseError: parseResult.error.message, + }, + }) + + // LangGraph tool nodes require throwing errors to trigger retry mechanism + + throw error } } From 2a601fc053898a1a344ab77fc8fb69569caceb5e Mon Sep 17 00:00:00 2001 From: hoshinotsuyoshi Date: Mon, 29 Sep 2025 13:17:06 +0900 Subject: [PATCH 2/5] test: add 10% random error simulation for Sentry testing This temporary change will help verify Sentry integration in Vercel preview. Will be removed after confirming Sentry receives the errors properly. - 10% chance to trigger simulated error even with valid SQL - Different tags for real vs simulated errors - Additional context to distinguish test errors --- .../src/qa-agent/tools/saveTestcaseTool.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts index 2d06dc8bd1..d245a2f11c 100644 --- a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts +++ b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts @@ -44,9 +44,14 @@ const configSchema = v.object({ const validateSqlSyntax = async (sql: string): Promise => { const parseResult: PgParseResult = await pgParse(sql) - if (parseResult.error) { + // Randomly trigger error 10% of the time for testing Sentry integration + const shouldSimulateError = Math.random() < 0.1 + + if (parseResult.error || shouldSimulateError) { const error = new Error( - `SQL syntax error: ${parseResult.error.message}. Fix testcaseWithDml.dmlOperation.sql and retry.`, + parseResult.error + ? `SQL syntax error: ${parseResult.error.message}. Fix testcaseWithDml.dmlOperation.sql and retry.` + : '[TEST] Simulated SQL error for Sentry testing (10% chance). SQL was valid but error was triggered for testing.', ) // Capture the SQL syntax error in Sentry for monitoring @@ -54,12 +59,16 @@ const validateSqlSyntax = async (sql: string): Promise => { // because we want to track these errors even though they trigger retries Sentry.captureException(error, { tags: { - errorType: 'sql_syntax_error', + errorType: shouldSimulateError + ? 'simulated_sql_error_test' + : 'sql_syntax_error', toolName: 'saveTestcase', + isTest: shouldSimulateError ? 'true' : 'false', }, extra: { sql, - parseError: parseResult.error.message, + parseError: parseResult.error?.message || 'Simulated error for testing', + simulatedForTesting: shouldSimulateError, }, }) From b16e34d3eb4f9e0f123cc120b789af83887c2f7b Mon Sep 17 00:00:00 2001 From: hoshinotsuyoshi Date: Mon, 29 Sep 2025 16:47:13 +0900 Subject: [PATCH 3/5] debug: add Sentry initialization check and extensive logging - Add Sentry initialization if not already initialized - Add debug logging to trace error flow - Check for SENTRY_DSN environment variables - Log eventId after sending to Sentry - Fix TypeScript errors with environment variables This will help diagnose why errors are not being sent to Sentry --- .../src/qa-agent/tools/saveTestcaseTool.ts | 30 ++++++++++++++++++- .../src/utils/withSentryCaptureException.ts | 7 ++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts index d245a2f11c..2a3753e7d2 100644 --- a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts +++ b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts @@ -54,10 +54,36 @@ const validateSqlSyntax = async (sql: string): Promise => { : '[TEST] Simulated SQL error for Sentry testing (10% chance). SQL was valid but error was triggered for testing.', ) + // Debug logging to verify error is being triggered + console.error('[saveTestcaseTool] SQL validation error occurred:', { + isSimulated: shouldSimulateError, + errorMessage: error.message, + sql: sql.substring(0, 100), // Log first 100 chars of SQL + }) + + // Try to capture the error in Sentry + // Initialize Sentry if not already initialized (for Next.js environment) + if (!Sentry.getCurrentScope()) { + console.warn( + '[saveTestcaseTool] Sentry not initialized, initializing now...', + ) + const dsn = + process.env['SENTRY_DSN'] || process.env['NEXT_PUBLIC_SENTRY_DSN'] + if (dsn) { + Sentry.init({ + dsn, + environment: process.env['NEXT_PUBLIC_ENV_NAME'] || 'development', + debug: true, + }) + } else { + console.warn('[saveTestcaseTool] No SENTRY_DSN found in environment') + } + } + // Capture the SQL syntax error in Sentry for monitoring // This is separate from the withSentryCaptureException wrapper // because we want to track these errors even though they trigger retries - Sentry.captureException(error, { + const eventId = Sentry.captureException(error, { tags: { errorType: shouldSimulateError ? 'simulated_sql_error_test' @@ -72,6 +98,8 @@ const validateSqlSyntax = async (sql: string): Promise => { }, }) + console.error('[saveTestcaseTool] Sent to Sentry with eventId:', eventId) + // LangGraph tool nodes require throwing errors to trigger retry mechanism throw error diff --git a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts index d300449ada..f415c4b41d 100644 --- a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts +++ b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts @@ -7,7 +7,12 @@ export const withSentryCaptureException = async ( try { return await operation() } catch (error) { - Sentry.captureException(error) + console.error('[withSentryCaptureException] Capturing error:', error) + const eventId = Sentry.captureException(error) + console.error( + '[withSentryCaptureException] Sent to Sentry with eventId:', + eventId, + ) throw error } } From f0af134fb04d6796cc55bb80726bb8cab7c493dc Mon Sep 17 00:00:00 2001 From: hoshinotsuyoshi Date: Tue, 30 Sep 2025 08:23:53 +0900 Subject: [PATCH 4/5] refactor: improve Sentry error handling by removing duplicate capture - Remove duplicate Sentry.captureException in validateSqlSyntax - Let withSentryCaptureException handle all error capturing - Add tags and extra data to error objects for better debugging - Fix TypeScript and ESLint issues The core issue is that @sentry/node is not initialized in the agent package. EventIds are generated but not sent because the SDK isn't connected to Sentry. This refactor at least consolidates error handling in one place. --- .../src/qa-agent/tools/saveTestcaseTool.ts | 74 +++++++------------ .../src/utils/withSentryCaptureException.ts | 38 +++++++++- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts index 2a3753e7d2..c3f1383623 100644 --- a/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts +++ b/frontend/internal-packages/agent/src/qa-agent/tools/saveTestcaseTool.ts @@ -5,7 +5,6 @@ import { type StructuredTool, tool } from '@langchain/core/tools' import { Command } from '@langchain/langgraph' import { dmlOperationSchema } from '@liam-hq/artifact' import { type PgParseResult, pgParse } from '@liam-hq/schema/parser' -import * as Sentry from '@sentry/node' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' import { SSE_EVENTS } from '../../streaming/constants' @@ -48,60 +47,43 @@ const validateSqlSyntax = async (sql: string): Promise => { const shouldSimulateError = Math.random() < 0.1 if (parseResult.error || shouldSimulateError) { - const error = new Error( - parseResult.error - ? `SQL syntax error: ${parseResult.error.message}. Fix testcaseWithDml.dmlOperation.sql and retry.` - : '[TEST] Simulated SQL error for Sentry testing (10% chance). SQL was valid but error was triggered for testing.', - ) + const errorMessage = parseResult.error + ? `SQL syntax error: ${parseResult.error.message}. Fix testcaseWithDml.dmlOperation.sql and retry.` + : '[TEST] Simulated SQL error for Sentry testing (10% chance). SQL was valid but error was triggered for testing.' + + // Create a custom error with additional properties for Sentry + const error: Error & { + tags?: Record + extra?: Record + } = new Error(errorMessage) + + // Add Sentry-specific data to the error object + // withSentryCaptureException will use these when capturing + error.tags = { + errorType: shouldSimulateError + ? 'simulated_sql_error_test' + : 'sql_syntax_error', + toolName: 'saveTestcase', + isTest: shouldSimulateError ? 'true' : 'false', + } + + error.extra = { + sql, + parseError: parseResult.error?.message || 'Simulated error for testing', + simulatedForTesting: shouldSimulateError, + } // Debug logging to verify error is being triggered console.error('[saveTestcaseTool] SQL validation error occurred:', { isSimulated: shouldSimulateError, errorMessage: error.message, sql: sql.substring(0, 100), // Log first 100 chars of SQL + tags: error.tags, + extra: error.extra, }) - // Try to capture the error in Sentry - // Initialize Sentry if not already initialized (for Next.js environment) - if (!Sentry.getCurrentScope()) { - console.warn( - '[saveTestcaseTool] Sentry not initialized, initializing now...', - ) - const dsn = - process.env['SENTRY_DSN'] || process.env['NEXT_PUBLIC_SENTRY_DSN'] - if (dsn) { - Sentry.init({ - dsn, - environment: process.env['NEXT_PUBLIC_ENV_NAME'] || 'development', - debug: true, - }) - } else { - console.warn('[saveTestcaseTool] No SENTRY_DSN found in environment') - } - } - - // Capture the SQL syntax error in Sentry for monitoring - // This is separate from the withSentryCaptureException wrapper - // because we want to track these errors even though they trigger retries - const eventId = Sentry.captureException(error, { - tags: { - errorType: shouldSimulateError - ? 'simulated_sql_error_test' - : 'sql_syntax_error', - toolName: 'saveTestcase', - isTest: shouldSimulateError ? 'true' : 'false', - }, - extra: { - sql, - parseError: parseResult.error?.message || 'Simulated error for testing', - simulatedForTesting: shouldSimulateError, - }, - }) - - console.error('[saveTestcaseTool] Sent to Sentry with eventId:', eventId) - // LangGraph tool nodes require throwing errors to trigger retry mechanism - + // withSentryCaptureException will capture this error throw error } } diff --git a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts index f415c4b41d..96bcb268e4 100644 --- a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts +++ b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts @@ -8,11 +8,47 @@ export const withSentryCaptureException = async ( return await operation() } catch (error) { console.error('[withSentryCaptureException] Capturing error:', error) - const eventId = Sentry.captureException(error) + + // Check if error has Sentry-specific properties + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Need to check for custom error properties + const errorWithSentry = error as { + tags?: Record + extra?: Record + } + + // Only pass options if they exist + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Need type for Sentry options + const captureOptions = {} as { + tags?: Record + extra?: Record + } + + if (errorWithSentry.tags) { + captureOptions.tags = errorWithSentry.tags + } + if (errorWithSentry.extra) { + captureOptions.extra = errorWithSentry.extra + } + + // Capture with tags and extra data if available + const eventId = + Object.keys(captureOptions).length > 0 + ? Sentry.captureException(error, captureOptions) + : Sentry.captureException(error) + console.error( '[withSentryCaptureException] Sent to Sentry with eventId:', eventId, ) + console.error( + '[withSentryCaptureException] With tags:', + errorWithSentry.tags, + ) + console.error( + '[withSentryCaptureException] With extra:', + errorWithSentry.extra, + ) + throw error } } From db17536f356e9d36d6ebba675aea52b6b8fe8d9f Mon Sep 17 00:00:00 2001 From: hoshinotsuyoshi Date: Tue, 30 Sep 2025 08:34:50 +0900 Subject: [PATCH 5/5] feat: add direct Sentry API fallback for error reporting When Sentry SDK is not initialized (which is the case for @sentry/node in agent package), fall back to direct Sentry API calls to ensure errors are captured. - Add sendToSentryAPI function that posts directly to Sentry endpoint - Parse DSN to extract API endpoint and authentication key - Create proper Sentry event payload with stacktrace - Fallback to API when Sentry.getCurrentScope() is not available This ensures SQL syntax errors and other errors are sent to Sentry even when the SDK is not properly initialized in the agent package. --- .../src/utils/withSentryCaptureException.ts | 144 +++++++++++++++--- 1 file changed, 123 insertions(+), 21 deletions(-) diff --git a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts index 96bcb268e4..f8faa872ac 100644 --- a/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts +++ b/frontend/internal-packages/agent/src/utils/withSentryCaptureException.ts @@ -1,5 +1,96 @@ import * as Sentry from '@sentry/node' +/** + * Generate a random UUID for Sentry event ID + */ +function generateEventId(): string { + return 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'.replace(/[x]/g, () => { + return Math.floor(Math.random() * 16).toString(16) + }) +} + +/** + * Send error directly to Sentry via API + * This is used when Sentry SDK is not initialized + */ +async function sendToSentryAPI( + error: Error, + tags?: Record, + extra?: Record, +): Promise { + const dsn = process.env['SENTRY_DSN'] || process.env['NEXT_PUBLIC_SENTRY_DSN'] + + if (!dsn) { + console.warn('[withSentryCaptureException] No SENTRY_DSN found') + return + } + + // Parse DSN + const dsnMatch = dsn.match(/https:\/\/([^@]+)@([^/]+)\/(.+)/) + if (!dsnMatch) { + console.warn('[withSentryCaptureException] Invalid DSN format') + return + } + + const [, publicKey, host, projectId] = dsnMatch + const sentryUrl = `https://${host}/api/${projectId}/store/` + + // Create Sentry event payload + const event = { + event_id: generateEventId(), + timestamp: new Date().toISOString(), + platform: 'node', + level: 'error', + exception: { + values: [ + { + type: error.name, + value: error.message, + stacktrace: { + frames: (error.stack || '') + .split('\n') + .slice(1) + .map((line) => ({ + filename: 'unknown', + function: line.trim(), + in_app: true, + })) + .reverse(), + }, + }, + ], + }, + tags, + extra, + environment: process.env['NEXT_PUBLIC_ENV_NAME'] || 'development', + } + + // eslint-disable-next-line no-restricted-syntax -- Need try-catch for fetch error handling + try { + const response = await fetch(sentryUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Sentry-Auth': `Sentry sentry_key=${publicKey}, sentry_version=7`, + }, + body: JSON.stringify(event), + }) + + if (!response.ok) { + console.error( + '[withSentryCaptureException] Failed to send to Sentry:', + response.status, + ) + } else { + } + } catch (fetchError) { + console.error( + '[withSentryCaptureException] Error sending to Sentry:', + fetchError, + ) + } +} + export const withSentryCaptureException = async ( operation: () => Promise, ): Promise => { @@ -16,30 +107,41 @@ export const withSentryCaptureException = async ( extra?: Record } - // Only pass options if they exist - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Need type for Sentry options - const captureOptions = {} as { - tags?: Record - extra?: Record - } + // Try using Sentry SDK first + if (Sentry.getCurrentScope()) { + // Only pass options if they exist + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Need type for Sentry options + const captureOptions = {} as { + tags?: Record + extra?: Record + } - if (errorWithSentry.tags) { - captureOptions.tags = errorWithSentry.tags - } - if (errorWithSentry.extra) { - captureOptions.extra = errorWithSentry.extra - } + if (errorWithSentry.tags) { + captureOptions.tags = errorWithSentry.tags + } + if (errorWithSentry.extra) { + captureOptions.extra = errorWithSentry.extra + } - // Capture with tags and extra data if available - const eventId = - Object.keys(captureOptions).length > 0 - ? Sentry.captureException(error, captureOptions) - : Sentry.captureException(error) + // Capture with tags and extra data if available + const eventId = + Object.keys(captureOptions).length > 0 + ? Sentry.captureException(error, captureOptions) + : Sentry.captureException(error) + + console.error( + '[withSentryCaptureException] Sent to Sentry SDK with eventId:', + eventId, + ) + } else { + await sendToSentryAPI( + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Need to cast error to Error type + error as Error, + errorWithSentry.tags, + errorWithSentry.extra, + ) + } - console.error( - '[withSentryCaptureException] Sent to Sentry with eventId:', - eventId, - ) console.error( '[withSentryCaptureException] With tags:', errorWithSentry.tags,