import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { buildApp } from '@/app'; import { prismaClient } from '@/infrastructure/database'; import { FastifyInstance } from 'fastify'; import { encryptCredential, generateCredentialSecret, issueIntegrationToken, } from '@/modules/catalog/products'; import { sessionsService } from '@/modules/ai-support/sessions'; /** * Covers specs/005-ai-support/quickstart.md Scenario 1 — requires a real ANTHROPIC_API_KEY * (spec.md Assumptions: this feature integrates a real LLM provider, not a mock). Skipped * entirely, not failed, when no real key is configured — see README's "AI Support" section for * how to supply one locally. */ const hasRealApiKey = /^sk-ant-/.test(process.env.ANTHROPIC_API_KEY ?? ''); describe.skipIf(!hasRealApiKey)( 'AI diagnosis — confidence decides the outcome (User Story 1)', () => { let app: FastifyInstance; const externalProductId = `TEST_AI_DIAG_PROD_${Date.now()}`; let secret: string; beforeAll(async () => { app = await buildApp(); const product = await prismaClient.product.create({ data: { externalProductId, name: 'AI Diagnosis Test Product', status: 'active' }, }); secret = generateCredentialSecret(); await prismaClient.productIntegration.create({ data: { productId: product.id, credentialRef: encryptCredential(secret), authMechanism: 'signed_token', allowedScope: { tenantIds: ['tenant-1'] }, status: 'active', rateLimitPerMinute: 1000, rateLimitPerUserPerMinute: 1000, }, }); // FR-006's grounding: at least one published entry must exist for a diagnosis to ever // reach the confidence-band decision instead of escalating on "no knowledge". await app .inject({ method: 'POST', url: `/admin/products/${externalProductId}/knowledge`, payload: { code: `KB-AITEST-${Date.now()}`, type: 'faq', problem: 'The application will not load past the loading screen.', recommendedSolution: 'Clear the browser cache and reload.', }, }) .then((r) => { const code = r.json().data.code; return app.inject({ method: 'PATCH', url: `/admin/knowledge/${code}/publish` }); }); }); afterAll(async () => { const product = await prismaClient.product.findUnique({ where: { externalProductId } }); if (product) { const tickets = await prismaClient.ticket.findMany({ where: { productId: product.id } }); const ticketIds = tickets.map((t) => t.id); const sessions = await prismaClient.aISupportSession.findMany({ where: { ticketId: { in: ticketIds } }, }); const sessionIds = sessions.map((s) => s.id); await prismaClient.aIKnowledgeReference.deleteMany({ where: { sessionId: { in: sessionIds } }, }); await prismaClient.aIDiagnosis.deleteMany({ where: { sessionId: { in: sessionIds } } }); await prismaClient.aIInteraction.deleteMany({ where: { sessionId: { in: sessionIds } } }); await prismaClient.aISupportSession.deleteMany({ where: { id: { in: sessionIds } } }); await prismaClient.ticketMessage.deleteMany({ where: { ticketId: { in: ticketIds } } }); await prismaClient.ticket.deleteMany({ where: { productId: product.id } }); await prismaClient.problem.deleteMany({ where: { productId: product.id } }); await prismaClient.knowledgeEntry.deleteMany({ where: { productId: product.id } }); await prismaClient.aIConfidencePolicy.deleteMany({ where: { productId: product.id } }); } await prismaClient.productIntegration.deleteMany({ where: { product: { externalProductId } }, }); await prismaClient.product.deleteMany({ where: { externalProductId } }); await app.close(); }); async function createTicketAndRunFirstTurn(problem: string): Promise { const token = issueIntegrationToken(secret, { externalProductId, tenantId: 'tenant-1', userId: 'user-1', }); const created = await app.inject({ method: 'POST', url: '/v1/support/requests', headers: { authorization: `Bearer ${token}` }, payload: { productId: externalProductId, tenantId: 'tenant-1', userId: 'user-1', source: 'test', problem, }, }); const ticketId = created.json().data.ticketId; // Bypass the queue — no worker runs during buildApp()-based tests, same convention as // ticket-attachments.test.ts's malware scanner call. await sessionsService.runFirstTurn(ticketId); return ticketId; } it('records a structured diagnosis with a confidence score, grounded in retrieved knowledge', async () => { const ticketId = await createTicketAndRunFirstTurn('The app is stuck on the loading screen.'); const response = await app.inject({ method: 'GET', url: `/tickets/${ticketId}/ai-session` }); expect(response.statusCode).toBe(200); const body = response.json().data; expect(body.diagnosis).not.toBeNull(); expect(typeof body.diagnosis.confidence).toBe('number'); expect(body.diagnosis.confidence).toBeGreaterThanOrEqual(0); expect(body.diagnosis.confidence).toBeLessThanOrEqual(1); }, 60000); it('escalates rather than diagnosing when no knowledge exists for the product', async () => { const noKnowledgeProduct = `TEST_AI_DIAG_NOKB_${Date.now()}`; const product = await prismaClient.product.create({ data: { externalProductId: noKnowledgeProduct, name: 'No KB product', status: 'active' }, }); const noKbSecret = generateCredentialSecret(); await prismaClient.productIntegration.create({ data: { productId: product.id, credentialRef: encryptCredential(noKbSecret), authMechanism: 'signed_token', allowedScope: { tenantIds: ['tenant-1'] }, status: 'active', rateLimitPerMinute: 1000, rateLimitPerUserPerMinute: 1000, }, }); const token = issueIntegrationToken(noKbSecret, { externalProductId: noKnowledgeProduct, tenantId: 'tenant-1', userId: 'user-1', }); const created = await app.inject({ method: 'POST', url: '/v1/support/requests', headers: { authorization: `Bearer ${token}` }, payload: { productId: noKnowledgeProduct, tenantId: 'tenant-1', userId: 'user-1', source: 'test', problem: 'Something is broken.', }, }); const ticketId = created.json().data.ticketId; await sessionsService.runFirstTurn(ticketId); const response = await app.inject({ method: 'GET', url: `/tickets/${ticketId}/ai-session` }); expect(response.json().data.status).toBe('escalated'); await prismaClient.aIDiagnosis.deleteMany({ where: { session: { ticketId } }, }); await prismaClient.aISupportSession.deleteMany({ where: { ticketId } }); await prismaClient.ticketMessage.deleteMany({ where: { ticketId } }); await prismaClient.ticket.deleteMany({ where: { id: ticketId } }); await prismaClient.problem.deleteMany({ where: { productId: product.id } }); await prismaClient.productIntegration.deleteMany({ where: { productId: product.id } }); await prismaClient.product.deleteMany({ where: { externalProductId: noKnowledgeProduct } }); }, 60000); it('a low highThreshold/lowThreshold makes the same diagnosis less likely to escalate on confidence alone', async () => { await app.inject({ method: 'PUT', url: `/admin/products/${externalProductId}/ai-policy`, payload: { highThreshold: 0.01, lowThreshold: 0.0, maxClarifyingQuestions: 3 }, }); const ticketId = await createTicketAndRunFirstTurn( 'The app is stuck on the loading screen again.', ); const response = await app.inject({ method: 'GET', url: `/tickets/${ticketId}/ai-session` }); // With lowThreshold at 0, "escalate" from confidence alone is impossible — any remaining // escalation would have to come from FR-006 (no knowledge), which doesn't apply here. expect(response.json().data.status).not.toBe('escalated'); }, 60000); }, );