Files
support_backend/tests/integration/ai-confidence-policy.test.ts
T
saqib mirandClaude Sonnet 5 82d02bcdcd feat: implement AI support agent (005) — diagnosis, tools, runbooks, verification
Real Anthropic Claude integration per explicit product decision: a
ticket's AI session diagnoses the problem via a structured-output call,
applies a DB-configurable confidence-band policy (FR-005), and on
"proceed" reasons and acts through a small permission/risk-gated tool
system (FR-011/FR-012), optionally walking a matching runbook step by
step with the application — never the model — owning the step index
(FR-015/FR-016). Resolution requires real tool evidence, never customer
claims alone (FR-018) — verifyProductResolution is a documented
fail-closed placeholder mirroring the existing malware-scanner precedent,
since no real per-product operational signal exists yet.

AISupportSession.status mirrors onto Ticket.status through 003-ticketing's
existing AI_ANALYZING/AI_TROUBLESHOOTING/AI_VERIFYING/AI_RESOLVED/
HUMAN_ESCALATION state machine, discovered during planning to have been
built anticipating this exact feature. Two circular module dependencies
(escalation<->sessions, tools<->sessions) were designed around rather than
found as bugs: escalation is a pure summary formatter with no state
dependencies of its own, and tools stays a clean leaf module with zero
dependency on ai-support/sessions. Ticket creation enqueues the first
diagnosis turn via the existing queue infrastructure (off the hot path of
the inbound SaaS integration endpoint); a human actor changing ticket
status ends the AI session via the event-bus scaffold that existed in
this codebase but had never been wired to anything.

A real Prisma limitation was found and fixed before it reached tests:
compound-unique upsert rejects null for a nullable key column, so
AIConfidencePolicy uses find-then-update/create instead, same fix class
004 already used for the same underlying limitation.

Adds 9 unit tests (confidence-band, tool-policy-gate, runbook-step-
advance) and 6 integration test files, including the two constitution-
required standing E2E scenarios. AI-independent tests were run against
real Postgres/Redis/MinIO (88 passed, 0 failed across the full suite,
including every pre-existing 002/003/004 test). The AI-dependent tests
compile and skip cleanly via describe.skipIf but were not run against a
live model — no ANTHROPIC_API_KEY was available in this session; a real
key must be supplied before this feature can actually run.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-02 17:44:52 +05:30

84 lines
3.3 KiB
TypeScript

import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { buildApp } from '@/app';
import { prismaClient } from '@/infrastructure/database';
import { FastifyInstance } from 'fastify';
/** Covers specs/005-ai-support/contracts/ai-support-contract.md's confidence-policy admin
* surface (FR-005) — no LLM call involved, so this runs unconditionally against a real
* Postgres, unlike the AI-diagnosis/reasoning tests in this same directory. */
describe('AI confidence policy — admin config (FR-005)', () => {
let app: FastifyInstance;
const externalProductId = `TEST_AI_POLICY_PROD_${Date.now()}`;
beforeAll(async () => {
app = await buildApp();
await prismaClient.product.create({
data: { externalProductId, name: 'AI Policy Test Product', status: 'active' },
});
});
afterAll(async () => {
const product = await prismaClient.product.findUnique({ where: { externalProductId } });
if (product) {
await prismaClient.aIConfidencePolicy.deleteMany({ where: { productId: product.id } });
}
await prismaClient.product.deleteMany({ where: { externalProductId } });
await app.close();
});
it('rejects a highThreshold at or below lowThreshold', async () => {
const response = await app.inject({
method: 'PUT',
url: `/admin/products/${externalProductId}/ai-policy`,
payload: { highThreshold: 0.4, lowThreshold: 0.4, maxClarifyingQuestions: 2 },
});
expect(response.statusCode).toBe(400);
});
it('upserts a product-wide policy and reflects it on GET, alongside the system defaults', async () => {
const putResponse = await app.inject({
method: 'PUT',
url: `/admin/products/${externalProductId}/ai-policy`,
payload: { highThreshold: 0.8, lowThreshold: 0.3, maxClarifyingQuestions: 3 },
});
expect(putResponse.statusCode).toBe(200);
expect(putResponse.json().data.highThreshold).toBe(0.8);
const getResponse = await app.inject({
method: 'GET',
url: `/admin/products/${externalProductId}/ai-policy`,
});
expect(getResponse.statusCode).toBe(200);
const body = getResponse.json().data;
expect(body.configured).toHaveLength(1);
expect(body.configured[0].highThreshold).toBe(0.8);
expect(body.systemDefaults).toHaveProperty('highThreshold');
expect(body.systemDefaults).toHaveProperty('lowThreshold');
expect(body.systemDefaults).toHaveProperty('maxClarifyingQuestions');
});
it('a second PUT for the same product (no category) updates the existing row rather than creating a duplicate', async () => {
await app.inject({
method: 'PUT',
url: `/admin/products/${externalProductId}/ai-policy`,
payload: { highThreshold: 0.9, lowThreshold: 0.2, maxClarifyingQuestions: 1 },
});
const getResponse = await app.inject({
method: 'GET',
url: `/admin/products/${externalProductId}/ai-policy`,
});
const configured = getResponse.json().data.configured;
expect(configured).toHaveLength(1);
expect(configured[0].highThreshold).toBe(0.9);
});
it('404s for an unregistered product', async () => {
const response = await app.inject({
method: 'PUT',
url: `/admin/products/TEST_NEVER_REGISTERED_${Date.now()}/ai-policy`,
payload: { highThreshold: 0.8, lowThreshold: 0.3, maxClarifyingQuestions: 2 },
});
expect(response.statusCode).toBe(404);
});
});