Real Anthropic Claude integration per explicit product decision: a ticket's AI session diagnoses the problem via a structured-output call, applies a DB-configurable confidence-band policy (FR-005), and on "proceed" reasons and acts through a small permission/risk-gated tool system (FR-011/FR-012), optionally walking a matching runbook step by step with the application — never the model — owning the step index (FR-015/FR-016). Resolution requires real tool evidence, never customer claims alone (FR-018) — verifyProductResolution is a documented fail-closed placeholder mirroring the existing malware-scanner precedent, since no real per-product operational signal exists yet. AISupportSession.status mirrors onto Ticket.status through 003-ticketing's existing AI_ANALYZING/AI_TROUBLESHOOTING/AI_VERIFYING/AI_RESOLVED/ HUMAN_ESCALATION state machine, discovered during planning to have been built anticipating this exact feature. Two circular module dependencies (escalation<->sessions, tools<->sessions) were designed around rather than found as bugs: escalation is a pure summary formatter with no state dependencies of its own, and tools stays a clean leaf module with zero dependency on ai-support/sessions. Ticket creation enqueues the first diagnosis turn via the existing queue infrastructure (off the hot path of the inbound SaaS integration endpoint); a human actor changing ticket status ends the AI session via the event-bus scaffold that existed in this codebase but had never been wired to anything. A real Prisma limitation was found and fixed before it reached tests: compound-unique upsert rejects null for a nullable key column, so AIConfidencePolicy uses find-then-update/create instead, same fix class 004 already used for the same underlying limitation. Adds 9 unit tests (confidence-band, tool-policy-gate, runbook-step- advance) and 6 integration test files, including the two constitution- required standing E2E scenarios. AI-independent tests were run against real Postgres/Redis/MinIO (88 passed, 0 failed across the full suite, including every pre-existing 002/003/004 test). The AI-dependent tests compile and skip cleanly via describe.skipIf but were not run against a live model — no ANTHROPIC_API_KEY was available in this session; a real key must be supplied before this feature can actually run. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
84 lines
3.3 KiB
TypeScript
84 lines
3.3 KiB
TypeScript
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
import { buildApp } from '@/app';
|
|
import { prismaClient } from '@/infrastructure/database';
|
|
import { FastifyInstance } from 'fastify';
|
|
|
|
/** Covers specs/005-ai-support/contracts/ai-support-contract.md's confidence-policy admin
|
|
* surface (FR-005) — no LLM call involved, so this runs unconditionally against a real
|
|
* Postgres, unlike the AI-diagnosis/reasoning tests in this same directory. */
|
|
describe('AI confidence policy — admin config (FR-005)', () => {
|
|
let app: FastifyInstance;
|
|
const externalProductId = `TEST_AI_POLICY_PROD_${Date.now()}`;
|
|
|
|
beforeAll(async () => {
|
|
app = await buildApp();
|
|
await prismaClient.product.create({
|
|
data: { externalProductId, name: 'AI Policy Test Product', status: 'active' },
|
|
});
|
|
});
|
|
|
|
afterAll(async () => {
|
|
const product = await prismaClient.product.findUnique({ where: { externalProductId } });
|
|
if (product) {
|
|
await prismaClient.aIConfidencePolicy.deleteMany({ where: { productId: product.id } });
|
|
}
|
|
await prismaClient.product.deleteMany({ where: { externalProductId } });
|
|
await app.close();
|
|
});
|
|
|
|
it('rejects a highThreshold at or below lowThreshold', async () => {
|
|
const response = await app.inject({
|
|
method: 'PUT',
|
|
url: `/admin/products/${externalProductId}/ai-policy`,
|
|
payload: { highThreshold: 0.4, lowThreshold: 0.4, maxClarifyingQuestions: 2 },
|
|
});
|
|
expect(response.statusCode).toBe(400);
|
|
});
|
|
|
|
it('upserts a product-wide policy and reflects it on GET, alongside the system defaults', async () => {
|
|
const putResponse = await app.inject({
|
|
method: 'PUT',
|
|
url: `/admin/products/${externalProductId}/ai-policy`,
|
|
payload: { highThreshold: 0.8, lowThreshold: 0.3, maxClarifyingQuestions: 3 },
|
|
});
|
|
expect(putResponse.statusCode).toBe(200);
|
|
expect(putResponse.json().data.highThreshold).toBe(0.8);
|
|
|
|
const getResponse = await app.inject({
|
|
method: 'GET',
|
|
url: `/admin/products/${externalProductId}/ai-policy`,
|
|
});
|
|
expect(getResponse.statusCode).toBe(200);
|
|
const body = getResponse.json().data;
|
|
expect(body.configured).toHaveLength(1);
|
|
expect(body.configured[0].highThreshold).toBe(0.8);
|
|
expect(body.systemDefaults).toHaveProperty('highThreshold');
|
|
expect(body.systemDefaults).toHaveProperty('lowThreshold');
|
|
expect(body.systemDefaults).toHaveProperty('maxClarifyingQuestions');
|
|
});
|
|
|
|
it('a second PUT for the same product (no category) updates the existing row rather than creating a duplicate', async () => {
|
|
await app.inject({
|
|
method: 'PUT',
|
|
url: `/admin/products/${externalProductId}/ai-policy`,
|
|
payload: { highThreshold: 0.9, lowThreshold: 0.2, maxClarifyingQuestions: 1 },
|
|
});
|
|
const getResponse = await app.inject({
|
|
method: 'GET',
|
|
url: `/admin/products/${externalProductId}/ai-policy`,
|
|
});
|
|
const configured = getResponse.json().data.configured;
|
|
expect(configured).toHaveLength(1);
|
|
expect(configured[0].highThreshold).toBe(0.9);
|
|
});
|
|
|
|
it('404s for an unregistered product', async () => {
|
|
const response = await app.inject({
|
|
method: 'PUT',
|
|
url: `/admin/products/TEST_NEVER_REGISTERED_${Date.now()}/ai-policy`,
|
|
payload: { highThreshold: 0.8, lowThreshold: 0.3, maxClarifyingQuestions: 2 },
|
|
});
|
|
expect(response.statusCode).toBe(404);
|
|
});
|
|
});
|