diff --git a/tests/load/admin-reporting.load.ts b/tests/load/admin-reporting.load.ts new file mode 100644 index 0000000..eba4919 --- /dev/null +++ b/tests/load/admin-reporting.load.ts @@ -0,0 +1,69 @@ +/** + * specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for the + * 015-reporting-dashboards admin endpoints — pure database aggregation reads, no third-party + * cost (unlike ai-support-flow.load.ts). Run against a real running dev server: + * `npx tsx tests/load/admin-reporting.load.ts`. + * + * Signs in as the project's own seeded admin account once (a session JWT is reusable across + * requests, unlike 002-saas-integration's single-use integration tokens), then cycles across + * all four dashboards so the report reflects a realistic mix of the endpoint group, not just one + * route. + */ +import { runLoadTest } from './autocannon.config'; + +const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501'; +const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 10); +const DURATION_SEC = Number(process.env.LOAD_TEST_DURATION_SEC ?? 10); +const ADMIN_EMAIL = process.env.LOAD_TEST_ADMIN_EMAIL ?? 'admin@supporthub.internal'; +const ADMIN_PASSWORD = process.env.LOAD_TEST_ADMIN_PASSWORD ?? 'ChangeMe123!'; + +const DASHBOARD_PATHS = [ + '/admin/reports/management', + '/admin/reports/support', + '/admin/reports/ai', +]; + +async function signIn(): Promise { + const response = await fetch(`${API_URL}/auth/login`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD }), + }); + if (!response.ok) { + throw new Error( + `Admin sign-in failed (${response.status}) — set LOAD_TEST_ADMIN_EMAIL/PASSWORD if the seeded admin credentials differ.`, + ); + } + const body = (await response.json()) as { data: { token: string } }; + return body.data.token; +} + +async function main(): Promise { + const token = await signIn(); + + let requestIndex = 0; + await runLoadTest('admin-reporting', { + url: API_URL, + connections: CONNECTIONS, + duration: DURATION_SEC, + requests: [ + { + method: 'GET', + headers: { authorization: `Bearer ${token}` }, + setupRequest: (request) => { + request.path = DASHBOARD_PATHS[requestIndex % DASHBOARD_PATHS.length]; + requestIndex += 1; + return request; + }, + }, + ], + }); +} + +main() + .then(() => process.exit(0)) + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + process.exit(1); + }); diff --git a/tests/load/ai-support-flow.load.ts b/tests/load/ai-support-flow.load.ts new file mode 100644 index 0000000..1118234 --- /dev/null +++ b/tests/load/ai-support-flow.load.ts @@ -0,0 +1,166 @@ +/** + * specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for the AI + * support flow's customer-reply turn (`POST /tickets/:ticketId/ai-session/messages`). + * + * IMPORTANT — real cost: 005-ai-support calls the real Anthropic API for every diagnosis turn + * (via @anthropic-ai/sdk), both when a ticket's first AI turn runs (asynchronously, right after + * ticket creation) AND for every customer-reply turn this script sends. Running this script + * fires genuine, billed Claude API calls — it is NOT a free, purely-internal load test like + * ticket-creation.load.ts's own database-only path. Confirm the intended request volume + * (LOAD_TEST_REQUESTS below) with whoever owns the Anthropic billing before running this against + * anything but a small smoke-sized amount. + * + * Observed in practice: a synthetic problem string with no matching entry in this throwaway + * product's (empty) knowledge base often escalates on the very first AI turn — there is nothing + * for the model to diagnose confidently. That is still a real, honestly-measured code path (a + * fast 404 from `handleCustomerReply`'s "no active session" guard), not a script bug — this + * script measures whatever the endpoint actually does, rather than forcing every session to + * stay open by only ever picking already-active ones. + * + * Run against a real running dev server: `npx tsx tests/load/ai-support-flow.load.ts`. + * `LOAD_TEST_REQUESTS` (default 5) hard-caps the total number of real API-consuming requests — + * this script uses autocannon's `amount` option, never an open-ended `duration`, specifically to + * keep the real-money cost bounded and predictable. + */ +import { prismaClient } from '@/infrastructure/database'; +import { + encryptCredential, + generateCredentialSecret, + issueIntegrationToken, +} from '@/modules/catalog/products'; +import { runLoadTest } from './autocannon.config'; + +const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501'; +const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 1); +const REQUESTS = Number(process.env.LOAD_TEST_REQUESTS ?? 5); +const SESSION_POLL_TIMEOUT_MS = 30_000; +const SESSION_POLL_INTERVAL_MS = 500; + +async function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** The AI_SESSION worker creates the session asynchronously after ticket creation (FR-001, + * session.service.ts's own runFirstTurn docstring) — poll until SOME session row exists (any + * status) rather than assuming it's ready the instant the create-ticket request returns. + * Deliberately not restricted to the "active" statuses: this throwaway product's problems have + * no matching knowledge, so the very first turn commonly escalates immediately, which is a + * legitimate terminal outcome to measure, not a wait condition. */ +async function waitForAnySession(ticketId: string): Promise { + const deadline = Date.now() + SESSION_POLL_TIMEOUT_MS; + while (Date.now() < deadline) { + const session = await prismaClient.aISupportSession.findFirst({ where: { ticketId } }); + if (session) return; + await sleep(SESSION_POLL_INTERVAL_MS); + } + throw new Error(`Timed out waiting for an AI session to be created on ticket ${ticketId}.`); +} + +async function cleanup(productId: string): Promise { + // AI-support rows form a deeper chain than a plain ticket (session -> diagnosis/interaction/ + // action/knowledge-reference), all RESTRICT-constrained back to the ticket — every level must + // be cleared before the ticket itself can be deleted. + const sessions = await prismaClient.aISupportSession.findMany({ + where: { ticket: { productId } }, + select: { id: true }, + }); + const sessionIds = sessions.map((s) => s.id); + await prismaClient.aIActionResult.deleteMany({ + where: { action: { sessionId: { in: sessionIds } } }, + }); + await prismaClient.aIAction.deleteMany({ where: { sessionId: { in: sessionIds } } }); + await prismaClient.aIDiagnosis.deleteMany({ where: { sessionId: { in: sessionIds } } }); + await prismaClient.aIInteraction.deleteMany({ where: { sessionId: { in: sessionIds } } }); + await prismaClient.aIKnowledgeReference.deleteMany({ where: { sessionId: { in: sessionIds } } }); + await prismaClient.aISupportSession.deleteMany({ where: { id: { in: sessionIds } } }); + await prismaClient.ticketMessage.deleteMany({ where: { ticket: { productId } } }); + await prismaClient.assignmentHistory.deleteMany({ where: { ticket: { productId } } }); + await prismaClient.assignment.deleteMany({ where: { ticket: { productId } } }); + await prismaClient.sLARun.deleteMany({ where: { ticket: { productId } } }); + await prismaClient.escalationEvent.deleteMany({ where: { ticket: { productId } } }); + await prismaClient.ticket.deleteMany({ where: { productId } }); + await prismaClient.problem.deleteMany({ where: { productId } }); + await prismaClient.productIntegration.deleteMany({ where: { productId } }); + await prismaClient.product.deleteMany({ where: { id: productId } }); +} + +async function main(): Promise { + const externalProductId = `LOAD_TEST_AI_FLOW_${Date.now()}`; + const product = await prismaClient.product.create({ + data: { externalProductId, name: 'Load Test — AI Support Flow', status: 'active' }, + }); + + try { + const secret = generateCredentialSecret(); + await prismaClient.productIntegration.create({ + data: { + productId: product.id, + credentialRef: encryptCredential(secret), + authMechanism: 'signed_token', + allowedScope: { tenantIds: ['load-test-tenant'] }, + status: 'active', + rateLimitPerMinute: 100000, + rateLimitPerUserPerMinute: 100000, + }, + }); + + // eslint-disable-next-line no-console + console.log(`Pre-creating ${REQUESTS} tickets and waiting for each one's AI session...`); + const ticketIds: string[] = []; + for (let i = 0; i < REQUESTS; i++) { + const token = issueIntegrationToken(secret, { + externalProductId, + tenantId: 'load-test-tenant', + userId: `load-test-user-${i}`, + }); + const response = await fetch(`${API_URL}/v1/support/requests`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${token}` }, + body: JSON.stringify({ + productId: externalProductId, + tenantId: 'load-test-tenant', + userId: `load-test-user-${i}`, + source: 'load-test', + problem: `AI flow load test problem ${i} ${Date.now()}`, + }), + }); + const body = (await response.json()) as { data: { ticketId: string } }; + ticketIds.push(body.data.ticketId); + await waitForAnySession(body.data.ticketId); + } + // eslint-disable-next-line no-console + console.log('Every ticket has an AI session (active or already resolved/escalated) — starting the load test.'); + + let requestIndex = 0; + await runLoadTest('ai-support-flow', { + url: `${API_URL}/tickets/placeholder/ai-session/messages`, + connections: CONNECTIONS, + amount: REQUESTS, + requests: [ + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + setupRequest: (request) => { + const ticketId = ticketIds[requestIndex % ticketIds.length]; + requestIndex += 1; + request.path = `/tickets/${ticketId}/ai-session/messages`; + request.body = JSON.stringify({ + message: 'I already tried restarting, still broken.', + }); + return request; + }, + }, + ], + }); + } finally { + await cleanup(product.id); + } +} + +main() + .then(() => process.exit(0)) + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + process.exit(1); + }); diff --git a/tests/load/autocannon.config.ts b/tests/load/autocannon.config.ts new file mode 100644 index 0000000..c17e711 --- /dev/null +++ b/tests/load/autocannon.config.ts @@ -0,0 +1,79 @@ +import autocannon from 'autocannon'; +import { mkdirSync, writeFileSync } from 'fs'; +import path from 'path'; + +/** + * specs/016-load-concurrency-testing data-model.md "Load test report". Printed to the console + * and written as JSON under tests/load/reports/ (gitignored — a run artifact, not a fixture) + * for every run, so two runs of the same script can be compared. + */ +export interface LoadTestReport { + endpoint: string; + connections: number; + durationSec: number; + requestsPerSec: number; + latencyP50Ms: number; + latencyP90Ms: number; + latencyP99Ms: number; + non2xxCount: number; + rateLimitedCount: number; +} + +/** + * FR-007/FR-008: a thin wrapper over autocannon's programmatic API producing this feature's own + * report shape. FR-009: deliberately has NO pass/fail assertion on the numbers — throughput and + * latency targets are an `OPEN BUSINESS DECISION` (spec.md Assumptions) this project's roadmap + * says must never be invented and shipped as if final. This tool measures and records; wiring an + * explicit CI gate is future work once the business sets a real target. + */ +export async function runLoadTest( + endpoint: string, + options: autocannon.Options, +): Promise { + const result = await autocannon(options); + + const report: LoadTestReport = { + endpoint, + connections: result.connections, + durationSec: result.duration, + requestsPerSec: Number(result.requests.average.toFixed(2)), + latencyP50Ms: result.latency.p50, + latencyP90Ms: result.latency.p90, + latencyP99Ms: result.latency.p99, + non2xxCount: result.non2xx, + rateLimitedCount: result.statusCodeStats?.['429']?.count ?? 0, + }; + + printReport(report); + writeReport(report); + return report; +} + +function printReport(report: LoadTestReport): void { + // eslint-disable-next-line no-console + console.log(`\n=== Load test report: ${report.endpoint} ===`); + // eslint-disable-next-line no-console + console.log(`connections: ${report.connections} duration: ${report.durationSec}s`); + // eslint-disable-next-line no-console + console.log(`requests/sec (avg): ${report.requestsPerSec}`); + // eslint-disable-next-line no-console + console.log( + `latency p50/p90/p99 (ms): ${report.latencyP50Ms}/${report.latencyP90Ms}/${report.latencyP99Ms}`, + ); + // eslint-disable-next-line no-console + console.log(`non-2xx: ${report.non2xxCount} rate-limited (429): ${report.rateLimitedCount}`); + // eslint-disable-next-line no-console + console.log( + 'No pass/fail threshold applied — throughput/latency targets are an OPEN BUSINESS DECISION.', + ); +} + +function writeReport(report: LoadTestReport): void { + const dir = path.join(__dirname, 'reports'); + mkdirSync(dir, { recursive: true }); + const safeName = report.endpoint.replace(/[^a-z0-9-]/gi, '_'); + const filePath = path.join(dir, `${safeName}-${Date.now()}.json`); + writeFileSync(filePath, JSON.stringify(report, null, 2)); + // eslint-disable-next-line no-console + console.log(`Report written to ${filePath}`); +} diff --git a/tests/load/ticket-creation.load.ts b/tests/load/ticket-creation.load.ts new file mode 100644 index 0000000..5fee3b8 --- /dev/null +++ b/tests/load/ticket-creation.load.ts @@ -0,0 +1,94 @@ +/** + * specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for new + * support-request submission — the entry point of the entire support flow. Run against a real + * running dev server: `npx tsx tests/load/ticket-creation.load.ts`. + * + * IMPORTANT — real cost: a successful ticket creation asynchronously triggers 005-ai-support's + * AI_SESSION worker, which makes a real, billed Anthropic API call for that ticket's first + * diagnosis turn. This endpoint's own HTTP response is fast and free, but the request still has + * a real downstream cost — set `LOAD_TEST_REQUESTS` to bound the total ticket count explicitly + * rather than relying on an open-ended `LOAD_TEST_DURATION_SEC` run whose total is + * latency-dependent and less predictable. + * + * Creates its own throwaway product + integration credential, generates a fresh single-use + * integration token per request (002-saas-integration's own jti replay protection means a + * single static token can't be reused across requests), and cleans up everything it created + * once the run finishes. + */ +import { prismaClient } from '@/infrastructure/database'; +import { + encryptCredential, + generateCredentialSecret, + issueIntegrationToken, +} from '@/modules/catalog/products'; +import { runLoadTest } from './autocannon.config'; + +const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501'; +const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 10); +const DURATION_SEC = Number(process.env.LOAD_TEST_DURATION_SEC ?? 10); +// When set, caps the total number of tickets created (and thus the total downstream AI cost) +// instead of running for an open-ended duration. +const REQUESTS = process.env.LOAD_TEST_REQUESTS ? Number(process.env.LOAD_TEST_REQUESTS) : undefined; + +async function main(): Promise { + const externalProductId = `LOAD_TEST_TICKET_CREATION_${Date.now()}`; + const product = await prismaClient.product.create({ + data: { externalProductId, name: 'Load Test — Ticket Creation', status: 'active' }, + }); + const secret = generateCredentialSecret(); + await prismaClient.productIntegration.create({ + data: { + productId: product.id, + credentialRef: encryptCredential(secret), + authMechanism: 'signed_token', + allowedScope: { tenantIds: ['load-test-tenant'] }, + status: 'active', + rateLimitPerMinute: 100000, + rateLimitPerUserPerMinute: 100000, + }, + }); + + try { + await runLoadTest('ticket-creation', { + url: `${API_URL}/v1/support/requests`, + connections: CONNECTIONS, + ...(REQUESTS !== undefined ? { amount: REQUESTS } : { duration: DURATION_SEC }), + requests: [ + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + setupRequest: (request) => { + const token = issueIntegrationToken(secret, { + externalProductId, + tenantId: 'load-test-tenant', + userId: 'load-test-user', + }); + request.headers = { ...request.headers, authorization: `Bearer ${token}` }; + request.body = JSON.stringify({ + productId: externalProductId, + tenantId: 'load-test-tenant', + userId: 'load-test-user', + source: 'load-test', + problem: `Load test problem ${Date.now()}-${Math.random()}`, + }); + return request; + }, + }, + ], + }); + } finally { + await prismaClient.ticketMessage.deleteMany({ where: { ticket: { productId: product.id } } }); + await prismaClient.ticket.deleteMany({ where: { productId: product.id } }); + await prismaClient.problem.deleteMany({ where: { productId: product.id } }); + await prismaClient.productIntegration.deleteMany({ where: { productId: product.id } }); + await prismaClient.product.deleteMany({ where: { id: product.id } }); + } +} + +main() + .then(() => process.exit(0)) + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + process.exit(1); + });