feat(016-load-concurrency-testing): US5 — autocannon load-test tooling
tests/load/autocannon.config.ts is a thin shared wrapper over autocannon's programmatic API producing this feature's own report shape (throughput, latency p50/p90/p99, non-2xx, rate-limited count) — printed and written to tests/load/reports/ (gitignored) for every run. No pass/fail threshold is applied (FR-009): throughput/latency targets are an OPEN BUSINESS DECISION per the roadmap's own convention, never invented. Three scripts cover the named critical endpoint groups: ticket-creation (pure DB path), admin-reporting (015-reporting-dashboards, pure DB path), and ai-support-flow (005-ai-support's real Anthropic API calls — clearly flagged as real, billed cost, run only at a small bounded amount rather than an open-ended duration). All three were run once at a small scale against the real dev server to confirm the tooling works end-to-end and cleans up fully after itself (verified via direct DB checks, not just script exit codes). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
saqib mir
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
cd8e408d7e
commit
8400a8db84
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for the
|
||||
* 015-reporting-dashboards admin endpoints — pure database aggregation reads, no third-party
|
||||
* cost (unlike ai-support-flow.load.ts). Run against a real running dev server:
|
||||
* `npx tsx tests/load/admin-reporting.load.ts`.
|
||||
*
|
||||
* Signs in as the project's own seeded admin account once (a session JWT is reusable across
|
||||
* requests, unlike 002-saas-integration's single-use integration tokens), then cycles across
|
||||
* all four dashboards so the report reflects a realistic mix of the endpoint group, not just one
|
||||
* route.
|
||||
*/
|
||||
import { runLoadTest } from './autocannon.config';
|
||||
|
||||
const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501';
|
||||
const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 10);
|
||||
const DURATION_SEC = Number(process.env.LOAD_TEST_DURATION_SEC ?? 10);
|
||||
const ADMIN_EMAIL = process.env.LOAD_TEST_ADMIN_EMAIL ?? 'admin@supporthub.internal';
|
||||
const ADMIN_PASSWORD = process.env.LOAD_TEST_ADMIN_PASSWORD ?? 'ChangeMe123!';
|
||||
|
||||
const DASHBOARD_PATHS = [
|
||||
'/admin/reports/management',
|
||||
'/admin/reports/support',
|
||||
'/admin/reports/ai',
|
||||
];
|
||||
|
||||
async function signIn(): Promise<string> {
|
||||
const response = await fetch(`${API_URL}/auth/login`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Admin sign-in failed (${response.status}) — set LOAD_TEST_ADMIN_EMAIL/PASSWORD if the seeded admin credentials differ.`,
|
||||
);
|
||||
}
|
||||
const body = (await response.json()) as { data: { token: string } };
|
||||
return body.data.token;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const token = await signIn();
|
||||
|
||||
let requestIndex = 0;
|
||||
await runLoadTest('admin-reporting', {
|
||||
url: API_URL,
|
||||
connections: CONNECTIONS,
|
||||
duration: DURATION_SEC,
|
||||
requests: [
|
||||
{
|
||||
method: 'GET',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
setupRequest: (request) => {
|
||||
request.path = DASHBOARD_PATHS[requestIndex % DASHBOARD_PATHS.length];
|
||||
requestIndex += 1;
|
||||
return request;
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for the AI
|
||||
* support flow's customer-reply turn (`POST /tickets/:ticketId/ai-session/messages`).
|
||||
*
|
||||
* IMPORTANT — real cost: 005-ai-support calls the real Anthropic API for every diagnosis turn
|
||||
* (via @anthropic-ai/sdk), both when a ticket's first AI turn runs (asynchronously, right after
|
||||
* ticket creation) AND for every customer-reply turn this script sends. Running this script
|
||||
* fires genuine, billed Claude API calls — it is NOT a free, purely-internal load test like
|
||||
* ticket-creation.load.ts's own database-only path. Confirm the intended request volume
|
||||
* (LOAD_TEST_REQUESTS below) with whoever owns the Anthropic billing before running this against
|
||||
* anything but a small smoke-sized amount.
|
||||
*
|
||||
* Observed in practice: a synthetic problem string with no matching entry in this throwaway
|
||||
* product's (empty) knowledge base often escalates on the very first AI turn — there is nothing
|
||||
* for the model to diagnose confidently. That is still a real, honestly-measured code path (a
|
||||
* fast 404 from `handleCustomerReply`'s "no active session" guard), not a script bug — this
|
||||
* script measures whatever the endpoint actually does, rather than forcing every session to
|
||||
* stay open by only ever picking already-active ones.
|
||||
*
|
||||
* Run against a real running dev server: `npx tsx tests/load/ai-support-flow.load.ts`.
|
||||
* `LOAD_TEST_REQUESTS` (default 5) hard-caps the total number of real API-consuming requests —
|
||||
* this script uses autocannon's `amount` option, never an open-ended `duration`, specifically to
|
||||
* keep the real-money cost bounded and predictable.
|
||||
*/
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { runLoadTest } from './autocannon.config';
|
||||
|
||||
const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501';
|
||||
const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 1);
|
||||
const REQUESTS = Number(process.env.LOAD_TEST_REQUESTS ?? 5);
|
||||
const SESSION_POLL_TIMEOUT_MS = 30_000;
|
||||
const SESSION_POLL_INTERVAL_MS = 500;
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** The AI_SESSION worker creates the session asynchronously after ticket creation (FR-001,
|
||||
* session.service.ts's own runFirstTurn docstring) — poll until SOME session row exists (any
|
||||
* status) rather than assuming it's ready the instant the create-ticket request returns.
|
||||
* Deliberately not restricted to the "active" statuses: this throwaway product's problems have
|
||||
* no matching knowledge, so the very first turn commonly escalates immediately, which is a
|
||||
* legitimate terminal outcome to measure, not a wait condition. */
|
||||
async function waitForAnySession(ticketId: string): Promise<void> {
|
||||
const deadline = Date.now() + SESSION_POLL_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const session = await prismaClient.aISupportSession.findFirst({ where: { ticketId } });
|
||||
if (session) return;
|
||||
await sleep(SESSION_POLL_INTERVAL_MS);
|
||||
}
|
||||
throw new Error(`Timed out waiting for an AI session to be created on ticket ${ticketId}.`);
|
||||
}
|
||||
|
||||
async function cleanup(productId: string): Promise<void> {
|
||||
// AI-support rows form a deeper chain than a plain ticket (session -> diagnosis/interaction/
|
||||
// action/knowledge-reference), all RESTRICT-constrained back to the ticket — every level must
|
||||
// be cleared before the ticket itself can be deleted.
|
||||
const sessions = await prismaClient.aISupportSession.findMany({
|
||||
where: { ticket: { productId } },
|
||||
select: { id: true },
|
||||
});
|
||||
const sessionIds = sessions.map((s) => s.id);
|
||||
await prismaClient.aIActionResult.deleteMany({
|
||||
where: { action: { sessionId: { in: sessionIds } } },
|
||||
});
|
||||
await prismaClient.aIAction.deleteMany({ where: { sessionId: { in: sessionIds } } });
|
||||
await prismaClient.aIDiagnosis.deleteMany({ where: { sessionId: { in: sessionIds } } });
|
||||
await prismaClient.aIInteraction.deleteMany({ where: { sessionId: { in: sessionIds } } });
|
||||
await prismaClient.aIKnowledgeReference.deleteMany({ where: { sessionId: { in: sessionIds } } });
|
||||
await prismaClient.aISupportSession.deleteMany({ where: { id: { in: sessionIds } } });
|
||||
await prismaClient.ticketMessage.deleteMany({ where: { ticket: { productId } } });
|
||||
await prismaClient.assignmentHistory.deleteMany({ where: { ticket: { productId } } });
|
||||
await prismaClient.assignment.deleteMany({ where: { ticket: { productId } } });
|
||||
await prismaClient.sLARun.deleteMany({ where: { ticket: { productId } } });
|
||||
await prismaClient.escalationEvent.deleteMany({ where: { ticket: { productId } } });
|
||||
await prismaClient.ticket.deleteMany({ where: { productId } });
|
||||
await prismaClient.problem.deleteMany({ where: { productId } });
|
||||
await prismaClient.productIntegration.deleteMany({ where: { productId } });
|
||||
await prismaClient.product.deleteMany({ where: { id: productId } });
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const externalProductId = `LOAD_TEST_AI_FLOW_${Date.now()}`;
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Load Test — AI Support Flow', status: 'active' },
|
||||
});
|
||||
|
||||
try {
|
||||
const secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['load-test-tenant'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 100000,
|
||||
rateLimitPerUserPerMinute: 100000,
|
||||
},
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Pre-creating ${REQUESTS} tickets and waiting for each one's AI session...`);
|
||||
const ticketIds: string[] = [];
|
||||
for (let i = 0; i < REQUESTS; i++) {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'load-test-tenant',
|
||||
userId: `load-test-user-${i}`,
|
||||
});
|
||||
const response = await fetch(`${API_URL}/v1/support/requests`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', authorization: `Bearer ${token}` },
|
||||
body: JSON.stringify({
|
||||
productId: externalProductId,
|
||||
tenantId: 'load-test-tenant',
|
||||
userId: `load-test-user-${i}`,
|
||||
source: 'load-test',
|
||||
problem: `AI flow load test problem ${i} ${Date.now()}`,
|
||||
}),
|
||||
});
|
||||
const body = (await response.json()) as { data: { ticketId: string } };
|
||||
ticketIds.push(body.data.ticketId);
|
||||
await waitForAnySession(body.data.ticketId);
|
||||
}
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Every ticket has an AI session (active or already resolved/escalated) — starting the load test.');
|
||||
|
||||
let requestIndex = 0;
|
||||
await runLoadTest('ai-support-flow', {
|
||||
url: `${API_URL}/tickets/placeholder/ai-session/messages`,
|
||||
connections: CONNECTIONS,
|
||||
amount: REQUESTS,
|
||||
requests: [
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
setupRequest: (request) => {
|
||||
const ticketId = ticketIds[requestIndex % ticketIds.length];
|
||||
requestIndex += 1;
|
||||
request.path = `/tickets/${ticketId}/ai-session/messages`;
|
||||
request.body = JSON.stringify({
|
||||
message: 'I already tried restarting, still broken.',
|
||||
});
|
||||
return request;
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
await cleanup(product.id);
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import autocannon from 'autocannon';
|
||||
import { mkdirSync, writeFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* specs/016-load-concurrency-testing data-model.md "Load test report". Printed to the console
|
||||
* and written as JSON under tests/load/reports/ (gitignored — a run artifact, not a fixture)
|
||||
* for every run, so two runs of the same script can be compared.
|
||||
*/
|
||||
export interface LoadTestReport {
|
||||
endpoint: string;
|
||||
connections: number;
|
||||
durationSec: number;
|
||||
requestsPerSec: number;
|
||||
latencyP50Ms: number;
|
||||
latencyP90Ms: number;
|
||||
latencyP99Ms: number;
|
||||
non2xxCount: number;
|
||||
rateLimitedCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* FR-007/FR-008: a thin wrapper over autocannon's programmatic API producing this feature's own
|
||||
* report shape. FR-009: deliberately has NO pass/fail assertion on the numbers — throughput and
|
||||
* latency targets are an `OPEN BUSINESS DECISION` (spec.md Assumptions) this project's roadmap
|
||||
* says must never be invented and shipped as if final. This tool measures and records; wiring an
|
||||
* explicit CI gate is future work once the business sets a real target.
|
||||
*/
|
||||
export async function runLoadTest(
|
||||
endpoint: string,
|
||||
options: autocannon.Options,
|
||||
): Promise<LoadTestReport> {
|
||||
const result = await autocannon(options);
|
||||
|
||||
const report: LoadTestReport = {
|
||||
endpoint,
|
||||
connections: result.connections,
|
||||
durationSec: result.duration,
|
||||
requestsPerSec: Number(result.requests.average.toFixed(2)),
|
||||
latencyP50Ms: result.latency.p50,
|
||||
latencyP90Ms: result.latency.p90,
|
||||
latencyP99Ms: result.latency.p99,
|
||||
non2xxCount: result.non2xx,
|
||||
rateLimitedCount: result.statusCodeStats?.['429']?.count ?? 0,
|
||||
};
|
||||
|
||||
printReport(report);
|
||||
writeReport(report);
|
||||
return report;
|
||||
}
|
||||
|
||||
function printReport(report: LoadTestReport): void {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`\n=== Load test report: ${report.endpoint} ===`);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`connections: ${report.connections} duration: ${report.durationSec}s`);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`requests/sec (avg): ${report.requestsPerSec}`);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`latency p50/p90/p99 (ms): ${report.latencyP50Ms}/${report.latencyP90Ms}/${report.latencyP99Ms}`,
|
||||
);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`non-2xx: ${report.non2xxCount} rate-limited (429): ${report.rateLimitedCount}`);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'No pass/fail threshold applied — throughput/latency targets are an OPEN BUSINESS DECISION.',
|
||||
);
|
||||
}
|
||||
|
||||
function writeReport(report: LoadTestReport): void {
|
||||
const dir = path.join(__dirname, 'reports');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const safeName = report.endpoint.replace(/[^a-z0-9-]/gi, '_');
|
||||
const filePath = path.join(dir, `${safeName}-${Date.now()}.json`);
|
||||
writeFileSync(filePath, JSON.stringify(report, null, 2));
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Report written to ${filePath}`);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* specs/016-load-concurrency-testing User Story 5 / FR-007: load/throughput baseline for new
|
||||
* support-request submission — the entry point of the entire support flow. Run against a real
|
||||
* running dev server: `npx tsx tests/load/ticket-creation.load.ts`.
|
||||
*
|
||||
* IMPORTANT — real cost: a successful ticket creation asynchronously triggers 005-ai-support's
|
||||
* AI_SESSION worker, which makes a real, billed Anthropic API call for that ticket's first
|
||||
* diagnosis turn. This endpoint's own HTTP response is fast and free, but the request still has
|
||||
* a real downstream cost — set `LOAD_TEST_REQUESTS` to bound the total ticket count explicitly
|
||||
* rather than relying on an open-ended `LOAD_TEST_DURATION_SEC` run whose total is
|
||||
* latency-dependent and less predictable.
|
||||
*
|
||||
* Creates its own throwaway product + integration credential, generates a fresh single-use
|
||||
* integration token per request (002-saas-integration's own jti replay protection means a
|
||||
* single static token can't be reused across requests), and cleans up everything it created
|
||||
* once the run finishes.
|
||||
*/
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { runLoadTest } from './autocannon.config';
|
||||
|
||||
const API_URL = process.env.LOAD_TEST_API_URL ?? 'http://localhost:4501';
|
||||
const CONNECTIONS = Number(process.env.LOAD_TEST_CONNECTIONS ?? 10);
|
||||
const DURATION_SEC = Number(process.env.LOAD_TEST_DURATION_SEC ?? 10);
|
||||
// When set, caps the total number of tickets created (and thus the total downstream AI cost)
|
||||
// instead of running for an open-ended duration.
|
||||
const REQUESTS = process.env.LOAD_TEST_REQUESTS ? Number(process.env.LOAD_TEST_REQUESTS) : undefined;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const externalProductId = `LOAD_TEST_TICKET_CREATION_${Date.now()}`;
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Load Test — Ticket Creation', status: 'active' },
|
||||
});
|
||||
const secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['load-test-tenant'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 100000,
|
||||
rateLimitPerUserPerMinute: 100000,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await runLoadTest('ticket-creation', {
|
||||
url: `${API_URL}/v1/support/requests`,
|
||||
connections: CONNECTIONS,
|
||||
...(REQUESTS !== undefined ? { amount: REQUESTS } : { duration: DURATION_SEC }),
|
||||
requests: [
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
setupRequest: (request) => {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'load-test-tenant',
|
||||
userId: 'load-test-user',
|
||||
});
|
||||
request.headers = { ...request.headers, authorization: `Bearer ${token}` };
|
||||
request.body = JSON.stringify({
|
||||
productId: externalProductId,
|
||||
tenantId: 'load-test-tenant',
|
||||
userId: 'load-test-user',
|
||||
source: 'load-test',
|
||||
problem: `Load test problem ${Date.now()}-${Math.random()}`,
|
||||
});
|
||||
return request;
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
await prismaClient.ticketMessage.deleteMany({ where: { ticket: { productId: product.id } } });
|
||||
await prismaClient.ticket.deleteMany({ where: { productId: product.id } });
|
||||
await prismaClient.problem.deleteMany({ where: { productId: product.id } });
|
||||
await prismaClient.productIntegration.deleteMany({ where: { productId: product.id } });
|
||||
await prismaClient.product.deleteMany({ where: { id: product.id } });
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user