feat(014-full-observability): real distributed tracing + first 5 business metrics
User Story 3: initializes a real OpenTelemetry TracerProvider (previously inert — getTracer() returned a no-op tracer with nothing ever exported). Adds ticket.create, ai.escalation, and orchestration.assignment spans covering both FR-006 cross-module paths, verified via a real, in-memory test exporter that confirms actual trace/parent-span nesting, not mocked. Also registers an AsyncLocalStorageContextManager (@opentelemetry/context-async-hooks) — without one, OTel's context API is a no-op that doesn't propagate across the await boundaries this feature's own event-bus subscribers rely on for span nesting; caught by the first version of the tracing integration test actually failing on real parent/child assertions, not assumed. Graceful degradation (FR-007) verified against a real, deliberately unreachable OTLP endpoint: the SDK's own background export path (what production actually exercises) never produces an unhandled rejection. Starts on the 11 named business-health metrics: AI session resolved/escalated outcomes (session.repository.ts, the single choke point every branch in session.service.ts funnels through), human-vs-AI resolution + resolution-time (a new TICKET_UPDATED/RESOLVED subscriber), escalation rate (a new subscriber on ESCALATION_TRIGGERED, published unconditionally since 008 but never previously consumed), and recurring problems (tickets.service.ts's existing problem-creation call site). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
saqib mir
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
acd3843aaf
commit
f75589d9ce
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { sessionsService, sessionRepository } from '@/modules/ai-support/sessions';
|
||||
import { getTestSpanExporter } from '@/infrastructure/observability';
|
||||
|
||||
/**
|
||||
* Covers specs/014-full-observability/quickstart.md Scenario 3, against a real running app.
|
||||
* Reads spans back from getTestSpanExporter() (a real TracerProvider, real spans — only the
|
||||
* export *destination* is swapped for an in-memory one, per research.md §4) rather than through
|
||||
* an external collector.
|
||||
*
|
||||
* Drives the escalation path directly via sessionsService.escalate(...) instead of through a
|
||||
* real AI reasoning turn — the tracing behavior under test (span creation/nesting) is identical
|
||||
* either way, and this avoids requiring a paid ANTHROPIC_API_KEY for every test run (see
|
||||
* ai-verification-and-escalation.test.ts's own `describe.skipIf(!hasRealApiKey)` for the
|
||||
* alternative this project already uses when a real reasoning call is actually required).
|
||||
*/
|
||||
describe('Cross-module trace (User Story 3)', () => {
|
||||
let app: FastifyInstance;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicketViaInboundRequest(): Promise<string> {
|
||||
const externalProductId = `TEST_TRACE_PROD_${Date.now()}_${Math.random().toString(36).slice(2)}`;
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Tracing Test Product', status: 'active' },
|
||||
});
|
||||
const secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: 'Needs a human, for tracing.',
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
it('produces a ticket.create span for ticket intake', async () => {
|
||||
getTestSpanExporter().reset();
|
||||
|
||||
await createTicketViaInboundRequest();
|
||||
|
||||
const spans = getTestSpanExporter().getFinishedSpans();
|
||||
const createSpan = spans.find((s) => s.name === 'ticket.create');
|
||||
expect(createSpan).toBeDefined();
|
||||
expect(createSpan?.attributes['ticket.id']).toBeTruthy();
|
||||
});
|
||||
|
||||
it('nests orchestration.assignment under ai.escalation, sharing one trace', async () => {
|
||||
getTestSpanExporter().reset();
|
||||
|
||||
const ticketId = await createTicketViaInboundRequest();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
await sessionsService.escalate(session, ticketId, 'Escalating for tracing test.');
|
||||
|
||||
const spans = getTestSpanExporter().getFinishedSpans();
|
||||
const escalationSpan = spans.find((s) => s.name === 'ai.escalation');
|
||||
const assignmentSpan = spans.find((s) => s.name === 'orchestration.assignment');
|
||||
|
||||
expect(escalationSpan).toBeDefined();
|
||||
expect(assignmentSpan).toBeDefined();
|
||||
expect(assignmentSpan?.spanContext().traceId).toBe(escalationSpan?.spanContext().traceId);
|
||||
expect(assignmentSpan?.parentSpanContext?.spanId).toBe(escalationSpan?.spanContext().spanId);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { BasicTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||
|
||||
/**
|
||||
* 014-full-observability FR-007/Quickstart Scenario 3 steps 3-4: an unreachable tracing
|
||||
* collector must never surface as an application-level failure. This exercises the real
|
||||
* OpenTelemetry SDK's actual background export path (BasicTracerProvider + BatchSpanProcessor +
|
||||
* a real OTLPTraceExporter, against a real, deliberately-unreachable address — not a mock) the
|
||||
* same way it runs in production: a span ends, the processor's own internal timer schedules the
|
||||
* export, and a failed export is caught by the SDK's own error handler — never left as an
|
||||
* unhandled rejection that could crash the process.
|
||||
*
|
||||
* (Deliberately does NOT call provider.forceFlush() to prove this — forceFlush() is documented
|
||||
* OpenTelemetry SDK behavior that *does* reject on a failed export, by design, so a caller that
|
||||
* explicitly asks "did my flush succeed?" can find out. This feature's own code never calls
|
||||
* forceFlush() on the request-handling path, only the SDK's own background timer does, which is
|
||||
* what this test exercises instead.)
|
||||
*/
|
||||
describe('Tracing graceful degradation', () => {
|
||||
let unhandledRejection: unknown;
|
||||
const onUnhandledRejection = (reason: unknown) => {
|
||||
unhandledRejection = reason;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
process.removeListener('unhandledRejection', onUnhandledRejection);
|
||||
});
|
||||
|
||||
it('does not produce an unhandled rejection when the background export to an unreachable endpoint fails', async () => {
|
||||
unhandledRejection = undefined;
|
||||
process.on('unhandledRejection', onUnhandledRejection);
|
||||
|
||||
const exporter = new OTLPTraceExporter({
|
||||
url: 'http://127.0.0.1:1/v1/traces', // port 1 — nothing listens there
|
||||
timeoutMillis: 500,
|
||||
});
|
||||
const provider = new BasicTracerProvider({
|
||||
spanProcessors: [
|
||||
new BatchSpanProcessor(exporter, { scheduledDelayMillis: 10, exportTimeoutMillis: 500 }),
|
||||
],
|
||||
});
|
||||
|
||||
const span = provider.getTracer('test').startSpan('unreachable-export-test');
|
||||
span.end(); // triggers the processor's own internal timer, not forceFlush()
|
||||
|
||||
// Long enough for the internal timer (10ms) + the failed connection attempt to resolve.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
|
||||
expect(unhandledRejection).toBeUndefined();
|
||||
|
||||
await provider.shutdown();
|
||||
}, 10000);
|
||||
});
|
||||
Reference in New Issue
Block a user