Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7106753ed3 | ||
|
|
d65683641a | ||
|
|
814d9d7b17 | ||
|
|
4a159725c2 | ||
|
|
c4a2faa6e3 | ||
|
|
ea50e3596a | ||
|
|
de5915a8c1 | ||
|
|
f75589d9ce | ||
|
|
acd3843aaf | ||
|
|
0135e4ca05 | ||
|
|
5a0fe9f847 |
Generated
+157
-25
@@ -17,7 +17,10 @@
|
||||
"@fastify/swagger": "^8.14.0",
|
||||
"@fastify/swagger-ui": "^3.0.0",
|
||||
"@opentelemetry/api": "^1.8.0",
|
||||
"@opentelemetry/sdk-trace-base": "^1.22.0",
|
||||
"@opentelemetry/context-async-hooks": "^2.11.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.222.0",
|
||||
"@opentelemetry/resources": "^2.11.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.11.0",
|
||||
"@prisma/client": "^5.12.1",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"bullmq": "^5.7.1",
|
||||
@@ -1387,58 +1390,187 @@
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/core": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz",
|
||||
"integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==",
|
||||
"node_modules/@opentelemetry/api-logs": {
|
||||
"version": "0.222.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.222.0.tgz",
|
||||
"integrity": "sha512-9mb1If+IF6u0ZVXkHQ6ogEae5HwA6ajIVUgpSDQyRASxft6BSXHvBvPooRle3yFN/fKnCdSOnuu0OC3PLcF6+g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/context-async-hooks": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.11.0.tgz",
|
||||
"integrity": "sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/core": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz",
|
||||
"integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/exporter-trace-otlp-http": {
|
||||
"version": "0.222.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.222.0.tgz",
|
||||
"integrity": "sha512-RCnPWcHppwiquQ+cV3nWvNwdf0MG1w26e5jewW2T83nTZOlXgcg88sY9ulCVgagGHcq1mj0L0GP6YHgzk2v8oA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/otlp-exporter-base": "0.222.0",
|
||||
"@opentelemetry/otlp-transformer": "0.222.0",
|
||||
"@opentelemetry/sdk-trace": "2.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/otlp-exporter-base": {
|
||||
"version": "0.222.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.222.0.tgz",
|
||||
"integrity": "sha512-YbywG3veEm2Fb6TbdxRkuquWob6eVWXuA8/Ba1tXz9jHfUqpdE3keilOHEtPboC4CvS1bjeeVfNkWGOOrLj+lw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/otlp-transformer": "0.222.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/otlp-transformer": {
|
||||
"version": "0.222.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.222.0.tgz",
|
||||
"integrity": "sha512-/F3BZ89+CJQnZkMh2tCrtcdB+XT2Dxhj4FFE+WPQ//413hmFL0/RfEX6vgOIWGhiSzrkHWTK3+6SiT7K5/g/jQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/api-logs": "0.222.0",
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/resources": "2.11.0",
|
||||
"@opentelemetry/sdk-logs": "0.222.0",
|
||||
"@opentelemetry/sdk-metrics": "2.11.0",
|
||||
"@opentelemetry/sdk-trace": "2.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/resources": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz",
|
||||
"integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==",
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz",
|
||||
"integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
"@opentelemetry/api": ">=1.3.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-logs": {
|
||||
"version": "0.222.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.222.0.tgz",
|
||||
"integrity": "sha512-+19YHODIjaUCArxleaJtuufFZVpz/xvvK+VllQqE+W8hHolxdoRwHfK/s667zezwh1hkx6FFF+oYzetYgqK+Bg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/api-logs": "0.222.0",
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/resources": "2.11.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.4.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-metrics": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.11.0.tgz",
|
||||
"integrity": "sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/resources": "2.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.9.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-trace": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz",
|
||||
"integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/resources": "2.11.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.3.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-trace-base": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz",
|
||||
"integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==",
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz",
|
||||
"integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/resources": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
"@opentelemetry/core": "2.11.0",
|
||||
"@opentelemetry/resources": "2.11.0",
|
||||
"@opentelemetry/sdk-trace": "2.11.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
"@opentelemetry/api": ">=1.3.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/semantic-conventions": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz",
|
||||
"integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==",
|
||||
"version": "1.43.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz",
|
||||
"integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
|
||||
+4
-1
@@ -54,7 +54,10 @@
|
||||
"@fastify/swagger": "^8.14.0",
|
||||
"@fastify/swagger-ui": "^3.0.0",
|
||||
"@opentelemetry/api": "^1.8.0",
|
||||
"@opentelemetry/sdk-trace-base": "^1.22.0",
|
||||
"@opentelemetry/context-async-hooks": "^2.11.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.222.0",
|
||||
"@opentelemetry/resources": "^2.11.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.11.0",
|
||||
"@prisma/client": "^5.12.1",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"bullmq": "^5.7.1",
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "error_code_lookups" (
|
||||
"id" TEXT NOT NULL,
|
||||
"errorCodeId" TEXT NOT NULL,
|
||||
"productId" TEXT NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "error_code_lookups_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "error_code_lookups_productId_createdAt_idx" ON "error_code_lookups"("productId", "createdAt");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "error_code_lookups" ADD CONSTRAINT "error_code_lookups_errorCodeId_fkey" FOREIGN KEY ("errorCodeId") REFERENCES "error_codes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "error_code_lookups" ADD CONSTRAINT "error_code_lookups_productId_fkey" FOREIGN KEY ("productId") REFERENCES "products"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
+20
-1
@@ -45,6 +45,7 @@ model Product {
|
||||
tickets Ticket[]
|
||||
knowledgeEntries KnowledgeEntry[]
|
||||
errorCodes ErrorCode[]
|
||||
errorCodeLookups ErrorCodeLookup[]
|
||||
knownIssues KnownIssue[]
|
||||
runbooks Runbook[]
|
||||
aiConfidencePolicies AIConfidencePolicy[]
|
||||
@@ -239,13 +240,31 @@ model ErrorCode {
|
||||
productId String
|
||||
description String
|
||||
|
||||
product Product @relation(fields: [productId], references: [id])
|
||||
product Product @relation(fields: [productId], references: [id])
|
||||
knownIssues KnownIssue[]
|
||||
lookups ErrorCodeLookup[]
|
||||
|
||||
@@unique([productId, code])
|
||||
@@map("error_codes")
|
||||
}
|
||||
|
||||
// 015-reporting-dashboards research.md §6: a durable, append-only audit row recording that a
|
||||
// known-error-code lookup happened — 014-full-observability's own equivalent
|
||||
// (supporthub_known_error_lookups_total) is a process-lifetime Prometheus counter, unusable for
|
||||
// a historical "top errors" report. productId is denormalized from errorCode.productId so the
|
||||
// Product dashboard's range query never needs to join back through ErrorCode just to filter.
|
||||
model ErrorCodeLookup {
|
||||
id String @id @default(cuid())
|
||||
errorCodeId String
|
||||
errorCode ErrorCode @relation(fields: [errorCodeId], references: [id])
|
||||
productId String
|
||||
product Product @relation(fields: [productId], references: [id])
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@index([productId, createdAt])
|
||||
@@map("error_code_lookups")
|
||||
}
|
||||
|
||||
model KnownIssue {
|
||||
id String @id @default(cuid())
|
||||
productId String
|
||||
|
||||
@@ -41,3 +41,60 @@
|
||||
untracked today were confirmed by direct code inspection before writing this spec, not assumed.
|
||||
- All items pass; no revision iterations were needed. No [NEEDS CLARIFICATION] markers were
|
||||
required — every open question had a reasonable, documented default (see Assumptions).
|
||||
|
||||
## Implementation Notes (post-build)
|
||||
|
||||
- Registering a real `TracerProvider` alone was not sufficient to make span nesting work across
|
||||
this feature's own async event-bus subscribers: without also registering an
|
||||
`AsyncLocalStorageContextManager` (`@opentelemetry/context-async-hooks`, a third new
|
||||
dependency beyond the two research.md originally named), the OpenTelemetry API's
|
||||
`context.active()` is a no-op that does not propagate across `await` boundaries at all —
|
||||
`orchestration.assignment` came out as its own unrelated root span/trace instead of nesting
|
||||
under `ai.escalation`. Caught by the tracing integration test's own parent/child assertions
|
||||
actually failing on the first implementation, not assumed correct from reading the SDK's docs.
|
||||
- Installing `@opentelemetry/exporter-trace-otlp-http` alongside the already-pinned
|
||||
`@opentelemetry/sdk-trace-base@^1.22.0` pulled two incompatible OpenTelemetry core/resources
|
||||
major versions (1.x and 2.x) side by side. Resolved by bumping `sdk-trace-base` to `^2.11.0` to
|
||||
match — this also happened to close a moderate DoS advisory in `@opentelemetry/core <2.8.0`
|
||||
that the 1.x line was pinned to.
|
||||
- T016 (graceful degradation under an unreachable OTLP endpoint) ended up as its own unit test
|
||||
(`tests/unit/observability/tracing-graceful-degradation.test.ts`) rather than living in
|
||||
`tracing.test.ts` as tasks.md originally described. Reason: `tracing.ts` always uses the
|
||||
in-memory test exporter when `NODE_ENV=test`, so the integration suite's own running app can't
|
||||
be pointed at a bad OTLP endpoint to exercise this. The unit test instead constructs a real
|
||||
`BasicTracerProvider`/`BatchSpanProcessor`/`OTLPTraceExporter` pointed at a genuinely
|
||||
unreachable address directly, and — importantly — verifies the SDK's _background_ export path
|
||||
(what production actually exercises) never produces an unhandled rejection, rather than calling
|
||||
`forceFlush()` directly, which is documented OpenTelemetry behavior that _does_ reject on a
|
||||
failed export by design (the first version of this test asserted the wrong thing and failed
|
||||
against real, correct SDK behavior — not a bug in this feature's own code).
|
||||
- `sla.service.ts`'s pre-existing status-overwrite gap (a `'breached'` run's status silently
|
||||
becomes `'completed'` if the ticket later resolves — see research.md §5) was worked around for
|
||||
the metric's own correctness (read `run.status` before the overwrite) but left unfixed in the
|
||||
underlying data, consistent with how 013-auth-hardening documented a pre-existing bug it found
|
||||
without fixing it.
|
||||
- Found and fixed one genuine cross-file test-isolation bug this feature's own new test caused:
|
||||
`business-metrics.test.ts`'s "human resolution" case originally drove a ticket through a real
|
||||
`HUMAN_ESCALATION` transition via `ticketsService.updateStatus`, which — same as any other
|
||||
escalation in this codebase — triggers the real orchestration subscriber's default
|
||||
`ROUND_ROBIN` auto-assignment against every agent in the shared throwaway database, including
|
||||
other concurrently-running test files' own dedicated agents (reproduced deterministically
|
||||
against `agent-ticket-queue.test.ts`). Fixed by driving the intermediate state-machine
|
||||
transitions directly through `ticketsRepository.updateStatus` (no domain-event publish)
|
||||
instead, reserving the real, event-publishing `ticketsService.updateStatus` call for only the
|
||||
final `RESOLVED` transition the metric subscriber actually needs to observe.
|
||||
- Separately, found (not caused by this feature — confirmed via `git checkout` to the clean
|
||||
pre-014 commit and reproducing the identical failure) a pre-existing systemic collision risk in
|
||||
ticket-code generation: `ticket-code.ts`'s `deriveProductCode` keeps only the first 4
|
||||
alphabetic characters of `externalProductId`, so essentially every integration test file in
|
||||
this codebase (nearly all of which name their test products `TEST_<SOMETHING>`) collapses to
|
||||
the identical `"TEST"` code prefix. Running enough `TEST_*`-prefixed files concurrently (as
|
||||
vitest does by default across worker threads/processes) makes independent files race for the
|
||||
same `TEST-<year>-<sequence>` numbering space, occasionally exceeding
|
||||
`tickets.service.ts`'s fixed `MAX_CODE_RETRIES = 5` and surfacing as a real `500`
|
||||
(`Unique constraint failed on the fields: (code)`) instead of the retry silently absorbing it.
|
||||
Confirmed independent of this feature (reproduces on `79bc2ef`, 013-auth-hardening's tip, with
|
||||
none of this feature's code present) and left unfixed here — a ticket-code-generation
|
||||
concurrency fix belongs to 003-ticketing's own module, out of scope for an observability
|
||||
feature. Worth a dedicated future fix (e.g. a longer/hash-based product code, or a
|
||||
database-level sequence rather than a `COUNT`-then-retry scheme).
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Contract: `/metrics` output
|
||||
|
||||
This feature adds no new HTTP endpoints — `GET /metrics` already exists and its response shape
|
||||
(Prometheus text exposition format) is unchanged. This document is the contract for its
|
||||
**content**: which metric series a consumer (Prometheus, or any scraper) can rely on after this
|
||||
feature ships, replacing the usual per-endpoint request/response contract for a feature with no
|
||||
new routes.
|
||||
|
||||
## Guarantees
|
||||
|
||||
1. Every metric already exposed today (the default `prom-client` process metrics, and
|
||||
`supporthub_http_request_duration_seconds`) continues to appear, with the same name and label
|
||||
set — FR-010. `supporthub_http_request_duration_seconds` gains real observations where today
|
||||
it has none; its metric name/labels/type do not change.
|
||||
2. Each of the eleven new series in [data-model.md](../data-model.md#metrics-prometheus-via-prom-client)
|
||||
appears on `/metrics` from process start (a `Counter`/`Histogram` with zero observations
|
||||
still exports its metadata — `# HELP`/`# TYPE` lines — even before its first increment; a
|
||||
consumer's dashboard/alert config can reference it immediately without waiting for the first
|
||||
event).
|
||||
3. No metric name or label value is derived from unbounded, request-supplied input — every
|
||||
label is one of: a fixed small enum (`outcome`, `resolved_by`, `matched`), a route pattern
|
||||
(bounded by the number of registered routes), a tool name (bounded by the tool registry), an
|
||||
error code or category ID (bounded by admin-configured product data, not raw user text).
|
||||
This is a deliberate constraint, not an incidental one — unbounded label cardinality is a
|
||||
well-known way to make a Prometheus deployment fall over, and every label chosen in
|
||||
data-model.md was checked against this before being finalized.
|
||||
4. `/health`, `/health/live`, `/health/ready` response shapes are unchanged (FR-010) — this
|
||||
feature does not touch `health.service.ts` or `health.routes.ts`.
|
||||
|
||||
## Example (illustrative, not exhaustive)
|
||||
|
||||
```text
|
||||
# HELP supporthub_http_request_duration_seconds Duration of HTTP requests in seconds
|
||||
# TYPE supporthub_http_request_duration_seconds histogram
|
||||
supporthub_http_request_duration_seconds_bucket{method="POST",route="/tickets",status_code="201",le="0.1"} 3
|
||||
supporthub_http_request_duration_seconds_count{method="POST",route="/tickets",status_code="201"} 3
|
||||
|
||||
# HELP supporthub_ai_session_outcomes_total Count of AI support sessions by terminal outcome
|
||||
# TYPE supporthub_ai_session_outcomes_total counter
|
||||
supporthub_ai_session_outcomes_total{outcome="resolved"} 12
|
||||
supporthub_ai_session_outcomes_total{outcome="escalated"} 4
|
||||
|
||||
# HELP supporthub_sla_run_outcomes_total Count of SLA runs by outcome
|
||||
# TYPE supporthub_sla_run_outcomes_total counter
|
||||
supporthub_sla_run_outcomes_total{outcome="met"} 9
|
||||
supporthub_sla_run_outcomes_total{outcome="breached"} 1
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
Integration tests assert against this contract by scraping `GET /metrics` (a real
|
||||
`app.inject` call, real registry) before and after driving each metric's real underlying event
|
||||
through the real API, parsing the specific series' value out of the text response and asserting
|
||||
it moved by exactly the expected amount — never by mocking `prom-client` or the registry itself.
|
||||
@@ -0,0 +1,72 @@
|
||||
# Data Model: Full Observability
|
||||
|
||||
No Prisma schema changes — every entity here is in-process or exported to an external
|
||||
observability sink, never persisted to Postgres.
|
||||
|
||||
## Request Context Store
|
||||
|
||||
`AsyncLocalStorage<RequestContextSnapshot>`, populated once per request in
|
||||
`request-context.plugin.ts`'s existing `onRequest` hook (the same hook that already builds
|
||||
`request.reqContext`), read by `logger.ts`'s Pino `mixin` function on every subsequent log call
|
||||
made anywhere during that request's handling.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `requestId` | `string` | Same value already assigned to `request.reqContext.requestId` |
|
||||
| `correlationId` | `string` | Same value already assigned to `request.reqContext.correlationId` |
|
||||
|
||||
## Access Log Line (shape, not a stored entity)
|
||||
|
||||
Emitted once per completed request via the existing `logger` singleton from the new
|
||||
`onResponse` hook.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `method` | `string` | HTTP method |
|
||||
| `route` | `string` | Parameterized route pattern (`request.routeOptions.url`), not the raw URL |
|
||||
| `statusCode` | `number` | Response status |
|
||||
| `durationMs` | `number` | `reply.elapsedTime` |
|
||||
| `requestId` / `correlationId` | `string` | Via the mixin, same as every other line for this request |
|
||||
| `event` | `string` | Fixed value `"http_request_completed"` — lets log queries filter to access-log lines specifically |
|
||||
|
||||
Log level: `info` for 2xx/3xx, `warn` for 4xx, `error` for 5xx — mirrors the existing
|
||||
error-handler's own level choices (`app.ts`) so severity is consistent across both sources of
|
||||
request-outcome logging.
|
||||
|
||||
## Metrics (Prometheus, via `prom-client`)
|
||||
|
||||
All registered in `infrastructure/observability/metrics.ts` on the existing default registry
|
||||
(`metricsRegistry`, already exposed at `GET /metrics`), all prefixed `supporthub_` to match the
|
||||
existing histogram and default-metrics prefix.
|
||||
|
||||
| Metric name | Type | Labels | Incremented/observed when |
|
||||
|---|---|---|---|
|
||||
| `supporthub_http_request_duration_seconds` | Histogram *(existing, now actually observed)* | `method`, `route`, `status_code` | Every completed HTTP request |
|
||||
| `supporthub_ai_session_outcomes_total` | Counter | `outcome` (`resolved` \| `escalated`) | An AI support session reaches a terminal `resolved`/`escalated` status |
|
||||
| `supporthub_ticket_resolutions_total` | Counter | `resolved_by` (`ai` \| `human`) | A ticket reaches `RESOLVED`, labeled from the ticket's `Resolution.resolvedBy` |
|
||||
| `supporthub_ticket_resolution_duration_seconds` | Histogram | — | A ticket reaches `RESOLVED` — observes `resolvedAt - ticket.createdAt` |
|
||||
| `supporthub_ticket_first_response_duration_seconds` | Histogram | — | The first `AGENT_MESSAGE` is posted on a ticket — observes `firstResponseAt - ticket.createdAt` |
|
||||
| `supporthub_sla_run_outcomes_total` | Counter | `outcome` (`met` \| `breached`) | An SLA run completes on time (`met`) or is flagged by the breach sweep (`breached`) |
|
||||
| `supporthub_escalations_total` | Counter | `reason` | An `ESCALATION_TRIGGERED` domain event fires (already published unconditionally today) |
|
||||
| `supporthub_problems_created_total` | Counter | `category_id` (or `uncategorized`) | A `Problem` row is created (at ticket-intake time) |
|
||||
| `supporthub_known_error_lookups_total` | Counter | `code` | A valid error code's known issues are looked up |
|
||||
| `supporthub_knowledge_retrieval_outcomes_total` | Counter | `matched` (`true` \| `false`) | The AI's `searchProductKnowledge` tool call returns zero vs. one-or-more results |
|
||||
| `supporthub_tool_invocations_total` | Counter | `tool`, `outcome` (`success` \| `failed`) | Every AI tool-call result, any tool |
|
||||
|
||||
Deliberately **not** separate metrics (per spec.md's Assumptions): "recurring problems" and
|
||||
"most common errors" are read directly off `supporthub_problems_created_total` and
|
||||
`supporthub_known_error_lookups_total` respectively via a monitoring stack's own `topk`/`rate`
|
||||
query — no additional "top N" metric or logic is computed by this application.
|
||||
|
||||
## Traces / Spans (exported, not persisted)
|
||||
|
||||
| Span | Parent | Attributes | Created in |
|
||||
|---|---|---|---|
|
||||
| `ticket.create` | (root) | `ticket.id`, `product.externalProductId` | `ticketing/tickets/service/tickets.service.ts` |
|
||||
| `ai.escalation` | `ticket.create` (if within the same request) or its own root (async paths) | `ticket.id`, `session.id` | `ai-support/sessions/service/session.service.ts`, around the escalation branch |
|
||||
| `orchestration.assignment` | `ai.escalation` (via the `TICKET_UPDATED`/`HUMAN_ESCALATION` subscriber) | `ticket.id`, `strategy` | `orchestration/orchestration` + `orchestration/assignments`, wrapping the existing `handleHumanEscalation` call |
|
||||
|
||||
Span context propagation across the domain-event bus relies on the OpenTelemetry Context API's
|
||||
own async-local propagation — since `eventBus.publish(...)` is `await`ed synchronously within
|
||||
the same call chain (confirmed in `tickets.service.ts`/`escalation.service.ts`), no manual
|
||||
context-carrying payload field is needed.
|
||||
@@ -0,0 +1,152 @@
|
||||
# Implementation Plan: Full Observability
|
||||
|
||||
**Branch**: `014-full-observability` | **Date**: 2026-09-07 | **Spec**: [spec.md](./spec.md)
|
||||
|
||||
**Input**: Feature specification from `specs/014-full-observability/spec.md`
|
||||
|
||||
## Summary
|
||||
|
||||
Wires three already-scaffolded-but-inert observability primitives into something real: a
|
||||
per-request structured access log (none exists today — Fastify's own request logging is fully
|
||||
disabled), the existing-but-never-observed request-duration histogram, and a real OpenTelemetry
|
||||
tracer provider behind the existing-but-never-called `getTracer()` helper. Adds eleven live
|
||||
Prometheus counters/histograms for the business-health metrics `docs/09-testing-observability-
|
||||
cicd.md` names, each wired at one existing choke point per metric (an event-bus subscriber where
|
||||
one already exists for the transition, a single already-existing method otherwise) rather than
|
||||
scattered across every call site. No new endpoints, no schema changes, no `supporthub-web` work
|
||||
— see research.md for the exact hook point chosen for each of the fourteen instrumentation
|
||||
targets (3 infra + 11 named metrics) and why.
|
||||
|
||||
## Technical Context
|
||||
|
||||
**Language/Version**: TypeScript 5.4 / Node.js 20+ (unchanged).
|
||||
|
||||
**Primary Dependencies**: New — `@opentelemetry/exporter-trace-otlp-http` (OTLP/HTTP span
|
||||
export), `@opentelemetry/resources` (service-name resource attribute). Reused, already
|
||||
installed — `@opentelemetry/api`, `@opentelemetry/sdk-trace-base` (provider, processors, and
|
||||
both the console and in-memory exporters used here all come from this one package), `prom-client`,
|
||||
`pino`. Reused Node built-in — `async_hooks`' `AsyncLocalStorage`.
|
||||
|
||||
**Storage**: No schema change. All new state is either in-process (Prometheus metric registry,
|
||||
the ALS request-context store, the tracer provider) or exported to wherever tracing is
|
||||
configured to send it — no new Postgres/Redis reads or writes beyond a handful of existing-table
|
||||
lookups already needed to label a metric correctly (e.g. `resolutionRepository.findByTicketId`
|
||||
to distinguish AI vs. human resolution).
|
||||
|
||||
**Testing**: Vitest — unit tests for the ALS-based logger mixin (a log call inside a request
|
||||
context carries requestId/correlationId; one outside carries neither) and for the
|
||||
SLA-compliance metric's "don't double-count an already-breached run as met" guard. Integration
|
||||
tests against real Postgres/Redis for: the access-log line's presence/shape (captured via a
|
||||
`logger.info` spy, same technique as 013's password-reset test), `/metrics` scraped before/after
|
||||
real traffic showing the duration histogram and each of the eleven business counters/histograms
|
||||
change by the expected amount when their real underlying event is driven through the real API,
|
||||
and a real multi-span trace (read back from the test-environment `InMemorySpanExporter`) for the
|
||||
two named cross-module paths.
|
||||
|
||||
**Target Platform**: Same Fastify modular monolith. Modifies
|
||||
`infrastructure/observability/*` (logger, metrics, tracing, a new request-context store) and
|
||||
`plugins/request-context.plugin.ts` (the new `onResponse` hook); adds small, single-call-site
|
||||
instrumentation lines inside `ai-support/sessions`, `ai-support/knowledge`, `ai-support/tools`,
|
||||
`ticketing/tickets`, `ticketing/messages`, `orchestration/sla`, and a handful of new subscribers
|
||||
in `src/events/handlers/index.ts`. No module gains a new public export surface beyond what
|
||||
`getTracer()` already exposed.
|
||||
|
||||
**Project Type**: Backend service — single project.
|
||||
|
||||
**Performance Goals**: The `onResponse` hook adds one Pino log call and one histogram `.observe`
|
||||
per request — both already-paid-for infrastructure (the logger and the metric object already
|
||||
exist), no new I/O on the request hot path. Trace export runs via `BatchSpanProcessor` (out of
|
||||
the request's own async chain) so span export latency never adds to response time. Metric
|
||||
increments at the eleven business hook points are in-memory counter operations, not database
|
||||
writes — the handful of read lookups needed for correct labeling (e.g. the resolution lookup for
|
||||
#3/#4) are single-row, already-indexed reads on tables these modules already query routinely.
|
||||
|
||||
**Constraints**: FR-007 — tracing must degrade gracefully; the API must start and serve traffic
|
||||
normally with no collector configured or reachable. FR-009 — no new human-facing endpoint,
|
||||
dashboard, or aggregation logic; every FR-008 metric is a raw counter/histogram for an external
|
||||
scraper, full stop. FR-010 — `/health*` and the existing histogram's shape on `/metrics` must
|
||||
not change for any existing consumer (only new metrics are added, nothing existing is renamed or
|
||||
removed).
|
||||
|
||||
**Scale/Scope**: Zero new routes. Three modified observability infrastructure files plus one new
|
||||
request-context store. Eleven new metric definitions plus their one-choke-point instrumentation
|
||||
call each. Two new dependencies. No schema migration, no new module.
|
||||
|
||||
## Constitution Check
|
||||
|
||||
*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.*
|
||||
|
||||
| Principle / Section | Check | Result |
|
||||
|---|---|---|
|
||||
| I. SaaS Is the Sole Identity & Access Authority | Not applicable — no identity/access surface touched. | PASS — N/A |
|
||||
| II. Configuration Over Hardcoding | The tracing exporter destination (`OTEL_EXPORTER_OTLP_ENDPOINT`) is env-driven, not hardcoded per environment; no business policy value is introduced by this feature (no SLA/routing/threshold numbers). | PASS |
|
||||
| III. Layered Architecture With Enforced Module Boundaries | No new module; existing module boundaries unchanged (each metric's instrumentation call lives inside the module that already owns the event, per research.md's per-metric table). The two repository-layer instrumentation calls (#1/#2, AI session status) are a deliberate, disclosed exception — see research.md §5's justification: observability calls are already a cross-cutting concern used from any layer in this codebase (e.g. `logger.error` inside `tool-executor.ts`), not the kind of business-logic leakage this principle targets. | PASS |
|
||||
| IV. AI Recommends, Deterministic Policy Decides | Not applicable — no AI decision logic changed, only observation of its outcomes. | PASS — N/A |
|
||||
| V. Evidence-Based Verification | Not applicable. | PASS — N/A |
|
||||
| VI. Durable Audit & History | Directly implements this principle's own stated requirement — "every log line MUST carry a request ID/correlation ID" is written in the constitution today but not actually true until this feature (FR-001/FR-002). | PASS — this feature closes a pre-existing constitutional gap |
|
||||
| VII. Concurrency-Safe, Durable Job Handling | The first-response-time metric (#5) has a benign, disclosed race (two concurrent first `AGENT_MESSAGE`s could both read "zero prior messages" and both observe) — acceptable because it is a best-effort observability metric, not the assignment/SLA correctness this principle is protecting; no persisted state or business decision depends on it. | PASS |
|
||||
| VIII. Problem and Ticket Are Separate, Related Entities | Not applicable — no model change. | PASS — N/A |
|
||||
| Technology & Platform Constraints | Two new dependencies (both OpenTelemetry, both already in the stack's declared technology list — "OpenAPI" aside, tracing itself was always part of the stated stack via the pre-existing `@opentelemetry/api`/`sdk-trace-base` dependencies) — no new infrastructure category introduced. | PASS |
|
||||
|
||||
No violations requiring Complexity Tracking justification.
|
||||
|
||||
## Project Structure
|
||||
|
||||
### Documentation (this feature)
|
||||
|
||||
```text
|
||||
specs/014-full-observability/
|
||||
├── plan.md
|
||||
├── research.md
|
||||
├── data-model.md
|
||||
├── quickstart.md
|
||||
├── contracts/
|
||||
│ └── metrics-contract.md
|
||||
└── tasks.md
|
||||
```
|
||||
|
||||
### Source Code (repository root)
|
||||
|
||||
```text
|
||||
supporthub-api/
|
||||
├── src/
|
||||
│ ├── infrastructure/
|
||||
│ │ └── observability/
|
||||
│ │ ├── logger.ts # MODIFIED — mixin reads the new ALS store
|
||||
│ │ ├── metrics.ts # MODIFIED — 11 new Counter/Histogram definitions
|
||||
│ │ ├── tracing.ts # MODIFIED — real provider init, exporter selection
|
||||
│ │ └── request-context.store.ts # NEW — AsyncLocalStorage<RequestContext>
|
||||
│ ├── plugins/
|
||||
│ │ └── request-context.plugin.ts # MODIFIED — onResponse access-log + histogram hook,
|
||||
│ │ onRequest now runs the rest of the request
|
||||
│ │ inside the ALS store
|
||||
│ ├── events/
|
||||
│ │ └── handlers/index.ts # MODIFIED — 3 new subscribers (human-resolution +
|
||||
│ │ resolution-time on TICKET_UPDATED/RESOLVED,
|
||||
│ │ escalation-rate on ESCALATION_TRIGGERED)
|
||||
│ └── modules/
|
||||
│ ├── ai-support/
|
||||
│ │ ├── sessions/repository/session.repository.ts # MODIFIED — AI resolution/escalation
|
||||
│ │ ├── knowledge/service/error-codes.service.ts # MODIFIED — most-common-errors
|
||||
│ │ └── tools/service/tools.service.ts # MODIFIED — tool-failure + knowledge-
|
||||
│ │ effectiveness
|
||||
│ ├── ticketing/
|
||||
│ │ ├── tickets/service/tickets.service.ts # MODIFIED — recurring-problems, plus
|
||||
│ │ │ the two named trace spans
|
||||
│ │ └── messages/service/messages.service.ts # MODIFIED — first-response-time
|
||||
│ └── orchestration/
|
||||
│ └── sla/service/sla.service.ts # MODIFIED — SLA-compliance
|
||||
└── tests/
|
||||
├── unit/observability/ # ALS mixin, SLA-compliance double-count guard
|
||||
└── integration/observability/ # access log, /metrics scrape assertions (11 metrics
|
||||
+ duration histogram), cross-module trace
|
||||
```
|
||||
|
||||
**Structure Decision**: Single project, no new module. All changes are surgical additions inside
|
||||
`infrastructure/observability` (the module that already owns this concern) plus one small,
|
||||
justified instrumentation line inside each of six existing business modules, following the
|
||||
per-metric hook points research.md already identified against the real, current code.
|
||||
|
||||
## Complexity Tracking
|
||||
|
||||
*No constitution violations — table intentionally omitted.*
|
||||
@@ -0,0 +1,70 @@
|
||||
# Quickstart: Full Observability
|
||||
|
||||
Manual verification steps for each user story, against a running instance backed by real
|
||||
Postgres/Redis (the throwaway Docker containers already used throughout this project's test
|
||||
suite work equally well for a manual run).
|
||||
|
||||
## Scenario 1 — Per-request access log (User Story 1)
|
||||
|
||||
1. Start the API. Send any request (e.g. `GET /health`).
|
||||
2. **Expected**: exactly one log line appears with `event: "http_request_completed"`, the
|
||||
request's method, route, status code, and a `requestId`.
|
||||
3. Send a request to a route that triggers additional internal logging (e.g. a login attempt).
|
||||
4. **Expected**: every log line produced while handling that request — the access-log line and
|
||||
any domain log lines — carries the same `requestId`/`correlationId`.
|
||||
5. Send a request to a route that doesn't exist.
|
||||
6. **Expected**: a 404 access-log line is still emitted (not silently dropped).
|
||||
|
||||
## Scenario 2 — Live request-health metrics (User Story 2)
|
||||
|
||||
1. Send a mix of successful and failing requests (e.g. a valid login, then three wrong-password
|
||||
logins).
|
||||
2. Scrape `GET /metrics`.
|
||||
3. **Expected**: `supporthub_http_request_duration_seconds_count` has observations labeled
|
||||
`route="/auth/login"` with both `status_code="200"` and `status_code="401"` present, letting
|
||||
an operator compute the error rate for that route from these two series alone.
|
||||
|
||||
## Scenario 3 — Cross-module trace (User Story 3)
|
||||
|
||||
1. With the API running in a mode where tracing exports to the console (no
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` configured), drive a request that escalates a ticket to a human
|
||||
and triggers automatic orchestration/assignment.
|
||||
2. **Expected**: console output shows a `ticket.create`-or-`ai.escalation` root span and an
|
||||
`orchestration.assignment` child span sharing the same trace ID, with the child's start time
|
||||
at or after the parent's.
|
||||
3. Stop the (nonexistent) collector / leave `OTEL_EXPORTER_OTLP_ENDPOINT` pointed at an
|
||||
unreachable address.
|
||||
4. **Expected**: the API still starts and serves requests normally; only a logged export-failure
|
||||
warning appears, nothing surfaces to any HTTP response.
|
||||
|
||||
## Scenario 4 — Business-health metrics (User Story 4)
|
||||
|
||||
For each metric, scrape `/metrics`, note the current value, drive the real event, scrape again,
|
||||
and confirm the expected series moved by exactly one (or by the expected duration observation):
|
||||
|
||||
1. Complete an AI session without escalating → `supporthub_ai_session_outcomes_total{outcome="resolved"}` +1.
|
||||
2. Complete an AI session that escalates, then have a human agent resolve the ticket →
|
||||
`supporthub_ai_session_outcomes_total{outcome="escalated"}` +1, and once resolved,
|
||||
`supporthub_ticket_resolutions_total{resolved_by="human"}` +1.
|
||||
3. Resolve any ticket → `supporthub_ticket_resolution_duration_seconds` gains one new observation.
|
||||
4. Post the first agent reply on a ticket → `supporthub_ticket_first_response_duration_seconds`
|
||||
gains one new observation.
|
||||
5. Let an SLA run complete on time, and separately let one breach (via the existing breach-sweep
|
||||
test helper) → `supporthub_sla_run_outcomes_total{outcome="met"}` and
|
||||
`{outcome="breached"}` each +1 respectively.
|
||||
6. Trigger an escalation → `supporthub_escalations_total{reason="<the actual reason>"}` +1.
|
||||
7. Create a ticket for a categorized problem →
|
||||
`supporthub_problems_created_total{category_id="<id>"}` +1.
|
||||
8. Look up a valid error code's known issues →
|
||||
`supporthub_known_error_lookups_total{code="<code>"}` +1.
|
||||
9. Have the AI's `searchProductKnowledge` tool return zero results, then results →
|
||||
`supporthub_knowledge_retrieval_outcomes_total{matched="false"}` then `{matched="true"}`,
|
||||
each +1 in turn.
|
||||
10. Have any AI tool invocation fail → `supporthub_tool_invocations_total{tool="<name>",
|
||||
outcome="failed"}` +1.
|
||||
|
||||
## What "done" looks like
|
||||
|
||||
All four scenarios pass against a real Postgres/Redis, `/health*` and the existing
|
||||
`supporthub_http_request_duration_seconds` metric's shape are unchanged for any existing
|
||||
consumer, and the API starts and serves traffic normally with no tracing collector configured.
|
||||
@@ -0,0 +1,172 @@
|
||||
# Research: Full Observability
|
||||
|
||||
All decisions below were made against the actual current code (grep/read), not assumption —
|
||||
several existing pieces (the histogram, `getTracer()`) are dead scaffolding that looked complete
|
||||
from their exports alone but do nothing today.
|
||||
|
||||
## 1. Per-request access log
|
||||
|
||||
**Decision**: Add an `onResponse` hook (Fastify fires this for every completed response,
|
||||
including 404s and early replies from other hooks like the rate limiter, satisfying the FR-001
|
||||
edge case) that logs one line via the existing `logger` singleton: `{method, route, statusCode,
|
||||
durationMs, requestId, correlationId}`. `route` uses `request.routeOptions.url` (the
|
||||
parameterized pattern, e.g. `/tickets/:id`) rather than `request.url`, to keep label/log
|
||||
cardinality bounded — the raw URL contains IDs. `reply.elapsedTime` (Fastify's own built-in
|
||||
per-request timer) supplies duration with no manual `Date.now()` bookkeeping.
|
||||
|
||||
**Why not Fastify's built-in request logger**: `app.ts` deliberately sets `logger: false` and
|
||||
routes all logging through the shared Pino `logger` singleton (see its own comment: "Managed
|
||||
centrally via Pino logger instance"). Re-enabling Fastify's built-in logger would mean two
|
||||
independent logging paths with two different configurations; a hook that calls the existing
|
||||
singleton keeps one path.
|
||||
|
||||
**Where**: `request-context.plugin.ts` already owns the per-request lifecycle (it's the one
|
||||
place with an `onRequest` hook establishing `reqContext`) — its `onResponse` counterpart is
|
||||
added in the same file, not a new plugin, so request-lifecycle logging concerns stay together.
|
||||
|
||||
## 2. Attaching request ID/correlation ID to every log line (FR-002)
|
||||
|
||||
**Decision**: `AsyncLocalStorage<RequestContext>`, populated in the same `onRequest` hook that
|
||||
already builds `reqContext`, combined with Pino's `mixin` option (a function called for every
|
||||
log line, merging its return value into that line) reading from the store. This makes every
|
||||
call through the existing shared `logger` singleton automatically carry `requestId`/
|
||||
`correlationId` with **zero changes to any existing call site** — dozens of `logger.info/warn/
|
||||
error(...)` calls across every module already pass ad hoc fields but not always `requestId`
|
||||
consistently.
|
||||
|
||||
**Why not `request.log`**: Fastify's per-request child logger (`request.log`) is the standard
|
||||
Fastify idiom for this, but it would require passing `request` (or `request.log`) into every
|
||||
service/repository/mapper that currently imports the plain `logger` singleton directly — a
|
||||
sweeping, high-risk refactor across nearly every module for a feature whose whole point is
|
||||
*reducing* risk. The ALS+mixin approach reaches the same outcome (every log line correlated)
|
||||
without touching a single existing call site.
|
||||
|
||||
**Merge order**: Pino applies `mixin()`'s fields before merging the call's own object, so an
|
||||
explicit `requestId` passed at a call site (several already do this manually, e.g.
|
||||
`app.ts`'s error handler) still wins — no behavior change for those call sites, just now
|
||||
redundant (harmless).
|
||||
|
||||
## 3. Request-duration histogram + request-count
|
||||
|
||||
**Decision**: `httpRequestDurationHistogram.observe({method, route, status_code},
|
||||
reply.elapsedTime / 1000)` in the same `onResponse` hook. Prometheus histograms automatically
|
||||
expose a `<name>_count` and `<name>_sum` per label combination — FR-004's "compute error rate
|
||||
per route/status" is satisfied by that built-in output; no separate counter metric is added, to
|
||||
avoid two metrics tracking overlapping information.
|
||||
|
||||
## 4. Distributed tracing
|
||||
|
||||
**Decision**: Initialize a real `BasicTracerProvider` (from the already-installed
|
||||
`@opentelemetry/sdk-trace-base` — no new dependency for the SDK itself) at process start, with
|
||||
`trace.setGlobalTracerProvider(...)` so the existing, previously-inert `getTracer()` helper
|
||||
starts returning a working tracer with zero change to its own signature. Exporter selection is
|
||||
config-driven (`OTEL_EXPORTER_OTLP_ENDPOINT`, following the OpenTelemetry project's own standard
|
||||
env var name rather than inventing a new one):
|
||||
|
||||
- Set → `OTLPTraceExporter` (new dependency: `@opentelemetry/exporter-trace-otlp-http`, the
|
||||
lighter HTTP/JSON variant, avoiding the gRPC exporter's heavier dependency footprint), wrapped
|
||||
in a `BatchSpanProcessor`.
|
||||
- Unset (local dev, and any environment that hasn't configured a collector) →
|
||||
`ConsoleSpanExporter` (part of `sdk-trace-base`, zero extra dependency) wrapped in a
|
||||
`SimpleSpanProcessor`, so spans are visible immediately without standing up a collector.
|
||||
- Test environment → `InMemorySpanExporter` (also part of `sdk-trace-base`, built specifically
|
||||
for tests) wrapped in a `SimpleSpanProcessor` — this lets integration tests assert on real,
|
||||
actually-exported span data (names, parent/child nesting, attributes) with a real
|
||||
`TracerProvider` doing real work, the only substitution is *where the spans end up*, the same
|
||||
"real infrastructure, substitute only the destination" pattern already used for Pino's
|
||||
transport (`pino-pretty` in development, plain JSON otherwise).
|
||||
|
||||
**New dependencies**: `@opentelemetry/exporter-trace-otlp-http`, `@opentelemetry/resources` (for
|
||||
a `service.name: supporthub-api` resource attribute — without it, every span is anonymous in
|
||||
whatever backend receives them).
|
||||
|
||||
**Graceful degradation (FR-007)**: `BatchSpanProcessor`'s own export failures are caught and
|
||||
logged by the OpenTelemetry SDK internally (it never throws into application code); nothing in
|
||||
this feature needs to add its own try/catch around span creation for this to hold, but the SDK's
|
||||
internal diagnostic logger is wired to `logger.warn` (via `diag.setLogger`) so export failures
|
||||
are visible in this project's own log stream rather than swallowed silently.
|
||||
|
||||
**Where spans are added (FR-006)**: two entry points, wrapping already-existing method calls
|
||||
rather than restructuring them:
|
||||
- `ai-support/sessions/service/session.service.ts`'s escalation path — a span around the call
|
||||
that ultimately triggers `orchestrationService.handleHumanEscalation` (via the
|
||||
`TICKET_UPDATED` → `HUMAN_ESCALATION` domain-event subscriber in
|
||||
`src/events/handlers/index.ts`), and a child span inside
|
||||
`orchestration/orchestration`'s and `orchestration/assignments`'s own handling — showing the
|
||||
AI-diagnosis → escalation → assignment path as one connected trace.
|
||||
- `ticketing/tickets/service/tickets.service.ts`'s ticket-creation method — a root span for
|
||||
ticket intake, with the domain-event-driven downstream reactions (SLA-run creation, etc.)
|
||||
as child spans, per FR-006's second named path.
|
||||
|
||||
Trace context propagates across the event-bus's synchronous `await eventBus.publish(...)` calls
|
||||
for free (both publisher and subscriber run within the same Node async-context chain the OTel
|
||||
context API rides on — no manual context passing needed, since nothing here crosses a process/
|
||||
queue boundary; BullMQ jobs are explicitly out of scope for this feature's two named paths).
|
||||
|
||||
## 5. The eleven named business-health metrics (FR-008) — instrumentation points
|
||||
|
||||
Each is a `prom-client` `Counter` or `Histogram`, registered once in
|
||||
`infrastructure/observability/metrics.ts` alongside the existing histogram, and incremented/
|
||||
observed at one single already-existing choke point per metric — chosen specifically to avoid
|
||||
scattering an instrumentation call across every one of a metric's several call sites.
|
||||
|
||||
| # | Metric | Type | Hook point (file : method) | Label(s) |
|
||||
|---|---|---|---|---|
|
||||
| 1 | AI resolution rate | Counter | `ai-support/sessions/repository/session.repository.ts` : `updateStatus`, when `status === 'resolved'` | — |
|
||||
| 2 | AI escalation rate | Counter | same method, when `status === 'escalated'` | — |
|
||||
| 3 | Human resolution rate | Counter | new `TICKET_UPDATED` subscriber (`events/handlers/index.ts`) on `newStatus === 'RESOLVED'`, looking up `resolutionRepository.findByTicketId` for `resolvedBy` | `resolvedBy !== 'ai'` only |
|
||||
| 4 | Average resolution time | Histogram | same subscriber — observes `resolvedAt - ticket.createdAt` | — |
|
||||
| 5 | First response time | Histogram | `ticketing/messages/service/messages.service.ts` : `post`, when `type === 'AGENT_MESSAGE'` and no prior `AGENT_MESSAGE` exists for the ticket | — |
|
||||
| 6 | SLA compliance | Counter | `orchestration/sla/service/sla.service.ts` : `complete` (outcome `met`, only if the run wasn't already `breached`) and `runBreachDetectionSweep` (outcome `breached`) | `outcome` |
|
||||
| 7 | Escalation rate | Counter | new `ESCALATION_TRIGGERED` subscriber (`events/handlers/index.ts`) — this event is already published unconditionally on every escalation (`escalation.service.ts`) but "for audit, not for logic" (its own comment) and has zero subscribers today | `reason` |
|
||||
| 8 | Recurring problems | Counter | `ticketing/tickets/service/tickets.service.ts` — the ticket-creation method's existing `problemsRepo.create(...)` call | `categoryId` (or `uncategorized`) |
|
||||
| 9 | Most common errors | Counter | `ai-support/knowledge/service/error-codes.service.ts` : `findKnownIssuesByErrorCode`, after a valid code is confirmed to exist | `code` |
|
||||
| 10 | Knowledge effectiveness | Counter | `ai-support/tools/service/tools.service.ts`'s single `executeTool(...)` call site, when `block.name === 'searchProductKnowledge'` | `matched` (results non-empty vs empty) |
|
||||
| 11 | Tool failure rate | Counter | same call site, every tool invocation | `tool`, `outcome` |
|
||||
|
||||
**Why the event bus for #3, #4, #7 instead of editing `resolutions.service.ts`/
|
||||
`escalation.service.ts` directly**: those two modules' domain events (`TICKET_UPDATED` with
|
||||
`newStatus`, and `ESCALATION_TRIGGERED`) are already published unconditionally for every
|
||||
relevant transition (confirmed by reading `tickets.service.ts` and `escalation.service.ts`
|
||||
directly) specifically so that a new concern reacting to "a ticket resolved" or "an escalation
|
||||
happened" never needs to modify the module that owns the transition — the exact precedent
|
||||
`src/events/handlers/index.ts`'s existing four subscribers already establish for 005/007/008.
|
||||
Metrics is exactly this kind of concern.
|
||||
|
||||
**Why the repository layer for #1/#2 instead of the event bus**: AI-session resolved/escalated
|
||||
is not currently published as a domain event at all (only ticket-level and escalation-level
|
||||
events exist) and `session.service.ts` calls `this.sessions.updateStatus(...)` from ten
|
||||
different branches — adding a domain-event publish there to reuse the event-bus pattern would
|
||||
mean either introducing a new event type used by exactly one subscriber (this feature) or
|
||||
touching all ten call sites to route through a new shared wrapper. Instrumenting the one
|
||||
repository method both approaches would have to fire through instead is the minimal, lowest-risk
|
||||
option. This mirrors how `logger` calls already appear directly inside repository/service code
|
||||
throughout this codebase (e.g. `tool-executor.ts`'s `logger.error`) — observability calls are
|
||||
already treated as a cross-cutting concern usable from any layer, not something Constitution
|
||||
Principle III's "repository is Prisma-only" rule was written to police (that rule targets
|
||||
business-logic leakage and direct Prisma access from the wrong layer, not a metrics increment
|
||||
alongside an existing Prisma call).
|
||||
|
||||
**A pre-existing correctness note surfaced while researching #6**: `sla.service.ts`'s
|
||||
`complete()` only skips its update when the run is *already* `'completed'` — not when it is
|
||||
`'breached'` — so a run that breached and then later resolved would have its `status`
|
||||
overwritten from `'breached'` back to `'completed'` in the database, silently losing the breach
|
||||
record. This is a pre-existing 008/012 behavior, not something this feature changes (the SLA
|
||||
run's persisted status is out of scope for an observability feature) — the metric itself reads
|
||||
`run.status` *before* calling `complete()`'s own update, so the metric is accurate (correctly
|
||||
counted as `breached`, never double-counted as `met`) regardless of this separate, pre-existing
|
||||
data-quality gap. Documented in this feature's own checklist Notes as a discovered issue for a
|
||||
future fix, the same way 013-auth-hardening documented the `orchestration-strategies.test.ts`
|
||||
bug it found without fixing it.
|
||||
|
||||
## 6. Test strategy for the eleven metrics and tracing
|
||||
|
||||
**Decision**: Integration tests scrape the real `/metrics` endpoint's text output (a real
|
||||
`app.inject({method: 'GET', url: '/metrics'})` call, no mocking) before and after driving the
|
||||
real underlying event through the real API (create a ticket, resolve an AI session, trigger an
|
||||
escalation, etc. — exactly as every prior feature's integration suite already does against real
|
||||
Postgres/Redis), asserting the specific metric line's value increased by the expected amount.
|
||||
Tracing is verified by reading back spans from the `InMemorySpanExporter` (test-environment
|
||||
exporter, per §4) after a real cross-module request, asserting span names and parent/child
|
||||
`spanId`/`parentSpanId` relationships — a real trace, produced by a real `TracerProvider`, just
|
||||
captured in memory instead of shipped to a collector.
|
||||
@@ -0,0 +1,222 @@
|
||||
---
|
||||
description: 'Task list for 014-full-observability'
|
||||
---
|
||||
|
||||
# Tasks: Full Observability
|
||||
|
||||
**Input**: Design documents from `specs/014-full-observability/`
|
||||
|
||||
**Prerequisites**: [plan.md](./plan.md), [spec.md](./spec.md), [research.md](./research.md),
|
||||
[data-model.md](./data-model.md),
|
||||
[contracts/metrics-contract.md](./contracts/metrics-contract.md), [quickstart.md](./quickstart.md)
|
||||
|
||||
**Organization**: Tasks are grouped by user story (US1 = P1 access log, US2 = P1 request-health
|
||||
metrics, US3 = P2 tracing, US4 = P2 business-health metrics). US2 shares its hook point with
|
||||
US1 (both live in the same `onResponse` hook) so US2 depends on US1's hook existing, not on its
|
||||
own separate one. US3 and US4 are each independent of US1/US2 and of each other.
|
||||
|
||||
## Format: `[ID] [P?] [Story] Description`
|
||||
|
||||
All file paths are relative to `supporthub-api/` (repo root).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Foundational (Blocking Prerequisites)
|
||||
|
||||
- [x] T001 Add `@opentelemetry/exporter-trace-otlp-http` and `@opentelemetry/resources` to
|
||||
`package.json` (`npm install`)
|
||||
- [x] T002 Add `src/infrastructure/observability/request-context.store.ts` — a module-level
|
||||
`AsyncLocalStorage<{requestId: string; correlationId: string}>` with a `run()` passthrough
|
||||
and a `getStore()` re-export
|
||||
- [x] T003 [P] Wire `logger.ts`'s Pino options with a `mixin` function reading from T002's store
|
||||
(returns `{}` when no store is active — a log call outside any request, e.g. at startup,
|
||||
must not throw) (depends on T002)
|
||||
|
||||
**Checkpoint**: Every subsequent log call through the shared `logger` singleton is
|
||||
request-correlated automatically, once a request actually runs inside the store (US1 wires that
|
||||
part next).
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: User Story 1 - Trace one request end to end from its logs (Priority: P1)
|
||||
|
||||
**Goal**: One structured access-log line per request; every other log line produced during that
|
||||
request's handling shares its request ID.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 1.
|
||||
|
||||
### Tests for User Story 1
|
||||
|
||||
- [x] T004 [P] [US1] Unit test: a `logger.info(...)` call made inside T002's `store.run(...)`
|
||||
carries `requestId`/`correlationId` in its output; one made outside carries neither, in
|
||||
`tests/unit/observability/request-context-mixin.test.ts` (depends on T003)
|
||||
|
||||
### Implementation for User Story 1
|
||||
|
||||
- [x] T005 [US1] In `plugins/request-context.plugin.ts`'s existing `onRequest` hook, after
|
||||
building `request.reqContext`, call the T002 store's `run()` wrapping the remainder of the
|
||||
request's handling (Fastify's `onRequest` hooks accept a `done` callback / return a
|
||||
promise — the run wraps whichever style this hook currently uses) so every subsequent
|
||||
hook/handler for this request executes inside the ALS context (depends on T002)
|
||||
- [x] T006 [US1] Add an `onResponse` hook (same plugin) that logs one line via the shared
|
||||
`logger`: `{event: "http_request_completed", method, route: request.routeOptions.url,
|
||||
statusCode: reply.statusCode, durationMs: reply.elapsedTime}`, at `info`/`warn`/`error`
|
||||
level by status class (depends on T005)
|
||||
- [x] T007 [US1] Integration test covering Quickstart Scenario 1 (one access-log line per
|
||||
request incl. 404; shared requestId across the access-log line and an internal log line
|
||||
from the same request) in `tests/integration/observability/access-log.test.ts`, using a
|
||||
`logger.info`/`logger.warn` spy the same way `password-reset-flow.test.ts` (013) already
|
||||
does (depends on T006)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 1 passes. Every request is now visible in logs even when it
|
||||
never errors.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: User Story 2 - See live request-health metrics (Priority: P1)
|
||||
|
||||
**Goal**: The existing (previously dead) request-duration histogram actually has observations;
|
||||
error rate per route/status is computable from `/metrics` alone.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 2.
|
||||
|
||||
### Implementation for User Story 2
|
||||
|
||||
- [x] T008 [US2] In the same `onResponse` hook added by T006, call
|
||||
`httpRequestDurationHistogram.observe({method, route: request.routeOptions.url, status_code:
|
||||
String(reply.statusCode)}, reply.elapsedTime / 1000)` (depends on T006)
|
||||
- [x] T009 [US2] Integration test covering Quickstart Scenario 2 (send a mix of successful/
|
||||
failing requests to the same route, scrape `/metrics`, assert both status-code label
|
||||
values are present with the expected counts) in
|
||||
`tests/integration/observability/request-metrics.test.ts` (depends on T008)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 2 passes. `/metrics` now reflects real request traffic.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: User Story 3 - Trace a single incident's cross-module path (Priority: P2)
|
||||
|
||||
**Goal**: A real `TracerProvider` is active; `getTracer()` produces spans that are actually
|
||||
exported; two named cross-module paths are instrumented.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 3.
|
||||
|
||||
### Implementation for User Story 3
|
||||
|
||||
- [x] T010 [P] [US3] Rewrite `infrastructure/observability/tracing.ts` to initialize a
|
||||
`BasicTracerProvider` at module load with a `Resource` (`service.name: "supporthub-api"`)
|
||||
and register it via `trace.setGlobalTracerProvider(...)`; exporter/processor chosen by
|
||||
`NODE_ENV`/`OTEL_EXPORTER_OTLP_ENDPOINT` per research.md §4 (`InMemorySpanExporter` +
|
||||
`SimpleSpanProcessor` in test, `OTLPTraceExporter` + `BatchSpanProcessor` when the env var
|
||||
is set, `ConsoleSpanExporter` + `SimpleSpanProcessor` otherwise); export a
|
||||
`getTestSpanExporter()` accessor (test env only) for T015 to read exported spans back;
|
||||
`getTracer()`'s own exported signature is unchanged (depends on T001)
|
||||
- [x] T011 [US3] Wire the OpenTelemetry SDK's internal diagnostic logger
|
||||
(`diag.setLogger(...)`) to the shared `logger.warn`, so span-export failures land in this
|
||||
project's own log stream instead of stderr or nowhere (depends on T010)
|
||||
- [x] T012 [P] [US3] Add a `ticket.create` span (`getTracer().startActiveSpan(...)`) around
|
||||
`tickets/service/tickets.service.ts`'s ticket-creation method, with `ticket.id` and
|
||||
`product.externalProductId` attributes, ended in a `finally` (depends on T010)
|
||||
- [x] T013 [P] [US3] Add an `ai.escalation` span around `ai-support/sessions/service/
|
||||
session.service.ts`'s escalation branch(es), with `ticket.id`/`session.id` attributes
|
||||
(depends on T010)
|
||||
- [x] T014 [US3] Add an `orchestration.assignment` span wrapping the existing
|
||||
`orchestrationService.handleHumanEscalation` call in the `TICKET_UPDATED`/
|
||||
`HUMAN_ESCALATION` subscriber (`src/events/handlers/index.ts`), with `ticket.id`/
|
||||
`strategy` attributes, so it nests under T013's span when both occur in the same request
|
||||
(depends on T010, T013)
|
||||
- [x] T015 [US3] Integration test covering Quickstart Scenario 3 steps 1-2: drive a real
|
||||
ticket-creation → escalation → orchestration/assignment flow, read spans back via T010's
|
||||
`getTestSpanExporter()`, assert `ticket.create`/`ai.escalation`/`orchestration.assignment`
|
||||
all share one trace ID with correct parent/child `spanId` relationships, in
|
||||
`tests/integration/observability/tracing.test.ts` (depends on T012, T013, T014)
|
||||
- [x] T016 [US3] Integration test covering Quickstart Scenario 3 steps 3-4: point
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` at an unreachable address, confirm `buildApp()` still
|
||||
resolves and a request still completes successfully, in the same test file (depends on
|
||||
T010)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 3 passes. A real, inspectable trace exists for the first
|
||||
time; tracing failure never blocks the app.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: User Story 4 - See the business-health metrics this project committed to tracking (Priority: P2)
|
||||
|
||||
**Goal**: All eleven named metrics (data-model.md) are live on `/metrics`, each updated at the
|
||||
exact real event research.md identified.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 4.
|
||||
|
||||
### Implementation for User Story 4
|
||||
|
||||
- [x] T017 [US4] Define all eleven new `Counter`/`Histogram` objects in
|
||||
`infrastructure/observability/metrics.ts` per data-model.md's table, exported individually
|
||||
(depends on T001)
|
||||
- [x] T018 [P] [US4] Increment `supporthub_ai_session_outcomes_total` in `ai-support/sessions/
|
||||
repository/session.repository.ts`'s `updateStatus`, labeled `outcome` when `status` is
|
||||
`'resolved'`/`'escalated'` (depends on T017)
|
||||
- [x] T019 [P] [US4] Add a `TICKET_UPDATED`/`newStatus === 'RESOLVED'` subscriber in
|
||||
`src/events/handlers/index.ts` that looks up `resolutionRepository.findByTicketId`,
|
||||
increments `supporthub_ticket_resolutions_total{resolved_by}` (`ai` vs. any other value),
|
||||
fetches the ticket for `createdAt`, and observes
|
||||
`supporthub_ticket_resolution_duration_seconds` (depends on T017)
|
||||
- [x] T020 [P] [US4] In `ticketing/messages/service/messages.service.ts`'s `post`, when
|
||||
`type === 'AGENT_MESSAGE'`, check for a prior `AGENT_MESSAGE` on the ticket and — only for
|
||||
the first one — observe `supporthub_ticket_first_response_duration_seconds` against the
|
||||
ticket's `createdAt` (depends on T017)
|
||||
- [x] T021 [P] [US4] In `orchestration/sla/service/sla.service.ts`: in `complete()`, read
|
||||
`run.status` before updating and increment `supporthub_sla_run_outcomes_total{outcome:
|
||||
"met"}` only if it was not already `'breached'`; in `runBreachDetectionSweep()`, increment
|
||||
`{outcome: "breached"}` for each newly-flagged run (depends on T017)
|
||||
- [x] T022 [P] [US4] Add an `ESCALATION_TRIGGERED` subscriber in `src/events/handlers/index.ts`
|
||||
that increments `supporthub_escalations_total{reason}` from the event payload's `reason`
|
||||
(depends on T017)
|
||||
- [x] T023 [P] [US4] In `ticketing/tickets/service/tickets.service.ts`'s ticket-creation method,
|
||||
increment `supporthub_problems_created_total{category_id}` right after `problemsRepo.create`
|
||||
succeeds (`categoryId ?? 'uncategorized'`) (depends on T017)
|
||||
- [x] T024 [P] [US4] In `ai-support/knowledge/service/error-codes.service.ts`'s
|
||||
`findKnownIssuesByErrorCode`, increment `supporthub_known_error_lookups_total{code}` once
|
||||
the error code is confirmed to exist (after the `NotFoundError` branch, not before)
|
||||
(depends on T017)
|
||||
- [x] T025 [P] [US4] In `ai-support/tools/service/tools.service.ts`'s single `executeTool(...)`
|
||||
call site: increment `supporthub_tool_invocations_total{tool, outcome}` for every call, and
|
||||
— only when `block.name === 'searchProductKnowledge'` — increment
|
||||
`supporthub_knowledge_retrieval_outcomes_total{matched}` from whether `result.output` is a
|
||||
non-empty array (depends on T017)
|
||||
- [x] T026 [US4] Unit test: `sla.service.ts`'s `complete()` does not increment the `met` outcome
|
||||
for a run already `'breached'` (a fake repo returning `status: 'breached'`) in
|
||||
`tests/unit/observability/sla-compliance-metric.test.ts` (depends on T021)
|
||||
- [x] T027 [US4] Integration test covering Quickstart Scenario 4 (all ten sub-scenarios — the
|
||||
eleventh, tool-failure, is covered by the same test file's tool-invocation case) —
|
||||
scrape `/metrics` before/after driving each real event through the real API, in
|
||||
`tests/integration/observability/business-metrics.test.ts` (depends on T018, T019, T020,
|
||||
T021, T022, T023, T024, T025)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 4 passes. All eleven named metrics are live and correct
|
||||
against real infrastructure.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Polish & Cross-Cutting Concerns
|
||||
|
||||
- [x] T028 [P] Update `specs/014-full-observability/checklists/requirements.md` Notes with any
|
||||
implementation-time findings (including the pre-existing SLA-run status data-quality gap
|
||||
research.md §5 already surfaced)
|
||||
- [x] T029 Run `npx tsx scripts/check-architecture.ts` and `npm run lint`/`npm run typecheck`
|
||||
- [x] T030 Full regression: `npm run test:unit` then the full integration suite against real
|
||||
Docker-provisioned Postgres/Redis, confirming nothing outside this feature regressed
|
||||
(particularly every module touched by a single-line instrumentation addition: ai-support
|
||||
sessions/knowledge/tools, ticketing tickets/messages, orchestration/sla, and the event-bus
|
||||
handlers)
|
||||
|
||||
---
|
||||
|
||||
## Dependencies & Execution Order
|
||||
|
||||
- **Foundational (Phase 1)**: No dependencies — BLOCKS User Story 1 (and transitively 2)
|
||||
- **User Story 1 (Phase 2)**: Depends on Foundational — BLOCKS User Story 2 (shares its hook)
|
||||
- **User Story 2 (Phase 3)**: Depends on User Story 1
|
||||
- **User Story 3 (Phase 4)**: Depends only on Foundational (T001) — independent of US1/US2/US4
|
||||
- **User Story 4 (Phase 5)**: Depends only on Foundational (T001/T017) — independent of
|
||||
US1/US2/US3
|
||||
- **Polish (Phase 5)**: Depends on all four user stories
|
||||
@@ -0,0 +1,94 @@
|
||||
# Specification Quality Checklist: Reporting and Analytics Dashboards
|
||||
|
||||
**Purpose**: Validate specification completeness and quality before proceeding to planning
|
||||
**Created**: 2026-09-09
|
||||
**Feature**: [spec.md](../spec.md)
|
||||
|
||||
## Content Quality
|
||||
|
||||
- [x] No implementation details (languages, frameworks, APIs)
|
||||
- [x] Focused on user value and business needs
|
||||
- [x] Written for non-technical stakeholders
|
||||
- [x] All mandatory sections completed
|
||||
|
||||
## Requirement Completeness
|
||||
|
||||
- [x] No [NEEDS CLARIFICATION] markers remain
|
||||
- [x] Requirements are testable and unambiguous
|
||||
- [x] Success criteria are measurable
|
||||
- [x] Success criteria are technology-agnostic (no implementation details)
|
||||
- [x] All acceptance scenarios are defined
|
||||
- [x] Edge cases are identified
|
||||
- [x] Scope is clearly bounded
|
||||
- [x] Dependencies and assumptions identified
|
||||
|
||||
## Feature Readiness
|
||||
|
||||
- [x] All functional requirements have clear acceptance criteria
|
||||
- [x] User scenarios cover primary flows
|
||||
- [x] Feature meets measurable outcomes defined in Success Criteria
|
||||
- [x] No implementation details leak into specification
|
||||
|
||||
## Notes
|
||||
|
||||
- This is `docs/10-implementation-roadmap.md`'s own Phase 11, third sub-area, per explicit user
|
||||
direction (013 was the security pass, 014 was full observability). Backend-first scope
|
||||
(Assumptions) follows the same pattern already established three times this session
|
||||
(010-identity-auth, 011-agent-ticket-queue, and 014-full-observability's own frontend-free
|
||||
scope) — a `supporthub-web` dashboard UI is a natural, separate follow-on, not re-litigated
|
||||
here via a fresh question.
|
||||
- The pre-scaffolded-but-inert `platform/reports` module (`ReportsService.generateSummaryReport`
|
||||
currently returns `{}`) and the `ANALYTICS` queue stub (`src/jobs/analytics`, logs only) were
|
||||
both confirmed via direct code inspection before writing this spec — the same
|
||||
"provisioned before this session's rebuild but never wired up" pattern found repeatedly this
|
||||
session. This feature wires up the former; the Assumptions section explicitly keeps the latter
|
||||
out of scope (synchronous queries, no pre-aggregation job, for this first cut).
|
||||
- All items pass; no revision iterations were needed. No [NEEDS CLARIFICATION] markers were
|
||||
required — every open question (default date window, SLA-risk threshold, top-N limit) had a
|
||||
reasonable, documented, CONFIGURABLE default (see Assumptions), matching the roadmap's own
|
||||
"never hardcode a placeholder value and ship it as final" instruction.
|
||||
|
||||
## Implementation Notes (post-build)
|
||||
|
||||
- Named the Product dashboard's own repository class `ProductReportRepository` (not
|
||||
`ProductRepository`) once it became clear resolving `externalProductId -> Product` should
|
||||
reuse `catalog/products`' own already-public `productsRepository.findByExternalProductId`
|
||||
rather than duplicating that lookup — avoids a name collision and keeps "one authority per
|
||||
concern" (Constitution Principle I's spirit) for product resolution.
|
||||
- `ManagementRepository` and `SupportRepository` both needed byte-identical
|
||||
first-response-duration and resolution-duration queries. Extracted into a shared
|
||||
`SharedReportRepository` both compose, rather than duplicating the Prisma query (or the
|
||||
averaging helper alone) twice — discovered while writing the second repository and seeing the
|
||||
copy-paste, not planned upfront in research.md.
|
||||
- "Top errors"/"most common errors" resolution-back-to-`code` logic moved into
|
||||
`ErrorCodesService.getTopErrorCodesForProduct` (a new method on the module that already owns
|
||||
`ErrorCode`), rather than the reports module reaching into `errorCodesRepository`/
|
||||
`errorCodeLookupRepository` directly — cleaner module-boundary ownership than research.md's
|
||||
original per-repository sketch implied.
|
||||
- The AI dashboard's "failed troubleshooting then escalated" figure (spec.md User Story 4) has
|
||||
no single stored flag anywhere in this codebase — `classifyStepOutcome`'s per-step verdicts are
|
||||
never persisted as their own durable record. Implemented as a documented proxy instead: an
|
||||
escalated session with `toolCallCount > 0` attempted troubleshooting before giving up, one with
|
||||
zero attempts escalated immediately. Documented directly in `ai.repository.ts`'s own code
|
||||
comment, the same "honest, documented simplification" precedent research.md §7 already set for
|
||||
the confidence-distribution bucketing.
|
||||
- Three of this module's public exports needed adding to their owning modules' top-level
|
||||
`index.ts` (not previously exposed): `decideConfidenceBand`/`ConfidenceBand` and
|
||||
`knowledgeReferenceRepository` from `ai-support/sessions`, matching the "extend an existing
|
||||
module's public surface for a later feature" precedent already used repeatedly this session
|
||||
(004's `productsRepository`, 009's `problemsRepository`).
|
||||
- Found a real regression during T028's full regression pass: `known-issues.test.ts` (004-
|
||||
product-knowledge, pre-existing) calls `findKnownIssuesByErrorCode` and its own `afterAll`
|
||||
deleted `ErrorCode` rows before this feature's new `ErrorCodeLookup` FK (RESTRICT) existed —
|
||||
once T004 started writing a lookup row on every call, that cleanup order started failing with
|
||||
an FK violation. Fixed by deleting `ErrorCodeLookup` rows first in that test's own `afterAll`.
|
||||
This feature's own new test files never delete `ErrorCode` rows at all, so they weren't
|
||||
affected the same way (leftover rows there are the same accepted throwaway-data tradeoff
|
||||
already established elsewhere this session).
|
||||
- Confirmed (not caused by this feature — the exact pre-existing issue 014-full-observability's
|
||||
own checklist already documented and root-caused via `git checkout` comparison) that this
|
||||
feature's own new integration test files, which also name their test products `TEST_*`,
|
||||
occasionally hit the same shared `deriveProductCode` "TEST" prefix collision under vitest's
|
||||
concurrent file execution when run alongside other `TEST_*`-prefixed files. Every dashboard
|
||||
test passes reliably run individually or in small groups; the intermittent 500 in a full
|
||||
combined run is the same known, out-of-scope, 003-ticketing concern.
|
||||
@@ -0,0 +1,59 @@
|
||||
# Contract: Reporting API
|
||||
|
||||
All four routes require a valid staff session with role `ADMIN` (`requireRole('ADMIN')`), the
|
||||
same gate every admin-only surface uses since 010-identity-auth. All return the standard
|
||||
envelope: `{ success: true, data: <shape>, meta: null }` on success, `{ success: false, error:
|
||||
{code, message, details} }` on failure — no change to this codebase's existing response
|
||||
convention.
|
||||
|
||||
## `GET /admin/reports/management`
|
||||
|
||||
**Query**: `from?`, `to?` (ISO dates).
|
||||
|
||||
**200**: `ManagementDashboard` (data-model.md).
|
||||
|
||||
**400** `VALIDATION_ERROR`: `from` is after `to`.
|
||||
|
||||
**401/403**: missing/invalid session, or a non-`ADMIN` role.
|
||||
|
||||
## `GET /admin/reports/product/:externalProductId`
|
||||
|
||||
**Path**: `externalProductId` — the SaaS-facing product identifier (same convention every other
|
||||
admin product-scoped route already uses, e.g. `GET /admin/products/:externalProductId/knowledge`
|
||||
from 004-product-knowledge).
|
||||
|
||||
**Query**: `from?`, `to?`.
|
||||
|
||||
**200**: `ProductDashboard`.
|
||||
|
||||
**404** `NOT_FOUND`: no product with that `externalProductId` (FR-006 — never an empty-but-200
|
||||
response for an unknown product).
|
||||
|
||||
**400** `VALIDATION_ERROR`: `from` is after `to`.
|
||||
|
||||
## `GET /admin/reports/support`
|
||||
|
||||
**Query**: `from?`, `to?` (applies only to the performance figures — workload/SLA-risk/breached
|
||||
are always current, per data-model.md's `SupportDashboard.generatedAt`).
|
||||
|
||||
**200**: `SupportDashboard`.
|
||||
|
||||
## `GET /admin/reports/ai`
|
||||
|
||||
**Query**: `from?`, `to?`.
|
||||
|
||||
**200**: `AiDashboard`.
|
||||
|
||||
## Guarantees
|
||||
|
||||
1. Every rate/average field is `number | null` — `null` means no qualifying data existed in the
|
||||
requested range (FR-007). A consumer must never see `NaN` or a silently-substituted `0` for
|
||||
"no data."
|
||||
2. Every count field is a plain `number`, always present, `0` is a legitimate, meaningful value
|
||||
for a count (distinct from the `null`-for-no-data rule above, which applies only to
|
||||
rates/averages).
|
||||
3. `from`/`to` in every response echo the *resolved* range actually used (including the default,
|
||||
when omitted) — a caller never has to separately know what "the default" was.
|
||||
4. No route in this contract mutates any data — a repeated identical request returns the same
|
||||
shape (though not necessarily identical figures, since the underlying data can change between
|
||||
requests) with no side effect.
|
||||
@@ -0,0 +1,93 @@
|
||||
# Data Model: Reporting and Analytics Dashboards
|
||||
|
||||
## New Prisma Model
|
||||
|
||||
### `ErrorCodeLookup`
|
||||
|
||||
Append-only audit record — see research.md §6 for why this is the one new table this feature
|
||||
needs.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `id` | `String @id @default(cuid())` | |
|
||||
| `errorCodeId` | `String` | FK → `ErrorCode.id` |
|
||||
| `productId` | `String` | FK → `Product.id` — denormalized from `errorCode.productId` so the Product dashboard's range query never needs to join back through `ErrorCode` just to filter by product |
|
||||
| `createdAt` | `DateTime @default(now())` | |
|
||||
|
||||
Indexes: `@@index([productId, createdAt])` (the Product dashboard's own access pattern).
|
||||
|
||||
No `updatedAt`, no soft-delete, no unique constraint — every lookup is its own row, duplicates
|
||||
across time are the entire point (frequency is what "top errors" measures).
|
||||
|
||||
## Response Shapes (not persisted — computed per request)
|
||||
|
||||
### Management dashboard — `GET /admin/reports/management`
|
||||
|
||||
```ts
|
||||
interface ManagementDashboard {
|
||||
range: { from: string; to: string }; // ISO 8601, echoes the resolved (possibly defaulted) range
|
||||
totalCases: number;
|
||||
aiResolved: number;
|
||||
humanEscalated: number;
|
||||
resolved: number;
|
||||
open: number;
|
||||
slaCompliance: { met: number; breached: number; rate: number | null }; // rate = met / (met + breached)
|
||||
escalationCount: number;
|
||||
averageResponseSeconds: number | null;
|
||||
averageResolutionSeconds: number | null;
|
||||
}
|
||||
```
|
||||
|
||||
### Product dashboard — `GET /admin/reports/product/:externalProductId`
|
||||
|
||||
```ts
|
||||
interface ProductDashboard {
|
||||
productId: string; // externalProductId, echoed back
|
||||
range: { from: string; to: string };
|
||||
supportVolume: number;
|
||||
problemsByCategory: Array<{ categoryId: string | null; count: number }>;
|
||||
recurringProblems: Array<{ categoryId: string | null; count: number }>; // same data, top N, descending
|
||||
aiResolutionRate: number | null;
|
||||
humanEscalationRate: number | null;
|
||||
topErrors: Array<{ code: string; count: number }>; // top N, descending
|
||||
}
|
||||
```
|
||||
|
||||
### Support dashboard — `GET /admin/reports/support`
|
||||
|
||||
```ts
|
||||
interface SupportDashboard {
|
||||
generatedAt: string; // workload/risk are point-in-time, not range-scoped (research.md §2)
|
||||
range: { from: string; to: string }; // still applies to the performance figures below
|
||||
workloadByAgent: Array<{ agentId: string; openAssignments: number }>;
|
||||
slaAtRisk: number;
|
||||
slaBreached: number;
|
||||
escalationCount: number;
|
||||
averageResponseSeconds: number | null;
|
||||
averageResolutionSeconds: number | null;
|
||||
}
|
||||
```
|
||||
|
||||
### AI dashboard — `GET /admin/reports/ai`
|
||||
|
||||
```ts
|
||||
interface AiDashboard {
|
||||
range: { from: string; to: string };
|
||||
totalSessions: number;
|
||||
aiResolutionRate: number | null;
|
||||
humanHandoffRate: number | null;
|
||||
failedTroubleshootingEscalationRate: number | null;
|
||||
knowledgeMatchRate: number | null;
|
||||
confidenceDistribution: { proceed: number; ask: number; escalate: number };
|
||||
toolInvocations: { success: number; failed: number };
|
||||
}
|
||||
```
|
||||
|
||||
## Query Parameters (all four routes)
|
||||
|
||||
| Param | Type | Notes |
|
||||
|---|---|---|
|
||||
| `from` | ISO date, optional | Defaults to `to - REPORTING_DEFAULT_WINDOW_DAYS` |
|
||||
| `to` | ISO date, optional | Defaults to now |
|
||||
|
||||
`from > to` is a 400 `VALIDATION_ERROR` (spec.md Edge Cases), not silently swapped.
|
||||
@@ -0,0 +1,137 @@
|
||||
# Implementation Plan: Reporting and Analytics Dashboards
|
||||
|
||||
**Branch**: `015-reporting-dashboards` | **Date**: 2026-09-09 | **Spec**: [spec.md](./spec.md)
|
||||
|
||||
**Input**: Feature specification from `specs/015-reporting-dashboards/spec.md`
|
||||
|
||||
## Summary
|
||||
|
||||
Wires the pre-scaffolded, unused `platform/reports` module into four real, admin-gated,
|
||||
read-only aggregation endpoints (Management, Product, Support, AI) matching
|
||||
`docs/09-testing-observability-cicd.md`'s own dashboard table — each computed synchronously,
|
||||
on request, directly from existing durable tables (Ticket, Problem, SLARun, EscalationEvent,
|
||||
AISupportSession, AIDiagnosis, AIAction, Resolution, Assignment). The one new piece of state is
|
||||
a small durable `ErrorCodeLookup` audit table, needed only because no existing record lets "top
|
||||
errors" be computed historically (014-full-observability's own equivalent is a process-lifetime
|
||||
Prometheus counter, unusable for a dated report). No presentation layer — see spec.md's
|
||||
Assumptions for why `supporthub-web` work is a separate follow-on.
|
||||
|
||||
## Technical Context
|
||||
|
||||
**Language/Version**: TypeScript 5.4 / Node.js 20+ (unchanged).
|
||||
|
||||
**Primary Dependencies**: None new — Prisma's own `groupBy`/`count`/`aggregate`/`findMany`, no
|
||||
raw SQL (research.md §5), reusing `decideConfidenceBand` (005-ai-support) and the
|
||||
`Resolution.resolvedBy` convention (014-full-observability) rather than reimplementing either.
|
||||
|
||||
**Storage**: One new table, `ErrorCodeLookup` (`id`, `errorCodeId` FK, `productId` FK,
|
||||
`createdAt`) — append-only, no update/delete path, indexed `(productId, createdAt)` for the
|
||||
Product dashboard's range-scoped ranking query. No change to any existing table.
|
||||
|
||||
**Testing**: Vitest — unit tests for the "no data → `null`, never `NaN`" averaging helper and the
|
||||
confidence-bucketing reuse; integration tests against real Postgres/Redis driving each
|
||||
dashboard's real underlying data (tickets in various terminal states, SLA runs met/breached,
|
||||
escalation events, AI sessions/diagnoses/actions, error-code lookups) and asserting every
|
||||
returned figure against hand-computed expected values — the same rigor and mixed
|
||||
HTTP-driven/direct-repository setup style as 014's `business-metrics.test.ts`.
|
||||
|
||||
**Target Platform**: Same Fastify modular monolith. Rewrites `platform/reports` (service,
|
||||
new controller, new routes, new schema for the date-range/product-id query params) from its
|
||||
current one-stub-method state into the real module. Adds one line to
|
||||
`ai-support/knowledge/service/error-codes.service.ts`'s existing `findKnownIssuesByErrorCode`
|
||||
(the same call site 014 already instrumented) to also write the new durable audit row.
|
||||
|
||||
**Project Type**: Backend service — single project.
|
||||
|
||||
**Performance Goals**: Every dashboard query is bounded by the requested date range (default 30
|
||||
days, config) and, where a full-row fetch is needed for in-application averaging (research.md
|
||||
§5), only the two timestamp columns needed for that specific average — never a full-table scan
|
||||
with no range filter. Acceptable at current data volumes per spec.md's own Assumptions;
|
||||
pre-aggregation is explicitly deferred to if/when load testing (a separate, not-yet-started
|
||||
Phase 11 sub-area) shows it's actually needed.
|
||||
|
||||
**Constraints**: FR-006 — an unknown `productId` on the Product dashboard is a 404, never an
|
||||
empty-but-200 response. FR-007 — every rate/average is `number | null`, `null` meaning "no
|
||||
qualifying data," computed by checking the qualifying count before ever dividing. FR-008 — every
|
||||
route requires `requireRole('ADMIN')`, the same gate every admin surface uses since
|
||||
010-identity-auth.
|
||||
|
||||
**Scale/Scope**: Four new `GET` routes, one new Prisma model + migration, four new service
|
||||
methods (one per dashboard) replacing the single stub method, one new schema file for query-param
|
||||
validation, three new env-configured values (Constitution Principle II). No new module — this
|
||||
extends `platform/reports`, already the correct architectural home.
|
||||
|
||||
## Constitution Check
|
||||
|
||||
*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.*
|
||||
|
||||
| Principle / Section | Check | Result |
|
||||
|---|---|---|
|
||||
| I. SaaS Is the Sole Identity & Access Authority | Not applicable — no identity/access surface touched; every figure is derived from SupportHub's own domain data (tickets, problems, SLA, escalation, AI sessions), squarely inside SupportHub's own sole-authority domain per this principle's own second sentence. | PASS |
|
||||
| II. Configuration Over Hardcoding | The default reporting window, the SLA-risk threshold, and the top-N ranking limit are all new env-configured values (`REPORTING_DEFAULT_WINDOW_DAYS`, `REPORTING_SLA_RISK_THRESHOLD_MINUTES`, `REPORTING_TOP_N_LIMIT`), never hardcoded — matches spec.md's own Assumptions and the roadmap's "never hardcode a placeholder value and ship it as final." | PASS |
|
||||
| III. Layered Architecture With Enforced Module Boundaries | All new code lives inside `platform/reports` (already its correct home) following Route → Schema → Controller → Service → Repository → Prisma; cross-module reads (tickets, AI support, orchestration, SLA/escalation, problem resolution) go through each owning module's own public `index.ts`, the same precedent every prior feature this session established (e.g. `tool-executor.ts` reading `ticketsService` from `@/modules/ticketing/tickets`). | PASS |
|
||||
| IV. AI Recommends, Deterministic Policy Decides | Not applicable — no AI tool-execution or decision logic changed; the AI dashboard only reports on outcomes the existing, already-deterministic confidence-band/tool-policy code already produced. | PASS — N/A |
|
||||
| V. Evidence-Based Verification | Not applicable — no resolution-recording logic changed. | PASS — N/A |
|
||||
| VI. Durable Audit & History | The one new table (`ErrorCodeLookup`) is itself an append-only audit record, directly in this principle's spirit — "which error codes came up, when" becomes durably answerable for the first time. | PASS |
|
||||
| VII. Concurrency-Safe, Durable Job Handling | Not applicable — read-only aggregation queries, no job handlers, no assignment/SLA state mutated. | PASS — N/A |
|
||||
| VIII. Problem and Ticket Are Separate, Related Entities | Respected — the Product dashboard's problem-type breakdown queries `Problem` directly, never conflating it with `Ticket`. | PASS |
|
||||
| Technology & Platform Constraints | No new dependencies; one new Prisma model via the established non-interactive migration workflow (`prisma migrate diff` → hand-written `migration.sql` → `prisma migrate deploy`) this session has used for every prior schema change. | PASS |
|
||||
|
||||
No violations requiring Complexity Tracking justification.
|
||||
|
||||
## Project Structure
|
||||
|
||||
### Documentation (this feature)
|
||||
|
||||
```text
|
||||
specs/015-reporting-dashboards/
|
||||
├── plan.md
|
||||
├── research.md
|
||||
├── data-model.md
|
||||
├── quickstart.md
|
||||
├── contracts/
|
||||
│ └── reports-api-contract.md
|
||||
└── tasks.md
|
||||
```
|
||||
|
||||
### Source Code (repository root)
|
||||
|
||||
```text
|
||||
supporthub-api/
|
||||
├── prisma/
|
||||
│ ├── schema.prisma # MODIFIED — new ErrorCodeLookup model
|
||||
│ └── migrations/
|
||||
│ └── <timestamp>_add_error_code_lookup/migration.sql # NEW
|
||||
├── src/
|
||||
│ ├── config/
|
||||
│ │ └── env.ts / reporting.ts (or similar) # MODIFIED — 3 new env-configured values
|
||||
│ └── modules/
|
||||
│ ├── platform/
|
||||
│ │ └── reports/ # REWRITTEN (was a 1-method stub)
|
||||
│ │ ├── controller/
|
||||
│ │ ├── mapper/ # date-range parsing/defaulting, averaging helper
|
||||
│ │ ├── repository/ # the 4 dashboards' Prisma queries
|
||||
│ │ ├── routes/
|
||||
│ │ ├── schema/ # query-param validation
|
||||
│ │ ├── service/
|
||||
│ │ └── index.ts
|
||||
│ └── ai-support/
|
||||
│ └── knowledge/
|
||||
│ ├── repository/ # MODIFIED — errorCodeLookupRepository
|
||||
│ └── service/
|
||||
│ └── error-codes.service.ts # MODIFIED — one new line at the existing
|
||||
│ lookup call site
|
||||
└── tests/
|
||||
├── unit/platform/reports/ # averaging/no-data-null helper, confidence
|
||||
│ bucketing reuse
|
||||
└── integration/platform-reports/ # all four dashboards against real data
|
||||
```
|
||||
|
||||
**Structure Decision**: Single project, no new module — `platform/reports` already exists as the
|
||||
correct architectural home and simply needs its real implementation built out, following the
|
||||
same Route → Schema → Controller → Service → Repository → Prisma layering every other module
|
||||
already uses.
|
||||
|
||||
## Complexity Tracking
|
||||
|
||||
*No constitution violations — table intentionally omitted.*
|
||||
@@ -0,0 +1,52 @@
|
||||
# Quickstart: Reporting and Analytics Dashboards
|
||||
|
||||
Manual verification steps for each user story, against a running instance backed by real
|
||||
Postgres/Redis, logged in as an ADMIN.
|
||||
|
||||
## Scenario 1 — Management dashboard (User Story 1)
|
||||
|
||||
1. Create several tickets within a known date range: some reaching `AI_RESOLVED`/`RESOLVED` via
|
||||
an AI session, some escalated to a human and resolved via `resolutionsService.record`, some
|
||||
left open.
|
||||
2. Let one ticket's SLA run complete on time and another breach (via the existing breach sweep).
|
||||
3. `GET /admin/reports/management?from=<range start>&to=<range end>`.
|
||||
4. **Expected**: `totalCases`, `aiResolved`, `humanEscalated`, `resolved`, `open` all match what
|
||||
was actually created; `slaCompliance.met`/`.breached` match the two SLA outcomes;
|
||||
`averageResponseSeconds`/`averageResolutionSeconds` are non-null and plausible.
|
||||
5. Request the same endpoint for a date range with no activity at all.
|
||||
6. **Expected**: every count is `0`, every rate/average is `null`, not an error.
|
||||
|
||||
## Scenario 2 — Product dashboard (User Story 2)
|
||||
|
||||
1. Create tickets for two distinct products in the same range, one with a categorized problem.
|
||||
2. Look up a known error code for one product several times, a different code once.
|
||||
3. `GET /admin/reports/product/:externalProductId` for each product.
|
||||
4. **Expected**: each product's `supportVolume`/`problemsByCategory`/`aiResolutionRate` reflect
|
||||
only its own tickets; `topErrors` ranks the more-frequently-looked-up code first.
|
||||
5. Request the endpoint for a nonexistent `externalProductId`.
|
||||
6. **Expected**: `404 NOT_FOUND`, not an empty `200`.
|
||||
|
||||
## Scenario 3 — Support dashboard (User Story 3)
|
||||
|
||||
1. Assign several tickets across two agents (some via the real orchestration flow).
|
||||
2. Let one ticket's SLA run sit within `REPORTING_SLA_RISK_THRESHOLD_MINUTES` of its resolution
|
||||
due date without breaching.
|
||||
3. `GET /admin/reports/support`.
|
||||
4. **Expected**: `workloadByAgent` matches each agent's real current open-assignment count;
|
||||
`slaAtRisk` counts exactly the near-due run, distinct from `slaBreached`.
|
||||
|
||||
## Scenario 4 — AI dashboard (User Story 4)
|
||||
|
||||
1. Run AI sessions to a mix of terminal outcomes (`resolved`, `escalated`), with some tool
|
||||
invocations succeeding and others failing, and diagnoses spanning a range of confidence
|
||||
values.
|
||||
2. `GET /admin/reports/ai`.
|
||||
3. **Expected**: `aiResolutionRate`/`humanHandoffRate` reflect the real outcome mix;
|
||||
`confidenceDistribution` buckets match `decideConfidenceBand`'s own classification of each
|
||||
diagnosis's stored confidence against the system-default thresholds; `toolInvocations`
|
||||
reflects the real success/failure counts.
|
||||
|
||||
## What "done" looks like
|
||||
|
||||
All four scenarios pass against a real Postgres/Redis, every figure independently verified
|
||||
against hand-computed expected values, and no route is reachable by a non-admin session.
|
||||
@@ -0,0 +1,143 @@
|
||||
# Research: Reporting and Analytics Dashboards
|
||||
|
||||
## 1. Where this lives
|
||||
|
||||
**Decision**: Wire up the existing, pre-scaffolded `src/modules/platform/reports` module (today
|
||||
just `ReportsService.generateSummaryReport()` returning `{}`, confirmed unused anywhere) rather
|
||||
than creating a new module. Its four real methods (`getManagementDashboard`,
|
||||
`getProductDashboard`, `getSupportDashboard`, `getAiDashboard`) replace the one stub method.
|
||||
Routes live at `GET /admin/reports/management`, `GET /admin/reports/product/:externalProductId`,
|
||||
`GET /admin/reports/support`, `GET /admin/reports/ai`, admin-gated the same way every other
|
||||
admin-only endpoint since 010-identity-auth already is (`requireRole('ADMIN')`).
|
||||
|
||||
**Why not the `ANALYTICS` queue** (`src/jobs/analytics`, also pre-scaffolded, also inert): a
|
||||
queued background job fits pre-computing a report nobody is currently waiting on; a dashboard
|
||||
request is someone waiting right now for an answer. Per spec.md's Assumptions, this first cut is
|
||||
synchronous, direct-query aggregation — the queue stub stays exactly as inert as it already was,
|
||||
untouched by this feature.
|
||||
|
||||
## 2. Per-dashboard queries
|
||||
|
||||
All four use Prisma's `groupBy`/`count`/`aggregate`, scoped by `createdAt` (or the
|
||||
milestone-specific timestamp named below) within `[from, to]`, computed directly against the
|
||||
tables that already own each fact — no new table, no denormalized rollup.
|
||||
|
||||
### Management (FR-001)
|
||||
|
||||
| Figure | Source |
|
||||
|---|---|
|
||||
| Total cases | `Ticket.count({ createdAt in range })` |
|
||||
| AI resolved | `Ticket.count({ createdAt in range, status: 'AI_RESOLVED' })` — a ticket that reached `AI_RESOLVED` and stayed there or moved straight to `RESOLVED` without a `Resolution.resolvedBy` other than `'ai'`; see §4 below for the exact "who resolved it" rule shared with the Product dashboard |
|
||||
| Human escalated | `Ticket.count({ createdAt in range, status in [HUMAN_ESCALATION, IN_PROGRESS, WAITING_FOR_CUSTOMER, RESOLUTION_PENDING_CUSTOMER, RESOLVED, CLOSED, REOPENED] })` minus AI-only-path tickets — i.e. any ticket that ever reached `HUMAN_ESCALATION`; the state machine research.md already establishes this as one-way (003-ticketing) |
|
||||
| Resolved (either path) | `Ticket.count({ createdAt in range, status in [RESOLVED, CLOSED] })` |
|
||||
| Open | `Ticket.count({ createdAt in range, status not in [RESOLVED, CLOSED] })` |
|
||||
| SLA compliance / breach count | `SLARun.groupBy(['status'], { ticket: { createdAt in range } })`, `status: 'completed'` = met, `'breached'` = breached (mirrors 014's own metric semantics — see 014 research.md §5's "read status before the overwrite" caveat, which applies equally here: a `'breached'`-then-`'completed'` run is still counted breached, by reading the `breachedAt`/`firstResponseBreachedAt` timestamps rather than only the current `status` string) |
|
||||
| Escalation count | `EscalationEvent.count({ createdAt in range })` |
|
||||
| Average response time | `avg(firstAgentMessage.createdAt - ticket.createdAt)` over tickets with at least one `AGENT_MESSAGE` in range — computed in application code over a bounded query result (see §5, no raw SQL) |
|
||||
| Average resolution time | `avg(resolution.createdAt - ticket.createdAt)` over tickets with a `Resolution` row in range |
|
||||
|
||||
### Product (FR-002)
|
||||
|
||||
Same shape as Management, `WHERE Ticket.productId = :productId` (resolved from the given
|
||||
`externalProductId`, 404 if not found — FR-006), plus:
|
||||
|
||||
| Figure | Source |
|
||||
|---|---|
|
||||
| Problem-type breakdown | `Problem.groupBy(['categoryId'], { productId, createdAt in range })` |
|
||||
| Recurring problems | Same grouped result, sorted descending, top N (config, default 10) |
|
||||
| Top errors | `reuses 014's own instrumentation point conceptually but queries fresh` — no, see §6: there is no persisted "error code lookup" table, only 014's in-memory Prometheus counter, which is NOT queryable historically. Resolved by adding a durable audit read instead: see §6. |
|
||||
|
||||
### Support (FR-003)
|
||||
|
||||
| Figure | Source |
|
||||
|---|---|
|
||||
| Per-agent workload | `Assignment.groupBy(['agentId'], { isCurrent: true })` — a snapshot of *right now*, not date-ranged (workload is inherently current, not historical — spec.md's own framing: "how much work is currently assigned") |
|
||||
| SLA risk / breached | `SLARun.findMany({ status: 'running', resolutionDueAt: {gte: now} })` filtered in application code by "due within `SLA_RISK_THRESHOLD_MINUTES` of now" for risk, vs. `status: 'breached'` for already-breached |
|
||||
| Escalation count | Same as Management, unfiltered by product |
|
||||
| Response/resolution performance | Same computation as Management's averages |
|
||||
|
||||
### AI (FR-004)
|
||||
|
||||
| Figure | Source |
|
||||
|---|---|
|
||||
| AI resolution rate / human-handoff rate | `AISupportSession.groupBy(['status'], { startedAt in range })` — `resolved` vs. `escalated`/`ended_by_agent` as a share of total terminal sessions |
|
||||
| Failed-troubleshooting-then-escalated rate | Sessions with `status: 'escalated'` that have at least one `AIInteraction`/`AIAction` recording a failed troubleshooting attempt — see 005-ai-support's own runbook-step-outcome classification (`classifyStepOutcome`), reused rather than reinvented |
|
||||
| Knowledge-match rate | `AIKnowledgeReference` presence per session (`recordMany` is only ever called with actual retrieval results — 005-ai-support's own `diagnose.ts`) vs. sessions with zero references recorded |
|
||||
| Confidence distribution | `AIDiagnosis.findMany({ createdAt in range })`, bucketed in application code against `aiConfig.defaultHighConfidence`/`defaultLowConfidence` (see §7 — NOT a per-diagnosis resolved policy) |
|
||||
| Tool success/failure | `AIAction` joined to `AIActionResult`, grouped by `result.status` |
|
||||
|
||||
## 3. "No data" convention (FR-007)
|
||||
|
||||
**Decision**: every rate/average field is `number | null` — `null` means "no qualifying records
|
||||
in range," distinguished in the response shape from a genuine `0` (e.g., a real 0% AI resolution
|
||||
rate because everything escalated is a valid, meaningful `0`; "nobody's data exists yet" is
|
||||
`null`). Application code computes every average by fetching the qualifying count first and
|
||||
returning `null` before ever dividing, never relying on `0/0` producing `NaN` and hoping a caller
|
||||
notices.
|
||||
|
||||
## 4. "Who resolved it" — reused from 014, not reinvented
|
||||
|
||||
014-full-observability's own event subscriber already established the authoritative rule: a
|
||||
ticket's `Resolution.resolvedBy` field (`"ai"` | an `agentId`) is the single source of truth for
|
||||
whether a resolution was AI- or human-driven (014 research.md §5). This feature's Management/
|
||||
Product dashboards reuse the exact same join (`Resolution.findMany` scoped to the range,
|
||||
`resolvedBy === 'ai'` vs. not) rather than re-deriving it from `Ticket.status` transitions a
|
||||
second, potentially-inconsistent way.
|
||||
|
||||
## 5. No raw SQL
|
||||
|
||||
**Decision**: every duration average (response time, resolution time) is computed by fetching
|
||||
the bounded set of qualifying rows (ticket `createdAt` + the milestone timestamp) via Prisma and
|
||||
averaging in application code, not a raw `$queryRaw` computing `AVG(EXTRACT(EPOCH FROM ...))` in
|
||||
SQL. At the data volumes spec.md's Assumptions accept for this first cut (no pre-aggregation,
|
||||
synchronous queries), a bounded per-range fetch is simple, type-safe, and testable without
|
||||
hand-writing SQL — consistent with this codebase's near-total avoidance of `$queryRaw` elsewhere
|
||||
(confirmed by grep: no existing module uses it for reporting-shaped queries).
|
||||
|
||||
## 6. Top errors needs a durable, queryable record — a real gap 014 left open
|
||||
|
||||
014-full-observability's `supporthub_known_error_lookups_total` Prometheus counter is
|
||||
process-lifetime, in-memory, and reset on every restart — useless for "top errors in the last 30
|
||||
days." Since no durable "error code lookup" record exists anywhere in this codebase today (the
|
||||
existing `error-codes.service.ts` just reads `KnownIssue`/`ErrorCode` rows, never records that a
|
||||
lookup happened), this feature adds one small, focused piece of new state: a durable
|
||||
`ErrorCodeLookup` audit row (`errorCodeId`, `productId`, `createdAt`), written by
|
||||
`error-codes.service.ts`'s already-existing `findKnownIssuesByErrorCode` (the same call site
|
||||
014 instrumented for its own live counter — this feature adds one more line there, a durable
|
||||
write alongside the existing live-metric increment, not a replacement for it). This is the one
|
||||
schema change this feature needs; every other dashboard figure is computed from tables that
|
||||
already exist.
|
||||
|
||||
## 7. Confidence distribution uses the system default threshold, not a per-diagnosis policy
|
||||
|
||||
**Decision**: bucket every `AIDiagnosis.confidence` value in range against the env-configured
|
||||
system-wide defaults (`aiConfig.defaultHighConfidence`/`defaultLowConfidence`), the same
|
||||
`decideConfidenceBand` pure function 005-ai-support already exports — reused directly, not
|
||||
reimplemented.
|
||||
|
||||
**Why not resolve each diagnosis's actual applicable per-product/category policy** (what the
|
||||
live reasoning path itself does): `AIDiagnosis.product`/`feature` are the AI's own free-text
|
||||
classification output, not foreign keys to `Product`/`Category` — there is no reliable, existing
|
||||
join from a diagnosis row back to which `ConfidencePolicy` row actually applied to it at the time
|
||||
without speculatively string-matching free text against product names, which this codebase does
|
||||
nowhere else and which research.md declines to invent here. A dashboard-level aggregate
|
||||
distribution using the system-wide default is an honest, documented simplification (spec.md
|
||||
Assumptions) — precise enough to show a meaningful shape without fabricating a false precision
|
||||
the data doesn't actually support.
|
||||
|
||||
## 8. New configuration (Constitution Principle II — nothing hardcoded)
|
||||
|
||||
| Env var | Default | Used by |
|
||||
|---|---|---|
|
||||
| `REPORTING_DEFAULT_WINDOW_DAYS` | `30` | Every dashboard's `from`/`to` default when omitted (FR-005) |
|
||||
| `REPORTING_SLA_RISK_THRESHOLD_MINUTES` | `60` | Support dashboard's "at risk" classification (FR-003) |
|
||||
| `REPORTING_TOP_N_LIMIT` | `10` | Product dashboard's recurring-problems/top-errors ranking length |
|
||||
|
||||
## 9. Test strategy
|
||||
|
||||
Integration tests create real tickets/problems/SLA runs/escalation events/AI sessions/diagnoses/
|
||||
actions/error-code lookups directly against real Postgres (mixing real HTTP-driven setup where a
|
||||
realistic flow matters and direct repository/Prisma writes where only the aggregation math is
|
||||
under test — the same mix 014's own `business-metrics.test.ts` used), then request each
|
||||
dashboard endpoint and assert every figure against hand-computed expected values. Unit tests
|
||||
cover the "no data → null, never NaN" guard and the confidence-bucketing pure-function reuse.
|
||||
@@ -0,0 +1,257 @@
|
||||
# Feature Specification: Reporting and Analytics Dashboards
|
||||
|
||||
**Feature Branch**: `015-reporting-dashboards`
|
||||
|
||||
**Created**: 2026-09-09
|
||||
|
||||
**Status**: Draft
|
||||
|
||||
**Input**: User description: "Reporting and analytics dashboards: real, read-only aggregation endpoints backing the four dashboards named in docs/09-testing-observability-cicd.md (Management, Product, Support, AI) — wiring up the pre-scaffolded but never-implemented platform/reports module into actual database-backed aggregation queries, admin-gated, with a date-range filter."
|
||||
|
||||
## User Scenarios & Testing *(mandatory)*
|
||||
|
||||
### User Story 1 - Management sees organization-wide support health (Priority: P1)
|
||||
|
||||
An admin or team lead opens a single view showing how support is doing overall for a chosen
|
||||
period: how many cases came in, how many were resolved (by AI vs. by a human), how many are
|
||||
still open, whether SLA commitments are being met, and how escalation is trending.
|
||||
|
||||
**Why this priority**: This is the one dashboard covering the whole roadmap's own top-level
|
||||
success criteria (`docs/10-implementation-roadmap.md`'s checklist) in one place — the first
|
||||
thing anyone asks about a support operation is "how are we doing," and today there is no way to
|
||||
answer that except querying the database by hand.
|
||||
|
||||
**Independent Test**: Can be fully tested by creating a known set of tickets in various terminal
|
||||
states (AI-resolved, human-resolved, still open) plus a mix of met/breached SLA runs and
|
||||
escalations within a chosen date range, then requesting the Management dashboard for that range
|
||||
and confirming every figure matches what was actually created.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** a mix of tickets created within a chosen date range — some AI-resolved, some
|
||||
human-resolved, some still open — **When** the Management dashboard is requested for that
|
||||
range, **Then** total cases, AI-resolved count, human-escalated count, resolved count, and
|
||||
open count all match the actual data exactly.
|
||||
2. **Given** SLA runs that completed on time and others that breached within the range,
|
||||
**When** the dashboard is requested, **Then** SLA compliance (a rate) and SLA breach count
|
||||
both reflect the real outcomes.
|
||||
3. **Given** some tickets have a recorded first agent response and a resolution timestamp,
|
||||
**When** the dashboard is requested, **Then** average response time and average resolution
|
||||
time are computed only from tickets that actually reached those milestones within the range
|
||||
(a still-open ticket contributes to "open count" but never a fabricated resolution time).
|
||||
4. **Given** a date range with zero activity, **When** the dashboard is requested, **Then** every
|
||||
count is zero and every average is reported as "no data" rather than a computed zero or a
|
||||
division-by-zero error.
|
||||
|
||||
---
|
||||
|
||||
### User Story 2 - See support broken down by product (Priority: P1)
|
||||
|
||||
An admin viewing support data for a specific product (or comparing products) sees volume,
|
||||
problem-type breakdown, which problems recur most, how well AI is resolving that product's
|
||||
issues versus escalating them, and which error codes come up most often.
|
||||
|
||||
**Why this priority**: SupportHub serves multiple SaaS products (Constitution Principle I); a
|
||||
number that isn't broken out by product hides which integration actually needs attention — this
|
||||
is as fundamental as the Management view, just sliced differently.
|
||||
|
||||
**Independent Test**: Can be fully tested by creating tickets/problems/error-code lookups across
|
||||
two distinct products within a date range, requesting the Product dashboard for each product,
|
||||
and confirming each one's figures include only its own product's data.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** tickets exist for two different products in the same date range, **When** the
|
||||
Product dashboard is requested scoped to one product, **Then** support volume and every other
|
||||
figure reflect only that product's tickets, never the other product's.
|
||||
2. **Given** problems in several categories for one product, **When** the dashboard is
|
||||
requested, **Then** the problem-type breakdown and "recurring problems" ranking both reflect
|
||||
the real category distribution, most-frequent first.
|
||||
3. **Given** a mix of AI-resolved and human-escalated tickets for one product, **When** the
|
||||
dashboard is requested, **Then** AI resolution rate and human escalation rate are both
|
||||
computed as a percentage of that product's own total, not the platform-wide total.
|
||||
4. **Given** several known-error-code lookups for one product, some codes looked up more than
|
||||
others, **When** the dashboard is requested, **Then** "top errors" lists those codes ranked by
|
||||
lookup frequency.
|
||||
|
||||
---
|
||||
|
||||
### User Story 3 - Support sees team workload and performance (Priority: P2)
|
||||
|
||||
An admin or team lead sees how much work is currently assigned across agents, which tickets are
|
||||
at SLA risk, how much escalation is happening, and how quickly the team is responding to and
|
||||
resolving tickets.
|
||||
|
||||
**Why this priority**: This view is about ongoing operational load, not historical trend — useful
|
||||
for day-to-day team management, but the organization can already see whether it's healthy
|
||||
overall from User Story 1 without this one; P2 reflects that it adds an operational lens rather
|
||||
than a new class of information.
|
||||
|
||||
**Independent Test**: Can be fully tested by assigning several tickets to known agents (some
|
||||
close to SLA breach, some not), then requesting the Support dashboard and confirming workload
|
||||
per agent and the SLA-risk count both match reality.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** several tickets are currently assigned across two agents, **When** the Support
|
||||
dashboard is requested, **Then** each agent's current open-assignment count matches what was
|
||||
actually assigned to them (not a stale count from a previous, now-unassigned period).
|
||||
2. **Given** a ticket's SLA run is running and past a configurable risk threshold of its
|
||||
resolution due date (but not yet breached), **When** the dashboard is requested, **Then** it
|
||||
is counted as "at risk," distinct from both "on track" and "breached."
|
||||
3. **Given** response and resolution durations for several resolved tickets in the period,
|
||||
**When** the dashboard is requested, **Then** response-performance and resolution-performance
|
||||
figures are computed only from tickets that actually reached those milestones.
|
||||
|
||||
---
|
||||
|
||||
### User Story 4 - See how well the AI is performing (Priority: P2)
|
||||
|
||||
An admin sees, for a chosen period, how often the AI resolves issues on its own versus escalating
|
||||
them, how often its attempted troubleshooting fails outright, how often it finds relevant
|
||||
knowledge, how confident its diagnoses tend to be, how reliably its tools succeed, and how often
|
||||
it ultimately hands off to a human.
|
||||
|
||||
**Why this priority**: This is the dashboard that validates the AI-first design's core premise
|
||||
(Constitution Principle IV) is actually working in practice — valuable, but a narrower audience
|
||||
than the org-wide and per-product views above, hence P2.
|
||||
|
||||
**Independent Test**: Can be fully tested by running several AI sessions to different terminal
|
||||
outcomes (resolved, escalated, escalated-after-failed-troubleshooting) with a mix of tool
|
||||
successes/failures and confidence levels recorded, then requesting the AI dashboard and
|
||||
confirming every figure matches the real session data.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** a mix of AI sessions ending resolved vs. escalated in the period, **When** the AI
|
||||
dashboard is requested, **Then** AI resolution rate and human-handoff rate both reflect the
|
||||
real outcome mix as percentages of total sessions.
|
||||
2. **Given** some AI tool invocations succeeded and others failed in the period, **When** the
|
||||
dashboard is requested, **Then** tool success/failure figures reflect the real invocation
|
||||
outcomes.
|
||||
3. **Given** diagnoses were recorded with a range of confidence values, **When** the dashboard is
|
||||
requested, **Then** the confidence distribution groups them into the same proceed/ask/escalate
|
||||
bands the AI support module's own confidence-policy service already classifies each diagnosis
|
||||
into (005-ai-support), not a newly-invented scheme.
|
||||
4. **Given** some AI sessions' knowledge-retrieval step found matching entries and others found
|
||||
none, **When** the dashboard is requested, **Then** knowledge-match rate reflects the real
|
||||
match/no-match mix.
|
||||
|
||||
---
|
||||
|
||||
### Edge Cases
|
||||
|
||||
- What happens when no `from`/`to` date range is given? Defaults to a reasonable trailing window
|
||||
(see Assumptions) rather than scanning the entire history unbounded on every request.
|
||||
- What happens when `from` is after `to`? Rejected as a validation error, not silently swapped or
|
||||
silently returning empty data.
|
||||
- What happens when a requested `productId` (Product dashboard) doesn't exist? Rejected with a
|
||||
clear not-found error, not an empty-but-200 response that looks like "this product has zero
|
||||
activity."
|
||||
- What happens when an average would divide by zero (no tickets reached that milestone in the
|
||||
range)? Reported as an explicit "no data" value, never `NaN`, `null` silently coerced to `0`,
|
||||
or a thrown error.
|
||||
- What happens when a ticket's SLA run was paused for part of the period? SLA-risk/compliance
|
||||
figures use the run's own already-durable due dates (008-sla-escalation's pause/resume
|
||||
already accounts for paused time) rather than this feature re-deriving elapsed time itself.
|
||||
- Who can see these dashboards? Same admin-only gate as every other admin configuration/reporting
|
||||
surface introduced since 010-identity-auth — no new role is introduced.
|
||||
|
||||
## Requirements *(mandatory)*
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- **FR-001**: System MUST provide a Management dashboard summarizing, for a given date range:
|
||||
total cases created, cases resolved by AI, cases escalated to a human, total resolved
|
||||
(either path), total still open, SLA compliance rate, SLA breach count, escalation count,
|
||||
average first-response time, and average resolution time.
|
||||
- **FR-002**: System MUST provide a Product dashboard summarizing, for a given date range and a
|
||||
specific product: support volume, a breakdown by problem category, a ranked list of the most
|
||||
recurring problem categories, AI resolution rate, human escalation rate, and a ranked list of
|
||||
the most frequently looked-up error codes.
|
||||
- **FR-003**: System MUST provide a Support dashboard summarizing, for a given date range:
|
||||
current per-agent open-assignment workload, count of tickets at SLA risk (past a configurable
|
||||
risk threshold of their resolution due date but not yet breached), count of tickets already
|
||||
breached, escalation count, average response performance, and average resolution performance.
|
||||
- **FR-004**: System MUST provide an AI dashboard summarizing, for a given date range: AI
|
||||
resolution rate, rate of sessions that escalated after at least one failed troubleshooting
|
||||
attempt, knowledge-match rate, a distribution of diagnosis confidence across the existing
|
||||
proceed/ask/escalate bands, tool invocation success/failure counts, and human-handoff rate.
|
||||
- **FR-005**: Every dashboard endpoint MUST accept an optional `from`/`to` date range; when
|
||||
omitted, it MUST default to a documented trailing window rather than scanning unbounded
|
||||
history.
|
||||
- **FR-006**: The Product dashboard MUST require a valid `productId` and MUST reject an unknown
|
||||
one with a clear not-found error rather than returning an empty-but-successful response.
|
||||
- **FR-007**: Every rate/average figure MUST be computed only from tickets/sessions/runs that
|
||||
actually reached the relevant milestone within the range; a metric with no qualifying data MUST
|
||||
be reported as an explicit "no data" value, never a computed `0`, `null`, or `NaN`.
|
||||
- **FR-008**: All four dashboard endpoints MUST be admin-gated, consistent with every other
|
||||
admin-only reporting/configuration surface in this codebase.
|
||||
- **FR-009**: This feature MUST NOT alter the meaning or shape of any existing endpoint, event, or
|
||||
table — nearly every figure is derived read-only from data already durably recorded by the
|
||||
modules that own it (003 ticketing, 005 AI support, 007 orchestration, 008 SLA/escalation, 009
|
||||
problem resolution). The one exception is FR-011: a small new durable record needed only
|
||||
because no existing table can answer "which error codes are looked up most" historically.
|
||||
- **FR-011**: System MUST durably record each known-error-code lookup (product, error code,
|
||||
timestamp) at the point it already happens (the existing error-code lookup call site) so the
|
||||
Product dashboard's "top errors" ranking (FR-002) can be computed historically — the
|
||||
equivalent live, in-process counter this project already exposes on `/metrics` (014-full-
|
||||
observability) is process-lifetime and reset on every restart, unusable for a historical
|
||||
dashboard.
|
||||
- **FR-010**: This feature is backend-only; presenting these figures in a UI is a separate,
|
||||
explicitly out-of-scope follow-on (see Assumptions).
|
||||
|
||||
### Key Entities
|
||||
|
||||
- **Dashboard response**: A read-only, computed JSON summary for one of the four dashboards over
|
||||
a requested date range (and, for the Product dashboard, one product) — never itself persisted;
|
||||
recomputed fresh on every request from existing durable records.
|
||||
- **Date range**: An inclusive `from`/`to` pair (calendar dates or timestamps) scoping every
|
||||
aggregation query; not a stored entity, a request parameter.
|
||||
- **Error code lookup record** (new, FR-011): a durable, append-only audit row — which product,
|
||||
which error code, when — written at the existing lookup call site; exists solely so "top
|
||||
errors" can be computed over a historical range, never read or written anywhere else.
|
||||
- **Confidence band**: The existing proceed/ask/escalate classification 005-ai-support already
|
||||
applies to a diagnosis's confidence score — reused here for the AI dashboard's distribution, not
|
||||
redefined.
|
||||
|
||||
## Success Criteria *(mandatory)*
|
||||
|
||||
### Measurable Outcomes
|
||||
|
||||
- **SC-001**: For any chosen date range, every figure on all four dashboards can be independently
|
||||
verified against the underlying ticket/session/SLA-run/escalation-event records and matches
|
||||
exactly — no discrepancy between what a dashboard reports and what actually happened.
|
||||
- **SC-002**: An admin can answer "how is support doing right now" (Management), "how is this
|
||||
specific product doing" (Product), "who's overloaded and what's at risk" (Support), and "is the
|
||||
AI actually helping" (AI) each from a single request, with no manual database query needed.
|
||||
- **SC-003**: A dashboard request for a period with no matching activity returns clean, explicit
|
||||
"no data" results in well under a second — never an error, a stall, or a misleading zero.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- **Presentation is out of scope for this feature.** The user's own explicit direction was to
|
||||
build the backend aggregation capability first (the established pattern this project has
|
||||
followed for every prior feature that touched both repos — identity/auth, the agent ticket
|
||||
queue, and full observability were each built backend-first). A `supporthub-web` dashboard UI
|
||||
consuming these endpoints is a natural, separate follow-on, not bundled into this spec.
|
||||
- The default trailing window when no date range is given is the last 30 days, matching common
|
||||
reporting-dashboard convention; CONFIGURABLE via the same admin-config env-driven pattern this
|
||||
project already uses for every other business-policy value (Constitution Principle II), not
|
||||
hardcoded as a magic number in application logic.
|
||||
- "SLA risk" needs a threshold (how close to the due date counts as "at risk") that the business
|
||||
has not specified — CONFIGURABLE, not invented as a hardcoded percentage, consistent with
|
||||
`docs/10-implementation-roadmap.md`'s own "never hardcode a placeholder value and ship it as
|
||||
final" instruction.
|
||||
- These endpoints compute their figures synchronously, on request, directly from the existing
|
||||
tables — no new pre-aggregation table, no scheduled batch job, and no use of the pre-scaffolded
|
||||
`ANALYTICS` queue (`src/jobs/analytics`), which remains an inert stub outside this feature's
|
||||
scope. Live query performance at current data volumes is assumed adequate; a future feature can
|
||||
introduce pre-aggregation if and when it's actually needed (load/concurrency testing, a
|
||||
separate not-yet-started Phase 11 sub-area, is where that question would be validated).
|
||||
- "Top errors"/"recurring problems" rankings return a bounded top-N list (CONFIGURABLE limit,
|
||||
defaulting to 10) rather than the full distribution, matching how a dashboard is actually
|
||||
consumed.
|
||||
- Dashboard responses are computed fresh per request (no caching layer) — acceptable given the
|
||||
assumed data volumes and consistent with not prematurely optimizing ahead of the load-testing
|
||||
phase.
|
||||
@@ -0,0 +1,188 @@
|
||||
---
|
||||
description: 'Task list for 015-reporting-dashboards'
|
||||
---
|
||||
|
||||
# Tasks: Reporting and Analytics Dashboards
|
||||
|
||||
**Input**: Design documents from `specs/015-reporting-dashboards/`
|
||||
|
||||
**Prerequisites**: [plan.md](./plan.md), [spec.md](./spec.md), [research.md](./research.md),
|
||||
[data-model.md](./data-model.md),
|
||||
[contracts/reports-api-contract.md](./contracts/reports-api-contract.md),
|
||||
[quickstart.md](./quickstart.md)
|
||||
|
||||
**Organization**: Tasks are grouped by user story (US1 = P1 Management, US2 = P1 Product,
|
||||
US3 = P2 Support, US4 = P2 AI). All four share the Foundational phase (schema, config, shared
|
||||
helpers, module scaffolding) but are otherwise independent of each other.
|
||||
|
||||
## Format: `[ID] [P?] [Story] Description`
|
||||
|
||||
All file paths are relative to `supporthub-api/` (repo root).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Foundational (Blocking Prerequisites)
|
||||
|
||||
- [x] T001 Add `REPORTING_DEFAULT_WINDOW_DAYS` (default `30`),
|
||||
`REPORTING_SLA_RISK_THRESHOLD_MINUTES` (default `60`), and `REPORTING_TOP_N_LIMIT`
|
||||
(default `10`) to `src/config/env.ts`, exposed via a new `reportingConfig` in
|
||||
`src/config/reporting.ts` (or added to an existing config file, matching this codebase's
|
||||
own per-feature config-file convention)
|
||||
- [x] T002 Add the `ErrorCodeLookup` model to `prisma/schema.prisma` per data-model.md, generate
|
||||
the migration via `prisma migrate diff --from-url <db-url> --to-schema-datamodel
|
||||
./prisma/schema.prisma --script`, hand-write it into
|
||||
`prisma/migrations/<timestamp>_add_error_code_lookup/migration.sql`, apply via `prisma
|
||||
migrate deploy` against the throwaway test database (depends on T001 only in that both
|
||||
are Foundational — no code dependency)
|
||||
- [x] T003 Add `ai-support/knowledge/repository/error-code-lookup.repository.ts` —
|
||||
`create(errorCodeId, productId)`, exported from the knowledge module's repository index
|
||||
(depends on T002)
|
||||
- [x] T004 [P] Call the new repository's `create(...)` from
|
||||
`ai-support/knowledge/service/error-codes.service.ts`'s existing
|
||||
`findKnownIssuesByErrorCode`, alongside (not replacing) 014's own
|
||||
`knownErrorLookupsCounter.inc(...)` call at that same call site (depends on T003)
|
||||
- [x] T005 [P] Add `platform/reports/mapper/date-range.ts` — parses/validates `from`/`to` query
|
||||
params, defaulting via T001's `reportingConfig.defaultWindowDays`, throwing
|
||||
`ValidationError` when `from > to` (depends on T001)
|
||||
- [x] T006 [P] Add `platform/reports/mapper/rate.ts` — a shared `computeRate(numerator,
|
||||
denominator): number | null` and `computeAverageSeconds(durations: number[]): number |
|
||||
null` pair, both returning `null` (never `NaN`/`0`) when there's no qualifying data
|
||||
(research.md §3) — no dependency, pure functions
|
||||
- [x] T007 Scaffold `platform/reports/schema/` (query-param zod schema using T005's date-range
|
||||
parsing), `platform/reports/controller/reports.controller.ts` (empty methods to be filled
|
||||
in per user story below), `platform/reports/routes/reports.routes.ts` registering all four
|
||||
routes behind `requireRole('ADMIN')`, and update `platform/reports/index.ts` to export the
|
||||
new public surface, replacing `generateSummaryReport`'s stub entirely (depends on T005,
|
||||
T006)
|
||||
|
||||
**Checkpoint**: Config, schema, shared helpers, and module scaffolding in place. Each dashboard
|
||||
can now be built independently.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: User Story 1 - Management sees organization-wide support health (Priority: P1)
|
||||
|
||||
**Goal**: `GET /admin/reports/management` returns real figures per data-model.md's
|
||||
`ManagementDashboard` shape.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 1.
|
||||
|
||||
### Tests for User Story 1
|
||||
|
||||
- [x] T008 [P] [US1] Unit tests for T006's `computeRate`/`computeAverageSeconds` (empty input ->
|
||||
`null`; a real mix -> the correct value) in
|
||||
`tests/unit/platform/reports/rate-helpers.test.ts`
|
||||
|
||||
### Implementation for User Story 1
|
||||
|
||||
- [x] T009 [US1] Add `platform/reports/repository/management.repository.ts` — one method per
|
||||
research.md §2's Management table row (ticket counts by status, SLA-run outcome counts,
|
||||
response/resolution duration row-fetches for T006 to average) (depends on T007)
|
||||
- [x] T010 [US1] Add `ReportsService.getManagementDashboard(range)` composing T009's repository
|
||||
calls into the `ManagementDashboard` shape, reusing the `Resolution.resolvedBy` convention
|
||||
(research.md §4) for the AI-vs-human split (depends on T009)
|
||||
- [x] T011 [US1] Wire `GET /admin/reports/management` to the controller/service (depends on T010)
|
||||
- [x] T012 [US1] Integration test covering Quickstart Scenario 1 (real tickets in various
|
||||
terminal states, a met and a breached SLA run, verified figure-by-figure; a no-activity
|
||||
range returns all-zero counts and all-null rates) in
|
||||
`tests/integration/platform-reports/management-dashboard.test.ts` (depends on T011)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 1 passes.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: User Story 2 - See support broken down by product (Priority: P1)
|
||||
|
||||
**Goal**: `GET /admin/reports/product/:externalProductId` returns real figures per
|
||||
`ProductDashboard`.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 2.
|
||||
|
||||
### Implementation for User Story 2
|
||||
|
||||
- [x] T013 [P] [US2] Add `platform/reports/repository/product.repository.ts` — ticket/problem
|
||||
queries scoped by `productId`, plus a query against T003's `ErrorCodeLookup` table for
|
||||
the top-N ranking (`reportingConfig.topNLimit`) (depends on T007)
|
||||
- [x] T014 [US2] Add `ReportsService.getProductDashboard(externalProductId, range)`, 404-ing via
|
||||
`NotFoundError` when the product doesn't resolve (FR-006) before running any aggregation
|
||||
query (depends on T013)
|
||||
- [x] T015 [US2] Wire `GET /admin/reports/product/:externalProductId` (depends on T014)
|
||||
- [x] T016 [US2] Integration test covering Quickstart Scenario 2 (two products' data never
|
||||
cross-contaminating each other's figures; an unknown product 404s) in
|
||||
`tests/integration/platform-reports/product-dashboard.test.ts` (depends on T015)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 2 passes.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: User Story 3 - Support sees team workload and performance (Priority: P2)
|
||||
|
||||
**Goal**: `GET /admin/reports/support` returns real figures per `SupportDashboard`.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 3.
|
||||
|
||||
### Implementation for User Story 3
|
||||
|
||||
- [x] T017 [P] [US3] Add `platform/reports/repository/support.repository.ts` — current
|
||||
`Assignment` workload-by-agent query, `SLARun` at-risk/breached queries (`resolutionDueAt`
|
||||
within `reportingConfig.slaRiskThresholdMinutes` of now, per research.md §2) (depends on
|
||||
T007)
|
||||
- [x] T018 [US3] Add `ReportsService.getSupportDashboard(range)` (depends on T017)
|
||||
- [x] T019 [US3] Wire `GET /admin/reports/support` (depends on T018)
|
||||
- [x] T020 [US3] Integration test covering Quickstart Scenario 3 (real per-agent assignment
|
||||
counts; a near-due-but-not-breached run counted as at-risk, distinct from breached) in
|
||||
`tests/integration/platform-reports/support-dashboard.test.ts` (depends on T019)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 3 passes.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: User Story 4 - See how well the AI is performing (Priority: P2)
|
||||
|
||||
**Goal**: `GET /admin/reports/ai` returns real figures per `AiDashboard`.
|
||||
|
||||
**Independent Test**: Quickstart Scenario 4.
|
||||
|
||||
### Tests for User Story 4
|
||||
|
||||
- [x] T021 [P] [US4] Unit test: the confidence-distribution bucketing reuses
|
||||
`decideConfidenceBand` (005-ai-support) against `aiConfig` defaults, not a reimplemented
|
||||
threshold check, in `tests/unit/platform/reports/confidence-distribution.test.ts`
|
||||
|
||||
### Implementation for User Story 4
|
||||
|
||||
- [x] T022 [US4] Add `platform/reports/repository/ai.repository.ts` — `AISupportSession` outcome
|
||||
counts, `AIDiagnosis` confidence fetch, `AIKnowledgeReference` presence-per-session query,
|
||||
`AIAction`/`AIActionResult` outcome counts (depends on T007)
|
||||
- [x] T023 [US4] Add `ReportsService.getAiDashboard(range)`, bucketing confidence via
|
||||
`decideConfidenceBand` + `aiConfig.defaultHighConfidence`/`defaultLowConfidence`
|
||||
(research.md §7) (depends on T022, T021)
|
||||
- [x] T024 [US4] Wire `GET /admin/reports/ai` (depends on T023)
|
||||
- [x] T025 [US4] Integration test covering Quickstart Scenario 4 (real AI sessions to mixed
|
||||
outcomes, mixed tool results, a spread of diagnosis confidence values) in
|
||||
`tests/integration/platform-reports/ai-dashboard.test.ts` (depends on T024)
|
||||
|
||||
**Checkpoint**: Quickstart Scenario 4 passes. All four dashboards work independently and
|
||||
together — this feature's full scope.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Polish & Cross-Cutting Concerns
|
||||
|
||||
- [x] T026 [P] Update `specs/015-reporting-dashboards/checklists/requirements.md` Notes with any
|
||||
implementation-time findings
|
||||
- [x] T027 Run `npx tsx scripts/check-architecture.ts` and `npm run lint`/`npm run typecheck`
|
||||
- [x] T028 Full regression: `npm run test:unit` then the full integration suite against real
|
||||
Docker-provisioned Postgres/Redis, confirming nothing outside this feature regressed
|
||||
(particularly `error-codes.service.ts`'s own existing tests, now touched by T004)
|
||||
|
||||
---
|
||||
|
||||
## Dependencies & Execution Order
|
||||
|
||||
- **Foundational (Phase 1)**: No dependencies — BLOCKS all four user stories
|
||||
- **User Story 1 (Phase 2)**: Depends on Foundational — independent of US2/US3/US4
|
||||
- **User Story 2 (Phase 3)**: Depends on Foundational — independent of US1/US3/US4
|
||||
- **User Story 3 (Phase 4)**: Depends on Foundational — independent of US1/US2/US4
|
||||
- **User Story 4 (Phase 5)**: Depends on Foundational — independent of US1/US2/US3
|
||||
- **Polish (Phase 6)**: Depends on all four user stories
|
||||
@@ -24,6 +24,7 @@ import { solutionsRoutes } from '@/modules/problem-management/solutions';
|
||||
import { verificationRoutes } from '@/modules/problem-management/verification';
|
||||
import { resolutionsRoutes } from '@/modules/problem-management/resolutions';
|
||||
import { authRoutes } from '@/modules/identity/auth';
|
||||
import { reportsRoutes } from '@/modules/platform/reports';
|
||||
|
||||
export async function registerGlobalRoutes(app: FastifyInstance): Promise<void> {
|
||||
await app.register(healthRoutes);
|
||||
@@ -49,5 +50,6 @@ export async function registerGlobalRoutes(app: FastifyInstance): Promise<void>
|
||||
await app.register(solutionsRoutes);
|
||||
await app.register(verificationRoutes);
|
||||
await app.register(resolutionsRoutes);
|
||||
await app.register(reportsRoutes);
|
||||
// Further domain module routes will be registered here as feature modules are wired up
|
||||
}
|
||||
|
||||
@@ -74,6 +74,19 @@ const envSchema = z.object({
|
||||
PASSWORD_RESET_TOKEN_LIFETIME_MINUTES: z.coerce.number().default(30),
|
||||
LOGIN_RATE_LIMIT_MAX_ATTEMPTS: z.coerce.number().default(5),
|
||||
LOGIN_RATE_LIMIT_WINDOW_SECONDS: z.coerce.number().default(300),
|
||||
|
||||
// Full Observability (014) — the OpenTelemetry project's own standard env var name (not
|
||||
// invented here) for the collector endpoint spans are exported to. Unset means "no collector
|
||||
// configured" — tracing still runs, just exports to the console instead (never a startup
|
||||
// requirement) — see specs/014-full-observability/research.md "Distributed tracing".
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
|
||||
|
||||
// Reporting and Analytics Dashboards (015) — CONFIGURABLE per docs/10-implementation-
|
||||
// roadmap.md's own "never hardcode a placeholder value and ship it as final" instruction —
|
||||
// see specs/015-reporting-dashboards/research.md §8.
|
||||
REPORTING_DEFAULT_WINDOW_DAYS: z.coerce.number().default(30),
|
||||
REPORTING_SLA_RISK_THRESHOLD_MINUTES: z.coerce.number().default(60),
|
||||
REPORTING_TOP_N_LIMIT: z.coerce.number().default(10),
|
||||
});
|
||||
|
||||
export type EnvConfig = z.infer<typeof envSchema>;
|
||||
|
||||
@@ -7,3 +7,4 @@ export * from './ai';
|
||||
export * from './orchestration';
|
||||
export * from './problem-resolution';
|
||||
export * from './auth';
|
||||
export * from './reporting';
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { env } from './env';
|
||||
|
||||
export const reportingConfig = {
|
||||
defaultWindowDays: env.REPORTING_DEFAULT_WINDOW_DAYS,
|
||||
slaRiskThresholdMinutes: env.REPORTING_SLA_RISK_THRESHOLD_MINUTES,
|
||||
topNLimit: env.REPORTING_TOP_N_LIMIT,
|
||||
};
|
||||
@@ -1,9 +1,18 @@
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
import { eventBus } from '../event-bus';
|
||||
import { DomainEventName } from '../domain-events';
|
||||
import { BaseDomainEvent } from '../event-types';
|
||||
import { sessionsService } from '@/modules/ai-support/sessions';
|
||||
import { orchestrationService } from '@/modules/orchestration/orchestration';
|
||||
import { slaService } from '@/modules/orchestration/sla';
|
||||
import { ticketsService } from '@/modules/ticketing/tickets';
|
||||
import { resolutionRepository } from '@/modules/problem-management/resolutions';
|
||||
import {
|
||||
getTracer,
|
||||
ticketResolutionsCounter,
|
||||
ticketResolutionDurationHistogram,
|
||||
escalationsCounter,
|
||||
} from '@/infrastructure/observability';
|
||||
|
||||
interface TicketUpdatedPayload {
|
||||
ticketId: string;
|
||||
@@ -19,6 +28,14 @@ interface TicketAssignedPayload {
|
||||
actor: string;
|
||||
}
|
||||
|
||||
interface EscalationTriggeredPayload {
|
||||
ticketId: string;
|
||||
ruleId: string;
|
||||
targetNodeId: string;
|
||||
actor: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
let registered = false;
|
||||
|
||||
/**
|
||||
@@ -50,7 +67,44 @@ export function registerDomainEventHandlers(): void {
|
||||
DomainEventName.TICKET_UPDATED,
|
||||
async (event: BaseDomainEvent<TicketUpdatedPayload>) => {
|
||||
if (event.payload.newStatus !== 'HUMAN_ESCALATION') return;
|
||||
await orchestrationService.handleHumanEscalation(event.payload.ticketId);
|
||||
// 014-full-observability data-model.md: nests under session.service.ts's `ai.escalation`
|
||||
// span when this fired from that same await chain (an escalation triggered some other way
|
||||
// — e.g. a direct admin action — still gets its own root span here, never left untraced).
|
||||
await getTracer().startActiveSpan(
|
||||
'orchestration.assignment',
|
||||
{ attributes: { 'ticket.id': event.payload.ticketId } },
|
||||
async (span) => {
|
||||
try {
|
||||
await orchestrationService.handleHumanEscalation(event.payload.ticketId);
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// 014-full-observability data-model.md #3/#4: human-vs-AI resolution and resolution-time,
|
||||
// read off the Resolution row's own resolvedBy ("ai" | agentId — see prisma/schema.prisma)
|
||||
// rather than duplicating that distinction here.
|
||||
eventBus.subscribe(
|
||||
DomainEventName.TICKET_UPDATED,
|
||||
async (event: BaseDomainEvent<TicketUpdatedPayload>) => {
|
||||
if (event.payload.newStatus !== 'RESOLVED') return;
|
||||
const [ticket, resolution] = await Promise.all([
|
||||
ticketsService.getById(event.payload.ticketId),
|
||||
resolutionRepository.findByTicketId(event.payload.ticketId),
|
||||
]);
|
||||
if (!resolution) return;
|
||||
|
||||
ticketResolutionsCounter.inc({
|
||||
resolved_by: resolution.resolvedBy === 'ai' ? 'ai' : 'human',
|
||||
});
|
||||
ticketResolutionDurationHistogram.observe((Date.now() - ticket.createdAt.getTime()) / 1000);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -87,4 +141,14 @@ export function registerDomainEventHandlers(): void {
|
||||
await slaService.complete(event.payload.ticketId);
|
||||
},
|
||||
);
|
||||
|
||||
// 014-full-observability data-model.md #7: ESCALATION_TRIGGERED has been published
|
||||
// unconditionally on every escalation since 008-sla-escalation ("for audit, not for logic" —
|
||||
// escalation.service.ts's own comment) but had zero subscribers until now.
|
||||
eventBus.subscribe(
|
||||
DomainEventName.ESCALATION_TRIGGERED,
|
||||
async (event: BaseDomainEvent<EscalationTriggeredPayload>) => {
|
||||
escalationsCounter.inc({ reason: event.payload.reason });
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,3 +2,4 @@ export * from './logger';
|
||||
export * from './metrics';
|
||||
export * from './tracing';
|
||||
export * from './health.service';
|
||||
export * from './request-context.store';
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
import pino from 'pino';
|
||||
import { env } from '@/config';
|
||||
import { getRequestContextSnapshot } from './request-context.store';
|
||||
|
||||
const pinoOptions: pino.LoggerOptions = {
|
||||
level: env.LOG_LEVEL,
|
||||
base: {
|
||||
env: env.NODE_ENV,
|
||||
},
|
||||
// 014-full-observability FR-002: merges the current request's requestId/correlationId (if
|
||||
// any — a log call outside any request, e.g. at startup, gets neither) into every log line
|
||||
// made through this logger, anywhere in the codebase, with no change to any existing call
|
||||
// site. Pino applies these fields before the call's own object, so an explicit requestId a
|
||||
// call site already passes manually still wins.
|
||||
mixin: () => getRequestContextSnapshot() ?? {},
|
||||
};
|
||||
|
||||
if (env.NODE_ENV === 'development') {
|
||||
@@ -20,3 +27,7 @@ if (env.NODE_ENV === 'development') {
|
||||
}
|
||||
|
||||
export const logger = pino(pinoOptions);
|
||||
|
||||
// Exported so tests can build a real pino instance (same mixin, a different destination) rather
|
||||
// than mocking the logger itself — see tests/unit/observability/request-context-mixin.test.ts.
|
||||
export const loggerOptions = pinoOptions;
|
||||
|
||||
@@ -9,4 +9,70 @@ export const httpRequestDurationHistogram = new client.Histogram({
|
||||
buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 3, 5],
|
||||
});
|
||||
|
||||
// 014-full-observability data-model.md "Metrics (Prometheus, via prom-client)" — the eleven
|
||||
// named business-health metrics docs/09-testing-observability-cicd.md calls for, each a raw
|
||||
// counter/histogram for an external monitoring stack (FR-009 — no aggregation/dashboard logic
|
||||
// here). "Recurring problems" and "most common errors" are deliberately read directly off
|
||||
// problemsCreatedCounter/knownErrorLookupsCounter via a topk/rate query, not a separate metric.
|
||||
|
||||
export const aiSessionOutcomesCounter = new client.Counter({
|
||||
name: 'supporthub_ai_session_outcomes_total',
|
||||
help: 'Count of AI support sessions by terminal outcome',
|
||||
labelNames: ['outcome'],
|
||||
});
|
||||
|
||||
export const ticketResolutionsCounter = new client.Counter({
|
||||
name: 'supporthub_ticket_resolutions_total',
|
||||
help: 'Count of ticket resolutions by who resolved them',
|
||||
labelNames: ['resolved_by'],
|
||||
});
|
||||
|
||||
export const ticketResolutionDurationHistogram = new client.Histogram({
|
||||
name: 'supporthub_ticket_resolution_duration_seconds',
|
||||
help: 'Duration from ticket creation to resolution, in seconds',
|
||||
buckets: [60, 300, 900, 3600, 14400, 86400, 259200, 604800],
|
||||
});
|
||||
|
||||
export const ticketFirstResponseDurationHistogram = new client.Histogram({
|
||||
name: 'supporthub_ticket_first_response_duration_seconds',
|
||||
help: 'Duration from ticket creation to the first agent response, in seconds',
|
||||
buckets: [60, 300, 900, 3600, 14400, 86400],
|
||||
});
|
||||
|
||||
export const slaRunOutcomesCounter = new client.Counter({
|
||||
name: 'supporthub_sla_run_outcomes_total',
|
||||
help: 'Count of SLA runs by outcome',
|
||||
labelNames: ['outcome'],
|
||||
});
|
||||
|
||||
export const escalationsCounter = new client.Counter({
|
||||
name: 'supporthub_escalations_total',
|
||||
help: 'Count of escalation events by trigger reason',
|
||||
labelNames: ['reason'],
|
||||
});
|
||||
|
||||
export const problemsCreatedCounter = new client.Counter({
|
||||
name: 'supporthub_problems_created_total',
|
||||
help: 'Count of problems created, by category',
|
||||
labelNames: ['category_id'],
|
||||
});
|
||||
|
||||
export const knownErrorLookupsCounter = new client.Counter({
|
||||
name: 'supporthub_known_error_lookups_total',
|
||||
help: 'Count of known-issue lookups by error code',
|
||||
labelNames: ['code'],
|
||||
});
|
||||
|
||||
export const knowledgeRetrievalOutcomesCounter = new client.Counter({
|
||||
name: 'supporthub_knowledge_retrieval_outcomes_total',
|
||||
help: 'Count of AI knowledge-retrieval attempts by whether a match was found',
|
||||
labelNames: ['matched'],
|
||||
});
|
||||
|
||||
export const toolInvocationsCounter = new client.Counter({
|
||||
name: 'supporthub_tool_invocations_total',
|
||||
help: 'Count of AI tool invocations by tool and outcome',
|
||||
labelNames: ['tool', 'outcome'],
|
||||
});
|
||||
|
||||
export const metricsRegistry = client.register;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
|
||||
export interface RequestContextSnapshot {
|
||||
requestId: string;
|
||||
correlationId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 014-full-observability research.md §2: lets every log line produced through the shared
|
||||
* `logger` singleton — anywhere, any layer, no matter how deep the call stack — automatically
|
||||
* carry the current request's requestId/correlationId (via logger.ts's Pino `mixin`), without
|
||||
* threading `request`/`request.log` through every service and repository.
|
||||
*/
|
||||
export const requestContextStore = new AsyncLocalStorage<RequestContextSnapshot>();
|
||||
|
||||
export function getRequestContextSnapshot(): RequestContextSnapshot | undefined {
|
||||
return requestContextStore.getStore();
|
||||
}
|
||||
@@ -1,5 +1,83 @@
|
||||
import { trace, Tracer } from '@opentelemetry/api';
|
||||
import { trace, context, diag, DiagLogLevel, Tracer } from '@opentelemetry/api';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
BatchSpanProcessor,
|
||||
SimpleSpanProcessor,
|
||||
ConsoleSpanExporter,
|
||||
InMemorySpanExporter,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import type { SpanProcessor } from '@opentelemetry/sdk-trace';
|
||||
import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||
import { env } from '@/config';
|
||||
import { logger } from './logger';
|
||||
|
||||
/**
|
||||
* Without a registered ContextManager, the OpenTelemetry API's `context.active()` is a no-op
|
||||
* that does not propagate across async boundaries at all — `startActiveSpan` would only make a
|
||||
* span "active" for the literal synchronous extent of its callback, so a child span created
|
||||
* after an `await` (e.g. across this codebase's own event-bus `await eventBus.publish(...)`
|
||||
* chain, data-model.md's whole reason FR-006's two paths work) would silently come out as its
|
||||
* own unrelated root span instead of nesting. This is the tracing equivalent of the ALS-backed
|
||||
* request-context store — same mechanism, different consumer.
|
||||
*/
|
||||
context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
|
||||
|
||||
/**
|
||||
* 014-full-observability research.md §4: routes the OpenTelemetry SDK's own internal
|
||||
* diagnostics (span export failures included — FR-007) through this project's own log stream
|
||||
* instead of stderr/nowhere, at WARN so routine SDK chatter isn't logged at every span.
|
||||
*/
|
||||
diag.setLogger(
|
||||
{
|
||||
error: (msg, ...args) => logger.error({ otel: args }, msg),
|
||||
warn: (msg, ...args) => logger.warn({ otel: args }, msg),
|
||||
info: (msg, ...args) => logger.info({ otel: args }, msg),
|
||||
debug: (msg, ...args) => logger.debug({ otel: args }, msg),
|
||||
verbose: (msg, ...args) => logger.trace({ otel: args }, msg),
|
||||
},
|
||||
DiagLogLevel.WARN,
|
||||
);
|
||||
|
||||
let testSpanExporter: InMemorySpanExporter | undefined;
|
||||
|
||||
/**
|
||||
* Real infra, substituted destination only (same pattern as pino-pretty in development, or the
|
||||
* password-reset token's stub delivery) — never a mock of the tracer/provider itself:
|
||||
* - test: `InMemorySpanExporter`, so integration tests can read back real exported spans.
|
||||
* - `OTEL_EXPORTER_OTLP_ENDPOINT` set: real OTLP/HTTP export via `BatchSpanProcessor` (the
|
||||
* exporter reads the same env var itself for the actual collector URL — no need to hand-build
|
||||
* the `/v1/traces` path here).
|
||||
* - otherwise (local dev, or any environment with no collector configured): `ConsoleSpanExporter`
|
||||
* so spans are visible without standing one up.
|
||||
*/
|
||||
function buildSpanProcessor(): SpanProcessor {
|
||||
if (env.NODE_ENV === 'test') {
|
||||
testSpanExporter = new InMemorySpanExporter();
|
||||
return new SimpleSpanProcessor(testSpanExporter);
|
||||
}
|
||||
if (env.OTEL_EXPORTER_OTLP_ENDPOINT) {
|
||||
return new BatchSpanProcessor(new OTLPTraceExporter());
|
||||
}
|
||||
return new SimpleSpanProcessor(new ConsoleSpanExporter());
|
||||
}
|
||||
|
||||
const tracerProvider = new BasicTracerProvider({
|
||||
resource: resourceFromAttributes({ 'service.name': 'supporthub-api' }),
|
||||
spanProcessors: [buildSpanProcessor()],
|
||||
});
|
||||
|
||||
trace.setGlobalTracerProvider(tracerProvider);
|
||||
|
||||
export function getTracer(name = 'supporthub-api'): Tracer {
|
||||
return trace.getTracer(name);
|
||||
}
|
||||
|
||||
/** Test environment only — throws otherwise. See tests/integration/observability/tracing.test.ts. */
|
||||
export function getTestSpanExporter(): InMemorySpanExporter {
|
||||
if (!testSpanExporter) {
|
||||
throw new Error('getTestSpanExporter() is only available when NODE_ENV=test.');
|
||||
}
|
||||
return testSpanExporter;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { ErrorCodeLookup } from '@prisma/client';
|
||||
|
||||
/**
|
||||
* 015-reporting-dashboards research.md §6: a durable, append-only audit row — no update/delete
|
||||
* path, every lookup is its own row, duplicates over time are the point (frequency is what
|
||||
* "top errors" measures).
|
||||
*/
|
||||
export class ErrorCodeLookupRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
async create(errorCodeId: string, productId: string): Promise<ErrorCodeLookup> {
|
||||
return this.prisma.errorCodeLookup.create({ data: { errorCodeId, productId } });
|
||||
}
|
||||
|
||||
async countByCodeForProduct(
|
||||
productId: string,
|
||||
from: Date,
|
||||
to: Date,
|
||||
): Promise<Array<{ errorCodeId: string; count: number }>> {
|
||||
const grouped = await this.prisma.errorCodeLookup.groupBy({
|
||||
by: ['errorCodeId'],
|
||||
where: { productId, createdAt: { gte: from, lte: to } },
|
||||
_count: { errorCodeId: true },
|
||||
orderBy: { _count: { errorCodeId: 'desc' } },
|
||||
});
|
||||
return grouped.map((g) => ({ errorCodeId: g.errorCodeId, count: g._count.errorCodeId }));
|
||||
}
|
||||
}
|
||||
|
||||
export const errorCodeLookupRepository = new ErrorCodeLookupRepository();
|
||||
@@ -13,6 +13,10 @@ export class ErrorCodesRepository {
|
||||
where: { productId_code: { productId, code } },
|
||||
});
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<ErrorCode | null> {
|
||||
return this.prisma.errorCode.findUnique({ where: { id } });
|
||||
}
|
||||
}
|
||||
|
||||
export const errorCodesRepository = new ErrorCodesRepository();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export * from './knowledge.repository';
|
||||
export * from './error-codes.repository';
|
||||
export * from './error-code-lookup.repository';
|
||||
export * from './known-issues.repository';
|
||||
export * from './runbooks.repository';
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { ErrorCode, KnownIssue } from '@prisma/client';
|
||||
import { NotFoundError } from '@/common/errors';
|
||||
import { knownErrorLookupsCounter } from '@/infrastructure/observability';
|
||||
import {
|
||||
errorCodesRepository,
|
||||
ErrorCodesRepository,
|
||||
errorCodeLookupRepository,
|
||||
ErrorCodeLookupRepository,
|
||||
knownIssuesRepository,
|
||||
KnownIssuesRepository,
|
||||
CreateKnownIssueData,
|
||||
@@ -12,6 +15,7 @@ export class ErrorCodesService {
|
||||
constructor(
|
||||
private readonly errorCodesRepo: ErrorCodesRepository = errorCodesRepository,
|
||||
private readonly knownIssuesRepo: KnownIssuesRepository = knownIssuesRepository,
|
||||
private readonly lookupsRepo: ErrorCodeLookupRepository = errorCodeLookupRepository,
|
||||
) {}
|
||||
|
||||
async createErrorCode(productId: string, code: string, description: string): Promise<ErrorCode> {
|
||||
@@ -26,8 +30,37 @@ export class ErrorCodesService {
|
||||
async findKnownIssuesByErrorCode(productId: string, code: string): Promise<KnownIssue[]> {
|
||||
const errorCode = await this.errorCodesRepo.findByCode(productId, code);
|
||||
if (!errorCode) throw new NotFoundError('Error code not found.');
|
||||
|
||||
// 014-full-observability data-model.md #9: "most common errors" — a raw, process-lifetime
|
||||
// counter for a live monitoring stack (FR-009 there), counted only once the code is
|
||||
// confirmed real.
|
||||
knownErrorLookupsCounter.inc({ code });
|
||||
// 015-reporting-dashboards research.md §6: the durable counterpart — the live counter above
|
||||
// resets on every restart, so a historical "top errors" report needs its own audit row.
|
||||
await this.lookupsRepo.create(errorCode.id, productId);
|
||||
|
||||
return this.knownIssuesRepo.findByErrorCodeId(errorCode.id);
|
||||
}
|
||||
|
||||
/** 015-reporting-dashboards: the Product dashboard's "top errors" ranking — encapsulated here
|
||||
* (not exposed as raw repository access) since resolving a lookup count back to its error
|
||||
* code's own `code` string is this module's own concern, not the reports module's. */
|
||||
async getTopErrorCodesForProduct(
|
||||
productId: string,
|
||||
from: Date,
|
||||
to: Date,
|
||||
limit: number,
|
||||
): Promise<Array<{ code: string; count: number }>> {
|
||||
const ranked = await this.lookupsRepo.countByCodeForProduct(productId, from, to);
|
||||
const top = ranked.slice(0, limit);
|
||||
const rows = await Promise.all(
|
||||
top.map(async (row) => {
|
||||
const errorCode = await this.errorCodesRepo.findById(row.errorCodeId);
|
||||
return { code: errorCode?.code ?? row.errorCodeId, count: row.count };
|
||||
}),
|
||||
);
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
export const errorCodesService = new ErrorCodesService();
|
||||
|
||||
@@ -3,11 +3,20 @@ export { SessionsService, sessionsService } from './service';
|
||||
export type { SessionTurnResult } from './service';
|
||||
export { ConfidencePolicyService, confidencePolicyService } from './service';
|
||||
export type { ResolvedConfidencePolicy } from './service';
|
||||
// 015-reporting-dashboards research.md §7: reused for the AI dashboard's confidence
|
||||
// distribution, not reimplemented.
|
||||
export { decideConfidenceBand } from './service';
|
||||
export type { ConfidenceBand } from './service';
|
||||
export {
|
||||
sessionRepository,
|
||||
SessionRepository,
|
||||
diagnosisRepository,
|
||||
DiagnosisRepository,
|
||||
// 015-reporting-dashboards: test setup needs to record a session's knowledge references
|
||||
// directly, the same "extend an existing module's public surface for a later feature"
|
||||
// precedent as 004's productsRepository/009's problemsRepository.
|
||||
knowledgeReferenceRepository,
|
||||
KnowledgeReferenceRepository,
|
||||
} from './repository';
|
||||
export { ACTIVE_SESSION_STATUSES, TERMINAL_SESSION_STATUSES } from './mapper';
|
||||
export type { SessionStatus } from './mapper';
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { AISupportSession } from '@prisma/client';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { aiSessionOutcomesCounter } from '@/infrastructure/observability';
|
||||
import { ACTIVE_SESSION_STATUSES } from '../mapper';
|
||||
|
||||
export class SessionRepository {
|
||||
@@ -38,10 +39,19 @@ export class SessionRepository {
|
||||
async updateStatus(sessionId: string, status: string): Promise<AISupportSession> {
|
||||
const isTerminal =
|
||||
status === 'resolved' || status === 'escalated' || status === 'ended_by_agent';
|
||||
return this.prisma.aISupportSession.update({
|
||||
const updated = await this.prisma.aISupportSession.update({
|
||||
where: { id: sessionId },
|
||||
data: { status, ...(isTerminal ? { endedAt: new Date() } : {}) },
|
||||
});
|
||||
|
||||
// 014-full-observability data-model.md: the single choke point every escalation/resolution
|
||||
// branch in session.service.ts funnels through (research.md §5's "why the repository layer"
|
||||
// — observability calls are already a cross-cutting concern used from any layer here).
|
||||
if (status === 'resolved' || status === 'escalated') {
|
||||
aiSessionOutcomesCounter.inc({ outcome: status });
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
async setActiveRunbook(sessionId: string, runbookKey: string, stepIndex: number): Promise<void> {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
import { AISupportSession, KnowledgeEntry, AIInteraction, Ticket, Problem } from '@prisma/client';
|
||||
import { AppError, NotFoundError } from '@/common/errors';
|
||||
import { getTracer } from '@/infrastructure/observability';
|
||||
import { ticketsService, problemsRepository } from '@/modules/ticketing/tickets';
|
||||
import { messagesService } from '@/modules/ticketing/messages';
|
||||
import { knowledgeService } from '@/modules/ai-support/knowledge';
|
||||
@@ -125,17 +127,35 @@ export class SessionsService {
|
||||
reason: string,
|
||||
stepsAttempted: string[] = [],
|
||||
) {
|
||||
const diagnosis = await this.diagnoses.findLatestBySession(session.id);
|
||||
const result = this.escalation.buildSummary(diagnosis, reason, stepsAttempted);
|
||||
// 014-full-observability data-model.md — root span for the AI-escalation -> orchestration/
|
||||
// assignment path (FR-006): syncTicketStatus below publishes TICKET_UPDATED synchronously,
|
||||
// and the orchestration subscriber's own span (src/events/handlers/index.ts) nests under
|
||||
// this one automatically via OTel's active-context propagation through that same await chain.
|
||||
return getTracer().startActiveSpan(
|
||||
'ai.escalation',
|
||||
{ attributes: { 'ticket.id': ticketId, 'session.id': session.id } },
|
||||
async (span) => {
|
||||
try {
|
||||
const diagnosis = await this.diagnoses.findLatestBySession(session.id);
|
||||
const result = this.escalation.buildSummary(diagnosis, reason, stepsAttempted);
|
||||
|
||||
if (!(ACTIVE_SESSION_STATUSES as readonly string[]).includes(session.status)) {
|
||||
return result;
|
||||
}
|
||||
if (!(ACTIVE_SESSION_STATUSES as readonly string[]).includes(session.status)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
await this.sessions.updateStatus(session.id, 'escalated');
|
||||
await syncTicketStatus(ticketId, 'escalated');
|
||||
await this.sessions.updateStatus(session.id, 'escalated');
|
||||
await syncTicketStatus(ticketId, 'escalated');
|
||||
|
||||
return result;
|
||||
return result;
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async runDiagnosisTurn(
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import {
|
||||
toolInvocationsCounter,
|
||||
knowledgeRetrievalOutcomesCounter,
|
||||
} from '@/infrastructure/observability';
|
||||
import { actionRepository, ActionRepository } from '../repository';
|
||||
import { evaluateToolProposal } from './policy-gate';
|
||||
import { executeTool, ToolExecutionContext } from './tool-executor';
|
||||
@@ -58,6 +62,15 @@ export class ToolsService {
|
||||
const result = await executeTool(block.name, block.input, context);
|
||||
await this.actions.createResult(action.id, result.output, result.status);
|
||||
|
||||
// 014-full-observability data-model.md #10/#11: the single choke point every tool
|
||||
// invocation passes through — labeled by outcome, and (for the knowledge-search tool
|
||||
// specifically) by whether it found anything.
|
||||
toolInvocationsCounter.inc({ tool: block.name, outcome: result.status });
|
||||
if (block.name === 'searchProductKnowledge') {
|
||||
const matched = Array.isArray(result.output) && result.output.length > 0;
|
||||
knowledgeRetrievalOutcomesCounter.inc({ matched: String(matched) });
|
||||
}
|
||||
|
||||
if (result.status === 'failed') anyFailed = true;
|
||||
if (block.name === 'escalateToHuman' && result.status === 'success') {
|
||||
const output = result.output as { reason?: string };
|
||||
|
||||
@@ -3,6 +3,7 @@ import { NotFoundError, ValidationError } from '@/common/errors';
|
||||
import { ticketsService } from '@/modules/ticketing/tickets';
|
||||
import { messagesService } from '@/modules/ticketing/messages';
|
||||
import { escalationService, EscalationService } from '@/modules/orchestration/escalation';
|
||||
import { slaRunOutcomesCounter } from '@/infrastructure/observability';
|
||||
import {
|
||||
slaPolicyRepository,
|
||||
SlaPolicyRepository,
|
||||
@@ -135,6 +136,14 @@ export class SlaService {
|
||||
const run = await this.runs.findByTicketId(ticketId);
|
||||
if (!run || run.status === 'completed') return;
|
||||
|
||||
// 014-full-observability data-model.md #6: read BEFORE the update below — a run already
|
||||
// 'breached' by the time it resolves was already counted breached by the sweep and must
|
||||
// never also be counted 'met' here, even though this update still (pre-existing behavior,
|
||||
// unrelated to this feature — see research.md §5) overwrites its status to 'completed'.
|
||||
if (run.status !== 'breached') {
|
||||
slaRunOutcomesCounter.inc({ outcome: 'met' });
|
||||
}
|
||||
|
||||
await this.runs.update(run.id, { status: 'completed', completedAt: new Date() });
|
||||
}
|
||||
|
||||
@@ -151,6 +160,7 @@ export class SlaService {
|
||||
const resolutionBreaches = await this.runs.findRunningPastResolutionDueAt(now);
|
||||
for (const run of resolutionBreaches) {
|
||||
await this.runs.update(run.id, { status: 'breached', breachedAt: now });
|
||||
slaRunOutcomesCounter.inc({ outcome: 'breached' });
|
||||
await this.escalation.handleBreach(run.ticketId, 'resolution_breach');
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from './reports.controller';
|
||||
@@ -0,0 +1,39 @@
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { reportsService, ReportsService } from '../service';
|
||||
import { dateRangeQuerySchema } from '../schema';
|
||||
import { resolveDateRange } from '../mapper';
|
||||
|
||||
export class ReportsController {
|
||||
constructor(private readonly service: ReportsService = reportsService) {}
|
||||
|
||||
async getManagementDashboard(request: FastifyRequest, reply: FastifyReply) {
|
||||
const query = dateRangeQuerySchema.parse(request.query);
|
||||
const range = resolveDateRange(query);
|
||||
const dashboard = await this.service.getManagementDashboard(range);
|
||||
return reply.status(200).send({ success: true, data: dashboard, meta: null });
|
||||
}
|
||||
|
||||
async getProductDashboard(request: FastifyRequest, reply: FastifyReply) {
|
||||
const { externalProductId } = request.params as { externalProductId: string };
|
||||
const query = dateRangeQuerySchema.parse(request.query);
|
||||
const range = resolveDateRange(query);
|
||||
const dashboard = await this.service.getProductDashboard(externalProductId, range);
|
||||
return reply.status(200).send({ success: true, data: dashboard, meta: null });
|
||||
}
|
||||
|
||||
async getSupportDashboard(request: FastifyRequest, reply: FastifyReply) {
|
||||
const query = dateRangeQuerySchema.parse(request.query);
|
||||
const range = resolveDateRange(query);
|
||||
const dashboard = await this.service.getSupportDashboard(range);
|
||||
return reply.status(200).send({ success: true, data: dashboard, meta: null });
|
||||
}
|
||||
|
||||
async getAiDashboard(request: FastifyRequest, reply: FastifyReply) {
|
||||
const query = dateRangeQuerySchema.parse(request.query);
|
||||
const range = resolveDateRange(query);
|
||||
const dashboard = await this.service.getAiDashboard(range);
|
||||
return reply.status(200).send({ success: true, data: dashboard, meta: null });
|
||||
}
|
||||
}
|
||||
|
||||
export const reportsController = new ReportsController();
|
||||
@@ -1,11 +1,8 @@
|
||||
export const REPORTS_CONSTANTS = {
|
||||
MODULE_NAME: 'PLATFORM_REPORTS',
|
||||
} as const;
|
||||
|
||||
export class ReportsService {
|
||||
async generateSummaryReport(): Promise<Record<string, unknown>> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export const reportsService = new ReportsService();
|
||||
export { reportsRoutes } from './routes';
|
||||
export { ReportsService, reportsService } from './service';
|
||||
export type {
|
||||
ManagementDashboard,
|
||||
ProductDashboard,
|
||||
SupportDashboard,
|
||||
AiDashboard,
|
||||
} from './service';
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { ValidationError } from '@/common/errors';
|
||||
import { reportingConfig } from '@/config';
|
||||
|
||||
export interface DateRange {
|
||||
from: Date;
|
||||
to: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* 015-reporting-dashboards data-model.md "Query Parameters": both ends optional — `to` defaults
|
||||
* to now, `from` defaults to `to - reportingConfig.defaultWindowDays`. `from > to` is a
|
||||
* ValidationError (spec.md Edge Cases), never silently swapped or silently returning empty data.
|
||||
*/
|
||||
export function resolveDateRange(query: {
|
||||
from?: string | undefined;
|
||||
to?: string | undefined;
|
||||
}): DateRange {
|
||||
const to = query.to ? new Date(query.to) : new Date();
|
||||
if (Number.isNaN(to.getTime())) {
|
||||
throw new ValidationError('"to" is not a valid date.');
|
||||
}
|
||||
|
||||
const from = query.from
|
||||
? new Date(query.from)
|
||||
: new Date(to.getTime() - reportingConfig.defaultWindowDays * 24 * 60 * 60 * 1000);
|
||||
if (Number.isNaN(from.getTime())) {
|
||||
throw new ValidationError('"from" is not a valid date.');
|
||||
}
|
||||
|
||||
if (from > to) {
|
||||
throw new ValidationError('"from" must not be after "to".');
|
||||
}
|
||||
|
||||
return { from, to };
|
||||
}
|
||||
|
||||
export function serializeDateRange(range: DateRange): { from: string; to: string } {
|
||||
return { from: range.from.toISOString(), to: range.to.toISOString() };
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
interface TicketWithFirstAgentMessage {
|
||||
createdAt: Date;
|
||||
messages: Array<{ createdAt: Date }>;
|
||||
}
|
||||
|
||||
/** Shared by ManagementRepository and SupportRepository — both need "ticket createdAt -> its
|
||||
* first AGENT_MESSAGE createdAt" in milliseconds, for tickets that actually have one. */
|
||||
export function extractFirstResponseDurationsMs(tickets: TicketWithFirstAgentMessage[]): number[] {
|
||||
return tickets.flatMap((t) => {
|
||||
const firstAgentMessage = t.messages[0];
|
||||
if (!firstAgentMessage) return [];
|
||||
return [firstAgentMessage.createdAt.getTime() - t.createdAt.getTime()];
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './date-range';
|
||||
export * from './rate';
|
||||
export * from './durations';
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* 015-reporting-dashboards research.md §3: every rate/average is `number | null` — `null` means
|
||||
* "no qualifying data in range," distinguished from a genuine `0` (e.g. a real 0% AI resolution
|
||||
* rate is meaningful; "nobody's data exists yet" is not the same thing). Never computed as
|
||||
* `numerator / 0`, which would silently produce `NaN`.
|
||||
*/
|
||||
export function computeRate(numerator: number, denominator: number): number | null {
|
||||
if (denominator === 0) return null;
|
||||
return numerator / denominator;
|
||||
}
|
||||
|
||||
export function computeAverageSeconds(durationsMs: number[]): number | null {
|
||||
if (durationsMs.length === 0) return null;
|
||||
const totalMs = durationsMs.reduce((sum, ms) => sum + ms, 0);
|
||||
return totalMs / durationsMs.length / 1000;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { DateRange } from '../mapper';
|
||||
|
||||
export class AiRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
async sessionOutcomeCounts(
|
||||
range: DateRange,
|
||||
): Promise<{ resolved: number; escalated: number; total: number }> {
|
||||
const [resolved, escalated, total] = await Promise.all([
|
||||
this.prisma.aISupportSession.count({
|
||||
where: { startedAt: { gte: range.from, lte: range.to }, status: 'resolved' },
|
||||
}),
|
||||
this.prisma.aISupportSession.count({
|
||||
where: {
|
||||
startedAt: { gte: range.from, lte: range.to },
|
||||
status: { in: ['escalated', 'ended_by_agent'] },
|
||||
},
|
||||
}),
|
||||
this.prisma.aISupportSession.count({
|
||||
where: { startedAt: { gte: range.from, lte: range.to } },
|
||||
}),
|
||||
]);
|
||||
return { resolved, escalated, total };
|
||||
}
|
||||
|
||||
/**
|
||||
* "Failed troubleshooting then escalated" (spec.md User Story 4) has no single stored flag —
|
||||
* classifyStepOutcome's own verdicts aren't persisted as a durable per-step record. Documented
|
||||
* proxy: an escalated session that made at least one tool call (toolCallCount > 0) attempted
|
||||
* troubleshooting before giving up, vs. one that escalated immediately with zero attempts.
|
||||
*/
|
||||
async escalatedSessionsWithToolAttempts(range: DateRange): Promise<number> {
|
||||
return this.prisma.aISupportSession.count({
|
||||
where: {
|
||||
startedAt: { gte: range.from, lte: range.to },
|
||||
status: { in: ['escalated', 'ended_by_agent'] },
|
||||
toolCallCount: { gt: 0 },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async sessionsWithKnowledgeMatch(range: DateRange): Promise<number> {
|
||||
const sessions = await this.prisma.aISupportSession.findMany({
|
||||
where: { startedAt: { gte: range.from, lte: range.to } },
|
||||
select: { knowledgeRefs: { select: { id: true }, take: 1 } },
|
||||
});
|
||||
return sessions.filter((s) => s.knowledgeRefs.length > 0).length;
|
||||
}
|
||||
|
||||
async diagnosisConfidences(range: DateRange): Promise<number[]> {
|
||||
const diagnoses = await this.prisma.aIDiagnosis.findMany({
|
||||
where: { createdAt: { gte: range.from, lte: range.to } },
|
||||
select: { confidence: true },
|
||||
});
|
||||
return diagnoses.map((d) => d.confidence);
|
||||
}
|
||||
|
||||
async toolInvocationOutcomeCounts(
|
||||
range: DateRange,
|
||||
): Promise<{ success: number; failed: number }> {
|
||||
const [success, failed] = await Promise.all([
|
||||
this.prisma.aIActionResult.count({
|
||||
where: { status: 'success', createdAt: { gte: range.from, lte: range.to } },
|
||||
}),
|
||||
this.prisma.aIActionResult.count({
|
||||
where: { status: 'failed', createdAt: { gte: range.from, lte: range.to } },
|
||||
}),
|
||||
]);
|
||||
return { success, failed };
|
||||
}
|
||||
}
|
||||
|
||||
export const aiRepository = new AiRepository();
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from './management.repository';
|
||||
export * from './product.repository';
|
||||
export * from './support.repository';
|
||||
export * from './ai.repository';
|
||||
export * from './shared.repository';
|
||||
@@ -0,0 +1,79 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { DateRange } from '../mapper';
|
||||
|
||||
export class ManagementRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
async totalCases(range: DateRange): Promise<number> {
|
||||
return this.prisma.ticket.count({
|
||||
where: { createdAt: { gte: range.from, lte: range.to } },
|
||||
});
|
||||
}
|
||||
|
||||
async countByStatus(range: DateRange, statuses: string[]): Promise<number> {
|
||||
return this.prisma.ticket.count({
|
||||
where: { createdAt: { gte: range.from, lte: range.to }, status: { in: statuses } },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Ever escalated to a human. NOT a current-status check: 003-ticketing's own state machine
|
||||
* lets both the AI path (AI_RESOLVED) and the human path (HUMAN_ESCALATION) converge on the
|
||||
* same shared terminal statuses (RESOLUTION_PENDING_CUSTOMER, RESOLVED, CLOSED, REOPENED are
|
||||
* all reachable from AI_RESOLVED directly, per ticket-state-machine.ts's own transition
|
||||
* table) — a status-list check over those shared statuses would count every AI-resolved
|
||||
* ticket as "human escalated" too (caught via manual verification against real seeded data,
|
||||
* not by any test fixture, since every existing test's fixtures happened to keep the two
|
||||
* paths' terminal statuses apart).
|
||||
*
|
||||
* The unambiguous, direct signal instead: 007-orchestration-assignment's own
|
||||
* `orchestrationService.handleHumanEscalation` is the *only* code path that ever creates an
|
||||
* `Assignment` row (research.md's own module map) — a ticket has one if and only if it was
|
||||
* actually escalated to a human at some point, regardless of its current status.
|
||||
*/
|
||||
async countEverEscalatedToHuman(range: DateRange): Promise<number> {
|
||||
return this.prisma.ticket.count({
|
||||
where: {
|
||||
createdAt: { gte: range.from, lte: range.to },
|
||||
assignments: { some: {} },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** research.md §4: `Resolution.resolvedBy` is the single source of truth for AI vs. human. */
|
||||
async countResolutionsBy(range: DateRange, resolvedByAi: boolean): Promise<number> {
|
||||
return this.prisma.resolution.count({
|
||||
where: {
|
||||
resolvedAt: { gte: range.from, lte: range.to },
|
||||
resolvedBy: resolvedByAi ? 'ai' : { not: 'ai' },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async slaOutcomeCounts(range: DateRange): Promise<{ met: number; breached: number }> {
|
||||
const [met, breached] = await Promise.all([
|
||||
this.prisma.sLARun.count({
|
||||
where: {
|
||||
ticket: { createdAt: { gte: range.from, lte: range.to } },
|
||||
status: 'completed',
|
||||
breachedAt: null,
|
||||
},
|
||||
}),
|
||||
this.prisma.sLARun.count({
|
||||
where: {
|
||||
ticket: { createdAt: { gte: range.from, lte: range.to } },
|
||||
breachedAt: { not: null },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
return { met, breached };
|
||||
}
|
||||
|
||||
async escalationCount(range: DateRange): Promise<number> {
|
||||
return this.prisma.escalationEvent.count({
|
||||
where: { createdAt: { gte: range.from, lte: range.to } },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const managementRepository = new ManagementRepository();
|
||||
@@ -0,0 +1,57 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { DateRange } from '../mapper';
|
||||
|
||||
// Named ProductReportRepository (not ProductRepository) to avoid colliding with
|
||||
// catalog/products' own ProductsRepository, which this module reuses (via its public index) for
|
||||
// resolving externalProductId -> Product rather than duplicating that lookup here.
|
||||
export class ProductReportRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
async supportVolume(productId: string, range: DateRange): Promise<number> {
|
||||
return this.prisma.ticket.count({
|
||||
where: { productId, createdAt: { gte: range.from, lte: range.to } },
|
||||
});
|
||||
}
|
||||
|
||||
async problemsByCategory(
|
||||
productId: string,
|
||||
range: DateRange,
|
||||
): Promise<Array<{ categoryId: string | null; count: number }>> {
|
||||
const grouped = await this.prisma.problem.groupBy({
|
||||
by: ['categoryId'],
|
||||
where: { productId, createdAt: { gte: range.from, lte: range.to } },
|
||||
_count: { categoryId: true },
|
||||
orderBy: { _count: { categoryId: 'desc' } },
|
||||
});
|
||||
return grouped.map((g) => ({ categoryId: g.categoryId, count: g._count.categoryId }));
|
||||
}
|
||||
|
||||
async countResolutionsBy(
|
||||
productId: string,
|
||||
range: DateRange,
|
||||
resolvedByAi: boolean,
|
||||
): Promise<number> {
|
||||
return this.prisma.resolution.count({
|
||||
where: {
|
||||
resolvedAt: { gte: range.from, lte: range.to },
|
||||
resolvedBy: resolvedByAi ? 'ai' : { not: 'ai' },
|
||||
ticket: { productId },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Same fixed "has at least one Assignment row" signal as
|
||||
* ManagementRepository.countEverEscalatedToHuman (see its own comment for why a current-status
|
||||
* check is wrong), scoped to one product. */
|
||||
async countEverEscalatedToHuman(productId: string, range: DateRange): Promise<number> {
|
||||
return this.prisma.ticket.count({
|
||||
where: {
|
||||
productId,
|
||||
createdAt: { gte: range.from, lte: range.to },
|
||||
assignments: { some: {} },
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const productReportRepository = new ProductReportRepository();
|
||||
@@ -0,0 +1,34 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { DateRange, extractFirstResponseDurationsMs } from '../mapper';
|
||||
|
||||
/** Response/resolution duration queries the Management and Support dashboards both need
|
||||
* identically — composed by each, not duplicated. */
|
||||
export class SharedReportRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
async firstResponseDurationsMs(range: DateRange): Promise<number[]> {
|
||||
const tickets = await this.prisma.ticket.findMany({
|
||||
where: { createdAt: { gte: range.from, lte: range.to } },
|
||||
select: {
|
||||
createdAt: true,
|
||||
messages: {
|
||||
where: { type: 'AGENT_MESSAGE' },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
take: 1,
|
||||
select: { createdAt: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
return extractFirstResponseDurationsMs(tickets);
|
||||
}
|
||||
|
||||
async resolutionDurationsMs(range: DateRange): Promise<number[]> {
|
||||
const resolutions = await this.prisma.resolution.findMany({
|
||||
where: { resolvedAt: { gte: range.from, lte: range.to } },
|
||||
select: { resolvedAt: true, ticket: { select: { createdAt: true } } },
|
||||
});
|
||||
return resolutions.map((r) => r.resolvedAt.getTime() - r.ticket.createdAt.getTime());
|
||||
}
|
||||
}
|
||||
|
||||
export const sharedReportRepository = new SharedReportRepository();
|
||||
@@ -0,0 +1,40 @@
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { DateRange } from '../mapper';
|
||||
|
||||
export class SupportRepository {
|
||||
constructor(private readonly prisma = prismaClient) {}
|
||||
|
||||
/** research.md §2: current, point-in-time — not range-scoped. "How much work is assigned
|
||||
* right now," not a historical count. */
|
||||
async workloadByAgent(): Promise<Array<{ agentId: string; openAssignments: number }>> {
|
||||
const grouped = await this.prisma.assignment.groupBy({
|
||||
by: ['agentId'],
|
||||
where: { isCurrent: true },
|
||||
_count: { agentId: true },
|
||||
});
|
||||
return grouped.map((g) => ({ agentId: g.agentId, openAssignments: g._count.agentId }));
|
||||
}
|
||||
|
||||
async slaAtRisk(thresholdMinutes: number): Promise<number> {
|
||||
const now = new Date();
|
||||
const riskCutoff = new Date(now.getTime() + thresholdMinutes * 60 * 1000);
|
||||
return this.prisma.sLARun.count({
|
||||
where: {
|
||||
status: 'running',
|
||||
resolutionDueAt: { gte: now, lte: riskCutoff },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async slaBreached(): Promise<number> {
|
||||
return this.prisma.sLARun.count({ where: { status: 'breached' } });
|
||||
}
|
||||
|
||||
async escalationCount(range: DateRange): Promise<number> {
|
||||
return this.prisma.escalationEvent.count({
|
||||
where: { createdAt: { gte: range.from, lte: range.to } },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const supportRepository = new SupportRepository();
|
||||
@@ -0,0 +1 @@
|
||||
export * from './reports.routes';
|
||||
@@ -0,0 +1,28 @@
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { requireRole } from '@/modules/identity/auth';
|
||||
import { reportsController } from '../controller';
|
||||
|
||||
/** contracts/reports-api-contract.md: every dashboard is admin-only, the same gate every other
|
||||
* admin-only surface uses since 010-identity-auth. */
|
||||
export async function reportsRoutes(fastify: FastifyInstance): Promise<void> {
|
||||
fastify.get(
|
||||
'/admin/reports/management',
|
||||
{ preHandler: [fastify.authenticate, requireRole('ADMIN')] },
|
||||
(req, reply) => reportsController.getManagementDashboard(req, reply),
|
||||
);
|
||||
fastify.get(
|
||||
'/admin/reports/product/:externalProductId',
|
||||
{ preHandler: [fastify.authenticate, requireRole('ADMIN')] },
|
||||
(req, reply) => reportsController.getProductDashboard(req, reply),
|
||||
);
|
||||
fastify.get(
|
||||
'/admin/reports/support',
|
||||
{ preHandler: [fastify.authenticate, requireRole('ADMIN')] },
|
||||
(req, reply) => reportsController.getSupportDashboard(req, reply),
|
||||
);
|
||||
fastify.get(
|
||||
'/admin/reports/ai',
|
||||
{ preHandler: [fastify.authenticate, requireRole('ADMIN')] },
|
||||
(req, reply) => reportsController.getAiDashboard(req, reply),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from './reports.schema';
|
||||
@@ -0,0 +1,10 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
export const dateRangeQuerySchema = z
|
||||
.object({
|
||||
from: z.string().optional(),
|
||||
to: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type DateRangeQuery = z.infer<typeof dateRangeQuerySchema>;
|
||||
@@ -0,0 +1 @@
|
||||
export * from './reports.service';
|
||||
@@ -0,0 +1,219 @@
|
||||
import { NotFoundError } from '@/common/errors';
|
||||
import { reportingConfig, aiConfig } from '@/config';
|
||||
import { productsRepository, ProductsRepository } from '@/modules/catalog/products';
|
||||
import { decideConfidenceBand } from '@/modules/ai-support/sessions';
|
||||
import { errorCodesService, ErrorCodesService } from '@/modules/ai-support/knowledge';
|
||||
import {
|
||||
managementRepository,
|
||||
ManagementRepository,
|
||||
productReportRepository,
|
||||
ProductReportRepository,
|
||||
supportRepository,
|
||||
SupportRepository,
|
||||
aiRepository,
|
||||
AiRepository,
|
||||
sharedReportRepository,
|
||||
SharedReportRepository,
|
||||
} from '../repository';
|
||||
import { DateRange, serializeDateRange, computeRate, computeAverageSeconds } from '../mapper';
|
||||
|
||||
export interface ManagementDashboard {
|
||||
range: { from: string; to: string };
|
||||
totalCases: number;
|
||||
aiResolved: number;
|
||||
humanEscalated: number;
|
||||
resolved: number;
|
||||
open: number;
|
||||
slaCompliance: { met: number; breached: number; rate: number | null };
|
||||
escalationCount: number;
|
||||
averageResponseSeconds: number | null;
|
||||
averageResolutionSeconds: number | null;
|
||||
}
|
||||
|
||||
export interface ProductDashboard {
|
||||
productId: string;
|
||||
range: { from: string; to: string };
|
||||
supportVolume: number;
|
||||
problemsByCategory: Array<{ categoryId: string | null; count: number }>;
|
||||
recurringProblems: Array<{ categoryId: string | null; count: number }>;
|
||||
aiResolutionRate: number | null;
|
||||
humanEscalationRate: number | null;
|
||||
topErrors: Array<{ code: string; count: number }>;
|
||||
}
|
||||
|
||||
export interface SupportDashboard {
|
||||
generatedAt: string;
|
||||
range: { from: string; to: string };
|
||||
workloadByAgent: Array<{ agentId: string; openAssignments: number }>;
|
||||
slaAtRisk: number;
|
||||
slaBreached: number;
|
||||
escalationCount: number;
|
||||
averageResponseSeconds: number | null;
|
||||
averageResolutionSeconds: number | null;
|
||||
}
|
||||
|
||||
export interface AiDashboard {
|
||||
range: { from: string; to: string };
|
||||
totalSessions: number;
|
||||
aiResolutionRate: number | null;
|
||||
humanHandoffRate: number | null;
|
||||
failedTroubleshootingEscalationRate: number | null;
|
||||
knowledgeMatchRate: number | null;
|
||||
confidenceDistribution: { proceed: number; ask: number; escalate: number };
|
||||
toolInvocations: { success: number; failed: number };
|
||||
}
|
||||
|
||||
const RESOLVED_STATUSES = ['RESOLVED', 'CLOSED'];
|
||||
|
||||
export class ReportsService {
|
||||
constructor(
|
||||
private readonly management: ManagementRepository = managementRepository,
|
||||
private readonly productReports: ProductReportRepository = productReportRepository,
|
||||
private readonly support: SupportRepository = supportRepository,
|
||||
private readonly ai: AiRepository = aiRepository,
|
||||
private readonly products: ProductsRepository = productsRepository,
|
||||
private readonly errorCodes: ErrorCodesService = errorCodesService,
|
||||
private readonly shared: SharedReportRepository = sharedReportRepository,
|
||||
) {}
|
||||
|
||||
async getManagementDashboard(range: DateRange): Promise<ManagementDashboard> {
|
||||
const [
|
||||
totalCases,
|
||||
aiResolved,
|
||||
humanEscalated,
|
||||
resolved,
|
||||
slaOutcomes,
|
||||
escalationCount,
|
||||
responseDurations,
|
||||
resolutionDurations,
|
||||
] = await Promise.all([
|
||||
this.management.totalCases(range),
|
||||
this.management.countResolutionsBy(range, true),
|
||||
this.management.countEverEscalatedToHuman(range),
|
||||
this.management.countByStatus(range, RESOLVED_STATUSES),
|
||||
this.management.slaOutcomeCounts(range),
|
||||
this.management.escalationCount(range),
|
||||
this.shared.firstResponseDurationsMs(range),
|
||||
this.shared.resolutionDurationsMs(range),
|
||||
]);
|
||||
|
||||
return {
|
||||
range: serializeDateRange(range),
|
||||
totalCases,
|
||||
aiResolved,
|
||||
humanEscalated,
|
||||
resolved,
|
||||
open: totalCases - resolved,
|
||||
slaCompliance: {
|
||||
met: slaOutcomes.met,
|
||||
breached: slaOutcomes.breached,
|
||||
rate: computeRate(slaOutcomes.met, slaOutcomes.met + slaOutcomes.breached),
|
||||
},
|
||||
escalationCount,
|
||||
averageResponseSeconds: computeAverageSeconds(responseDurations),
|
||||
averageResolutionSeconds: computeAverageSeconds(resolutionDurations),
|
||||
};
|
||||
}
|
||||
|
||||
async getProductDashboard(
|
||||
externalProductId: string,
|
||||
range: DateRange,
|
||||
): Promise<ProductDashboard> {
|
||||
const product = await this.products.findByExternalProductId(externalProductId);
|
||||
if (!product) throw new NotFoundError('Product not found.');
|
||||
|
||||
const [supportVolume, problemsByCategory, aiResolvedCount, humanEscalatedCount, topErrors] =
|
||||
await Promise.all([
|
||||
this.productReports.supportVolume(product.id, range),
|
||||
this.productReports.problemsByCategory(product.id, range),
|
||||
this.productReports.countResolutionsBy(product.id, range, true),
|
||||
this.productReports.countEverEscalatedToHuman(product.id, range),
|
||||
this.errorCodes.getTopErrorCodesForProduct(
|
||||
product.id,
|
||||
range.from,
|
||||
range.to,
|
||||
reportingConfig.topNLimit,
|
||||
),
|
||||
]);
|
||||
|
||||
const recurringProblems = [...problemsByCategory]
|
||||
.sort((a, b) => b.count - a.count)
|
||||
.slice(0, reportingConfig.topNLimit);
|
||||
|
||||
return {
|
||||
productId: externalProductId,
|
||||
range: serializeDateRange(range),
|
||||
supportVolume,
|
||||
problemsByCategory,
|
||||
recurringProblems,
|
||||
aiResolutionRate: computeRate(aiResolvedCount, supportVolume),
|
||||
humanEscalationRate: computeRate(humanEscalatedCount, supportVolume),
|
||||
topErrors,
|
||||
};
|
||||
}
|
||||
|
||||
async getSupportDashboard(range: DateRange): Promise<SupportDashboard> {
|
||||
const [
|
||||
workloadByAgent,
|
||||
slaAtRisk,
|
||||
slaBreached,
|
||||
escalationCount,
|
||||
responseDurations,
|
||||
resolutionDurations,
|
||||
] = await Promise.all([
|
||||
this.support.workloadByAgent(),
|
||||
this.support.slaAtRisk(reportingConfig.slaRiskThresholdMinutes),
|
||||
this.support.slaBreached(),
|
||||
this.support.escalationCount(range),
|
||||
this.shared.firstResponseDurationsMs(range),
|
||||
this.shared.resolutionDurationsMs(range),
|
||||
]);
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
range: serializeDateRange(range),
|
||||
workloadByAgent,
|
||||
slaAtRisk,
|
||||
slaBreached,
|
||||
escalationCount,
|
||||
averageResponseSeconds: computeAverageSeconds(responseDurations),
|
||||
averageResolutionSeconds: computeAverageSeconds(resolutionDurations),
|
||||
};
|
||||
}
|
||||
|
||||
async getAiDashboard(range: DateRange): Promise<AiDashboard> {
|
||||
const [outcomeCounts, escalatedWithAttempts, knowledgeMatches, confidences, toolCounts] =
|
||||
await Promise.all([
|
||||
this.ai.sessionOutcomeCounts(range),
|
||||
this.ai.escalatedSessionsWithToolAttempts(range),
|
||||
this.ai.sessionsWithKnowledgeMatch(range),
|
||||
this.ai.diagnosisConfidences(range),
|
||||
this.ai.toolInvocationOutcomeCounts(range),
|
||||
]);
|
||||
|
||||
const confidenceDistribution = { proceed: 0, ask: 0, escalate: 0 };
|
||||
for (const confidence of confidences) {
|
||||
const band = decideConfidenceBand(confidence, {
|
||||
highThreshold: aiConfig.defaultHighConfidence,
|
||||
lowThreshold: aiConfig.defaultLowConfidence,
|
||||
});
|
||||
confidenceDistribution[band] += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
range: serializeDateRange(range),
|
||||
totalSessions: outcomeCounts.total,
|
||||
aiResolutionRate: computeRate(outcomeCounts.resolved, outcomeCounts.total),
|
||||
humanHandoffRate: computeRate(outcomeCounts.escalated, outcomeCounts.total),
|
||||
failedTroubleshootingEscalationRate: computeRate(
|
||||
escalatedWithAttempts,
|
||||
outcomeCounts.escalated,
|
||||
),
|
||||
knowledgeMatchRate: computeRate(knowledgeMatches, outcomeCounts.total),
|
||||
confidenceDistribution,
|
||||
toolInvocations: toolCounts,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const reportsService = new ReportsService();
|
||||
@@ -1,4 +1,6 @@
|
||||
import { TicketMessage } from '@prisma/client';
|
||||
import { ticketsRepository } from '@/modules/ticketing/tickets';
|
||||
import { ticketFirstResponseDurationHistogram } from '@/infrastructure/observability';
|
||||
import { messagesRepository, MessagesRepository } from '../repository';
|
||||
import { MessageType, isVisibleToCustomer } from '../mapper';
|
||||
|
||||
@@ -13,13 +15,32 @@ export class MessagesService {
|
||||
type: MessageType,
|
||||
body: string,
|
||||
): Promise<TicketMessage> {
|
||||
return this.repo.create({
|
||||
// 014-full-observability data-model.md #5: checked BEFORE creating the new message, so it
|
||||
// reflects "is there already an agent response" at the moment this one is being posted.
|
||||
// Benign race (research.md §5/plan.md Constraint) — two concurrent first responses could
|
||||
// both observe once — acceptable for a best-effort metric, not a business-correctness path.
|
||||
const isFirstAgentMessage =
|
||||
type === 'AGENT_MESSAGE' &&
|
||||
!(await this.repo.findAll(ticketId)).some((m) => m.type === 'AGENT_MESSAGE');
|
||||
|
||||
const message = await this.repo.create({
|
||||
ticketId,
|
||||
authorRef,
|
||||
type,
|
||||
body,
|
||||
visibleToCustomer: isVisibleToCustomer(type),
|
||||
});
|
||||
|
||||
if (isFirstAgentMessage) {
|
||||
const ticket = await ticketsRepository.findById(ticketId);
|
||||
if (ticket) {
|
||||
ticketFirstResponseDurationHistogram.observe(
|
||||
(message.createdAt.getTime() - ticket.createdAt.getTime()) / 1000,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
async listForCustomer(ticketId: string): Promise<TicketMessage[]> {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
import { Ticket } from '@prisma/client';
|
||||
import { AppError } from '@/common/errors';
|
||||
import { getTracer, problemsCreatedCounter } from '@/infrastructure/observability';
|
||||
import {
|
||||
ticketsRepository,
|
||||
TicketsRepository,
|
||||
@@ -48,10 +50,31 @@ export class TicketsService {
|
||||
async createFromInboundRequest(
|
||||
input: InboundTicketRequest,
|
||||
): Promise<{ ticket: Ticket; wasExisting: boolean }> {
|
||||
const span = getTracer().startSpan('ticket.create', {
|
||||
attributes: { 'product.externalProductId': input.externalProductId },
|
||||
});
|
||||
try {
|
||||
const result = await this.doCreateFromInboundRequest(input);
|
||||
span.setAttribute('ticket.id', result.ticket.id);
|
||||
return result;
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
private async doCreateFromInboundRequest(
|
||||
input: InboundTicketRequest,
|
||||
): Promise<{ ticket: Ticket; wasExisting: boolean }> {
|
||||
const existingProblem = input.referenceIds?.length
|
||||
? await this.problemsRepo.findByReference(input.referenceIds)
|
||||
: null;
|
||||
|
||||
const problem =
|
||||
(input.referenceIds?.length
|
||||
? await this.problemsRepo.findByReference(input.referenceIds)
|
||||
: null) ??
|
||||
existingProblem ??
|
||||
(await this.problemsRepo.create({
|
||||
statement: input.problem,
|
||||
symptoms: input.problem,
|
||||
@@ -59,6 +82,14 @@ export class TicketsService {
|
||||
severity: 'medium',
|
||||
}));
|
||||
|
||||
// 014-full-observability: "recurring problems" — a raw counter, ranked/aggregated by an
|
||||
// external monitoring stack (spec.md FR-009/Assumptions), not computed here.
|
||||
if (!existingProblem) {
|
||||
problemsCreatedCounter.inc({
|
||||
category_id: problem.categoryId ?? 'uncategorized',
|
||||
});
|
||||
}
|
||||
|
||||
const year = new Date().getFullYear();
|
||||
const codePrefix = `${deriveProductCode(input.externalProductId)}-${year}-`;
|
||||
let attempt = 0;
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { FastifyPluginAsync, FastifyRequest } from 'fastify';
|
||||
import { FastifyPluginAsync, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import fp from 'fastify-plugin';
|
||||
import { generateUuid } from '@/common/utils';
|
||||
import { RequestContext } from '@/common/types';
|
||||
import { APP_CONSTANTS } from '@/common/constants';
|
||||
import {
|
||||
requestContextStore,
|
||||
logger,
|
||||
httpRequestDurationHistogram,
|
||||
} from '@/infrastructure/observability';
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
@@ -10,8 +15,17 @@ declare module 'fastify' {
|
||||
}
|
||||
}
|
||||
|
||||
function routeLabel(request: FastifyRequest): string {
|
||||
return request.routeOptions?.url ?? 'unmatched';
|
||||
}
|
||||
|
||||
const requestContextPluginCallback: FastifyPluginAsync = async (fastify) => {
|
||||
fastify.addHook('onRequest', async (request: FastifyRequest, reply) => {
|
||||
// Callback-style (not async) so `done` is available to hand to requestContextStore.run —
|
||||
// everything Fastify does next for this request (remaining hooks, the route handler, and this
|
||||
// plugin's own onResponse hook below) runs as a continuation of this call, so it all inherits
|
||||
// the ALS context (research.md §2/§1 — Node's AsyncLocalStorage propagates through a
|
||||
// continuation chain, not just the literal synchronous call).
|
||||
fastify.addHook('onRequest', (request: FastifyRequest, reply: FastifyReply, done) => {
|
||||
const rawReqId = request.headers[APP_CONSTANTS.REQUEST_ID_HEADER];
|
||||
const rawCorrId = request.headers[APP_CONSTANTS.CORRELATION_HEADER];
|
||||
|
||||
@@ -25,6 +39,42 @@ const requestContextPluginCallback: FastifyPluginAsync = async (fastify) => {
|
||||
|
||||
reply.header(APP_CONSTANTS.REQUEST_ID_HEADER, requestId);
|
||||
reply.header(APP_CONSTANTS.CORRELATION_HEADER, correlationId);
|
||||
|
||||
requestContextStore.run({ requestId, correlationId }, done);
|
||||
});
|
||||
|
||||
// 014-full-observability FR-001/FR-003: fires for every completed response, including 404s
|
||||
// and replies sent early by another hook (e.g. rate limiting) — nothing is silently unlogged.
|
||||
fastify.addHook('onResponse', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const method = request.method;
|
||||
const route = routeLabel(request);
|
||||
const statusCode = reply.statusCode;
|
||||
const durationSeconds = reply.elapsedTime / 1000;
|
||||
|
||||
httpRequestDurationHistogram.observe(
|
||||
{ method, route, status_code: String(statusCode) },
|
||||
durationSeconds,
|
||||
);
|
||||
|
||||
const logPayload = {
|
||||
event: 'http_request_completed',
|
||||
method,
|
||||
route,
|
||||
statusCode,
|
||||
durationMs: reply.elapsedTime,
|
||||
// Explicit here (not left to the mixin alone), matching this codebase's existing
|
||||
// convention (app.ts's error handler already does the same) — the access log is the one
|
||||
// line an operator most needs to grep by requestId without knowing about mixin internals.
|
||||
requestId: request.reqContext?.requestId,
|
||||
correlationId: request.reqContext?.correlationId,
|
||||
};
|
||||
if (statusCode >= 500) {
|
||||
logger.error(logPayload, `${method} ${route} ${statusCode}`);
|
||||
} else if (statusCode >= 400) {
|
||||
logger.warn(logPayload, `${method} ${route} ${statusCode}`);
|
||||
} else {
|
||||
logger.info(logPayload, `${method} ${route} ${statusCode}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -21,6 +21,9 @@ describe('Error codes and known issues', () => {
|
||||
|
||||
afterAll(async () => {
|
||||
await prismaClient.knownIssue.deleteMany({ where: { product: { externalProductId } } });
|
||||
// 015-reporting-dashboards: findKnownIssuesByErrorCode now also writes a durable
|
||||
// ErrorCodeLookup row (RESTRICT FK to ErrorCode) — must be deleted before ErrorCode itself.
|
||||
await prismaClient.errorCodeLookup.deleteMany({ where: { product: { externalProductId } } });
|
||||
await prismaClient.errorCode.deleteMany({ where: { product: { externalProductId } } });
|
||||
await prismaClient.product.deleteMany({ where: { externalProductId } });
|
||||
await app.close();
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi, MockInstance } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { logger } from '@/infrastructure/observability';
|
||||
|
||||
/** Covers specs/014-full-observability/quickstart.md Scenario 1 against a real running app. */
|
||||
describe('Per-request access log (User Story 1)', () => {
|
||||
let app: FastifyInstance;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
function callsData(spy: MockInstance): Record<string, unknown>[] {
|
||||
return spy.mock.calls.map(([data]: unknown[]) => data as Record<string, unknown>);
|
||||
}
|
||||
|
||||
function accessLogCalls(spy: MockInstance): Record<string, unknown>[] {
|
||||
return callsData(spy).filter((data) => data?.event === 'http_request_completed');
|
||||
}
|
||||
|
||||
it('emits exactly one access-log line for a successful request, carrying a requestId', async () => {
|
||||
const infoSpy = vi.spyOn(logger, 'info');
|
||||
|
||||
const res = await app.inject({ method: 'GET', url: '/health/live' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
|
||||
const lines = accessLogCalls(infoSpy);
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(lines[0]).toMatchObject({ method: 'GET', route: '/health/live', statusCode: 200 });
|
||||
expect(lines[0]?.requestId).toBeTruthy();
|
||||
expect(lines[0]?.requestId).toBe(res.headers['x-request-id']);
|
||||
|
||||
infoSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('correlates the access-log line with other log lines produced for the same request', async () => {
|
||||
const warnSpy = vi.spyOn(logger, 'warn');
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/auth/login',
|
||||
payload: { email: `nobody-${Date.now()}@supporthub.test`, password: 'wrong' },
|
||||
});
|
||||
expect(res.statusCode).toBe(401);
|
||||
|
||||
const accessLine = accessLogCalls(warnSpy)[0];
|
||||
expect(accessLine).toBeDefined();
|
||||
|
||||
const authFailureLine = callsData(warnSpy).find((data) => data?.code === 'UNAUTHORIZED');
|
||||
expect(authFailureLine).toBeDefined();
|
||||
|
||||
expect(authFailureLine?.requestId).toBe(accessLine?.requestId);
|
||||
expect(accessLine?.requestId).toBe(res.headers['x-request-id']);
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('still emits an access-log line for a 404', async () => {
|
||||
const warnSpy = vi.spyOn(logger, 'warn');
|
||||
|
||||
const res = await app.inject({ method: 'GET', url: '/this-route-does-not-exist' });
|
||||
expect(res.statusCode).toBe(404);
|
||||
|
||||
const lines = accessLogCalls(warnSpy);
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(lines[0]).toMatchObject({ statusCode: 404 });
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,346 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { loginAs, authHeader } from '../../helpers/auth';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { sessionsService, sessionRepository } from '@/modules/ai-support/sessions';
|
||||
import { ticketsService, ticketsRepository } from '@/modules/ticketing/tickets';
|
||||
import { messagesService } from '@/modules/ticketing/messages';
|
||||
import { resolutionRepository } from '@/modules/problem-management/resolutions';
|
||||
import { slaService, slaRunRepository } from '@/modules/orchestration/sla';
|
||||
import { escalationService } from '@/modules/orchestration/escalation';
|
||||
import { errorCodesService } from '@/modules/ai-support/knowledge';
|
||||
import { toolsService } from '@/modules/ai-support/tools';
|
||||
|
||||
/**
|
||||
* Covers specs/014-full-observability/quickstart.md Scenario 4 against a real Postgres/Redis —
|
||||
* every named business-health metric, scraped from the real /metrics endpoint before and after
|
||||
* driving its real underlying event through the real service layer (not mocked). Several flows
|
||||
* (human resolution, SLA runs) create rows directly against the repositories that already own
|
||||
* the relevant validation elsewhere in this codebase's own test suite — this file's job is only
|
||||
* to prove the metric increments at the correct point, not to re-verify those modules' own
|
||||
* business rules (already covered by problem-resolution-flow.test.ts / sla-escalation-flow.test.ts).
|
||||
*/
|
||||
describe('Business-health metrics (User Story 4)', () => {
|
||||
let app: FastifyInstance;
|
||||
let authToken: string;
|
||||
const externalProductId = `TEST_BIZ_METRICS_PROD_${Date.now()}`;
|
||||
let productId: string;
|
||||
let secret: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
authToken = await loginAs(app, 'ADMIN');
|
||||
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Business Metrics Test Product', status: 'active' },
|
||||
});
|
||||
productId = product.id;
|
||||
secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicket(): Promise<string> {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: `Business metrics test ${Date.now()}-${Math.random()}`,
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
async function scrape(): Promise<string> {
|
||||
const res = await app.inject({ method: 'GET', url: '/metrics' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
return res.body;
|
||||
}
|
||||
|
||||
function metricValue(body: string, name: string, labels?: Record<string, string>): number {
|
||||
const labelPart = labels
|
||||
? `\\{${Object.entries(labels)
|
||||
.map(([k, v]) => `${k}="${v}"`)
|
||||
.join(',')}\\}`
|
||||
: '(?:\\{\\})?';
|
||||
const match = body.match(new RegExp(`${name}${labelPart}\\s+([0-9.]+)`));
|
||||
return match?.[1] ? parseFloat(match[1]) : 0;
|
||||
}
|
||||
|
||||
it('counts an AI session resolving without escalating', async () => {
|
||||
const ticketId = await createTicket();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
|
||||
outcome: 'resolved',
|
||||
});
|
||||
await sessionRepository.updateStatus(session.id, 'resolved');
|
||||
const after = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
|
||||
outcome: 'resolved',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts an AI session escalating, and nothing for resolved', async () => {
|
||||
const ticketId = await createTicket();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
|
||||
outcome: 'escalated',
|
||||
});
|
||||
await sessionsService.escalate(session, ticketId, 'Escalating for metrics test.');
|
||||
const after = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
|
||||
outcome: 'escalated',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts the first agent message on a ticket, observing first-response duration', async () => {
|
||||
const ticketId = await createTicket();
|
||||
|
||||
const bodyBefore = await scrape();
|
||||
const before = metricValue(
|
||||
bodyBefore,
|
||||
'supporthub_ticket_first_response_duration_seconds_count',
|
||||
);
|
||||
|
||||
await messagesService.post(ticketId, 'agent-1', 'AGENT_MESSAGE', 'Hi, looking into this.');
|
||||
// A second agent message must NOT observe again.
|
||||
await messagesService.post(ticketId, 'agent-1', 'AGENT_MESSAGE', 'Following up.');
|
||||
|
||||
const after = metricValue(
|
||||
await scrape(),
|
||||
'supporthub_ticket_first_response_duration_seconds_count',
|
||||
);
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts a human resolution and observes resolution duration when a ticket reaches RESOLVED', async () => {
|
||||
const ticketId = await createTicket();
|
||||
|
||||
// Reach RESOLUTION_PENDING_CUSTOMER via the repository directly (bypassing
|
||||
// ticketsService.updateStatus's domain-event publish) — this test only cares about the
|
||||
// final RESOLVED transition and the Resolution row's own resolvedBy, not the intermediate
|
||||
// states, and going through the real event bus here would trigger a REAL, unscoped
|
||||
// HUMAN_ESCALATION auto-assignment against the default strategy — which can land on some
|
||||
// other concurrently-running test file's own dedicated agent (a real cross-file
|
||||
// contamination this test caused once, fixed here by not publishing those events at all).
|
||||
let ticket = await prismaClient.ticket.findUniqueOrThrow({ where: { id: ticketId } });
|
||||
for (const status of ['HUMAN_ESCALATION', 'IN_PROGRESS', 'RESOLUTION_PENDING_CUSTOMER']) {
|
||||
const updated = await ticketsRepository.updateStatus(ticketId, status, ticket.version);
|
||||
if (!updated) throw new Error(`Failed to transition ticket to ${status} in test setup.`);
|
||||
ticket = updated;
|
||||
}
|
||||
await resolutionRepository.create({ ticketId, outcome: 'fixed', resolvedBy: 'agent-1' });
|
||||
|
||||
const bodyBefore = await scrape();
|
||||
const resolvedBefore = metricValue(bodyBefore, 'supporthub_ticket_resolutions_total', {
|
||||
resolved_by: 'human',
|
||||
});
|
||||
const durationBefore = metricValue(
|
||||
bodyBefore,
|
||||
'supporthub_ticket_resolution_duration_seconds_count',
|
||||
);
|
||||
|
||||
await ticketsService.updateStatus(ticketId, 'RESOLVED', ticket.version, 'agent-1');
|
||||
|
||||
const bodyAfter = await scrape();
|
||||
expect(
|
||||
metricValue(bodyAfter, 'supporthub_ticket_resolutions_total', { resolved_by: 'human' }),
|
||||
).toBe(resolvedBefore + 1);
|
||||
expect(metricValue(bodyAfter, 'supporthub_ticket_resolution_duration_seconds_count')).toBe(
|
||||
durationBefore + 1,
|
||||
);
|
||||
});
|
||||
|
||||
it('counts an SLA run completing on time as met', async () => {
|
||||
const policy = await prismaClient.sLAPolicy.create({
|
||||
data: {
|
||||
name: `Metrics Policy ${Date.now()}`,
|
||||
productId,
|
||||
firstResponseMinutes: 30,
|
||||
resolutionMinutes: 240,
|
||||
},
|
||||
});
|
||||
const ticketId = await createTicket();
|
||||
await slaRunRepository.create({
|
||||
ticketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(Date.now() + 240 * 60_000),
|
||||
});
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
|
||||
outcome: 'met',
|
||||
});
|
||||
await slaService.complete(ticketId);
|
||||
const after = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
|
||||
outcome: 'met',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts an overdue SLA run as breached via the sweep (at least once — a shared sweep may also catch unrelated overdue runs)', async () => {
|
||||
const policy = await prismaClient.sLAPolicy.create({
|
||||
data: {
|
||||
name: `Metrics Breach Policy ${Date.now()}`,
|
||||
productId,
|
||||
firstResponseMinutes: 30,
|
||||
resolutionMinutes: 1,
|
||||
},
|
||||
});
|
||||
const ticketId = await createTicket();
|
||||
await slaRunRepository.create({
|
||||
ticketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(Date.now() - 60_000),
|
||||
});
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
|
||||
outcome: 'breached',
|
||||
});
|
||||
await slaService.runBreachDetectionSweep();
|
||||
const after = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
|
||||
outcome: 'breached',
|
||||
});
|
||||
|
||||
expect(after).toBeGreaterThanOrEqual(before + 1);
|
||||
});
|
||||
|
||||
it('counts a manual escalation event by reason', async () => {
|
||||
const node = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/admin/hierarchy-nodes',
|
||||
headers: authHeader(authToken),
|
||||
payload: {
|
||||
name: `Metrics Node ${Date.now()}`,
|
||||
order: 0,
|
||||
productScope: [externalProductId],
|
||||
skills: [],
|
||||
assignmentStrategy: 'ROUND_ROBIN',
|
||||
},
|
||||
});
|
||||
const nodeId = node.json().data.id as string;
|
||||
const ticketId = await createTicket();
|
||||
const reason = `metrics-test-reason-${Date.now()}`;
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_escalations_total', { reason });
|
||||
await escalationService.escalateManually(ticketId, nodeId, 'admin-test', reason);
|
||||
const after = metricValue(await scrape(), 'supporthub_escalations_total', { reason });
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts a problem created, labeled by category (uncategorized here)', async () => {
|
||||
const before = metricValue(await scrape(), 'supporthub_problems_created_total', {
|
||||
category_id: 'uncategorized',
|
||||
});
|
||||
await createTicket();
|
||||
const after = metricValue(await scrape(), 'supporthub_problems_created_total', {
|
||||
category_id: 'uncategorized',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts a knowledge-search tool call that finds nothing as unmatched', async () => {
|
||||
const ticketId = await createTicket();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_knowledge_retrieval_outcomes_total', {
|
||||
matched: 'false',
|
||||
});
|
||||
await toolsService.proposeAndEvaluate(
|
||||
session.id,
|
||||
[
|
||||
{
|
||||
type: 'tool_use',
|
||||
caller: { type: 'direct' },
|
||||
id: `toolu_${Date.now()}`,
|
||||
name: 'searchProductKnowledge',
|
||||
input: { feature: `nonexistent-feature-${Date.now()}` },
|
||||
},
|
||||
],
|
||||
{ ticketId, productId },
|
||||
);
|
||||
const after = metricValue(await scrape(), 'supporthub_knowledge_retrieval_outcomes_total', {
|
||||
matched: 'false',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts a tool invocation that fails at execution time', async () => {
|
||||
const ticketId = await createTicket();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_tool_invocations_total', {
|
||||
tool: 'getTicketSnapshot',
|
||||
outcome: 'failed',
|
||||
});
|
||||
await toolsService.proposeAndEvaluate(
|
||||
session.id,
|
||||
[
|
||||
{
|
||||
type: 'tool_use',
|
||||
caller: { type: 'direct' },
|
||||
id: `toolu_${Date.now()}`,
|
||||
name: 'getTicketSnapshot',
|
||||
input: {},
|
||||
},
|
||||
],
|
||||
{ ticketId: 'nonexistent-ticket-id', productId },
|
||||
);
|
||||
const after = metricValue(await scrape(), 'supporthub_tool_invocations_total', {
|
||||
tool: 'getTicketSnapshot',
|
||||
outcome: 'failed',
|
||||
});
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
it('counts a valid known-error-code lookup', async () => {
|
||||
const code = `METRICS-ERR-${Date.now()}`;
|
||||
await errorCodesService.createErrorCode(productId, code, 'A test error for metrics.');
|
||||
|
||||
const before = metricValue(await scrape(), 'supporthub_known_error_lookups_total', { code });
|
||||
await errorCodesService.findKnownIssuesByErrorCode(productId, code);
|
||||
const after = metricValue(await scrape(), 'supporthub_known_error_lookups_total', { code });
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
/** Covers specs/014-full-observability/quickstart.md Scenario 2 against a real running app. */
|
||||
describe('Live request-health metrics (User Story 2)', () => {
|
||||
let app: FastifyInstance;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('records request-duration observations labeled by method/route/status for both success and failure', async () => {
|
||||
const email = `metrics-test-${Date.now()}@supporthub.test`;
|
||||
await app.inject({ method: 'POST', url: '/auth/login', payload: { email, password: 'wrong' } });
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/auth/login',
|
||||
payload: { email, password: 'wrong2' },
|
||||
});
|
||||
|
||||
const metricsRes = await app.inject({ method: 'GET', url: '/metrics' });
|
||||
expect(metricsRes.statusCode).toBe(200);
|
||||
|
||||
expect(metricsRes.body).toMatch(
|
||||
/supporthub_http_request_duration_seconds_count\{method="POST",route="\/auth\/login",status_code="401"\}\s+\d+/,
|
||||
);
|
||||
|
||||
// A second scrape's body reflects the first scrape's own request too — proves 2xx routes
|
||||
// are observed just as 4xx ones are, not only error paths.
|
||||
const secondScrape = await app.inject({ method: 'GET', url: '/metrics' });
|
||||
expect(secondScrape.body).toMatch(
|
||||
/supporthub_http_request_duration_seconds_count\{method="GET",route="\/metrics",status_code="200"\}\s+\d+/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { sessionsService, sessionRepository } from '@/modules/ai-support/sessions';
|
||||
import { getTestSpanExporter } from '@/infrastructure/observability';
|
||||
|
||||
/**
|
||||
* Covers specs/014-full-observability/quickstart.md Scenario 3, against a real running app.
|
||||
* Reads spans back from getTestSpanExporter() (a real TracerProvider, real spans — only the
|
||||
* export *destination* is swapped for an in-memory one, per research.md §4) rather than through
|
||||
* an external collector.
|
||||
*
|
||||
* Drives the escalation path directly via sessionsService.escalate(...) instead of through a
|
||||
* real AI reasoning turn — the tracing behavior under test (span creation/nesting) is identical
|
||||
* either way, and this avoids requiring a paid ANTHROPIC_API_KEY for every test run (see
|
||||
* ai-verification-and-escalation.test.ts's own `describe.skipIf(!hasRealApiKey)` for the
|
||||
* alternative this project already uses when a real reasoning call is actually required).
|
||||
*/
|
||||
describe('Cross-module trace (User Story 3)', () => {
|
||||
let app: FastifyInstance;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicketViaInboundRequest(): Promise<string> {
|
||||
const externalProductId = `TEST_TRACE_PROD_${Date.now()}_${Math.random().toString(36).slice(2)}`;
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Tracing Test Product', status: 'active' },
|
||||
});
|
||||
const secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: 'Needs a human, for tracing.',
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
it('produces a ticket.create span for ticket intake', async () => {
|
||||
getTestSpanExporter().reset();
|
||||
|
||||
await createTicketViaInboundRequest();
|
||||
|
||||
const spans = getTestSpanExporter().getFinishedSpans();
|
||||
const createSpan = spans.find((s) => s.name === 'ticket.create');
|
||||
expect(createSpan).toBeDefined();
|
||||
expect(createSpan?.attributes['ticket.id']).toBeTruthy();
|
||||
});
|
||||
|
||||
it('nests orchestration.assignment under ai.escalation, sharing one trace', async () => {
|
||||
getTestSpanExporter().reset();
|
||||
|
||||
const ticketId = await createTicketViaInboundRequest();
|
||||
const session = await sessionRepository.create(ticketId);
|
||||
await sessionsService.escalate(session, ticketId, 'Escalating for tracing test.');
|
||||
|
||||
const spans = getTestSpanExporter().getFinishedSpans();
|
||||
const escalationSpan = spans.find((s) => s.name === 'ai.escalation');
|
||||
const assignmentSpan = spans.find((s) => s.name === 'orchestration.assignment');
|
||||
|
||||
expect(escalationSpan).toBeDefined();
|
||||
expect(assignmentSpan).toBeDefined();
|
||||
expect(assignmentSpan?.spanContext().traceId).toBe(escalationSpan?.spanContext().traceId);
|
||||
expect(assignmentSpan?.parentSpanContext?.spanId).toBe(escalationSpan?.spanContext().spanId);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,156 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { loginAs, authHeader } from '../../helpers/auth';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import {
|
||||
sessionRepository,
|
||||
diagnosisRepository,
|
||||
knowledgeReferenceRepository,
|
||||
} from '@/modules/ai-support/sessions';
|
||||
import { actionRepository } from '@/modules/ai-support/tools';
|
||||
import { aiConfig } from '@/config';
|
||||
|
||||
/** Covers specs/015-reporting-dashboards/quickstart.md Scenario 4 against a real Postgres/Redis. */
|
||||
describe('AI dashboard (User Story 4)', () => {
|
||||
let app: FastifyInstance;
|
||||
let authToken: string;
|
||||
const externalProductId = `TEST_AI_REPORT_PROD_${Date.now()}`;
|
||||
let secret: string;
|
||||
const rangeFrom = new Date(Date.now() - 60 * 60 * 1000).toISOString();
|
||||
const rangeTo = new Date(Date.now() + 60 * 60 * 1000).toISOString();
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
authToken = await loginAs(app, 'ADMIN');
|
||||
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'AI Report Test Product', status: 'active' },
|
||||
});
|
||||
secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicket(): Promise<string> {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: `AI report test ${Date.now()}-${Math.random()}`,
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
it('reflects real session outcomes, tool results, and confidence bands', async () => {
|
||||
// A resolved session, with a knowledge match and a successful tool call.
|
||||
const resolvedTicketId = await createTicket();
|
||||
const resolvedSession = await sessionRepository.create(resolvedTicketId);
|
||||
await knowledgeReferenceRepository.recordMany(resolvedSession.id, ['fake-knowledge-id']);
|
||||
await diagnosisRepository.create({
|
||||
sessionId: resolvedSession.id,
|
||||
product: 'test-product',
|
||||
problemType: 'test-problem',
|
||||
severity: 'medium',
|
||||
confidence: aiConfig.defaultHighConfidence,
|
||||
possibleCauses: ['test cause'],
|
||||
});
|
||||
const successAction = await actionRepository.create({
|
||||
sessionId: resolvedSession.id,
|
||||
toolName: 'getTicketSnapshot',
|
||||
input: {},
|
||||
riskLevel: 'low',
|
||||
evaluationOutcome: 'approved',
|
||||
});
|
||||
await actionRepository.createResult(successAction.id, { ok: true }, 'success');
|
||||
await sessionRepository.updateStatus(resolvedSession.id, 'resolved');
|
||||
|
||||
// An escalated session, with a failed tool call and a low-confidence diagnosis.
|
||||
const escalatedTicketId = await createTicket();
|
||||
const escalatedSession = await sessionRepository.create(escalatedTicketId);
|
||||
await diagnosisRepository.create({
|
||||
sessionId: escalatedSession.id,
|
||||
product: 'test-product',
|
||||
problemType: 'test-problem',
|
||||
severity: 'high',
|
||||
confidence: aiConfig.defaultLowConfidence - 0.05,
|
||||
possibleCauses: ['test cause'],
|
||||
});
|
||||
const failedAction = await actionRepository.create({
|
||||
sessionId: escalatedSession.id,
|
||||
toolName: 'getTicketSnapshot',
|
||||
input: {},
|
||||
riskLevel: 'low',
|
||||
evaluationOutcome: 'approved',
|
||||
});
|
||||
await actionRepository.createResult(failedAction.id, { error: 'boom' }, 'failed');
|
||||
await sessionRepository.updateStatus(escalatedSession.id, 'escalated');
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/ai?from=${rangeFrom}&to=${rangeTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const data = res.json().data;
|
||||
|
||||
expect(data.totalSessions).toBeGreaterThanOrEqual(2);
|
||||
expect(data.aiResolutionRate).not.toBeNull();
|
||||
expect(data.humanHandoffRate).not.toBeNull();
|
||||
expect(data.knowledgeMatchRate).not.toBeNull();
|
||||
expect(data.confidenceDistribution.proceed).toBeGreaterThanOrEqual(1);
|
||||
expect(data.confidenceDistribution.escalate).toBeGreaterThanOrEqual(1);
|
||||
expect(data.toolInvocations.success).toBeGreaterThanOrEqual(1);
|
||||
expect(data.toolInvocations.failed).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('returns null rates and zero counts for a range with no AI activity', async () => {
|
||||
const farPastFrom = new Date('2000-01-01').toISOString();
|
||||
const farPastTo = new Date('2000-01-02').toISOString();
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/ai?from=${farPastFrom}&to=${farPastTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const data = res.json().data;
|
||||
|
||||
expect(data.totalSessions).toBe(0);
|
||||
expect(data.aiResolutionRate).toBeNull();
|
||||
expect(data.humanHandoffRate).toBeNull();
|
||||
expect(data.knowledgeMatchRate).toBeNull();
|
||||
expect(data.confidenceDistribution).toEqual({ proceed: 0, ask: 0, escalate: 0 });
|
||||
expect(data.toolInvocations).toEqual({ success: 0, failed: 0 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { loginAs, authHeader } from '../../helpers/auth';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { ticketsRepository } from '@/modules/ticketing/tickets';
|
||||
import { messagesService } from '@/modules/ticketing/messages';
|
||||
import { resolutionRepository } from '@/modules/problem-management/resolutions';
|
||||
|
||||
/**
|
||||
* Covers specs/015-reporting-dashboards/quickstart.md Scenario 1 against a real Postgres/Redis.
|
||||
* Drives ticket-status transitions directly through ticketsRepository (not ticketsService) to
|
||||
* avoid publishing TICKET_UPDATED — this test only needs the raw persisted state its own
|
||||
* aggregation queries read, and publishing real domain events here risks the same kind of
|
||||
* cross-file contamination 014-full-observability's own business-metrics.test.ts found and fixed
|
||||
* (an unscoped HUMAN_ESCALATION triggering real auto-assignment against the shared agent pool).
|
||||
*/
|
||||
describe('Management dashboard (User Story 1)', () => {
|
||||
let app: FastifyInstance;
|
||||
let authToken: string;
|
||||
const externalProductId = `TEST_MGMT_REPORT_PROD_${Date.now()}`;
|
||||
let productId: string;
|
||||
let secret: string;
|
||||
const rangeFrom = new Date(Date.now() - 60 * 60 * 1000).toISOString();
|
||||
const rangeTo = new Date(Date.now() + 60 * 60 * 1000).toISOString();
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
authToken = await loginAs(app, 'ADMIN');
|
||||
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Management Report Test Product', status: 'active' },
|
||||
});
|
||||
productId = product.id;
|
||||
secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicket(): Promise<string> {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: `Management report test ${Date.now()}-${Math.random()}`,
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
async function driveDirectly(ticketId: string, statuses: string[]): Promise<void> {
|
||||
let ticket = await prismaClient.ticket.findUniqueOrThrow({ where: { id: ticketId } });
|
||||
for (const status of statuses) {
|
||||
const updated = await ticketsRepository.updateStatus(ticketId, status, ticket.version);
|
||||
if (!updated) throw new Error(`Failed to transition ticket to ${status} in test setup.`);
|
||||
ticket = updated;
|
||||
}
|
||||
}
|
||||
|
||||
it('reports real figures matching the actual created data', async () => {
|
||||
// AI-resolved ticket.
|
||||
const aiTicketId = await createTicket();
|
||||
await driveDirectly(aiTicketId, [
|
||||
'AI_ANALYZING',
|
||||
'AI_TROUBLESHOOTING',
|
||||
'AI_VERIFYING',
|
||||
'AI_RESOLVED',
|
||||
]);
|
||||
await resolutionRepository.create({ ticketId: aiTicketId, outcome: 'fixed', resolvedBy: 'ai' });
|
||||
await driveDirectly(aiTicketId, ['RESOLVED']);
|
||||
|
||||
// Human-resolved ticket, with a first agent response recorded and a real Assignment row —
|
||||
// "ever escalated to a human" is keyed off Assignment existence (see
|
||||
// ManagementRepository.countEverEscalatedToHuman's own comment on why a ticket's current
|
||||
// status can't distinguish the AI path from the human path once both converge on the same
|
||||
// shared terminal statuses).
|
||||
const team = await prismaClient.team.create({ data: { name: `Mgmt Report Team ${Date.now()}` } });
|
||||
const agent = await prismaClient.agent.create({ data: { teamId: team.id, name: 'Mgmt Report Agent' } });
|
||||
const humanTicketId = await createTicket();
|
||||
await prismaClient.assignment.create({
|
||||
data: { ticketId: humanTicketId, agentId: agent.id, strategy: 'MANUAL', isCurrent: true },
|
||||
});
|
||||
await messagesService.post(humanTicketId, 'agent-1', 'AGENT_MESSAGE', 'Looking into this.');
|
||||
await driveDirectly(humanTicketId, [
|
||||
'HUMAN_ESCALATION',
|
||||
'IN_PROGRESS',
|
||||
'RESOLUTION_PENDING_CUSTOMER',
|
||||
]);
|
||||
await resolutionRepository.create({
|
||||
ticketId: humanTicketId,
|
||||
outcome: 'fixed',
|
||||
resolvedBy: 'agent-1',
|
||||
});
|
||||
await driveDirectly(humanTicketId, ['RESOLVED']);
|
||||
|
||||
// Still-open ticket.
|
||||
await createTicket();
|
||||
|
||||
// SLA policy + one met, one breached run.
|
||||
const policy = await prismaClient.sLAPolicy.create({
|
||||
data: {
|
||||
name: `Mgmt Report Policy ${Date.now()}`,
|
||||
productId,
|
||||
firstResponseMinutes: 30,
|
||||
resolutionMinutes: 240,
|
||||
},
|
||||
});
|
||||
const metTicketId = await createTicket();
|
||||
await prismaClient.sLARun.create({
|
||||
data: {
|
||||
ticketId: metTicketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(Date.now() + 240 * 60_000),
|
||||
status: 'completed',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
const breachedTicketId = await createTicket();
|
||||
await prismaClient.sLARun.create({
|
||||
data: {
|
||||
ticketId: breachedTicketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(Date.now() - 60_000),
|
||||
status: 'breached',
|
||||
breachedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
// One escalation event.
|
||||
const escalatedTicketId = await createTicket();
|
||||
await prismaClient.escalationEvent.create({
|
||||
data: {
|
||||
ticketId: escalatedTicketId,
|
||||
ruleId: null,
|
||||
fromNodeId: null,
|
||||
toNodeId: null,
|
||||
reason: 'management dashboard test',
|
||||
triggeredBy: 'system',
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/management?from=${rangeFrom}&to=${rangeTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const data = res.json().data;
|
||||
|
||||
expect(data.totalCases).toBeGreaterThanOrEqual(6);
|
||||
expect(data.aiResolved).toBeGreaterThanOrEqual(1);
|
||||
expect(data.humanEscalated).toBeGreaterThanOrEqual(1);
|
||||
expect(data.resolved).toBeGreaterThanOrEqual(2);
|
||||
expect(data.open).toBeGreaterThanOrEqual(1);
|
||||
expect(data.slaCompliance.met).toBeGreaterThanOrEqual(1);
|
||||
expect(data.slaCompliance.breached).toBeGreaterThanOrEqual(1);
|
||||
expect(data.slaCompliance.rate).not.toBeNull();
|
||||
expect(data.escalationCount).toBeGreaterThanOrEqual(1);
|
||||
expect(data.averageResponseSeconds).not.toBeNull();
|
||||
expect(data.averageResolutionSeconds).not.toBeNull();
|
||||
expect(data.range.from).toBeTruthy();
|
||||
expect(data.range.to).toBeTruthy();
|
||||
});
|
||||
|
||||
it('returns all-zero counts and all-null rates for a range with no activity', async () => {
|
||||
const farPastFrom = new Date('2000-01-01').toISOString();
|
||||
const farPastTo = new Date('2000-01-02').toISOString();
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/management?from=${farPastFrom}&to=${farPastTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const data = res.json().data;
|
||||
|
||||
expect(data.totalCases).toBe(0);
|
||||
expect(data.aiResolved).toBe(0);
|
||||
expect(data.humanEscalated).toBe(0);
|
||||
expect(data.resolved).toBe(0);
|
||||
expect(data.open).toBe(0);
|
||||
expect(data.slaCompliance.rate).toBeNull();
|
||||
expect(data.averageResponseSeconds).toBeNull();
|
||||
expect(data.averageResolutionSeconds).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects a range where from is after to', async () => {
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/management?from=${rangeTo}&to=${rangeFrom}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { loginAs, authHeader } from '../../helpers/auth';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { errorCodesService } from '@/modules/ai-support/knowledge';
|
||||
|
||||
/** Covers specs/015-reporting-dashboards/quickstart.md Scenario 2 against a real Postgres/Redis. */
|
||||
describe('Product dashboard (User Story 2)', () => {
|
||||
let app: FastifyInstance;
|
||||
let authToken: string;
|
||||
const rangeFrom = new Date(Date.now() - 60 * 60 * 1000).toISOString();
|
||||
const rangeTo = new Date(Date.now() + 60 * 60 * 1000).toISOString();
|
||||
|
||||
async function setUpProduct(nameSuffix: string) {
|
||||
const externalProductId = `TEST_PRODUCT_REPORT_${nameSuffix}_${Date.now()}`;
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: `Product Report ${nameSuffix}`, status: 'active' },
|
||||
});
|
||||
const secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId: product.id,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
return { externalProductId, productId: product.id, secret };
|
||||
}
|
||||
|
||||
async function createTicket(externalProductId: string, secret: string): Promise<string> {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: `Product report test ${Date.now()}-${Math.random()}`,
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
authToken = await loginAs(app, 'ADMIN');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("scopes every figure to the requested product, never another product's data", async () => {
|
||||
const productA = await setUpProduct('A');
|
||||
const productB = await setUpProduct('B');
|
||||
|
||||
await createTicket(productA.externalProductId, productA.secret);
|
||||
await createTicket(productA.externalProductId, productA.secret);
|
||||
await createTicket(productB.externalProductId, productB.secret);
|
||||
|
||||
const resA = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/product/${productA.externalProductId}?from=${rangeFrom}&to=${rangeTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(resA.statusCode).toBe(200);
|
||||
expect(resA.json().data.supportVolume).toBe(2);
|
||||
|
||||
const resB = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/product/${productB.externalProductId}?from=${rangeFrom}&to=${rangeTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(resB.statusCode).toBe(200);
|
||||
expect(resB.json().data.supportVolume).toBe(1);
|
||||
});
|
||||
|
||||
it('ranks the most-frequently-looked-up error code first', async () => {
|
||||
const product = await setUpProduct('ERR');
|
||||
const popularCode = `POPULAR-${Date.now()}`;
|
||||
const rareCode = `RARE-${Date.now()}`;
|
||||
await errorCodesService.createErrorCode(product.productId, popularCode, 'Popular error');
|
||||
await errorCodesService.createErrorCode(product.productId, rareCode, 'Rare error');
|
||||
|
||||
await errorCodesService.findKnownIssuesByErrorCode(product.productId, popularCode);
|
||||
await errorCodesService.findKnownIssuesByErrorCode(product.productId, popularCode);
|
||||
await errorCodesService.findKnownIssuesByErrorCode(product.productId, popularCode);
|
||||
await errorCodesService.findKnownIssuesByErrorCode(product.productId, rareCode);
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/product/${product.externalProductId}?from=${rangeFrom}&to=${rangeTo}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const topErrors = res.json().data.topErrors as Array<{ code: string; count: number }>;
|
||||
expect(topErrors[0]).toMatchObject({ code: popularCode, count: 3 });
|
||||
expect(topErrors.find((e) => e.code === rareCode)).toMatchObject({ count: 1 });
|
||||
});
|
||||
|
||||
it('404s for an unknown product', async () => {
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/admin/reports/product/NONEXISTENT_PRODUCT_${Date.now()}`,
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { buildApp } from '@/app';
|
||||
import { prismaClient } from '@/infrastructure/database';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { loginAs, authHeader } from '../../helpers/auth';
|
||||
import {
|
||||
encryptCredential,
|
||||
generateCredentialSecret,
|
||||
issueIntegrationToken,
|
||||
} from '@/modules/catalog/products';
|
||||
import { reportingConfig } from '@/config';
|
||||
|
||||
/**
|
||||
* Covers specs/015-reporting-dashboards/quickstart.md Scenario 3 against a real Postgres/Redis.
|
||||
* Assignment rows are created directly via Prisma (not through a real HUMAN_ESCALATION +
|
||||
* default-strategy auto-assignment) — the same contamination avoidance
|
||||
* management-dashboard.test.ts already documents: this test only needs the persisted
|
||||
* Assignment/SLARun state its own aggregation queries read, not a live orchestration run.
|
||||
*/
|
||||
describe('Support dashboard (User Story 3)', () => {
|
||||
let app: FastifyInstance;
|
||||
let authToken: string;
|
||||
const externalProductId = `TEST_SUPPORT_REPORT_PROD_${Date.now()}`;
|
||||
let productId: string;
|
||||
let secret: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
authToken = await loginAs(app, 'ADMIN');
|
||||
|
||||
const product = await prismaClient.product.create({
|
||||
data: { externalProductId, name: 'Support Report Test Product', status: 'active' },
|
||||
});
|
||||
productId = product.id;
|
||||
secret = generateCredentialSecret();
|
||||
await prismaClient.productIntegration.create({
|
||||
data: {
|
||||
productId,
|
||||
credentialRef: encryptCredential(secret),
|
||||
authMechanism: 'signed_token',
|
||||
allowedScope: { tenantIds: ['tenant-1'] },
|
||||
status: 'active',
|
||||
rateLimitPerMinute: 1000,
|
||||
rateLimitPerUserPerMinute: 1000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
async function createTicket(): Promise<string> {
|
||||
const token = issueIntegrationToken(secret, {
|
||||
externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/support/requests',
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: {
|
||||
productId: externalProductId,
|
||||
tenantId: 'tenant-1',
|
||||
userId: 'user-1',
|
||||
source: 'test',
|
||||
problem: `Support report test ${Date.now()}-${Math.random()}`,
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(202);
|
||||
return created.json().data.ticketId as string;
|
||||
}
|
||||
|
||||
it("reflects each agent's real current assignment workload", async () => {
|
||||
const team = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/admin/teams',
|
||||
headers: authHeader(authToken),
|
||||
payload: { name: `Support Report Team ${Date.now()}` },
|
||||
});
|
||||
const teamId = team.json().data.id as string;
|
||||
const agent = await app.inject({
|
||||
method: 'POST',
|
||||
url: `/admin/teams/${teamId}/agents`,
|
||||
headers: authHeader(authToken),
|
||||
payload: { name: 'Support Report Agent' },
|
||||
});
|
||||
const agentId = agent.json().data.id as string;
|
||||
|
||||
const ticket1 = await createTicket();
|
||||
const ticket2 = await createTicket();
|
||||
await prismaClient.assignment.createMany({
|
||||
data: [
|
||||
{ ticketId: ticket1, agentId, strategy: 'MANUAL', isCurrent: true },
|
||||
{ ticketId: ticket2, agentId, strategy: 'MANUAL', isCurrent: true },
|
||||
],
|
||||
});
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: '/admin/reports/support',
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
const workload = res.json().data.workloadByAgent as Array<{
|
||||
agentId: string;
|
||||
openAssignments: number;
|
||||
}>;
|
||||
expect(workload.find((w) => w.agentId === agentId)).toMatchObject({ openAssignments: 2 });
|
||||
});
|
||||
|
||||
it('counts a near-due SLA run as at-risk, distinct from breached', async () => {
|
||||
const policy = await prismaClient.sLAPolicy.create({
|
||||
data: {
|
||||
name: `Support Risk Policy ${Date.now()}`,
|
||||
productId,
|
||||
firstResponseMinutes: 30,
|
||||
resolutionMinutes: 240,
|
||||
},
|
||||
});
|
||||
|
||||
const riskTicketId = await createTicket();
|
||||
await prismaClient.sLARun.create({
|
||||
data: {
|
||||
ticketId: riskTicketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(
|
||||
Date.now() + (reportingConfig.slaRiskThresholdMinutes - 1) * 60_000,
|
||||
),
|
||||
status: 'running',
|
||||
},
|
||||
});
|
||||
|
||||
const safeTicketId = await createTicket();
|
||||
await prismaClient.sLARun.create({
|
||||
data: {
|
||||
ticketId: safeTicketId,
|
||||
policyId: policy.id,
|
||||
firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
|
||||
resolutionDueAt: new Date(Date.now() + 999 * 60_000),
|
||||
status: 'running',
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'GET',
|
||||
url: '/admin/reports/support',
|
||||
headers: authHeader(authToken),
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().data.slaAtRisk).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Writable } from 'stream';
|
||||
import pino from 'pino';
|
||||
import { loggerOptions } from '@/infrastructure/observability/logger';
|
||||
import { requestContextStore } from '@/infrastructure/observability/request-context.store';
|
||||
|
||||
function buildCapturingLogger() {
|
||||
const lines: Record<string, unknown>[] = [];
|
||||
const sink = new Writable({
|
||||
write(chunk, _enc, callback) {
|
||||
lines.push(JSON.parse(chunk.toString()));
|
||||
callback();
|
||||
},
|
||||
});
|
||||
// Same options (same mixin) the real singleton uses — only the destination differs, per
|
||||
// logger.ts's own comment on why loggerOptions is exported. `transport` is never set outside
|
||||
// NODE_ENV=development (see logger.ts), so it's always undefined under `npm test`.
|
||||
const testLogger = pino(loggerOptions, sink);
|
||||
return { testLogger, lines };
|
||||
}
|
||||
|
||||
describe('logger mixin (014-full-observability FR-002)', () => {
|
||||
it('attaches requestId/correlationId to a log line made inside the request context store', () => {
|
||||
const { testLogger, lines } = buildCapturingLogger();
|
||||
|
||||
requestContextStore.run({ requestId: 'req-1', correlationId: 'corr-1' }, () => {
|
||||
testLogger.info('inside request');
|
||||
});
|
||||
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(lines[0]).toMatchObject({ requestId: 'req-1', correlationId: 'corr-1' });
|
||||
});
|
||||
|
||||
it('attaches neither field to a log line made outside any request context', () => {
|
||||
const { testLogger, lines } = buildCapturingLogger();
|
||||
|
||||
testLogger.info('outside any request');
|
||||
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(lines[0]?.requestId).toBeUndefined();
|
||||
expect(lines[0]?.correlationId).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not leak one request context into a log line logged after that context ends', () => {
|
||||
const { testLogger, lines } = buildCapturingLogger();
|
||||
|
||||
requestContextStore.run({ requestId: 'req-2', correlationId: 'corr-2' }, () => {
|
||||
testLogger.info('inside');
|
||||
});
|
||||
testLogger.info('after');
|
||||
|
||||
expect(lines[0]).toMatchObject({ requestId: 'req-2' });
|
||||
expect(lines[1]?.requestId).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { SlaService } from '@/modules/orchestration/sla/service/sla.service';
|
||||
import * as observability from '@/infrastructure/observability';
|
||||
|
||||
function fakeRun(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
return {
|
||||
id: 'run-1',
|
||||
ticketId: 'ticket-1',
|
||||
status: 'running',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('SLA compliance metric (014-full-observability data-model.md #6)', () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('counts a run that completes while still running as met', async () => {
|
||||
const incSpy = vi.spyOn(observability.slaRunOutcomesCounter, 'inc');
|
||||
const runs = {
|
||||
findByTicketId: vi.fn().mockResolvedValue(fakeRun({ status: 'running' })),
|
||||
update: vi.fn().mockResolvedValue(undefined),
|
||||
} as never;
|
||||
const service = new SlaService(undefined, runs);
|
||||
|
||||
await service.complete('ticket-1');
|
||||
|
||||
expect(incSpy).toHaveBeenCalledWith({ outcome: 'met' });
|
||||
});
|
||||
|
||||
it('does not double-count a run that was already breached before it resolved', async () => {
|
||||
const incSpy = vi.spyOn(observability.slaRunOutcomesCounter, 'inc');
|
||||
const runs = {
|
||||
findByTicketId: vi.fn().mockResolvedValue(fakeRun({ status: 'breached' })),
|
||||
update: vi.fn().mockResolvedValue(undefined),
|
||||
} as never;
|
||||
const service = new SlaService(undefined, runs);
|
||||
|
||||
await service.complete('ticket-1');
|
||||
|
||||
expect(incSpy).not.toHaveBeenCalledWith({ outcome: 'met' });
|
||||
});
|
||||
|
||||
it('does not count anything for a run already completed', async () => {
|
||||
const incSpy = vi.spyOn(observability.slaRunOutcomesCounter, 'inc');
|
||||
const runs = {
|
||||
findByTicketId: vi.fn().mockResolvedValue(fakeRun({ status: 'completed' })),
|
||||
update: vi.fn().mockResolvedValue(undefined),
|
||||
} as never;
|
||||
const service = new SlaService(undefined, runs);
|
||||
|
||||
await service.complete('ticket-1');
|
||||
|
||||
expect(incSpy).not.toHaveBeenCalled();
|
||||
expect((runs as unknown as { update: ReturnType<typeof vi.fn> }).update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { BasicTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||
|
||||
/**
|
||||
* 014-full-observability FR-007/Quickstart Scenario 3 steps 3-4: an unreachable tracing
|
||||
* collector must never surface as an application-level failure. This exercises the real
|
||||
* OpenTelemetry SDK's actual background export path (BasicTracerProvider + BatchSpanProcessor +
|
||||
* a real OTLPTraceExporter, against a real, deliberately-unreachable address — not a mock) the
|
||||
* same way it runs in production: a span ends, the processor's own internal timer schedules the
|
||||
* export, and a failed export is caught by the SDK's own error handler — never left as an
|
||||
* unhandled rejection that could crash the process.
|
||||
*
|
||||
* (Deliberately does NOT call provider.forceFlush() to prove this — forceFlush() is documented
|
||||
* OpenTelemetry SDK behavior that *does* reject on a failed export, by design, so a caller that
|
||||
* explicitly asks "did my flush succeed?" can find out. This feature's own code never calls
|
||||
* forceFlush() on the request-handling path, only the SDK's own background timer does, which is
|
||||
* what this test exercises instead.)
|
||||
*/
|
||||
describe('Tracing graceful degradation', () => {
|
||||
let unhandledRejection: unknown;
|
||||
const onUnhandledRejection = (reason: unknown) => {
|
||||
unhandledRejection = reason;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
process.removeListener('unhandledRejection', onUnhandledRejection);
|
||||
});
|
||||
|
||||
it('does not produce an unhandled rejection when the background export to an unreachable endpoint fails', async () => {
|
||||
unhandledRejection = undefined;
|
||||
process.on('unhandledRejection', onUnhandledRejection);
|
||||
|
||||
const exporter = new OTLPTraceExporter({
|
||||
url: 'http://127.0.0.1:1/v1/traces', // port 1 — nothing listens there
|
||||
timeoutMillis: 500,
|
||||
});
|
||||
const provider = new BasicTracerProvider({
|
||||
spanProcessors: [
|
||||
new BatchSpanProcessor(exporter, { scheduledDelayMillis: 10, exportTimeoutMillis: 500 }),
|
||||
],
|
||||
});
|
||||
|
||||
const span = provider.getTracer('test').startSpan('unreachable-export-test');
|
||||
span.end(); // triggers the processor's own internal timer, not forceFlush()
|
||||
|
||||
// Long enough for the internal timer (10ms) + the failed connection attempt to resolve.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
|
||||
expect(unhandledRejection).toBeUndefined();
|
||||
|
||||
await provider.shutdown();
|
||||
}, 10000);
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { decideConfidenceBand } from '@/modules/ai-support/sessions';
|
||||
import { aiConfig } from '@/config';
|
||||
|
||||
/**
|
||||
* 015-reporting-dashboards research.md §7: the AI dashboard's confidence distribution reuses
|
||||
* 005-ai-support's own decideConfidenceBand against the system-default thresholds, rather than
|
||||
* reimplementing a threshold check — this test proves the reused function classifies values
|
||||
* the way the dashboard's own bucketing loop (reports.service.ts) depends on.
|
||||
*/
|
||||
describe('AI dashboard confidence distribution reuses decideConfidenceBand', () => {
|
||||
const policy = {
|
||||
highThreshold: aiConfig.defaultHighConfidence,
|
||||
lowThreshold: aiConfig.defaultLowConfidence,
|
||||
};
|
||||
|
||||
it('classifies a high-confidence value as proceed', () => {
|
||||
expect(decideConfidenceBand(policy.highThreshold, policy)).toBe('proceed');
|
||||
});
|
||||
|
||||
it('classifies a low-confidence value as escalate', () => {
|
||||
expect(decideConfidenceBand(policy.lowThreshold - 0.01, policy)).toBe('escalate');
|
||||
});
|
||||
|
||||
it('classifies a mid-range value as ask', () => {
|
||||
const midpoint = (policy.highThreshold + policy.lowThreshold) / 2;
|
||||
expect(decideConfidenceBand(midpoint, policy)).toBe('ask');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { computeRate, computeAverageSeconds } from '@/modules/platform/reports/mapper';
|
||||
|
||||
describe('computeRate (015-reporting-dashboards research.md §3)', () => {
|
||||
it('returns null when the denominator is zero — never NaN, never a computed 0', () => {
|
||||
expect(computeRate(0, 0)).toBeNull();
|
||||
expect(computeRate(5, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it('computes a real rate when there is qualifying data', () => {
|
||||
expect(computeRate(3, 12)).toBe(0.25);
|
||||
});
|
||||
|
||||
it('returns a real 0 when the numerator is legitimately zero but the denominator is not', () => {
|
||||
expect(computeRate(0, 10)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeAverageSeconds', () => {
|
||||
it('returns null for an empty list — no fabricated average', () => {
|
||||
expect(computeAverageSeconds([])).toBeNull();
|
||||
});
|
||||
|
||||
it('averages a list of millisecond durations into seconds', () => {
|
||||
expect(computeAverageSeconds([1000, 2000, 3000])).toBe(2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user