saqib mirandClaude Sonnet 5
de5915a8c1
Completes User Story 4's eleven named metrics: first-response-time (messages.service.ts's post(), guarded against double-counting a ticket's second agent message), SLA compliance (sla.service.ts's complete()/runBreachDetectionSweep(), with a guard so a run already breached by the sweep is never also counted "met" when it later resolves), most-common-errors (error-codes.service.ts, counted only once a code is confirmed real), and tool-failure-rate/knowledge- effectiveness (tools.service.ts's single executeTool call site). Verified end-to-end against real Postgres/Redis by scraping the real /metrics endpoint before and after driving each metric's actual underlying event through the real service layer — including a genuine tool-execution failure (a nonexistent ticket ID) rather than a simulated one. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
93 lines
3.7 KiB
TypeScript
93 lines
3.7 KiB
TypeScript
import Anthropic from '@anthropic-ai/sdk';
|
|
import {
|
|
toolInvocationsCounter,
|
|
knowledgeRetrievalOutcomesCounter,
|
|
} from '@/infrastructure/observability';
|
|
import { actionRepository, ActionRepository } from '../repository';
|
|
import { evaluateToolProposal } from './policy-gate';
|
|
import { executeTool, ToolExecutionContext } from './tool-executor';
|
|
|
|
export interface ProposeAndEvaluateResult {
|
|
toolResults: Anthropic.ToolResultBlockParam[];
|
|
escalationRequested: { reason: string } | null;
|
|
anyFailed: boolean;
|
|
}
|
|
|
|
export class ToolsService {
|
|
constructor(private readonly actions: ActionRepository = actionRepository) {}
|
|
|
|
/**
|
|
* FR-011/FR-012/FR-013/FR-014: every `tool_use` block in a reasoning turn's response passes
|
|
* through the deterministic gate before anything runs; every proposal, its evaluation, and its
|
|
* result (if any) is durably recorded. Returns the `tool_result` blocks the caller feeds back
|
|
* into the next reasoning call, plus whether an approved `escalateToHuman` call fired (the
|
|
* session, not this module, decides what to do with that — see tool-executor.ts's module
|
|
* comment on why this stays decoupled from ai-support/sessions).
|
|
*/
|
|
async proposeAndEvaluate(
|
|
sessionId: string,
|
|
toolUseBlocks: Anthropic.ToolUseBlock[],
|
|
context: ToolExecutionContext,
|
|
): Promise<ProposeAndEvaluateResult> {
|
|
const toolResults: Anthropic.ToolResultBlockParam[] = [];
|
|
let escalationRequested: { reason: string } | null = null;
|
|
let anyFailed = false;
|
|
|
|
for (const block of toolUseBlocks) {
|
|
const evaluation = evaluateToolProposal(block.name, { productId: context.productId });
|
|
|
|
const action = await this.actions.create({
|
|
sessionId,
|
|
toolName: block.name,
|
|
input: block.input,
|
|
riskLevel: evaluation.riskLevel,
|
|
evaluationOutcome: evaluation.outcome,
|
|
refusalReason: evaluation.refusalReason ?? undefined,
|
|
approvedBy: evaluation.outcome === 'approved' ? 'system-policy' : undefined,
|
|
});
|
|
|
|
if (evaluation.outcome !== 'approved') {
|
|
toolResults.push({
|
|
type: 'tool_result',
|
|
tool_use_id: block.id,
|
|
content:
|
|
evaluation.outcome === 'pending_approval'
|
|
? 'This action requires human approval and has not been executed. Do not assume it succeeded.'
|
|
: `This action was refused: ${evaluation.refusalReason}`,
|
|
is_error: evaluation.outcome === 'refused',
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const result = await executeTool(block.name, block.input, context);
|
|
await this.actions.createResult(action.id, result.output, result.status);
|
|
|
|
// 014-full-observability data-model.md #10/#11: the single choke point every tool
|
|
// invocation passes through — labeled by outcome, and (for the knowledge-search tool
|
|
// specifically) by whether it found anything.
|
|
toolInvocationsCounter.inc({ tool: block.name, outcome: result.status });
|
|
if (block.name === 'searchProductKnowledge') {
|
|
const matched = Array.isArray(result.output) && result.output.length > 0;
|
|
knowledgeRetrievalOutcomesCounter.inc({ matched: String(matched) });
|
|
}
|
|
|
|
if (result.status === 'failed') anyFailed = true;
|
|
if (block.name === 'escalateToHuman' && result.status === 'success') {
|
|
const output = result.output as { reason?: string };
|
|
escalationRequested = { reason: output.reason ?? 'The AI requested escalation.' };
|
|
}
|
|
|
|
toolResults.push({
|
|
type: 'tool_result',
|
|
tool_use_id: block.id,
|
|
content: JSON.stringify(result.output),
|
|
is_error: result.status === 'failed',
|
|
});
|
|
}
|
|
|
|
return { toolResults, escalationRequested, anyFailed };
|
|
}
|
|
}
|
|
|
|
export const toolsService = new ToolsService();
|