support_backend/tests/integration/observability/business-metrics.test.ts

import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { buildApp } from '@/app';
import { prismaClient } from '@/infrastructure/database';
import { FastifyInstance } from 'fastify';
import { loginAs, authHeader } from '../../helpers/auth';
import {
  encryptCredential,
  generateCredentialSecret,
  issueIntegrationToken,
} from '@/modules/catalog/products';
import { sessionsService, sessionRepository } from '@/modules/ai-support/sessions';
import { ticketsService, ticketsRepository } from '@/modules/ticketing/tickets';
import { messagesService } from '@/modules/ticketing/messages';
import { resolutionRepository } from '@/modules/problem-management/resolutions';
import { slaService, slaRunRepository } from '@/modules/orchestration/sla';
import { escalationService } from '@/modules/orchestration/escalation';
import { errorCodesService } from '@/modules/ai-support/knowledge';
import { toolsService } from '@/modules/ai-support/tools';

/**
 * Covers specs/014-full-observability/quickstart.md Scenario 4 against a real Postgres/Redis —
 * every named business-health metric, scraped from the real /metrics endpoint before and after
 * driving its real underlying event through the real service layer (not mocked). Several flows
 * (human resolution, SLA runs) create rows directly against the repositories that already own
 * the relevant validation elsewhere in this codebase's own test suite — this file's job is only
 * to prove the metric increments at the correct point, not to re-verify those modules' own
 * business rules (already covered by problem-resolution-flow.test.ts / sla-escalation-flow.test.ts).
 */
describe('Business-health metrics (User Story 4)', () => {
  let app: FastifyInstance;
  let authToken: string;
  const externalProductId = `TEST_BIZ_METRICS_PROD_${Date.now()}`;
  let productId: string;
  let secret: string;

  beforeAll(async () => {
    app = await buildApp();
    authToken = await loginAs(app, 'ADMIN');

    const product = await prismaClient.product.create({
      data: { externalProductId, name: 'Business Metrics Test Product', status: 'active' },
    });
    productId = product.id;
    secret = generateCredentialSecret();
    await prismaClient.productIntegration.create({
      data: {
        productId,
        credentialRef: encryptCredential(secret),
        authMechanism: 'signed_token',
        allowedScope: { tenantIds: ['tenant-1'] },
        status: 'active',
        rateLimitPerMinute: 1000,
        rateLimitPerUserPerMinute: 1000,
      },
    });
  });

  afterAll(async () => {
    await app.close();
  });

  async function createTicket(): Promise<string> {
    const token = issueIntegrationToken(secret, {
      externalProductId,
      tenantId: 'tenant-1',
      userId: 'user-1',
    });
    const created = await app.inject({
      method: 'POST',
      url: '/v1/support/requests',
      headers: { authorization: `Bearer ${token}` },
      payload: {
        productId: externalProductId,
        tenantId: 'tenant-1',
        userId: 'user-1',
        source: 'test',
        problem: `Business metrics test ${Date.now()}-${Math.random()}`,
      },
    });
    expect(created.statusCode).toBe(202);
    return created.json().data.ticketId as string;
  }

  async function scrape(): Promise<string> {
    const res = await app.inject({ method: 'GET', url: '/metrics' });
    expect(res.statusCode).toBe(200);
    return res.body;
  }

  function metricValue(body: string, name: string, labels?: Record<string, string>): number {
    const labelPart = labels
      ? `\\{${Object.entries(labels)
          .map(([k, v]) => `${k}="${v}"`)
          .join(',')}\\}`
      : '(?:\\{\\})?';
    const match = body.match(new RegExp(`${name}${labelPart}\\s+([0-9.]+)`));
    return match?.[1] ? parseFloat(match[1]) : 0;
  }

  it('counts an AI session resolving without escalating', async () => {
    const ticketId = await createTicket();
    const session = await sessionRepository.create(ticketId);

    const before = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
      outcome: 'resolved',
    });
    await sessionRepository.updateStatus(session.id, 'resolved');
    const after = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
      outcome: 'resolved',
    });

    expect(after).toBe(before + 1);
  });

  it('counts an AI session escalating, and nothing for resolved', async () => {
    const ticketId = await createTicket();
    const session = await sessionRepository.create(ticketId);

    const before = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
      outcome: 'escalated',
    });
    await sessionsService.escalate(session, ticketId, 'Escalating for metrics test.');
    const after = metricValue(await scrape(), 'supporthub_ai_session_outcomes_total', {
      outcome: 'escalated',
    });

    expect(after).toBe(before + 1);
  });

  it('counts the first agent message on a ticket, observing first-response duration', async () => {
    const ticketId = await createTicket();

    const bodyBefore = await scrape();
    const before = metricValue(
      bodyBefore,
      'supporthub_ticket_first_response_duration_seconds_count',
    );

    await messagesService.post(ticketId, 'agent-1', 'AGENT_MESSAGE', 'Hi, looking into this.');
    // A second agent message must NOT observe again.
    await messagesService.post(ticketId, 'agent-1', 'AGENT_MESSAGE', 'Following up.');

    const after = metricValue(
      await scrape(),
      'supporthub_ticket_first_response_duration_seconds_count',
    );
    expect(after).toBe(before + 1);
  });

  it('counts a human resolution and observes resolution duration when a ticket reaches RESOLVED', async () => {
    const ticketId = await createTicket();

    // Reach RESOLUTION_PENDING_CUSTOMER via the repository directly (bypassing
    // ticketsService.updateStatus's domain-event publish) — this test only cares about the
    // final RESOLVED transition and the Resolution row's own resolvedBy, not the intermediate
    // states, and going through the real event bus here would trigger a REAL, unscoped
    // HUMAN_ESCALATION auto-assignment against the default strategy — which can land on some
    // other concurrently-running test file's own dedicated agent (a real cross-file
    // contamination this test caused once, fixed here by not publishing those events at all).
    let ticket = await prismaClient.ticket.findUniqueOrThrow({ where: { id: ticketId } });
    for (const status of ['HUMAN_ESCALATION', 'IN_PROGRESS', 'RESOLUTION_PENDING_CUSTOMER']) {
      const updated = await ticketsRepository.updateStatus(ticketId, status, ticket.version);
      if (!updated) throw new Error(`Failed to transition ticket to ${status} in test setup.`);
      ticket = updated;
    }
    await resolutionRepository.create({ ticketId, outcome: 'fixed', resolvedBy: 'agent-1' });

    const bodyBefore = await scrape();
    const resolvedBefore = metricValue(bodyBefore, 'supporthub_ticket_resolutions_total', {
      resolved_by: 'human',
    });
    const durationBefore = metricValue(
      bodyBefore,
      'supporthub_ticket_resolution_duration_seconds_count',
    );

    await ticketsService.updateStatus(ticketId, 'RESOLVED', ticket.version, 'agent-1');

    const bodyAfter = await scrape();
    expect(
      metricValue(bodyAfter, 'supporthub_ticket_resolutions_total', { resolved_by: 'human' }),
    ).toBe(resolvedBefore + 1);
    expect(metricValue(bodyAfter, 'supporthub_ticket_resolution_duration_seconds_count')).toBe(
      durationBefore + 1,
    );
  });

  it('counts an SLA run completing on time as met', async () => {
    const policy = await prismaClient.sLAPolicy.create({
      data: {
        name: `Metrics Policy ${Date.now()}`,
        productId,
        firstResponseMinutes: 30,
        resolutionMinutes: 240,
      },
    });
    const ticketId = await createTicket();
    await slaRunRepository.create({
      ticketId,
      policyId: policy.id,
      firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
      resolutionDueAt: new Date(Date.now() + 240 * 60_000),
    });

    const before = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
      outcome: 'met',
    });
    await slaService.complete(ticketId);
    const after = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
      outcome: 'met',
    });

    expect(after).toBe(before + 1);
  });

  it('counts an overdue SLA run as breached via the sweep (at least once — a shared sweep may also catch unrelated overdue runs)', async () => {
    const policy = await prismaClient.sLAPolicy.create({
      data: {
        name: `Metrics Breach Policy ${Date.now()}`,
        productId,
        firstResponseMinutes: 30,
        resolutionMinutes: 1,
      },
    });
    const ticketId = await createTicket();
    await slaRunRepository.create({
      ticketId,
      policyId: policy.id,
      firstResponseDueAt: new Date(Date.now() + 30 * 60_000),
      resolutionDueAt: new Date(Date.now() - 60_000),
    });

    const before = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
      outcome: 'breached',
    });
    await slaService.runBreachDetectionSweep();
    const after = metricValue(await scrape(), 'supporthub_sla_run_outcomes_total', {
      outcome: 'breached',
    });

    expect(after).toBeGreaterThanOrEqual(before + 1);
  });

  it('counts a manual escalation event by reason', async () => {
    const node = await app.inject({
      method: 'POST',
      url: '/admin/hierarchy-nodes',
      headers: authHeader(authToken),
      payload: {
        name: `Metrics Node ${Date.now()}`,
        order: 0,
        productScope: [externalProductId],
        skills: [],
        assignmentStrategy: 'ROUND_ROBIN',
      },
    });
    const nodeId = node.json().data.id as string;
    const ticketId = await createTicket();
    const reason = `metrics-test-reason-${Date.now()}`;

    const before = metricValue(await scrape(), 'supporthub_escalations_total', { reason });
    await escalationService.escalateManually(ticketId, nodeId, 'admin-test', reason);
    const after = metricValue(await scrape(), 'supporthub_escalations_total', { reason });

    expect(after).toBe(before + 1);
  });

  it('counts a problem created, labeled by category (uncategorized here)', async () => {
    const before = metricValue(await scrape(), 'supporthub_problems_created_total', {
      category_id: 'uncategorized',
    });
    await createTicket();
    const after = metricValue(await scrape(), 'supporthub_problems_created_total', {
      category_id: 'uncategorized',
    });

    expect(after).toBe(before + 1);
  });

  it('counts a knowledge-search tool call that finds nothing as unmatched', async () => {
    const ticketId = await createTicket();
    const session = await sessionRepository.create(ticketId);

    const before = metricValue(await scrape(), 'supporthub_knowledge_retrieval_outcomes_total', {
      matched: 'false',
    });
    await toolsService.proposeAndEvaluate(
      session.id,
      [
        {
          type: 'tool_use',
          caller: { type: 'direct' },
          id: `toolu_${Date.now()}`,
          name: 'searchProductKnowledge',
          input: { feature: `nonexistent-feature-${Date.now()}` },
        },
      ],
      { ticketId, productId },
    );
    const after = metricValue(await scrape(), 'supporthub_knowledge_retrieval_outcomes_total', {
      matched: 'false',
    });

    expect(after).toBe(before + 1);
  });

  it('counts a tool invocation that fails at execution time', async () => {
    const ticketId = await createTicket();
    const session = await sessionRepository.create(ticketId);

    const before = metricValue(await scrape(), 'supporthub_tool_invocations_total', {
      tool: 'getTicketSnapshot',
      outcome: 'failed',
    });
    await toolsService.proposeAndEvaluate(
      session.id,
      [
        {
          type: 'tool_use',
          caller: { type: 'direct' },
          id: `toolu_${Date.now()}`,
          name: 'getTicketSnapshot',
          input: {},
        },
      ],
      { ticketId: 'nonexistent-ticket-id', productId },
    );
    const after = metricValue(await scrape(), 'supporthub_tool_invocations_total', {
      tool: 'getTicketSnapshot',
      outcome: 'failed',
    });

    expect(after).toBe(before + 1);
  });

  it('counts a valid known-error-code lookup', async () => {
    const code = `METRICS-ERR-${Date.now()}`;
    await errorCodesService.createErrorCode(productId, code, 'A test error for metrics.');

    const before = metricValue(await scrape(), 'supporthub_known_error_lookups_total', { code });
    await errorCodesService.findKnownIssuesByErrorCode(productId, code);
    const after = metricValue(await scrape(), 'supporthub_known_error_lookups_total', { code });

    expect(after).toBe(before + 1);
  });
});