support_backend/tests/concurrency/sla-race.test.ts

import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { buildApp } from '@/app';
import { prismaClient } from '@/infrastructure/database';
import { FastifyInstance } from 'fastify';
import { loginAs, authHeader } from '../helpers/auth';
import {
  encryptCredential,
  generateCredentialSecret,
  issueIntegrationToken,
} from '@/modules/catalog/products';
import { slaService } from '@/modules/orchestration/sla';

/**
 * specs/016-load-concurrency-testing User Story 2 / FR-002 / SC-002: concurrent pause, resume,
 * and breach-sweep activity against the same SLA run must always leave it in one
 * internally-consistent, auditable state — never a state with contradictory fields (paused with
 * no pause timestamp, or a legitimately breached run silently reverted to running by a racing
 * resume). Before research.md §2's fix (SLARun.version + SlaRunRepository.updateWithVersion),
 * SlaService.pause/resume/complete/runBreachDetectionSweep each did a plain read-then-write with
 * no guard, so two racing calls could clobber each other's writes.
 */
describe('SLA pause/resume/sweep race (User Story 2)', () => {
  let app: FastifyInstance;
  let authToken: string;
  const externalProductId = `TEST_SLA_RACE_PROD_${Date.now()}`;
  const skillTag = `sla_race_skill_${Date.now()}`;
  let productId: string;
  let teamId: string;
  let agentId: string;
  let nodeId: string;
  let policyId: string;
  let secret: string;
  const createdTicketIds: string[] = [];

  async function createTicketAndAssign(): Promise<string> {
    const token = issueIntegrationToken(secret, {
      externalProductId,
      tenantId: 'tenant-1',
      userId: 'user-1',
    });
    const created = await app.inject({
      method: 'POST',
      url: '/v1/support/requests',
      headers: { authorization: `Bearer ${token}` },
      payload: {
        productId: externalProductId,
        tenantId: 'tenant-1',
        userId: 'user-1',
        source: 'test',
        problem: `SLA race test ${Date.now()}-${Math.random()}`,
      },
    });
    expect(created.statusCode).toBe(202);
    const ticketId = created.json().data.ticketId as string;
    createdTicketIds.push(ticketId);

    const ticket = await prismaClient.ticket.findUniqueOrThrow({ where: { id: ticketId } });
    await app.inject({
      method: 'PATCH',
      url: `/tickets/${ticketId}/status`,
      headers: authHeader(authToken),
      payload: { status: 'HUMAN_ESCALATION', expectedVersion: ticket.version },
    });
    return ticketId;
  }

  beforeAll(async () => {
    app = await buildApp();
    authToken = await loginAs(app, 'ADMIN');

    const product = await prismaClient.product.create({
      data: { externalProductId, name: 'SLA Race Test Product', status: 'active' },
    });
    productId = product.id;
    secret = generateCredentialSecret();
    await prismaClient.productIntegration.create({
      data: {
        productId,
        credentialRef: encryptCredential(secret),
        authMechanism: 'signed_token',
        allowedScope: { tenantIds: ['tenant-1'] },
        status: 'active',
        rateLimitPerMinute: 1000,
        rateLimitPerUserPerMinute: 1000,
      },
    });

    const team = await app.inject({
      method: 'POST',
      url: '/admin/teams',
      headers: authHeader(authToken),
      payload: { name: `SLA Race Team ${Date.now()}` },
    });
    teamId = team.json().data.id;

    const agent = await app.inject({
      method: 'POST',
      url: `/admin/teams/${teamId}/agents`,
      headers: authHeader(authToken),
      payload: { name: 'SLA Race Agent' },
    });
    agentId = agent.json().data.id;
    await app.inject({
      method: 'PUT',
      url: `/admin/agents/${agentId}/skills/${skillTag}`,
      headers: authHeader(authToken),
      payload: { level: 3 },
    });

    const node = await app.inject({
      method: 'POST',
      url: '/admin/hierarchy-nodes',
      headers: authHeader(authToken),
      payload: {
        name: 'SLA Race Node',
        order: 0,
        productScope: [externalProductId],
        skills: [skillTag],
        assignmentStrategy: 'ROUND_ROBIN',
      },
    });
    nodeId = node.json().data.id;

    const policy = await app.inject({
      method: 'POST',
      url: '/admin/sla-policies',
      headers: authHeader(authToken),
      payload: { name: 'SLA Race Policy', productId, firstResponseMinutes: 30, resolutionMinutes: 240 },
    });
    policyId = policy.json().data.id;
  });

  afterAll(async () => {
    const ticketFilter = { ticketId: { in: createdTicketIds } };
    await prismaClient.escalationEvent.deleteMany({ where: ticketFilter });
    await prismaClient.sLARun.deleteMany({ where: ticketFilter });
    await prismaClient.sLAPolicy.deleteMany({ where: { id: policyId } });
    await prismaClient.assignmentHistory.deleteMany({ where: ticketFilter });
    await prismaClient.assignment.deleteMany({ where: ticketFilter });
    await prismaClient.hierarchyNode.deleteMany({ where: { id: nodeId } });
    await prismaClient.agentSkill.deleteMany({ where: { agentId } });
    await prismaClient.agent.deleteMany({ where: { teamId } });
    await prismaClient.team.deleteMany({ where: { id: teamId } });
    await prismaClient.ticketMessage.deleteMany({ where: ticketFilter });
    await prismaClient.ticket.deleteMany({ where: { id: { in: createdTicketIds } } });
    await prismaClient.problem.deleteMany({ where: { productId } });
    await prismaClient.productIntegration.deleteMany({ where: { productId } });
    await prismaClient.product.deleteMany({ where: { id: productId } });
    await app.close();
  });

  function assertInternallyConsistent(run: {
    status: string;
    pausedAt: Date | null;
    resumedAt: Date | null;
    breachedAt: Date | null;
  }) {
    if (run.status === 'paused') {
      expect(run.pausedAt).not.toBeNull();
    } else {
      expect(run.pausedAt).toBeNull();
    }
    // A run the sweep has genuinely marked breached must never be silently reverted to running
    // by a racing resume — status and breachedAt must agree with each other.
    if (run.status === 'breached') {
      expect(run.breachedAt).not.toBeNull();
    }
  }

  it(
    'leaves an internally-consistent final state under concurrent pause/resume/sweep',
    async () => {
      const ticketId = await createTicketAndAssign();

      // Force the run's resolution due date into the past so the breach sweep genuinely has
      // something real to detect concurrently with pause/resume, not a no-op query.
      await prismaClient.sLARun.update({
        where: { ticketId },
        data: { resolutionDueAt: new Date(Date.now() - 60_000) },
      });

      await Promise.all([
        slaService.pause(ticketId),
        slaService.resume(ticketId),
        slaService.runBreachDetectionSweep(),
        slaService.pause(ticketId),
        slaService.resume(ticketId),
      ]);

      const run = await prismaClient.sLARun.findUniqueOrThrow({ where: { ticketId } });
      assertInternallyConsistent(run);
    },
    30000,
  );

  it(
    'holds consistently across 10 repeated runs (SC-002: zero contradictory-state outcomes)',
    async () => {
      for (let run = 0; run < 10; run++) {
        const ticketId = await createTicketAndAssign();
        await prismaClient.sLARun.update({
          where: { ticketId },
          data: { resolutionDueAt: new Date(Date.now() - 60_000) },
        });

        await Promise.all([
          slaService.pause(ticketId),
          slaService.resume(ticketId),
          slaService.runBreachDetectionSweep(),
        ]);

        const finalRun = await prismaClient.sLARun.findUniqueOrThrow({ where: { ticketId } });
        assertInternallyConsistent(finalRun);
      }
    },
    60000,
  );
});