feat: add crm dead-letter operations and benchmark summary

2026-06-30 23:07:43 +08:00
parent 90ad9e90a3
commit 9891fe9ef3
22 changed files with 1284 additions and 110 deletions
--- a/scripts/api-performance-benchmark.js
+++ b/scripts/api-performance-benchmark.js
@@ -15,6 +15,9 @@ const CONCURRENCY = envNumber('PERF_CONCURRENCY', 6);
 const RAMP_SECONDS = envNumber('PERF_RAMP_SECONDS', 3, { allowZero: true });
 const INCLUDE_WRITES = envBool('PERF_INCLUDE_WRITES', false);
 const INCLUDE_LEADERBOARD = envBool('PERF_INCLUDE_LEADERBOARD', false);
+const PRACTICE_FLOW_RATIO = Math.min(Math.max(envNumber('PERF_PRACTICE_FLOW_RATIO', 0.15, { allowZero: true }), 0), 1);
+const PRACTICE_FLOW_ANSWERS = envNumber('PERF_PRACTICE_FLOW_ANSWERS', 3);
+const ENSURE_SVIP_FOR_WRITES = envBool('PERF_ENSURE_SVIP_FOR_WRITES', true);
 const START_PORT = Number(process.env.PERF_API_PORT || 0) || 0;
 const TENANT_CODE = process.env.PERF_TENANT_CODE || 'master';
 const QUESTION_LIMIT = envNumber('PERF_QUESTION_LIMIT', 20);
@@ -111,7 +114,7 @@ async function requestJson(endpoint, context, timeoutMs = DEFAULT_TIMEOUT_MS) {
        message: payload?.error || payload?.message || response.statusText,
      };
    }
-    return { ok: true, status: response.status, elapsedMs, bytes: JSON.stringify(payload).length };
+    return { ok: true, status: response.status, elapsedMs, bytes: JSON.stringify(payload).length, payload };
  } catch (error) {
    return {
      ok: false,
@@ -124,6 +127,164 @@ async function requestJson(endpoint, context, timeoutMs = DEFAULT_TIMEOUT_MS) {
  }
 }

+function sampleFromResult(name, result, startedAt, finishedAt) {
+  return {
+    name,
+    ok: result.ok,
+    status: result.status,
+    elapsedMs: result.elapsedMs,
+    startedAt,
+    finishedAt,
+  };
+}
+
+function pickAnswerPayload(question) {
+  const subQuestions = Array.isArray(question.subQuestions) ? question.subQuestions : [];
+  if (subQuestions.length) {
+    return {
+      subAnswers: subQuestions.slice(0, Math.max(PRACTICE_FLOW_ANSWERS, 1)).map((subQuestion, index) => {
+        const subCorrectIndices = Array.isArray(subQuestion.correctOptionIndices) ? subQuestion.correctOptionIndices : [];
+        const subCorrectIndex = subQuestion.correctOptionIndex;
+        const subAnswer = {
+          subQuestionId: subQuestion.id || subQuestion.subQuestionId || subQuestion.key || `sub_${index + 1}`,
+          selectedOptions: [],
+          answerText: '',
+        };
+        if (subCorrectIndices.length) {
+          subAnswer.selectedOptions = subCorrectIndices.map(item => String(item));
+        } else if (subCorrectIndex !== null && subCorrectIndex !== undefined && Number.isFinite(Number(subCorrectIndex))) {
+          subAnswer.selectedOptions = [String(Number(subCorrectIndex))];
+        } else if (Array.isArray(subQuestion.options) && subQuestion.options.length) {
+          subAnswer.selectedOptions = ['0'];
+        } else {
+          subAnswer.answerText = 'performance benchmark answer';
+          subAnswer.selfJudgedCorrect = true;
+        }
+        return subAnswer;
+      }),
+    };
+  }
+  const correctIndices = Array.isArray(question.correctOptionIndices) ? question.correctOptionIndices : [];
+  const correctIndex = question.correctOptionIndex;
+  if (correctIndices.length) {
+    return { selectedOptions: correctIndices.map(item => String(item)) };
+  }
+  if (correctIndex !== null && correctIndex !== undefined && Number.isFinite(Number(correctIndex))) {
+    return { selectedOptions: [String(Number(correctIndex))] };
+  }
+  if (Array.isArray(question.options) && question.options.length) {
+    return { selectedOptions: ['0'] };
+  }
+  return { answerText: 'performance benchmark answer', selfJudgedCorrect: true };
+}
+
+async function runPracticeFlow(context, samples, errors) {
+  if (!context.blueprint || !context.collection) return false;
+  const flowStartedAt = performance.now();
+  const create = await requestJson({
+    name: 'learning.practice_flow.create',
+    method: 'POST',
+    path: '/api/learning/practice-sessions',
+    body: () => ({
+      blueprintId: context.blueprint.id,
+      collectionId: context.collection.id,
+      mode: context.blueprint.mode || 'sequential',
+      questionLimit: Math.min(QUESTION_LIMIT, Math.max(PRACTICE_FLOW_ANSWERS, 3)),
+      metadata: { source: 'api-performance-benchmark' },
+    }),
+  }, context);
+  const flowFinishedCreate = performance.now();
+  samples.push(sampleFromResult('learning.practice_flow.create', create, flowStartedAt, flowFinishedCreate));
+  if (!create.ok) {
+    pushError(errors, 'learning.practice_flow.create', create);
+    return true;
+  }
+
+  const practiceSessionId = create.payload?.item?.id;
+  if (!practiceSessionId) {
+    pushError(errors, 'learning.practice_flow.create', {
+      status: 200,
+      elapsedMs: create.elapsedMs,
+      error: 'PRACTICE_SESSION_ID_MISSING',
+      message: 'Practice session create response did not include item.id',
+    });
+    return true;
+  }
+
+  const detailStartedAt = performance.now();
+  const detail = await requestJson({
+    name: 'learning.practice_flow.detail',
+    method: 'GET',
+    path: '/api/learning/practice-sessions/detail',
+    query: { practiceSessionId },
+  }, context);
+  const detailFinishedAt = performance.now();
+  samples.push(sampleFromResult('learning.practice_flow.detail', detail, detailStartedAt, detailFinishedAt));
+  if (!detail.ok) {
+    pushError(errors, 'learning.practice_flow.detail', detail);
+    return true;
+  }
+
+  const questions = Array.isArray(detail.payload?.item?.questions) ? detail.payload.item.questions : [];
+  for (const question of questions.slice(0, PRACTICE_FLOW_ANSWERS)) {
+    if (!question?.id) continue;
+    const answerStartedAt = performance.now();
+    const answer = await requestJson({
+      name: 'learning.practice_flow.answer',
+      method: 'POST',
+      path: '/api/learning/answers',
+      body: () => ({
+        practiceSessionId,
+        questionId: question.id,
+        ...pickAnswerPayload(question),
+      }),
+    }, context);
+    const answerFinishedAt = performance.now();
+    samples.push(sampleFromResult('learning.practice_flow.answer', answer, answerStartedAt, answerFinishedAt));
+    if (!answer.ok) {
+      pushError(errors, 'learning.practice_flow.answer', answer);
+      return true;
+    }
+  }
+
+  const submitStartedAt = performance.now();
+  const submit = await requestJson({
+    name: 'learning.practice_flow.submit',
+    method: 'POST',
+    path: '/api/learning/practice-sessions/submit',
+    body: () => ({ practiceSessionId }),
+  }, context);
+  const submitFinishedAt = performance.now();
+  samples.push(sampleFromResult('learning.practice_flow.submit', submit, submitStartedAt, submitFinishedAt));
+  if (!submit.ok) {
+    pushError(errors, 'learning.practice_flow.submit', submit);
+    return true;
+  }
+
+  const reportStartedAt = performance.now();
+  const report = await requestJson({
+    name: 'learning.practice_flow.report',
+    method: 'GET',
+    path: '/api/learning/practice-sessions/report',
+    query: { practiceSessionId },
+  }, context);
+  const reportFinishedAt = performance.now();
+  samples.push(sampleFromResult('learning.practice_flow.report', report, reportStartedAt, reportFinishedAt));
+  if (!report.ok) pushError(errors, 'learning.practice_flow.report', report);
+  return true;
+}
+
+function pushError(errors, endpointName, result) {
+  if (errors.length >= MAX_ERRORS_TO_KEEP) return;
+  errors.push({
+    endpoint: endpointName,
+    status: result.status,
+    error: result.error,
+    message: result.message,
+    elapsedMs: round(result.elapsedMs),
+  });
+}
+
 async function waitForHealth(timeoutMs = 20_000) {
  const started = Date.now();
  let lastError = null;
@@ -203,8 +364,14 @@ async function discoverBenchmarkContext() {
      select pu.id, coalesce(pu.name, pu.username, pu.phone, pu.legacy_id, pu.id::text) as name
      from public.tenant_memberships tm
      join public.platform_users pu on pu.id = tm.user_id
+      left join public.entitlements e on e.tenant_id = tm.tenant_id
+        and e.user_id = pu.id
+        and e.entitlement_type = 'svip'
+        and e.status = 'active'
+        and (e.expires_at is null or e.expires_at > now())
      where tm.tenant_id = $1 and tm.status = 'active' and tm.role = 'student'
-      order by pu.created_at asc
+      group by pu.id, pu.name, pu.username, pu.phone, pu.legacy_id, pu.created_at
+      order by case when count(e.id) > 0 then 0 else 1 end, pu.created_at asc
      limit 1
      `,
      [tenant.id],
@@ -304,6 +471,39 @@ async function discoverBenchmarkContext() {
        )
      : null;

+    const hasActiveEntitlement = user
+      ? await one(
+          pool,
+          `
+          select id
+          from public.entitlements
+          where tenant_id = $1
+            and user_id = $2
+            and entitlement_type = 'svip'
+            and status = 'active'
+            and (expires_at is null or expires_at > now())
+          limit 1
+          `,
+          [tenant.id, user.id],
+        )
+      : null;
+    let benchmarkEntitlementCreated = false;
+    if (INCLUDE_WRITES && ENSURE_SVIP_FOR_WRITES && user && !hasActiveEntitlement) {
+      await pool.query(
+        `
+        insert into public.entitlements (
+          tenant_id, user_id, entitlement_type, scope_type, source_type,
+          starts_at, expires_at, status, metadata
+        )
+        values ($1, $2, 'svip', 'tenant', 'performance_benchmark',
+                now(), now() + interval '1 day', 'active',
+                '{"createdBy":"api-performance-benchmark"}'::jsonb)
+        `,
+        [tenant.id, user.id],
+      );
+      benchmarkEntitlementCreated = true;
+    }
+
    const dbStats = {
      before: await captureDbStats(pool),
      tableCounts: await many(
@@ -327,7 +527,13 @@ async function discoverBenchmarkContext() {

    return {
      tenant,
-      user,
+      user: {
+        ...user,
+        entitlement: {
+          hadActiveSvip: Boolean(hasActiveEntitlement),
+          benchmarkEntitlementCreated,
+        },
+      },
      entry,
      node,
      collection,
@@ -426,24 +632,23 @@ function createEndpointCatalog(context) {
      { name: 'catalog.handbook_entries', weight: 5, method: 'GET', path: '/api/catalog/handbook-entries', query: { chapterId: context.handbookChapter.id, includeContent: 'true' } },
    );
  }
-  if (INCLUDE_WRITES && context.blueprint && context.collection) {
-    endpoints.push({
-      name: 'learning.practice_session.create',
-      weight: 1,
-      method: 'POST',
-      path: '/api/learning/practice-sessions',
-      body: () => ({
-        blueprintId: context.blueprint.id,
-        collectionId: context.collection.id,
-        mode: context.blueprint.mode || 'sequential',
-        questionLimit: Math.min(5, QUESTION_LIMIT),
-      }),
-    });
-  }
-
  return expandWeightedEndpoints(endpoints);
 }

+function benchmarkEndpointNames(weightedEndpoints, context) {
+  const names = new Set(weightedEndpoints.map(endpoint => endpoint.name));
+  if (INCLUDE_WRITES && context.blueprint && context.collection && PRACTICE_FLOW_RATIO > 0) {
+    for (const name of [
+      'learning.practice_flow.create',
+      'learning.practice_flow.detail',
+      'learning.practice_flow.answer',
+      'learning.practice_flow.submit',
+      'learning.practice_flow.report',
+    ]) names.add(name);
+  }
+  return [...names].sort();
+}
+
 function expandWeightedEndpoints(endpoints) {
  const weighted = [];
  for (const endpoint of endpoints) {
@@ -542,33 +747,22 @@ async function workerLoop(workerId, weightedEndpoints, context, stopAt, samples,
  const rampDelay = RAMP_SECONDS > 0 ? (workerId / Math.max(CONCURRENCY, 1)) * RAMP_SECONDS * 1000 : 0;
  if (rampDelay > 0) await sleep(rampDelay);
  while (performance.now() < stopAt) {
+    if (INCLUDE_WRITES && PRACTICE_FLOW_RATIO > 0 && Math.random() < PRACTICE_FLOW_RATIO) {
+      await runPracticeFlow(context, samples, errors);
+      continue;
+    }
    const endpoint = pickEndpoint(weightedEndpoints);
    const startedAt = performance.now();
    const result = await requestJson(endpoint, context);
    const finishedAt = performance.now();
-    samples.push({
-      name: endpoint.name,
-      ok: result.ok,
-      status: result.status,
-      elapsedMs: result.elapsedMs,
-      startedAt,
-      finishedAt,
-    });
-    if (!result.ok && errors.length < MAX_ERRORS_TO_KEEP) {
-      errors.push({
-        endpoint: endpoint.name,
-        status: result.status,
-        error: result.error,
-        message: result.message,
-        elapsedMs: round(result.elapsedMs),
-      });
-    }
+    samples.push(sampleFromResult(endpoint.name, result, startedAt, finishedAt));
+    if (!result.ok) pushError(errors, endpoint.name, result);
  }
 }

 async function runBenchmark(context) {
  const weightedEndpoints = createEndpointCatalog(context);
-  const endpointNames = [...new Set(weightedEndpoints.map(endpoint => endpoint.name))].sort();
+  const endpointNames = benchmarkEndpointNames(weightedEndpoints, context);
  const samples = [];
  const errors = [];
  const startedAt = new Date();
@@ -596,6 +790,8 @@ async function runBenchmark(context) {
      concurrency: CONCURRENCY,
      rampSeconds: RAMP_SECONDS,
      includeWrites: INCLUDE_WRITES,
+      practiceFlowRatio: INCLUDE_WRITES ? PRACTICE_FLOW_RATIO : 0,
+      practiceFlowAnswers: INCLUDE_WRITES ? PRACTICE_FLOW_ANSWERS : 0,
      questionLimit: QUESTION_LIMIT,
      timeoutMs: DEFAULT_TIMEOUT_MS,
    },
@@ -635,6 +831,10 @@ function markdownReport(report) {
  lines.push(`- 持续时间：${report.config.durationSeconds}s`);
  lines.push(`- Ramp：${report.config.rampSeconds}s`);
  lines.push(`- 写入压测：${report.config.includeWrites ? '开启' : '关闭'}`);
+  if (report.config.includeWrites) {
+    lines.push(`- 刷题闭环比例：${Math.round((report.config.practiceFlowRatio || 0) * 100)}%`);
+    lines.push(`- 每个刷题闭环提交答案数：${report.config.practiceFlowAnswers || 0}`);
+  }
  lines.push(`- 请求超时：${report.config.timeoutMs}ms`);
  lines.push('');
  lines.push('## 总览');
@@ -682,7 +882,8 @@ function markdownReport(report) {
  }
  lines.push('## 说明');
  lines.push('');
-  lines.push('- 默认压测是只读混合工作负载，适合在真实迁移库上做烟测。');
+  lines.push('- 默认压测是只读混合工作负载，适合在真实迁移库上做烟测；开启写入后会执行“创建练习 session -> 读取详情 -> 提交答案 -> 交卷 -> 读取报告”的刷题闭环。');
+  lines.push('- 并发 worker 是无停顿请求流，不能直接等同于真实在线人数；真实学生有读题、思考、翻页和网络间隔，应结合前端埋点估算每人 RPS。');
  lines.push('- 结果只能代表当前本机 Docker、API 进程和数据库状态；4 核 16G 云服务器需要按 runbook 跑阶梯并发。');
  lines.push('- 若 P95 明显升高，下一步应结合 PostgreSQL 慢 SQL、`pg_stat_activity` 和 API 日志定位。');
  lines.push('');