chore: require real data api benchmark evidence

2026-06-30 13:13:21 +08:00
parent 6e17e77aaf
commit 6d5ea62bf5
6 changed files with 136 additions and 5 deletions
--- a/README.md
+++ b/README.md
@@ -394,7 +394,7 @@ cp docs/refactor/production-launch-evidence.template.json docs/refactor/producti
 npm run launch:gate -- --evidence docs/refactor/production-launch-evidence.json
 ```

-`production-launch-evidence.json` 不入 Git，里面只记录验收摘要、artifact 路径、审批人和时间，不保存真实 access token、支付密钥、对象存储密钥或用户隐私明细。门禁会要求以下证据全部齐备并通过：`readiness:production`、`readiness:production:db`、真实 `smoke:auth:remote`、`test:rls`、PocketBase production dry-run、`pb:import:validate`、`pb:import:sample`、API/worker/Taro 构建、`audit:runtime`、`@codex-security` 扫描，以及备份、回滚、真实数据抽样、生产 provider、对象存储控制、支付对账和三套 H5 `runtime-config.json` 人工确认。
+`production-launch-evidence.json` 不入 Git，里面只记录验收摘要、artifact 路径、审批人和时间，不保存真实 access token、支付密钥、对象存储密钥或用户隐私明细。门禁会要求以下证据全部齐备并通过：`readiness:production`、`readiness:production:db`、真实 `smoke:auth:remote`、`test:rls`、PocketBase production dry-run、`pb:import:validate`、`pb:import:sample`、真实数据 API 读路径压测、API/worker/Taro 构建、`audit:runtime`、`@codex-security` 扫描，以及备份、回滚、真实数据抽样、生产 provider、对象存储控制、支付对账和三套 H5 `runtime-config.json` 人工确认。

 模板文件：

--- a/docs/refactor/next-development-todo.md
+++ b/docs/refactor/next-development-todo.md
@@ -88,7 +88,7 @@
   - 生产 `.env` 模板和 `npm run readiness:production` / `npm run readiness:production:db` 已补，后续上云必须作为验收 gate。
   - Auth/JWKS 上云后必须临时设置 `AUTH_SMOKE_*` 环境变量并运行 `npm run smoke:auth:remote`，真实 access token 不得写入仓库、前端配置或日志。
   - 本地/预生产必须同时跑 `npm run test:rls`，它验证运行时 JWT claim 下的租户隔离，和 `readiness:production:db` 的静态 policy 检查互补。
-   - 已补 `npm run launch:gate` 生产上线证据门禁和 `docs/refactor/production-launch-evidence.template.json` 模板；最终切换前必须把 readiness、远程 Auth、RLS、生产 dry-run、导入校验、`pb:import:sample` 业务抽样、API/worker/Taro、运行时审计、`@codex-security`、备份/回滚/真实抽样/生产 provider 等证据填入本地 `production-launch-evidence.json` 并通过门禁。
+   - 已补 `npm run launch:gate` 生产上线证据门禁和 `docs/refactor/production-launch-evidence.template.json` 模板；最终切换前必须把 readiness、远程 Auth、RLS、生产 dry-run、导入校验、`pb:import:sample` 业务抽样、真实数据 API 读路径压测、API/worker/Taro、运行时审计、`@codex-security`、备份/回滚/真实抽样/生产 provider 等证据填入本地 `production-launch-evidence.json` 并通过门禁。
   - 确认数据库迁移流程、备份恢复、日志、告警。
   - 准备 API 容器部署和 Supabase 云端/自托管连接方案。
   - 已补 `npm run perf:api:local` 和 `docs/refactor/performance-benchmark-runbook.md`，可在本地或云端对真实迁移数据做只读混合压测；4 核 16G 正式容量报告需上云后按 6/30/50/100 阶梯并发复跑并归档到本地上线证据。
--- a/docs/refactor/performance-benchmark-runbook.md
+++ b/docs/refactor/performance-benchmark-runbook.md
@@ -151,4 +151,37 @@ npm run perf:api:local
 npm run launch:gate
 ```

+`launch:gate` 会强制检查一条真实数据读路径压测证据：
+
+```json
+{
+  "id": "performance.api-real-data-read",
+  "status": "pass",
+  "command": "PERF_START_SERVER=false PERF_API_BASE=https://api.example.com PERF_DURATION_SECONDS=300 PERF_CONCURRENCY=30 PERF_RAMP_SECONDS=30 PERF_INCLUDE_WRITES=false npm run perf:api:local > docs/refactor/launch-artifacts/api-real-data-read-benchmark.log",
+  "completedAt": "2026-06-30T10:48:00+08:00",
+  "artifact": "launch-artifacts/api-real-data-read-benchmark.log",
+  "summary": {
+    "errors": 0,
+    "errorRate": 0,
+    "p95Ms": 0,
+    "p99Ms": 0,
+    "concurrency": 30,
+    "durationSeconds": 300,
+    "includeWrites": false
+  }
+}
+```
+
+上线门禁的最低机器阈值：
+
+- `errors = 0`
+- `errorRate <= 0.001`
+- `p95Ms <= 300`
+- `p99Ms <= 800`
+- `concurrency >= 30`
+- `durationSeconds >= 120`
+- `includeWrites = false`
+
+这些字段应从 `perf:api:local` JSON 报告的 `summary.errors`、`summary.errorRate`、`summary.latencyOk.p95Ms`、`summary.latencyOk.p99Ms`、`config.concurrency`、`config.durationSeconds`、`config.includeWrites` 转写到证据摘要。更高的 50/100 并发、写入混合场景和容量结论仍应作为人工容量报告归档；门禁只负责挡住明显不达标的基础读路径。
+
 证据中只记录报告路径、并发矩阵、P95/P99、错误率和结论，不保存真实 token、支付密钥、用户隐私或完整响应。
--- a/docs/refactor/production-launch-evidence.template.json
+++ b/docs/refactor/production-launch-evidence.template.json
@@ -87,6 +87,22 @@
        "pass": 0
      }
    },
+    {
+      "id": "performance.api-real-data-read",
+      "status": "pass",
+      "command": "PERF_START_SERVER=false PERF_API_BASE=https://api.example.com PERF_DURATION_SECONDS=300 PERF_CONCURRENCY=30 PERF_RAMP_SECONDS=30 PERF_INCLUDE_WRITES=false npm run perf:api:local > docs/refactor/launch-artifacts/api-real-data-read-benchmark.log",
+      "completedAt": "2026-06-30T10:48:00+08:00",
+      "artifact": "launch-artifacts/api-real-data-read-benchmark.log",
+      "summary": {
+        "errors": 0,
+        "errorRate": 0,
+        "p95Ms": 0,
+        "p99Ms": 0,
+        "concurrency": 30,
+        "durationSeconds": 300,
+        "includeWrites": false
+      }
+    },
    {
      "id": "api.integration",
      "status": "pass",
--- a/scripts/production-launch-gate-test.js
+++ b/scripts/production-launch-gate-test.js
@@ -12,6 +12,21 @@ function isoNow() {
  return new Date().toISOString();
 }

+function sampleSummaryValue(expectedValue) {
+  if (!expectedValue || typeof expectedValue !== 'object' || Array.isArray(expectedValue)) return expectedValue;
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'eq')) return expectedValue.eq;
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'oneOf')) return expectedValue.oneOf?.[0];
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'lte')) return expectedValue.lte;
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'lt')) return Number(expectedValue.lt) - 1;
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'gte')) return expectedValue.gte;
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'gt')) return Number(expectedValue.gt) + 1;
+  return expectedValue;
+}
+
+function sampleSummary(summarySpec) {
+  return Object.fromEntries(Object.entries(summarySpec || {}).map(([key, value]) => [key, sampleSummaryValue(value)]));
+}
+
 function createEvidence(tempDir, overrides = {}) {
  const artifactDir = path.join(tempDir, 'launch-artifacts');
  fs.mkdirSync(artifactDir, { recursive: true });
@@ -25,7 +40,7 @@ function createEvidence(tempDir, overrides = {}) {
      command: `npm run ${spec.commandIncludes} -- recorded-for-launch-gate`,
      completedAt: isoNow(),
      artifact,
-      summary: { ...spec.summary },
+      summary: sampleSummary(spec.summary),
    };
  });

@@ -105,6 +120,18 @@ assert.ok(
  'migration profile mismatch should be reported as a blocker',
 );

+const slowBenchmark = runGate(tempDir => {
+  const evidence = createEvidence(tempDir);
+  const item = evidence.checks.find(check => check.id === 'performance.api-real-data-read');
+  item.summary.p95Ms = 301;
+  return evidence;
+});
+assert.notEqual(slowBenchmark.status, 0, 'slow API benchmark should fail launch gate');
+assert.ok(
+  slowBenchmark.payload.checks?.some(item => item.id === 'check.performance.api-real-data-read.summary' && item.status === 'blocker'),
+  'slow API benchmark should be reported as a blocker',
+);
+
 const missingBusinessSampling = runGate(tempDir => {
  const evidence = createEvidence(tempDir);
  evidence.checks = evidence.checks.filter(item => item.id !== 'migration.pb-import-sample');
--- a/scripts/production-launch-gate.js
+++ b/scripts/production-launch-gate.js
@@ -56,6 +56,20 @@ const gateChecks = [
    commandIncludes: 'pb:import:sample',
    summary: { fail: 0 },
  },
+  {
+    id: 'performance.api-real-data-read',
+    label: 'Real-data API read benchmark',
+    commandIncludes: 'perf:api:local',
+    summary: {
+      errors: 0,
+      errorRate: { lte: 0.001 },
+      p95Ms: { lte: 300 },
+      p99Ms: { lte: 800 },
+      concurrency: { gte: 30 },
+      durationSeconds: { gte: 120 },
+      includeWrites: false,
+    },
+  },
  {
    id: 'api.integration',
    label: 'API integration regression',
@@ -235,13 +249,54 @@ function compareSummary(actualSummary, expectedSummary) {
      continue;
    }
    const actualValue = valueAt(actualSummary, key);
-    if (actualValue !== expectedValue) {
-      failures.push(`${key} expected ${JSON.stringify(expectedValue)} but got ${JSON.stringify(actualValue)}`);
+    const comparisonFailure = compareExpectedValue(actualValue, expectedValue);
+    if (comparisonFailure) {
+      failures.push(`${key} ${comparisonFailure}`);
    }
  }
  return failures;
 }

+function isComparatorSpec(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  return ['eq', 'lt', 'lte', 'gt', 'gte', 'oneOf'].some(key => Object.prototype.hasOwnProperty.call(value, key));
+}
+
+function compareExpectedValue(actualValue, expectedValue) {
+  if (!isComparatorSpec(expectedValue)) {
+    if (actualValue !== expectedValue) {
+      return `expected ${JSON.stringify(expectedValue)} but got ${JSON.stringify(actualValue)}`;
+    }
+    return '';
+  }
+
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'eq') && actualValue !== expectedValue.eq) {
+    return `expected ${JSON.stringify(expectedValue.eq)} but got ${JSON.stringify(actualValue)}`;
+  }
+  if (Object.prototype.hasOwnProperty.call(expectedValue, 'oneOf')) {
+    const choices = Array.isArray(expectedValue.oneOf) ? expectedValue.oneOf : [];
+    if (!choices.includes(actualValue)) {
+      return `expected one of ${JSON.stringify(choices)} but got ${JSON.stringify(actualValue)}`;
+    }
+  }
+
+  const numericChecks = [
+    ['lt', (actual, expected) => actual < expected, '<'],
+    ['lte', (actual, expected) => actual <= expected, '<='],
+    ['gt', (actual, expected) => actual > expected, '>'],
+    ['gte', (actual, expected) => actual >= expected, '>='],
+  ];
+  for (const [key, predicate, label] of numericChecks) {
+    if (!Object.prototype.hasOwnProperty.call(expectedValue, key)) continue;
+    const actualNumber = Number(actualValue);
+    const expectedNumber = Number(expectedValue[key]);
+    if (!Number.isFinite(actualNumber) || !Number.isFinite(expectedNumber) || !predicate(actualNumber, expectedNumber)) {
+      return `expected ${label} ${expectedNumber} but got ${JSON.stringify(actualValue)}`;
+    }
+  }
+  return '';
+}
+
 function resultCollector() {
  const checks = [];
  function pass(id, message, details = {}) {