From b1146787abc7da171903639e0943598033cf8c9d Mon Sep 17 00:00:00 2001 From: xiong Date: Thu, 30 Jul 2026 16:14:41 +0800 Subject: [PATCH] perf: add reproducible concurrency test suite --- tools/performance/README.md | 51 ++++++-- tools/performance/RESULTS-2026-07-30.md | 48 ++++++++ tools/performance/run-local.sh | 154 ++++++++++++++++++++++++ tools/performance/tiku-api.js | 146 ++++++++++++++++------ 4 files changed, 352 insertions(+), 47 deletions(-) create mode 100644 tools/performance/RESULTS-2026-07-30.md create mode 100755 tools/performance/run-local.sh diff --git a/tools/performance/README.md b/tools/performance/README.md index 24fb3ee..636e65a 100644 --- a/tools/performance/README.md +++ b/tools/performance/README.md @@ -1,12 +1,49 @@ -# TIKU API performance scenarios +# TIKU API 并发测试 -Run the API against a prepared PostgreSQL database, then execute: +这套测试用固定到达率验证单个 Tiku.Api 实例，并把 PostgreSQL 与 Redis 放进真实请求链路。它不是单测，也不会使用内存数据库。 + +## 场景 + +| 模式 | 请求 | 依赖含义 | +| --- | --- | --- | +| `hot` | `GET /api/catalog/regions` | 租户解析 + 热输出缓存，代表高缓存命中读流量 | +| `cold` | 同一目录接口，每次附加唯一查询参数 | 绕过输出缓存复用，持续执行租户解析与 PostgreSQL 目录查询 | +| `ready` | `GET /api/health/ready` | 每次检查 PostgreSQL、Redis 以及消息/outbox 就绪状态 | +| `mixed` | 70% hot、20% cold、10% ready | 默认的读多型业务流量 | + +`setup()` 会强制校验 readiness 响应中的 `database=true`、`redis.configured=true` 和 `redis.ready=true`，所以 Redis 没接上时测试会直接失败，不会给出误导性的容量数字。 + +## 一键本机测试 + +前置条件：本机 PostgreSQL 已有迁移和种子数据，Docker 中存在名为 `tiku-redis` 的 Redis 容器。脚本会构建并启动一个独立的 Release API（默认端口 5091），关闭应用限流以测应用与依赖本身，并使用本机 k6 或 `grafana/k6` Docker 镜像发压。 ```bash -TIKU_BASE_URL=http://localhost:5000 \ -TIKU_TENANT_CODE= \ -TIKU_ACCESS_TOKEN= \ -k6 run tools/performance/tiku-api.js +tools/performance/run-local.sh ``` -Do not store access tokens or database credentials in this directory. The default threshold is only a smoke gate; release comparison uses the same dataset and environment before and after a change, and requires at least a 50% p95 improvement for the targeted endpoint. +常用参数： + +```bash +TIKU_MODE=mixed TIKU_RATE=250 TIKU_DURATION=30s tools/performance/run-local.sh +TIKU_MODE=cold TIKU_RATE=100 TIKU_DURATION=60s tools/performance/run-local.sh +TIKU_MODE=ready TIKU_RATE=100 TIKU_DURATION=30s tools/performance/run-local.sh +``` + +可配置变量： + +- `TIKU_MODE`：`mixed`（默认）、`hot`、`cold` 或 `ready`。 +- `TIKU_RATE`：目标每秒迭代数，默认 100；每次迭代恰好发一个请求。 +- `TIKU_DURATION`：持续时间，默认 30 秒。 +- `TIKU_TENANT_CODE`：有效租户 slug，默认 `demo-crm-school`。 +- `TIKU_API_PORT`：独立 API 端口，默认 5091。 +- `TIKU_REDIS_CONTAINER`：Redis 容器名，默认 `tiku-redis`。 +- `TIKU_DATABASE_NAME`：本机 PostgreSQL 数据库名，默认 `tiku`。 +- `DATABASE_URL`：需要覆盖 API 默认开发连接串时设置；不要写入仓库。 + +结果写入忽略提交的 `artifacts/performance/<时间>-<模式>-<速率>/`，包含 k6 JSON 汇总、API 日志、主机信息、PostgreSQL 与 Redis 测试前后快照。k6 阈值是错误率低于 1%、p95 低于 500 ms、p99 低于 1 s，并且不允许掉迭代。 + +## 解释结果 + +从较低 `TIKU_RATE` 逐步翻倍。当首次出现 `dropped_iterations > 0`、错误率达到 1%，或延迟阈值失败时，上一档可视为当前机器、当前数据集、单实例配置下的保守持续吞吐。生产容量还要在接近生产的独立压测机、数据量、网络、连接池和观测配置下复测，不能直接按本机核数线性外推。 + +请勿把访问令牌、数据库密码或 Redis 密码写进此目录或结果文件。 diff --git a/tools/performance/RESULTS-2026-07-30.md b/tools/performance/RESULTS-2026-07-30.md new file mode 100644 index 0000000..4e92e4e --- /dev/null +++ b/tools/performance/RESULTS-2026-07-30.md @@ -0,0 +1,48 @@ +# 2026-07-30 本机并发基线 + +## 结论 + +- 默认 `mixed` 读流量在 10 秒短测中完整通过 5000 RPS；6000 RPS 开始出现少量 dropped iterations，8000 RPS 已明显饱和。因此这台机器上的单实例短时安全线应按 **5000 RPS 以下**理解，而不是把 8000 RPS 当成容量。 +- 每个请求都执行 PostgreSQL + Redis 依赖检查的 `ready` 场景，2000 RPS 完整通过；4000 RPS 完成 39,969/40,000 次目标迭代，出现 32 次 dropped iterations 与 8 次 503，严格门槛未通过。因此依赖重型场景的已验证短时安全线是 **至少 2000 RPS，4000 RPS 已在边缘**。 +- 这些数字是本机、小种子数据、10 秒短测结果，不是生产 SLA 或长期持续容量。 + +## 环境 + +- Mac17,3，10 个逻辑 CPU，16 GiB 内存，macOS 26.5.2，arm64。 +- .NET SDK 10.0.301，ASP.NET Core Runtime 10.0.9，Release 单实例。 +- PostgreSQL 与 API 在本机运行；Redis 7 运行在 Docker `tiku-redis`。 +- k6 运行在 `grafana/k6:latest` Docker 容器。 +- API 业务限流关闭，后台任务关闭，默认日志级别调到 Warning，避免测到限流器或日志终端吞吐。 +- 租户使用 `demo-crm-school`，目录响应为小数据集。 + +## 结果 + +### 混合读流量 + +流量比例为 70% 热目录、20% 唯一查询参数的冷目录、10% PostgreSQL + Redis readiness。 + +| 目标 RPS | 完成迭代 | 错误率 | dropped | p95 | p99 | 判定 | +| ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1000 | 10,001 | 0% | 0 | 1.31 ms | 1.97 ms | 通过 | +| 2000 | 20,001 | 0% | 0 | 1.32 ms | 6.54 ms | 通过 | +| 4000 | 40,001 | 0% | 0 | 1.43 ms | 6.38 ms | 通过 | +| 5000 | 50,001 | 0.004% | 0 | 7.93 ms | 62.29 ms | 通过 | +| 6000 | 59,859 | 0% | 142 | 5.58 ms | 29.00 ms | 未通过：开始掉迭代 | +| 7000 | 69,776 | 0% | 226 | 32.27 ms | 84.21 ms | 未通过：掉迭代 | +| 8000 | 62,186 | 0.01% | 17,815 | 936.11 ms | 1.06 s | 饱和 | + +### PostgreSQL + Redis 全请求依赖检查 + +4000 RPS 的独立复测结果： + +- 39,969 次完成迭代，实际约 3957 RPS。 +- 8/39,971 HTTP 请求失败，错误率 0.02%；API 日志记录 8 次 readiness 503。 +- dropped iterations 32；p95 19.33 ms，p99 84.07 ms，最大 384.24 ms。 +- PostgreSQL `xact_commit` 增加 79,931，`blks_hit` 增加 23,661，`blks_read` 不变，说明本次数据页全部命中 PostgreSQL shared buffers。 +- Redis `total_commands_processed` 增加 39,989，`rejected_connections` 保持 0，证明 Redis 确实参与了几乎每次请求。 + +对应原始结果位于忽略提交的 `artifacts/performance/20260730-161031-ready-4000rps/`。 + +## 适用边界 + +当前目录数据很小，而且 PostgreSQL 页面全在内存中；生产数据量上升后，冷查询、索引质量与返回体大小会显著改变结果。API、PostgreSQL 和 Redis 也在同一台开发机上，会产生资源竞争且没有真实网络延迟。正式容量结论至少还需要：独立发压机、接近生产的数据量、30 分钟以上稳态测试、认证读写场景，以及对 CPU、GC、Npgsql 连接池、PostgreSQL locks/I/O 与 Redis latency 的时序观测。 diff --git a/tools/performance/run-local.sh b/tools/performance/run-local.sh new file mode 100755 index 0000000..a883445 --- /dev/null +++ b/tools/performance/run-local.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +api_port="${TIKU_API_PORT:-5091}" +mode="${TIKU_MODE:-mixed}" +rate="${TIKU_RATE:-100}" +duration="${TIKU_DURATION:-30s}" +tenant_code="${TIKU_TENANT_CODE:-demo-crm-school}" +redis_container="${TIKU_REDIS_CONTAINER:-tiku-redis}" +database_name="${TIKU_DATABASE_NAME:-tiku}" +run_stamp="$(date +%Y%m%d-%H%M%S)" +result_dir="${repo_root}/artifacts/performance/${run_stamp}-${mode}-${rate}rps" +api_pid="" + +cleanup() { + if [[ -n "${api_pid}" ]] && kill -0 "${api_pid}" 2>/dev/null; then + kill "${api_pid}" 2>/dev/null || true + wait "${api_pid}" 2>/dev/null || true + fi + api_pid="" +} +trap cleanup EXIT INT TERM + +mkdir -p "${result_dir}" + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the Redis readiness check" >&2 + exit 1 +fi +if ! docker inspect "${redis_container}" >/dev/null 2>&1; then + echo "Redis container '${redis_container}' does not exist" >&2 + exit 1 +fi +if [[ "$(docker inspect -f '{{.State.Running}}' "${redis_container}")" != "true" ]]; then + echo "Redis container '${redis_container}' is not running" >&2 + exit 1 +fi +docker exec "${redis_container}" redis-cli ping | grep -qx PONG + +if ! command -v psql >/dev/null 2>&1; then + echo "psql is required to capture PostgreSQL metrics" >&2 + exit 1 +fi +psql -d "${database_name}" -Atqc 'select 1' | grep -qx 1 + +( + cd "${repo_root}" + dotnet build TIKU-BACKEND.slnx -c Release --no-restore +) + +host_url="http://127.0.0.1:${api_port}" +load_url="${host_url}" + +( + cd "${repo_root}" + exec env \ + ASPNETCORE_ENVIRONMENT=Development \ + ASPNETCORE_URLS="http://0.0.0.0:${api_port}" \ + ConnectionStrings__Redis=localhost:6379 \ + RateLimiting__Enabled=false \ + BackgroundProcessing__Enabled=false \ + Serilog__MinimumLevel__Default=Warning \ + dotnet run --project Tiku.Api -c Release --no-build --no-launch-profile +) >"${result_dir}/api.log" 2>&1 & +api_pid=$! + +ready_body="" +for _ in $(seq 1 60); do + if ready_body="$(curl -fsS "${host_url}/api/health/ready" 2>/dev/null)"; then + break + fi + if ! kill -0 "${api_pid}" 2>/dev/null; then + echo "Tiku.Api stopped during startup; see ${result_dir}/api.log" >&2 + exit 1 + fi + sleep 1 +done + +if [[ -z "${ready_body}" ]]; then + echo "Tiku.Api did not become ready; see ${result_dir}/api.log" >&2 + exit 1 +fi +printf '%s\n' "${ready_body}" >"${result_dir}/readiness.json" +if ! printf '%s' "${ready_body}" | grep -q '"database":true'; then + echo "PostgreSQL is not ready: ${ready_body}" >&2 + exit 1 +fi +if ! printf '%s' "${ready_body}" | grep -q '"redis":{"configured":true,"ready":true}'; then + echo "Redis is not configured and ready: ${ready_body}" >&2 + exit 1 +fi + +capture_dependencies() { + local suffix="$1" + psql -d "${database_name}" -P pager=off -c \ + "select datname, numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched from pg_stat_database where datname = current_database();" \ + >"${result_dir}/postgres-${suffix}.txt" + docker exec "${redis_container}" redis-cli INFO stats \ + >"${result_dir}/redis-${suffix}.txt" +} + +capture_dependencies before +{ + sw_vers + printf 'logical_cpu: ' + sysctl -n hw.logicalcpu + printf 'memory_bytes: ' + sysctl -n hw.memsize + dotnet --info +} >"${result_dir}/host.txt" + +set +e +if command -v k6 >/dev/null 2>&1; then + load_url="${host_url}" + k6 run \ + --summary-export "${result_dir}/summary.json" \ + -e "TIKU_BASE_URL=${load_url}" \ + -e "TIKU_HOST_HEADER=localhost:${api_port}" \ + -e "TIKU_TENANT_CODE=${tenant_code}" \ + -e "TIKU_MODE=${mode}" \ + -e "TIKU_RATE=${rate}" \ + -e "TIKU_DURATION=${duration}" \ + "${repo_root}/tools/performance/tiku-api.js" \ + | tee "${result_dir}/k6.txt" + k6_exit=${PIPESTATUS[0]} +else + load_url="http://host.docker.internal:${api_port}" + docker run --rm \ + -v "${repo_root}:/work:ro" \ + -v "${result_dir}:/results" \ + grafana/k6:latest \ + run \ + --summary-export /results/summary.json \ + -e "TIKU_BASE_URL=${load_url}" \ + -e "TIKU_HOST_HEADER=localhost:${api_port}" \ + -e "TIKU_TENANT_CODE=${tenant_code}" \ + -e "TIKU_MODE=${mode}" \ + -e "TIKU_RATE=${rate}" \ + -e "TIKU_DURATION=${duration}" \ + /work/tools/performance/tiku-api.js \ + | tee "${result_dir}/k6.txt" + k6_exit=${PIPESTATUS[0]} +fi +set -e + +cleanup +capture_dependencies after +printf 'mode=%s\nrate=%s\nduration=%s\ntenant=%s\nk6_exit=%s\n' \ + "${mode}" "${rate}" "${duration}" "${tenant_code}" "${k6_exit}" \ + >"${result_dir}/run.env" + +echo "Results: ${result_dir}" +exit "${k6_exit}" diff --git a/tools/performance/tiku-api.js b/tools/performance/tiku-api.js index ca9cf21..c5d1061 100644 --- a/tools/performance/tiku-api.js +++ b/tools/performance/tiku-api.js @@ -1,59 +1,125 @@ import http from 'k6/http'; -import { check, sleep } from 'k6'; +import { check, fail } from 'k6'; -const baseUrl = __ENV.TIKU_BASE_URL || 'http://localhost:5000'; -const tenantCode = __ENV.TIKU_TENANT_CODE || ''; -const accessToken = __ENV.TIKU_ACCESS_TOKEN || ''; +const baseUrl = (__ENV.TIKU_BASE_URL || 'http://localhost:5091').replace(/\/$/, ''); +const tenantCode = __ENV.TIKU_TENANT_CODE || 'demo-crm-school'; +const hostHeader = __ENV.TIKU_HOST_HEADER || ''; +const mode = (__ENV.TIKU_MODE || 'mixed').toLowerCase(); +const rate = positiveInteger('TIKU_RATE', 100); +const duration = __ENV.TIKU_DURATION || '30s'; +const preAllocatedVUs = positiveInteger('TIKU_PRE_ALLOCATED_VUS', Math.max(20, Math.ceil(rate / 20))); +const maxVUs = positiveInteger('TIKU_MAX_VUS', Math.max(100, preAllocatedVUs * 4)); + +if (!['mixed', 'hot', 'cold', 'ready'].includes(mode)) { + throw new Error(`Unsupported TIKU_MODE '${mode}'. Use mixed, hot, cold, or ready.`); +} export const options = { + discardResponseBodies: true, + summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'], scenarios: { - public_catalog: { - executor: 'constant-vus', - vus: Number(__ENV.TIKU_PUBLIC_VUS || 10), - duration: __ENV.TIKU_DURATION || '30s', - exec: 'publicCatalog', - }, - scoreline_page: { - executor: 'constant-vus', - vus: Number(__ENV.TIKU_SCORELINE_VUS || 10), - duration: __ENV.TIKU_DURATION || '30s', - exec: 'scorelinePage', - }, - tenant_bootstrap: { - executor: 'constant-vus', - vus: Number(__ENV.TIKU_BACKOFFICE_VUS || 5), - duration: __ENV.TIKU_DURATION || '30s', - exec: 'tenantBootstrap', - startTime: '1s', + api: { + executor: 'constant-arrival-rate', + exec: 'apiTraffic', + rate, + timeUnit: '1s', + duration, + preAllocatedVUs, + maxVUs, + gracefulStop: '5s', }, }, thresholds: { + checks: ['rate>0.99'], http_req_failed: ['rate<0.01'], - http_req_duration: ['p(95)<1000'], + http_req_duration: ['p(95)<500', 'p(99)<1000'], + dropped_iterations: ['count==0'], }, }; -function tenantParams(authenticated = false) { - const headers = tenantCode ? { 'x-tenant-code': tenantCode } : {}; - if (authenticated && accessToken) headers.Authorization = `Bearer ${accessToken}`; - return { headers }; +const tenantHeaders = { + 'x-tenant-code': tenantCode, + Accept: 'application/json', + ...(hostHeader ? { Host: hostHeader } : {}), +}; + +const platformHeaders = hostHeader ? { Host: hostHeader } : {}; + +export function setup() { + const ready = http.get(`${baseUrl}/api/health/ready`, { + headers: platformHeaders, + responseType: 'text', + tags: { endpoint: 'ready', phase: 'setup' }, + }); + if (ready.status !== 200) { + fail(`API dependencies are not ready: status=${ready.status} body=${ready.body}`); + } + + let readiness; + try { + readiness = ready.json(); + } catch (error) { + fail(`Readiness endpoint did not return JSON: ${error}`); + } + if (!readiness.database || !readiness.redis?.configured || !readiness.redis?.ready) { + fail(`PostgreSQL and Redis must both be configured and ready: ${JSON.stringify(readiness)}`); + } + + const catalog = http.get(`${baseUrl}/api/catalog/regions`, { + headers: tenantHeaders, + tags: { endpoint: 'catalog_hot', phase: 'setup' }, + }); + if (catalog.status !== 200) { + fail(`Tenant '${tenantCode}' is not usable: status=${catalog.status} body=${catalog.body}`); + } } -export function publicCatalog() { - const response = http.get(`${baseUrl}/api/catalog/regions`, tenantParams()); - check(response, { 'catalog is 200': (result) => result.status === 200 }); - sleep(0.1); +export function apiTraffic() { + const selected = selectEndpoint(); + let response; + + if (selected === 'ready') { + response = http.get(`${baseUrl}/api/health/ready`, { + headers: platformHeaders, + tags: { endpoint: 'ready' }, + }); + } else if (selected === 'cold') { + // A unique query string bypasses output-cache reuse while exercising the same + // tenant resolution and PostgreSQL-backed catalog query as the hot request. + const cacheBuster = `${__VU}-${__ITER}`; + response = http.get(`${baseUrl}/api/catalog/regions?loadProbe=${cacheBuster}`, { + headers: tenantHeaders, + tags: { endpoint: 'catalog_cold' }, + }); + } else { + response = http.get(`${baseUrl}/api/catalog/regions`, { + headers: tenantHeaders, + tags: { endpoint: 'catalog_hot' }, + }); + } + + check(response, { + [`${selected} returned 200`]: (result) => result.status === 200, + }); } -export function scorelinePage() { - const response = http.get(`${baseUrl}/api/scoreline/records?page=1&pageSize=20`, tenantParams()); - check(response, { 'scoreline is 200': (result) => result.status === 200 }); - sleep(0.1); +function selectEndpoint() { + if (mode !== 'mixed') return mode; + + // Stable 70/20/10 traffic mix: cached catalog / uncached catalog / dependency readiness. + const bucket = (__ITER + __VU) % 10; + if (bucket === 0) return 'ready'; + if (bucket <= 2) return 'cold'; + return 'hot'; } -export function tenantBootstrap() { - if (!accessToken) return; - const response = http.get(`${baseUrl}/api/tenant-backoffice/ui-bootstrap`, tenantParams(true)); - check(response, { 'bootstrap is 200': (result) => result.status === 200 }); - sleep(0.1); +function positiveInteger(name, fallback) { + const raw = __ENV[name]; + if (raw === undefined || raw === '') return fallback; + + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer, got '${raw}'.`); + } + return value; }