xiongyuxing/gongxue-base
1
0
Fork 0
forked from wangziqi/gongxue-base
Code Pull Requests Activity

chore: gate production postgres tuning and taro handoff

Browse Source
This commit is contained in:
Codex
2026-07-01 04:27:19 +08:00
parent 6bdb2a175a
commit 69b4d3b62d
14 changed files with 702 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

296
scripts/postgres-tuning-evidence.js
View File

@@ -1,30 +1,38 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import pg from 'pg';
import {
defaultPostgresTuningProfile,
getPostgresTuningProfile,
getPostgresTuningSettingNames,
} from './lib/postgres-tuning-profile.js';
const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
const outputDir = process.env.PG_TUNING_OUTPUT_DIR || 'docs/refactor/launch-artifacts';
const settingNames = [
'max_connections',
'shared_buffers',
'effective_cache_size',
'work_mem',
'maintenance_work_mem',
'autovacuum_work_mem',
'wal_buffers',
'min_wal_size',
'max_wal_size',
'checkpoint_timeout',
'checkpoint_completion_target',
'effective_io_concurrency',
'random_page_cost',
'jit',
'log_min_duration_statement',
'idle_in_transaction_session_timeout',
'statement_timeout',
'lock_timeout',
];
function parseArgs(argv) {
const options = {
profile: process.env.PG_TUNING_PROFILE || defaultPostgresTuningProfile,
strict: process.env.PG_TUNING_STRICT === 'true',
printSql: false,
json: false,
};
for (let index = 2; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--strict') options.strict = true;
else if (arg === '--json') options.json = true;
else if (arg === '--print-sql') options.printSql = true;
else if (arg === '--profile') {
options.profile = argv[index + 1] || options.profile;
index += 1;
} else if (arg.startsWith('--profile=')) {
options.profile = arg.slice('--profile='.length);
}
}
options.profileDefinition = getPostgresTuningProfile(options.profile);
options.settingNames = getPostgresTuningSettingNames(options.profile);
return options;
}
function shanghaiStamp(date = new Date()) {
const parts = Object.fromEntries(
@@ -59,19 +67,184 @@ function redactUrl(value) {
return value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:[redacted]@');
}
function pgUnitMultiplier(unit) {
const text = String(unit || '').trim().toLowerCase();
if (!text) return 1;
if (text === 'ms') return 1;
if (text === 's') return 1000;
if (text === 'min') return 60_000;
const memoryMatch = text.match(/^(\d+)?\s*(b|kb|mb|gb|tb)$/);
if (!memoryMatch) return 1;
const blockCount = Number(memoryMatch[1] || 1);
const unitName = memoryMatch[2];
const factors = {
b: 1,
kb: 1024,
mb: 1024 ** 2,
gb: 1024 ** 3,
tb: 1024 ** 4,
};
return blockCount * factors[unitName];
}
function formatBytes(bytes) {
if (!Number.isFinite(bytes)) return '';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let value = bytes;
let unitIndex = 0;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex += 1;
}
const rounded = value >= 10 || Number.isInteger(value) ? value.toFixed(0) : value.toFixed(1);
return `${rounded}${units[unitIndex]}`;
}
function formatDuration(ms) {
if (!Number.isFinite(ms)) return '';
if (ms === 0) return '0ms';
if (ms % 60_000 === 0) return `${ms / 60_000}min`;
if (ms % 1000 === 0) return `${ms / 1000}s`;
return `${ms}ms`;
}
function normalizedSettingValue(setting) {
const number = Number(setting.setting);
if (!Number.isFinite(number)) return `${setting.setting}${setting.unit || ''}`;
const unit = String(setting.unit || '').trim();
if (!unit) return String(setting.setting);
if (/^(?:\d+)?\s*(?:b|kb|mb|gb|tb)$/i.test(unit)) {
return formatBytes(number * pgUnitMultiplier(unit));
}
if (/^(?:ms|s|min)$/i.test(unit)) {
return formatDuration(number * pgUnitMultiplier(unit));
}
return `${setting.setting}${unit}`;
}
function explainSetting(setting) {
const value = `${setting.setting}${setting.unit || ''}`;
return {
name: setting.name,
value,
value: normalizedSettingValue(setting),
rawSetting: setting.setting,
unit: setting.unit || null,
source: setting.source,
pendingRestart: setting.pending_restart,
};
}
function evaluate(settingsRows) {
function parseSettingNumber(setting) {
if (!setting) return Number.NaN;
const number = Number(setting.setting);
return Number.isFinite(number) ? number : Number.NaN;
}
function parseBytes(value) {
if (typeof value === 'number') return value;
const text = String(value || '').trim().toLowerCase();
if (!text) return Number.NaN;
const match = text.match(/^(-?\d+(?:\.\d+)?)\s*([a-z]+)?$/);
if (!match) return Number.NaN;
const amount = Number(match[1]);
const unit = match[2] || 'b';
const factors = {
b: 1,
byte: 1,
bytes: 1,
kb: 1024,
mb: 1024 ** 2,
gb: 1024 ** 3,
tb: 1024 ** 4,
};
return amount * (factors[unit] || Number.NaN);
}
function parseDurationMs(value) {
if (typeof value === 'number') return value;
const text = String(value || '').trim().toLowerCase();
if (!text) return Number.NaN;
const match = text.match(/^(-?\d+(?:\.\d+)?)\s*([a-z]+)?$/);
if (!match) return Number.NaN;
const amount = Number(match[1]);
const unit = match[2] || 'ms';
const factors = {
ms: 1,
s: 1000,
sec: 1000,
secs: 1000,
second: 1000,
seconds: 1000,
min: 60_000,
mins: 60_000,
minute: 60_000,
minutes: 60_000,
h: 3_600_000,
hour: 3_600_000,
hours: 3_600_000,
};
return amount * (factors[unit] || Number.NaN);
}
function comparableValue(setting, rule) {
if (!setting) return Number.NaN;
if (rule.type === 'bytes') return Number(setting.setting) * pgUnitMultiplier(setting.unit);
if (rule.type === 'duration') return Number(setting.setting) * pgUnitMultiplier(setting.unit);
return parseSettingNumber(setting);
}
function recommendedComparator(rule, key) {
if (rule.type === 'bytes') return parseBytes(rule[key]);
if (rule.type === 'duration') return parseDurationMs(rule[key]);
return Number(rule[key]);
}
function evaluateProfile(settingsRows, profile) {
const byName = new Map(settingsRows.map(row => [row.name, row]));
const failures = [];
const warnings = [];
for (const [name, rule] of Object.entries(profile.settings)) {
const setting = byName.get(name);
if (!setting) {
failures.push(`${name} is missing from pg_settings evidence.`);
continue;
}
if (setting.pending_restart) {
failures.push(`${name} has pending_restart=true; restart PostgreSQL before capacity testing.`);
}
if (rule.exact !== undefined) {
const actual = String(setting.setting || '').toLowerCase();
if (actual !== String(rule.exact).toLowerCase()) {
failures.push(`${name} should be ${rule.exact} for ${profile.label}, got ${setting.setting}.`);
}
continue;
}
if (rule.min !== undefined) {
const actual = comparableValue(setting, rule);
const expected = recommendedComparator(rule, 'min');
if (!Number.isFinite(actual) || actual < expected) {
failures.push(`${name} is below ${rule.min} for ${profile.label}; current value is ${normalizedSettingValue(setting)}.`);
}
}
if (rule.max !== undefined) {
const actual = comparableValue(setting, rule);
const expected = recommendedComparator(rule, 'max');
if (!Number.isFinite(actual) || actual > expected) {
failures.push(`${name} is above ${rule.max} for ${profile.label}; current value is ${normalizedSettingValue(setting)}.`);
}
}
if (rule.recommended !== undefined && String(setting.source || '').toLowerCase() === 'default') {
warnings.push(`${name} still comes from default; recommended ${profile.name} value is ${rule.recommended}.`);
}
}
return { failures, warnings };
}
function evaluate(settingsRows, profile, options, pgStatStatements) {
const byName = new Map(settingsRows.map(row => [row.name, row]));
const warnings = [];
const failures = [];
const maxConnections = Number(byName.get('max_connections')?.setting || 0);
const jit = String(byName.get('jit')?.setting || '').toLowerCase();
const statementTimeout = String(byName.get('statement_timeout')?.setting || '');
@@ -84,9 +257,19 @@ function evaluate(settingsRows) {
if (idleTimeout === '0') warnings.push('idle_in_transaction_session_timeout is disabled; long idle transactions can block migrations and writes.');
if (lockTimeout === '0') warnings.push('lock_timeout is disabled; ordinary API requests may wait too long behind locks.');
if (settingsRows.some(row => row.pending_restart)) warnings.push('Some PostgreSQL settings have pending_restart=true; restart is required before capacity testing.');
if (pgStatStatements?.unavailable) warnings.push('pg_stat_statements is unavailable; enable it before production capacity acceptance.');
const profileEvaluation = evaluateProfile(settingsRows, profile);
warnings.push(...profileEvaluation.warnings);
if (options.strict) {
failures.push(...profileEvaluation.failures);
if (pgStatStatements?.unavailable) failures.push('pg_stat_statements is unavailable in strict mode.');
}
return {
status: warnings.length ? 'warn' : 'pass',
status: failures.length ? 'fail' : options.strict ? 'pass' : warnings.length ? 'warn' : 'pass',
profile: profile.name,
failures,
warnings,
};
}
@@ -99,9 +282,17 @@ function markdown(report) {
'',
`数据库：${report.databaseUrl}`,
'',
`调参 profile：${report.profile.label} (${report.profile.name})`,
'',
`评估：${report.evaluation.status}`,
'',
];
if (report.evaluation.failures.length) {
lines.push('## 阻断项');
lines.push('');
for (const failure of report.evaluation.failures) lines.push(`- ${failure}`);
lines.push('');
}
if (report.evaluation.warnings.length) {
lines.push('## 警告');
lines.push('');
@@ -158,7 +349,30 @@ function markdown(report) {
return `${lines.join('\n')}\n`;
}
function sqlQuote(value) {
return String(value).replaceAll("'", "''");
}
function alterSystemSql(profile) {
const lines = [
`-- PostgreSQL tuning profile: ${profile.label}`,
'-- Review current values and take a snapshot before applying this file.',
'-- Run npm run perf:postgres:evidence -- --strict after reload/restart.',
'',
];
for (const [name, rule] of Object.entries(profile.settings)) {
lines.push(`alter system set ${name} = '${sqlQuote(rule.recommended)}';`);
}
lines.push('select pg_reload_conf();');
return `${lines.join('\n')}\n`;
}
async function main() {
const options = parseArgs(process.argv);
if (options.printSql) {
console.log(alterSystemSql(options.profileDefinition));
return;
}
const pool = new pg.Pool({ connectionString: databaseUrl, max: 2 });
try {
const settingsRaw = await query(
@@ -169,7 +383,7 @@ async function main() {
where name = any($1::text[])
order by array_position($1::text[], name)
`,
[settingNames],
[options.settingNames],
);
const activityByWait = await query(
pool,
@@ -231,7 +445,12 @@ async function main() {
const report = {
generatedAt: new Date().toISOString(),
databaseUrl: redactUrl(databaseUrl),
evaluation: evaluate(settingsRaw),
profile: {
name: options.profileDefinition.name,
label: options.profileDefinition.label,
description: options.profileDefinition.description,
},
evaluation: evaluate(settingsRaw, options.profileDefinition, options, pgStatStatements),
settings: settingsRaw.map(explainSetting),
activityByWait,
databaseStats,
@@ -239,15 +458,32 @@ async function main() {
largeRelations,
pgStatStatements,
};
const pendingRestart = settingsRaw.filter(row => row.pending_restart).length;
await fs.mkdir(outputDir, { recursive: true });
const jsonPath = path.join(outputDir, `postgres-tuning-evidence-${shanghaiStamp()}.json`);
const mdPath = jsonPath.replace(/\.json$/, '.md');
await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
await fs.writeFile(mdPath, markdown(report), 'utf8');
console.log(`[pg-evidence] status=${report.evaluation.status}`);
for (const warning of report.evaluation.warnings) console.log(`[pg-evidence] warning: ${warning}`);
console.log(`[pg-evidence] wrote ${jsonPath}`);
console.log(`[pg-evidence] wrote ${mdPath}`);
const payload = {
status: report.evaluation.status,
profile: report.profile.name,
warnings: report.evaluation.warnings.length,
failures: report.evaluation.failures.length,
pgStatStatementsAvailable: !pgStatStatements?.unavailable,
pendingRestart,
jsonPath,
mdPath,
};
if (options.json) {
console.log(JSON.stringify(payload, null, 2));
} else {
console.log(`[pg-evidence] status=${report.evaluation.status}`);
for (const failure of report.evaluation.failures) console.log(`[pg-evidence] failure: ${failure}`);
for (const warning of report.evaluation.warnings) console.log(`[pg-evidence] warning: ${warning}`);
console.log(`[pg-evidence] wrote ${jsonPath}`);
console.log(`[pg-evidence] wrote ${mdPath}`);
}
if (options.strict && report.evaluation.failures.length > 0) process.exitCode = 1;
} finally {
await pool.end();
}