forked from wangziqi/gongxue-base
499 lines
17 KiB
JavaScript
499 lines
17 KiB
JavaScript
import fs from 'node:fs/promises';
|
||
import path from 'node:path';
|
||
import pg from 'pg';
|
||
import {
|
||
defaultPostgresTuningProfile,
|
||
getPostgresTuningProfile,
|
||
getPostgresTuningSettingNames,
|
||
} from './lib/postgres-tuning-profile.js';
|
||
|
||
const databaseUrl = process.env.DATABASE_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
|
||
const outputDir = process.env.PG_TUNING_OUTPUT_DIR || 'docs/refactor/launch-artifacts';
|
||
|
||
function parseArgs(argv) {
|
||
const options = {
|
||
profile: process.env.PG_TUNING_PROFILE || defaultPostgresTuningProfile,
|
||
strict: process.env.PG_TUNING_STRICT === 'true',
|
||
printSql: false,
|
||
json: false,
|
||
};
|
||
for (let index = 2; index < argv.length; index += 1) {
|
||
const arg = argv[index];
|
||
if (arg === '--strict') options.strict = true;
|
||
else if (arg === '--json') options.json = true;
|
||
else if (arg === '--print-sql') options.printSql = true;
|
||
else if (arg === '--profile') {
|
||
options.profile = argv[index + 1] || options.profile;
|
||
index += 1;
|
||
} else if (arg.startsWith('--profile=')) {
|
||
options.profile = arg.slice('--profile='.length);
|
||
}
|
||
}
|
||
options.profileDefinition = getPostgresTuningProfile(options.profile);
|
||
options.settingNames = getPostgresTuningSettingNames(options.profile);
|
||
return options;
|
||
}
|
||
|
||
function shanghaiStamp(date = new Date()) {
|
||
const parts = Object.fromEntries(
|
||
new Intl.DateTimeFormat('en-CA', {
|
||
timeZone: 'Asia/Shanghai',
|
||
year: 'numeric',
|
||
month: '2-digit',
|
||
day: '2-digit',
|
||
hour: '2-digit',
|
||
minute: '2-digit',
|
||
second: '2-digit',
|
||
hour12: false,
|
||
}).formatToParts(date).map(part => [part.type, part.value]),
|
||
);
|
||
return `${parts.year}${parts.month}${parts.day}-${parts.hour}${parts.minute}${parts.second}`;
|
||
}
|
||
|
||
async function query(pool, sql, params = []) {
|
||
const result = await pool.query(sql, params);
|
||
return result.rows;
|
||
}
|
||
|
||
async function maybeQuery(pool, sql, params = []) {
|
||
try {
|
||
return await query(pool, sql, params);
|
||
} catch (error) {
|
||
return { unavailable: true, message: error?.message || String(error) };
|
||
}
|
||
}
|
||
|
||
function redactUrl(value) {
|
||
return value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:[redacted]@');
|
||
}
|
||
|
||
function pgUnitMultiplier(unit) {
|
||
const text = String(unit || '').trim().toLowerCase();
|
||
if (!text) return 1;
|
||
if (text === 'ms') return 1;
|
||
if (text === 's') return 1000;
|
||
if (text === 'min') return 60_000;
|
||
const memoryMatch = text.match(/^(\d+)?\s*(b|kb|mb|gb|tb)$/);
|
||
if (!memoryMatch) return 1;
|
||
const blockCount = Number(memoryMatch[1] || 1);
|
||
const unitName = memoryMatch[2];
|
||
const factors = {
|
||
b: 1,
|
||
kb: 1024,
|
||
mb: 1024 ** 2,
|
||
gb: 1024 ** 3,
|
||
tb: 1024 ** 4,
|
||
};
|
||
return blockCount * factors[unitName];
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (!Number.isFinite(bytes)) return '';
|
||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||
let value = bytes;
|
||
let unitIndex = 0;
|
||
while (value >= 1024 && unitIndex < units.length - 1) {
|
||
value /= 1024;
|
||
unitIndex += 1;
|
||
}
|
||
const rounded = value >= 10 || Number.isInteger(value) ? value.toFixed(0) : value.toFixed(1);
|
||
return `${rounded}${units[unitIndex]}`;
|
||
}
|
||
|
||
function formatDuration(ms) {
|
||
if (!Number.isFinite(ms)) return '';
|
||
if (ms === 0) return '0ms';
|
||
if (ms % 60_000 === 0) return `${ms / 60_000}min`;
|
||
if (ms % 1000 === 0) return `${ms / 1000}s`;
|
||
return `${ms}ms`;
|
||
}
|
||
|
||
function normalizedSettingValue(setting) {
|
||
const number = Number(setting.setting);
|
||
if (!Number.isFinite(number)) return `${setting.setting}${setting.unit || ''}`;
|
||
const unit = String(setting.unit || '').trim();
|
||
if (!unit) return String(setting.setting);
|
||
if (/^(?:\d+)?\s*(?:b|kb|mb|gb|tb)$/i.test(unit)) {
|
||
return formatBytes(number * pgUnitMultiplier(unit));
|
||
}
|
||
if (/^(?:ms|s|min)$/i.test(unit)) {
|
||
return formatDuration(number * pgUnitMultiplier(unit));
|
||
}
|
||
return `${setting.setting}${unit}`;
|
||
}
|
||
|
||
function explainSetting(setting) {
|
||
return {
|
||
name: setting.name,
|
||
value: normalizedSettingValue(setting),
|
||
rawSetting: setting.setting,
|
||
unit: setting.unit || null,
|
||
source: setting.source,
|
||
pendingRestart: setting.pending_restart,
|
||
};
|
||
}
|
||
|
||
function parseSettingNumber(setting) {
|
||
if (!setting) return Number.NaN;
|
||
const number = Number(setting.setting);
|
||
return Number.isFinite(number) ? number : Number.NaN;
|
||
}
|
||
|
||
function parseBytes(value) {
|
||
if (typeof value === 'number') return value;
|
||
const text = String(value || '').trim().toLowerCase();
|
||
if (!text) return Number.NaN;
|
||
const match = text.match(/^(-?\d+(?:\.\d+)?)\s*([a-z]+)?$/);
|
||
if (!match) return Number.NaN;
|
||
const amount = Number(match[1]);
|
||
const unit = match[2] || 'b';
|
||
const factors = {
|
||
b: 1,
|
||
byte: 1,
|
||
bytes: 1,
|
||
kb: 1024,
|
||
mb: 1024 ** 2,
|
||
gb: 1024 ** 3,
|
||
tb: 1024 ** 4,
|
||
};
|
||
return amount * (factors[unit] || Number.NaN);
|
||
}
|
||
|
||
function parseDurationMs(value) {
|
||
if (typeof value === 'number') return value;
|
||
const text = String(value || '').trim().toLowerCase();
|
||
if (!text) return Number.NaN;
|
||
const match = text.match(/^(-?\d+(?:\.\d+)?)\s*([a-z]+)?$/);
|
||
if (!match) return Number.NaN;
|
||
const amount = Number(match[1]);
|
||
const unit = match[2] || 'ms';
|
||
const factors = {
|
||
ms: 1,
|
||
s: 1000,
|
||
sec: 1000,
|
||
secs: 1000,
|
||
second: 1000,
|
||
seconds: 1000,
|
||
min: 60_000,
|
||
mins: 60_000,
|
||
minute: 60_000,
|
||
minutes: 60_000,
|
||
h: 3_600_000,
|
||
hour: 3_600_000,
|
||
hours: 3_600_000,
|
||
};
|
||
return amount * (factors[unit] || Number.NaN);
|
||
}
|
||
|
||
function comparableValue(setting, rule) {
|
||
if (!setting) return Number.NaN;
|
||
if (rule.type === 'bytes') return Number(setting.setting) * pgUnitMultiplier(setting.unit);
|
||
if (rule.type === 'duration') return Number(setting.setting) * pgUnitMultiplier(setting.unit);
|
||
return parseSettingNumber(setting);
|
||
}
|
||
|
||
function recommendedComparator(rule, key) {
|
||
if (rule.type === 'bytes') return parseBytes(rule[key]);
|
||
if (rule.type === 'duration') return parseDurationMs(rule[key]);
|
||
return Number(rule[key]);
|
||
}
|
||
|
||
function evaluateProfile(settingsRows, profile) {
|
||
const byName = new Map(settingsRows.map(row => [row.name, row]));
|
||
const failures = [];
|
||
const warnings = [];
|
||
|
||
for (const [name, rule] of Object.entries(profile.settings)) {
|
||
const setting = byName.get(name);
|
||
if (!setting) {
|
||
failures.push(`${name} is missing from pg_settings evidence.`);
|
||
continue;
|
||
}
|
||
if (setting.pending_restart) {
|
||
failures.push(`${name} has pending_restart=true; restart PostgreSQL before capacity testing.`);
|
||
}
|
||
if (rule.exact !== undefined) {
|
||
const actual = String(setting.setting || '').toLowerCase();
|
||
if (actual !== String(rule.exact).toLowerCase()) {
|
||
failures.push(`${name} should be ${rule.exact} for ${profile.label}, got ${setting.setting}.`);
|
||
}
|
||
if (String(setting.source || '').toLowerCase() === 'default') {
|
||
warnings.push(`${name} still comes from default; recommended ${profile.name} value is ${rule.recommended}.`);
|
||
}
|
||
continue;
|
||
}
|
||
if (rule.min !== undefined) {
|
||
const actual = comparableValue(setting, rule);
|
||
const expected = recommendedComparator(rule, 'min');
|
||
if (!Number.isFinite(actual) || actual < expected) {
|
||
failures.push(`${name} is below ${rule.min} for ${profile.label}; current value is ${normalizedSettingValue(setting)}.`);
|
||
}
|
||
}
|
||
if (rule.max !== undefined) {
|
||
const actual = comparableValue(setting, rule);
|
||
const expected = recommendedComparator(rule, 'max');
|
||
if (!Number.isFinite(actual) || actual > expected) {
|
||
failures.push(`${name} is above ${rule.max} for ${profile.label}; current value is ${normalizedSettingValue(setting)}.`);
|
||
}
|
||
}
|
||
if (rule.recommended !== undefined && String(setting.source || '').toLowerCase() === 'default') {
|
||
warnings.push(`${name} still comes from default; recommended ${profile.name} value is ${rule.recommended}.`);
|
||
}
|
||
}
|
||
|
||
return { failures, warnings };
|
||
}
|
||
|
||
function evaluate(settingsRows, profile, options, pgStatStatements) {
|
||
const byName = new Map(settingsRows.map(row => [row.name, row]));
|
||
const warnings = [];
|
||
const failures = [];
|
||
const maxConnections = Number(byName.get('max_connections')?.setting || 0);
|
||
const jit = String(byName.get('jit')?.setting || '').toLowerCase();
|
||
const statementTimeout = String(byName.get('statement_timeout')?.setting || '');
|
||
const idleTimeout = String(byName.get('idle_in_transaction_session_timeout')?.setting || '');
|
||
const lockTimeout = String(byName.get('lock_timeout')?.setting || '');
|
||
|
||
if (maxConnections > 150) warnings.push('max_connections is high for a 4 vCPU database; prefer API/pooler limits over direct connections.');
|
||
if (jit === 'on') warnings.push('jit is on; short OLTP-style question-bank API queries usually start safer with jit=off.');
|
||
if (statementTimeout === '0') warnings.push('statement_timeout is disabled; production API should have a bounded global timeout and import jobs should override per session.');
|
||
if (idleTimeout === '0') warnings.push('idle_in_transaction_session_timeout is disabled; long idle transactions can block migrations and writes.');
|
||
if (lockTimeout === '0') warnings.push('lock_timeout is disabled; ordinary API requests may wait too long behind locks.');
|
||
if (settingsRows.some(row => row.pending_restart)) warnings.push('Some PostgreSQL settings have pending_restart=true; restart is required before capacity testing.');
|
||
if (pgStatStatements?.unavailable) warnings.push('pg_stat_statements is unavailable; enable it before production capacity acceptance.');
|
||
|
||
const profileEvaluation = evaluateProfile(settingsRows, profile);
|
||
warnings.push(...profileEvaluation.warnings);
|
||
if (options.strict) {
|
||
failures.push(...profileEvaluation.failures);
|
||
if (pgStatStatements?.unavailable) failures.push('pg_stat_statements is unavailable in strict mode.');
|
||
}
|
||
|
||
return {
|
||
status: failures.length ? 'fail' : options.strict ? 'pass' : warnings.length ? 'warn' : 'pass',
|
||
profile: profile.name,
|
||
failures,
|
||
warnings,
|
||
};
|
||
}
|
||
|
||
function markdown(report) {
|
||
const lines = [
|
||
'# PostgreSQL 调参与运行证据',
|
||
'',
|
||
`生成时间：${new Date(report.generatedAt).toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' })}`,
|
||
'',
|
||
`数据库：${report.databaseUrl}`,
|
||
'',
|
||
`调参 profile：${report.profile.label} (${report.profile.name})`,
|
||
'',
|
||
`评估：${report.evaluation.status}`,
|
||
'',
|
||
];
|
||
if (report.evaluation.failures.length) {
|
||
lines.push('## 阻断项');
|
||
lines.push('');
|
||
for (const failure of report.evaluation.failures) lines.push(`- ${failure}`);
|
||
lines.push('');
|
||
}
|
||
if (report.evaluation.warnings.length) {
|
||
lines.push('## 警告');
|
||
lines.push('');
|
||
for (const warning of report.evaluation.warnings) lines.push(`- ${warning}`);
|
||
lines.push('');
|
||
}
|
||
lines.push('## 关键参数');
|
||
lines.push('');
|
||
lines.push('| 参数 | 当前值 | 来源 | 待重启 |');
|
||
lines.push('| --- | ---: | --- | --- |');
|
||
for (const item of report.settings) {
|
||
lines.push(`| ${item.name} | ${item.value} | ${item.source} | ${item.pendingRestart ? '是' : '否'} |`);
|
||
}
|
||
lines.push('');
|
||
lines.push('## 连接与等待');
|
||
lines.push('');
|
||
lines.push('| state | wait_event_type | wait_event | count |');
|
||
lines.push('| --- | --- | --- | ---: |');
|
||
for (const item of report.activityByWait) {
|
||
lines.push(`| ${item.state || '-'} | ${item.wait_event_type || '-'} | ${item.wait_event || '-'} | ${item.count} |`);
|
||
}
|
||
lines.push('');
|
||
lines.push('## 缓存与事务');
|
||
lines.push('');
|
||
lines.push('| datname | commits | rollbacks | cache_hit_ratio |');
|
||
lines.push('| --- | ---: | ---: | ---: |');
|
||
for (const item of report.databaseStats) {
|
||
lines.push(`| ${item.datname} | ${item.xact_commit} | ${item.xact_rollback} | ${item.cache_hit_ratio ?? '-'} |`);
|
||
}
|
||
lines.push('');
|
||
lines.push('## 大表规模');
|
||
lines.push('');
|
||
lines.push('| 表 | 估算行数 | 总大小 | 索引大小 |');
|
||
lines.push('| --- | ---: | ---: | ---: |');
|
||
for (const item of report.largeRelations) {
|
||
lines.push(`| ${item.relation} | ${item.estimated_rows} | ${item.total_size} | ${item.index_size} |`);
|
||
}
|
||
lines.push('');
|
||
if (Array.isArray(report.pgStatStatements)) {
|
||
lines.push('## pg_stat_statements Top SQL');
|
||
lines.push('');
|
||
lines.push('| calls | total_exec_ms | mean_exec_ms | rows | query |');
|
||
lines.push('| ---: | ---: | ---: | ---: | --- |');
|
||
for (const item of report.pgStatStatements) {
|
||
lines.push(`| ${item.calls} | ${item.total_exec_ms} | ${item.mean_exec_ms} | ${item.rows} | ${String(item.query || '').replaceAll('|', '\\|')} |`);
|
||
}
|
||
lines.push('');
|
||
} else {
|
||
lines.push('## pg_stat_statements');
|
||
lines.push('');
|
||
lines.push(`未采集：${report.pgStatStatements?.message || 'extension/view unavailable'}`);
|
||
lines.push('');
|
||
}
|
||
return `${lines.join('\n')}\n`;
|
||
}
|
||
|
||
function sqlQuote(value) {
|
||
return String(value).replaceAll("'", "''");
|
||
}
|
||
|
||
function alterSystemSql(profile) {
|
||
const lines = [
|
||
`-- PostgreSQL tuning profile: ${profile.label}`,
|
||
'-- Review current values and take a snapshot before applying this file.',
|
||
'-- Run npm run perf:postgres:evidence -- --strict after reload/restart.',
|
||
'',
|
||
];
|
||
for (const [name, rule] of Object.entries(profile.settings)) {
|
||
lines.push(`alter system set ${name} = '${sqlQuote(rule.recommended)}';`);
|
||
}
|
||
lines.push('select pg_reload_conf();');
|
||
return `${lines.join('\n')}\n`;
|
||
}
|
||
|
||
async function main() {
|
||
const options = parseArgs(process.argv);
|
||
if (options.printSql) {
|
||
console.log(alterSystemSql(options.profileDefinition));
|
||
return;
|
||
}
|
||
const pool = new pg.Pool({ connectionString: databaseUrl, max: 2 });
|
||
try {
|
||
const settingsRaw = await query(
|
||
pool,
|
||
`
|
||
select name, setting, unit, source, pending_restart
|
||
from pg_settings
|
||
where name = any($1::text[])
|
||
order by array_position($1::text[], name)
|
||
`,
|
||
[options.settingNames],
|
||
);
|
||
const activityByWait = await query(
|
||
pool,
|
||
`
|
||
select state, wait_event_type, wait_event, count(*)::int as count
|
||
from pg_stat_activity
|
||
where datname = current_database()
|
||
group by state, wait_event_type, wait_event
|
||
order by count desc, state nulls last
|
||
`,
|
||
);
|
||
const databaseStats = await query(
|
||
pool,
|
||
`
|
||
select datname,
|
||
xact_commit::text,
|
||
xact_rollback::text,
|
||
blks_read::text,
|
||
blks_hit::text,
|
||
round(blks_hit * 100.0 / nullif(blks_hit + blks_read, 0), 2)::text as cache_hit_ratio
|
||
from pg_stat_database
|
||
where datname = current_database()
|
||
`,
|
||
);
|
||
const bgwriter = await maybeQuery(
|
||
pool,
|
||
`
|
||
select checkpoints_timed::text, checkpoints_req::text,
|
||
checkpoint_write_time::text, checkpoint_sync_time::text
|
||
from pg_stat_bgwriter
|
||
`,
|
||
);
|
||
const largeRelations = await query(
|
||
pool,
|
||
`
|
||
select relid::regclass::text as relation,
|
||
n_live_tup::bigint::text as estimated_rows,
|
||
pg_size_pretty(pg_total_relation_size(relid)) as total_size,
|
||
pg_size_pretty(pg_indexes_size(relid)) as index_size
|
||
from pg_stat_user_tables
|
||
where schemaname = 'public'
|
||
order by pg_total_relation_size(relid) desc
|
||
limit 20
|
||
`,
|
||
);
|
||
const pgStatStatements = await maybeQuery(
|
||
pool,
|
||
`
|
||
select calls::bigint::text,
|
||
round(total_exec_time::numeric, 2)::text as total_exec_ms,
|
||
round(mean_exec_time::numeric, 2)::text as mean_exec_ms,
|
||
rows::bigint::text,
|
||
left(regexp_replace(query, '\\s+', ' ', 'g'), 180) as query
|
||
from pg_stat_statements
|
||
order by total_exec_time desc
|
||
limit 10
|
||
`,
|
||
);
|
||
const report = {
|
||
generatedAt: new Date().toISOString(),
|
||
databaseUrl: redactUrl(databaseUrl),
|
||
profile: {
|
||
name: options.profileDefinition.name,
|
||
label: options.profileDefinition.label,
|
||
description: options.profileDefinition.description,
|
||
},
|
||
evaluation: evaluate(settingsRaw, options.profileDefinition, options, pgStatStatements),
|
||
settings: settingsRaw.map(explainSetting),
|
||
activityByWait,
|
||
databaseStats,
|
||
bgwriter,
|
||
largeRelations,
|
||
pgStatStatements,
|
||
};
|
||
const pendingRestart = settingsRaw.filter(row => row.pending_restart).length;
|
||
await fs.mkdir(outputDir, { recursive: true });
|
||
const jsonPath = path.join(outputDir, `postgres-tuning-evidence-${shanghaiStamp()}.json`);
|
||
const mdPath = jsonPath.replace(/\.json$/, '.md');
|
||
await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||
await fs.writeFile(mdPath, markdown(report), 'utf8');
|
||
const payload = {
|
||
status: report.evaluation.status,
|
||
profile: report.profile.name,
|
||
warnings: report.evaluation.warnings.length,
|
||
failures: report.evaluation.failures.length,
|
||
pgStatStatementsAvailable: !pgStatStatements?.unavailable,
|
||
pendingRestart,
|
||
jsonPath,
|
||
mdPath,
|
||
};
|
||
if (options.json) {
|
||
console.log(JSON.stringify(payload, null, 2));
|
||
} else {
|
||
console.log(`[pg-evidence] status=${report.evaluation.status}`);
|
||
for (const failure of report.evaluation.failures) console.log(`[pg-evidence] failure: ${failure}`);
|
||
for (const warning of report.evaluation.warnings) console.log(`[pg-evidence] warning: ${warning}`);
|
||
console.log(`[pg-evidence] wrote ${jsonPath}`);
|
||
console.log(`[pg-evidence] wrote ${mdPath}`);
|
||
}
|
||
if (options.strict && report.evaluation.failures.length > 0) process.exitCode = 1;
|
||
} finally {
|
||
await pool.end();
|
||
}
|
||
}
|
||
|
||
main().catch(error => {
|
||
console.error(error);
|
||
process.exit(1);
|
||
});
|