feat: alert usage overage worker failures

This commit is contained in:
Codex
2026-06-30 21:43:49 +08:00
parent ac5d136b35
commit 90ad9e90a3
13 changed files with 262 additions and 24 deletions

View File

@@ -66,7 +66,8 @@ async function runOnce() {
const result = await processPlatformUsageOverageBatch();
console.log(
`[worker] platform-usage-overage batch processed=${result.processed}`
+ ` created=${result.created} skipped=${result.skipped} totalCents=${result.totalCents}`,
+ ` created=${result.created} skipped=${result.skipped}`
+ ` failed=${result.failed} totalCents=${result.totalCents}`,
);
return;
}

View File

@@ -7,6 +7,7 @@ export interface PlatformUsageOverageWorkerResult {
created: number;
skipped: number;
totalCents: number;
failed: number;
}
const MONTH_RE = /^\d{4}-\d{2}$/;
@@ -49,6 +50,9 @@ function monthPeriod(monthText: string) {
const [yearText, monthNumberText] = normalized.split('-');
const year = Number(yearText);
const monthNumber = Number(monthNumberText);
if (monthNumber < 1 || monthNumber > 12) {
throw new Error(`Invalid platform usage overage month: ${monthText}. Month must be between 01 and 12.`);
}
const periodStart = `${yearText}-${monthNumberText}-01`;
const periodEnd = new Date(Date.UTC(year, monthNumber, 0)).toISOString().slice(0, 10);
return { periodStart, periodEnd };
@@ -60,6 +64,86 @@ function dueDateText(days: number) {
return now.toISOString().slice(0, 10);
}
function truncate(value: unknown, max = 900) {
return String(value ?? '').slice(0, max);
}
function redactSensitiveValue(value: unknown, parentKey = '', depth = 0): unknown {
if (value === null || value === undefined) return value;
if (depth > 8) return '[REDACTED_DEPTH_LIMIT]';
if (
/(?:password|passwd|secret|token|credential|private[_-]?key|api[_-]?key|app[_-]?secret|authorization|cookie|session|cert|signature|database[_-]?url|connection[_-]?string)$/i
.test(parentKey)
) {
return '[REDACTED]';
}
if (Array.isArray(value)) return value.map(item => redactSensitiveValue(item, parentKey, depth + 1));
if (typeof value === 'object') {
const output: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
output[key] = redactSensitiveValue(item, key, depth + 1);
}
return output;
}
if (typeof value === 'string') {
return value
.replace(/\bpostgres(?:ql)?:\/\/[^\s"'<>]+/gi, '[REDACTED_DATABASE_URL]')
.replace(/\b(?:bearer|basic)\s+[a-z0-9._~+/-]+=*/gi, '[REDACTED_AUTH_HEADER]')
.replace(/\b((?:access[_-]?token|refresh[_-]?token|api[_-]?key|secret|password))=([^&\s]+)/gi, '$1=[REDACTED]');
}
return value;
}
function errorCode(error: unknown) {
if (typeof error === 'object' && error !== null && 'code' in error) {
return truncate((error as { code?: unknown }).code, 120);
}
return 'PLATFORM_USAGE_OVERAGE_WORKER_FAILED';
}
function errorDetails(error: unknown) {
const details: Record<string, unknown> = {
code: errorCode(error),
message: truncate(error instanceof Error ? error.message : String(error)),
name: error instanceof Error ? error.name : typeof error,
};
if (typeof error === 'object' && error !== null) {
const record = error as Record<string, unknown>;
for (const key of ['detail', 'hint', 'where', 'schema', 'table', 'constraint']) {
if (record[key] !== undefined) details[key] = truncate(record[key], 300);
}
}
return redactSensitiveValue(details) as Record<string, unknown>;
}
async function recordUsageOverageWorkerFailure(input: {
month: string;
periodStart?: string | null;
periodEnd?: string | null;
dueDate?: string | null;
limit?: number | null;
error: unknown;
}) {
await pool.query(
`
insert into public.audit_logs (tenant_id, actor_user_id, action, target_type, target_id, details)
values (null, null, 'platform.invoice.usage_overage_worker_failed', 'tenant_invoice_batch', $1, $2::jsonb)
`,
[
input.periodStart && input.periodEnd ? `${input.periodStart}_${input.periodEnd}` : input.month,
JSON.stringify({
month: input.month,
periodStart: input.periodStart || null,
periodEnd: input.periodEnd || null,
dueDate: input.dueDate || null,
limit: input.limit || null,
workerId: config.platformUsageOverageWorkerId,
error: errorDetails(input.error),
}),
],
);
}
export async function processPlatformUsageOverageBatch(options: {
limit?: number;
month?: string;
@@ -67,14 +151,20 @@ export async function processPlatformUsageOverageBatch(options: {
} = {}): Promise<PlatformUsageOverageWorkerResult> {
const limit = positiveInteger(options.limit ?? config.platformUsageOverageBatchSize, 100, 500);
const month = targetMonth(options.month || config.platformUsageOverageMonth);
const { periodStart, periodEnd } = monthPeriod(month);
let periodStart: string | null = null;
let periodEnd: string | null = null;
let dueDate: string | null = null;
const client = await pool.connect();
try {
const period = monthPeriod(month);
periodStart = period.periodStart;
periodEnd = period.periodEnd;
dueDate = dueDateText(options.dueDays ?? config.platformUsageOverageDueDays);
await client.query('begin');
const result = await processUsageOverageInvoices(client, {
periodStart,
periodEnd,
dueDate: dueDateText(options.dueDays ?? config.platformUsageOverageDueDays),
dueDate,
status: 'issued',
note: `平台自动生成 ${periodStart}${periodEnd} 用量超额服务费账单`,
dryRun: false,
@@ -87,9 +177,18 @@ export async function processPlatformUsageOverageBatch(options: {
created: Number(result.createdCount || 0),
skipped: Number(result.skippedCount || 0),
totalCents: Number(result.totalCents || 0),
failed: 0,
};
} catch (error) {
await client.query('rollback').catch(() => {});
await recordUsageOverageWorkerFailure({
month,
periodStart,
periodEnd,
dueDate,
limit,
error,
}).catch(() => {});
throw error;
} finally {
client.release();