249 lines
8.3 KiB
TypeScript
249 lines
8.3 KiB
TypeScript
/**
|
|
* Empirical Benchmark & Profiling Harness for Multi-Worker Act Asset Baking
|
|
*
|
|
* Measures:
|
|
* 1. Wall-clock duration & Speedup (MAX_WORKERS=1, 4, 8, 16)
|
|
* 2. Multi-core CPU utilization (summed across all threads of all descendant processes)
|
|
* 3. Memory stability & Peak RSS (sampled across entire process tree every 100ms)
|
|
* 4. Deterministic output parity (comparing index.json between sequential and parallel)
|
|
* 5. Clean process/thread termination with zero orphaned processes
|
|
*/
|
|
|
|
import { spawn } from 'node:child_process'
|
|
import { readdirSync, readFileSync, rmSync, existsSync, writeFileSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import { performance } from 'node:perf_hooks'
|
|
|
|
export interface BenchResult {
|
|
workers: number
|
|
wallTimeSec: number
|
|
speedup: number
|
|
peakRssMb: number
|
|
avgCpuPercent: number
|
|
peakCpuPercent: number
|
|
peakThreads: number
|
|
exitCode: number
|
|
scenesCount: number
|
|
outDir: string
|
|
}
|
|
|
|
function getDescendantPids(rootPid: number): number[] {
|
|
const pids = [rootPid]
|
|
try {
|
|
const allDirs = readdirSync('/proc').filter(d => /^\d+$/.test(d))
|
|
for (const d of allDirs) {
|
|
try {
|
|
const stat = readFileSync(`/proc/${d}/stat`, 'utf8')
|
|
const closeParen = stat.lastIndexOf(')')
|
|
if (closeParen !== -1) {
|
|
const rest = stat.slice(closeParen + 2).split(' ')
|
|
const ppid = parseInt(rest[1] || '0', 10)
|
|
const pid = parseInt(d, 10)
|
|
if (pids.includes(ppid) && !pids.includes(pid)) {
|
|
pids.push(pid)
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
} catch {}
|
|
return pids
|
|
}
|
|
|
|
function getProcessTreeMetrics(rootPid: number): { rssMb: number; totalCpuTicks: number; threadCount: number } {
|
|
const pids = getDescendantPids(rootPid)
|
|
let totalRssKb = 0
|
|
let totalThreads = 0
|
|
let totalCpuTicks = 0
|
|
|
|
for (const pid of pids) {
|
|
try {
|
|
const statusContent = readFileSync(`/proc/${pid}/status`, 'utf8')
|
|
for (const line of statusContent.split('\n')) {
|
|
if (line.startsWith('VmRSS:')) {
|
|
const parts = line.split(/\s+/)
|
|
totalRssKb += parseInt(parts[1] || '0', 10)
|
|
} else if (line.startsWith('Threads:')) {
|
|
const parts = line.split(/\s+/)
|
|
totalThreads += parseInt(parts[1] || '0', 10)
|
|
}
|
|
}
|
|
|
|
const taskDir = `/proc/${pid}/task`
|
|
if (existsSync(taskDir)) {
|
|
const tids = readdirSync(taskDir)
|
|
for (const tid of tids) {
|
|
try {
|
|
const statContent = readFileSync(`/proc/${pid}/task/${tid}/stat`, 'utf8')
|
|
const closeParen = statContent.lastIndexOf(')')
|
|
if (closeParen !== -1) {
|
|
const rest = statContent.slice(closeParen + 2).split(' ')
|
|
const utime = parseInt(rest[11] || '0', 10)
|
|
const stime = parseInt(rest[12] || '0', 10)
|
|
totalCpuTicks += utime + stime
|
|
}
|
|
} catch {}
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
return {
|
|
rssMb: totalRssKb / 1024,
|
|
totalCpuTicks,
|
|
threadCount: totalThreads,
|
|
}
|
|
}
|
|
|
|
export async function runBench(
|
|
workers: number,
|
|
filterLevel: string,
|
|
outDir: string,
|
|
archiveDir: string = 'samples/d2',
|
|
): Promise<BenchResult> {
|
|
if (existsSync(outDir)) {
|
|
rmSync(outDir, { recursive: true, force: true })
|
|
}
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const args = ['tsx', 'scripts/pack-act-assets.ts', archiveDir, outDir]
|
|
const env = {
|
|
...process.env,
|
|
MAX_WORKERS: String(workers),
|
|
FILTER_LEVEL: filterLevel,
|
|
SKIP_ENTITIES: '1',
|
|
}
|
|
|
|
const tStart = performance.now()
|
|
const child = spawn('npx', args, { env, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
const pid = child.pid!
|
|
|
|
let peakRss = 0
|
|
let peakThreads = 0
|
|
let peakCpu = 0
|
|
const cpuSamples: number[] = []
|
|
|
|
let prevTicks = 0
|
|
let prevTime = performance.now()
|
|
|
|
const interval = setInterval(() => {
|
|
const now = performance.now()
|
|
const metrics = getProcessTreeMetrics(pid)
|
|
if (metrics.rssMb > peakRss) peakRss = metrics.rssMb
|
|
if (metrics.threadCount > peakThreads) peakThreads = metrics.threadCount
|
|
|
|
const deltaMs = now - prevTime
|
|
const deltaTicks = metrics.totalCpuTicks - prevTicks
|
|
if (prevTicks > 0 && deltaMs > 50) {
|
|
// 100 ticks per second = 1 full CPU core at 100%
|
|
// CPU percent = (deltaTicks / (deltaMs / 10)) * 100 = (deltaTicks / deltaMs) * 1000
|
|
const cpuPercent = (deltaTicks / (deltaMs / 1000))
|
|
if (cpuPercent > 0) {
|
|
cpuSamples.push(cpuPercent)
|
|
if (cpuPercent > peakCpu) peakCpu = cpuPercent
|
|
}
|
|
}
|
|
|
|
prevTicks = metrics.totalCpuTicks
|
|
prevTime = now
|
|
}, 100)
|
|
|
|
let stdout = ''
|
|
let stderr = ''
|
|
|
|
child.stdout.on('data', (d) => {
|
|
stdout += d.toString()
|
|
})
|
|
child.stderr.on('data', (d) => {
|
|
stderr += d.toString()
|
|
})
|
|
|
|
child.on('close', (code) => {
|
|
clearInterval(interval)
|
|
const tEnd = performance.now()
|
|
const wallTimeSec = (tEnd - tStart) / 1000
|
|
|
|
const avgCpu = cpuSamples.length > 0 ? cpuSamples.reduce((a, b) => a + b, 0) / cpuSamples.length : 0
|
|
const sceneMatches = stdout.match(/\[\s*\d+\/\s*\d+\]/g)
|
|
const scenesCount = sceneMatches ? sceneMatches.length : 0
|
|
|
|
if (code !== 0) {
|
|
console.error(`Benchmark failed with code ${code}`)
|
|
console.error('STDERR:', stderr)
|
|
console.error('STDOUT:', stdout)
|
|
reject(new Error(`Exit code ${code}: ${stderr}`))
|
|
return
|
|
}
|
|
|
|
resolve({
|
|
workers,
|
|
wallTimeSec,
|
|
speedup: 1.0,
|
|
peakRssMb: peakRss,
|
|
avgCpuPercent: avgCpu,
|
|
peakCpuPercent: peakCpu,
|
|
peakThreads,
|
|
exitCode: code ?? 0,
|
|
scenesCount,
|
|
outDir,
|
|
})
|
|
})
|
|
})
|
|
}
|
|
|
|
async function main() {
|
|
console.log('========================================================================')
|
|
console.log(' EMPIRICAL MULTI-WORKER BAKING BENCHMARK & PROFILING HARNESS (TREE)')
|
|
console.log('========================================================================')
|
|
|
|
const filterLevel = process.env.BENCH_FILTER_LEVEL || '1,2'
|
|
const workerConfigs = process.env.BENCH_WORKERS
|
|
? process.env.BENCH_WORKERS.split(',').map(s => parseInt(s.trim(), 10))
|
|
: [1, 4, 8, 16]
|
|
const results: BenchResult[] = []
|
|
|
|
let baselineSec = 0
|
|
|
|
for (const w of workerConfigs) {
|
|
const outDir = `samples/d2-packs-bench-tree-w${w}`
|
|
console.log(`\n[RUN] Benchmarking MAX_WORKERS=${w} on Level ${filterLevel} (${w === 1 ? 'Sequential' : `${w} Workers`})...`)
|
|
const res = await runBench(w, filterLevel, outDir)
|
|
if (w === 1) {
|
|
baselineSec = res.wallTimeSec
|
|
res.speedup = 1.0
|
|
} else {
|
|
res.speedup = baselineSec / res.wallTimeSec
|
|
}
|
|
results.push(res)
|
|
console.log(
|
|
` → Done in ${res.wallTimeSec.toFixed(2)}s | Speedup: ${res.speedup.toFixed(2)}x | Peak RSS: ${res.peakRssMb.toFixed(1)} MB | Avg CPU: ${res.avgCpuPercent.toFixed(0)}% (Peak: ${res.peakCpuPercent.toFixed(0)}%) | Peak Threads: ${res.peakThreads}`,
|
|
)
|
|
}
|
|
|
|
console.log('\n========================================================================')
|
|
console.log(' BENCHMARK SUMMARY MATRIX (TREE PROFILED)')
|
|
console.log('========================================================================')
|
|
console.log(
|
|
'Workers | Wall Time (s) | Speedup | Peak RSS (MB) | Avg CPU % | Peak CPU % | Threads | Status',
|
|
)
|
|
console.log(
|
|
'----------------------------------------------------------------------------------',
|
|
)
|
|
for (const r of results) {
|
|
console.log(
|
|
`${String(r.workers).padStart(7)} | ${r.wallTimeSec.toFixed(2).padStart(13)} | ${r.speedup.toFixed(2).padStart(7)}x | ${r.peakRssMb.toFixed(1).padStart(13)} | ${r.avgCpuPercent.toFixed(0).padStart(9)}% | ${r.peakCpuPercent.toFixed(0).padStart(10)}% | ${String(r.peakThreads).padStart(7)} | PASS`,
|
|
)
|
|
}
|
|
console.log('========================================================================\n')
|
|
|
|
const jsonPath = process.env.BENCH_JSON_PATH || '/usr/local/google/home/taodao/diablo2-web/.agents/teamwork_preview_challenger_m2_2/benchmark-tree-results.json'
|
|
writeFileSync(jsonPath, JSON.stringify(results, null, 2))
|
|
console.log(`Saved tree benchmark metrics to ${jsonPath}`)
|
|
}
|
|
|
|
if (process.argv[1]?.endsWith('benchmark-act-concurrency.ts')) {
|
|
main().catch((err) => {
|
|
console.error('Benchmark error:', err)
|
|
process.exit(1)
|
|
})
|
|
}
|