/** * Empirical Benchmark & Profiling Harness for Multi-Worker Act Asset Baking * * Measures: * 1. Wall-clock duration & Speedup (MAX_WORKERS=1, 4, 8, 16) * 2. Multi-core CPU utilization (summed across all threads of all descendant processes) * 3. Memory stability & Peak RSS (sampled across entire process tree every 100ms) * 4. Deterministic output parity (comparing index.json between sequential and parallel) * 5. Clean process/thread termination with zero orphaned processes */ import { spawn } from 'node:child_process' import { readdirSync, readFileSync, rmSync, existsSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { performance } from 'node:perf_hooks' export interface BenchResult { workers: number wallTimeSec: number speedup: number peakRssMb: number avgCpuPercent: number peakCpuPercent: number peakThreads: number exitCode: number scenesCount: number outDir: string } function getDescendantPids(rootPid: number): number[] { const pids = [rootPid] try { const allDirs = readdirSync('/proc').filter(d => /^\d+$/.test(d)) for (const d of allDirs) { try { const stat = readFileSync(`/proc/${d}/stat`, 'utf8') const closeParen = stat.lastIndexOf(')') if (closeParen !== -1) { const rest = stat.slice(closeParen + 2).split(' ') const ppid = parseInt(rest[1] || '0', 10) const pid = parseInt(d, 10) if (pids.includes(ppid) && !pids.includes(pid)) { pids.push(pid) } } } catch {} } } catch {} return pids } function getProcessTreeMetrics(rootPid: number): { rssMb: number; totalCpuTicks: number; threadCount: number } { const pids = getDescendantPids(rootPid) let totalRssKb = 0 let totalThreads = 0 let totalCpuTicks = 0 for (const pid of pids) { try { const statusContent = readFileSync(`/proc/${pid}/status`, 'utf8') for (const line of statusContent.split('\n')) { if (line.startsWith('VmRSS:')) { const parts = line.split(/\s+/) totalRssKb += parseInt(parts[1] || '0', 10) } else if (line.startsWith('Threads:')) { const parts = line.split(/\s+/) totalThreads += parseInt(parts[1] || '0', 10) } } const taskDir = `/proc/${pid}/task` if (existsSync(taskDir)) { const tids = readdirSync(taskDir) for (const tid of tids) { try { const statContent = readFileSync(`/proc/${pid}/task/${tid}/stat`, 'utf8') const closeParen = statContent.lastIndexOf(')') if (closeParen !== -1) { const rest = statContent.slice(closeParen + 2).split(' ') const utime = parseInt(rest[11] || '0', 10) const stime = parseInt(rest[12] || '0', 10) totalCpuTicks += utime + stime } } catch {} } } } catch {} } return { rssMb: totalRssKb / 1024, totalCpuTicks, threadCount: totalThreads, } } export async function runBench( workers: number, filterLevel: string, outDir: string, archiveDir: string = 'samples/d2', ): Promise { if (existsSync(outDir)) { rmSync(outDir, { recursive: true, force: true }) } return new Promise((resolve, reject) => { const args = ['tsx', 'scripts/pack-act-assets.ts', archiveDir, outDir] const env = { ...process.env, MAX_WORKERS: String(workers), FILTER_LEVEL: filterLevel, SKIP_ENTITIES: '1', } const tStart = performance.now() const child = spawn('npx', args, { env, stdio: ['ignore', 'pipe', 'pipe'] }) const pid = child.pid! let peakRss = 0 let peakThreads = 0 let peakCpu = 0 const cpuSamples: number[] = [] let prevTicks = 0 let prevTime = performance.now() const interval = setInterval(() => { const now = performance.now() const metrics = getProcessTreeMetrics(pid) if (metrics.rssMb > peakRss) peakRss = metrics.rssMb if (metrics.threadCount > peakThreads) peakThreads = metrics.threadCount const deltaMs = now - prevTime const deltaTicks = metrics.totalCpuTicks - prevTicks if (prevTicks > 0 && deltaMs > 50) { // 100 ticks per second = 1 full CPU core at 100% // CPU percent = (deltaTicks / (deltaMs / 10)) * 100 = (deltaTicks / deltaMs) * 1000 const cpuPercent = (deltaTicks / (deltaMs / 1000)) if (cpuPercent > 0) { cpuSamples.push(cpuPercent) if (cpuPercent > peakCpu) peakCpu = cpuPercent } } prevTicks = metrics.totalCpuTicks prevTime = now }, 100) let stdout = '' let stderr = '' child.stdout.on('data', (d) => { stdout += d.toString() }) child.stderr.on('data', (d) => { stderr += d.toString() }) child.on('close', (code) => { clearInterval(interval) const tEnd = performance.now() const wallTimeSec = (tEnd - tStart) / 1000 const avgCpu = cpuSamples.length > 0 ? cpuSamples.reduce((a, b) => a + b, 0) / cpuSamples.length : 0 const sceneMatches = stdout.match(/\[\s*\d+\/\s*\d+\]/g) const scenesCount = sceneMatches ? sceneMatches.length : 0 if (code !== 0) { console.error(`Benchmark failed with code ${code}`) console.error('STDERR:', stderr) console.error('STDOUT:', stdout) reject(new Error(`Exit code ${code}: ${stderr}`)) return } resolve({ workers, wallTimeSec, speedup: 1.0, peakRssMb: peakRss, avgCpuPercent: avgCpu, peakCpuPercent: peakCpu, peakThreads, exitCode: code ?? 0, scenesCount, outDir, }) }) }) } async function main() { console.log('========================================================================') console.log(' EMPIRICAL MULTI-WORKER BAKING BENCHMARK & PROFILING HARNESS (TREE)') console.log('========================================================================') const filterLevel = process.env.BENCH_FILTER_LEVEL || '1,2' const workerConfigs = process.env.BENCH_WORKERS ? process.env.BENCH_WORKERS.split(',').map(s => parseInt(s.trim(), 10)) : [1, 4, 8, 16] const results: BenchResult[] = [] let baselineSec = 0 for (const w of workerConfigs) { const outDir = `samples/d2-packs-bench-tree-w${w}` console.log(`\n[RUN] Benchmarking MAX_WORKERS=${w} on Level ${filterLevel} (${w === 1 ? 'Sequential' : `${w} Workers`})...`) const res = await runBench(w, filterLevel, outDir) if (w === 1) { baselineSec = res.wallTimeSec res.speedup = 1.0 } else { res.speedup = baselineSec / res.wallTimeSec } results.push(res) console.log( ` → Done in ${res.wallTimeSec.toFixed(2)}s | Speedup: ${res.speedup.toFixed(2)}x | Peak RSS: ${res.peakRssMb.toFixed(1)} MB | Avg CPU: ${res.avgCpuPercent.toFixed(0)}% (Peak: ${res.peakCpuPercent.toFixed(0)}%) | Peak Threads: ${res.peakThreads}`, ) } console.log('\n========================================================================') console.log(' BENCHMARK SUMMARY MATRIX (TREE PROFILED)') console.log('========================================================================') console.log( 'Workers | Wall Time (s) | Speedup | Peak RSS (MB) | Avg CPU % | Peak CPU % | Threads | Status', ) console.log( '----------------------------------------------------------------------------------', ) for (const r of results) { console.log( `${String(r.workers).padStart(7)} | ${r.wallTimeSec.toFixed(2).padStart(13)} | ${r.speedup.toFixed(2).padStart(7)}x | ${r.peakRssMb.toFixed(1).padStart(13)} | ${r.avgCpuPercent.toFixed(0).padStart(9)}% | ${r.peakCpuPercent.toFixed(0).padStart(10)}% | ${String(r.peakThreads).padStart(7)} | PASS`, ) } console.log('========================================================================\n') const jsonPath = process.env.BENCH_JSON_PATH || '/usr/local/google/home/taodao/diablo2-web/.agents/teamwork_preview_challenger_m2_2/benchmark-tree-results.json' writeFileSync(jsonPath, JSON.stringify(results, null, 2)) console.log(`Saved tree benchmark metrics to ${jsonPath}`) } if (process.argv[1]?.endsWith('benchmark-act-concurrency.ts')) { main().catch((err) => { console.error('Benchmark error:', err) process.exit(1) }) }