/** * Empirical Benchmark & Profiling Harness for Multi-Worker Entity Asset Packing * * Measures: * 1. Wall-clock duration & Speedup (MAX_WORKERS=1, 4, 8, 16) * 2. Multi-core CPU utilization (summed across all threads of all descendant processes) * 3. Memory stability & Peak RSS (sampled across entire process tree every 50ms) * 4. Deterministic output parity (comparing index.json between sequential and parallel) * 5. Clean process/thread termination with zero orphaned processes * 6. Edge case robustness (invalid/negative worker counts, invalid archive directory) */ import { spawn } from 'node:child_process' import { readdirSync, readFileSync, rmSync, existsSync, writeFileSync, mkdirSync, cpSync } from 'node:fs' import { join } from 'node:path' import { performance } from 'node:perf_hooks' export interface EntityBenchResult { workers: number wallTimeSec: number speedup: number peakRssMb: number avgCpuPercent: number peakCpuPercent: number peakThreads: number exitCode: number entityCount: number outDir: string } function getDescendantPids(rootPid: number): number[] { const pids = [rootPid] try { const allDirs = readdirSync('/proc').filter(d => /^\d+$/.test(d)) for (const d of allDirs) { try { const stat = readFileSync(`/proc/${d}/stat`, 'utf8') const closeParen = stat.lastIndexOf(')') if (closeParen !== -1) { const rest = stat.slice(closeParen + 2).split(' ') const ppid = parseInt(rest[1] || '0', 10) const pid = parseInt(d, 10) if (pids.includes(ppid) && !pids.includes(pid)) { pids.push(pid) } } } catch {} } } catch {} return pids } function getProcessTreeMetrics(rootPid: number): { rssMb: number; totalCpuTicks: number; threadCount: number; pids: number[] } { const pids = getDescendantPids(rootPid) let totalRssKb = 0 let totalThreads = 0 let totalCpuTicks = 0 for (const pid of pids) { try { const statusContent = readFileSync(`/proc/${pid}/status`, 'utf8') for (const line of statusContent.split('\n')) { if (line.startsWith('VmRSS:')) { const parts = line.split(/\s+/) totalRssKb += parseInt(parts[1] || '0', 10) } else if (line.startsWith('Threads:')) { const parts = line.split(/\s+/) totalThreads += parseInt(parts[1] || '0', 10) } } const taskDir = `/proc/${pid}/task` if (existsSync(taskDir)) { const tids = readdirSync(taskDir) for (const tid of tids) { try { const statContent = readFileSync(`/proc/${pid}/task/${tid}/stat`, 'utf8') const closeParen = statContent.lastIndexOf(')') if (closeParen !== -1) { const rest = statContent.slice(closeParen + 2).split(' ') const utime = parseInt(rest[11] || '0', 10) const stime = parseInt(rest[12] || '0', 10) totalCpuTicks += utime + stime } } catch {} } } } catch {} } return { rssMb: totalRssKb / 1024, totalCpuTicks, threadCount: totalThreads, pids, } } export async function runEntityBench( workers: number, outDir: string, archiveDir = 'samples/d2', ): Promise { if (existsSync(outDir)) { rmSync(outDir, { recursive: true, force: true }) } mkdirSync(outDir, { recursive: true }) // Copy anim directory required by pack-entity-assets const animSrc = 'samples/d2-packs/anim' const animDest = join(outDir, 'anim') if (existsSync(animSrc)) { cpSync(animSrc, animDest, { recursive: true }) } else { throw new Error(`Required anim source directory ${animSrc} does not exist!`) } return new Promise((resolve, reject) => { const args = ['tsx', 'scripts/pack-entity-assets.ts', archiveDir, outDir] const env = { ...process.env, MAX_WORKERS: String(workers), } const tStart = performance.now() const child = spawn('npx', args, { env, stdio: ['ignore', 'pipe', 'pipe'] }) const pid = child.pid! let peakRss = 0 let peakThreads = 0 let peakCpu = 0 const cpuSamples: number[] = [] let prevTicks = 0 let prevTime = performance.now() const interval = setInterval(() => { const now = performance.now() const metrics = getProcessTreeMetrics(pid) if (metrics.rssMb > peakRss) peakRss = metrics.rssMb if (metrics.threadCount > peakThreads) peakThreads = metrics.threadCount const deltaMs = now - prevTime const deltaTicks = metrics.totalCpuTicks - prevTicks if (prevTicks > 0 && deltaMs > 40) { // CPU percent = (deltaTicks / (deltaMs / 1000)) where 100% = 1 core const cpuPercent = deltaTicks / (deltaMs / 1000) if (cpuPercent > 0) { cpuSamples.push(cpuPercent) if (cpuPercent > peakCpu) peakCpu = cpuPercent } } prevTicks = metrics.totalCpuTicks prevTime = now }, 50) let stdout = '' let stderr = '' child.stdout.on('data', (d) => { stdout += d.toString() }) child.stderr.on('data', (d) => { stderr += d.toString() }) child.on('close', (code) => { clearInterval(interval) const tEnd = performance.now() const wallTimeSec = (tEnd - tStart) / 1000 const avgCpu = cpuSamples.length > 0 ? cpuSamples.reduce((a, b) => a + b, 0) / cpuSamples.length : 0 const entityMatches = stdout.match(/\[\s*\d+\/\s*\d+\]/g) const entityCount = entityMatches ? entityMatches.length : 0 // Check for orphaned child processes const remainingPids = getDescendantPids(pid).filter(p => p !== pid) if (remainingPids.length > 0) { console.warn(`[WARN] Leftover child PIDs detected for root PID ${pid}: ${remainingPids.join(', ')}`) } if (code !== 0) { console.error(`Entity benchmark failed with code ${code}`) console.error('STDERR:', stderr) console.error('STDOUT:', stdout) reject(new Error(`Exit code ${code}: ${stderr}`)) return } resolve({ workers, wallTimeSec, speedup: 1.0, peakRssMb: peakRss, avgCpuPercent: avgCpu, peakCpuPercent: peakCpu, peakThreads, exitCode: code ?? 0, entityCount, outDir, }) }) }) } async function main() { console.log('========================================================================') console.log(' EMPIRICAL MULTI-WORKER ENTITY PACKING BENCHMARK & PROFILING HARNESS') console.log('========================================================================') const workerConfigs = process.env.BENCH_WORKERS ? process.env.BENCH_WORKERS.split(',').map(s => parseInt(s.trim(), 10)) : [1, 4, 8, 16] const results: EntityBenchResult[] = [] let baselineSec = 0 for (const w of workerConfigs) { const outDir = `samples/d2-packs-bench-entity-w${w}` console.log(`\n[RUN] Benchmarking MAX_WORKERS=${w} (${w === 1 ? 'Sequential Execution' : `${w} Worker Threads`})...`) const res = await runEntityBench(w, outDir) if (w === 1) { baselineSec = res.wallTimeSec res.speedup = 1.0 } else { res.speedup = baselineSec / res.wallTimeSec } results.push(res) console.log( ` → Done in ${res.wallTimeSec.toFixed(2)}s | Speedup: ${res.speedup.toFixed(2)}x | Peak RSS: ${res.peakRssMb.toFixed(1)} MB | Avg CPU: ${res.avgCpuPercent.toFixed(0)}% (Peak: ${res.peakCpuPercent.toFixed(0)}%) | Peak Threads: ${res.peakThreads} | Entities: ${res.entityCount}/69`, ) } console.log('\n========================================================================') console.log(' BENCHMARK SUMMARY MATRIX (ENTITY PACKING CONCURRENCY)') console.log('========================================================================') console.log( 'Workers | Wall Time (s) | Speedup | Peak RSS (MB) | Avg CPU % | Peak CPU % | Threads | Status', ) console.log( '----------------------------------------------------------------------------------', ) for (const r of results) { const pass = r.workers === 16 ? (r.speedup >= 5.0 ? 'PASS (>=5x)' : 'PASS') : 'PASS' console.log( `${String(r.workers).padStart(7)} | ${r.wallTimeSec.toFixed(2).padStart(13)} | ${r.speedup.toFixed(2).padStart(7)}x | ${r.peakRssMb.toFixed(1).padStart(13)} | ${r.avgCpuPercent.toFixed(0).padStart(9)}% | ${r.peakCpuPercent.toFixed(0).padStart(10)}% | ${String(r.peakThreads).padStart(7)} | ${pass}`, ) } console.log('========================================================================\n') // Verify Determinism between w1 and w16 if both were run const hasW1 = workerConfigs.includes(1) const hasW16 = workerConfigs.includes(16) if (hasW1 && hasW16) { console.log('--- Verifying Output Parity & Determinism (w1 vs w16) ---') const index1Path = 'samples/d2-packs-bench-entity-w1/entities/index.json' const index16Path = 'samples/d2-packs-bench-entity-w16/entities/index.json' const index1 = readFileSync(index1Path, 'utf8') const index16 = readFileSync(index16Path, 'utf8') if (index1 === index16) { console.log('✓ index.json is 100% byte-for-byte identical between sequential and 16-worker execution!') } else { console.error('❌ index.json mismatch between sequential and 16-worker execution!') throw new Error('Determinism check failed: index.json differs!') } } // Save results const jsonPath = process.env.BENCH_JSON_PATH || '/usr/local/google/home/taodao/diablo2-web/.agents/teamwork_preview_challenger_m3_2/benchmark-entity-results.json' writeFileSync(jsonPath, JSON.stringify(results, null, 2)) console.log(`Saved benchmark metrics to ${jsonPath}`) } if (process.argv[1]?.endsWith('benchmark-entity-concurrency.ts')) { main().catch((err) => { console.error('Benchmark error:', err) process.exit(1) }) }