282 lines
9.8 KiB
TypeScript
282 lines
9.8 KiB
TypeScript
/**
|
|
* Empirical Benchmark & Profiling Harness for Multi-Worker Entity Asset Packing
|
|
*
|
|
* Measures:
|
|
* 1. Wall-clock duration & Speedup (MAX_WORKERS=1, 4, 8, 16)
|
|
* 2. Multi-core CPU utilization (summed across all threads of all descendant processes)
|
|
* 3. Memory stability & Peak RSS (sampled across entire process tree every 50ms)
|
|
* 4. Deterministic output parity (comparing index.json between sequential and parallel)
|
|
* 5. Clean process/thread termination with zero orphaned processes
|
|
* 6. Edge case robustness (invalid/negative worker counts, invalid archive directory)
|
|
*/
|
|
|
|
import { spawn } from 'node:child_process'
|
|
import { readdirSync, readFileSync, rmSync, existsSync, writeFileSync, mkdirSync, cpSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import { performance } from 'node:perf_hooks'
|
|
|
|
export interface EntityBenchResult {
|
|
workers: number
|
|
wallTimeSec: number
|
|
speedup: number
|
|
peakRssMb: number
|
|
avgCpuPercent: number
|
|
peakCpuPercent: number
|
|
peakThreads: number
|
|
exitCode: number
|
|
entityCount: number
|
|
outDir: string
|
|
}
|
|
|
|
function getDescendantPids(rootPid: number): number[] {
|
|
const pids = [rootPid]
|
|
try {
|
|
const allDirs = readdirSync('/proc').filter(d => /^\d+$/.test(d))
|
|
for (const d of allDirs) {
|
|
try {
|
|
const stat = readFileSync(`/proc/${d}/stat`, 'utf8')
|
|
const closeParen = stat.lastIndexOf(')')
|
|
if (closeParen !== -1) {
|
|
const rest = stat.slice(closeParen + 2).split(' ')
|
|
const ppid = parseInt(rest[1] || '0', 10)
|
|
const pid = parseInt(d, 10)
|
|
if (pids.includes(ppid) && !pids.includes(pid)) {
|
|
pids.push(pid)
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
} catch {}
|
|
return pids
|
|
}
|
|
|
|
function getProcessTreeMetrics(rootPid: number): { rssMb: number; totalCpuTicks: number; threadCount: number; pids: number[] } {
|
|
const pids = getDescendantPids(rootPid)
|
|
let totalRssKb = 0
|
|
let totalThreads = 0
|
|
let totalCpuTicks = 0
|
|
|
|
for (const pid of pids) {
|
|
try {
|
|
const statusContent = readFileSync(`/proc/${pid}/status`, 'utf8')
|
|
for (const line of statusContent.split('\n')) {
|
|
if (line.startsWith('VmRSS:')) {
|
|
const parts = line.split(/\s+/)
|
|
totalRssKb += parseInt(parts[1] || '0', 10)
|
|
} else if (line.startsWith('Threads:')) {
|
|
const parts = line.split(/\s+/)
|
|
totalThreads += parseInt(parts[1] || '0', 10)
|
|
}
|
|
}
|
|
|
|
const taskDir = `/proc/${pid}/task`
|
|
if (existsSync(taskDir)) {
|
|
const tids = readdirSync(taskDir)
|
|
for (const tid of tids) {
|
|
try {
|
|
const statContent = readFileSync(`/proc/${pid}/task/${tid}/stat`, 'utf8')
|
|
const closeParen = statContent.lastIndexOf(')')
|
|
if (closeParen !== -1) {
|
|
const rest = statContent.slice(closeParen + 2).split(' ')
|
|
const utime = parseInt(rest[11] || '0', 10)
|
|
const stime = parseInt(rest[12] || '0', 10)
|
|
totalCpuTicks += utime + stime
|
|
}
|
|
} catch {}
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
|
|
return {
|
|
rssMb: totalRssKb / 1024,
|
|
totalCpuTicks,
|
|
threadCount: totalThreads,
|
|
pids,
|
|
}
|
|
}
|
|
|
|
export async function runEntityBench(
|
|
workers: number,
|
|
outDir: string,
|
|
archiveDir = 'samples/d2',
|
|
): Promise<EntityBenchResult> {
|
|
if (existsSync(outDir)) {
|
|
rmSync(outDir, { recursive: true, force: true })
|
|
}
|
|
mkdirSync(outDir, { recursive: true })
|
|
|
|
// Copy anim directory required by pack-entity-assets
|
|
const animSrc = 'samples/d2-packs/anim'
|
|
const animDest = join(outDir, 'anim')
|
|
if (existsSync(animSrc)) {
|
|
cpSync(animSrc, animDest, { recursive: true })
|
|
} else {
|
|
throw new Error(`Required anim source directory ${animSrc} does not exist!`)
|
|
}
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const args = ['tsx', 'scripts/pack-entity-assets.ts', archiveDir, outDir]
|
|
const env = {
|
|
...process.env,
|
|
MAX_WORKERS: String(workers),
|
|
}
|
|
|
|
const tStart = performance.now()
|
|
const child = spawn('npx', args, { env, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
const pid = child.pid!
|
|
|
|
let peakRss = 0
|
|
let peakThreads = 0
|
|
let peakCpu = 0
|
|
const cpuSamples: number[] = []
|
|
|
|
let prevTicks = 0
|
|
let prevTime = performance.now()
|
|
|
|
const interval = setInterval(() => {
|
|
const now = performance.now()
|
|
const metrics = getProcessTreeMetrics(pid)
|
|
if (metrics.rssMb > peakRss) peakRss = metrics.rssMb
|
|
if (metrics.threadCount > peakThreads) peakThreads = metrics.threadCount
|
|
|
|
const deltaMs = now - prevTime
|
|
const deltaTicks = metrics.totalCpuTicks - prevTicks
|
|
if (prevTicks > 0 && deltaMs > 40) {
|
|
// CPU percent = (deltaTicks / (deltaMs / 1000)) where 100% = 1 core
|
|
const cpuPercent = deltaTicks / (deltaMs / 1000)
|
|
if (cpuPercent > 0) {
|
|
cpuSamples.push(cpuPercent)
|
|
if (cpuPercent > peakCpu) peakCpu = cpuPercent
|
|
}
|
|
}
|
|
|
|
prevTicks = metrics.totalCpuTicks
|
|
prevTime = now
|
|
}, 50)
|
|
|
|
let stdout = ''
|
|
let stderr = ''
|
|
|
|
child.stdout.on('data', (d) => {
|
|
stdout += d.toString()
|
|
})
|
|
child.stderr.on('data', (d) => {
|
|
stderr += d.toString()
|
|
})
|
|
|
|
child.on('close', (code) => {
|
|
clearInterval(interval)
|
|
const tEnd = performance.now()
|
|
const wallTimeSec = (tEnd - tStart) / 1000
|
|
|
|
const avgCpu = cpuSamples.length > 0 ? cpuSamples.reduce((a, b) => a + b, 0) / cpuSamples.length : 0
|
|
const entityMatches = stdout.match(/\[\s*\d+\/\s*\d+\]/g)
|
|
const entityCount = entityMatches ? entityMatches.length : 0
|
|
|
|
// Check for orphaned child processes
|
|
const remainingPids = getDescendantPids(pid).filter(p => p !== pid)
|
|
if (remainingPids.length > 0) {
|
|
console.warn(`[WARN] Leftover child PIDs detected for root PID ${pid}: ${remainingPids.join(', ')}`)
|
|
}
|
|
|
|
if (code !== 0) {
|
|
console.error(`Entity benchmark failed with code ${code}`)
|
|
console.error('STDERR:', stderr)
|
|
console.error('STDOUT:', stdout)
|
|
reject(new Error(`Exit code ${code}: ${stderr}`))
|
|
return
|
|
}
|
|
|
|
resolve({
|
|
workers,
|
|
wallTimeSec,
|
|
speedup: 1.0,
|
|
peakRssMb: peakRss,
|
|
avgCpuPercent: avgCpu,
|
|
peakCpuPercent: peakCpu,
|
|
peakThreads,
|
|
exitCode: code ?? 0,
|
|
entityCount,
|
|
outDir,
|
|
})
|
|
})
|
|
})
|
|
}
|
|
|
|
async function main() {
|
|
console.log('========================================================================')
|
|
console.log(' EMPIRICAL MULTI-WORKER ENTITY PACKING BENCHMARK & PROFILING HARNESS')
|
|
console.log('========================================================================')
|
|
|
|
const workerConfigs = process.env.BENCH_WORKERS
|
|
? process.env.BENCH_WORKERS.split(',').map(s => parseInt(s.trim(), 10))
|
|
: [1, 4, 8, 16]
|
|
const results: EntityBenchResult[] = []
|
|
|
|
let baselineSec = 0
|
|
|
|
for (const w of workerConfigs) {
|
|
const outDir = `samples/d2-packs-bench-entity-w${w}`
|
|
console.log(`\n[RUN] Benchmarking MAX_WORKERS=${w} (${w === 1 ? 'Sequential Execution' : `${w} Worker Threads`})...`)
|
|
const res = await runEntityBench(w, outDir)
|
|
if (w === 1) {
|
|
baselineSec = res.wallTimeSec
|
|
res.speedup = 1.0
|
|
} else {
|
|
res.speedup = baselineSec / res.wallTimeSec
|
|
}
|
|
results.push(res)
|
|
console.log(
|
|
` → Done in ${res.wallTimeSec.toFixed(2)}s | Speedup: ${res.speedup.toFixed(2)}x | Peak RSS: ${res.peakRssMb.toFixed(1)} MB | Avg CPU: ${res.avgCpuPercent.toFixed(0)}% (Peak: ${res.peakCpuPercent.toFixed(0)}%) | Peak Threads: ${res.peakThreads} | Entities: ${res.entityCount}/69`,
|
|
)
|
|
}
|
|
|
|
console.log('\n========================================================================')
|
|
console.log(' BENCHMARK SUMMARY MATRIX (ENTITY PACKING CONCURRENCY)')
|
|
console.log('========================================================================')
|
|
console.log(
|
|
'Workers | Wall Time (s) | Speedup | Peak RSS (MB) | Avg CPU % | Peak CPU % | Threads | Status',
|
|
)
|
|
console.log(
|
|
'----------------------------------------------------------------------------------',
|
|
)
|
|
for (const r of results) {
|
|
const pass = r.workers === 16 ? (r.speedup >= 5.0 ? 'PASS (>=5x)' : 'PASS') : 'PASS'
|
|
console.log(
|
|
`${String(r.workers).padStart(7)} | ${r.wallTimeSec.toFixed(2).padStart(13)} | ${r.speedup.toFixed(2).padStart(7)}x | ${r.peakRssMb.toFixed(1).padStart(13)} | ${r.avgCpuPercent.toFixed(0).padStart(9)}% | ${r.peakCpuPercent.toFixed(0).padStart(10)}% | ${String(r.peakThreads).padStart(7)} | ${pass}`,
|
|
)
|
|
}
|
|
console.log('========================================================================\n')
|
|
|
|
// Verify Determinism between w1 and w16 if both were run
|
|
const hasW1 = workerConfigs.includes(1)
|
|
const hasW16 = workerConfigs.includes(16)
|
|
if (hasW1 && hasW16) {
|
|
console.log('--- Verifying Output Parity & Determinism (w1 vs w16) ---')
|
|
const index1Path = 'samples/d2-packs-bench-entity-w1/entities/index.json'
|
|
const index16Path = 'samples/d2-packs-bench-entity-w16/entities/index.json'
|
|
const index1 = readFileSync(index1Path, 'utf8')
|
|
const index16 = readFileSync(index16Path, 'utf8')
|
|
|
|
if (index1 === index16) {
|
|
console.log('✓ index.json is 100% byte-for-byte identical between sequential and 16-worker execution!')
|
|
} else {
|
|
console.error('❌ index.json mismatch between sequential and 16-worker execution!')
|
|
throw new Error('Determinism check failed: index.json differs!')
|
|
}
|
|
}
|
|
|
|
// Save results
|
|
const jsonPath = process.env.BENCH_JSON_PATH || '/usr/local/google/home/taodao/diablo2-web/.agents/teamwork_preview_challenger_m3_2/benchmark-entity-results.json'
|
|
writeFileSync(jsonPath, JSON.stringify(results, null, 2))
|
|
console.log(`Saved benchmark metrics to ${jsonPath}`)
|
|
}
|
|
|
|
if (process.argv[1]?.endsWith('benchmark-entity-concurrency.ts')) {
|
|
main().catch((err) => {
|
|
console.error('Benchmark error:', err)
|
|
process.exit(1)
|
|
})
|
|
}
|