/** * MPQ fixture packer: write a real MPQ v1 archive containing the generated * fixtures. * * This exists so the *whole* user path can be exercised before any Blizzard data * is available: drop an archive in, have the engine open it, list it, decode map * members from it, and render a map. It is also the only way to test the * reader's stored-member path and its sector logic against something whose * contents are known byte for byte. * * The archive is written to the v1 rules the reader implements: both tables * encrypted with their fixed keys, members stored either raw or as zlib sectors * with a compression-mask byte per sector, and — deliberately — a mix of both so * one archive covers both code paths. * * This is tooling, not engine code: it is not shipped and not imported by the app. * * Usage: node scripts/make-mpq-fixture.ts [output.mpq] */ import { readFile, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { BLOCK_TABLE_KEY, HASH_NAME_A, HASH_NAME_B, HASH_TABLE_KEY, HASH_TABLE_OFFSET, encryptBlock, hashString, normalizeName, } from '../src/mpq/crypt.ts' /** v1 header size in bytes. */ const HEADER_SIZE = 32 /** Archive signature ('MPQ\x1A'). */ const MAGIC = 0x1a51504d /** Sector size shift: sectors are `512 << shift` bytes. */ const SECTOR_SHIFT = 3 /** Resulting sector size. */ const SECTOR_SIZE = 512 << SECTOR_SHIFT /** Flag: entry exists. */ const FILE_EXISTS = 0x80000000 /** Flag: member uses the multi-codec compression path (sector table present). */ const FILE_COMPRESS = 0x00000200 /** * Flag: member is stored as one unit, with no sector offset table. * * This is not optional for stored members: a reader that sees neither this flag * nor `FILE_COMPRESS` takes the sector path and reads a sector table out of the * file's own data. Omitting it produces an archive only a lenient reader can * open — which is exactly how this was found, by an independent reader failing * on the stored members while the compressed ones read fine. */ const FILE_SINGLE_UNIT = 0x01000000 /** Compression mask byte for zlib. */ const MASK_ZLIB = 0x02 /** One member to pack. */ interface Member { /** Archive name (either separator). */ readonly name: string /** Contents. */ readonly data: Uint8Array /** Whether to store it through the compressed sector path. */ readonly compress: boolean } /** * Deflate one buffer with the platform compressor. * * @param data - input bytes. * @returns the zlib stream. */ async function deflate(data: Uint8Array): Promise { const stream = new Blob([data as BlobPart]).stream().pipeThrough(new CompressionStream('deflate')) return new Uint8Array(await new Response(stream).arrayBuffer()) } /** * Encode one member's stored bytes and flags. * * @param member - the member to encode. * @returns the stored bytes, stored size, flags and (when compressed) layout. */ async function encodeMember(member: Member): Promise<{ stored: Uint8Array; flags: number }> { if (!member.compress) { // Stored members have no sector offset table at all, which is what // SINGLE_UNIT declares. return { stored: member.data, flags: (FILE_EXISTS | FILE_SINGLE_UNIT) >>> 0 } } const sectorCount = Math.max(1, Math.ceil(member.data.byteLength / SECTOR_SIZE)) const tableSize = (sectorCount + 1) * 4 const payloads: Uint8Array[] = [] for (let sector = 0; sector < sectorCount; sector += 1) { const from = sector * SECTOR_SIZE const raw = member.data.subarray(from, Math.min(from + SECTOR_SIZE, member.data.byteLength)) const deflated = await deflate(raw) // A sector that did not shrink is stored raw, which is what the reader // detects by comparing the stored size to the expected size. payloads.push(deflated.byteLength < raw.byteLength ? concat([new Uint8Array([MASK_ZLIB]), deflated]) : raw) } const total = tableSize + payloads.reduce((sum, payload) => sum + payload.byteLength, 0) const stored = new Uint8Array(total) const view = new DataView(stored.buffer) let at = tableSize payloads.forEach((payload, sector) => { // Offsets are relative to the block start and include the table itself, // which is why the first entry equals the table size. view.setUint32(sector * 4, at, true) stored.set(payload, at) at += payload.byteLength }) view.setUint32(sectorCount * 4, total, true) return { stored, flags: (FILE_EXISTS | FILE_COMPRESS) >>> 0 } } /** * Concatenate buffers. * * @param parts - the buffers. * @returns their concatenation. */ function concat(parts: readonly Uint8Array[]): Uint8Array { const total = parts.reduce((sum, part) => sum + part.byteLength, 0) const out = new Uint8Array(total) let at = 0 for (const part of parts) { out.set(part, at); at += part.byteLength } return out } /** * Write an MPQ v1 archive. * * @param members - members to pack. * @returns the archive bytes. */ async function writeMpq(members: readonly Member[]): Promise { const encoded = await Promise.all(members.map(async member => ({ name: normalizeName(member.name), ...await encodeMember(member), fileSize: member.data.byteLength, }))) // Lay out the file data first, then the two tables. let cursor = HEADER_SIZE const placed = encoded.map(entry => { const at = cursor cursor += entry.stored.byteLength return { ...entry, filePos: at } }) const hashTableSize = Math.max(4, nextPowerOfTwo(placed.length * 2)) const hashTablePos = cursor cursor += hashTableSize * 16 const blockTablePos = cursor cursor += placed.length * 16 const archiveSize = cursor const out = new Uint8Array(archiveSize) const view = new DataView(out.buffer) placed.forEach(entry => { out.set(entry.stored, entry.filePos) }) // Block table: position, stored size, file size, flags. placed.forEach((entry, index) => { const at = blockTablePos + index * 16 view.setUint32(at, entry.filePos, true) view.setUint32(at + 4, entry.stored.byteLength, true) view.setUint32(at + 8, entry.fileSize, true) view.setUint32(at + 12, entry.flags, true) }) // Hash table: one entry per member at its probe position, free slots marked // with the 0xFFFFFFFF sentinel the reader stops on. const hashTable = new Uint8Array(hashTableSize * 16) const hashView = new DataView(hashTable.buffer) for (let slot = 0; slot < hashTableSize; slot += 1) { hashView.setUint32(slot * 16 + 12, 0xffffffff, true) } placed.forEach((entry, index) => { const hashA = hashString(entry.name, HASH_NAME_A) const hashB = hashString(entry.name, HASH_NAME_B) let slot = hashString(entry.name, HASH_TABLE_OFFSET) % hashTableSize // Linear probing, exactly as the reader walks it. while (hashView.getUint32(slot * 16 + 12, true) !== 0xffffffff) slot = (slot + 1) % hashTableSize const at = slot * 16 hashView.setUint32(at, hashA, true) hashView.setUint32(at + 4, hashB, true) hashView.setUint16(at + 8, 0, true) // locale hashView.setUint16(at + 10, 0, true) // platform hashView.setUint32(at + 12, index, true) }) encryptBlock(hashTable, HASH_TABLE_KEY) const blockTable = new Uint8Array(out.subarray(blockTablePos, blockTablePos + placed.length * 16)) encryptBlock(blockTable, BLOCK_TABLE_KEY) out.set(hashTable, hashTablePos) out.set(blockTable, blockTablePos) view.setUint32(0x00, MAGIC, true) view.setUint32(0x04, HEADER_SIZE, true) view.setUint32(0x08, archiveSize, true) view.setUint16(0x0c, 0, true) // format version 1 view.setUint16(0x0e, SECTOR_SHIFT, true) view.setUint32(0x10, hashTablePos, true) view.setUint32(0x14, blockTablePos, true) view.setUint32(0x18, hashTableSize, true) view.setUint32(0x1c, placed.length, true) return out } /** * Round a value up to a power of two. * * @param value - the value. * @returns the next power of two. */ function nextPowerOfTwo(value: number): number { let result = 1 while (result < value) result *= 2 return result } const dir = process.argv[2] if (dir === undefined) { console.error('usage: node scripts/make-mpq-fixture.ts [output.mpq]') process.exit(2) } const output = process.argv[3] ?? join(dir, 'fixture.mpq') // The map members are the point; one of them is stored raw so the archive // exercises both the stored and the compressed read paths. const names = [ 'fixture.ds1', 'fixture.dt1', 'palette.pal', 'fixture.dc6', 'actor.dc6', 'data/global/excel/monstats.txt', 'data/global/excel/experience.txt', 'data/global/excel/weapons.txt', 'data/global/excel/armor.txt', 'data/global/excel/misc.txt', 'data/global/excel/magicprefix.txt', 'data/global/excel/magicsuffix.txt', 'data/global/excel/skills.txt', 'data/global/excel/npcs.txt', 'data/global/excel/quests.txt', 'data/local/string.tbl', ] const members: Member[] = [] for (const name of names) { const data = new Uint8Array(await readFile(join(dir, name))) members.push({ name, data, compress: name !== 'palette.pal' }) } members.push({ name: '(listfile)', data: new TextEncoder().encode(`${names.join('\r\n')}\r\n`), compress: false, }) const archive = await writeMpq(members) await writeFile(output, archive) console.log(`wrote ${output} (${String(archive.byteLength)} bytes, ${String(members.length)} members)`) console.log(` stored : ${members.filter(member => !member.compress).map(member => member.name).join(', ')}`) console.log(` zlib : ${members.filter(member => member.compress).map(member => member.name).join(', ')}`)