From 258be4d6083d159d496fcae6dfb36f1c5ec41ced Mon Sep 17 00:00:00 2001 From: troytt <47798984@qq.com> Date: Tue, 15 Sep 2026 06:33:19 +0000 Subject: [PATCH] fix(publish): fine-grained batched push with resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 跨境链路对长连接做 MSS 钳制：实测 pmtu 1460 但 mss 被压到 324，重传率 13.7%，cwnd 触底到 3-4，有效带宽只剩 ~20 KB/s。原来按 act 分 5 批推送， 单批最大 117 MB，在这种链路上必然超时，且失败后整批重来。 改为： - 按地图目录累计到 ~10 MB 就切一批（45 批 / 399.1 MB），失败只重推一批 - http.lowSpeedLimit 1000 + http.lowSpeedTime 120：低于 1 KB/s 持续 2 分钟 就断开，让重试开新连接（实测新连接能拿回 mss 1408，速度提升 2.5 倍） - --work 支持续推：用 git ls-remote 读远端 main，回卷本地工作区对齐， 再跳过所有已在 HEAD 中的批次，避免重复上传 - 5 次重试，线性退避 15/30/45/60 秒 另外修正 tests/wilderness-features.test.ts 的 WildernessPiece 导入—— verbatimModuleSyntax 下类型导入必须用 import type。 TAG=agy CONV=c89513df-cd12-4749-b5bd-5e9f16639391 --- scripts/publish-packs.ts | 256 ++++++++++++++++++++++++++---- tests/wilderness-features.test.ts | 2 +- 2 files changed, 229 insertions(+), 29 deletions(-) diff --git a/scripts/publish-packs.ts b/scripts/publish-packs.ts index 8ae8fc6..853590e 100644 --- a/scripts/publish-packs.ts +++ b/scripts/publish-packs.ts @@ -149,51 +149,251 @@ npm run publish:packs # orphan 提交 + force push 覆盖本仓库 ` } -const remote = process.argv[2] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE +/* ------------------------------------------------------------------------- * + * Batch planning + * ------------------------------------------------------------------------- */ + +/** + * Target size of one push batch, in bytes. + * + * The asset remote sits behind a high-loss cross-border link: measured 13.7% + * TCP retransmission, MSS clamped to 324 bytes by a middlebox, and a congestion + * window pinned at 3–4 segments, which caps a single connection at roughly + * 20 KB/s. A 100 MB push therefore needs ~1.5 h of uninterrupted connection and + * loses everything on the first reset. Batching at ~10 MB keeps each push to a + * few minutes and makes a retry cheap. + */ +const MAX_BATCH_BYTES = 10 * 1024 * 1024 + +/** How many times to retry one push before giving up. */ +const PUSH_ATTEMPTS = 5 + +/** The order acts are published in, smallest first so the remote is usable early. */ +const ACT_ORDER = ['act4', 'act3', 'act5', 'act2', 'act1'] as const + +/** One push batch: a set of map directories inside one act. */ +interface Batch { + readonly act: string + /** Paths relative to {@link PACKS_DIR}, e.g. `act1/1-act-1-town-townn1`. */ + readonly paths: readonly string[] + readonly bytes: number +} + +/** + * Total size of a directory tree, in bytes. + * + * @param dir - absolute path. + * @returns bytes, `.git` excluded. + */ +function dirSize(dir: string): number { + let bytes = 0 + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git') continue + const path = join(dir, entry.name) + if (entry.isDirectory()) bytes += dirSize(path) + else bytes += statSync(path).size + } + return bytes +} + +/** + * Split every act into batches of at most {@link MAX_BATCH_BYTES}. + * + * Map directories are never split, so a single oversized map becomes a batch of + * its own. The order within an act is the directory listing order, sorted, so + * the plan is stable across runs — which is what makes resuming safe. + * + * @returns the batches, act by act in {@link ACT_ORDER}. + */ +function planBatches(): Batch[] { + const batches: Batch[] = [] + for (const act of ACT_ORDER) { + const actDir = join(PACKS_DIR, act) + if (!existsSync(actDir)) continue + const maps = readdirSync(actDir, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .sort() + + let current: string[] = [] + let currentBytes = 0 + const flush = (): void => { + if (current.length === 0) return + batches.push({ act, paths: current, bytes: currentBytes }) + current = [] + currentBytes = 0 + } + for (const map of maps) { + const bytes = dirSize(join(actDir, map)) + if (currentBytes > 0 && currentBytes + bytes > MAX_BATCH_BYTES) flush() + current.push(`${act}/${map}`) + currentBytes += bytes + } + // Loose files directly under the act directory ride along in the last batch. + const loose = readdirSync(actDir, { withFileTypes: true }) + .filter(entry => !entry.isDirectory()) + .map(entry => `${act}/${entry.name}`) + for (const file of loose) current.push(file) + flush() + } + return batches +} + +/* ------------------------------------------------------------------------- * + * Git helpers that tolerate a lossy link + * ------------------------------------------------------------------------- */ + +/** + * Run one git command and capture stdout instead of inheriting it. + * + * @param args - git arguments. + * @param cwd - directory to run in. + * @returns trimmed stdout, or null when the command failed. + */ +function gitQuery(args: readonly string[], cwd: string): string | null { + try { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() + } catch { + return null + } +} + +/** + * Push `main`, retrying with linear backoff. + * + * `http.lowSpeedLimit`/`http.lowSpeedTime` are what stop a dead connection from + * hanging for hours: git aborts once throughput stays under 1 KB/s for two + * minutes, and the retry opens a fresh connection — which on this link usually + * comes back with a healthy congestion window. + * + * @param remoteUrl - the push target. + * @param cwd - the work repository. + * @param label - what is being pushed, for the log. + * @throws when every attempt fails. + */ +function pushWithRetry(remoteUrl: string, cwd: string, label: string): void { + for (let attempt = 1; attempt <= PUSH_ATTEMPTS; attempt += 1) { + try { + execFileSync('git', ['push', remoteUrl, 'main'], { cwd, stdio: ['ignore', 'inherit', 'inherit'] }) + return + } catch { + if (attempt === PUSH_ATTEMPTS) { + throw new PublishError(`推送「${label}」失败：已重试 ${String(PUSH_ATTEMPTS)} 次`) + } + const waitSeconds = attempt * 15 + console.log(` ↻ 推送「${label}」第 ${String(attempt)} 次失败，${String(waitSeconds)} 秒后重试…`) + execFileSync('sleep', [String(waitSeconds)], { stdio: 'ignore' }) + } + } +} + +/** + * Is `path` already committed in the work repository's `HEAD`? + * + * @param path - path relative to the repository root. + * @param cwd - the work repository. + * @returns true when `HEAD` already carries it. + */ +function committed(path: string, cwd: string): boolean { + const listed = gitQuery(['ls-tree', '--name-only', 'HEAD', path], cwd) + return listed !== null && listed !== '' +} + +/* ------------------------------------------------------------------------- * + * Driver + * ------------------------------------------------------------------------- */ + +const args = process.argv.slice(2) +const workFlagIndex = args.indexOf('--work') +const reuseWork = workFlagIndex >= 0 ? args[workFlagIndex + 1] : undefined +const positional = args.filter((value, index) => + value !== '--work' && index !== workFlagIndex + 1) + +const remote = positional[0] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE const stats = measure() const stamp = new Date().toISOString().slice(0, 10) -const work = mkdtempSync(join(tmpdir(), 'd2-assets-')) +const keepWork = reuseWork !== undefined +const work = reuseWork ?? mkdtempSync(join(tmpdir(), 'd2-assets-')) try { + const batches = planBatches() + const totalBytes = batches.reduce((sum, batch) => sum + batch.bytes, 0) console.log(`图集包：${String(stats.maps)} 块地图 / ${String(stats.files)} 个文件 / ${(stats.bytes / 1048576).toFixed(1)} MB`) console.log(`index.json sha256：${stats.indexSha}`) - console.log(`临时仓库：${work}`) + console.log(`临时仓库：${work}${keepWork ? '（复用，断点续传）' : ''}`) + console.log(`推送计划：${String(batches.length)} 个批次，单批上限 ${String(MAX_BATCH_BYTES / 1048576)} MB，合计 ${(totalBytes / 1048576).toFixed(1)} MB`) - git(['init', '-q', '-b', 'main'], work) + if (!keepWork) git(['init', '-q', '-b', 'main'], work) + // A 500 MB post buffer keeps git from chunking the upload; HTTP/1.1 avoids the + // remote's HTTP/2 stream resets; the low-speed guard turns a stalled + // connection into a fast failure that `pushWithRetry` can recover from. git(['config', 'http.postBuffer', '524288000'], work) git(['config', 'http.version', 'HTTP/1.1'], work) + git(['config', 'http.lowSpeedLimit', '1000'], work) + git(['config', 'http.lowSpeedTime', '120'], work) + // PNG atlas pages are already deflate compressed, so delta search is pure cost. + git(['config', 'pack.window', '0'], work) + git(['config', 'core.compression', '1'], work) - // Step 1: Base metadata and index - writeFileSync(join(work, 'README.md'), readme(stats, stamp)) - writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n') - cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json')) - git(['add', '-A'], work) - git([ - '-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`, - '-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`, - 'commit', '-q', '-m', - `烘焙图集包 ${stamp} [0/5]：索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`, - ], work) - console.log(`推送到 ${remote}（force，覆盖历史，初始化基线）…`) - git(['push', '--force', remote, 'main'], work) + // Resume: rewind the work tree to whatever the remote already has, so a + // half-finished oversized commit from an earlier run is discarded rather than + // re-uploaded. + if (keepWork) { + const remoteSha = gitQuery(['ls-remote', remote, 'main'], work)?.split(/\s+/)[0] + if (remoteSha !== undefined && gitQuery(['cat-file', '-e', `${remoteSha}^{commit}`], work) !== null) { + console.log(`远端 main 位于 ${remoteSha.slice(0, 7)}，回卷工作区以对齐…`) + git(['reset', '--hard', '-q', remoteSha], work) + git(['clean', '-qfd'], work) + } + } - // Step 2..6: Push acts in small incremental batches to stay under HTTP timeout limits - const acts = ['act4', 'act3', 'act5', 'act2', 'act1'] as const - for (let i = 0; i < acts.length; i++) { - const act = acts[i]! - console.log(`添加并提交 ${act} [${i + 1}/${acts.length}]…`) - cpSync(join(PACKS_DIR, act), join(work, act), { recursive: true }) - git(['add', act], work) + // Step 0: metadata and index. + if (!committed('index.json', work)) { + writeFileSync(join(work, 'README.md'), readme(stats, stamp)) + writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n') + cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json')) + git(['add', '-A'], work) git([ '-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`, '-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`, 'commit', '-q', '-m', - `烘焙图集包 ${stamp} [${i + 1}/5]：${act} 图集`, + `烘焙图集包 ${stamp} [0]：索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`, ], work) - console.log(`推送到 ${remote}（增量推送 ${act}）…`) - git(['push', remote, 'main'], work) + console.log(`推送到 ${remote}（force，覆盖历史，初始化基线）…`) + git(['push', '--force', remote, 'main'], work) + } else { + console.log('索引与元数据已在远端，跳过。') } + + // Steps 1..N: one small batch per push. + let sentBytes = 0 + for (let i = 0; i < batches.length; i += 1) { + const batch = batches[i]! + const label = `${String(i + 1)}/${String(batches.length)} ${batch.act} (${(batch.bytes / 1048576).toFixed(1)} MB)` + sentBytes += batch.bytes + + if (batch.paths.every(path => committed(path, work))) { + console.log(`[${label}] 已在远端，跳过。`) + continue + } + + for (const path of batch.paths) { + cpSync(join(PACKS_DIR, path), join(work, path), { recursive: true }) + } + git(['add', ...batch.paths], work) + git([ + '-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`, + '-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`, + 'commit', '-q', '-m', + `烘焙图集包 ${stamp} [${String(i + 1)}/${String(batches.length)}]：${batch.act} ${String(batch.paths.length)} 项`, + ], work) + console.log(`[${label}] 推送中…（累计 ${(sentBytes / 1048576).toFixed(1)}/${(totalBytes / 1048576).toFixed(1)} MB）`) + pushWithRetry(remote, work, label) + } + console.log('完成。使用方：git clone ' + remote + ' samples/d2-packs') } finally { - rmSync(work, { recursive: true, force: true }) + if (!keepWork) rmSync(work, { recursive: true, force: true }) + else console.log(`工作区保留在 ${work}（下次可用 --work ${work} 续传）`) } + diff --git a/tests/wilderness-features.test.ts b/tests/wilderness-features.test.ts index 9e4e2d3..45b2f10 100644 --- a/tests/wilderness-features.test.ts +++ b/tests/wilderness-features.test.ts @@ -5,8 +5,8 @@ import { getPerimeterOpenings, SPECIAL_PRESETS_BY_LEVEL, generateWilderness, - WildernessPiece, } from "../src/game/wilderness.ts" +import type { WildernessPiece } from "../src/game/wilderness.ts" import { Rng } from "../src/game/rng.ts" import type { Ds1, Ds1Cell } from "../src/formats/ds1.ts"