fix(publish): fine-grained batched push with resume

跨境链路对长连接做 MSS 钳制:实测 pmtu 1460 但 mss 被压到 324,重传率
13.7%,cwnd 触底到 3-4,有效带宽只剩 ~20 KB/s。原来按 act 分 5 批推送,
单批最大 117 MB,在这种链路上必然超时,且失败后整批重来。

改为:
- 按地图目录累计到 ~10 MB 就切一批(45 批 / 399.1 MB),失败只重推一批
- http.lowSpeedLimit 1000 + http.lowSpeedTime 120:低于 1 KB/s 持续 2 分钟
  就断开,让重试开新连接(实测新连接能拿回 mss 1408,速度提升 2.5 倍)
- --work 支持续推:用 git ls-remote 读远端 main,回卷本地工作区对齐,
  再跳过所有已在 HEAD 中的批次,避免重复上传
- 5 次重试,线性退避 15/30/45/60 秒

另外修正 tests/wilderness-features.test.ts 的 WildernessPiece 导入——
verbatimModuleSyntax 下类型导入必须用 import type。

TAG=agy
CONV=c89513df-cd12-4749-b5bd-5e9f16639391
This commit is contained in:
troytt 2026-09-15 06:33:19 +00:00
parent de384f0744
commit 258be4d608
2 changed files with 229 additions and 29 deletions

View File

@ -149,51 +149,251 @@ npm run publish:packs # orphan 提交 + force push 覆盖本仓库
` `
} }
const remote = process.argv[2] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE /* ------------------------------------------------------------------------- *
* Batch planning
* ------------------------------------------------------------------------- */
/**
* Target size of one push batch, in bytes.
*
* The asset remote sits behind a high-loss cross-border link: measured 13.7%
* TCP retransmission, MSS clamped to 324 bytes by a middlebox, and a congestion
* window pinned at 3–4 segments, which caps a single connection at roughly
* 20 KB/s. A 100 MB push therefore needs ~1.5 h of uninterrupted connection and
* loses everything on the first reset. Batching at ~10 MB keeps each push to a
* few minutes and makes a retry cheap.
*/
const MAX_BATCH_BYTES = 10 * 1024 * 1024
/** How many times to retry one push before giving up. */
const PUSH_ATTEMPTS = 5
/** The order acts are published in, smallest first so the remote is usable early. */
const ACT_ORDER = ['act4', 'act3', 'act5', 'act2', 'act1'] as const
/** One push batch: a set of map directories inside one act. */
interface Batch {
readonly act: string
/** Paths relative to {@link PACKS_DIR}, e.g. `act1/1-act-1-town-townn1`. */
readonly paths: readonly string[]
readonly bytes: number
}
/**
* Total size of a directory tree, in bytes.
*
* @param dir - absolute path.
* @returns bytes, `.git` excluded.
*/
function dirSize(dir: string): number {
let bytes = 0
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.name === '.git') continue
const path = join(dir, entry.name)
if (entry.isDirectory()) bytes += dirSize(path)
else bytes += statSync(path).size
}
return bytes
}
/**
* Split every act into batches of at most {@link MAX_BATCH_BYTES}.
*
* Map directories are never split, so a single oversized map becomes a batch of
* its own. The order within an act is the directory listing order, sorted, so
* the plan is stable across runs — which is what makes resuming safe.
*
* @returns the batches, act by act in {@link ACT_ORDER}.
*/
function planBatches(): Batch[] {
const batches: Batch[] = []
for (const act of ACT_ORDER) {
const actDir = join(PACKS_DIR, act)
if (!existsSync(actDir)) continue
const maps = readdirSync(actDir, { withFileTypes: true })
.filter(entry => entry.isDirectory())
.map(entry => entry.name)
.sort()
let current: string[] = []
let currentBytes = 0
const flush = (): void => {
if (current.length === 0) return
batches.push({ act, paths: current, bytes: currentBytes })
current = []
currentBytes = 0
}
for (const map of maps) {
const bytes = dirSize(join(actDir, map))
if (currentBytes > 0 && currentBytes + bytes > MAX_BATCH_BYTES) flush()
current.push(`${act}/${map}`)
currentBytes += bytes
}
// Loose files directly under the act directory ride along in the last batch.
const loose = readdirSync(actDir, { withFileTypes: true })
.filter(entry => !entry.isDirectory())
.map(entry => `${act}/${entry.name}`)
for (const file of loose) current.push(file)
flush()
}
return batches
}
/* ------------------------------------------------------------------------- *
* Git helpers that tolerate a lossy link
* ------------------------------------------------------------------------- */
/**
* Run one git command and capture stdout instead of inheriting it.
*
* @param args - git arguments.
* @param cwd - directory to run in.
* @returns trimmed stdout, or null when the command failed.
*/
function gitQuery(args: readonly string[], cwd: string): string | null {
try {
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()
} catch {
return null
}
}
/**
* Push `main`, retrying with linear backoff.
*
* `http.lowSpeedLimit`/`http.lowSpeedTime` are what stop a dead connection from
* hanging for hours: git aborts once throughput stays under 1 KB/s for two
* minutes, and the retry opens a fresh connection — which on this link usually
* comes back with a healthy congestion window.
*
* @param remoteUrl - the push target.
* @param cwd - the work repository.
* @param label - what is being pushed, for the log.
* @throws when every attempt fails.
*/
function pushWithRetry(remoteUrl: string, cwd: string, label: string): void {
for (let attempt = 1; attempt <= PUSH_ATTEMPTS; attempt += 1) {
try {
execFileSync('git', ['push', remoteUrl, 'main'], { cwd, stdio: ['ignore', 'inherit', 'inherit'] })
return
} catch {
if (attempt === PUSH_ATTEMPTS) {
throw new PublishError(`推送「${label}」失败:已重试 ${String(PUSH_ATTEMPTS)} 次`)
}
const waitSeconds = attempt * 15
console.log(` ↻ 推送「${label}」第 ${String(attempt)} 次失败,${String(waitSeconds)} 秒后重试…`)
execFileSync('sleep', [String(waitSeconds)], { stdio: 'ignore' })
}
}
}
/**
* Is `path` already committed in the work repository's `HEAD`?
*
* @param path - path relative to the repository root.
* @param cwd - the work repository.
* @returns true when `HEAD` already carries it.
*/
function committed(path: string, cwd: string): boolean {
const listed = gitQuery(['ls-tree', '--name-only', 'HEAD', path], cwd)
return listed !== null && listed !== ''
}
/* ------------------------------------------------------------------------- *
* Driver
* ------------------------------------------------------------------------- */
const args = process.argv.slice(2)
const workFlagIndex = args.indexOf('--work')
const reuseWork = workFlagIndex >= 0 ? args[workFlagIndex + 1] : undefined
const positional = args.filter((value, index) =>
value !== '--work' && index !== workFlagIndex + 1)
const remote = positional[0] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE
const stats = measure() const stats = measure()
const stamp = new Date().toISOString().slice(0, 10) const stamp = new Date().toISOString().slice(0, 10)
const work = mkdtempSync(join(tmpdir(), 'd2-assets-')) const keepWork = reuseWork !== undefined
const work = reuseWork ?? mkdtempSync(join(tmpdir(), 'd2-assets-'))
try { try {
const batches = planBatches()
const totalBytes = batches.reduce((sum, batch) => sum + batch.bytes, 0)
console.log(`图集包:${String(stats.maps)} 块地图 / ${String(stats.files)} 个文件 / ${(stats.bytes / 1048576).toFixed(1)} MB`) console.log(`图集包:${String(stats.maps)} 块地图 / ${String(stats.files)} 个文件 / ${(stats.bytes / 1048576).toFixed(1)} MB`)
console.log(`index.json sha256:${stats.indexSha}`) console.log(`index.json sha256:${stats.indexSha}`)
console.log(`临时仓库:${work}`) console.log(`临时仓库:${work}${keepWork ? '(复用,断点续传)' : ''}`)
console.log(`推送计划:${String(batches.length)} 个批次,单批上限 ${String(MAX_BATCH_BYTES / 1048576)} MB,合计 ${(totalBytes / 1048576).toFixed(1)} MB`)
git(['init', '-q', '-b', 'main'], work) if (!keepWork) git(['init', '-q', '-b', 'main'], work)
// A 500 MB post buffer keeps git from chunking the upload; HTTP/1.1 avoids the
// remote's HTTP/2 stream resets; the low-speed guard turns a stalled
// connection into a fast failure that `pushWithRetry` can recover from.
git(['config', 'http.postBuffer', '524288000'], work) git(['config', 'http.postBuffer', '524288000'], work)
git(['config', 'http.version', 'HTTP/1.1'], work) git(['config', 'http.version', 'HTTP/1.1'], work)
git(['config', 'http.lowSpeedLimit', '1000'], work)
git(['config', 'http.lowSpeedTime', '120'], work)
// PNG atlas pages are already deflate compressed, so delta search is pure cost.
git(['config', 'pack.window', '0'], work)
git(['config', 'core.compression', '1'], work)
// Step 1: Base metadata and index // Resume: rewind the work tree to whatever the remote already has, so a
writeFileSync(join(work, 'README.md'), readme(stats, stamp)) // half-finished oversized commit from an earlier run is discarded rather than
writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n') // re-uploaded.
cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json')) if (keepWork) {
git(['add', '-A'], work) const remoteSha = gitQuery(['ls-remote', remote, 'main'], work)?.split(/\s+/)[0]
git([ if (remoteSha !== undefined && gitQuery(['cat-file', '-e', `${remoteSha}^{commit}`], work) !== null) {
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`, console.log(`远端 main 位于 ${remoteSha.slice(0, 7)},回卷工作区以对齐…`)
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`, git(['reset', '--hard', '-q', remoteSha], work)
'commit', '-q', '-m', git(['clean', '-qfd'], work)
`烘焙图集包 ${stamp} [0/5]:索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`, }
], work) }
console.log(`推送到 ${remote}(force,覆盖历史,初始化基线)…`)
git(['push', '--force', remote, 'main'], work)
// Step 2..6: Push acts in small incremental batches to stay under HTTP timeout limits // Step 0: metadata and index.
const acts = ['act4', 'act3', 'act5', 'act2', 'act1'] as const if (!committed('index.json', work)) {
for (let i = 0; i < acts.length; i++) { writeFileSync(join(work, 'README.md'), readme(stats, stamp))
const act = acts[i]! writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n')
console.log(`添加并提交 ${act} [${i + 1}/${acts.length}]…`) cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json'))
cpSync(join(PACKS_DIR, act), join(work, act), { recursive: true }) git(['add', '-A'], work)
git(['add', act], work)
git([ git([
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`, '-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`,
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`, '-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`,
'commit', '-q', '-m', 'commit', '-q', '-m',
`烘焙图集包 ${stamp} [${i + 1}/5]:${act} 图集`, `烘焙图集包 ${stamp} [0]:索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`,
], work) ], work)
console.log(`推送到 ${remote}(增量推送 ${act})…`) console.log(`推送到 ${remote}(force,覆盖历史,初始化基线)…`)
git(['push', remote, 'main'], work) git(['push', '--force', remote, 'main'], work)
} else {
console.log('索引与元数据已在远端,跳过。')
} }
// Steps 1..N: one small batch per push.
let sentBytes = 0
for (let i = 0; i < batches.length; i += 1) {
const batch = batches[i]!
const label = `${String(i + 1)}/${String(batches.length)} ${batch.act} (${(batch.bytes / 1048576).toFixed(1)} MB)`
sentBytes += batch.bytes
if (batch.paths.every(path => committed(path, work))) {
console.log(`[${label}] 已在远端,跳过。`)
continue
}
for (const path of batch.paths) {
cpSync(join(PACKS_DIR, path), join(work, path), { recursive: true })
}
git(['add', ...batch.paths], work)
git([
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`,
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`,
'commit', '-q', '-m',
`烘焙图集包 ${stamp} [${String(i + 1)}/${String(batches.length)}]:${batch.act} ${String(batch.paths.length)} 项`,
], work)
console.log(`[${label}] 推送中…(累计 ${(sentBytes / 1048576).toFixed(1)}/${(totalBytes / 1048576).toFixed(1)} MB)`)
pushWithRetry(remote, work, label)
}
console.log('完成。使用方:git clone ' + remote + ' samples/d2-packs') console.log('完成。使用方:git clone ' + remote + ' samples/d2-packs')
} finally { } finally {
rmSync(work, { recursive: true, force: true }) if (!keepWork) rmSync(work, { recursive: true, force: true })
else console.log(`工作区保留在 ${work}(下次可用 --work ${work} 续传)`)
} }

View File

@ -5,8 +5,8 @@ import {
getPerimeterOpenings, getPerimeterOpenings,
SPECIAL_PRESETS_BY_LEVEL, SPECIAL_PRESETS_BY_LEVEL,
generateWilderness, generateWilderness,
WildernessPiece,
} from "../src/game/wilderness.ts" } from "../src/game/wilderness.ts"
import type { WildernessPiece } from "../src/game/wilderness.ts"
import { Rng } from "../src/game/rng.ts" import { Rng } from "../src/game/rng.ts"
import type { Ds1, Ds1Cell } from "../src/formats/ds1.ts" import type { Ds1, Ds1Cell } from "../src/formats/ds1.ts"