fix(publish): fine-grained batched push with resume
跨境链路对长连接做 MSS 钳制：实测 pmtu 1460 但 mss 被压到 324，重传率 13.7%，cwnd 触底到 3-4，有效带宽只剩 ~20 KB/s。原来按 act 分 5 批推送， 单批最大 117 MB，在这种链路上必然超时，且失败后整批重来。 改为： - 按地图目录累计到 ~10 MB 就切一批（45 批 / 399.1 MB），失败只重推一批 - http.lowSpeedLimit 1000 + http.lowSpeedTime 120：低于 1 KB/s 持续 2 分钟 就断开，让重试开新连接（实测新连接能拿回 mss 1408，速度提升 2.5 倍） - --work 支持续推：用 git ls-remote 读远端 main，回卷本地工作区对齐， 再跳过所有已在 HEAD 中的批次，避免重复上传 - 5 次重试，线性退避 15/30/45/60 秒 另外修正 tests/wilderness-features.test.ts 的 WildernessPiece 导入—— verbatimModuleSyntax 下类型导入必须用 import type。 TAG=agy CONV=c89513df-cd12-4749-b5bd-5e9f16639391
This commit is contained in:
parent
de384f0744
commit
258be4d608
|
|
@ -149,51 +149,251 @@ npm run publish:packs # orphan 提交 + force push 覆盖本仓库
|
|||
`
|
||||
}
|
||||
|
||||
const remote = process.argv[2] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE
|
||||
/* ------------------------------------------------------------------------- *
|
||||
* Batch planning
|
||||
* ------------------------------------------------------------------------- */
|
||||
|
||||
/**
|
||||
* Target size of one push batch, in bytes.
|
||||
*
|
||||
* The asset remote sits behind a high-loss cross-border link: measured 13.7%
|
||||
* TCP retransmission, MSS clamped to 324 bytes by a middlebox, and a congestion
|
||||
* window pinned at 3–4 segments, which caps a single connection at roughly
|
||||
* 20 KB/s. A 100 MB push therefore needs ~1.5 h of uninterrupted connection and
|
||||
* loses everything on the first reset. Batching at ~10 MB keeps each push to a
|
||||
* few minutes and makes a retry cheap.
|
||||
*/
|
||||
const MAX_BATCH_BYTES = 10 * 1024 * 1024
|
||||
|
||||
/** How many times to retry one push before giving up. */
|
||||
const PUSH_ATTEMPTS = 5
|
||||
|
||||
/** The order acts are published in, smallest first so the remote is usable early. */
|
||||
const ACT_ORDER = ['act4', 'act3', 'act5', 'act2', 'act1'] as const
|
||||
|
||||
/** One push batch: a set of map directories inside one act. */
|
||||
interface Batch {
|
||||
readonly act: string
|
||||
/** Paths relative to {@link PACKS_DIR}, e.g. `act1/1-act-1-town-townn1`. */
|
||||
readonly paths: readonly string[]
|
||||
readonly bytes: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Total size of a directory tree, in bytes.
|
||||
*
|
||||
* @param dir - absolute path.
|
||||
* @returns bytes, `.git` excluded.
|
||||
*/
|
||||
function dirSize(dir: string): number {
|
||||
let bytes = 0
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === '.git') continue
|
||||
const path = join(dir, entry.name)
|
||||
if (entry.isDirectory()) bytes += dirSize(path)
|
||||
else bytes += statSync(path).size
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
/**
|
||||
* Split every act into batches of at most {@link MAX_BATCH_BYTES}.
|
||||
*
|
||||
* Map directories are never split, so a single oversized map becomes a batch of
|
||||
* its own. The order within an act is the directory listing order, sorted, so
|
||||
* the plan is stable across runs — which is what makes resuming safe.
|
||||
*
|
||||
* @returns the batches, act by act in {@link ACT_ORDER}.
|
||||
*/
|
||||
function planBatches(): Batch[] {
|
||||
const batches: Batch[] = []
|
||||
for (const act of ACT_ORDER) {
|
||||
const actDir = join(PACKS_DIR, act)
|
||||
if (!existsSync(actDir)) continue
|
||||
const maps = readdirSync(actDir, { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory())
|
||||
.map(entry => entry.name)
|
||||
.sort()
|
||||
|
||||
let current: string[] = []
|
||||
let currentBytes = 0
|
||||
const flush = (): void => {
|
||||
if (current.length === 0) return
|
||||
batches.push({ act, paths: current, bytes: currentBytes })
|
||||
current = []
|
||||
currentBytes = 0
|
||||
}
|
||||
for (const map of maps) {
|
||||
const bytes = dirSize(join(actDir, map))
|
||||
if (currentBytes > 0 && currentBytes + bytes > MAX_BATCH_BYTES) flush()
|
||||
current.push(`${act}/${map}`)
|
||||
currentBytes += bytes
|
||||
}
|
||||
// Loose files directly under the act directory ride along in the last batch.
|
||||
const loose = readdirSync(actDir, { withFileTypes: true })
|
||||
.filter(entry => !entry.isDirectory())
|
||||
.map(entry => `${act}/${entry.name}`)
|
||||
for (const file of loose) current.push(file)
|
||||
flush()
|
||||
}
|
||||
return batches
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------- *
|
||||
* Git helpers that tolerate a lossy link
|
||||
* ------------------------------------------------------------------------- */
|
||||
|
||||
/**
|
||||
* Run one git command and capture stdout instead of inheriting it.
|
||||
*
|
||||
* @param args - git arguments.
|
||||
* @param cwd - directory to run in.
|
||||
* @returns trimmed stdout, or null when the command failed.
|
||||
*/
|
||||
function gitQuery(args: readonly string[], cwd: string): string | null {
|
||||
try {
|
||||
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Push `main`, retrying with linear backoff.
|
||||
*
|
||||
* `http.lowSpeedLimit`/`http.lowSpeedTime` are what stop a dead connection from
|
||||
* hanging for hours: git aborts once throughput stays under 1 KB/s for two
|
||||
* minutes, and the retry opens a fresh connection — which on this link usually
|
||||
* comes back with a healthy congestion window.
|
||||
*
|
||||
* @param remoteUrl - the push target.
|
||||
* @param cwd - the work repository.
|
||||
* @param label - what is being pushed, for the log.
|
||||
* @throws when every attempt fails.
|
||||
*/
|
||||
function pushWithRetry(remoteUrl: string, cwd: string, label: string): void {
|
||||
for (let attempt = 1; attempt <= PUSH_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
execFileSync('git', ['push', remoteUrl, 'main'], { cwd, stdio: ['ignore', 'inherit', 'inherit'] })
|
||||
return
|
||||
} catch {
|
||||
if (attempt === PUSH_ATTEMPTS) {
|
||||
throw new PublishError(`推送「${label}」失败：已重试 ${String(PUSH_ATTEMPTS)} 次`)
|
||||
}
|
||||
const waitSeconds = attempt * 15
|
||||
console.log(` ↻ 推送「${label}」第 ${String(attempt)} 次失败，${String(waitSeconds)} 秒后重试…`)
|
||||
execFileSync('sleep', [String(waitSeconds)], { stdio: 'ignore' })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Is `path` already committed in the work repository's `HEAD`?
|
||||
*
|
||||
* @param path - path relative to the repository root.
|
||||
* @param cwd - the work repository.
|
||||
* @returns true when `HEAD` already carries it.
|
||||
*/
|
||||
function committed(path: string, cwd: string): boolean {
|
||||
const listed = gitQuery(['ls-tree', '--name-only', 'HEAD', path], cwd)
|
||||
return listed !== null && listed !== ''
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------- *
|
||||
* Driver
|
||||
* ------------------------------------------------------------------------- */
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const workFlagIndex = args.indexOf('--work')
|
||||
const reuseWork = workFlagIndex >= 0 ? args[workFlagIndex + 1] : undefined
|
||||
const positional = args.filter((value, index) =>
|
||||
value !== '--work' && index !== workFlagIndex + 1)
|
||||
|
||||
const remote = positional[0] ?? process.env['D2_ASSETS_REMOTE'] ?? DEFAULT_REMOTE
|
||||
const stats = measure()
|
||||
const stamp = new Date().toISOString().slice(0, 10)
|
||||
const work = mkdtempSync(join(tmpdir(), 'd2-assets-'))
|
||||
const keepWork = reuseWork !== undefined
|
||||
const work = reuseWork ?? mkdtempSync(join(tmpdir(), 'd2-assets-'))
|
||||
|
||||
try {
|
||||
const batches = planBatches()
|
||||
const totalBytes = batches.reduce((sum, batch) => sum + batch.bytes, 0)
|
||||
console.log(`图集包：${String(stats.maps)} 块地图 / ${String(stats.files)} 个文件 / ${(stats.bytes / 1048576).toFixed(1)} MB`)
|
||||
console.log(`index.json sha256：${stats.indexSha}`)
|
||||
console.log(`临时仓库：${work}`)
|
||||
console.log(`临时仓库：${work}${keepWork ? '（复用，断点续传）' : ''}`)
|
||||
console.log(`推送计划：${String(batches.length)} 个批次，单批上限 ${String(MAX_BATCH_BYTES / 1048576)} MB，合计 ${(totalBytes / 1048576).toFixed(1)} MB`)
|
||||
|
||||
git(['init', '-q', '-b', 'main'], work)
|
||||
if (!keepWork) git(['init', '-q', '-b', 'main'], work)
|
||||
// A 500 MB post buffer keeps git from chunking the upload; HTTP/1.1 avoids the
|
||||
// remote's HTTP/2 stream resets; the low-speed guard turns a stalled
|
||||
// connection into a fast failure that `pushWithRetry` can recover from.
|
||||
git(['config', 'http.postBuffer', '524288000'], work)
|
||||
git(['config', 'http.version', 'HTTP/1.1'], work)
|
||||
git(['config', 'http.lowSpeedLimit', '1000'], work)
|
||||
git(['config', 'http.lowSpeedTime', '120'], work)
|
||||
// PNG atlas pages are already deflate compressed, so delta search is pure cost.
|
||||
git(['config', 'pack.window', '0'], work)
|
||||
git(['config', 'core.compression', '1'], work)
|
||||
|
||||
// Step 1: Base metadata and index
|
||||
writeFileSync(join(work, 'README.md'), readme(stats, stamp))
|
||||
writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n')
|
||||
cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json'))
|
||||
git(['add', '-A'], work)
|
||||
git([
|
||||
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`,
|
||||
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`,
|
||||
'commit', '-q', '-m',
|
||||
`烘焙图集包 ${stamp} [0/5]：索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`,
|
||||
], work)
|
||||
console.log(`推送到 ${remote}（force，覆盖历史，初始化基线）…`)
|
||||
git(['push', '--force', remote, 'main'], work)
|
||||
// Resume: rewind the work tree to whatever the remote already has, so a
|
||||
// half-finished oversized commit from an earlier run is discarded rather than
|
||||
// re-uploaded.
|
||||
if (keepWork) {
|
||||
const remoteSha = gitQuery(['ls-remote', remote, 'main'], work)?.split(/\s+/)[0]
|
||||
if (remoteSha !== undefined && gitQuery(['cat-file', '-e', `${remoteSha}^{commit}`], work) !== null) {
|
||||
console.log(`远端 main 位于 ${remoteSha.slice(0, 7)}，回卷工作区以对齐…`)
|
||||
git(['reset', '--hard', '-q', remoteSha], work)
|
||||
git(['clean', '-qfd'], work)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2..6: Push acts in small incremental batches to stay under HTTP timeout limits
|
||||
const acts = ['act4', 'act3', 'act5', 'act2', 'act1'] as const
|
||||
for (let i = 0; i < acts.length; i++) {
|
||||
const act = acts[i]!
|
||||
console.log(`添加并提交 ${act} [${i + 1}/${acts.length}]…`)
|
||||
cpSync(join(PACKS_DIR, act), join(work, act), { recursive: true })
|
||||
git(['add', act], work)
|
||||
// Step 0: metadata and index.
|
||||
if (!committed('index.json', work)) {
|
||||
writeFileSync(join(work, 'README.md'), readme(stats, stamp))
|
||||
writeFileSync(join(work, '.gitattributes'), '*.png binary -diff\n*.json text eol=lf\n')
|
||||
cpSync(join(PACKS_DIR, 'index.json'), join(work, 'index.json'))
|
||||
git(['add', '-A'], work)
|
||||
git([
|
||||
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`,
|
||||
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`,
|
||||
'commit', '-q', '-m',
|
||||
`烘焙图集包 ${stamp} [${i + 1}/5]：${act} 图集`,
|
||||
`烘焙图集包 ${stamp} [0]：索引与元数据\n\nindex.json sha256: ${stats.indexSha}\n\n由 troytt/diablo2-web 的 npm run pack:data 生成。`,
|
||||
], work)
|
||||
console.log(`推送到 ${remote}（增量推送 ${act}）…`)
|
||||
git(['push', remote, 'main'], work)
|
||||
console.log(`推送到 ${remote}（force，覆盖历史，初始化基线）…`)
|
||||
git(['push', '--force', remote, 'main'], work)
|
||||
} else {
|
||||
console.log('索引与元数据已在远端，跳过。')
|
||||
}
|
||||
|
||||
// Steps 1..N: one small batch per push.
|
||||
let sentBytes = 0
|
||||
for (let i = 0; i < batches.length; i += 1) {
|
||||
const batch = batches[i]!
|
||||
const label = `${String(i + 1)}/${String(batches.length)} ${batch.act} (${(batch.bytes / 1048576).toFixed(1)} MB)`
|
||||
sentBytes += batch.bytes
|
||||
|
||||
if (batch.paths.every(path => committed(path, work))) {
|
||||
console.log(`[${label}] 已在远端，跳过。`)
|
||||
continue
|
||||
}
|
||||
|
||||
for (const path of batch.paths) {
|
||||
cpSync(join(PACKS_DIR, path), join(work, path), { recursive: true })
|
||||
}
|
||||
git(['add', ...batch.paths], work)
|
||||
git([
|
||||
'-c', `user.name=${process.env['GIT_AUTHOR_NAME'] ?? 'Dao Tao'}`,
|
||||
'-c', `user.email=${process.env['GIT_AUTHOR_EMAIL'] ?? 'taodao@google.com'}`,
|
||||
'commit', '-q', '-m',
|
||||
`烘焙图集包 ${stamp} [${String(i + 1)}/${String(batches.length)}]：${batch.act} ${String(batch.paths.length)} 项`,
|
||||
], work)
|
||||
console.log(`[${label}] 推送中…（累计 ${(sentBytes / 1048576).toFixed(1)}/${(totalBytes / 1048576).toFixed(1)} MB）`)
|
||||
pushWithRetry(remote, work, label)
|
||||
}
|
||||
|
||||
console.log('完成。使用方：git clone ' + remote + ' samples/d2-packs')
|
||||
} finally {
|
||||
rmSync(work, { recursive: true, force: true })
|
||||
if (!keepWork) rmSync(work, { recursive: true, force: true })
|
||||
else console.log(`工作区保留在 ${work}（下次可用 --work ${work} 续传）`)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import {
|
|||
getPerimeterOpenings,
|
||||
SPECIAL_PRESETS_BY_LEVEL,
|
||||
generateWilderness,
|
||||
WildernessPiece,
|
||||
} from "../src/game/wilderness.ts"
|
||||
import type { WildernessPiece } from "../src/game/wilderness.ts"
|
||||
import { Rng } from "../src/game/rng.ts"
|
||||
import type { Ds1, Ds1Cell } from "../src/formats/ds1.ts"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue