fix: batch Git LFS uploads

This commit is contained in:
Tom Boullay
2026-04-27 23:17:56 +02:00
parent 799e61c92e
commit b084c0e20e
2 changed files with 60 additions and 14 deletions
+47
View File
@@ -3,6 +3,8 @@ import { Octokit } from '@octokit/rest'
import { LFS_EXTENSIONS } from './constants'
import type { PushFile, RemoteFile } from './types'
const LFS_BATCH_SIZE = 100
// ---------------------------------------------------------------------------
// Octokit helpers
// ---------------------------------------------------------------------------
@@ -55,6 +57,24 @@ function parseLfsPointer(content: string): { oid: string; size: number } | null
return { oid: oidMatch[1], size: parseInt(sizeMatch[1], 10) }
}
function formatElapsed(startedAt: number) {
return `${((performance.now() - startedAt) / 1000).toFixed(1)}s`
}
function logInfo(step: string, action: string, startedAt: number, details?: Record<string, unknown>) {
console.info(`[INFO] ${step} -> ${action} | Timer: ${formatElapsed(startedAt)}`, details || '')
}
function chunkArray<T>(items: T[], size: number) {
const chunks: T[][] = []
for (let i = 0; i < items.length; i += size) {
chunks.push(items.slice(i, i + size))
}
return chunks
}
interface LfsObject {
oid: string
size: number
@@ -76,6 +96,25 @@ async function uploadToLfs(
): Promise<void> {
if (objects.length === 0) return
const batches = chunkArray(objects, LFS_BATCH_SIZE)
for (let i = 0; i < batches.length; i++) {
await uploadToLfsBatch(owner, repo, batches[i], i + 1, batches.length)
}
}
async function uploadToLfsBatch(
owner: string,
repo: string,
objects: LfsObject[],
batchNumber: number,
totalBatches: number,
): Promise<void> {
const startedAt = performance.now()
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} started`, startedAt, {
objects: objects.length,
})
const token = process.env.GITHUB_TOKEN!
const lfsUrl = `https://github.com/${owner}/${repo}.git/info/lfs/objects/batch`
@@ -96,6 +135,10 @@ async function uploadToLfs(
if (!batchRes.ok) {
const text = await batchRes.text()
console.error(`[ERROR] Git LFS -> Batch ${batchNumber}/${totalBatches} failed | Timer: ${formatElapsed(startedAt)}`, {
objects: objects.length,
status: batchRes.status,
})
throw new Error(`LFS batch request failed (${batchRes.status}): ${text}`)
}
@@ -165,6 +208,10 @@ async function uploadToLfs(
}
}
}
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} done`, startedAt, {
objects: objects.length,
})
}
// ---------------------------------------------------------------------------