fix: batch Git LFS uploads
This commit is contained in:
@@ -3,6 +3,8 @@ import { Octokit } from '@octokit/rest'
|
||||
import { LFS_EXTENSIONS } from './constants'
|
||||
import type { PushFile, RemoteFile } from './types'
|
||||
|
||||
const LFS_BATCH_SIZE = 100
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Octokit helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -55,6 +57,24 @@ function parseLfsPointer(content: string): { oid: string; size: number } | null
|
||||
return { oid: oidMatch[1], size: parseInt(sizeMatch[1], 10) }
|
||||
}
|
||||
|
||||
function formatElapsed(startedAt: number) {
|
||||
return `${((performance.now() - startedAt) / 1000).toFixed(1)}s`
|
||||
}
|
||||
|
||||
function logInfo(step: string, action: string, startedAt: number, details?: Record<string, unknown>) {
|
||||
console.info(`[INFO] ${step} -> ${action} | Timer: ${formatElapsed(startedAt)}`, details || '')
|
||||
}
|
||||
|
||||
function chunkArray<T>(items: T[], size: number) {
|
||||
const chunks: T[][] = []
|
||||
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
chunks.push(items.slice(i, i + size))
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
interface LfsObject {
|
||||
oid: string
|
||||
size: number
|
||||
@@ -76,6 +96,25 @@ async function uploadToLfs(
|
||||
): Promise<void> {
|
||||
if (objects.length === 0) return
|
||||
|
||||
const batches = chunkArray(objects, LFS_BATCH_SIZE)
|
||||
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
await uploadToLfsBatch(owner, repo, batches[i], i + 1, batches.length)
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadToLfsBatch(
|
||||
owner: string,
|
||||
repo: string,
|
||||
objects: LfsObject[],
|
||||
batchNumber: number,
|
||||
totalBatches: number,
|
||||
): Promise<void> {
|
||||
const startedAt = performance.now()
|
||||
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} started`, startedAt, {
|
||||
objects: objects.length,
|
||||
})
|
||||
|
||||
const token = process.env.GITHUB_TOKEN!
|
||||
const lfsUrl = `https://github.com/${owner}/${repo}.git/info/lfs/objects/batch`
|
||||
|
||||
@@ -96,6 +135,10 @@ async function uploadToLfs(
|
||||
|
||||
if (!batchRes.ok) {
|
||||
const text = await batchRes.text()
|
||||
console.error(`[ERROR] Git LFS -> Batch ${batchNumber}/${totalBatches} failed | Timer: ${formatElapsed(startedAt)}`, {
|
||||
objects: objects.length,
|
||||
status: batchRes.status,
|
||||
})
|
||||
throw new Error(`LFS batch request failed (${batchRes.status}): ${text}`)
|
||||
}
|
||||
|
||||
@@ -165,6 +208,10 @@ async function uploadToLfs(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} done`, startedAt, {
|
||||
objects: objects.length,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user