Tom Boullay
429 lines
12 KiB
TypeScript
429 lines
12 KiB
TypeScript
import { createHash } from 'crypto'
|
|
import { Octokit } from '@octokit/rest'
|
|
import { LFS_EXTENSIONS } from './constants'
|
|
import { isRecord } from './guards'
|
|
import type { PushFile, RemoteFile } from './types'
|
|
|
|
const LFS_BATCH_SIZE = 100
|
|
|
|
type LogDetails = Record<string, string | number | boolean | undefined>
|
|
|
|
function isHttpError(err: unknown): err is { status: number } {
|
|
return isRecord(err) && typeof err.status === 'number'
|
|
}
|
|
|
|
function getOctokit(): Octokit {
|
|
const token = process.env.GITHUB_TOKEN
|
|
if (!token) throw new Error('GITHUB_TOKEN non configure')
|
|
return new Octokit({ auth: token })
|
|
}
|
|
|
|
function parseRepoUrl(): { owner: string; repo: string } {
|
|
const url = process.env.GIT_REPO_URL?.trim()
|
|
if (!url) throw new Error('GIT_REPO_URL non configure')
|
|
|
|
const cleanRepoName = (repo: string) => repo.replace(/\/+$/, '').replace(/\.git$/, '')
|
|
const shortMatch = url.match(/^([^/\s:]+)\/([^/\s]+)$/)
|
|
if (shortMatch) {
|
|
return { owner: shortMatch[1], repo: cleanRepoName(shortMatch[2]) }
|
|
}
|
|
|
|
const sshMatch = url.match(/github\.com:([^/\s]+)\/(.+)$/)
|
|
if (sshMatch) {
|
|
return { owner: sshMatch[1], repo: cleanRepoName(sshMatch[2]) }
|
|
}
|
|
|
|
if (URL.canParse(url)) {
|
|
const parsed = new URL(url)
|
|
const pathParts = parsed.pathname
|
|
.replace(/^\/+|\/+$/g, '')
|
|
.split('/')
|
|
.filter(Boolean)
|
|
|
|
if (parsed.hostname === 'github.com' && pathParts.length >= 2) {
|
|
return { owner: pathParts[0], repo: cleanRepoName(pathParts[1]) }
|
|
}
|
|
}
|
|
|
|
throw new Error(`Format GIT_REPO_URL invalide: "${url}"`)
|
|
}
|
|
|
|
function isLfsFile(filePath: string): boolean {
|
|
const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase()
|
|
return LFS_EXTENSIONS.has(ext)
|
|
}
|
|
|
|
function buildLfsPointer(sha256: string, size: number): string {
|
|
return `version https://git-lfs.github.com/spec/v1\noid sha256:${sha256}\nsize ${size}\n`
|
|
}
|
|
|
|
function parseLfsPointer(content: string): { oid: string; size: number } | null {
|
|
if (!content.startsWith('version https://git-lfs.github.com/spec/v1')) return null
|
|
const sizeMatch = content.match(/^size (\d+)$/m)
|
|
const oidMatch = content.match(/^oid sha256:([a-f0-9]{64})$/m)
|
|
if (!sizeMatch || !oidMatch) return null
|
|
return { oid: oidMatch[1], size: parseInt(sizeMatch[1], 10) }
|
|
}
|
|
|
|
function formatElapsed(startedAt: number) {
|
|
return `${((performance.now() - startedAt) / 1000).toFixed(1)}s`
|
|
}
|
|
|
|
function logInfo(step: string, action: string, startedAt: number, details?: LogDetails) {
|
|
console.info(`[INFO] ${step} -> ${action} | Timer: ${formatElapsed(startedAt)}`, details || '')
|
|
}
|
|
|
|
function chunkArray<T>(items: T[], size: number) {
|
|
const chunks: T[][] = []
|
|
|
|
for (let i = 0; i < items.length; i += size) {
|
|
chunks.push(items.slice(i, i + size))
|
|
}
|
|
|
|
return chunks
|
|
}
|
|
|
|
interface LfsObject {
|
|
oid: string
|
|
size: number
|
|
contentBase64: string
|
|
}
|
|
|
|
interface LfsBatchAction {
|
|
href: string
|
|
header?: Record<string, string>
|
|
}
|
|
|
|
interface LfsBatchObject {
|
|
oid: string
|
|
size: number
|
|
actions?: {
|
|
upload?: LfsBatchAction
|
|
verify?: LfsBatchAction
|
|
}
|
|
error?: { code: number; message: string }
|
|
}
|
|
|
|
function isStringRecord(value: unknown): value is Record<string, string> {
|
|
return isRecord(value) && Object.values(value).every((entry) => typeof entry === 'string')
|
|
}
|
|
|
|
function parseLfsAction(value: unknown): LfsBatchAction | undefined {
|
|
if (!isRecord(value) || typeof value.href !== 'string') return undefined
|
|
return {
|
|
href: value.href,
|
|
header: isStringRecord(value.header) ? value.header : undefined,
|
|
}
|
|
}
|
|
|
|
function parseLfsBatchObject(value: unknown): LfsBatchObject | null {
|
|
if (!isRecord(value) || typeof value.oid !== 'string' || typeof value.size !== 'number') return null
|
|
|
|
const actions = isRecord(value.actions)
|
|
? {
|
|
upload: parseLfsAction(value.actions.upload),
|
|
verify: parseLfsAction(value.actions.verify),
|
|
}
|
|
: undefined
|
|
|
|
const error = isRecord(value.error) && typeof value.error.code === 'number' && typeof value.error.message === 'string'
|
|
? { code: value.error.code, message: value.error.message }
|
|
: undefined
|
|
|
|
return { oid: value.oid, size: value.size, actions, error }
|
|
}
|
|
|
|
function parseLfsBatchResponse(value: unknown): LfsBatchObject[] {
|
|
if (!isRecord(value) || !Array.isArray(value.objects)) {
|
|
throw new Error('LFS batch response invalide')
|
|
}
|
|
|
|
const objects: LfsBatchObject[] = []
|
|
for (const object of value.objects) {
|
|
const parsed = parseLfsBatchObject(object)
|
|
if (!parsed) {
|
|
throw new Error('LFS batch object invalide')
|
|
}
|
|
objects.push(parsed)
|
|
}
|
|
|
|
return objects
|
|
}
|
|
|
|
async function uploadToLfs(
|
|
owner: string,
|
|
repo: string,
|
|
objects: LfsObject[],
|
|
): Promise<void> {
|
|
if (objects.length === 0) return
|
|
|
|
const batches = chunkArray(objects, LFS_BATCH_SIZE)
|
|
|
|
for (let i = 0; i < batches.length; i++) {
|
|
await uploadToLfsBatch(owner, repo, batches[i], i + 1, batches.length)
|
|
}
|
|
}
|
|
|
|
async function uploadToLfsBatch(
|
|
owner: string,
|
|
repo: string,
|
|
objects: LfsObject[],
|
|
batchNumber: number,
|
|
totalBatches: number,
|
|
): Promise<void> {
|
|
const startedAt = performance.now()
|
|
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} started`, startedAt, {
|
|
objects: objects.length,
|
|
})
|
|
|
|
const token = process.env.GITHUB_TOKEN!
|
|
const lfsUrl = `https://github.com/${owner}/${repo}.git/info/lfs/objects/batch`
|
|
|
|
const batchRes = await fetch(lfsUrl, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Accept': 'application/vnd.git-lfs+json',
|
|
'Content-Type': 'application/vnd.git-lfs+json',
|
|
'Authorization': `token ${token}`,
|
|
},
|
|
body: JSON.stringify({
|
|
operation: 'upload',
|
|
transfers: ['basic'],
|
|
objects: objects.map((o) => ({ oid: o.oid, size: o.size })),
|
|
}),
|
|
})
|
|
|
|
if (!batchRes.ok) {
|
|
const text = await batchRes.text()
|
|
console.error(`[ERROR] Git LFS -> Batch ${batchNumber}/${totalBatches} failed | Timer: ${formatElapsed(startedAt)}`, {
|
|
objects: objects.length,
|
|
status: batchRes.status,
|
|
})
|
|
throw new Error(`LFS batch request failed (${batchRes.status}): ${text}`)
|
|
}
|
|
|
|
const batchObjects = parseLfsBatchResponse(await batchRes.json())
|
|
|
|
const objectMap = new Map(objects.map((o) => [o.oid, o]))
|
|
|
|
for (const obj of batchObjects) {
|
|
if (obj.error) {
|
|
throw new Error(`LFS error for ${obj.oid}: ${obj.error.message} (${obj.error.code})`)
|
|
}
|
|
|
|
if (!obj.actions?.upload) continue
|
|
|
|
const local = objectMap.get(obj.oid)
|
|
if (!local) continue
|
|
|
|
const uploadAction = obj.actions.upload
|
|
const headers: Record<string, string> = {
|
|
'Content-Type': 'application/octet-stream',
|
|
...uploadAction.header,
|
|
}
|
|
|
|
const body = Buffer.from(local.contentBase64, 'base64')
|
|
|
|
const uploadRes = await fetch(uploadAction.href, {
|
|
method: 'PUT',
|
|
headers,
|
|
body,
|
|
})
|
|
|
|
if (!uploadRes.ok) {
|
|
const text = await uploadRes.text()
|
|
throw new Error(`LFS upload failed for ${obj.oid} (${uploadRes.status}): ${text}`)
|
|
}
|
|
|
|
if (obj.actions.verify) {
|
|
const verifyAction = obj.actions.verify
|
|
const verifyHeaders: Record<string, string> = {
|
|
'Accept': 'application/vnd.git-lfs+json',
|
|
'Content-Type': 'application/vnd.git-lfs+json',
|
|
...verifyAction.header,
|
|
}
|
|
|
|
const verifyRes = await fetch(verifyAction.href, {
|
|
method: 'POST',
|
|
headers: verifyHeaders,
|
|
body: JSON.stringify({ oid: obj.oid, size: obj.size }),
|
|
})
|
|
|
|
if (!verifyRes.ok) {
|
|
const text = await verifyRes.text()
|
|
throw new Error(`LFS verify failed for ${obj.oid} (${verifyRes.status}): ${text}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} done`, startedAt, {
|
|
objects: objects.length,
|
|
})
|
|
}
|
|
|
|
export async function getRemoteFolder(
|
|
folderPath: string,
|
|
): Promise<{ exists: boolean; files: RemoteFile[] }> {
|
|
const octokit = getOctokit()
|
|
const { owner, repo } = parseRepoUrl()
|
|
const branch = process.env.GIT_BRANCH ?? 'main'
|
|
|
|
try {
|
|
const { data } = await octokit.repos.getContent({
|
|
owner,
|
|
repo,
|
|
path: folderPath,
|
|
ref: branch,
|
|
})
|
|
|
|
if (!Array.isArray(data)) {
|
|
return { exists: false, files: [] }
|
|
}
|
|
|
|
const files: RemoteFile[] = await Promise.all(
|
|
data.map(async (f): Promise<RemoteFile> => {
|
|
if (!isLfsFile(f.name) || f.size > 1024) {
|
|
return { name: f.name, size: f.size }
|
|
}
|
|
|
|
try {
|
|
const { data: fileData } = await octokit.repos.getContent({
|
|
owner,
|
|
repo,
|
|
path: `${folderPath}/${f.name}`,
|
|
ref: branch,
|
|
})
|
|
|
|
if (!Array.isArray(fileData) && 'content' in fileData && fileData.content) {
|
|
const content = Buffer.from(fileData.content, 'base64').toString('utf-8')
|
|
const pointer = parseLfsPointer(content)
|
|
if (pointer) {
|
|
return { name: f.name, size: pointer.size }
|
|
}
|
|
}
|
|
} catch (err) {
|
|
if (!isHttpError(err) || err.status !== 404) throw err
|
|
}
|
|
|
|
return { name: f.name, size: f.size }
|
|
}),
|
|
)
|
|
|
|
return { exists: true, files }
|
|
} catch (err: unknown) {
|
|
if (isHttpError(err) && err.status === 404) {
|
|
return { exists: false, files: [] }
|
|
}
|
|
throw err
|
|
}
|
|
}
|
|
|
|
export async function pushAllToGitHub(
|
|
files: PushFile[],
|
|
deletePaths: string[],
|
|
commitMessage: string,
|
|
): Promise<{ commitUrl: string }> {
|
|
const octokit = getOctokit()
|
|
const { owner, repo } = parseRepoUrl()
|
|
const branch = process.env.GIT_BRANCH ?? 'main'
|
|
|
|
const lfsFiles: { path: string; contentBase64: string; oid: string; size: number }[] = []
|
|
const regularFiles: PushFile[] = []
|
|
|
|
for (const f of files) {
|
|
if (isLfsFile(f.path)) {
|
|
const buf = Buffer.from(f.contentBase64, 'base64')
|
|
const oid = createHash('sha256').update(buf).digest('hex')
|
|
lfsFiles.push({ ...f, oid, size: buf.length })
|
|
} else {
|
|
regularFiles.push(f)
|
|
}
|
|
}
|
|
|
|
if (lfsFiles.length > 0) {
|
|
await uploadToLfs(
|
|
owner,
|
|
repo,
|
|
lfsFiles.map((f) => ({ oid: f.oid, size: f.size, contentBase64: f.contentBase64 })),
|
|
)
|
|
}
|
|
|
|
const { data: ref } = await octokit.git.getRef({
|
|
owner,
|
|
repo,
|
|
ref: `heads/${branch}`,
|
|
})
|
|
const latestCommitSha = ref.object.sha
|
|
|
|
const { data: commit } = await octokit.git.getCommit({
|
|
owner,
|
|
repo,
|
|
commit_sha: latestCommitSha,
|
|
})
|
|
|
|
const allFiles = [...regularFiles, ...lfsFiles]
|
|
|
|
const blobResults = await Promise.all(
|
|
allFiles.map((f) => {
|
|
const lfs = lfsFiles.find((lf) => lf.path === f.path)
|
|
if (lfs) {
|
|
const pointer = buildLfsPointer(lfs.oid, lfs.size)
|
|
return octokit.git.createBlob({
|
|
owner,
|
|
repo,
|
|
content: Buffer.from(pointer, 'utf-8').toString('base64'),
|
|
encoding: 'base64',
|
|
})
|
|
}
|
|
return octokit.git.createBlob({
|
|
owner,
|
|
repo,
|
|
content: f.contentBase64,
|
|
encoding: 'base64',
|
|
})
|
|
}),
|
|
)
|
|
|
|
const newFilePaths = new Set(files.map((f) => f.path))
|
|
const deleteEntries = deletePaths
|
|
.filter((p) => !newFilePaths.has(p))
|
|
.map((p) => ({
|
|
path: p,
|
|
mode: '100644' as const,
|
|
type: 'blob' as const,
|
|
sha: null,
|
|
}))
|
|
|
|
const { data: newTree } = await octokit.git.createTree({
|
|
owner,
|
|
repo,
|
|
base_tree: commit.tree.sha,
|
|
tree: [
|
|
...allFiles.map((f, i) => ({
|
|
path: f.path,
|
|
mode: '100644' as const,
|
|
type: 'blob' as const,
|
|
sha: blobResults[i].data.sha,
|
|
})),
|
|
...deleteEntries,
|
|
],
|
|
})
|
|
|
|
const { data: newCommit } = await octokit.git.createCommit({
|
|
owner,
|
|
repo,
|
|
message: commitMessage,
|
|
tree: newTree.sha,
|
|
parents: [latestCommitSha],
|
|
})
|
|
|
|
await octokit.git.updateRef({
|
|
owner,
|
|
repo,
|
|
ref: `heads/${branch}`,
|
|
sha: newCommit.sha,
|
|
})
|
|
|
|
return { commitUrl: newCommit.html_url }
|
|
}
|