diff --git a/.env.example b/.env.example index 931fbf9..67708bc 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ UPLOAD_SECRET_KEY=your-secret-key-here +GIT_PROVIDER=gitea GIT_USERNAME=your-gitea-username GIT_TOKEN=your-git-provider-token GIT_BRANCH=main diff --git a/README.md b/README.md index 5f71f2e..3764f86 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Built for La Fabrik Durable's internal use, but open-sourced for anyone looking - [**Next.js 16.2.5** (App Router)](https://nextjs.org/docs/app/getting-started/installation) + [React 19](https://react.dev/learn/creating-a-react-app) + [TypeScript](https://www.typescriptlang.org/docs/) - [**Three.js**](https://threejs.org/docs/#manual/en/introduction/Creating-a-scene) ([React Three Fiber](https://r3f.docs.pmnd.rs/getting-started/introduction) + [Drei](https://drei.docs.pmnd.rs/getting-started/introduction)) for 3D preview - [**Tailwind CSS**](https://v3.tailwindcss.com/docs/installation) for styling -- [**Octokit**](https://github.com/octokit/rest.js/#readme) for pushing via a GitHub-compatible API (GitHub or Gitea) +- [**Octokit**](https://github.com/octokit/rest.js/#readme) + provider adapters for GitHub and Gitea uploads - [**Nextcloud WebDAV**](https://docs.nextcloud.com/server/latest/developer_manual/client_apis/WebDAV/index.html) for Drive archiving with automatic versioning - [**Sharp**](https://sharp.pixelplumbing.com/install/) for server-side texture compression - [**npm lockfile + Coolify** (Docker)](https://coolify.io/docs/applications/build-packs/dockerfile) for hosting @@ -206,7 +206,15 @@ lib/ ├── diff-files.ts # File diff classification (new/changed/unchanged/deleted) ├── sanitize.ts # Filename sanitization ├── auth.ts # Upload secret validation (timing-safe) -├── github.ts # Octokit helpers for GitHub-compatible remotes +├── git/ # Git provider layer selected by env +│ ├── config.ts # GIT_PROVIDER/GIT_REPO_URL/GIT_TOKEN parsing +│ ├── content.ts # Shared remote folder/file content helpers +│ ├── http.ts # Shared Git API request helpers +│ ├── index.ts # Public getRemoteFolder/pushAllToGit facade +│ ├── lfs.ts # Shared Git LFS upload helpers +│ └── providers/ +│ ├── gitea.ts # Gitea Contents API implementation +│ └── github.ts # GitHub Git Data API implementation ├── nextcloud.ts # Nextcloud WebDAV client (native fetch, cached config) ├── upload-staging.ts # Temporary server-side staging and prepared asset reuse ├── upload-lock.ts # Lightweight in-memory per-folder upload lock @@ -237,6 +245,7 @@ Copy `.env.example` to `.env.local` and fill in the values: ```env UPLOAD_SECRET_KEY=your-secret-key-here +GIT_PROVIDER=gitea GIT_USERNAME=your-gitea-username GIT_TOKEN=your-git-provider-token GIT_BRANCH=main @@ -251,6 +260,7 @@ NEXTCLOUD_BASE_PATH=Models | Variable | Description | Required | |----------|-------------|----------| | `UPLOAD_SECRET_KEY` | Secret key for upload authentication | Yes | +| `GIT_PROVIDER` | Git provider adapter to use: `github` or `gitea`. If omitted, it is inferred from `GIT_REPO_URL` (`github.com` → GitHub, anything else → Gitea). | No | | `GIT_USERNAME` | Git username for Git LFS Basic auth on Gitea. Required for Gitea when LFS files are uploaded. | Gitea LFS | | `GIT_TOKEN` | Git provider token with repository read/write access. `GITHUB_TOKEN` is still accepted for backward compatibility. | Yes | | `GIT_BRANCH` | Target branch (default: main) | No | @@ -262,6 +272,34 @@ NEXTCLOUD_BASE_PATH=Models > GitHub tokens need `Contents: Read and write`. Gitea tokens need repository read/write access. +### Git provider selection + +The upload routes call a small provider layer in `lib/git/`: + +- `lib/git/config.ts` reads `GIT_PROVIDER`, `GIT_REPO_URL`, `GIT_TOKEN`, `GIT_USERNAME`, and `GIT_BRANCH` +- `lib/git/providers/github.ts` handles GitHub commits with the Git Data API +- `lib/git/providers/gitea.ts` handles Gitea commits with the Contents API +- `lib/git/lfs.ts` handles Git LFS upload/auth for both providers + +For GitHub: + +```env +GIT_PROVIDER=github +GIT_TOKEN=ghp_xxx +GIT_REPO_URL=https://github.com/org/repo.git +GIT_BRANCH=main +``` + +For Gitea: + +```env +GIT_PROVIDER=gitea +GIT_USERNAME=your-gitea-username +GIT_TOKEN=token_xxx +GIT_REPO_URL=https://git.fabrik.mathieu-chavanel.fr/math-pixel/La-Fabrik +GIT_BRANCH=main +``` + > To create a Nextcloud public share token: Nextcloud > Files > select folder > Share > Create public share > set permissions (write access required) > copy the share link and extract the token ### Production (Coolify / Docker) @@ -280,6 +318,7 @@ After a security patch: docker build -t upload-gltf . docker run -p 3000:3000 \ -e UPLOAD_SECRET_KEY=your-key \ + -e GIT_PROVIDER=gitea \ -e GIT_USERNAME=your-gitea-username \ -e GIT_TOKEN=token_xxx \ -e GIT_REPO_URL=https://git.fabrik.mathieu-chavanel.fr/math-pixel/La-Fabrik \ diff --git a/app/api/upload/check/route.ts b/app/api/upload/check/route.ts index 868b282..4665953 100644 --- a/app/api/upload/check/route.ts +++ b/app/api/upload/check/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { validateUploadSecret } from '@/lib/auth' -import { getRemoteFolder } from '@/lib/github' +import { getRemoteFolder } from '@/lib/git' import { classifyFileChanges } from '@/lib/diff-files' import { getModelFolderPath } from '@/lib/model-paths' import { ensurePreparedStagingAssets } from '@/lib/upload-staging' diff --git a/app/api/upload/git/route.ts b/app/api/upload/git/route.ts index 7f20fcc..0261c77 100644 --- a/app/api/upload/git/route.ts +++ b/app/api/upload/git/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server' import { validateUploadSecret } from '@/lib/auth' -import { getRemoteFolder, pushAllToGit } from '@/lib/github' +import { getRemoteFolder, pushAllToGit } from '@/lib/git' import { buildCommitMessage } from '@/lib/commit-message' import { classifyFileChanges } from '@/lib/diff-files' import { getModelFolderPath } from '@/lib/model-paths' diff --git a/lib/git/config.ts b/lib/git/config.ts new file mode 100644 index 0000000..0afe469 --- /dev/null +++ b/lib/git/config.ts @@ -0,0 +1,85 @@ +import type { GitProviderName, GitRemoteConfig } from './types' + +const DEFAULT_GIT_BRANCH = 'main' + +function getGitProviderOverride(): GitProviderName | undefined { + const provider = process.env.GIT_PROVIDER?.trim().toLowerCase() + if (!provider) return undefined + + if (provider !== 'github' && provider !== 'gitea') { + throw new Error(`GIT_PROVIDER invalide: "${provider}"`) + } + + return provider +} + +function getGitToken() { + const token = process.env.GIT_TOKEN?.trim() || process.env.GITHUB_TOKEN?.trim() + if (!token) throw new Error('GIT_TOKEN non configure') + return token +} + +function cleanRepoName(repo: string) { + return repo.replace(/\/+$/, '').replace(/\.git$/, '') +} + +function resolveProvider(host: string): GitProviderName { + const override = getGitProviderOverride() + if (override) return override + + return host.toLowerCase() === 'github.com' ? 'github' : 'gitea' +} + +function buildRemoteConfig(host: string, owner: string, repo: string, protocol = 'https:'): GitRemoteConfig { + const provider = resolveProvider(host) + const origin = `${protocol === 'http:' ? 'http' : 'https'}://${host}` + + return { + apiBaseUrl: provider === 'github' ? 'https://api.github.com' : `${origin}/api/v1`, + lfsBatchUrl: `${origin}/${owner}/${repo}.git/info/lfs/objects/batch`, + owner, + provider, + repo, + token: getGitToken(), + username: process.env.GIT_USERNAME?.trim() || undefined, + webUrl: `${origin}/${owner}/${repo}`, + } +} + +export function getGitBranch() { + return process.env.GIT_BRANCH?.trim() || DEFAULT_GIT_BRANCH +} + +export function readGitRemoteConfig(): GitRemoteConfig { + const url = process.env.GIT_REPO_URL?.trim() + if (!url) throw new Error('GIT_REPO_URL non configure') + + const shortMatch = url.match(/^([^/\s:]+)\/([^/\s]+)$/) + if (shortMatch) { + const providerOverride = getGitProviderOverride() + if (providerOverride === 'gitea') { + throw new Error('GIT_REPO_URL doit inclure le host pour Gitea') + } + + return buildRemoteConfig('github.com', shortMatch[1], cleanRepoName(shortMatch[2])) + } + + const scpLikeMatch = !url.includes('://') ? url.match(/^(?:[^@\s]+@)?([^:\s]+):([^/\s]+)\/([^/\s]+)$/) : null + if (scpLikeMatch) { + return buildRemoteConfig(scpLikeMatch[1], scpLikeMatch[2], cleanRepoName(scpLikeMatch[3])) + } + + if (URL.canParse(url)) { + const parsed = new URL(url) + const pathParts = parsed.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean) + + if ((parsed.protocol === 'https:' || parsed.protocol === 'http:' || parsed.protocol === 'ssh:') && pathParts.length >= 2) { + return buildRemoteConfig(parsed.hostname, pathParts[0], cleanRepoName(pathParts[1]), parsed.protocol) + } + } + + throw new Error(`Format GIT_REPO_URL invalide: "${url}"`) +} diff --git a/lib/git/content.ts b/lib/git/content.ts new file mode 100644 index 0000000..e0a009c --- /dev/null +++ b/lib/git/content.ts @@ -0,0 +1,102 @@ +import { isRecord } from '@/lib/guards' +import type { RemoteFile } from '@/lib/types' +import { decodeBase64Content, encodePath, getRepoApiPath, isHttpError, requestGitJson } from './http' +import { isLfsFile, parseLfsPointer } from './lfs' +import type { GitRemoteConfig } from './types' + +interface GitContentEntry { + content?: string + name: string + path?: string + sha?: string + size?: number + type?: string +} + +function isGitContentEntry(value: unknown): value is GitContentEntry { + return isRecord(value) + && typeof value.name === 'string' + && (value.path === undefined || typeof value.path === 'string') + && (value.sha === undefined || typeof value.sha === 'string') + && (value.size === undefined || typeof value.size === 'number') + && (value.content === undefined || typeof value.content === 'string') + && (value.type === undefined || typeof value.type === 'string') +} + +function getContentEntrySize(entry: GitContentEntry) { + return entry.size ?? 0 +} + +async function getRemoteContent(remote: GitRemoteConfig, path: string, branch: string) { + const query = new URLSearchParams({ ref: branch }) + return requestGitJson(remote, `${getRepoApiPath(remote)}/contents/${encodePath(path)}?${query.toString()}`) +} + +export function getRequiredContentEntrySha(entry: GitContentEntry, path: string) { + if (!entry.sha) { + throw new Error(`SHA Git manquant pour ${path}`) + } + + return entry.sha +} + +export async function getRemoteFileEntry(remote: GitRemoteConfig, path: string, branch: string): Promise { + try { + const data = await getRemoteContent(remote, path, branch) + return isGitContentEntry(data) ? data : null + } catch (err) { + if (isHttpError(err) && err.status === 404) return null + throw err + } +} + +export async function readRemoteFolder( + remote: GitRemoteConfig, + branch: string, + folderPath: string, +): Promise<{ exists: boolean; files: RemoteFile[] }> { + try { + const data = await getRemoteContent(remote, folderPath, branch) + + if (!Array.isArray(data)) { + throw new Error(`Le chemin distant ${folderPath} existe mais ce n'est pas un dossier`) + } + + const files: RemoteFile[] = await Promise.all( + data.map(async (entry: unknown): Promise => { + if (!isGitContentEntry(entry)) { + throw new Error(`Reponse Git invalide pour ${folderPath}`) + } + + const size = getContentEntrySize(entry) + + if (!isLfsFile(entry.name) || size > 1024) { + return { name: entry.name, size } + } + + try { + const fileData = await getRemoteFileEntry(remote, `${folderPath}/${entry.name}`, branch) + + if (fileData?.content) { + const content = decodeBase64Content(fileData.content) + const pointer = parseLfsPointer(content) + if (pointer) { + return { name: entry.name, size: pointer.size } + } + } + } catch (err) { + if (!isHttpError(err) || err.status !== 404) throw err + } + + return { name: entry.name, size } + }), + ) + + return { exists: true, files } + } catch (err: unknown) { + if (isHttpError(err) && err.status === 404) { + return { exists: false, files: [] } + } + throw err + } +} diff --git a/lib/git/http.ts b/lib/git/http.ts new file mode 100644 index 0000000..e0d0e17 --- /dev/null +++ b/lib/git/http.ts @@ -0,0 +1,47 @@ +import { isRecord } from '@/lib/guards' +import type { GitRemoteConfig } from './types' + +export class GitApiError extends Error { + constructor(message: string, public status: number) { + super(message) + } +} + +export function isHttpError(err: unknown): err is { status: number } { + return isRecord(err) && typeof err.status === 'number' +} + +export function encodePath(path: string) { + return path.split('/').map(encodeURIComponent).join('/') +} + +export function getRepoApiPath(remote: GitRemoteConfig) { + return `/repos/${encodeURIComponent(remote.owner)}/${encodeURIComponent(remote.repo)}` +} + +export function decodeBase64Content(content: string) { + return Buffer.from(content.replace(/\s/g, ''), 'base64').toString('utf-8') +} + +export async function requestGitJson( + remote: GitRemoteConfig, + path: string, + init: { method?: string; body?: unknown } = {}, +): Promise { + const res = await fetch(`${remote.apiBaseUrl}${path}`, { + method: init.method ?? 'GET', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Authorization': `token ${remote.token}`, + }, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + }) + + if (!res.ok) { + const text = await res.text() + throw new GitApiError(text || `Git API request failed (${res.status})`, res.status) + } + + return res.json() +} diff --git a/lib/git/index.ts b/lib/git/index.ts new file mode 100644 index 0000000..250e3b4 --- /dev/null +++ b/lib/git/index.ts @@ -0,0 +1,24 @@ +import { getGitBranch, readGitRemoteConfig } from './config' +import { createGiteaProvider } from './providers/gitea' +import { createGitHubProvider } from './providers/github' +import type { GitProvider } from './types' +import type { PushFile } from '@/lib/types' + +function createGitProvider(): GitProvider { + const remote = readGitRemoteConfig() + const branch = getGitBranch() + + if (remote.provider === 'github') { + return createGitHubProvider(remote, branch) + } + + return createGiteaProvider(remote, branch) +} + +export async function getRemoteFolder(folderPath: string) { + return createGitProvider().getRemoteFolder(folderPath) +} + +export async function pushAllToGit(files: PushFile[], deletePaths: string[], commitMessage: string) { + return createGitProvider().pushFiles({ files, deletePaths, commitMessage }) +} diff --git a/lib/git/lfs.ts b/lib/git/lfs.ts new file mode 100644 index 0000000..8c23613 --- /dev/null +++ b/lib/git/lfs.ts @@ -0,0 +1,233 @@ +import { createHash } from 'crypto' +import { LFS_EXTENSIONS } from '@/lib/constants' +import { isRecord } from '@/lib/guards' +import type { GitRemoteConfig, LfsObject, LfsPushFile } from './types' +import type { PushFile } from '@/lib/types' + +const LFS_BATCH_SIZE = 100 + +type LogDetails = Record + +interface LfsBatchAction { + href: string + header?: Record +} + +interface LfsBatchObject { + oid: string + size: number + actions?: { + upload?: LfsBatchAction + verify?: LfsBatchAction + } + error?: { code: number; message: string } +} + +export function isLfsFile(filePath: string): boolean { + const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase() + return LFS_EXTENSIONS.has(ext) +} + +function formatElapsed(startedAt: number) { + return `${((performance.now() - startedAt) / 1000).toFixed(1)}s` +} + +function logInfo(step: string, action: string, startedAt: number, details?: LogDetails) { + console.info(`[INFO] ${step} -> ${action} | Timer: ${formatElapsed(startedAt)}`, details || '') +} + +function chunkArray(items: T[], size: number) { + const chunks: T[][] = [] + + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)) + } + + return chunks +} + +function isStringRecord(value: unknown): value is Record { + return isRecord(value) && Object.values(value).every((entry) => typeof entry === 'string') +} + +function parseLfsAction(value: unknown): LfsBatchAction | undefined { + if (!isRecord(value) || typeof value.href !== 'string') return undefined + return { + href: value.href, + header: isStringRecord(value.header) ? value.header : undefined, + } +} + +function parseLfsBatchObject(value: unknown): LfsBatchObject | null { + if (!isRecord(value) || typeof value.oid !== 'string' || typeof value.size !== 'number') return null + + const actions = isRecord(value.actions) + ? { + upload: parseLfsAction(value.actions.upload), + verify: parseLfsAction(value.actions.verify), + } + : undefined + + const error = isRecord(value.error) && typeof value.error.code === 'number' && typeof value.error.message === 'string' + ? { code: value.error.code, message: value.error.message } + : undefined + + return { oid: value.oid, size: value.size, actions, error } +} + +function parseLfsBatchResponse(value: unknown): LfsBatchObject[] { + if (!isRecord(value) || !Array.isArray(value.objects)) { + throw new Error('LFS batch response invalide') + } + + const objects: LfsBatchObject[] = [] + for (const object of value.objects) { + const parsed = parseLfsBatchObject(object) + if (!parsed) { + throw new Error('LFS batch object invalide') + } + objects.push(parsed) + } + + return objects +} + +function getLfsAuthorizationHeader(remote: GitRemoteConfig) { + if (remote.provider === 'github') return `token ${remote.token}` + + if (!remote.username) { + throw new Error('GIT_USERNAME non configure pour Git LFS sur Gitea') + } + + return `Basic ${Buffer.from(`${remote.username}:${remote.token}`, 'utf-8').toString('base64')}` +} + +export function buildLfsPointer(sha256: string, size: number): string { + return `version https://git-lfs.github.com/spec/v1\noid sha256:${sha256}\nsize ${size}\n` +} + +export function parseLfsPointer(content: string): { oid: string; size: number } | null { + if (!content.startsWith('version https://git-lfs.github.com/spec/v1')) return null + const sizeMatch = content.match(/^size (\d+)$/m) + const oidMatch = content.match(/^oid sha256:([a-f0-9]{64})$/m) + if (!sizeMatch || !oidMatch) return null + return { oid: oidMatch[1], size: parseInt(sizeMatch[1], 10) } +} + +export function splitLfsFiles(files: PushFile[]) { + const lfsFiles: LfsPushFile[] = [] + const regularFiles: PushFile[] = [] + + for (const file of files) { + if (isLfsFile(file.path)) { + const buffer = Buffer.from(file.contentBase64, 'base64') + const oid = createHash('sha256').update(buffer).digest('hex') + lfsFiles.push({ ...file, oid, size: buffer.length }) + } else { + regularFiles.push(file) + } + } + + return { lfsFiles, regularFiles } +} + +export async function uploadToLfs(remote: GitRemoteConfig, objects: LfsObject[]): Promise { + if (objects.length === 0) return + + const batches = chunkArray(objects, LFS_BATCH_SIZE) + + for (let i = 0; i < batches.length; i++) { + await uploadToLfsBatch(remote, batches[i], i + 1, batches.length) + } +} + +async function uploadToLfsBatch( + remote: GitRemoteConfig, + objects: LfsObject[], + batchNumber: number, + totalBatches: number, +): Promise { + const startedAt = performance.now() + logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} started`, startedAt, { + objects: objects.length, + }) + + const batchRes = await fetch(remote.lfsBatchUrl, { + method: 'POST', + headers: { + 'Accept': 'application/vnd.git-lfs+json', + 'Content-Type': 'application/vnd.git-lfs+json', + 'Authorization': getLfsAuthorizationHeader(remote), + }, + body: JSON.stringify({ + operation: 'upload', + transfers: ['basic'], + objects: objects.map((object) => ({ oid: object.oid, size: object.size })), + }), + }) + + if (!batchRes.ok) { + const text = await batchRes.text() + console.error(`[ERROR] Git LFS -> Batch ${batchNumber}/${totalBatches} failed | Timer: ${formatElapsed(startedAt)}`, { + objects: objects.length, + status: batchRes.status, + }) + throw new Error(`LFS batch request failed (${batchRes.status}): ${text}`) + } + + const batchData: unknown = await batchRes.json() + const batchObjects = parseLfsBatchResponse(batchData) + const objectMap = new Map(objects.map((object) => [object.oid, object])) + + for (const object of batchObjects) { + if (object.error) { + throw new Error(`LFS error for ${object.oid}: ${object.error.message} (${object.error.code})`) + } + + if (!object.actions?.upload) continue + + const local = objectMap.get(object.oid) + if (!local) continue + + const uploadAction = object.actions.upload + const headers: Record = { + 'Content-Type': 'application/octet-stream', + ...uploadAction.header, + } + + const uploadRes = await fetch(uploadAction.href, { + method: 'PUT', + headers, + body: Buffer.from(local.contentBase64, 'base64'), + }) + + if (!uploadRes.ok) { + const text = await uploadRes.text() + throw new Error(`LFS upload failed for ${object.oid} (${uploadRes.status}): ${text}`) + } + + if (object.actions.verify) { + const verifyAction = object.actions.verify + const verifyHeaders: Record = { + 'Accept': 'application/vnd.git-lfs+json', + 'Content-Type': 'application/vnd.git-lfs+json', + ...verifyAction.header, + } + + const verifyRes = await fetch(verifyAction.href, { + method: 'POST', + headers: verifyHeaders, + body: JSON.stringify({ oid: object.oid, size: object.size }), + }) + + if (!verifyRes.ok) { + const text = await verifyRes.text() + throw new Error(`LFS verify failed for ${object.oid} (${verifyRes.status}): ${text}`) + } + } + } + + logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} done`, startedAt, { + objects: objects.length, + }) +} diff --git a/lib/git/providers/gitea.ts b/lib/git/providers/gitea.ts new file mode 100644 index 0000000..6c71f0f --- /dev/null +++ b/lib/git/providers/gitea.ts @@ -0,0 +1,102 @@ +import { getRemoteFileEntry, getRequiredContentEntrySha, readRemoteFolder } from '../content' +import { getRepoApiPath, requestGitJson } from '../http' +import { buildLfsPointer, splitLfsFiles, uploadToLfs } from '../lfs' +import type { GitProvider, GitRemoteConfig, LfsPushFile, PushFilesParams } from '../types' +import type { PushFile } from '@/lib/types' +import { isRecord } from '@/lib/guards' + +interface GiteaFileOperation { + content?: string + operation: 'create' | 'update' | 'delete' + path: string + sha?: string +} + +function getGiteaCommitUrl(value: unknown, remote: GitRemoteConfig, branch: string) { + if (isRecord(value) && isRecord(value.commit)) { + if (typeof value.commit.html_url === 'string') return value.commit.html_url + if (typeof value.commit.sha === 'string') return `${remote.webUrl}/commit/${value.commit.sha}` + } + + return `${remote.webUrl}/commits/branch/${branch}` +} + +function buildCommittedFiles(regularFiles: PushFile[], lfsFiles: LfsPushFile[]): PushFile[] { + return [ + ...regularFiles, + ...lfsFiles.map((file) => ({ + path: file.path, + contentBase64: Buffer.from(buildLfsPointer(file.oid, file.size), 'utf-8').toString('base64'), + })), + ] +} + +async function buildGiteaOperations( + remote: GitRemoteConfig, + branch: string, + committedFiles: PushFile[], + deletePaths: string[], +) { + const newFilePaths = new Set(committedFiles.map((file) => file.path)) + const operations: GiteaFileOperation[] = [] + + for (const file of committedFiles) { + const existing = await getRemoteFileEntry(remote, file.path, branch) + operations.push({ + content: file.contentBase64, + operation: existing ? 'update' : 'create', + path: file.path, + sha: existing ? getRequiredContentEntrySha(existing, file.path) : undefined, + }) + } + + for (const path of deletePaths) { + if (newFilePaths.has(path)) continue + + const existing = await getRemoteFileEntry(remote, path, branch) + if (!existing) continue + + operations.push({ + operation: 'delete', + path, + sha: getRequiredContentEntrySha(existing, path), + }) + } + + return operations +} + +export function createGiteaProvider(remote: GitRemoteConfig, branch: string): GitProvider { + return { + getRemoteFolder(folderPath) { + return readRemoteFolder(remote, branch, folderPath) + }, + + async pushFiles({ files, deletePaths, commitMessage }: PushFilesParams) { + const { lfsFiles, regularFiles } = splitLfsFiles(files) + + await uploadToLfs( + remote, + lfsFiles.map((file) => ({ oid: file.oid, size: file.size, contentBase64: file.contentBase64 })), + ) + + const committedFiles = buildCommittedFiles(regularFiles, lfsFiles) + const operations = await buildGiteaOperations(remote, branch, committedFiles, deletePaths) + + if (operations.length === 0) { + return { commitUrl: `${remote.webUrl}/commits/branch/${branch}` } + } + + const data = await requestGitJson(remote, `${getRepoApiPath(remote)}/contents`, { + method: 'POST', + body: { + branch, + files: operations, + message: commitMessage, + }, + }) + + return { commitUrl: getGiteaCommitUrl(data, remote, branch) } + }, + } +} diff --git a/lib/git/providers/github.ts b/lib/git/providers/github.ts new file mode 100644 index 0000000..5c67ec0 --- /dev/null +++ b/lib/git/providers/github.ts @@ -0,0 +1,106 @@ +import { Octokit } from '@octokit/rest' +import { readRemoteFolder } from '../content' +import { buildLfsPointer, splitLfsFiles, uploadToLfs } from '../lfs' +import type { GitProvider, GitRemoteConfig, PushFilesParams } from '../types' + +export function createGitHubProvider(remote: GitRemoteConfig, branch: string): GitProvider { + const octokit = new Octokit({ + auth: remote.token, + baseUrl: remote.apiBaseUrl, + }) + + return { + getRemoteFolder(folderPath) { + return readRemoteFolder(remote, branch, folderPath) + }, + + async pushFiles({ files, deletePaths, commitMessage }: PushFilesParams) { + const { lfsFiles, regularFiles } = splitLfsFiles(files) + + await uploadToLfs( + remote, + lfsFiles.map((file) => ({ oid: file.oid, size: file.size, contentBase64: file.contentBase64 })), + ) + + const { owner, repo } = remote + const { data: ref } = await octokit.git.getRef({ + owner, + repo, + ref: `heads/${branch}`, + }) + const latestCommitSha = ref.object.sha + + const { data: commit } = await octokit.git.getCommit({ + owner, + repo, + commit_sha: latestCommitSha, + }) + + const allFiles = [...regularFiles, ...lfsFiles] + + const blobResults = await Promise.all( + allFiles.map((file) => { + const lfs = lfsFiles.find((lfsFile) => lfsFile.path === file.path) + if (lfs) { + const pointer = buildLfsPointer(lfs.oid, lfs.size) + return octokit.git.createBlob({ + owner, + repo, + content: Buffer.from(pointer, 'utf-8').toString('base64'), + encoding: 'base64', + }) + } + + return octokit.git.createBlob({ + owner, + repo, + content: file.contentBase64, + encoding: 'base64', + }) + }), + ) + + const newFilePaths = new Set(files.map((file) => file.path)) + const deleteEntries = deletePaths + .filter((path) => !newFilePaths.has(path)) + .map((path) => ({ + path, + mode: '100644' as const, + type: 'blob' as const, + sha: null, + })) + + const { data: newTree } = await octokit.git.createTree({ + owner, + repo, + base_tree: commit.tree.sha, + tree: [ + ...allFiles.map((file, index) => ({ + path: file.path, + mode: '100644' as const, + type: 'blob' as const, + sha: blobResults[index].data.sha, + })), + ...deleteEntries, + ], + }) + + const { data: newCommit } = await octokit.git.createCommit({ + owner, + repo, + message: commitMessage, + tree: newTree.sha, + parents: [latestCommitSha], + }) + + await octokit.git.updateRef({ + owner, + repo, + ref: `heads/${branch}`, + sha: newCommit.sha, + }) + + return { commitUrl: newCommit.html_url || `${remote.webUrl}/commit/${newCommit.sha}` } + }, + } +} diff --git a/lib/git/types.ts b/lib/git/types.ts new file mode 100644 index 0000000..4ce3f5b --- /dev/null +++ b/lib/git/types.ts @@ -0,0 +1,36 @@ +import type { PushFile, RemoteFile } from '@/lib/types' + +export type GitProviderName = 'github' | 'gitea' + +export interface GitRemoteConfig { + apiBaseUrl: string + lfsBatchUrl: string + owner: string + provider: GitProviderName + repo: string + token: string + username?: string + webUrl: string +} + +export interface LfsObject { + oid: string + size: number + contentBase64: string +} + +export type LfsPushFile = PushFile & { + oid: string + size: number +} + +export interface PushFilesParams { + commitMessage: string + deletePaths: string[] + files: PushFile[] +} + +export interface GitProvider { + getRemoteFolder(folderPath: string): Promise<{ exists: boolean; files: RemoteFile[] }> + pushFiles(params: PushFilesParams): Promise<{ commitUrl: string }> +} diff --git a/lib/github.ts b/lib/github.ts deleted file mode 100644 index 8ebf360..0000000 --- a/lib/github.ts +++ /dev/null @@ -1,640 +0,0 @@ -import { createHash } from 'crypto' -import { Octokit } from '@octokit/rest' -import { LFS_EXTENSIONS } from './constants' -import { isRecord } from './guards' -import type { PushFile, RemoteFile } from './types' - -const LFS_BATCH_SIZE = 100 - -type LogDetails = Record - -interface GitRemoteConfig { - apiBaseUrl: string - lfsBatchUrl: string - owner: string - provider: 'github' | 'gitea' - repo: string - token: string - webUrl: string -} - -class GitApiError extends Error { - constructor(message: string, public status: number) { - super(message) - } -} - -function isHttpError(err: unknown): err is { status: number } { - return isRecord(err) && typeof err.status === 'number' -} - -function encodePath(path: string) { - return path.split('/').map(encodeURIComponent).join('/') -} - -function getRepoApiPath(remote: GitRemoteConfig) { - return `/repos/${encodeURIComponent(remote.owner)}/${encodeURIComponent(remote.repo)}` -} - -async function requestGitJson( - remote: GitRemoteConfig, - path: string, - init: { method?: string; body?: unknown } = {}, -): Promise { - const res = await fetch(`${remote.apiBaseUrl}${path}`, { - method: init.method ?? 'GET', - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json', - 'Authorization': `token ${remote.token}`, - }, - body: init.body === undefined ? undefined : JSON.stringify(init.body), - }) - - if (!res.ok) { - const text = await res.text() - throw new GitApiError(text || `Git API request failed (${res.status})`, res.status) - } - - return res.json() -} - -function getGitToken() { - const token = process.env.GIT_TOKEN?.trim() || process.env.GITHUB_TOKEN?.trim() - if (!token) throw new Error('GIT_TOKEN non configure') - return token -} - -function getGitUsername() { - return process.env.GIT_USERNAME?.trim() -} - -function getLfsAuthorizationHeader(remote: GitRemoteConfig) { - if (remote.provider === 'github') return `token ${remote.token}` - - const username = getGitUsername() - if (!username) { - throw new Error('GIT_USERNAME non configure pour Git LFS sur Gitea') - } - - return `Basic ${Buffer.from(`${username}:${remote.token}`, 'utf-8').toString('base64')}` -} - -function getOctokit(remote: GitRemoteConfig): Octokit { - return new Octokit({ - auth: remote.token, - baseUrl: remote.apiBaseUrl, - }) -} - -function cleanRepoName(repo: string) { - return repo.replace(/\/+$/, '').replace(/\.git$/, '') -} - -function buildRemoteConfig(host: string, owner: string, repo: string, protocol = 'https:'): GitRemoteConfig { - const normalizedHost = host.toLowerCase() - const origin = `${protocol === 'http:' ? 'http' : 'https'}://${host}` - const isGitHub = normalizedHost === 'github.com' - - return { - apiBaseUrl: isGitHub ? 'https://api.github.com' : `${origin}/api/v1`, - lfsBatchUrl: `${origin}/${owner}/${repo}.git/info/lfs/objects/batch`, - owner, - provider: isGitHub ? 'github' : 'gitea', - repo, - token: getGitToken(), - webUrl: `${origin}/${owner}/${repo}`, - } -} - -function parseRepoUrl(): GitRemoteConfig { - const url = process.env.GIT_REPO_URL?.trim() - if (!url) throw new Error('GIT_REPO_URL non configure') - - const shortMatch = url.match(/^([^/\s:]+)\/([^/\s]+)$/) - if (shortMatch) { - return buildRemoteConfig('github.com', shortMatch[1], cleanRepoName(shortMatch[2])) - } - - const scpLikeMatch = !url.includes('://') ? url.match(/^(?:[^@\s]+@)?([^:\s]+):([^/\s]+)\/([^/\s]+)$/) : null - if (scpLikeMatch) { - return buildRemoteConfig(scpLikeMatch[1], scpLikeMatch[2], cleanRepoName(scpLikeMatch[3])) - } - - if (URL.canParse(url)) { - const parsed = new URL(url) - const pathParts = parsed.pathname - .replace(/^\/+|\/+$/g, '') - .split('/') - .filter(Boolean) - - if ((parsed.protocol === 'https:' || parsed.protocol === 'http:' || parsed.protocol === 'ssh:') && pathParts.length >= 2) { - return buildRemoteConfig(parsed.hostname, pathParts[0], cleanRepoName(pathParts[1]), parsed.protocol) - } - } - - throw new Error(`Format GIT_REPO_URL invalide: "${url}"`) -} - -function isLfsFile(filePath: string): boolean { - const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase() - return LFS_EXTENSIONS.has(ext) -} - -function buildLfsPointer(sha256: string, size: number): string { - return `version https://git-lfs.github.com/spec/v1\noid sha256:${sha256}\nsize ${size}\n` -} - -function parseLfsPointer(content: string): { oid: string; size: number } | null { - if (!content.startsWith('version https://git-lfs.github.com/spec/v1')) return null - const sizeMatch = content.match(/^size (\d+)$/m) - const oidMatch = content.match(/^oid sha256:([a-f0-9]{64})$/m) - if (!sizeMatch || !oidMatch) return null - return { oid: oidMatch[1], size: parseInt(sizeMatch[1], 10) } -} - -interface GitContentEntry { - content?: string - name: string - path?: string - sha?: string - size?: number - type?: string -} - -function isGitContentEntry(value: unknown): value is GitContentEntry { - return isRecord(value) - && typeof value.name === 'string' - && (value.path === undefined || typeof value.path === 'string') - && (value.sha === undefined || typeof value.sha === 'string') - && (value.size === undefined || typeof value.size === 'number') - && (value.content === undefined || typeof value.content === 'string') - && (value.type === undefined || typeof value.type === 'string') -} - -function getContentEntrySize(entry: GitContentEntry) { - return entry.size ?? 0 -} - -function getRequiredContentEntrySha(entry: GitContentEntry, path: string) { - if (!entry.sha) { - throw new Error(`SHA Git manquant pour ${path}`) - } - - return entry.sha -} - -function decodeBase64Content(content: string) { - return Buffer.from(content.replace(/\s/g, ''), 'base64').toString('utf-8') -} - -async function getRemoteContent(remote: GitRemoteConfig, path: string, branch: string) { - const query = new URLSearchParams({ ref: branch }) - return requestGitJson(remote, `${getRepoApiPath(remote)}/contents/${encodePath(path)}?${query.toString()}`) -} - -async function getRemoteFileEntry(remote: GitRemoteConfig, path: string, branch: string): Promise { - try { - const data = await getRemoteContent(remote, path, branch) - return isGitContentEntry(data) ? data : null - } catch (err) { - if (isHttpError(err) && err.status === 404) return null - throw err - } -} - -function formatElapsed(startedAt: number) { - return `${((performance.now() - startedAt) / 1000).toFixed(1)}s` -} - -function logInfo(step: string, action: string, startedAt: number, details?: LogDetails) { - console.info(`[INFO] ${step} -> ${action} | Timer: ${formatElapsed(startedAt)}`, details || '') -} - -function chunkArray(items: T[], size: number) { - const chunks: T[][] = [] - - for (let i = 0; i < items.length; i += size) { - chunks.push(items.slice(i, i + size)) - } - - return chunks -} - -interface LfsObject { - oid: string - size: number - contentBase64: string -} - -type LfsPushFile = PushFile & { - oid: string - size: number -} - -interface LfsBatchAction { - href: string - header?: Record -} - -interface LfsBatchObject { - oid: string - size: number - actions?: { - upload?: LfsBatchAction - verify?: LfsBatchAction - } - error?: { code: number; message: string } -} - -function isStringRecord(value: unknown): value is Record { - return isRecord(value) && Object.values(value).every((entry) => typeof entry === 'string') -} - -function parseLfsAction(value: unknown): LfsBatchAction | undefined { - if (!isRecord(value) || typeof value.href !== 'string') return undefined - return { - href: value.href, - header: isStringRecord(value.header) ? value.header : undefined, - } -} - -function parseLfsBatchObject(value: unknown): LfsBatchObject | null { - if (!isRecord(value) || typeof value.oid !== 'string' || typeof value.size !== 'number') return null - - const actions = isRecord(value.actions) - ? { - upload: parseLfsAction(value.actions.upload), - verify: parseLfsAction(value.actions.verify), - } - : undefined - - const error = isRecord(value.error) && typeof value.error.code === 'number' && typeof value.error.message === 'string' - ? { code: value.error.code, message: value.error.message } - : undefined - - return { oid: value.oid, size: value.size, actions, error } -} - -function parseLfsBatchResponse(value: unknown): LfsBatchObject[] { - if (!isRecord(value) || !Array.isArray(value.objects)) { - throw new Error('LFS batch response invalide') - } - - const objects: LfsBatchObject[] = [] - for (const object of value.objects) { - const parsed = parseLfsBatchObject(object) - if (!parsed) { - throw new Error('LFS batch object invalide') - } - objects.push(parsed) - } - - return objects -} - -async function uploadToLfs( - remote: GitRemoteConfig, - objects: LfsObject[], -): Promise { - if (objects.length === 0) return - - const batches = chunkArray(objects, LFS_BATCH_SIZE) - - for (let i = 0; i < batches.length; i++) { - await uploadToLfsBatch(remote, batches[i], i + 1, batches.length) - } -} - -async function uploadToLfsBatch( - remote: GitRemoteConfig, - objects: LfsObject[], - batchNumber: number, - totalBatches: number, -): Promise { - const startedAt = performance.now() - logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} started`, startedAt, { - objects: objects.length, - }) - - const batchRes = await fetch(remote.lfsBatchUrl, { - method: 'POST', - headers: { - 'Accept': 'application/vnd.git-lfs+json', - 'Content-Type': 'application/vnd.git-lfs+json', - 'Authorization': getLfsAuthorizationHeader(remote), - }, - body: JSON.stringify({ - operation: 'upload', - transfers: ['basic'], - objects: objects.map((o) => ({ oid: o.oid, size: o.size })), - }), - }) - - if (!batchRes.ok) { - const text = await batchRes.text() - console.error(`[ERROR] Git LFS -> Batch ${batchNumber}/${totalBatches} failed | Timer: ${formatElapsed(startedAt)}`, { - objects: objects.length, - status: batchRes.status, - }) - throw new Error(`LFS batch request failed (${batchRes.status}): ${text}`) - } - - const batchData: unknown = await batchRes.json() - const batchObjects = parseLfsBatchResponse(batchData) - - const objectMap = new Map(objects.map((o) => [o.oid, o])) - - for (const obj of batchObjects) { - if (obj.error) { - throw new Error(`LFS error for ${obj.oid}: ${obj.error.message} (${obj.error.code})`) - } - - if (!obj.actions?.upload) continue - - const local = objectMap.get(obj.oid) - if (!local) continue - - const uploadAction = obj.actions.upload - const headers: Record = { - 'Content-Type': 'application/octet-stream', - ...uploadAction.header, - } - - const body = Buffer.from(local.contentBase64, 'base64') - - const uploadRes = await fetch(uploadAction.href, { - method: 'PUT', - headers, - body, - }) - - if (!uploadRes.ok) { - const text = await uploadRes.text() - throw new Error(`LFS upload failed for ${obj.oid} (${uploadRes.status}): ${text}`) - } - - if (obj.actions.verify) { - const verifyAction = obj.actions.verify - const verifyHeaders: Record = { - 'Accept': 'application/vnd.git-lfs+json', - 'Content-Type': 'application/vnd.git-lfs+json', - ...verifyAction.header, - } - - const verifyRes = await fetch(verifyAction.href, { - method: 'POST', - headers: verifyHeaders, - body: JSON.stringify({ oid: obj.oid, size: obj.size }), - }) - - if (!verifyRes.ok) { - const text = await verifyRes.text() - throw new Error(`LFS verify failed for ${obj.oid} (${verifyRes.status}): ${text}`) - } - } - } - - logInfo('Git LFS', `Batch ${batchNumber}/${totalBatches} done`, startedAt, { - objects: objects.length, - }) -} - -interface GiteaFileOperation { - content?: string - operation: 'create' | 'update' | 'delete' - path: string - sha?: string -} - -function getGiteaCommitUrl(value: unknown, remote: GitRemoteConfig, branch: string) { - if (isRecord(value) && isRecord(value.commit)) { - if (typeof value.commit.html_url === 'string') return value.commit.html_url - if (typeof value.commit.sha === 'string') return `${remote.webUrl}/commit/${value.commit.sha}` - } - - return `${remote.webUrl}/commits/branch/${branch}` -} - -async function pushAllToGitea( - remote: GitRemoteConfig, - regularFiles: PushFile[], - lfsFiles: LfsPushFile[], - deletePaths: string[], - commitMessage: string, - branch: string, -): Promise<{ commitUrl: string }> { - const committedFiles: PushFile[] = [ - ...regularFiles, - ...lfsFiles.map((file) => ({ - path: file.path, - contentBase64: Buffer.from(buildLfsPointer(file.oid, file.size), 'utf-8').toString('base64'), - })), - ] - const newFilePaths = new Set(committedFiles.map((file) => file.path)) - const operations: GiteaFileOperation[] = [] - - for (const file of committedFiles) { - const existing = await getRemoteFileEntry(remote, file.path, branch) - operations.push({ - content: file.contentBase64, - operation: existing ? 'update' : 'create', - path: file.path, - sha: existing ? getRequiredContentEntrySha(existing, file.path) : undefined, - }) - } - - for (const path of deletePaths) { - if (newFilePaths.has(path)) continue - - const existing = await getRemoteFileEntry(remote, path, branch) - if (!existing) continue - - operations.push({ - operation: 'delete', - path, - sha: getRequiredContentEntrySha(existing, path), - }) - } - - if (operations.length === 0) { - return { commitUrl: `${remote.webUrl}/commits/branch/${branch}` } - } - - const data = await requestGitJson(remote, `${getRepoApiPath(remote)}/contents`, { - method: 'POST', - body: { - branch, - files: operations, - message: commitMessage, - }, - }) - - return { commitUrl: getGiteaCommitUrl(data, remote, branch) } -} - -export async function getRemoteFolder( - folderPath: string, -): Promise<{ exists: boolean; files: RemoteFile[] }> { - const remote = parseRepoUrl() - const branch = process.env.GIT_BRANCH ?? 'main' - - try { - const data = await getRemoteContent(remote, folderPath, branch) - - if (!Array.isArray(data)) { - throw new Error(`Le chemin distant ${folderPath} existe mais ce n'est pas un dossier`) - } - - const files: RemoteFile[] = await Promise.all( - data.map(async (f: unknown): Promise => { - if (!isGitContentEntry(f)) { - throw new Error(`Reponse Git invalide pour ${folderPath}`) - } - - const size = getContentEntrySize(f) - - if (!isLfsFile(f.name) || size > 1024) { - return { name: f.name, size } - } - - try { - const fileData = await getRemoteFileEntry(remote, `${folderPath}/${f.name}`, branch) - - if (fileData?.content) { - const content = decodeBase64Content(fileData.content) - const pointer = parseLfsPointer(content) - if (pointer) { - return { name: f.name, size: pointer.size } - } - } - } catch (err) { - if (!isHttpError(err) || err.status !== 404) throw err - } - - return { name: f.name, size } - }), - ) - - return { exists: true, files } - } catch (err: unknown) { - if (isHttpError(err) && err.status === 404) { - return { exists: false, files: [] } - } - throw err - } -} - -export async function pushAllToGit( - files: PushFile[], - deletePaths: string[], - commitMessage: string, -): Promise<{ commitUrl: string }> { - const remote = parseRepoUrl() - const branch = process.env.GIT_BRANCH ?? 'main' - - const lfsFiles: LfsPushFile[] = [] - const regularFiles: PushFile[] = [] - - for (const f of files) { - if (isLfsFile(f.path)) { - const buf = Buffer.from(f.contentBase64, 'base64') - const oid = createHash('sha256').update(buf).digest('hex') - lfsFiles.push({ ...f, oid, size: buf.length }) - } else { - regularFiles.push(f) - } - } - - if (lfsFiles.length > 0) { - await uploadToLfs( - remote, - lfsFiles.map((f) => ({ oid: f.oid, size: f.size, contentBase64: f.contentBase64 })), - ) - } - - if (remote.provider === 'gitea') { - return pushAllToGitea(remote, regularFiles, lfsFiles, deletePaths, commitMessage, branch) - } - - const octokit = getOctokit(remote) - const { owner, repo } = remote - - const { data: ref } = await octokit.git.getRef({ - owner, - repo, - ref: `heads/${branch}`, - }) - const latestCommitSha = ref.object.sha - - const { data: commit } = await octokit.git.getCommit({ - owner, - repo, - commit_sha: latestCommitSha, - }) - - const allFiles = [...regularFiles, ...lfsFiles] - - const blobResults = await Promise.all( - allFiles.map((f) => { - const lfs = lfsFiles.find((lf) => lf.path === f.path) - if (lfs) { - const pointer = buildLfsPointer(lfs.oid, lfs.size) - return octokit.git.createBlob({ - owner, - repo, - content: Buffer.from(pointer, 'utf-8').toString('base64'), - encoding: 'base64', - }) - } - return octokit.git.createBlob({ - owner, - repo, - content: f.contentBase64, - encoding: 'base64', - }) - }), - ) - - const newFilePaths = new Set(files.map((f) => f.path)) - const deleteEntries = deletePaths - .filter((p) => !newFilePaths.has(p)) - .map((p) => ({ - path: p, - mode: '100644' as const, - type: 'blob' as const, - sha: null, - })) - - const { data: newTree } = await octokit.git.createTree({ - owner, - repo, - base_tree: commit.tree.sha, - tree: [ - ...allFiles.map((f, i) => ({ - path: f.path, - mode: '100644' as const, - type: 'blob' as const, - sha: blobResults[i].data.sha, - })), - ...deleteEntries, - ], - }) - - const { data: newCommit } = await octokit.git.createCommit({ - owner, - repo, - message: commitMessage, - tree: newTree.sha, - parents: [latestCommitSha], - }) - - await octokit.git.updateRef({ - owner, - repo, - ref: `heads/${branch}`, - sha: newCommit.sha, - }) - - return { commitUrl: newCommit.html_url || `${remote.webUrl}/commit/${newCommit.sha}` } -}