refactor: stage uploads before drive and git delivery
This commit is contained in:
Tom Boullay
+51
-15
@@ -10,6 +10,12 @@ export interface CheckResult {
|
||||
diffs: FileDiff[]
|
||||
}
|
||||
|
||||
export interface StageResult {
|
||||
stagingId: string
|
||||
folderName: string
|
||||
filesCount: number
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared FormData builder
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -49,15 +55,17 @@ function buildUploadFormData(
|
||||
* Throws on auth/network errors so callers can surface them to the user.
|
||||
*/
|
||||
export async function checkFolderDiffs(
|
||||
folder: FolderEntry,
|
||||
stagingId: string,
|
||||
secret: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CheckResult> {
|
||||
const formData = buildUploadFormData(folder)
|
||||
const res = await fetch('/api/upload/check', {
|
||||
method: 'POST',
|
||||
headers: { 'x-upload-secret': secret.trim() },
|
||||
body: formData,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-upload-secret': secret.trim(),
|
||||
},
|
||||
body: JSON.stringify({ stagingId }),
|
||||
signal,
|
||||
})
|
||||
|
||||
@@ -75,24 +83,51 @@ export async function checkFolderDiffs(
|
||||
return { exists: true, diffs: (data.diffs || []) as FileDiff[] }
|
||||
}
|
||||
|
||||
export async function stageUpload(
|
||||
folder: FolderEntry,
|
||||
secret: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<StageResult> {
|
||||
const formData = buildUploadFormData(folder)
|
||||
const res = await fetch('/api/upload/stage', {
|
||||
method: 'POST',
|
||||
headers: { 'x-upload-secret': secret.trim() },
|
||||
body: formData,
|
||||
signal,
|
||||
})
|
||||
|
||||
const data = await res.json()
|
||||
|
||||
if (!res.ok || !data.success) {
|
||||
throw new Error(data.error || `Erreur serveur (${res.status})`)
|
||||
}
|
||||
|
||||
return {
|
||||
stagingId: data.stagingId,
|
||||
folderName: data.folderName,
|
||||
filesCount: data.filesCount,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Upload original files to Nextcloud Drive
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Upload original files to Nextcloud Drive (no Blender compression). */
|
||||
export async function uploadDrive(
|
||||
folder: FolderEntry,
|
||||
stagingId: string,
|
||||
secret: string,
|
||||
action: 'new' | 'replace',
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
const formData = buildUploadFormData(folder, { action })
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/upload/drive', {
|
||||
method: 'POST',
|
||||
headers: { 'x-upload-secret': secret.trim() },
|
||||
body: formData,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-upload-secret': secret.trim(),
|
||||
},
|
||||
body: JSON.stringify({ stagingId, action }),
|
||||
signal,
|
||||
})
|
||||
const data = await res.json()
|
||||
@@ -112,20 +147,21 @@ export async function uploadDrive(
|
||||
|
||||
/** Upload files to GitHub (with Blender compression). */
|
||||
export async function uploadGit(
|
||||
folder: FolderEntry,
|
||||
stagingId: string,
|
||||
secret: string,
|
||||
onProgress: (pct: number) => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ success: boolean; filename?: string; error?: string }> {
|
||||
const formData = buildUploadFormData(folder)
|
||||
|
||||
onProgress(10)
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/upload/git', {
|
||||
method: 'POST',
|
||||
headers: { 'x-upload-secret': secret.trim() },
|
||||
body: formData,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-upload-secret': secret.trim(),
|
||||
},
|
||||
body: JSON.stringify({ stagingId }),
|
||||
signal,
|
||||
})
|
||||
|
||||
@@ -137,7 +173,7 @@ export async function uploadGit(
|
||||
}
|
||||
|
||||
onProgress(100)
|
||||
return { success: true, filename: folder.folderName }
|
||||
return { success: true, filename: data.folderName }
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === 'AbortError') {
|
||||
return { success: false, error: 'Upload annule' }
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import { randomUUID } from 'crypto'
|
||||
import { dirname, join } from 'path'
|
||||
import { mkdir, readdir, readFile, rm, stat, writeFile } from 'fs/promises'
|
||||
import { existsSync } from 'fs'
|
||||
import { TMP_DIR } from '@/lib/constants'
|
||||
import { prepareGitAssets } from '@/lib/prepare-git-assets'
|
||||
import type { ParsedFile, PreparedAssetSummary } from '@/lib/types'
|
||||
|
||||
const STAGING_ROOT = join(TMP_DIR, 'staging')
|
||||
const STAGING_TTL_MS = 60 * 60 * 1000
|
||||
|
||||
interface StagedOriginalFile {
|
||||
filename: string
|
||||
size: number
|
||||
isModel: boolean
|
||||
}
|
||||
|
||||
interface StagedPreparedData {
|
||||
modelFilename: string
|
||||
compressed: boolean
|
||||
compressionError?: string
|
||||
assetSummaries: PreparedAssetSummary[]
|
||||
}
|
||||
|
||||
interface StagingManifest {
|
||||
stagingId: string
|
||||
folderName: string
|
||||
createdAt: number
|
||||
originals: StagedOriginalFile[]
|
||||
prepared?: StagedPreparedData
|
||||
}
|
||||
|
||||
interface PushFile {
|
||||
path: string
|
||||
contentBase64: string
|
||||
}
|
||||
|
||||
interface PreparedStageAssetsResult {
|
||||
folderName: string
|
||||
filesToPush: PushFile[]
|
||||
modelFilename: string
|
||||
assetSummaries: PreparedAssetSummary[]
|
||||
compressed: boolean
|
||||
compressionError?: string
|
||||
}
|
||||
|
||||
function getStageDir(stagingId: string) {
|
||||
return join(STAGING_ROOT, stagingId)
|
||||
}
|
||||
|
||||
function getOriginalDir(stagingId: string) {
|
||||
return join(getStageDir(stagingId), 'original')
|
||||
}
|
||||
|
||||
function getPreparedDir(stagingId: string) {
|
||||
return join(getStageDir(stagingId), 'prepared')
|
||||
}
|
||||
|
||||
function getManifestPath(stagingId: string) {
|
||||
return join(getStageDir(stagingId), 'manifest.json')
|
||||
}
|
||||
|
||||
async function ensureParentDir(filePath: string) {
|
||||
await mkdir(dirname(filePath), { recursive: true })
|
||||
}
|
||||
|
||||
async function writeManifest(manifest: StagingManifest) {
|
||||
await ensureParentDir(getManifestPath(manifest.stagingId))
|
||||
await writeFile(getManifestPath(manifest.stagingId), JSON.stringify(manifest, null, 2), 'utf-8')
|
||||
}
|
||||
|
||||
export async function cleanupExpiredStagingUploads() {
|
||||
if (!existsSync(STAGING_ROOT)) return
|
||||
|
||||
const entries = await readdir(STAGING_ROOT, { withFileTypes: true })
|
||||
const now = Date.now()
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
|
||||
const stagingId = entry.name
|
||||
|
||||
try {
|
||||
const manifest = await readStagedManifest(stagingId)
|
||||
if (now - manifest.createdAt > STAGING_TTL_MS) {
|
||||
await cleanupStagingUpload(stagingId)
|
||||
}
|
||||
} catch {
|
||||
await cleanupStagingUpload(stagingId).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createStagingUpload(folderName: string, parsedFiles: ParsedFile[]) {
|
||||
await cleanupExpiredStagingUploads()
|
||||
|
||||
const stagingId = randomUUID()
|
||||
const originalDir = getOriginalDir(stagingId)
|
||||
await mkdir(originalDir, { recursive: true })
|
||||
|
||||
const originals: StagedOriginalFile[] = []
|
||||
|
||||
for (const file of parsedFiles) {
|
||||
const filePath = join(originalDir, file.filename)
|
||||
await ensureParentDir(filePath)
|
||||
await writeFile(filePath, file.buffer)
|
||||
originals.push({ filename: file.filename, size: file.buffer.length, isModel: file.isModel })
|
||||
}
|
||||
|
||||
const manifest: StagingManifest = {
|
||||
stagingId,
|
||||
folderName,
|
||||
createdAt: Date.now(),
|
||||
originals,
|
||||
}
|
||||
|
||||
await writeManifest(manifest)
|
||||
|
||||
return {
|
||||
stagingId,
|
||||
folderName,
|
||||
filesCount: originals.length,
|
||||
}
|
||||
}
|
||||
|
||||
export async function readStagedManifest(stagingId: string): Promise<StagingManifest> {
|
||||
const manifestPath = getManifestPath(stagingId)
|
||||
const content = await readFile(manifestPath, 'utf-8')
|
||||
return JSON.parse(content) as StagingManifest
|
||||
}
|
||||
|
||||
async function readOriginalParsedFiles(stagingId: string, manifest: StagingManifest): Promise<ParsedFile[]> {
|
||||
const originalDir = getOriginalDir(stagingId)
|
||||
|
||||
return Promise.all(
|
||||
manifest.originals.map(async (file) => ({
|
||||
filename: file.filename,
|
||||
buffer: await readFile(join(originalDir, file.filename)),
|
||||
isModel: file.isModel,
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
async function buildPreparedPushFiles(stagingId: string, manifest: StagingManifest): Promise<PushFile[]> {
|
||||
const preparedDir = getPreparedDir(stagingId)
|
||||
|
||||
return Promise.all(
|
||||
manifest.originals.map(async (file) => {
|
||||
const buffer = await readFile(join(preparedDir, file.filename))
|
||||
return {
|
||||
path: `public/models/${manifest.folderName}/${file.filename}`,
|
||||
contentBase64: buffer.toString('base64'),
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
export async function ensurePreparedStagingAssets(stagingId: string): Promise<PreparedStageAssetsResult> {
|
||||
const manifest = await readStagedManifest(stagingId)
|
||||
|
||||
if (!manifest.prepared) {
|
||||
const parsedFiles = await readOriginalParsedFiles(stagingId, manifest)
|
||||
const prepared = await prepareGitAssets({ folderName: manifest.folderName, parsedFiles })
|
||||
const preparedDir = getPreparedDir(stagingId)
|
||||
await mkdir(preparedDir, { recursive: true })
|
||||
|
||||
for (const file of prepared.filesToPush) {
|
||||
const filename = file.path.split('/').pop()
|
||||
if (!filename) continue
|
||||
const outputPath = join(preparedDir, filename)
|
||||
await ensureParentDir(outputPath)
|
||||
await writeFile(outputPath, Buffer.from(file.contentBase64, 'base64'))
|
||||
}
|
||||
|
||||
manifest.prepared = {
|
||||
modelFilename: prepared.modelFilename,
|
||||
compressed: prepared.compressed,
|
||||
compressionError: prepared.compressionError,
|
||||
assetSummaries: prepared.assetSummaries,
|
||||
}
|
||||
|
||||
await writeManifest(manifest)
|
||||
}
|
||||
|
||||
return {
|
||||
folderName: manifest.folderName,
|
||||
filesToPush: await buildPreparedPushFiles(stagingId, manifest),
|
||||
modelFilename: manifest.prepared.modelFilename,
|
||||
assetSummaries: manifest.prepared.assetSummaries,
|
||||
compressed: manifest.prepared.compressed,
|
||||
compressionError: manifest.prepared.compressionError,
|
||||
}
|
||||
}
|
||||
|
||||
export async function readStagedOriginalFiles(stagingId: string): Promise<{ folderName: string; files: ParsedFile[] }> {
|
||||
const manifest = await readStagedManifest(stagingId)
|
||||
return {
|
||||
folderName: manifest.folderName,
|
||||
files: await readOriginalParsedFiles(stagingId, manifest),
|
||||
}
|
||||
}
|
||||
|
||||
export async function cleanupStagingUpload(stagingId: string) {
|
||||
await rm(getStageDir(stagingId), { recursive: true, force: true })
|
||||
}
|
||||
|
||||
export async function stagingExists(stagingId: string): Promise<boolean> {
|
||||
try {
|
||||
const info = await stat(getStageDir(stagingId))
|
||||
return info.isDirectory()
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user