refactor: stage uploads before drive and git delivery

lib/upload-staging.ts

@@ -0,0 +1,214 @@
+import { randomUUID } from 'crypto'
+import { dirname, join } from 'path'
+import { mkdir, readdir, readFile, rm, stat, writeFile } from 'fs/promises'
+import { existsSync } from 'fs'
+import { TMP_DIR } from '@/lib/constants'
+import { prepareGitAssets } from '@/lib/prepare-git-assets'
+import type { ParsedFile, PreparedAssetSummary } from '@/lib/types'
+
+const STAGING_ROOT = join(TMP_DIR, 'staging')
+const STAGING_TTL_MS = 60 * 60 * 1000
+
+interface StagedOriginalFile {
+  filename: string
+  size: number
+  isModel: boolean
+}
+
+interface StagedPreparedData {
+  modelFilename: string
+  compressed: boolean
+  compressionError?: string
+  assetSummaries: PreparedAssetSummary[]
+}
+
+interface StagingManifest {
+  stagingId: string
+  folderName: string
+  createdAt: number
+  originals: StagedOriginalFile[]
+  prepared?: StagedPreparedData
+}
+
+interface PushFile {
+  path: string
+  contentBase64: string
+}
+
+interface PreparedStageAssetsResult {
+  folderName: string
+  filesToPush: PushFile[]
+  modelFilename: string
+  assetSummaries: PreparedAssetSummary[]
+  compressed: boolean
+  compressionError?: string
+}
+
+function getStageDir(stagingId: string) {
+  return join(STAGING_ROOT, stagingId)
+}
+
+function getOriginalDir(stagingId: string) {
+  return join(getStageDir(stagingId), 'original')
+}
+
+function getPreparedDir(stagingId: string) {
+  return join(getStageDir(stagingId), 'prepared')
+}
+
+function getManifestPath(stagingId: string) {
+  return join(getStageDir(stagingId), 'manifest.json')
+}
+
+async function ensureParentDir(filePath: string) {
+  await mkdir(dirname(filePath), { recursive: true })
+}
+
+async function writeManifest(manifest: StagingManifest) {
+  await ensureParentDir(getManifestPath(manifest.stagingId))
+  await writeFile(getManifestPath(manifest.stagingId), JSON.stringify(manifest, null, 2), 'utf-8')
+}
+
+export async function cleanupExpiredStagingUploads() {
+  if (!existsSync(STAGING_ROOT)) return
+
+  const entries = await readdir(STAGING_ROOT, { withFileTypes: true })
+  const now = Date.now()
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+
+    const stagingId = entry.name
+
+    try {
+      const manifest = await readStagedManifest(stagingId)
+      if (now - manifest.createdAt > STAGING_TTL_MS) {
+        await cleanupStagingUpload(stagingId)
+      }
+    } catch {
+      await cleanupStagingUpload(stagingId).catch(() => {})
+    }
+  }
+}
+
+export async function createStagingUpload(folderName: string, parsedFiles: ParsedFile[]) {
+  await cleanupExpiredStagingUploads()
+
+  const stagingId = randomUUID()
+  const originalDir = getOriginalDir(stagingId)
+  await mkdir(originalDir, { recursive: true })
+
+  const originals: StagedOriginalFile[] = []
+
+  for (const file of parsedFiles) {
+    const filePath = join(originalDir, file.filename)
+    await ensureParentDir(filePath)
+    await writeFile(filePath, file.buffer)
+    originals.push({ filename: file.filename, size: file.buffer.length, isModel: file.isModel })
+  }
+
+  const manifest: StagingManifest = {
+    stagingId,
+    folderName,
+    createdAt: Date.now(),
+    originals,
+  }
+
+  await writeManifest(manifest)
+
+  return {
+    stagingId,
+    folderName,
+    filesCount: originals.length,
+  }
+}
+
+export async function readStagedManifest(stagingId: string): Promise<StagingManifest> {
+  const manifestPath = getManifestPath(stagingId)
+  const content = await readFile(manifestPath, 'utf-8')
+  return JSON.parse(content) as StagingManifest
+}
+
+async function readOriginalParsedFiles(stagingId: string, manifest: StagingManifest): Promise<ParsedFile[]> {
+  const originalDir = getOriginalDir(stagingId)
+
+  return Promise.all(
+    manifest.originals.map(async (file) => ({
+      filename: file.filename,
+      buffer: await readFile(join(originalDir, file.filename)),
+      isModel: file.isModel,
+    })),
+  )
+}
+
+async function buildPreparedPushFiles(stagingId: string, manifest: StagingManifest): Promise<PushFile[]> {
+  const preparedDir = getPreparedDir(stagingId)
+
+  return Promise.all(
+    manifest.originals.map(async (file) => {
+      const buffer = await readFile(join(preparedDir, file.filename))
+      return {
+        path: `public/models/${manifest.folderName}/${file.filename}`,
+        contentBase64: buffer.toString('base64'),
+      }
+    }),
+  )
+}
+
+export async function ensurePreparedStagingAssets(stagingId: string): Promise<PreparedStageAssetsResult> {
+  const manifest = await readStagedManifest(stagingId)
+
+  if (!manifest.prepared) {
+    const parsedFiles = await readOriginalParsedFiles(stagingId, manifest)
+    const prepared = await prepareGitAssets({ folderName: manifest.folderName, parsedFiles })
+    const preparedDir = getPreparedDir(stagingId)
+    await mkdir(preparedDir, { recursive: true })
+
+    for (const file of prepared.filesToPush) {
+      const filename = file.path.split('/').pop()
+      if (!filename) continue
+      const outputPath = join(preparedDir, filename)
+      await ensureParentDir(outputPath)
+      await writeFile(outputPath, Buffer.from(file.contentBase64, 'base64'))
+    }
+
+    manifest.prepared = {
+      modelFilename: prepared.modelFilename,
+      compressed: prepared.compressed,
+      compressionError: prepared.compressionError,
+      assetSummaries: prepared.assetSummaries,
+    }
+
+    await writeManifest(manifest)
+  }
+
+  return {
+    folderName: manifest.folderName,
+    filesToPush: await buildPreparedPushFiles(stagingId, manifest),
+    modelFilename: manifest.prepared.modelFilename,
+    assetSummaries: manifest.prepared.assetSummaries,
+    compressed: manifest.prepared.compressed,
+    compressionError: manifest.prepared.compressionError,
+  }
+}
+
+export async function readStagedOriginalFiles(stagingId: string): Promise<{ folderName: string; files: ParsedFile[] }> {
+  const manifest = await readStagedManifest(stagingId)
+  return {
+    folderName: manifest.folderName,
+    files: await readOriginalParsedFiles(stagingId, manifest),
+  }
+}
+
+export async function cleanupStagingUpload(stagingId: string) {
+  await rm(getStageDir(stagingId), { recursive: true, force: true })
+}
+
+export async function stagingExists(stagingId: string): Promise<boolean> {
+  try {
+    const info = await stat(getStageDir(stagingId))
+    return info.isDirectory()
+  } catch {
+    return false
+  }
+}