feat: add ktx2 texture fallback flow
This commit is contained in:
@@ -5,6 +5,10 @@ GIT_TOKEN=your-git-provider-token
|
|||||||
GIT_BRANCH=main
|
GIT_BRANCH=main
|
||||||
GIT_REPO_URL=https://git.example.com/your-org/your-repo
|
GIT_REPO_URL=https://git.example.com/your-org/your-repo
|
||||||
|
|
||||||
|
# Optional texture optimization
|
||||||
|
TOKTX_PATH=toktx
|
||||||
|
TEXTURE_MAX_SIZE=1024
|
||||||
|
|
||||||
# Nextcloud Drive (public share WebDAV)
|
# Nextcloud Drive (public share WebDAV)
|
||||||
NEXTCLOUD_URL=https://cloud.example.com
|
NEXTCLOUD_URL=https://cloud.example.com
|
||||||
NEXTCLOUD_SHARE_TOKEN=your-public-share-token
|
NEXTCLOUD_SHARE_TOKEN=your-public-share-token
|
||||||
|
|||||||
+13
-2
@@ -24,15 +24,26 @@ RUN npm run build
|
|||||||
# --- Stage 3: Production -----------------------------------------------------
|
# --- Stage 3: Production -----------------------------------------------------
|
||||||
FROM node:20-slim AS runner
|
FROM node:20-slim AS runner
|
||||||
|
|
||||||
|
ARG KTX_SOFTWARE_VERSION=4.4.2
|
||||||
|
|
||||||
LABEL maintainer="La Fabrik Durable"
|
LABEL maintainer="La Fabrik Durable"
|
||||||
LABEL description="Secure GLTF upload interface with Draco compression and Git push"
|
LABEL description="Secure GLTF upload interface with Draco/KTX2 compression and Git push"
|
||||||
|
|
||||||
# Blender is required for server-side Draco GLB export.
|
# Blender is required for server-side Draco GLB export.
|
||||||
|
# KTX-Software provides toktx for optional KTX2 texture delivery.
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
bzip2 \
|
||||||
blender \
|
blender \
|
||||||
|
ca-certificates \
|
||||||
tini \
|
tini \
|
||||||
curl \
|
curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& curl -fsSL \
|
||||||
|
"https://github.com/KhronosGroup/KTX-Software/releases/download/v${KTX_SOFTWARE_VERSION}/KTX-Software-${KTX_SOFTWARE_VERSION}-Linux-x86_64.tar.bz2" \
|
||||||
|
-o /tmp/ktx-software.tar.bz2 \
|
||||||
|
&& mkdir -p /opt/ktx-software \
|
||||||
|
&& tar -xjf /tmp/ktx-software.tar.bz2 -C /opt/ktx-software --strip-components=1 \
|
||||||
|
&& ln -s /opt/ktx-software/bin/toktx /usr/local/bin/toktx \
|
||||||
|
&& rm -rf /tmp/ktx-software.tar.bz2 /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
A secure web interface for uploading `model.gltf` with its associated `.bin` file and textures with two outputs:
|
A secure web interface for uploading `model.gltf` with its associated `.bin` file and textures with two outputs:
|
||||||
|
|
||||||
- **Nextcloud Drive** — Archives the original files with automatic versioning (VF/V1/V2...), so artists always have a history of past versions
|
- **Nextcloud Drive** — Archives the original files with automatic versioning (VF/V1/V2...), so artists always have a history of past versions
|
||||||
- **Git remote** — Delivers Draco-compressed GLB assets by default, with an optional GLTF delivery mode for specific models
|
- **Git remote** — Delivers Draco-compressed GLB assets by default, with an optional GLTF delivery mode using KTX2 textures, then WebP/original fallbacks for specific models
|
||||||
|
|
||||||
Built for La Fabrik Durable's internal use, but open-sourced for anyone looking for a similar solution. The app validates the upload locally, stages it server-side, then compares file diffs to avoid unnecessary uploads and commits. The Drive upload serves as the source of truth and version history, while the Git upload delivers the prepared assets to developers.
|
Built for La Fabrik Durable's internal use, but open-sourced for anyone looking for a similar solution. The app validates the upload locally, stages it server-side, then compares file diffs to avoid unnecessary uploads and commits. The Drive upload serves as the source of truth and version history, while the Git upload delivers the prepared assets to developers.
|
||||||
|
|
||||||
@@ -14,7 +14,8 @@ Built for La Fabrik Durable's internal use, but open-sourced for anyone looking
|
|||||||
- [**Tailwind CSS**](https://v3.tailwindcss.com/docs/installation) for styling
|
- [**Tailwind CSS**](https://v3.tailwindcss.com/docs/installation) for styling
|
||||||
- [**Octokit**](https://github.com/octokit/rest.js/#readme) + provider adapters for GitHub and Gitea uploads
|
- [**Octokit**](https://github.com/octokit/rest.js/#readme) + provider adapters for GitHub and Gitea uploads
|
||||||
- [**Nextcloud WebDAV**](https://docs.nextcloud.com/server/latest/developer_manual/client_apis/WebDAV/index.html) for Drive archiving with automatic versioning
|
- [**Nextcloud WebDAV**](https://docs.nextcloud.com/server/latest/developer_manual/client_apis/WebDAV/index.html) for Drive archiving with automatic versioning
|
||||||
- [**Sharp**](https://sharp.pixelplumbing.com/install/) for server-side texture compression
|
- [**Sharp**](https://sharp.pixelplumbing.com/install/) for server-side WebP texture fallback
|
||||||
|
- [**KTX-Software**](https://github.com/KhronosGroup/KTX-Software) (`toktx`) for optional KTX2 texture delivery
|
||||||
- [**npm lockfile + Coolify** (Docker)](https://coolify.io/docs/applications/build-packs/dockerfile) for hosting
|
- [**npm lockfile + Coolify** (Docker)](https://coolify.io/docs/applications/build-packs/dockerfile) for hosting
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
@@ -87,7 +88,7 @@ Invalid or unknown asset names still block the upload.
|
|||||||
### Upload flow: Drive first, then Git
|
### Upload flow: Drive first, then Git
|
||||||
|
|
||||||
5. **Drive upload (archiving)** — Original files from the staging area are uploaded to the Nextcloud Drive with automatic versioning (see below). This serves as the artists' source of truth and version history. If the Drive upload fails, a modal asks the user whether to send to Git only or cancel entirely.
|
5. **Drive upload (archiving)** — Original files from the staging area are uploaded to the Nextcloud Drive with automatic versioning (see below). This serves as the artists' source of truth and version history. If the Drive upload fails, a modal asks the user whether to send to Git only or cancel entirely.
|
||||||
6. **Git upload (delivery to devs)** — The prepared Git payload is reused from staging. By default, Blender exports a single `model.glb` with Draco compression. If Blender cannot process a specific model, the upload falls back to the separate `model.gltf` + `.bin` + compressed textures workflow and shows a warning. For specific models, "Envoyer en GLTF" keeps that separate GLTF delivery mode from the start.
|
6. **Git upload (delivery to devs)** — The prepared Git payload is reused from staging. By default, Blender exports a single `model.glb` with Draco compression. If Blender cannot process a specific model, the upload falls back to the separate `model.gltf` + `.bin` + optimized textures workflow and shows a warning. In separate GLTF delivery, each texture tries one optimized output in order: KTX2 with `KHR_texture_basisu`, then WebP through Sharp, then the original texture if all optimization fails. For specific models, "Envoyer en GLTF" keeps that separate GLTF delivery mode from the start.
|
||||||
|
|
||||||
### Drive versioning (Nextcloud WebDAV)
|
### Drive versioning (Nextcloud WebDAV)
|
||||||
|
|
||||||
@@ -153,7 +154,7 @@ Commit sections:
|
|||||||
Symbols: `✅` new — `🔄` modified — `↔️` unchanged (model always re-pushed) — `❌` deleted
|
Symbols: `✅` new — `🔄` modified — `↔️` unchanged (model always re-pushed) — `❌` deleted
|
||||||
|
|
||||||
7. Orphan files (present on remote but not in the new upload) are deleted in the same commit
|
7. Orphan files (present on remote but not in the new upload) are deleted in the same commit
|
||||||
8. Default Git delivery pushes `model.glb` when Blender compression succeeds. If Blender fails, the app falls back to separate `model.gltf`, `.bin`, and compressed textures with a warning. "Envoyer en GLTF" always uses separate GLTF delivery.
|
8. Default Git delivery pushes `model.glb` when Blender compression succeeds. If Blender fails, the app falls back to separate `model.gltf`, `.bin`, and one delivered texture per source: `.ktx2` when possible, `.webp` when KTX2 fails, or the original file when both optimizations fail. "Envoyer en GLTF" always uses separate GLTF delivery.
|
||||||
|
|
||||||
Uploaded models are pushed to `public/models/<folderName>/` in the target repo.
|
Uploaded models are pushed to `public/models/<folderName>/` in the target repo.
|
||||||
|
|
||||||
@@ -162,6 +163,7 @@ Uploaded models are pushed to `public/models/<folderName>/` in the target repo.
|
|||||||
- Large uploads are staged once, but the Drive upload remains sequential.
|
- Large uploads are staged once, but the Drive upload remains sequential.
|
||||||
- Git LFS batch uploads are sequential by batch.
|
- Git LFS batch uploads are sequential by batch.
|
||||||
- Default GLB Draco delivery reduces Git LFS usage by replacing many support files with one compressed model file.
|
- Default GLB Draco delivery reduces Git LFS usage by replacing many support files with one compressed model file.
|
||||||
|
- KTX2 is currently applied to separate GLTF delivery only. Embedding KTX2 inside the default GLB would require a glTF-aware optimizer step after Blender, not just image preprocessing before Blender.
|
||||||
- Uploads expect a single `model.gltf` file plus optional flat support files (`.bin`, `.png`, `.jpg`, `.jpeg`, `.webp`).
|
- Uploads expect a single `model.gltf` file plus optional flat support files (`.bin`, `.png`, `.jpg`, `.jpeg`, `.webp`).
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
@@ -221,14 +223,15 @@ lib/
|
|||||||
├── asset-classification.ts # Group assets by family for commit messages
|
├── asset-classification.ts # Group assets by family for commit messages
|
||||||
├── asset-naming.ts # Allowed asset families and naming convention helpers
|
├── asset-naming.ts # Allowed asset families and naming convention helpers
|
||||||
├── blender.ts # Blender Draco compression helper
|
├── blender.ts # Blender Draco compression helper
|
||||||
|
├── texture-compression.ts # KTX2/WebP texture delivery helpers
|
||||||
├── commit-message.ts # Commit message builder
|
├── commit-message.ts # Commit message builder
|
||||||
├── parse-upload.ts # FormData parser + validation
|
├── parse-upload.ts # FormData parser + validation
|
||||||
├── validate-folder.ts # Client-side folder validation (discriminated union)
|
├── validate-folder.ts # Client-side folder validation (discriminated union)
|
||||||
└── format-bytes.ts # Byte formatting utility
|
└── format-bytes.ts # Byte formatting utility
|
||||||
scripts/
|
scripts/
|
||||||
└── compress.py # Blender Draco compression script
|
└── compress.py # Blender Draco compression script
|
||||||
Dockerfile # Multi-stage build: Node 20 slim + Blender + tini
|
Dockerfile # Multi-stage build: Node 20 slim + Blender + KTX-Software + tini
|
||||||
docker-entrypoint.sh # Upload temp setup + Blender availability check
|
docker-entrypoint.sh # Upload temp setup + Blender/toktx availability check
|
||||||
```
|
```
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
@@ -250,6 +253,9 @@ GIT_USERNAME=your-gitea-username
|
|||||||
GIT_TOKEN=your-git-provider-token
|
GIT_TOKEN=your-git-provider-token
|
||||||
GIT_BRANCH=main
|
GIT_BRANCH=main
|
||||||
GIT_REPO_URL=https://git.example.com/your-org/your-repo
|
GIT_REPO_URL=https://git.example.com/your-org/your-repo
|
||||||
|
# Optional texture optimization
|
||||||
|
TOKTX_PATH=toktx
|
||||||
|
TEXTURE_MAX_SIZE=1024
|
||||||
# Nextcloud Drive (public share WebDAV)
|
# Nextcloud Drive (public share WebDAV)
|
||||||
NEXTCLOUD_URL=https://cloud.example.com
|
NEXTCLOUD_URL=https://cloud.example.com
|
||||||
NEXTCLOUD_SHARE_TOKEN=your-public-share-token
|
NEXTCLOUD_SHARE_TOKEN=your-public-share-token
|
||||||
@@ -265,6 +271,8 @@ NEXTCLOUD_BASE_PATH=Models
|
|||||||
| `GIT_TOKEN` | Git provider token with repository read/write access. `GITHUB_TOKEN` is still accepted for backward compatibility. | Yes |
|
| `GIT_TOKEN` | Git provider token with repository read/write access. `GITHUB_TOKEN` is still accepted for backward compatibility. | Yes |
|
||||||
| `GIT_BRANCH` | Target branch (default: main) | No |
|
| `GIT_BRANCH` | Target branch (default: main) | No |
|
||||||
| `GIT_REPO_URL` | Target GitHub or Gitea repository URL (`owner/repo`, HTTPS, or SSH) | Yes |
|
| `GIT_REPO_URL` | Target GitHub or Gitea repository URL (`owner/repo`, HTTPS, or SSH) | Yes |
|
||||||
|
| `TOKTX_PATH` | Path to the `toktx` binary used for KTX2 texture generation (default: `toktx`). | No |
|
||||||
|
| `TEXTURE_MAX_SIZE` | Maximum texture dimension used for KTX2/WebP separate GLTF delivery (default: `1024`). | No |
|
||||||
| `NEXTCLOUD_URL` | Nextcloud instance URL | Yes |
|
| `NEXTCLOUD_URL` | Nextcloud instance URL | Yes |
|
||||||
| `NEXTCLOUD_SHARE_TOKEN` | Public share token (the part after `/s/` in the share link) | Yes |
|
| `NEXTCLOUD_SHARE_TOKEN` | Public share token (the part after `/s/` in the share link) | Yes |
|
||||||
| `NEXTCLOUD_SHARE_PASSWORD` | Public share password (empty if none) | No |
|
| `NEXTCLOUD_SHARE_PASSWORD` | Public share password (empty if none) | No |
|
||||||
@@ -327,7 +335,7 @@ docker run -p 3000:3000 \
|
|||||||
upload-gltf
|
upload-gltf
|
||||||
```
|
```
|
||||||
|
|
||||||
The Docker image runs the Next.js app, Blender Draco compression, and server-side asset preparation in a single container. The `docker-entrypoint.sh` script creates the upload temp directory and reports Blender availability before launching the app.
|
The Docker image runs the Next.js app, Blender Draco compression, KTX2 texture generation, and server-side asset preparation in a single container. The `docker-entrypoint.sh` script creates the upload temp directory and reports Blender/toktx availability before launching the app.
|
||||||
|
|
||||||
### Secret rotation
|
### Secret rotation
|
||||||
|
|
||||||
@@ -366,9 +374,10 @@ git push origin main && git push gitea main
|
|||||||
|------|------------|
|
|------|------------|
|
||||||
| 3D Models | `.gltf` |
|
| 3D Models | `.gltf` |
|
||||||
| Binary buffers | `.bin` |
|
| Binary buffers | `.bin` |
|
||||||
| Textures | `.png`, `.jpg`, `.jpeg`, `.webp` |
|
| Input textures | `.png`, `.jpg`, `.jpeg`, `.webp` |
|
||||||
|
| Generated Git textures | `.ktx2`, `.webp`, or original source extension as fallback |
|
||||||
|
|
||||||
Git delivery outputs `.glb` by default, or keeps the source `.gltf` structure when "Envoyer en GLTF" is selected.
|
Git delivery outputs `.glb` by default, or keeps the source `.gltf` structure when "Envoyer en GLTF" is selected. Separate GLTF delivery tries KTX2 first, then WebP, then the original texture. When KTX2 is selected, `model.gltf` references it through `KHR_texture_basisu`.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+8
-2
@@ -8,8 +8,14 @@ export default function Home() {
|
|||||||
Upload GLTF
|
Upload GLTF
|
||||||
</h1>
|
</h1>
|
||||||
<p className="text-gray-400 text-base leading-relaxed">
|
<p className="text-gray-400 text-base leading-relaxed">
|
||||||
Deposez vos fichiers 3D — ils seront archives sur le Drive
|
Envoyer genere un GLB quand possible : plus performant, ideal pour la deco
|
||||||
<br />avec versioning, puis envoyes aux devs via Git
|
et les modeles simples.
|
||||||
|
<br />
|
||||||
|
Envoyer en GLTF garde des fichiers separes, moins optimises mais plus
|
||||||
|
pratiques pour inspecter et bidouiller les noeuds.
|
||||||
|
<br />
|
||||||
|
A privilegier pour les packs de relance, ebikes, pylones et modeles
|
||||||
|
que les devs doivent ajuster.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,14 @@ else
|
|||||||
echo "[upload-gltf] WARNING: Blender not found. GLB Draco compression will fall back to separate GLTF delivery."
|
echo "[upload-gltf] WARNING: Blender not found. GLB Draco compression will fall back to separate GLTF delivery."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if command -v toktx > /dev/null 2>&1; then
|
||||||
|
TOKTX_VERSION=$(toktx --version 2>/dev/null | head -n 1)
|
||||||
|
echo "[upload-gltf] toktx found: $TOKTX_VERSION"
|
||||||
|
echo "[upload-gltf] KTX2 texture delivery is enabled."
|
||||||
|
else
|
||||||
|
echo "[upload-gltf] WARNING: toktx not found. Texture delivery will fall back to WebP."
|
||||||
|
fi
|
||||||
|
|
||||||
echo "[upload-gltf] Ready. Launching application..."
|
echo "[upload-gltf] Ready. Launching application..."
|
||||||
|
|
||||||
exec "$@"
|
exec "$@"
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ export const ASSET_EXTENSIONS = new Set(['.bin'])
|
|||||||
export const ALL_ALLOWED_EXTENSIONS = new Set([...MODEL_EXTENSIONS, ...TEXTURE_EXTENSIONS, ...ASSET_EXTENSIONS])
|
export const ALL_ALLOWED_EXTENSIONS = new Set([...MODEL_EXTENSIONS, ...TEXTURE_EXTENSIONS, ...ASSET_EXTENSIONS])
|
||||||
|
|
||||||
/** Extensions tracked by Git LFS (must match .gitattributes) */
|
/** Extensions tracked by Git LFS (must match .gitattributes) */
|
||||||
export const LFS_EXTENSIONS = new Set(['.glb', '.gltf', '.bin', '.png', '.jpg', '.jpeg', '.webp'])
|
export const LFS_EXTENSIONS = new Set(['.glb', '.gltf', '.bin', '.png', '.jpg', '.jpeg', '.webp', '.ktx2'])
|
||||||
|
|
||||||
export const TMP_DIR = '/tmp/assets'
|
export const TMP_DIR = '/tmp/assets'
|
||||||
|
|
||||||
|
|||||||
+182
-14
@@ -3,7 +3,7 @@ import { existsSync } from 'fs'
|
|||||||
import { mkdir, readFile, rm, writeFile } from 'fs/promises'
|
import { mkdir, readFile, rm, writeFile } from 'fs/promises'
|
||||||
import { extname, join } from 'path'
|
import { extname, join } from 'path'
|
||||||
import { compressWithBlender } from '@/lib/blender'
|
import { compressWithBlender } from '@/lib/blender'
|
||||||
import { compressTextureBuffer } from '@/lib/texture-compression'
|
import { compressTextureBuffer, prepareTextureDelivery } from '@/lib/texture-compression'
|
||||||
import { classifyAssetCategory } from '@/lib/asset-classification'
|
import { classifyAssetCategory } from '@/lib/asset-classification'
|
||||||
import { normalizeTextureFilename } from '@/lib/asset-naming'
|
import { normalizeTextureFilename } from '@/lib/asset-naming'
|
||||||
import { TEXTURE_EXTENSIONS, TMP_DIR } from '@/lib/constants'
|
import { TEXTURE_EXTENSIONS, TMP_DIR } from '@/lib/constants'
|
||||||
@@ -18,6 +18,17 @@ interface PrepareGitAssetsParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }
|
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }
|
||||||
|
type JsonObject = { [key: string]: JsonValue }
|
||||||
|
|
||||||
|
interface PreparedTexturePlan {
|
||||||
|
filename: string
|
||||||
|
buffer: Buffer
|
||||||
|
category: PreparedAssetSummary['category']
|
||||||
|
compressed: boolean
|
||||||
|
format: 'ktx2' | 'webp' | 'original'
|
||||||
|
}
|
||||||
|
|
||||||
|
const KHR_TEXTURE_BASISU = 'KHR_texture_basisu'
|
||||||
|
|
||||||
function isJsonValue(value: unknown): value is JsonValue {
|
function isJsonValue(value: unknown): value is JsonValue {
|
||||||
if (value === null) return true
|
if (value === null) return true
|
||||||
@@ -43,6 +54,34 @@ function parseJsonValue(content: string) {
|
|||||||
return parsed
|
return parsed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isJsonObject(value: JsonValue): value is JsonObject {
|
||||||
|
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getJsonObjectArray(value: JsonValue | undefined) {
|
||||||
|
if (!Array.isArray(value)) return null
|
||||||
|
return value.every(isJsonObject) ? value : null
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStringArray(value: JsonValue | undefined) {
|
||||||
|
if (!Array.isArray(value)) return []
|
||||||
|
return value.filter((entry): entry is string => typeof entry === 'string')
|
||||||
|
}
|
||||||
|
|
||||||
|
function addExtensionUsed(gltf: JsonObject, extensionName: string) {
|
||||||
|
const extensionsUsed = new Set(getStringArray(gltf.extensionsUsed))
|
||||||
|
extensionsUsed.add(extensionName)
|
||||||
|
gltf.extensionsUsed = [...extensionsUsed]
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOrCreateExtensions(value: JsonObject): JsonObject {
|
||||||
|
if (!isJsonObject(value.extensions)) {
|
||||||
|
value.extensions = {}
|
||||||
|
}
|
||||||
|
|
||||||
|
return value.extensions
|
||||||
|
}
|
||||||
|
|
||||||
function getTextureFilenameMap(parsedFiles: ParsedFile[]) {
|
function getTextureFilenameMap(parsedFiles: ParsedFile[]) {
|
||||||
const filenameMap = new Map<string, string>()
|
const filenameMap = new Map<string, string>()
|
||||||
const normalizedGroups = new Map<string, Array<{ original: string; normalized: string }>>()
|
const normalizedGroups = new Map<string, Array<{ original: string; normalized: string }>>()
|
||||||
@@ -97,11 +136,115 @@ function rewriteGltfUris(value: JsonValue, filenameMap: Map<string, string>): Js
|
|||||||
return rewritten
|
return rewritten
|
||||||
}
|
}
|
||||||
|
|
||||||
function prepareModelBuffer(buffer: Buffer, filenameMap: Map<string, string>) {
|
function addKtx2TextureExtensions(value: JsonValue, ktx2Filenames: Set<string>) {
|
||||||
|
if (ktx2Filenames.size === 0 || !isJsonObject(value)) return value
|
||||||
|
|
||||||
|
const images = getJsonObjectArray(value.images)
|
||||||
|
const textures = getJsonObjectArray(value.textures)
|
||||||
|
if (!images || !textures) return value
|
||||||
|
|
||||||
|
let hasKtx2Texture = false
|
||||||
|
|
||||||
|
for (let imageIndex = 0; imageIndex < images.length; imageIndex++) {
|
||||||
|
const image = images[imageIndex]
|
||||||
|
const uri = image.uri
|
||||||
|
if (typeof uri !== 'string') continue
|
||||||
|
|
||||||
|
const filename = getReferencedFilename(uri)
|
||||||
|
if (!filename || !ktx2Filenames.has(filename)) continue
|
||||||
|
|
||||||
|
image.mimeType = 'image/ktx2'
|
||||||
|
|
||||||
|
for (const texture of textures) {
|
||||||
|
if (texture.source !== imageIndex) continue
|
||||||
|
|
||||||
|
const extensions = getOrCreateExtensions(texture)
|
||||||
|
extensions[KHR_TEXTURE_BASISU] = { source: imageIndex }
|
||||||
|
delete texture.source
|
||||||
|
hasKtx2Texture = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasKtx2Texture) {
|
||||||
|
addExtensionUsed(value, KHR_TEXTURE_BASISU)
|
||||||
|
const extensionsRequired = new Set(getStringArray(value.extensionsRequired))
|
||||||
|
extensionsRequired.add(KHR_TEXTURE_BASISU)
|
||||||
|
value.extensionsRequired = [...extensionsRequired]
|
||||||
|
}
|
||||||
|
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
function prepareModelBuffer(
|
||||||
|
buffer: Buffer,
|
||||||
|
filenameMap: Map<string, string>,
|
||||||
|
ktx2Filenames = new Set<string>(),
|
||||||
|
) {
|
||||||
if (filenameMap.size === 0) return buffer
|
if (filenameMap.size === 0) return buffer
|
||||||
|
|
||||||
const parsed = parseJsonValue(buffer.toString('utf-8'))
|
const parsed = parseJsonValue(buffer.toString('utf-8'))
|
||||||
return Buffer.from(JSON.stringify(rewriteGltfUris(parsed, filenameMap), null, 2), 'utf-8')
|
const rewritten = rewriteGltfUris(parsed, filenameMap)
|
||||||
|
const withKtx2 = addKtx2TextureExtensions(rewritten, ktx2Filenames)
|
||||||
|
|
||||||
|
return Buffer.from(JSON.stringify(withKtx2, null, 2), 'utf-8')
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatCompressionWarnings(warnings: string[]) {
|
||||||
|
if (warnings.length === 0) return undefined
|
||||||
|
if (warnings.length === 1) return warnings[0]
|
||||||
|
return `${warnings[0]} (${warnings.length - 1} autre(s) texture(s) en fallback WebP.)`
|
||||||
|
}
|
||||||
|
|
||||||
|
function reserveDeliveryFilename(filename: string, usedStems: Set<string>) {
|
||||||
|
const extension = extname(filename)
|
||||||
|
const stem = filename.slice(0, -extension.length)
|
||||||
|
let candidateStem = stem
|
||||||
|
let suffix = 2
|
||||||
|
|
||||||
|
while (usedStems.has(candidateStem.toLowerCase())) {
|
||||||
|
candidateStem = `${stem}_${suffix}`
|
||||||
|
suffix += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
usedStems.add(candidateStem.toLowerCase())
|
||||||
|
return `${candidateStem}${extension}`
|
||||||
|
}
|
||||||
|
|
||||||
|
async function prepareTexturePlans(
|
||||||
|
parsedFiles: ParsedFile[],
|
||||||
|
textureFilenameMap: Map<string, string>,
|
||||||
|
) {
|
||||||
|
const plans = new Map<string, PreparedTexturePlan>()
|
||||||
|
const warnings: string[] = []
|
||||||
|
const usedDeliveryStems = new Set<string>()
|
||||||
|
|
||||||
|
for (const pf of parsedFiles) {
|
||||||
|
const ext = extname(pf.filename).toLowerCase()
|
||||||
|
if (!TEXTURE_EXTENSIONS.has(ext)) continue
|
||||||
|
|
||||||
|
const normalizedFilename = textureFilenameMap.get(pf.filename.toLowerCase()) || pf.filename
|
||||||
|
const categoryFilename = normalizeTextureFilename(normalizedFilename) || normalizedFilename
|
||||||
|
const category = classifyAssetCategory(categoryFilename)
|
||||||
|
const deliveryFilename = reserveDeliveryFilename(normalizedFilename, usedDeliveryStems)
|
||||||
|
const delivery = await prepareTextureDelivery(deliveryFilename, pf.buffer)
|
||||||
|
|
||||||
|
if (delivery.warning) {
|
||||||
|
warnings.push(delivery.warning)
|
||||||
|
}
|
||||||
|
|
||||||
|
plans.set(pf.filename.toLowerCase(), {
|
||||||
|
filename: delivery.asset.filename,
|
||||||
|
buffer: delivery.asset.buffer,
|
||||||
|
category,
|
||||||
|
compressed: delivery.asset.compressed,
|
||||||
|
format: delivery.format,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
plans,
|
||||||
|
warning: formatCompressionWarnings(warnings),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function prepareSeparateFiles(
|
async function prepareSeparateFiles(
|
||||||
@@ -111,16 +254,30 @@ async function prepareSeparateFiles(
|
|||||||
) {
|
) {
|
||||||
const filesToPush: PushFile[] = []
|
const filesToPush: PushFile[] = []
|
||||||
const assetSummaries: PreparedAssetSummary[] = []
|
const assetSummaries: PreparedAssetSummary[] = []
|
||||||
|
const { plans: texturePlans, warning: textureCompressionWarning } = await prepareTexturePlans(
|
||||||
|
parsedFiles,
|
||||||
|
textureFilenameMap,
|
||||||
|
)
|
||||||
|
const deliveryFilenameMap = new Map(textureFilenameMap)
|
||||||
|
const ktx2Filenames = new Set<string>()
|
||||||
let modelFilename = ''
|
let modelFilename = ''
|
||||||
let compressed = false
|
let compressed = false
|
||||||
let compressionError: string | undefined
|
let compressionError: string | undefined = textureCompressionWarning
|
||||||
|
|
||||||
|
for (const [originalFilename, texturePlan] of texturePlans) {
|
||||||
|
deliveryFilenameMap.set(originalFilename, texturePlan.filename)
|
||||||
|
|
||||||
|
if (texturePlan.format === 'ktx2') {
|
||||||
|
ktx2Filenames.add(texturePlan.filename.toLowerCase())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (const pf of parsedFiles) {
|
for (const pf of parsedFiles) {
|
||||||
let content = pf.buffer
|
let content = pf.buffer
|
||||||
let filename = pf.filename
|
let filename = pf.filename
|
||||||
|
|
||||||
if (pf.isModel) {
|
if (pf.isModel) {
|
||||||
content = prepareModelBuffer(pf.buffer, textureFilenameMap)
|
content = prepareModelBuffer(pf.buffer, deliveryFilenameMap, ktx2Filenames)
|
||||||
modelFilename = pf.filename
|
modelFilename = pf.filename
|
||||||
|
|
||||||
assetSummaries.push({
|
assetSummaries.push({
|
||||||
@@ -128,24 +285,35 @@ async function prepareSeparateFiles(
|
|||||||
kind: 'model',
|
kind: 'model',
|
||||||
compressed: false,
|
compressed: false,
|
||||||
})
|
})
|
||||||
|
} else if (texturePlans.has(pf.filename.toLowerCase())) {
|
||||||
|
const texturePlan = texturePlans.get(pf.filename.toLowerCase())
|
||||||
|
if (!texturePlan) continue
|
||||||
|
|
||||||
|
compressed ||= texturePlan.compressed
|
||||||
|
|
||||||
|
assetSummaries.push({
|
||||||
|
filename: texturePlan.filename,
|
||||||
|
kind: 'texture',
|
||||||
|
category: texturePlan.category,
|
||||||
|
compressed: texturePlan.compressed,
|
||||||
|
})
|
||||||
|
|
||||||
|
filesToPush.push({
|
||||||
|
path: getModelAssetPath(folderName, texturePlan.filename),
|
||||||
|
contentBase64: texturePlan.buffer.toString('base64'),
|
||||||
|
})
|
||||||
|
|
||||||
|
continue
|
||||||
} else {
|
} else {
|
||||||
filename = textureFilenameMap.get(pf.filename.toLowerCase()) || pf.filename
|
filename = textureFilenameMap.get(pf.filename.toLowerCase()) || pf.filename
|
||||||
const categoryFilename = textureFilenameMap.get(pf.filename.toLowerCase()) || normalizeTextureFilename(pf.filename) || pf.filename
|
const categoryFilename = textureFilenameMap.get(pf.filename.toLowerCase()) || normalizeTextureFilename(pf.filename) || pf.filename
|
||||||
const category = classifyAssetCategory(categoryFilename)
|
const category = classifyAssetCategory(categoryFilename)
|
||||||
|
|
||||||
const textureResult = await compressTextureBuffer(filename, pf.buffer)
|
|
||||||
content = textureResult.buffer
|
|
||||||
compressed ||= textureResult.compressed
|
|
||||||
|
|
||||||
if (textureResult.error && !compressionError) {
|
|
||||||
compressionError = textureResult.error
|
|
||||||
}
|
|
||||||
|
|
||||||
assetSummaries.push({
|
assetSummaries.push({
|
||||||
filename,
|
filename,
|
||||||
kind: category === 'assets' ? 'asset' : 'texture',
|
kind: category === 'assets' ? 'asset' : 'texture',
|
||||||
category,
|
category,
|
||||||
compressed: textureResult.compressed,
|
compressed: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+156
-2
@@ -1,13 +1,133 @@
|
|||||||
import { extname } from 'path'
|
import { randomUUID } from 'crypto'
|
||||||
|
import { spawn } from 'child_process'
|
||||||
|
import { mkdir, readFile, rm } from 'fs/promises'
|
||||||
|
import { extname, join } from 'path'
|
||||||
|
import { tmpdir } from 'os'
|
||||||
import sharp from 'sharp'
|
import sharp from 'sharp'
|
||||||
import { getErrorMessage } from './guards'
|
import { getErrorMessage } from './guards'
|
||||||
|
|
||||||
interface TextureCompressionResult {
|
export interface TextureCompressionResult {
|
||||||
buffer: Buffer
|
buffer: Buffer
|
||||||
compressed: boolean
|
compressed: boolean
|
||||||
error?: string
|
error?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface TextureDeliveryAsset {
|
||||||
|
filename: string
|
||||||
|
buffer: Buffer
|
||||||
|
compressed: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TextureDeliveryResult {
|
||||||
|
asset: TextureDeliveryAsset
|
||||||
|
format: 'ktx2' | 'webp' | 'original'
|
||||||
|
warning?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_TEXTURE_SIZE = 1024
|
||||||
|
const WEBP_QUALITY = 82
|
||||||
|
const KTX2_EXTENSION = '.ktx2'
|
||||||
|
const WEBP_EXTENSION = '.webp'
|
||||||
|
|
||||||
|
function getTextureMaxSize() {
|
||||||
|
const value = Number(process.env.TEXTURE_MAX_SIZE || DEFAULT_TEXTURE_SIZE)
|
||||||
|
return Number.isFinite(value) && value > 0 ? value : DEFAULT_TEXTURE_SIZE
|
||||||
|
}
|
||||||
|
|
||||||
|
function replaceExtension(filename: string, extension: string) {
|
||||||
|
return filename.replace(/\.[^.]+$/, extension)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTextureFamily(filename: string) {
|
||||||
|
return filename
|
||||||
|
.replace(/\.[^.]+$/, '')
|
||||||
|
.split(/[_-]/)[0]
|
||||||
|
.toLowerCase()
|
||||||
|
}
|
||||||
|
|
||||||
|
function getKtx2Flags(filename: string) {
|
||||||
|
const family = getTextureFamily(filename)
|
||||||
|
const flags = ['--t2', '--genmipmap']
|
||||||
|
|
||||||
|
if (family === 'color' || family === 'diffuse') {
|
||||||
|
flags.push('--assign_oetf', 'srgb')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (family === 'normal') {
|
||||||
|
flags.push('--encode', 'uastc', '--zcmp', '18')
|
||||||
|
return flags
|
||||||
|
}
|
||||||
|
|
||||||
|
flags.push('--encode', 'etc1s', '--clevel', '2', '--qlevel', '128')
|
||||||
|
return flags
|
||||||
|
}
|
||||||
|
|
||||||
|
function runToktx(args: string[]) {
|
||||||
|
const toktxPath = process.env.TOKTX_PATH || 'toktx'
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
const output: string[] = []
|
||||||
|
const proc = spawn(toktxPath, args)
|
||||||
|
|
||||||
|
proc.stdout.on('data', (data: Buffer) => output.push(data.toString()))
|
||||||
|
proc.stderr.on('data', (data: Buffer) => output.push(data.toString()))
|
||||||
|
proc.on('error', (err) => reject(err))
|
||||||
|
proc.on('close', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolve()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
reject(new Error(output.join('').trim() || `toktx exited with code ${code}`))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeResizedPng(input: Buffer, outputPath: string) {
|
||||||
|
await sharp(input)
|
||||||
|
.resize(getTextureMaxSize(), getTextureMaxSize(), { fit: 'inside', withoutEnlargement: true })
|
||||||
|
.png()
|
||||||
|
.toFile(outputPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function compressTextureToKtx2(filename: string, buffer: Buffer): Promise<TextureDeliveryAsset> {
|
||||||
|
const tmpFolder = join(/* turbopackIgnore: true */ tmpdir(), `upload-gltf-ktx2-${randomUUID()}`)
|
||||||
|
const inputPath = join(/* turbopackIgnore: true */ tmpFolder, 'texture.png')
|
||||||
|
const outputFilename = replaceExtension(filename, KTX2_EXTENSION)
|
||||||
|
const outputPath = join(/* turbopackIgnore: true */ tmpFolder, outputFilename)
|
||||||
|
|
||||||
|
await mkdir(/* turbopackIgnore: true */ tmpFolder, { recursive: true })
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeResizedPng(buffer, inputPath)
|
||||||
|
await runToktx([...getKtx2Flags(filename), outputPath, inputPath])
|
||||||
|
|
||||||
|
return {
|
||||||
|
filename: outputFilename,
|
||||||
|
buffer: await readFile(/* turbopackIgnore: true */ outputPath),
|
||||||
|
compressed: true,
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await rm(/* turbopackIgnore: true */ tmpFolder, { recursive: true, force: true }).catch((err) => {
|
||||||
|
console.warn('[WARN] KTX2 temp cleanup failed', {
|
||||||
|
filename,
|
||||||
|
error: getErrorMessage(err),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function compressTextureToWebp(filename: string, buffer: Buffer): Promise<TextureDeliveryAsset> {
|
||||||
|
return {
|
||||||
|
filename: replaceExtension(filename, WEBP_EXTENSION),
|
||||||
|
buffer: await sharp(buffer)
|
||||||
|
.resize(getTextureMaxSize(), getTextureMaxSize(), { fit: 'inside', withoutEnlargement: true })
|
||||||
|
.webp({ quality: WEBP_QUALITY })
|
||||||
|
.toBuffer(),
|
||||||
|
compressed: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function compressTextureBuffer(
|
export async function compressTextureBuffer(
|
||||||
filename: string,
|
filename: string,
|
||||||
buffer: Buffer,
|
buffer: Buffer,
|
||||||
@@ -46,3 +166,37 @@ export async function compressTextureBuffer(
|
|||||||
|
|
||||||
return { buffer, compressed: false }
|
return { buffer, compressed: false }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function prepareTextureDelivery(
|
||||||
|
filename: string,
|
||||||
|
buffer: Buffer,
|
||||||
|
): Promise<TextureDeliveryResult> {
|
||||||
|
try {
|
||||||
|
return {
|
||||||
|
asset: await compressTextureToKtx2(filename, buffer),
|
||||||
|
format: 'ktx2',
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const ktx2Error = getErrorMessage(err, String(err))
|
||||||
|
|
||||||
|
try {
|
||||||
|
return {
|
||||||
|
asset: await compressTextureToWebp(filename, buffer),
|
||||||
|
format: 'webp',
|
||||||
|
warning: `Compression KTX2 impossible pour ${filename}. Fallback WebP utilise. Detail : ${ktx2Error}`,
|
||||||
|
}
|
||||||
|
} catch (webpErr) {
|
||||||
|
const webpError = getErrorMessage(webpErr, String(webpErr))
|
||||||
|
|
||||||
|
return {
|
||||||
|
asset: {
|
||||||
|
filename,
|
||||||
|
buffer,
|
||||||
|
compressed: false,
|
||||||
|
},
|
||||||
|
format: 'original',
|
||||||
|
warning: `Compression KTX2 impossible pour ${filename}. Compression WebP impossible. Texture originale conservee. Details : ${ktx2Error} / ${webpError}`,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user