| Server IP : 217.154.3.148 / Your IP : 216.73.216.90 Web Server : Apache System : Linux blissful-wright.217-154-3-148.plesk.page 6.8.0-124-generic #124-Ubuntu SMP PREEMPT_DYNAMIC Tue May 26 13:00:45 UTC 2026 x86_64 User : gracious-boyd_hfhh1h8vo9n ( 10000) PHP Version : 8.3.31 Disable Function : opcache_get_status MySQL : OFF | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : /var/www/vhosts/gracious-boyd.217-154-3-148.plesk.page/nextjs/components/ |
Upload File : |
'use client';
/* eslint-disable @next/next/no-img-element -- canvas overlays and data URLs require <img> */
import { processMask } from './MaskPost'
import {
boxBlur,
morphDilate,
morphErode,
maskAnd,
maskMinus,
maskOr,
sobel,
removeSmallComponents,
fillSmallHoles,
} from '../utils/maskOps'
import { refineMaskEdges } from '../lib/render/mask_edge_refine'
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { buildGroutMaskFromGrid, subtractGroutFromTiles } from '../lib/render/grout_mask'
import { buildLightMap, lumaFromRgba } from '../lib/render/light_map'
import { buildShadowMap } from '../lib/render/shadow_map'
import { buildSpecularMap } from '../lib/render/specular_map'
import { shadeTileOverlayInPlace } from '../lib/render/composite'
import { renderTilePatternImageData } from '../lib/render/tile_pattern'
type RansacAxisMeta = { T?: number | null; a?: number | null; conf?: number | null };
type RansacMeta = {
mode?: string;
coverage?: number | null;
v?: RansacAxisMeta;
h?: RansacAxisMeta;
};
type MixMeta = {
mode?: string;
wu?: number;
wh?: number;
fblur?: number;
he_mask_available?: boolean;
heur_error?: string;
note?: string;
u2_source?: string | null;
he_source?: string | null;
weights?: { u2: number; heur: number };
fblur_used?: number;
u2_coverage?: number;
heur_coverage?: number;
weight_adjustments?: string[];
};
type GridPolySegment = {
x1: number;
y1: number;
x2: number;
y2: number;
};
type GridPoly = {
v?: GridPolySegment[];
h?: GridPolySegment[];
};
type OrtWasmConfig = {
wasmPaths?: string;
simd?: boolean;
numThreads?: number;
};
type OrtModule = typeof import('onnxruntime-web') & { env?: { wasm?: OrtWasmConfig } };
type OrtGlobal = typeof globalThis & { ort?: { env?: { wasm?: OrtWasmConfig } } };
type LightMaskResponse = { shadow_mask?: string | null; light_mask?: string | null };
type MaskPreset = 'tiles-default' | 'tiles-detail';
type MaskMeta = {
polarity?: 'white' | 'black';
coverage?: number | null;
preset?: MaskPreset | string | null;
tag?: string;
source?: string | null;
};
type TileMaskMeta = {
classes: string[];
colorMap: Record<string, [number, number, number]>;
coverage?: Record<string, number>;
coverageAbsolute?: Record<string, number>;
rawCoverage?: Record<string, number>;
occluderCoverage?: number | null;
occludedPixels?: number | null;
occluderPixels?: number | null;
occludedCoverage?: number | null;
model?: string | null;
source?: string | null;
};
type TileMaskVersions = {
combined?: string | null;
raw?: string | null;
occluder?: string | null;
};
type TileMaskViewMode = 'combined' | 'raw' | 'occluder';
type TileSegModel = 'unet_latest' | 'unet_old' | 'u2net' | 'none';
const TILE_MASK_VIEW_LABELS: Record<TileMaskViewMode, string> = {
combined: 'Final mask',
raw: 'Raw UNet',
occluder: 'Occluder',
};
const TILE_MASK_VIEW_TITLES: Record<TileMaskViewMode, string> = {
combined: 'Combined tile mask (tiles without occluder overlaps).',
raw: 'Raw tiles directly from the UNet before occluder removal.',
occluder: 'Detected occluder mask (areas that block tiles).',
};
const COVERAGE_DESCRIPTIONS = {
interior: 'Tile/grout coverage normalized to the interior (occluder pixels removed).',
absolute: 'Tile/grout coverage relative to the entire image area.',
occluder: 'Share of the total image that is occupied by occluders.',
occluded: 'Share of the tile area that was reclassified as occluded.',
};
const MAX_UPLOAD_DIMENSION = Number.isFinite(Number(process.env.NEXT_PUBLIC_MAX_UPLOAD_DIM ?? '2048'))
? Number(process.env.NEXT_PUBLIC_MAX_UPLOAD_DIM ?? '2048')
: 2048;
const TILE_MODEL_OPTIONS: { value: TileSegModel; label: string }[] = [
{ value: 'unet_latest', label: 'UNet latest' },
{ value: 'unet_old', label: 'UNet legacy' },
{ value: 'u2net', label: 'U²-Net' },
{ value: 'none', label: 'None' },
];
async function downscaleImageFileIfNeeded(file: File, maxDimension: number) {
const firstUrl = URL.createObjectURL(file);
try {
if (!(maxDimension > 0)) {
return { file, url: firstUrl, downscaled: false };
}
const img = await loadImage(firstUrl);
const largestSide = Math.max(img.naturalWidth || 0, img.naturalHeight || 0);
if (!(largestSide > maxDimension)) {
return { file, url: firstUrl, downscaled: false };
}
const scale = maxDimension / largestSide;
const targetWidth = Math.max(1, Math.round((img.naturalWidth || 0) * scale));
const targetHeight = Math.max(1, Math.round((img.naturalHeight || 0) * scale));
const canvas = document.createElement('canvas');
canvas.width = targetWidth;
canvas.height = targetHeight;
const ctx = canvas.getContext('2d');
if (!ctx) {
return { file, url: firstUrl, downscaled: false };
}
ctx.drawImage(img, 0, 0, targetWidth, targetHeight);
const blob: Blob = await new Promise((resolve, reject) => {
canvas.toBlob((b) => (b ? resolve(b) : reject(new Error('toBlob failed'))), file.type || 'image/jpeg', 0.92);
});
const processedFile = new File([blob], file.name, { type: blob.type, lastModified: Date.now() });
const processedUrl = URL.createObjectURL(processedFile);
URL.revokeObjectURL(firstUrl);
return { file: processedFile, url: processedUrl, downscaled: true };
} catch (error) {
URL.revokeObjectURL(firstUrl);
console.warn('downscale failed, falling back to original', error);
const fallbackUrl = URL.createObjectURL(file);
return { file, url: fallbackUrl, downscaled: false };
}
}
type AnalyzeResponse = {
ok: boolean;
image: { width: number; height: number };
tile_w_px: number;
tile_h_px: number;
grout_px: number;
tile_w_mm?: number | null;
tile_h_mm?: number | null;
grout_mm?: number | null;
grid: { vlines: number[]; hlines: number[] };
source?: string;
tile_model?: string | null;
fg_mask?: string | null;
fg_mask_meta?: MaskMeta | null;
debug_overlay?: string | null;
name?: string;
type?: string;
ransac?: RansacMeta;
mix?: MixMeta;
grid_poly?: GridPoly | null;
mask_preset?: MaskPreset | null;
mask_versions?: {
normalized?: string | null;
fused?: string | null;
raw?: string | null;
overlay?: string | null;
cutout?: string | null;
} | null;
tile_mask?: string | null;
tile_mask_meta?: TileMaskMeta | null;
tile_mask_versions?: TileMaskVersions | null;
mask_metrics?: {
coverage?: number | null;
coverageNorm?: number | null;
coverageRaw?: number | null;
backgroundCoverage?: number | null;
occluderCoverage?: number | null;
seedCoverage?: number | null;
fusedCoverage?: number | null;
fusedMean?: number | null;
preset?: MaskPreset | string | null;
latticeLeak?: number | null;
latticeLeakPct?: number | null;
latticeInteriorCoverage?: number | null;
warnings?: string[] | null;
autotuneMs?: number | null;
source?: string | null;
} | null;
mask_debug?: any;
};
type MaskTelemetryPayload = {
analysisId: string;
presetRequested: MaskPreset | string | null;
presetApplied: MaskPreset | string | null;
coverage: number | null;
coverageNorm: number | null;
coverageRaw: number | null;
backgroundCoverage: number | null;
seedCoverage: number | null;
interiorCoverage: number | null;
leakPct: number | null;
warnings: string[];
autotuneMs: number | null;
latencyMs: number | null;
imageWidth: number | null;
imageHeight: number | null;
tileWidth: number | null;
tileHeight: number | null;
groutWidth: number | null;
maskLattice: boolean;
maskIllumination: boolean;
maskAutotune: boolean;
maskAutotuneBudget: number;
maskThinFrac: number;
maskMinAreaFrac: number;
ransacMode: string;
wu: number;
wh: number;
fblur: number;
source: string | null;
fileBytes: number | null;
fileType: string | null;
mixMode: string | null;
mixWu: number | null;
mixWh: number | null;
mixBlur: number | null;
mixU2Coverage: number | null;
mixHeurCoverage: number | null;
fallbackUsed: boolean;
fallbackReasons: string[];
fallbackOriginalCoverage: number | null;
fallbackPipelineCoverage: number | null;
fallbackLeakPct: number | null;
tileMaskTiles?: number | null;
tileMaskGrout?: number | null;
tileMaskTilesRaw?: number | null;
tileMaskGroutRaw?: number | null;
tileMaskOccluder?: number | null;
tileMaskTilesAbs?: number | null;
tileMaskGroutAbs?: number | null;
tileMaskOccluded?: number | null;
tileMaskOccluderPixels?: number | null;
tileMaskOccludedPixels?: number | null;
tileModel?: string | null;
};
const BASE_PATH = (process.env.NEXT_PUBLIC_BASE_PATH || '').replace(/\/$/, '');
const TELEMETRY_ENDPOINT = `${BASE_PATH}/api/mask-telemetry`;
const CLIENT_VERSION = process.env.NEXT_PUBLIC_APP_VERSION ?? null;
function runtimeBasePath(): string {
if (BASE_PATH) return BASE_PATH;
if (typeof window === 'undefined') return '/nextjs';
const parts = window.location.pathname.split('/').filter(Boolean);
if (parts.length > 0) return `/${parts[0]}`;
return '/nextjs';
}
function dispatchMaskTelemetry(payload: MaskTelemetryPayload) {
try {
const body = JSON.stringify({
...payload,
clientVersion: CLIENT_VERSION ?? undefined,
});
if (typeof navigator !== 'undefined' && typeof navigator.sendBeacon === 'function') {
try {
const blob = new Blob([body], { type: 'application/json' });
const sent = navigator.sendBeacon(TELEMETRY_ENDPOINT, blob);
if (sent) return;
} catch (err) {
console.warn('mask telemetry beacon failed', err);
}
}
void fetch(TELEMETRY_ENDPOINT, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body,
keepalive: true,
}).catch((err) => {
console.warn('mask telemetry post failed', err);
});
} catch (err) {
console.warn('mask telemetry serialization failed', err);
}
}
const MASK_FALLBACK_THRESHOLD = Number(process.env.NEXT_PUBLIC_MASK_FALLBACK_THRESHOLD ?? '0.14');
const OCCLUDER_GROW_PX = 0;
// Hard cut-out for tiles: any feather/blur here makes the tile grid "ghost" over occluder edges.
const OCCLUDER_FEATHER_PX = 0;
const OCCLUDER_ALPHA_THRESHOLD = 160;
const OCCLUDER_NO_PAINT_DILATE_PX = 2;
const OCCLUDER_LEAK_GUARD_PX = 2;
const OCCLUDER_COMPOSITE_ERODE_PX = 2;
const OCCLUDER_MIN_COVERAGE = 0.18;
const OCCLUDER_TARGET_COVERAGE = 0.28;
const OCCLUDER_MAX_COVERAGE = 0.48;
const OCCLUDER_AUTO_GROW_MAX_RADIUS = 14;
type MaskPreference = 'auto' | 'processed' | 'server' | 'client' | 'fused' | 'raw';
type Pattern = { id: string; name: string; src: string };
type TileDesign = { id: string; name: string; src: string };
/** Einfache eingebaute Muster */
const PATTERNS: Pattern[] = [
{
id: 'checker',
name: 'Checker',
src:
'data:image/svg+xml;utf8,' +
encodeURIComponent(
`<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">
<rect width="64" height="64" fill="#d8d8d8"/>
<rect width="32" height="32" x="0" y="0" fill="#b9b9b9"/>
<rect width="32" height="32" x="32" y="32" fill="#b9b9b9"/>
</svg>`
),
},
{
id: 'light',
name: 'Light',
src:
'data:image/svg+xml;utf8,' +
encodeURIComponent(
`<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">
<rect width="64" height="64" fill="#e6e6e6"/>
</svg>`
),
},
{
id: 'dark',
name: 'Dark',
src:
'data:image/svg+xml;utf8,' +
encodeURIComponent(
`<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">
<rect width="64" height="64" fill="#a8a8a8"/>
</svg>`
),
},
];
const TILE_DESIGNS: TileDesign[] = [
{ id: 'white_glossy', name: 'White glossy', src: `/tile_designs/white_glossy.svg` },
{ id: 'stone_gray', name: 'Stone gray', src: `/tile_designs/stone_gray.svg` },
{ id: 'demo_blue', name: 'Demo blue', src: `/tile_designs/demo_blue.svg` },
];
function computeIsOblique(gridPoly: GridPoly | null | undefined): boolean {
if (!gridPoly) return false;
const segments: Array<{ dx: number; dy: number; kind: 'v' | 'h' }> = [];
for (const seg of gridPoly.v ?? []) {
segments.push({ dx: seg.x2 - seg.x1, dy: seg.y2 - seg.y1, kind: 'v' });
}
for (const seg of gridPoly.h ?? []) {
segments.push({ dx: seg.x2 - seg.x1, dy: seg.y2 - seg.y1, kind: 'h' });
}
if (segments.length < 4) return false;
const deviations: number[] = [];
for (const s of segments) {
const ang = Math.abs(Math.atan2(s.dy, s.dx)) * (180 / Math.PI); // 0..180
const dev =
s.kind === 'h' ? Math.min(ang, Math.abs(180 - ang)) : Math.abs(90 - ang);
if (Number.isFinite(dev)) deviations.push(dev);
}
if (!deviations.length) return false;
deviations.sort((a, b) => a - b);
const median = deviations[Math.floor(deviations.length / 2)] ?? 0;
return median >= 10; // ~10°+ implies oblique/perspective view
}
function loadImage(url: string): Promise<HTMLImageElement> {
return new Promise((res, rej) => {
const img = new Image();
img.onload = () => res(img);
img.onerror = (e) => rej(e);
img.crossOrigin = 'anonymous';
img.src = url;
});
}
/** Graustufen-Maskenbild -> Canvas, dessen ALPHA = Grau (0..255) ist */
async function prepareMaskAlphaCanvas(
maskUrl: string,
w: number,
h: number,
opts: { mode?: 'raw' | 'auto' } = {},
): Promise<HTMLCanvasElement> {
const mimg = await loadImage(maskUrl);
const mc = document.createElement('canvas');
mc.width = w; mc.height = h;
const mctx = mc.getContext('2d')!;
(mctx as any).imageSmoothingEnabled = false;
// Maske in Zielgröße ziehen
mctx.drawImage(mimg, 0, 0, w, h);
const data = mctx.getImageData(0, 0, w, h);
const p = data.data;
// Analyse channel statistics to cope with masks that use grayscale instead of alpha.
let alphaMin = 255;
let alphaMax = 0;
let redMin = 255;
let redMax = 0;
for (let i = 0; i < p.length; i += 4) {
const alpha = p[i + 3];
if (alpha < alphaMin) alphaMin = alpha;
if (alpha > alphaMax) alphaMax = alpha;
const red = p[i];
if (red < redMin) redMin = red;
if (red > redMax) redMax = red;
}
const alphaVaries = alphaMax > alphaMin;
const redVaries = redMax > redMin;
const useAlpha = alphaVaries || !redVaries ? true : false;
let channelMin = useAlpha ? alphaMin : redMin;
let channelMax = useAlpha ? alphaMax : redMax;
if (channelMax === channelMin) channelMax = channelMin + 1; // avoid div-by-zero
const channelRange = channelMax - channelMin;
// Estimate raw coverage relative to the active channel.
let solid = 0;
for (let i = 0; i < p.length; i += 4) {
const raw = useAlpha ? p[i + 3] : p[i];
const norm = (raw - channelMin) / channelRange;
if (norm >= 0.5) solid += 1;
}
const rawCoverage = solid / (w * h);
const invert = rawCoverage > 0.98;
const effectiveCoverage = invert ? Math.max(0, Math.min(1, 1 - rawCoverage)) : rawCoverage;
const mode = opts.mode ?? 'auto';
if (mode === 'raw') {
solid = 0;
for (let i = 0; i < p.length; i += 4) {
const raw = useAlpha ? p[i + 3] : p[i];
let norm = (raw - channelMin) / channelRange;
norm = Math.max(0, Math.min(1, norm));
if (invert) norm = 1 - norm;
const alpha = Math.round(norm * 255);
p[i] = 255; p[i + 1] = 255; p[i + 2] = 255;
p[i + 3] = alpha;
if (alpha >= 128) solid += 1;
}
mctx.putImageData(data, 0, 0);
const storedCoverage = solid / (w * h);
mc.dataset.maskCoverage = storedCoverage.toFixed(3);
mc.dataset.maskCoverageRaw = storedCoverage.toString();
mc.dataset.maskCoverageRawInput = effectiveCoverage.toString();
mc.dataset.maskCoverageTarget = storedCoverage.toFixed(3);
mc.dataset.maskThreshold = '0.5';
mc.dataset.maskSource = maskUrl;
mc.dataset.maskInverted = invert ? 'true' : 'false';
mc.dataset.maskTag = mc.dataset.maskTag ?? 'maskRaw';
return mc;
}
// Normalize resulting alpha to the full 0..255 range for stronger masking.
let normMin = 1;
let normMax = 0;
let nearZero = 0;
let nearOne = 0;
for (let i = 0; i < p.length; i += 4) {
const raw = useAlpha ? p[i + 3] : p[i];
let norm = (raw - channelMin) / channelRange;
norm = Math.max(0, Math.min(1, norm));
if (invert) norm = 1 - norm;
if (norm < normMin) normMin = norm;
if (norm > normMax) normMax = norm;
if (norm <= 0.05) nearZero += 1;
if (norm >= 0.95) nearOne += 1;
}
solid = 0;
const baseCoverage = effectiveCoverage;
const binaryRatio = (nearZero + nearOne) / Math.max(1, (w * h));
const assumeBinary = binaryRatio > 0.985 && nearZero > 0 && nearOne > 0;
let boostGamma = baseCoverage < 0.05 ? 0.32 : baseCoverage < 0.12 ? 0.38 : baseCoverage < 0.24 ? 0.45 : 0.52;
let solidThreshold = baseCoverage < 0.05 ? 0.32 : baseCoverage < 0.12 ? 0.36 : baseCoverage < 0.24 ? 0.42 : 0.48;
if (assumeBinary) {
boostGamma = 1;
solidThreshold = 0.5;
}
const targetCoverage = Math.min(0.95, Math.max(baseCoverage * 1.2, baseCoverage + 0.02));
const thresholdScore = solidThreshold;
const floorThreshold = assumeBinary ? solidThreshold : Math.max(0.04, solidThreshold * 0.35);
for (let i = 0; i < p.length; i += 4) {
const raw = useAlpha ? p[i + 3] : p[i];
let norm = (raw - channelMin) / channelRange;
norm = Math.max(0, Math.min(1, norm));
if (invert) norm = 1 - norm;
const boosted = Math.pow(norm, boostGamma);
let hard: number;
if (boosted >= solidThreshold) {
hard = 1;
} else if (boosted <= floorThreshold) {
hard = 0;
} else {
const span = Math.max(1e-4, solidThreshold - floorThreshold);
hard = Math.max(0, Math.min(1, (boosted - floorThreshold) / span));
}
const alpha = Math.round(hard * 255);
if (hard >= 0.5) solid += 1;
p[i] = 255; p[i + 1] = 255; p[i + 2] = 255;
p[i + 3] = alpha;
}
mctx.putImageData(data, 0, 0);
const storedCoverage = solid / (w * h);
mc.dataset.maskCoverage = storedCoverage.toFixed(3);
mc.dataset.maskCoverageRaw = storedCoverage.toString();
mc.dataset.maskCoverageRawInput = effectiveCoverage.toString();
mc.dataset.maskCoverageTarget = targetCoverage.toFixed(3);
mc.dataset.maskThreshold = thresholdScore.toFixed(3);
mc.dataset.maskSource = maskUrl;
mc.dataset.maskInverted = invert ? 'true' : 'false';
const stats = {
alphaMin,
alphaMax,
redMin,
redMax,
useAlpha,
channelMin,
channelMax,
rawCoverage: effectiveCoverage.toFixed(3),
storedCoverage: storedCoverage.toFixed(3),
targetCoverage: targetCoverage.toFixed(3),
threshold: thresholdScore.toFixed(3),
invert,
normMin: normMin.toFixed(3),
normMax: normMax.toFixed(3),
binaryRatio: binaryRatio.toFixed(3),
assumeBinary,
};
if (storedCoverage < 0.02 || storedCoverage > 0.98) {
console.info('[mask] coverage extreme', stats);
} else {
console.log('[mask] prepared', stats);
}
return mc;
}
async function prepareTileClassAlphaCanvas(
tileMaskUrl: string,
tileMaskMeta: TileMaskMeta,
className: string,
w: number,
h: number,
): Promise<HTMLCanvasElement> {
const img = await loadImage(tileMaskUrl);
const mc = document.createElement('canvas');
mc.width = w;
mc.height = h;
const mctx = mc.getContext('2d', { willReadFrequently: true })!;
(mctx as any).imageSmoothingEnabled = false;
mctx.drawImage(img, 0, 0, w, h);
const data = mctx.getImageData(0, 0, w, h).data;
// Prefer id-based lookup when available (server may expose `classIdLegend` with id->name).
const legend = (tileMaskMeta as any)?.classIdLegend as Record<string, string> | undefined;
const resolvedId = legend
? Object.entries(legend).find(([, name]) => String(name).toLowerCase() === className.toLowerCase())?.[0] ?? null
: null;
const classIndex = tileMaskMeta.classes.indexOf(className);
const rgb =
(resolvedId ? tileMaskMeta.colorMap?.[String(resolvedId)] : null) ??
(classIndex >= 0 ? tileMaskMeta.colorMap?.[String(classIndex)] : null) ??
null;
if (!rgb) {
const empty = new Uint8Array(w * h);
const canvas = canvasFromMask(empty, w, h);
canvas.dataset.maskTag = `tile:${className}:no-color`;
return canvas;
}
const out = new Uint8Array(w * h);
const [tr, tg, tb] = rgb;
for (let i = 0, p = 0; i < data.length; i += 4, p++) {
const r = data[i];
const g = data[i + 1];
const b = data[i + 2];
// Tolerate tiny interpolation drift (should be 0 with imageSmoothingDisabled, but be safe).
out[p] = Math.abs(r - tr) <= 2 && Math.abs(g - tg) <= 2 && Math.abs(b - tb) <= 2 ? 255 : 0;
}
const canvas = canvasFromMask(out, w, h);
canvas.dataset.maskTag = `tile:${className}`;
return canvas;
}
async function prepareIgnoreMaskCanvas(
tileMaskUrl: string,
tileMaskMeta: TileMaskMeta,
w: number,
h: number,
): Promise<HTMLCanvasElement | null> {
const legend = (tileMaskMeta as any)?.classIdLegend as Record<string, string> | undefined;
const ignoreId =
legend
? Object.entries(legend).find(([, name]) => String(name).toLowerCase().includes('ignore'))?.[0] ?? null
: null;
const candidateId = ignoreId ?? (tileMaskMeta.colorMap?.['255'] ? '255' : null);
if (!candidateId) return null;
const rgb = tileMaskMeta.colorMap?.[String(candidateId)] ?? null;
if (!rgb) return null;
const img = await loadImage(tileMaskUrl);
const mc = document.createElement('canvas');
mc.width = w;
mc.height = h;
const mctx = mc.getContext('2d', { willReadFrequently: true })!;
(mctx as any).imageSmoothingEnabled = false;
mctx.drawImage(img, 0, 0, w, h);
const data = mctx.getImageData(0, 0, w, h).data;
const [tr, tg, tb] = rgb;
const out = new Uint8Array(w * h);
for (let i = 0, p = 0; i < data.length; i += 4, p++) {
const r = data[i];
const g = data[i + 1];
const b = data[i + 2];
out[p] = Math.abs(r - tr) <= 2 && Math.abs(g - tg) <= 2 && Math.abs(b - tb) <= 2 ? 255 : 0;
}
const canvas = canvasFromMask(out, w, h);
canvas.dataset.maskTag = `tile:ignore`;
return canvas;
}
function extractBinaryMask(canvas: HTMLCanvasElement, invert: boolean): Uint8Array | null {
const width = canvas.width;
const height = canvas.height;
if (!width || !height) return null;
const ctx = canvas.getContext('2d', { willReadFrequently: true });
if (!ctx) return null;
const image = ctx.getImageData(0, 0, width, height);
const src = image.data;
const binary = new Uint8Array(width * height);
const threshold = OCCLUDER_ALPHA_THRESHOLD;
let solidCount = 0;
for (let i = 0, j = 0; i < binary.length; i++, j += 4) {
const alpha = src[j + 3];
const solid = alpha >= threshold ? 1 : 0;
const value = invert ? (solid ? 0 : 255) : solid ? 255 : 0;
binary[i] = value;
if (value) solidCount += 1;
}
if (solidCount === 0 && threshold > 1) {
for (let i = 0, j = 0; i < binary.length; i++, j += 4) {
const alpha = src[j + 3];
const solid = alpha > 0 ? 1 : 0;
const value = invert ? (solid ? 0 : 255) : solid ? 255 : 0;
binary[i] = value;
}
}
return binary;
}
function buildGridRoiMask(
grid: { vlines?: number[]; hlines?: number[] } | null,
band: { yMin: number; yMax: number } | null,
width: number,
height: number,
): Uint8Array | null {
if (!grid || !band) return null;
const v = (grid.vlines ?? []).filter((n) => Number.isFinite(n));
if (v.length < 2) return null;
const xMin = Math.max(0, Math.min(...v));
const xMax = Math.min(width, Math.max(...v));
const yMin = Math.max(0, Math.min(height, Math.round(band.yMin)));
const yMax = Math.max(yMin, Math.min(height, Math.round(band.yMax)));
if (xMax - xMin < Math.round(width * 0.05)) return null;
if (yMax - yMin < Math.round(height * 0.05)) return null;
const mask = new Uint8Array(width * height);
for (let y = yMin; y < yMax; y++) {
const row = y * width;
for (let x = xMin; x < xMax; x++) {
mask[row + x] = 255;
}
}
return mask;
}
function refineOccluderEdges(
occluder: Uint8Array,
rgba: Uint8ClampedArray,
width: number,
height: number,
trimPx: number,
edgeThreshold = 28,
edgeRejectMask?: Uint8Array | null,
): Uint8Array {
const radius = Math.max(0, Math.round(trimPx));
if (!radius) return occluder;
const luma = lumaFromRgba(rgba, width, height);
const blurRadius = Math.max(2, Math.round(radius * 0.75));
const blurred = boxBlur(luma, width, height, blurRadius);
const lumaBlur = new Uint8Array(blurred.length);
for (let i = 0; i < blurred.length; i++) {
const v = blurred[i] ?? 0;
lumaBlur[i] = v < 0 ? 0 : v > 255 ? 255 : Math.round(v);
}
const edges = sobel(lumaBlur, width, height);
const threshold = Math.max(18, Math.min(100, Math.round(edgeThreshold + radius * 5)));
const edgeMask = new Uint8Array(edges.length);
for (let i = 0; i < edges.length; i++) {
edgeMask[i] = edges[i] >= threshold ? 255 : 0;
}
if (edgeRejectMask && edgeRejectMask.length === edgeMask.length) {
for (let i = 0; i < edgeMask.length; i++) {
if (edgeRejectMask[i]) edgeMask[i] = 0;
}
}
const eroded = morphErode(occluder, width, height, radius);
const outerBand = maskMinus(occluder, eroded);
const edgeSupport = maskAnd(edgeMask, outerBand);
const edgeCore = morphDilate(edgeSupport, width, height, Math.max(1, Math.round(radius * 0.5)));
const trimmed = maskOr(eroded, edgeCore);
const trimmedCoverage = coverageOfMask(trimmed);
if (trimmedCoverage < 0.02) return occluder;
return trimmed;
}
function coverageOfMask(mask: Uint8Array | null): number {
if (!mask || !mask.length) return 0;
let solid = 0;
for (let i = 0; i < mask.length; i++) if (mask[i]) solid += 1;
return solid / mask.length;
}
function buildEdgeBand(mask: Uint8Array, width: number, height: number, radius = 1): Uint8Array {
const dil = morphDilate(mask, width, height, Math.max(1, radius))
const ero = morphErode(mask, width, height, Math.max(1, radius))
return maskMinus(dil, ero)
}
function edgeRoughness(mask: Uint8Array, width: number, height: number): number {
const edge = buildEdgeBand(mask, width, height, 1)
let edgeCount = 0
let totalVariation = 0
for (let y = 0; y < height; y++) {
const row = y * width
for (let x = 0; x < width; x++) {
const idx = row + x
if (!edge[idx]) continue
edgeCount += 1
const v = mask[idx] ? 1 : 0
if (x + 1 < width) totalVariation += Math.abs(v - (mask[idx + 1] ? 1 : 0))
if (y + 1 < height) totalVariation += Math.abs(v - (mask[idx + width] ? 1 : 0))
}
}
if (!edgeCount) return 0
return totalVariation / edgeCount
}
function haloRisk(alpha: Uint8Array, hard: Uint8Array): number {
const len = Math.min(alpha.length, hard.length)
if (!len) return 0
let leak = 0
let active = 0
for (let i = 0; i < len; i++) {
const a = alpha[i]
if (a > 0) {
active += 1
if (!hard[i]) leak += 1
}
}
return active ? leak / active : 0
}
function canvasFromMask(mask: Uint8Array, width: number, height: number): HTMLCanvasElement {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d')!;
const image = ctx.createImageData(width, height);
const dst = image.data;
for (let i = 0, j = 0; i < mask.length; i++, j += 4) {
const value = mask[i] ? 255 : 0;
dst[j] = 255;
dst[j + 1] = 255;
dst[j + 2] = 255;
dst[j + 3] = value;
}
ctx.putImageData(image, 0, 0);
return canvas;
}
function applyFeather(canvas: HTMLCanvasElement, featherPx: number): HTMLCanvasElement {
const amount = Math.max(0, featherPx);
if (!amount) return canvas;
const width = canvas.width;
const height = canvas.height;
const feathered = document.createElement('canvas');
feathered.width = width;
feathered.height = height;
const fctx = feathered.getContext('2d')!;
const prevFilter = fctx.filter ?? 'none';
fctx.filter = amount > 0 ? `blur(${amount}px)` : 'none';
fctx.drawImage(canvas, 0, 0, width, height);
fctx.filter = prevFilter;
return feathered;
}
function canvasFromGrayU8(gray: Uint8Array, width: number, height: number): HTMLCanvasElement {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d')!;
const image = ctx.createImageData(width, height);
const dst = image.data;
for (let i = 0, j = 0; i < gray.length; i++, j += 4) {
const v = gray[i] ?? 0;
dst[j] = v;
dst[j + 1] = v;
dst[j + 2] = v;
dst[j + 3] = 255;
}
ctx.putImageData(image, 0, 0);
return canvas;
}
function canvasFromAlphaU8(alpha: Uint8Array, width: number, height: number): HTMLCanvasElement {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d')!;
const image = ctx.createImageData(width, height);
const dst = image.data;
for (let i = 0, j = 0; i < alpha.length; i++, j += 4) {
const v = alpha[i] ?? 0;
dst[j] = 255;
dst[j + 1] = 255;
dst[j + 2] = 255;
dst[j + 3] = v;
}
ctx.putImageData(image, 0, 0);
return canvas;
}
function extractAlphaU8(canvas: HTMLCanvasElement): Uint8Array | null {
const width = canvas.width;
const height = canvas.height;
if (!width || !height) return null;
const ctx = canvas.getContext('2d', { willReadFrequently: true });
if (!ctx) return null;
const data = ctx.getImageData(0, 0, width, height).data;
const out = new Uint8Array(width * height);
for (let i = 0, p = 0; i < data.length; i += 4, p++) {
out[p] = data[i + 3];
}
return out;
}
function canvasFromGrayU8Preview(
gray: Uint8Array,
width: number,
height: number,
opts?: { autoContrast?: boolean; gamma?: number },
): HTMLCanvasElement {
const autoContrast = Boolean(opts?.autoContrast);
const gamma = typeof opts?.gamma === 'number' && Number.isFinite(opts.gamma) && opts.gamma > 0 ? opts.gamma : 1;
let min = 255;
let max = 0;
if (autoContrast) {
for (let i = 0; i < gray.length; i++) {
const v = gray[i] ?? 0;
if (v < min) min = v;
if (v > max) max = v;
}
if (max - min < 4) {
min = 0;
max = 255;
}
} else {
min = 0;
max = 255;
}
const range = Math.max(1, max - min);
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d')!;
const image = ctx.createImageData(width, height);
const dst = image.data;
for (let i = 0, j = 0; i < gray.length; i++, j += 4) {
const raw = gray[i] ?? 0;
let v = Math.max(0, Math.min(1, (raw - min) / range));
if (gamma !== 1) v = Math.pow(v, 1 / gamma);
const out = Math.max(0, Math.min(255, Math.round(v * 255)));
dst[j] = out;
dst[j + 1] = out;
dst[j + 2] = out;
dst[j + 3] = 255;
}
ctx.putImageData(image, 0, 0);
return canvas;
}
function buildOccluderMaskCanvas(
base: HTMLCanvasElement,
opts: {
invert?: boolean;
grow?: number;
shrink?: number;
feather?: number;
tag?: string;
fallbackCanvas?: HTMLCanvasElement | null;
fallbackTag?: string;
autoGrowMaxRadius?: number;
}
): HTMLCanvasElement | null {
const width = base.width;
const height = base.height;
if (!width || !height) return null;
const binary = extractBinaryMask(base, opts.invert ?? false);
if (!binary) return null;
const grow = Math.max(0, Math.round(opts.grow ?? 0));
const shrink = Math.max(0, Math.round(opts.shrink ?? 0));
const feather = Math.max(0, Math.round(opts.feather ?? 0));
const autoGrowMaxRadius = Math.max(
0,
Math.round(opts.autoGrowMaxRadius ?? OCCLUDER_AUTO_GROW_MAX_RADIUS),
);
const baseCoverage = coverageOfMask(binary);
let expanded = grow > 0 ? morphDilate(binary, width, height, grow) : binary;
let coverage = coverageOfMask(expanded);
if (coverage > 0 && coverage < OCCLUDER_MIN_COVERAGE) {
const targetCoverage = Math.min(
OCCLUDER_MAX_COVERAGE,
Math.max(OCCLUDER_TARGET_COVERAGE, coverage * 2.5),
);
const missing = Math.max(0, targetCoverage - coverage);
if (missing > 0) {
const coverageBefore = coverage;
const approxPixels = missing * width * height;
const estimatedRadius = Math.min(
autoGrowMaxRadius,
Math.max(1, Math.ceil(Math.sqrt(approxPixels / Math.PI))),
);
const appliedRadius = Math.min(
autoGrowMaxRadius,
Math.max(estimatedRadius, Math.max(1, grow || 1)),
);
expanded = morphDilate(expanded, width, height, appliedRadius);
let newCoverage = coverageOfMask(expanded);
if (newCoverage < targetCoverage && appliedRadius < autoGrowMaxRadius) {
const extraRadius = Math.min(autoGrowMaxRadius, appliedRadius + 2);
expanded = morphDilate(expanded, width, height, extraRadius);
newCoverage = coverageOfMask(expanded);
}
if (newCoverage !== coverageBefore) {
console.info('[occluder] auto-grow applied', {
tag: opts.tag ?? base.dataset.maskTag ?? 'mask',
baseCoverage: Number(baseCoverage.toFixed(3)),
coverageBefore: Number(coverageBefore.toFixed(3)),
coverageAfter: Number(newCoverage.toFixed(3)),
radius: appliedRadius,
targetCoverage: Number(targetCoverage.toFixed(3)),
});
}
coverage = newCoverage;
}
}
if (coverage < OCCLUDER_MIN_COVERAGE && opts.fallbackCanvas) {
const fallbackInvert = opts.fallbackCanvas.dataset?.maskInverted === 'true';
const fallbackBinary = extractBinaryMask(opts.fallbackCanvas, fallbackInvert);
if (fallbackBinary) {
const coverageBeforeFallback = coverage;
const combined = maskOr(expanded, fallbackBinary);
const combinedCoverage = coverageOfMask(combined);
if (combinedCoverage > coverage) {
expanded = combined;
coverage = Math.min(combinedCoverage, OCCLUDER_MAX_COVERAGE);
console.info('[occluder] fallback union', {
tag: opts.tag ?? base.dataset.maskTag ?? 'mask',
fallback: opts.fallbackTag ?? 'maskUrl',
baseCoverage: Number(baseCoverage.toFixed(3)),
coverageBefore: Number(coverageBeforeFallback.toFixed(3)),
coverageAfter: Number(coverage.toFixed(3)),
});
}
}
}
// Halo reduction: after solidify/grow, shrink the occluder slightly (1–4px typical).
// This avoids a visible "cutout margin" around objects while keeping the mask hole-free.
if (shrink > 0 && coverage > 0) {
const beforeShrinkCoverage = coverage;
const eroded = morphErode(expanded, width, height, shrink);
const afterShrinkCoverage = coverageOfMask(eroded);
// Guard: do not collapse the mask completely.
if (afterShrinkCoverage > 0.005 || beforeShrinkCoverage <= 0.02) {
expanded = eroded;
coverage = afterShrinkCoverage;
console.info('[occluder] shrink applied', {
tag: opts.tag ?? base.dataset.maskTag ?? 'mask',
shrink,
coverageBefore: Number(beforeShrinkCoverage.toFixed(3)),
coverageAfter: Number(afterShrinkCoverage.toFixed(3)),
});
}
}
const occluderCanvas = canvasFromMask(expanded, width, height);
const finalCanvas = applyFeather(occluderCanvas, feather);
finalCanvas.dataset.maskCoverage = coverage.toFixed(3);
finalCanvas.dataset.maskCoverageRaw = coverage.toString();
finalCanvas.dataset.maskCoverageRawInput = base.dataset.maskCoverageRawInput ?? base.dataset.maskCoverageRaw ?? coverage.toString();
finalCanvas.dataset.maskTag = opts.tag ?? `${base.dataset.maskTag ?? 'mask'}:occluder`;
finalCanvas.dataset.maskSource = `${base.dataset.maskSource ?? ''}#occluder`;
finalCanvas.dataset.maskInverted = (opts.invert ?? false) ? 'true' : 'false';
finalCanvas.dataset.maskBaseCoverage = baseCoverage.toFixed(3);
return finalCanvas;
}
/** Draw perspective grid segments (resp.grid_poly) onto any canvas context */
function drawPolyOverlay(
ctx: CanvasRenderingContext2D,
poly: GridPoly | null | undefined,
stroke = '#10b981',
alpha = 0.9,
) {
if (!ctx || !poly) return;
const { width, height } = ctx.canvas;
ctx.save();
ctx.globalAlpha = alpha;
ctx.lineWidth = 1;
ctx.strokeStyle = stroke;
ctx.beginPath();
const drawSegments = (segments?: GridPolySegment[]) => {
if (!segments?.length) return;
for (const seg of segments) {
const x1 = Number(seg?.x1);
const y1 = Number(seg?.y1);
const x2 = Number(seg?.x2);
const y2 = Number(seg?.y2);
if (!Number.isFinite(x1) || !Number.isFinite(y1) || !Number.isFinite(x2) || !Number.isFinite(y2)) continue;
const outOfView =
(x1 < -50 && x2 < -50) ||
(x1 > width + 50 && x2 > width + 50) ||
(y1 < -50 && y2 < -50) ||
(y1 > height + 50 && y2 > height + 50);
if (outOfView) continue;
ctx.moveTo(x1 + 0.5, y1 + 0.5);
ctx.lineTo(x2 + 0.5, y2 + 0.5);
}
};
drawSegments(poly.v);
drawSegments(poly.h);
ctx.stroke();
ctx.restore();
}
export default function TileSwapApp() {
// Datei & Analyse
const [file, setFile] = useState<File|null>(null);
const [imgUrl, setImgUrl] = useState<string>('');
const [resp, setResp] = useState<AnalyzeResponse|null>(null);
const [showRelight, setShowRelight] = useState(true);
const [error, setError] = useState<string| null>(null);
// Muster / Overlay
const [basePatternId, setBasePatternId] = useState<string>('checker');
const [tileDesignId, setTileDesignId] = useState<string>('stone_gray');
const [showOriginalTiles, setShowOriginalTiles] = useState<boolean>(false);
const [overlayAlpha, setOverlayAlpha] = useState<number>(0.85);
const [showGrout, setShowGrout] = useState<boolean>(true);
const [showPersp, setShowPersp] = useState<boolean>(false);
const [advancedTileRender, setAdvancedTileRender] = useState<boolean>(true);
const [groutWidthPx, setGroutWidthPx] = useState<number>(5);
const [tileGroutOpacity, setTileGroutOpacity] = useState<number>(0.9);
const [tileShadowStrength, setTileShadowStrength] = useState<number>(0.35);
const [tileSpecStrength, setTileSpecStrength] = useState<number>(0.25);
const [occluderFeatherPx, setOccluderFeatherPx] = useState<number>(1);
// einfache Steuerung
const rotationDeg = 0;
const offsetX = 0;
const offsetY = 0;
// Maske (neu)
const [maskUrl, setMaskUrl] = useState<string | null>(null);
const [maskProc, setMaskProc] = useState<string | null>(null);
const [maskProcMeta, setMaskProcMeta] = useState<MaskMeta | null>(null);
const [occluderOn, setOccluderOn] = useState<boolean>(true);
const [showOccDebug, setShowOccDebug] = useState<boolean>(false);
const [renderDebug, setRenderDebug] = useState<Record<string, string | null> | null>(null);
const [maskBusy, setMaskBusy] = useState<boolean>(false);
const [maskGrowPx, setMaskGrowPx] = useState<number>(8);
const [maskPreBlur, setMaskPreBlur] = useState<number>(1);
const [maskThresh, setMaskThresh] = useState<number>(0.4);
const [maskFinalBlur, setMaskFinalBlur] = useState<number>(2);
const [occluderShrinkPx, setOccluderShrinkPx] = useState<number>(2);
const [occluderEdgeTrimPx, setOccluderEdgeTrimPx] = useState<number>(4);
const [occluderLeakGuardPx, setOccluderLeakGuardPx] = useState<number>(OCCLUDER_LEAK_GUARD_PX);
const [tileFeatherPx, setTileFeatherPx] = useState<number>(0);
const [showTilesRenderedOnly, setShowTilesRenderedOnly] = useState<boolean>(false);
const [ignoreTileBandClip, setIgnoreTileBandClip] = useState<boolean>(false);
const [logRenderDebug, setLogRenderDebug] = useState<boolean>(false);
const [tileMaskUrl, setTileMaskUrl] = useState<string | null>(null);
const [tileMaskMeta, setTileMaskMeta] = useState<TileMaskMeta | null>(null);
const [tileMaskVersions, setTileMaskVersions] = useState<TileMaskVersions | null>(null);
const [tileMaskViewMode, setTileMaskViewMode] = useState<TileMaskViewMode>('combined');
const [modelVersion, setModelVersion] = useState<TileSegModel>('unet_latest');
const [maskPreset, setMaskPreset] = useState<MaskPreset>('tiles-default');
const [maskAppliedPreset, setMaskAppliedPreset] = useState<MaskPreset | null>(null);
const [maskLatticeEnabled, setMaskLatticeEnabled] = useState<boolean>(true);
const [maskIlluminationEnabled, setMaskIlluminationEnabled] = useState<boolean>(true);
const [maskAutotuneEnabled, setMaskAutotuneEnabled] = useState<boolean>(true);
const [maskAutotuneMs, setMaskAutotuneMs] = useState<number>(150);
const [maskThinFrac, setMaskThinFrac] = useState<number>(0.06);
const [maskMinAreaFrac, setMaskMinAreaFrac] = useState<number>(0.25);
const [maskPreference, setMaskPreference] = useState<MaskPreference>('auto');
const [shadowStrength, setShadowStrength] = useState<number>(1);
const [lightStrength, setLightStrength] = useState<number>(1);
const maskAlphaCanvasRef = useRef<HTMLCanvasElement | null>(null);
const maskCanvasCacheRef = useRef<Map<string, HTMLCanvasElement>>(new Map());
const occluderCanvasCacheRef = useRef<Map<string, HTMLCanvasElement>>(new Map());
const occluderBinaryRef = useRef<Uint8Array | null>(null);
const advancedOccluderTrimCanvasRef = useRef<HTMLCanvasElement | null>(null);
const tileClassMaskCacheRef = useRef<Map<string, HTMLCanvasElement>>(new Map());
const imgUrlRef = useRef<string | null>(null);
const formatPercent = (value?: number | null) =>
value != null && Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : '—';
const heuristicsDisabled = modelVersion === 'unet_latest';
const tileCoverageEntries = useMemo(() => (
tileMaskMeta?.coverage ? Object.entries(tileMaskMeta.coverage) : []
), [tileMaskMeta]);
const tileCoverageAbsoluteEntries = useMemo(() => (
tileMaskMeta?.coverageAbsolute ? Object.entries(tileMaskMeta.coverageAbsolute) : []
), [tileMaskMeta]);
const tileCoverageRawEntries = useMemo(() => (
tileMaskMeta?.rawCoverage ? Object.entries(tileMaskMeta.rawCoverage) : []
), [tileMaskMeta]);
const tileOccluderCoverage = tileMaskMeta?.occluderCoverage ?? null;
const tileOccludedCoverage = tileMaskMeta?.occludedCoverage ?? null;
const tileOccluderPixels = tileMaskMeta?.occluderPixels ?? null;
const tileOccludedPixels = tileMaskMeta?.occludedPixels ?? null;
const tileCoverageTiles = tileMaskMeta?.coverage?.tiles ?? null;
const tileCoverageGrout = tileMaskMeta?.coverage?.grout ?? null;
const tileWarnings = (tileMaskMeta as any)?.warnings as string[] | undefined;
const tilesMissingWarning = Array.isArray(tileWarnings) && tileWarnings.includes('tiles_missing');
const allowTileOverlay = !tilesMissingWarning && (tileCoverageTiles ?? 0) > 0.005;
const showGroutMissing = (() => {
if (!(tileCoverageGrout != null && tileCoverageGrout < 0.005)) return false;
// For the 2-class tiles model, grout is heuristic (generated from tiles/grid), so a "missing" warning
// based on model output is misleading.
const rawLegend = (tileMaskMeta as any)?.classIdLegendRaw as Record<string, string> | undefined;
const isTwoClassTilesModel =
rawLegend != null &&
Object.keys(rawLegend).length === 3 &&
String(rawLegend['0']).toLowerCase() === 'background' &&
String(rawLegend['1']).toLowerCase() === 'tiles';
if (isTwoClassTilesModel) return false;
if (typeof tileMaskMeta?.source === 'string' && tileMaskMeta.source.includes('grout_heur')) return false;
return true;
})();
const showTilesMissing = tilesMissingWarning || (tileCoverageTiles != null && tileCoverageTiles <= 0);
const tileBand = tileMaskMeta?.tileBand ?? null;
const isOblique = useMemo(() => computeIsOblique(resp?.grid_poly ?? null), [resp?.grid_poly]);
const effectiveOccluderGrowPx = isOblique ? 1 : OCCLUDER_GROW_PX;
const effectiveOccluderShrinkPx = occluderShrinkPx;
const effectiveOccluderEdgeTrimPx = isOblique ? Math.min(2, occluderEdgeTrimPx) : occluderEdgeTrimPx;
const effectiveOccluderLeakGuardPx = Math.max(0, Math.round(
isOblique ? Math.min(2, occluderLeakGuardPx) : occluderLeakGuardPx,
));
const effectiveTileFeatherPx = isOblique ? Math.min(1, tileFeatherPx) : tileFeatherPx;
const effectiveOccluderFeatherPx = isOblique ? Math.min(3, occluderFeatherPx) : occluderFeatherPx;
const occluderMethodSource = String(
resp?.mask_metrics?.source
?? (resp?.mask_debug?.originMeta as any)?.source
?? (resp?.fg_mask_meta as any)?.source
?? resp?.fg_mask_meta?.tag
?? ''
).toLowerCase();
const preferSegOccluder =
occluderMethodSource.includes('sam_hq') ||
occluderMethodSource.includes('sam-hq');
const finalMaskUrl = resp?.fg_mask ?? resp?.mask_versions?.normalized ?? null;
const tileMaskCombinedSource = tileMaskVersions?.combined ?? tileMaskUrl;
const tileMaskRawSource = tileMaskVersions?.raw ?? null;
// Only allow depth-derived occluder debug; never fall back to tile/grid masks.
// If backend rejects the depth occluder as grid-like, do not show it as occluder.
const occluderDebugSource = (resp as any)?.depth_debug?.rejected
? null
: (resp?.depth_debug?.occluderUrl ?? resp?.depth_debug?.maskRawUrl ?? null);
useEffect(() => {
if (tileMaskViewMode === 'raw' && !tileMaskRawSource) {
if (tileMaskCombinedSource) {
setTileMaskViewMode('combined');
} else if (occluderDebugSource) {
setTileMaskViewMode('occluder');
}
} else if (tileMaskViewMode === 'occluder' && !occluderDebugSource) {
if (tileMaskCombinedSource) {
setTileMaskViewMode('combined');
} else if (tileMaskRawSource) {
setTileMaskViewMode('raw');
}
}
}, [tileMaskViewMode, tileMaskCombinedSource, tileMaskRawSource, occluderDebugSource]);
const tileMaskDisplayUrl = useMemo(() => {
if (tileMaskViewMode === 'raw') {
return tileMaskRawSource ?? tileMaskCombinedSource ?? null;
}
if (tileMaskViewMode === 'occluder') {
return occluderDebugSource ?? null; // do not fall back to tile mask; avoid grid/pattern confusion
}
return tileMaskCombinedSource ?? null;
}, [tileMaskViewMode, tileMaskCombinedSource, tileMaskRawSource, occluderDebugSource]);
const tileMaskViewOptions = useMemo(() => ([
{
mode: 'combined' as TileMaskViewMode,
label: TILE_MASK_VIEW_LABELS.combined,
title: TILE_MASK_VIEW_TITLES.combined,
available: Boolean(tileMaskCombinedSource),
},
{
mode: 'raw' as TileMaskViewMode,
label: TILE_MASK_VIEW_LABELS.raw,
title: TILE_MASK_VIEW_TITLES.raw,
available: Boolean(tileMaskRawSource),
},
{
mode: 'occluder' as TileMaskViewMode,
label: TILE_MASK_VIEW_LABELS.occluder,
title: TILE_MASK_VIEW_TITLES.occluder,
available: Boolean(occluderDebugSource),
},
]), [tileMaskCombinedSource, tileMaskRawSource, occluderDebugSource]);
const tileColorForClass = useCallback((cls: string) => {
if (!tileMaskMeta) return 'rgb(0,0,0)';
const idx = tileMaskMeta.classes.indexOf(cls);
if (idx === -1) return 'rgb(0,0,0)';
const tuple = tileMaskMeta.colorMap?.[String(idx)];
if (!tuple) return 'rgb(0,0,0)';
return `rgb(${tuple[0]}, ${tuple[1]}, ${tuple[2]})`;
}, [tileMaskMeta]);
useEffect(() => {
const defaults = maskPreset === 'tiles-detail'
? { lattice: true, illum: true, autotune: true, autotuneMs: 200, thin: 0.05, minArea: 0.20 }
: { lattice: true, illum: true, autotune: true, autotuneMs: 150, thin: 0.06, minArea: 0.25 };
setMaskLatticeEnabled(defaults.lattice);
setMaskIlluminationEnabled(defaults.illum);
setMaskAutotuneEnabled(defaults.autotune);
setMaskAutotuneMs(defaults.autotuneMs);
setMaskThinFrac(defaults.thin);
setMaskMinAreaFrac(defaults.minArea);
}, [maskPreset]);
useEffect(() => () => {
if (imgUrlRef.current) {
URL.revokeObjectURL(imgUrlRef.current);
imgUrlRef.current = null;
}
}, []);
const pipelineFallback = resp?.mask_debug?.pipeline?.fallback as
| { reason?: Record<string, boolean>; usedOriginal?: boolean; originalCoverage?: number; pipelineCoverage?: number; leakPct?: number }
| undefined;
const fallbackReasons = pipelineFallback?.reason
? Object.entries(pipelineFallback.reason)
.filter(([, flagged]) => flagged)
.map(([key]) => key)
: [];
const rawMaskWarnings = resp?.mask_metrics?.warnings?.filter((w): w is string => typeof w === 'string' && w.length > 0) ?? [];
const maskWarnings = heuristicsDisabled ? [] : rawMaskWarnings;
const ensureMaskCanvas = useCallback(async () => {
const canvas = canvasRef.current;
const candidates: { source: string; tag: string; polarity?: 'white' | 'black' }[] = [];
let clientMaskPrepared: HTMLCanvasElement | null = null;
const maskPreferenceEffective = preferSegOccluder ? 'raw' : maskPreference;
const depthOccluderUrl = (resp as any)?.depth_debug?.rejected
? null
: (resp as any)?.depth_debug?.occluderUrl ?? null;
const depthOccCoverage = Number((resp as any)?.depth_debug?.occluderCoverage ?? NaN);
const rawCoverage = Number((resp as any)?.mask_metrics?.coverageRaw ?? (resp as any)?.mask_metrics?.coverage ?? NaN);
const depthUnderCoverage = Number.isFinite(depthOccCoverage)
&& Number.isFinite(rawCoverage)
&& (rawCoverage - depthOccCoverage) > 0.03;
if (typeof depthOccluderUrl === 'string' && depthOccluderUrl.length > 0 && !depthUnderCoverage && maskPreferenceEffective !== 'raw' && maskPreferenceEffective !== 'server') {
candidates.push({ source: depthOccluderUrl, tag: 'depthOccluder', polarity: 'white' });
}
if (maskProc) candidates.push({ source: maskProc, tag: 'maskProc', polarity: maskProcMeta?.polarity });
const maskVersions = resp?.mask_versions;
const wantServerRaw = maskPreferenceEffective === 'server' && !maskVersions?.normalized;
const wantRawMask = maskPreferenceEffective === 'raw' || maskPreferenceEffective === 'auto';
if (maskVersions?.raw && wantRawMask) {
candidates.push({ source: maskVersions.raw, tag: 'maskRaw', polarity: 'white' });
}
if (!preferSegOccluder) {
if (maskVersions?.normalized) {
candidates.push({ source: maskVersions.normalized, tag: 'maskNormalized', polarity: 'white' });
}
if (maskVersions?.fused) {
candidates.push({ source: maskVersions.fused, tag: 'maskFused', polarity: 'white' });
}
if (resp?.fg_mask && (wantServerRaw || wantRawMask)) {
candidates.push({ source: resp.fg_mask, tag: 'resp.fg_mask', polarity: resp?.fg_mask_meta?.polarity });
}
if (maskUrl) candidates.push({ source: maskUrl, tag: 'maskUrl' });
}
if (!canvas || candidates.length === 0) {
maskAlphaCanvasRef.current = null;
return null;
}
const uniqueCandidates = Array.from(
new Map(candidates.map((c) => [c.source, c])).values()
);
const preferenceOrder = (tag: string): number => {
const prefRankings: Record<MaskPreference, string[]> = {
auto: ['maskNormalized', 'maskProc', 'maskFused', 'resp.fg_mask', 'maskUrl', 'maskRaw'],
processed: ['maskProc', 'maskNormalized', 'resp.fg_mask', 'maskFused', 'maskUrl', 'maskRaw'],
server: ['maskNormalized', 'resp.fg_mask', 'maskProc', 'maskFused', 'maskUrl', 'maskRaw'],
client: ['maskUrl', 'maskProc', 'maskNormalized', 'resp.fg_mask', 'maskFused', 'maskRaw'],
fused: ['maskFused', 'maskNormalized', 'maskProc', 'resp.fg_mask', 'maskUrl', 'maskRaw'],
raw: ['maskRaw', 'maskNormalized', 'maskProc', 'resp.fg_mask', 'maskFused', 'maskUrl'],
};
if (preferSegOccluder) {
return tag === 'maskRaw' ? 0 : 99;
}
const ranking = prefRankings[maskPreferenceEffective] ?? prefRankings.auto;
const idx = ranking.indexOf(tag);
return idx >= 0 ? idx : ranking.length;
};
uniqueCandidates.sort((a, b) => preferenceOrder(a.tag) - preferenceOrder(b.tag));
const evaluations: {
canvas: HTMLCanvasElement;
source: string;
tag: string;
coverage: number;
rawCoverage: number;
score: number;
invert: boolean;
suspicious: boolean;
}[] = [];
const rawEvaluations: any[] = [];
for (const { source, tag, polarity } of uniqueCandidates as Array<{ source: string; tag: string; polarity?: 'white' | 'black' }>) {
const cacheKey = tag === 'maskRaw' ? `${source}#raw` : source;
let prepared = maskCanvasCacheRef.current.get(cacheKey);
const needsNew =
!prepared ||
prepared.width !== canvas.width ||
prepared.height !== canvas.height;
if (needsNew) {
prepared = await prepareMaskAlphaCanvas(
source,
canvas.width,
canvas.height,
tag === 'maskRaw' ? { mode: 'raw' } : undefined,
);
prepared.dataset.maskSource = source;
prepared.dataset.maskTag = tag;
maskCanvasCacheRef.current.set(cacheKey, prepared);
}
if (!prepared) continue;
if (tag === 'maskUrl') {
clientMaskPrepared = prepared;
}
const storedCoverage = Number(prepared.dataset.maskCoverage ?? prepared.dataset.maskCoverageRaw ?? '0');
const rawCoverage = Number(prepared.dataset.maskCoverageRawInput ?? prepared.dataset.maskCoverageRaw ?? '0');
const targetCoverage = Number(prepared.dataset.maskCoverageTarget ?? prepared.dataset.maskCoverage ?? '0');
const thresholdScore = Number(prepared.dataset.maskThreshold ?? '0');
let invert = prepared.dataset.maskInverted === 'true';
if (polarity) {
invert = polarity === 'black';
prepared.dataset.maskInverted = invert ? 'true' : 'false';
}
const rawScore = Math.abs(storedCoverage - 0.5);
const score = (tag !== 'maskProc' && tag !== 'maskRaw' && (storedCoverage <= 0.01 || storedCoverage >= 0.99))
? Number.POSITIVE_INFINITY
: (tag === 'maskProc' ? rawScore * 0.25 : tag === 'maskRaw' ? rawScore * 0.35 : rawScore);
const coverageMismatch = Math.abs(storedCoverage - rawCoverage);
const suspicious =
!Number.isFinite(rawCoverage) ||
!Number.isFinite(storedCoverage) ||
(rawCoverage >= 0.6 && storedCoverage <= 0.05) ||
(rawCoverage <= 0.05 && storedCoverage >= 0.5) ||
coverageMismatch >= 0.35 ||
storedCoverage >= 0.7 ||
storedCoverage <= 0.005;
evaluations.push({
canvas: prepared,
source,
tag,
coverage: storedCoverage,
rawCoverage,
invert,
score,
suspicious,
});
rawEvaluations.push({
tag,
coverage: storedCoverage,
raw: rawCoverage,
target: targetCoverage,
threshold: thresholdScore,
invert,
size: `${prepared.width}x${prepared.height}`,
mismatch: Number.isFinite(coverageMismatch) ? Number(coverageMismatch.toFixed(3)) : coverageMismatch,
suspicious,
});
}
if (rawEvaluations.length) {
console.info('[occluder] mask candidates', rawEvaluations.map((entry) => ({
tag: entry.tag,
coverage: Number.isFinite(entry.coverage) ? entry.coverage.toFixed(3) : String(entry.coverage),
raw: Number.isFinite(entry.raw) ? entry.raw.toFixed(3) : String(entry.raw),
target: Number.isFinite(entry.target) ? entry.target.toFixed(3) : String(entry.target),
threshold: Number.isFinite(entry.threshold) ? entry.threshold.toFixed(3) : String(entry.threshold),
mismatch: Number.isFinite(entry.mismatch) ? entry.mismatch.toFixed(3) : String(entry.mismatch),
invert: entry.invert,
suspicious: entry.suspicious,
size: entry.size,
})));
}
if (evaluations.length === 0) {
maskAlphaCanvasRef.current = null;
return null;
}
const depthEval = evaluations.find((entry) => entry.tag === 'depthOccluder');
if (depthEval) {
const baseline = evaluations
.filter((entry) => entry.tag !== 'depthOccluder')
.filter((entry) => Number.isFinite(entry.coverage))
.sort((a, b) => b.coverage - a.coverage)[0];
if (baseline && Number.isFinite(depthEval.coverage)) {
const delta = baseline.coverage - depthEval.coverage;
if (delta > 0.08) {
depthEval.score = Number.POSITIVE_INFINITY;
depthEval.suspicious = true;
} else if (delta > 0.03) {
depthEval.score += 0.5;
depthEval.suspicious = true;
}
}
}
evaluations.sort((a, b) => a.score - b.score);
const forcedTag = preferSegOccluder
? 'maskRaw'
: maskPreferenceEffective === 'processed'
? 'maskProc'
: maskPreferenceEffective === 'server'
? (resp?.mask_versions?.normalized ? 'maskNormalized' : 'resp.fg_mask')
: maskPreferenceEffective === 'client'
? 'maskUrl'
: maskPreferenceEffective === 'fused'
? 'maskFused'
: maskPreferenceEffective === 'raw'
? 'maskRaw'
: null;
const prefer = (entry: typeof evaluations[number]) => {
if (!Number.isFinite(entry.score)) return false;
if (forcedTag) {
return entry.tag === forcedTag;
}
return !entry.suspicious;
};
let best = evaluations.find(prefer);
if (!best) {
best = evaluations.find((entry) => Number.isFinite(entry.score));
}
if (forcedTag && (!best || best.tag !== forcedTag)) {
const fallback = evaluations.find((entry) => entry.tag === forcedTag);
if (fallback) {
console.warn('[occluder] forced mask unavailable, falling back to first candidate', { forcedTag, coverage: fallback.coverage.toFixed(3) });
best = fallback;
}
}
if (!best) {
if (!maskAlphaCanvasRef.current?.dataset.maskCoverageWarned) {
console.info('[occluder] no usable mask candidates', evaluations.map((entry) => ({
tag: entry.tag,
coverage: entry.coverage.toFixed(3),
raw: entry.rawCoverage.toFixed(3),
})));
if (maskAlphaCanvasRef.current?.dataset) maskAlphaCanvasRef.current.dataset.maskCoverageWarned = '1';
}
maskAlphaCanvasRef.current = null;
return null;
}
const rawOccluderTag = best.tag === 'maskRaw';
const shrinkPx = rawOccluderTag ? 0 : effectiveOccluderShrinkPx;
const occluderKey = `${best.source}|grow=${effectiveOccluderGrowPx}|shrink=${shrinkPx}|feather=${OCCLUDER_FEATHER_PX}|invert=${best.invert ? 1 : 0}|oblique=${isOblique ? 1 : 0}`;
let processedCanvas = occluderCanvasCacheRef.current.get(occluderKey) ?? null;
if (!processedCanvas) {
const built = buildOccluderMaskCanvas(best.canvas, {
invert: best.invert,
grow: effectiveOccluderGrowPx,
shrink: shrinkPx,
feather: OCCLUDER_FEATHER_PX,
tag: `${best.tag}:occluder`,
fallbackCanvas: preferSegOccluder
? null
: (clientMaskPrepared && clientMaskPrepared !== best.canvas ? clientMaskPrepared : null),
fallbackTag: preferSegOccluder ? undefined : (clientMaskPrepared?.dataset.maskTag ?? 'maskUrl'),
autoGrowMaxRadius: preferSegOccluder ? 0 : (isOblique ? 8 : OCCLUDER_AUTO_GROW_MAX_RADIUS),
});
if (built && built !== best.canvas) {
processedCanvas = built;
occluderCanvasCacheRef.current.set(occluderKey, processedCanvas);
}
}
const finalCanvas = processedCanvas ?? best.canvas;
const finalCoverage = Number(finalCanvas.dataset.maskCoverage ?? finalCanvas.dataset.maskCoverageRaw ?? best.coverage ?? 0);
if (maskAlphaCanvasRef.current !== finalCanvas) {
console.info('[occluder] using mask', {
tag: best.tag,
coverage: Number.isFinite(finalCoverage) ? finalCoverage.toFixed(3) : String(finalCoverage),
raw: best.rawCoverage.toFixed(3),
invert: best.invert,
suspicious: best.suspicious,
score: Number.isFinite(best.score) ? Number(best.score.toFixed(3)) : best.score,
});
}
maskAlphaCanvasRef.current = finalCanvas;
return maskAlphaCanvasRef.current;
}, [
maskProc,
maskProcMeta,
maskUrl,
resp?.fg_mask,
resp?.fg_mask_meta,
resp?.mask_versions,
maskPreference,
preferSegOccluder,
effectiveOccluderGrowPx,
effectiveOccluderShrinkPx,
isOblique,
]);
const [wu, setWu] = useState<number>(1.0);
const [wh, setWh] = useState<number>(0.9);
const [fblur, setFblur] = useState<number>(1);
const [ransacMode, setRansacMode] = useState<'off' | 'auto' | '1'>('auto');
const [analyzeBusy, setAnalyzeBusy] = useState<boolean>(false);
const analyzeAbort = useRef<AbortController | null>(null);
useEffect(() => {
setMaskAppliedPreset(resp?.mask_metrics?.preset ?? resp?.mask_preset ?? null);
}, [resp?.mask_metrics?.preset, resp?.mask_preset]);
useEffect(() => {
occluderCanvasCacheRef.current.clear();
maskAlphaCanvasRef.current = null;
occluderBinaryRef.current = null;
advancedOccluderTrimCanvasRef.current = null;
}, [
maskProc,
maskUrl,
resp?.mask_versions?.normalized,
resp?.mask_versions?.fused,
resp?.fg_mask,
occluderShrinkPx,
occluderLeakGuardPx,
tileFeatherPx,
isOblique,
]);
/** Auto-Postprocessing der Masken (Dilation/Threshold) */
useEffect(() => {
let cancel = false;
async function run() {
const normalized = resp?.mask_versions?.normalized ?? null;
if (normalized) {
if (!cancel) {
setMaskProc(normalized);
setMaskProcMeta({
...(resp?.fg_mask_meta ?? {}),
coverage: resp?.mask_metrics?.coverage ?? resp?.fg_mask_meta?.coverage ?? null,
preset: resp?.mask_preset ?? null,
tag: 'server-normalized',
});
}
return;
}
const mk = resp?.fg_mask ?? null;
if (!mk) {
setMaskProc(null);
setMaskProcMeta(null);
return;
}
try {
const out = await processMask(mk, maskGrowPx, maskPreBlur, maskThresh, maskFinalBlur);
if (!cancel) {
setMaskProc(out);
setMaskProcMeta({
...(resp?.fg_mask_meta ?? { polarity: 'white' }),
tag: 'client-processed',
});
}
} catch (e) {
console.error('mask postprocess failed', e);
}
}
run();
return () => { cancel = true; };
}, [resp?.mask_versions?.normalized, resp?.fg_mask, resp?.fg_mask_meta, resp?.mask_metrics?.coverage, resp?.mask_preset, maskGrowPx, maskPreBlur, maskThresh, maskFinalBlur]);
// Canvas Refs
const canvasRef = useRef<HTMLCanvasElement | null>(null);
// Upload
const onFile = async (f: File) => {
setError(null);
setResp(null);
setMaskUrl(null);
setMaskProc(null);
setMaskProcMeta(null);
setTileMaskUrl(null);
setTileMaskMeta(null);
setTileMaskViewMode('combined');
setMaskPreference('auto');
setMaskAppliedPreset(null);
analyzeAbort.current?.abort();
try {
setMaskBusy(true);
const prepared = await downscaleImageFileIfNeeded(f, MAX_UPLOAD_DIMENSION);
if (imgUrlRef.current) {
URL.revokeObjectURL(imgUrlRef.current);
}
imgUrlRef.current = prepared.url;
setImgUrl(prepared.url);
setFile(prepared.file);
const url = await computeClientMask(prepared.file);
setMaskUrl(url);
console.info('Client mask computed:', url ? 'ok' : 'empty');
} catch (e) {
console.warn('Client-side mask failed, continue without:', e);
setMaskUrl(null);
} finally {
setMaskBusy(false);
}
};
useEffect(() => {
if (!file) return;
const controller = new AbortController();
analyzeAbort.current?.abort();
analyzeAbort.current = controller;
const run = async () => {
setAnalyzeBusy(true);
setError(null);
const toFinite = (value: unknown): number | null =>
typeof value === 'number' && Number.isFinite(value) ? value : null;
const analysisId =
typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function'
? crypto.randomUUID()
: `mask-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
const startedAt =
typeof performance !== 'undefined' && typeof performance.now === 'function'
? performance.now()
: Date.now();
try {
const fd = new FormData();
fd.append('file', file, file.name || 'upload.bin');
const params = new URLSearchParams({
ransac: ransacMode,
wu: wu.toString(),
wh: wh.toString(),
fblur: fblur.toString(),
maskPreset,
});
params.set('model', modelVersion);
params.set('maskLattice', maskLatticeEnabled ? '1' : '0');
params.set('maskIllumination', maskIlluminationEnabled ? '1' : '0');
params.set('maskAutotune', maskAutotuneEnabled ? '1' : '0');
params.set('maskAutotuneMs', Math.max(0, Math.round(maskAutotuneMs)).toString());
params.set('maskThinWidthFrac', maskThinFrac.toFixed(3));
params.set('maskMinAreaFrac', maskMinAreaFrac.toFixed(3));
const res = await fetch(`${BASE_PATH}/api/analyze-mix?${params.toString()}`, {
method: 'POST',
body: fd,
signal: controller.signal,
});
if (!res.ok) {
const text = await res.text().catch(() => '');
throw new Error(`Analyze mix failed ${res.status}: ${text}`);
}
const json = (await res.json()) as AnalyzeResponse;
const finishedAt =
typeof performance !== 'undefined' && typeof performance.now === 'function'
? performance.now()
: Date.now();
const latencyMsRaw = finishedAt - startedAt;
const latencyMs = Number.isFinite(latencyMsRaw) ? Math.max(0, Math.round(latencyMsRaw)) : null;
const maskMetrics = json.mask_metrics ?? null;
const appliedPreset = (maskMetrics?.preset ?? json.mask_preset ?? null) as MaskPreset | string | null;
const fallbackInfo = (json.mask_debug?.pipeline?.fallback ?? null) as
| { reason?: Record<string, boolean>; usedOriginal?: boolean; originalCoverage?: number; pipelineCoverage?: number; leakPct?: number }
| null;
const fallbackReasons = fallbackInfo?.reason
? Object.entries(fallbackInfo.reason)
.filter(([, flagged]) => Boolean(flagged))
.map(([key]) => key)
: [];
const warnings = Array.isArray(maskMetrics?.warnings)
? maskMetrics!.warnings.filter((w): w is string => typeof w === 'string' && w.length > 0)
: [];
const mix = json.mix;
const mixWeights = mix?.weights ?? null;
const telemetryTileModel = json.tile_mask_meta?.model ?? json.tile_model ?? modelVersion;
dispatchMaskTelemetry({
analysisId,
presetRequested: maskPreset,
presetApplied: appliedPreset,
coverage: toFinite(maskMetrics?.coverage ?? null),
coverageNorm: toFinite(maskMetrics?.coverageNorm ?? maskMetrics?.coverage ?? null),
coverageRaw: toFinite(maskMetrics?.coverageRaw ?? null),
backgroundCoverage: toFinite(maskMetrics?.backgroundCoverage ?? null),
seedCoverage: toFinite(maskMetrics?.seedCoverage ?? null),
interiorCoverage: toFinite(maskMetrics?.latticeInteriorCoverage ?? null),
leakPct: toFinite(maskMetrics?.latticeLeakPct ?? maskMetrics?.latticeLeak ?? null),
warnings,
autotuneMs: toFinite(maskMetrics?.autotuneMs ?? null),
latencyMs,
imageWidth: toFinite(json.image?.width ?? null),
imageHeight: toFinite(json.image?.height ?? null),
tileWidth: toFinite(json.tile_w_px),
tileHeight: toFinite(json.tile_h_px),
groutWidth: toFinite(json.grout_px),
maskLattice: maskLatticeEnabled,
maskIllumination: maskIlluminationEnabled,
maskAutotune: maskAutotuneEnabled,
maskAutotuneBudget: maskAutotuneMs,
maskThinFrac,
maskMinAreaFrac,
ransacMode,
wu,
wh,
fblur,
source: json.source ?? null,
fileBytes: toFinite(file.size),
fileType: file.type ? file.type : null,
mixMode: mix?.mode ?? null,
mixWu: toFinite((mixWeights?.u2 ?? mix?.wu) ?? null),
mixWh: toFinite((mixWeights?.heur ?? mix?.wh) ?? null),
mixBlur: toFinite((mix?.fblur_used ?? mix?.fblur) ?? null),
mixU2Coverage: toFinite(mix?.u2_coverage ?? null),
mixHeurCoverage: toFinite(mix?.heur_coverage ?? null),
fallbackUsed: Boolean(fallbackInfo?.usedOriginal),
fallbackReasons,
fallbackOriginalCoverage: toFinite(fallbackInfo?.originalCoverage ?? null),
fallbackPipelineCoverage: toFinite(fallbackInfo?.pipelineCoverage ?? null),
fallbackLeakPct: toFinite(fallbackInfo?.leakPct ?? null),
tileMaskTiles: toFinite(json.tile_mask_meta?.coverage?.tiles ?? null),
tileMaskGrout: toFinite(json.tile_mask_meta?.coverage?.grout ?? null),
tileMaskTilesRaw: toFinite(json.tile_mask_meta?.rawCoverage?.tiles ?? null),
tileMaskGroutRaw: toFinite(json.tile_mask_meta?.rawCoverage?.grout ?? null),
tileMaskOccluder: toFinite(json.tile_mask_meta?.occluderCoverage ?? null),
tileMaskTilesAbs: toFinite(json.tile_mask_meta?.coverageAbsolute?.tiles ?? null),
tileMaskGroutAbs: toFinite(json.tile_mask_meta?.coverageAbsolute?.grout ?? null),
tileMaskOccluded: toFinite(json.tile_mask_meta?.occludedCoverage ?? null),
tileMaskOccluderPixels: toFinite(json.tile_mask_meta?.occluderPixels ?? null),
tileMaskOccludedPixels: toFinite(json.tile_mask_meta?.occludedPixels ?? null),
tileModel: telemetryTileModel,
});
setMaskAppliedPreset(appliedPreset ?? null);
setResp(json);
setTileMaskUrl(json.tile_mask ?? json.tile_mask_versions?.combined ?? null);
setTileMaskMeta(json.tile_mask_meta ?? null);
setTileMaskVersions(json.tile_mask_versions ?? null);
setTileMaskViewMode('combined');
} catch (err: unknown) {
if (err instanceof Error) {
if (err.name === 'AbortError') return;
setError(err.message);
} else {
setError(String(err));
}
setTileMaskUrl(null);
setTileMaskMeta(null);
setTileMaskVersions(null);
setTileMaskViewMode('combined');
} finally {
if (analyzeAbort.current === controller) analyzeAbort.current = null;
setAnalyzeBusy(false);
}
};
run();
return () => {
controller.abort();
};
}, [file, ransacMode, wu, wh, fblur, maskPreset, maskLatticeEnabled, maskIlluminationEnabled, maskAutotuneEnabled, maskAutotuneMs, maskThinFrac, maskMinAreaFrac, modelVersion]);
// Rendering
useEffect(() => {
(async () => {
const cnv = canvasRef.current;
if (!cnv || !imgUrl) return;
const img = await loadImage(imgUrl);
cnv.width = img.naturalWidth;
cnv.height = img.naturalHeight;
const ctx = cnv.getContext('2d')!;
ctx.clearRect(0,0,cnv.width, cnv.height);
let advancedAllowedMatte: HTMLCanvasElement | null = null;
let advancedTileAlphaCanvas: HTMLCanvasElement | null = null;
let advancedOccluderTrimCanvas: HTMLCanvasElement | null = null;
// 1) Overlay in Offscreen zeichnen (bestehende Grid-Logik abgespeckt)
const overlay = document.createElement('canvas');
overlay.width = cnv.width; overlay.height = cnv.height;
const octx = overlay.getContext('2d')!;
octx.clearRect(0,0,overlay.width, overlay.height);
const pat = PATTERNS.find(p => p.id === basePatternId) ?? PATTERNS[0];
const tex = await loadImage(pat.src);
const design = TILE_DESIGNS.find((d) => d.id === tileDesignId) ?? TILE_DESIGNS[0];
const tileTex = await loadImage(`${runtimeBasePath()}${design.src}`);
const tw = resp?.tile_w_px ?? 0;
const th = resp?.tile_h_px ?? 0;
const grid = resp?.grid ?? null;
// --- Advanced image-space tiles (synthetic grout + light/shadow/spec) ---
const tilesMaskUrl = tileMaskVersions?.combined ?? tileMaskVersions?.raw ?? tileMaskUrl ?? null;
const tilesMaskSource = tileMaskVersions?.combined
? 'combined'
: tileMaskVersions?.raw
? 'raw'
: tileMaskUrl
? 'tile_mask'
: 'none';
const canAdvanced =
Boolean(advancedTileRender) &&
Boolean(grid?.vlines?.length) &&
Boolean(grid?.hlines?.length) &&
Boolean(tilesMaskUrl) &&
Boolean(tileMaskMeta);
const canOverlayNow =
!showOriginalTiles &&
(advancedTileRender ? (allowTileOverlay || showTilesRenderedOnly) : allowTileOverlay && overlayAlpha > 0);
const overlayDrawAlpha = advancedTileRender ? 1 : overlayAlpha;
if (tw > 0 && th > 0 && canOverlayNow) {
if (canAdvanced && tilesMaskUrl && tileMaskMeta && grid) {
const key = `${tilesMaskUrl}|cls=tiles|${overlay.width}x${overlay.height}`;
let tilesCanvas = tileClassMaskCacheRef.current.get(key) ?? null;
if (!tilesCanvas) {
tilesCanvas = await prepareTileClassAlphaCanvas(
tilesMaskUrl,
tileMaskMeta,
'tiles',
overlay.width,
overlay.height,
);
tileClassMaskCacheRef.current.set(key, tilesCanvas);
}
const tilesBinary = extractBinaryMask(tilesCanvas, false) ?? new Uint8Array(overlay.width * overlay.height);
if (tilesBinary.length !== overlay.width * overlay.height) {
throw new Error(
`[render] tilesBinary dimension mismatch: len=${tilesBinary.length} expected=${overlay.width * overlay.height}`,
);
}
const gridRoiMask = buildGridRoiMask(grid, tileBand, overlay.width, overlay.height);
const occluderBinaryForTiles = (() => {
if (occluderBinaryRef.current) return occluderBinaryRef.current;
const maskAlphaCanvas = maskAlphaCanvasRef.current;
if (!maskAlphaCanvas) return null;
return extractBinaryMask(maskAlphaCanvas, false);
})();
let tilesBinaryEffective = gridRoiMask ?? tilesBinary;
if (occluderBinaryForTiles && tileBand) {
const merged = new Uint8Array(tilesBinary.length);
merged.set(tilesBinaryEffective);
const yMin = Math.max(0, Math.min(overlay.height, Math.round(tileBand.yMin)));
const yMax = Math.max(yMin, Math.min(overlay.height, Math.round(tileBand.yMax)));
for (let y = yMin; y < yMax; y++) {
const row = y * overlay.width;
for (let x = 0; x < overlay.width; x++) {
const idx = row + x;
if (occluderBinaryForTiles[idx]) merged[idx] = 255;
}
}
tilesBinaryEffective = merged;
}
const tilePattern = renderTilePatternImageData({
patternImage: tileTex,
canvasSize: { w: overlay.width, h: overlay.height },
grid,
tilePaddingPx: 0,
});
const renderGrid = { vlines: tilePattern.vlines, hlines: tilePattern.hlines };
const groutMask = showGrout
? buildGroutMaskFromGrid({
width: overlay.width,
height: overlay.height,
vlines: renderGrid.vlines,
hlines: renderGrid.hlines,
groutWidthPx,
tilesMask: tilesBinaryEffective,
tilesDilatePx: 2,
}).groutMask
: new Uint8Array(overlay.width * overlay.height);
const orig = document.createElement('canvas');
orig.width = overlay.width;
orig.height = overlay.height;
const origCtx = orig.getContext('2d', { willReadFrequently: true })!;
origCtx.drawImage(img, 0, 0, overlay.width, overlay.height);
const origData = origCtx.getImageData(0, 0, overlay.width, overlay.height);
const cleanupBase = Math.max(24, Math.round((overlay.width * overlay.height) * 0.00004));
const tileMinComponentPx = Math.max(50, Math.round(cleanupBase * 1.5));
const tileMaxHolePx = Math.max(40, Math.round(cleanupBase * 1.2));
const occluderMinComponentPx = Math.max(80, Math.round(cleanupBase * 2.0));
const occluderMaxHolePx = Math.max(60, Math.round(cleanupBase * 1.5));
const edgeRejectMask = showGrout
? morphDilate(groutMask, overlay.width, overlay.height, 1)
: null;
const maskAlphaCanvas = occluderOn ? await ensureMaskCanvas() : null;
const occluderBinaryRaw = maskAlphaCanvas ? (extractBinaryMask(maskAlphaCanvas, false) ?? null) : null;
const occluderTag = maskAlphaCanvas?.dataset?.maskTag ?? '';
const occluderIsRaw = occluderTag.includes('maskRaw');
const occluderEdgeTrim = occluderIsRaw ? 0 : effectiveOccluderEdgeTrimPx;
const occluderEdgeWidth = occluderIsRaw ? 1 : Math.max(1, Math.round(occluderEdgeTrim || 2));
const occluderSnapSearchPx = occluderIsRaw ? 1 : Math.max(2, occluderEdgeWidth + 1);
const occluderSnapGradientMin = occluderIsRaw ? 10 : 12;
const occluderBandSmoothPx = occluderIsRaw ? 0 : Math.max(1, Math.min(3, Math.round(occluderEdgeWidth / 2)));
const occluderBandSmoothThreshold = occluderIsRaw ? 0.58 : 0.56;
const occluderSigmaColor = occluderIsRaw ? 12 : 10;
const occluderSigmaSpace = occluderIsRaw ? occluderEdgeWidth : Math.max(1, occluderEdgeWidth);
const tileArea = (tw > 0 && th > 0) ? (tw * th) : 0;
const occluderMinComponentPxEff = occluderIsRaw ? Math.min(24, occluderMinComponentPx) : occluderMinComponentPx;
const occluderMaxHolePxEff = occluderIsRaw
? Math.max(occluderMaxHolePx, Math.round(Math.max(40, tileArea * 2.5)))
: occluderMaxHolePx;
let occluderBinary = (occluderBinaryRaw && occluderEdgeTrim > 0)
? refineOccluderEdges(
occluderBinaryRaw,
origData.data,
overlay.width,
overlay.height,
occluderEdgeTrim,
undefined,
edgeRejectMask,
)
: occluderBinaryRaw;
if (occluderBinary) {
const beforeCleanup = occluderBinary;
const beforeCoverage = coverageOfMask(beforeCleanup);
let cleaned = removeSmallComponents(
occluderBinary,
overlay.width,
overlay.height,
occluderMinComponentPxEff,
);
cleaned = fillSmallHoles(
cleaned,
overlay.width,
overlay.height,
occluderMaxHolePxEff,
);
const afterCoverage = coverageOfMask(cleaned);
const cleanupRollback = beforeCoverage > 0.02 && afterCoverage < beforeCoverage * 0.78;
if (cleanupRollback) {
console.info('[occluder] cleanup rollback', {
before: Number(beforeCoverage.toFixed(4)),
after: Number(afterCoverage.toFixed(4)),
minComponentPx: occluderMinComponentPxEff,
maxHolePx: occluderMaxHolePxEff,
});
occluderBinary = beforeCleanup;
} else {
occluderBinary = cleaned;
}
}
let occluderAlpha: Uint8Array | null = null;
let rawRescueBase: Uint8Array | null = null;
let rawRescueBand: Uint8Array | null = null;
const allowRawRescue = !preferSegOccluder;
if (occluderBinary) {
const rawMaskSource = resp?.mask_versions?.raw ?? resp?.fg_mask ?? null;
if (!occluderIsRaw && rawMaskSource && allowRawRescue) {
const rawKey = `${rawMaskSource}#raw`;
let rawCanvas = maskCanvasCacheRef.current.get(rawKey) ?? null;
const needsRaw =
!rawCanvas ||
rawCanvas.width !== overlay.width ||
rawCanvas.height !== overlay.height;
if (needsRaw) {
rawCanvas = await prepareMaskAlphaCanvas(
rawMaskSource,
overlay.width,
overlay.height,
{ mode: 'raw' },
);
rawCanvas.dataset.maskTag = rawCanvas.dataset.maskTag ?? 'maskRaw';
maskCanvasCacheRef.current.set(rawKey, rawCanvas);
}
const rawInvert = rawCanvas?.dataset.maskInverted === 'true';
const rawBinary = rawCanvas ? extractBinaryMask(rawCanvas, rawInvert) : null;
if (rawBinary && rawBinary.length === occluderBinary.length) {
const rawCoverage = coverageOfMask(rawBinary);
const rawCoverageOk = rawCoverage >= 0.02 && rawCoverage <= 0.85;
if (!rawCoverageOk) {
console.info('[occluder] raw rescue skipped', {
reason: 'coverage-out-of-range',
rawCoverage: Number(rawCoverage.toFixed(4)),
source: rawMaskSource,
});
}
let bandPad = 0;
if (tileBand) {
const bandHeight = Math.max(1, Math.round(tileBand.yMax - tileBand.yMin));
bandPad = Math.max(8, Math.min(24, Math.round(bandHeight * 0.12)));
}
if (rawCoverageOk) {
rawRescueBase = rawBinary;
if (tileBand) {
const bandMask = new Uint8Array(rawBinary.length);
const yMin = Math.max(0, Math.floor(tileBand.yMin - bandPad));
const yMax = Math.min(overlay.height, Math.ceil(tileBand.yMax + bandPad));
for (let y = yMin; y < yMax; y++) {
const row = y * overlay.width;
bandMask.fill(255, row, row + overlay.width);
}
rawRescueBand = bandMask;
}
if (gridRoiMask) {
const roiCoverage = coverageOfMask(gridRoiMask);
if (roiCoverage >= 0.2) {
rawRescueBand = rawRescueBand ? maskAnd(rawRescueBand, gridRoiMask) : gridRoiMask;
} else {
console.info('[occluder] raw rescue skip grid ROI clamp', {
roiCoverage: Number(roiCoverage.toFixed(4)),
});
}
}
}
}
}
const refined = refineMaskEdges({
mask: occluderBinary,
imageRGBA: origData.data,
width: overlay.width,
height: overlay.height,
edgeWidthPx: occluderEdgeWidth,
mode: 'occluder',
cleanRadius: occluderIsRaw ? 0 : 1,
snapEdges: !occluderIsRaw,
hardThreshold: 0.5,
snapMode: 'gradient',
snapSearchPx: occluderSnapSearchPx,
snapGradientMin: occluderSnapGradientMin,
bandSmoothPx: occluderBandSmoothPx,
bandSmoothThreshold: occluderBandSmoothThreshold,
sigmaColor: occluderSigmaColor,
sigmaSpace: occluderSigmaSpace,
alphaClampToHard: occluderIsRaw,
});
occluderBinary = refined.maskHard;
occluderAlpha = refined.maskAlpha;
if (rawRescueBase && allowRawRescue) {
let rescue = maskMinus(rawRescueBase, occluderBinary);
if (rawRescueBand) {
rescue = maskAnd(rescue, rawRescueBand);
}
const rescueMinComponentPx = Math.max(12, Math.round((overlay.width * overlay.height) * 0.00004));
rescue = removeSmallComponents(rescue, overlay.width, overlay.height, rescueMinComponentPx);
let rescuePixels = 0;
for (let i = 0; i < rescue.length; i++) if (rescue[i]) rescuePixels += 1;
if (rescuePixels > 0) {
occluderBinary = maskOr(occluderBinary, rescue);
if (occluderAlpha) {
for (let i = 0; i < rescue.length; i++) {
if (rescue[i]) occluderAlpha[i] = 255;
}
}
console.info('[occluder] raw rescue applied', {
rescuePixels,
rescueCoverage: Number((rescuePixels / rescue.length).toFixed(4)),
rawCoverage: Number(coverageOfMask(rawRescueBase).toFixed(4)),
baseCoverage: Number(coverageOfMask(occluderBinary).toFixed(4)),
rescueMinComponentPx,
source: rawMaskSource ?? 'unknown',
});
} else {
console.info('[occluder] raw rescue skipped', {
rescuePixels,
rescueMinComponentPx,
});
}
}
if (!occluderIsRaw && resp?.mask_versions?.raw && allowRawRescue) {
const rawTileKey = `${resp.mask_versions.raw}#raw`;
let rawTileCanvas = maskCanvasCacheRef.current.get(rawTileKey) ?? null;
const needsRawTile =
!rawTileCanvas ||
rawTileCanvas.width !== overlay.width ||
rawTileCanvas.height !== overlay.height;
if (needsRawTile) {
rawTileCanvas = await prepareMaskAlphaCanvas(
resp.mask_versions.raw,
overlay.width,
overlay.height,
{ mode: 'raw' },
);
rawTileCanvas.dataset.maskTag = rawTileCanvas.dataset.maskTag ?? 'maskRaw';
maskCanvasCacheRef.current.set(rawTileKey, rawTileCanvas);
}
const rawTileInvert = rawTileCanvas?.dataset.maskInverted === 'true';
const rawTileBinary = rawTileCanvas ? extractBinaryMask(rawTileCanvas, rawTileInvert) : null;
if (rawTileBinary && rawTileBinary.length === occluderBinary.length) {
const growPx = Math.max(4, Math.min(12, Math.round((occluderEdgeTrim || 4) + 2)));
const seed = morphDilate(occluderBinary, overlay.width, overlay.height, growPx);
let rescue = maskAnd(rawTileBinary, seed);
rescue = maskMinus(rescue, occluderBinary);
if (tileBand) {
const bandHeight = Math.max(1, Math.round(tileBand.yMax - tileBand.yMin));
const bandPad = Math.max(8, Math.min(24, Math.round(bandHeight * 0.12)));
const yMin = Math.max(0, Math.floor(tileBand.yMin - bandPad));
const yMax = Math.min(overlay.height, Math.ceil(tileBand.yMax + bandPad));
for (let y = 0; y < overlay.height; y++) {
if (y >= yMin && y < yMax) continue;
const row = y * overlay.width;
rescue.fill(0, row, row + overlay.width);
}
}
const rescueMinComponentPx = Math.max(20, Math.round((overlay.width * overlay.height) * 0.00006));
rescue = removeSmallComponents(rescue, overlay.width, overlay.height, rescueMinComponentPx);
let rescuePixels = 0;
for (let i = 0; i < rescue.length; i++) if (rescue[i]) rescuePixels += 1;
if (rescuePixels > 0) {
occluderBinary = maskOr(occluderBinary, rescue);
if (occluderAlpha) {
for (let i = 0; i < rescue.length; i++) {
if (rescue[i]) occluderAlpha[i] = 255;
}
}
console.info('[occluder] tile-raw rescue applied', {
rescuePixels,
rescueCoverage: Number((rescuePixels / rescue.length).toFixed(4)),
rawCoverage: Number(coverageOfMask(rawTileBinary).toFixed(4)),
baseCoverage: Number(coverageOfMask(occluderBinary).toFixed(4)),
rescueMinComponentPx,
growPx,
});
} else {
console.info('[occluder] tile-raw rescue skipped', {
rescuePixels,
rescueMinComponentPx,
growPx,
});
}
}
}
if (occluderIsRaw && occluderBinaryRaw) {
occluderBinary = maskOr(occluderBinary, occluderBinaryRaw);
if (occluderAlpha) {
for (let i = 0; i < occluderBinary.length; i++) {
if (occluderBinary[i] && occluderAlpha[i] < 255) occluderAlpha[i] = 255;
}
}
occluderBinary = fillSmallHoles(
occluderBinary,
overlay.width,
overlay.height,
occluderMaxHolePxEff,
);
}
}
occluderBinaryRef.current = occluderBinary;
let ignoreMaskCanvas: HTMLCanvasElement | null = null;
if (tileMaskMeta && tilesMaskUrl) {
const ignoreKey = `${tilesMaskUrl}|cls=ignore|${overlay.width}x${overlay.height}`;
ignoreMaskCanvas = tileClassMaskCacheRef.current.get(ignoreKey) ?? null;
if (!ignoreMaskCanvas) {
ignoreMaskCanvas = await prepareIgnoreMaskCanvas(
tilesMaskUrl,
tileMaskMeta,
overlay.width,
overlay.height,
);
if (ignoreMaskCanvas) tileClassMaskCacheRef.current.set(ignoreKey, ignoreMaskCanvas);
}
}
const ignoreBinary = ignoreMaskCanvas ? extractBinaryMask(ignoreMaskCanvas, false) : null;
const tileMaskCandidate = showGrout
? subtractGroutFromTiles(tilesBinaryEffective, groutMask)
: tilesBinaryEffective;
let tileMaskFinal = tileMaskCandidate;
if (occluderBinary) {
const occDil = morphDilate(
occluderBinary,
overlay.width,
overlay.height,
OCCLUDER_NO_PAINT_DILATE_PX,
);
const noPaint = ignoreBinary ? maskOr(occDil, ignoreBinary) : occDil;
tileMaskFinal = maskMinus(tileMaskCandidate, noPaint);
} else if (ignoreBinary) {
tileMaskFinal = maskMinus(tileMaskCandidate, ignoreBinary);
}
tileMaskFinal = removeSmallComponents(
tileMaskFinal,
overlay.width,
overlay.height,
tileMinComponentPx,
);
tileMaskFinal = fillSmallHoles(
tileMaskFinal,
overlay.width,
overlay.height,
tileMaxHolePx,
);
const tileRefined = refineMaskEdges({
mask: tileMaskFinal,
imageRGBA: origData.data,
width: overlay.width,
height: overlay.height,
edgeWidthPx: Math.max(1, Math.round(effectiveTileFeatherPx || 2)),
mode: 'tile',
cleanRadius: 1,
snapEdges: true,
hardThreshold: 0.55,
snapMode: 'gradient',
snapSearchPx: Math.max(3, Math.round((effectiveTileFeatherPx || 2) + 2)),
snapGradientMin: 14,
alphaClampToHard: true,
bandSmoothPx: 2,
bandSmoothThreshold: 0.6,
});
tileMaskFinal = tileRefined.maskHard;
tileMaskFinal = removeSmallComponents(
tileMaskFinal,
overlay.width,
overlay.height,
tileMinComponentPx,
);
tileMaskFinal = fillSmallHoles(
tileMaskFinal,
overlay.width,
overlay.height,
tileMaxHolePx,
);
if (occluderBinary && effectiveOccluderLeakGuardPx > 0) {
const leakGuard = morphDilate(
tileMaskFinal,
overlay.width,
overlay.height,
effectiveOccluderLeakGuardPx,
);
occluderBinary = maskMinus(occluderBinary, leakGuard);
if (occluderAlpha) {
for (let i = 0; i < leakGuard.length; i++) {
if (leakGuard[i] > 0) occluderAlpha[i] = 0;
}
}
}
occluderBinaryRef.current = occluderBinary;
let tileAlphaFinal = new Uint8Array(tileMaskFinal);
if (occluderAlpha) {
for (let i = 0; i < tileAlphaFinal.length; i++) {
tileAlphaFinal[i] = Math.round((tileAlphaFinal[i] * (255 - occluderAlpha[i])) / 255);
}
}
const tileFillCanvas = canvasFromMask(tileMaskFinal, overlay.width, overlay.height);
advancedAllowedMatte = tileFillCanvas;
const occluderBinaryForDebug = (() => {
if (occluderBinaryRef.current) return occluderBinaryRef.current;
const maskAlphaCanvas = maskAlphaCanvasRef.current;
if (!maskAlphaCanvas) return null;
const bin = extractBinaryMask(maskAlphaCanvas, false);
return bin ?? null;
})();
const countSolid = (mask: Uint8Array | null) => {
if (!mask || !mask.length) return 0;
let solid = 0;
for (let i = 0; i < mask.length; i++) if (mask[i] > 0) solid += 1;
return solid;
};
const gridStats = (() => {
const v = renderGrid.vlines ?? [];
const h = renderGrid.hlines ?? [];
const monotonic = (arr: number[]) => arr.every((val, idx) => idx === 0 || val >= arr[idx - 1]!);
const inRange = (arr: number[], min: number, max: number) => arr.every((val) => val >= min && val <= max);
const bounds = (arr: number[]) => ({
min: arr.length ? Math.min(...arr) : null,
max: arr.length ? Math.max(...arr) : null,
});
return {
vCount: v.length,
hCount: h.length,
vBounds: bounds(v),
hBounds: bounds(h),
vMonotonic: monotonic(v),
hMonotonic: monotonic(h),
vInBounds: inRange(v, -4, overlay.width + 4),
hInBounds: inRange(h, -4, overlay.height + 4),
};
})();
const tileEdgeBand = (showOccDebug || logRenderDebug)
? buildEdgeBand(tileMaskFinal, overlay.width, overlay.height, 1)
: null;
const occluderEdgeBand = (showOccDebug || logRenderDebug) && occluderBinary
? buildEdgeBand(occluderBinary, overlay.width, overlay.height, 1)
: null;
if (showOccDebug || logRenderDebug) {
const serverCounts = (tileMaskMeta as any)?.classPixelCounts as Record<string, number> | undefined;
const serverCountsRaw = (tileMaskMeta as any)?.classPixelCountsRaw as Record<string, number> | undefined;
console.info('[render] advanced sanity', {
image: { w: overlay.width, h: overlay.height },
tilesMaskSource,
tilesBinary: {
solid: countSolid(tilesBinaryEffective),
coverage: Number(coverageOfMask(tilesBinaryEffective).toFixed(4)),
},
tilesBinaryGrid: gridRoiMask
? {
solid: countSolid(gridRoiMask),
coverage: Number(coverageOfMask(gridRoiMask).toFixed(4)),
}
: null,
tilesBinaryRaw: {
solid: countSolid(tilesBinary),
coverage: Number(coverageOfMask(tilesBinary).toFixed(4)),
},
tileFillMask: {
solid: countSolid(tileMaskFinal),
coverage: Number(coverageOfMask(tileMaskFinal).toFixed(4)),
},
groutMask: {
solid: countSolid(groutMask),
coverage: Number(coverageOfMask(groutMask).toFixed(4)),
},
occluder: occluderBinaryForDebug
? {
solid: countSolid(occluderBinaryForDebug),
coverage: Number(coverageOfMask(occluderBinaryForDebug).toFixed(4)),
}
: null,
edgeMetrics: {
tileEdgeRoughness: Number(edgeRoughness(tileMaskFinal, overlay.width, overlay.height).toFixed(4)),
occluderEdgeRoughness: occluderBinaryForDebug
? Number(edgeRoughness(occluderBinaryForDebug, overlay.width, overlay.height).toFixed(4))
: null,
haloRisk: Number(haloRisk(tileAlphaFinal, tileMaskFinal).toFixed(5)),
},
cleanup: {
tileMinComponentPx,
tileMaxHolePx,
occluderMinComponentPx,
occluderMaxHolePx,
},
tileBand,
allowTileOverlay,
grid: gridStats,
serverClassPixelCounts: serverCounts ?? null,
serverClassPixelCountsRaw: serverCountsRaw ?? null,
});
}
// Render a grid-aligned tile pattern (no homography) and hard-clip to the tile fill region.
octx.clearRect(0, 0, overlay.width, overlay.height);
octx.putImageData(tilePattern.imageData, 0, 0);
octx.save();
octx.globalCompositeOperation = 'destination-in';
octx.drawImage(tileFillCanvas, 0, 0);
octx.restore();
// Synthetic grout pass
if (showGrout) {
const groutCanvas = canvasFromMask(groutMask, overlay.width, overlay.height);
const groutRgb = [30, 30, 30] as const;
const softGrout = applyFeather(groutCanvas, 1);
const groutLayer = document.createElement('canvas');
groutLayer.width = overlay.width;
groutLayer.height = overlay.height;
const gctx = groutLayer.getContext('2d')!;
gctx.clearRect(0, 0, groutLayer.width, groutLayer.height);
gctx.fillStyle = `rgb(${groutRgb[0]},${groutRgb[1]},${groutRgb[2]})`;
gctx.fillRect(0, 0, groutLayer.width, groutLayer.height);
gctx.globalCompositeOperation = 'destination-in';
gctx.drawImage(softGrout, 0, 0);
gctx.globalCompositeOperation = 'source-over';
octx.save();
octx.globalAlpha = Math.max(0, Math.min(1, tileGroutOpacity));
octx.drawImage(groutLayer, 0, 0);
octx.restore();
}
// Shading maps from original image + occluder
if (occluderBinary) {
advancedOccluderTrimCanvas = canvasFromMask(occluderBinary, overlay.width, overlay.height);
advancedOccluderTrimCanvasRef.current = advancedOccluderTrimCanvas;
} else {
advancedOccluderTrimCanvasRef.current = null;
}
const tilesShadeMask = tileMaskFinal;
const light = buildLightMap({
rgba: origData.data,
width: overlay.width,
height: overlay.height,
tilesMask: tilesShadeMask,
blurRadiusPx: 48,
});
const shadow = occluderBinary
? buildShadowMap({
occluderMask: occluderBinary,
tilesMask: tilesBinaryEffective,
width: overlay.width,
height: overlay.height,
dilatePx: 10,
blurRadiusPx: 18,
strength: tileShadowStrength,
})
: { data: new Uint8Array(overlay.width * overlay.height), width: overlay.width, height: overlay.height };
const spec = buildSpecularMap({
rgba: origData.data,
width: overlay.width,
height: overlay.height,
tilesMask: tilesShadeMask,
blurRadiusPx: 6,
postBlurPx: 3,
strength: tileSpecStrength,
});
const overlayData = octx.getImageData(0, 0, overlay.width, overlay.height);
shadeTileOverlayInPlace({
overlay: overlayData,
lightMap: light.data,
shadowMap: shadow.data,
specMap: spec.data,
params: { specStrength: tileSpecStrength },
});
octx.putImageData(overlayData, 0, 0);
const tileAlphaCanvas = canvasFromAlphaU8(tileAlphaFinal, overlay.width, overlay.height);
advancedTileAlphaCanvas = tileAlphaCanvas;
octx.save();
octx.globalCompositeOperation = 'destination-in';
octx.drawImage(tileAlphaCanvas, 0, 0);
octx.restore();
if (showOccDebug) {
const tileMaskFinalCanvas = canvasFromGrayU8(tileMaskFinal, overlay.width, overlay.height);
const tileAlphaFinalCanvas = canvasFromGrayU8(tileAlphaFinal, overlay.width, overlay.height);
const occluderHardUrl = occluderBinary
? canvasFromGrayU8(occluderBinary, overlay.width, overlay.height).toDataURL('image/png')
: null;
const occluderFeatherUrl = occluderBinary
? (occluderAlpha ? canvasFromGrayU8(occluderAlpha, overlay.width, overlay.height).toDataURL('image/png') : null)
: null;
setRenderDebug({
tilesMaskUrl: canvasFromGrayU8(tileMaskFinal, overlay.width, overlay.height).toDataURL('image/png'),
tileMaskFinalUrl: tileMaskFinalCanvas.toDataURL('image/png'),
tileAlphaFinalUrl: tileAlphaFinalCanvas.toDataURL('image/png'),
tileEdgeBandUrl: tileEdgeBand
? canvasFromGrayU8(tileEdgeBand, overlay.width, overlay.height).toDataURL('image/png')
: null,
groutMaskUrl: canvasFromGrayU8(groutMask, overlay.width, overlay.height).toDataURL('image/png'),
lightMapUrl: canvasFromGrayU8Preview(light.data, overlay.width, overlay.height, { autoContrast: true, gamma: 1.2 }).toDataURL('image/png'),
shadowMapUrl: canvasFromGrayU8Preview(shadow.data, overlay.width, overlay.height, { autoContrast: true, gamma: 1.2 }).toDataURL('image/png'),
specMapUrl: canvasFromGrayU8Preview(spec.data, overlay.width, overlay.height, { autoContrast: true, gamma: 1.2 }).toDataURL('image/png'),
tilesRenderedUrl: overlay.toDataURL('image/png'),
occluderHardUrl,
occluderFeatherUrl,
occluderEdgeBandUrl: occluderEdgeBand
? canvasFromGrayU8(occluderEdgeBand, overlay.width, overlay.height).toDataURL('image/png')
: null,
});
} else if (renderDebug) {
setRenderDebug(null);
}
} else {
// Legacy image-space grid rendering (full-frame)
const grout = showGrout ? (resp?.grout_px ?? 0) : 0;
const cols = Math.ceil(overlay.width / tw) + 2;
const rows = Math.ceil(overlay.height / th) + 2;
if (rotationDeg !== 0) {
octx.save();
octx.translate(overlay.width / 2, overlay.height / 2);
octx.rotate((rotationDeg * Math.PI) / 180);
octx.translate(-overlay.width / 2, -overlay.height / 2);
}
for (let r = -1; r < rows; r++) {
for (let c = -1; c < cols; c++) {
const x0 = c * tw + offsetX;
const y0 = r * th + offsetY;
const x1 = x0 + tw;
const y1 = y0 + th;
const shrink = grout / 2;
const px = Math.round(x0 + shrink);
const py = Math.round(y0 + shrink);
const pw = Math.max(0, Math.round(x1 - x0 - grout));
const ph = Math.max(0, Math.round(y1 - y0 - grout));
if (pw > 0 && ph > 0) {
octx.drawImage(tex, px, py, pw, ph);
}
}
}
if (rotationDeg !== 0) octx.restore();
if (showGrout && grid) {
octx.save();
octx.fillStyle = 'rgba(0,0,0,0.15)';
for (const vx of grid.vlines) {
octx.fillRect(Math.round(vx - grout / 2), 0, Math.max(1, Math.round(grout)), overlay.height);
}
for (const hy of grid.hlines) {
octx.fillRect(0, Math.round(hy - grout / 2), overlay.width, Math.max(1, Math.round(grout)));
}
octx.restore();
}
}
}
// 2) Mask vorbereiten
const maskAlphaCanvas = occluderOn ? await ensureMaskCanvas() : null;
const baseOccluderCanvas = advancedOccluderTrimCanvas ?? maskAlphaCanvas;
const compositeOccluderCanvas = baseOccluderCanvas
? buildCompositeOccluderCanvas(
baseOccluderCanvas,
Math.max(OCCLUDER_COMPOSITE_ERODE_PX, effectiveOccluderShrinkPx),
)
: null;
if (showTilesRenderedOnly) {
ctx.clearRect(0, 0, cnv.width, cnv.height);
ctx.save();
if (tileBand && !ignoreTileBandClip) {
ctx.beginPath();
ctx.rect(0, tileBand.yMin, cnv.width, tileBand.yMax - tileBand.yMin);
ctx.clip();
}
ctx.globalAlpha = overlayDrawAlpha;
ctx.drawImage(overlay, 0, 0);
ctx.restore();
return;
}
if (!maskAlphaCanvas) {
if (showTilesRenderedOnly) {
ctx.clearRect(0, 0, cnv.width, cnv.height);
ctx.drawImage(overlay, 0, 0);
return;
}
// Kein Occluder aktiv -> Original (+ optional Overlay, if enabled)
ctx.drawImage(img, 0, 0, cnv.width, cnv.height);
if (canOverlayNow) {
ctx.save();
if (tileBand && !ignoreTileBandClip) {
ctx.beginPath();
ctx.rect(0, tileBand.yMin, cnv.width, tileBand.yMax - tileBand.yMin);
ctx.clip();
}
ctx.globalAlpha = overlayDrawAlpha;
ctx.drawImage(overlay, 0, 0);
ctx.restore();
}
} else {
// 3) Hintergrund- und Vordergrund-Layer erzeugen
const fgCanvas = document.createElement('canvas');
fgCanvas.width = cnv.width; fgCanvas.height = cnv.height;
const fgCtx = fgCanvas.getContext('2d')!;
fgCtx.clearRect(0, 0, cnv.width, cnv.height);
fgCtx.drawImage(img, 0, 0, cnv.width, cnv.height);
fgCtx.globalCompositeOperation = 'destination-in';
// Hard cut for occluder compositing avoids visible halos around objects.
if (compositeOccluderCanvas) fgCtx.drawImage(compositeOccluderCanvas, 0, 0);
const bgCanvas = document.createElement('canvas');
bgCanvas.width = cnv.width; bgCanvas.height = cnv.height;
const bgCtx = bgCanvas.getContext('2d')!;
bgCtx.clearRect(0, 0, cnv.width, cnv.height);
bgCtx.drawImage(img, 0, 0, cnv.width, cnv.height);
bgCtx.globalCompositeOperation = 'destination-out';
// Hard cut for occluder compositing avoids visible halos around objects.
if (compositeOccluderCanvas) bgCtx.drawImage(compositeOccluderCanvas, 0, 0);
// 4) Apply tile overlay only where tiles are allowed (i.e. not occluded).
// Feathering is applied to the *tile matte* and clipped back to the allowed region to avoid
// any tile "ghosting" over occluders.
if (canOverlayNow) {
let tilesMaskCanvas: HTMLCanvasElement | null = null;
const tilesMaskUrl = tileMaskVersions?.raw ?? tileMaskVersions?.combined ?? tileMaskUrl ?? null;
if (advancedTileRender && advancedAllowedMatte) {
tilesMaskCanvas = advancedTileAlphaCanvas ?? advancedAllowedMatte;
} else if (advancedTileRender && tilesMaskUrl && tileMaskMeta) {
const key = `${tilesMaskUrl}|cls=tiles|${overlay.width}x${overlay.height}`;
tilesMaskCanvas = tileClassMaskCacheRef.current.get(key) ?? null;
if (!tilesMaskCanvas) {
tilesMaskCanvas = await prepareTileClassAlphaCanvas(
tilesMaskUrl,
tileMaskMeta,
'tiles',
overlay.width,
overlay.height,
);
tileClassMaskCacheRef.current.set(key, tilesMaskCanvas);
}
}
applyTileAllowedMaskFromOccluder(
overlay,
advancedOccluderTrimCanvas ?? compositeOccluderCanvas ?? maskAlphaCanvas,
advancedTileRender ? 0 : effectiveTileFeatherPx,
tilesMaskCanvas,
0,
);
if (showOccDebug) {
const baseOnly = document.createElement('canvas');
baseOnly.width = cnv.width;
baseOnly.height = cnv.height;
const bctx = baseOnly.getContext('2d')!;
bctx.drawImage(bgCanvas, 0, 0);
bctx.drawImage(fgCanvas, 0, 0);
const baseOnlyUrl = baseOnly.toDataURL('image/png');
const tilesOnlyUrl = overlay.toDataURL('image/png');
setRenderDebug((prev) => (prev ? { ...prev, finalBaseUrl: baseOnlyUrl, finalTilesOnlyUrl: tilesOnlyUrl } : prev));
}
} else {
// Ensure we don't accidentally draw a full-frame overlay when tiles are missing.
const octx2 = overlay.getContext('2d')!;
octx2.clearRect(0, 0, overlay.width, overlay.height);
}
ctx.drawImage(bgCanvas, 0, 0);
if (canOverlayNow) {
ctx.save();
if (tileBand && !ignoreTileBandClip) {
ctx.beginPath();
ctx.rect(0, tileBand.yMin, cnv.width, tileBand.yMax - tileBand.yMin);
ctx.clip();
}
ctx.globalAlpha = overlayDrawAlpha;
ctx.drawImage(overlay, 0, 0);
ctx.restore();
}
ctx.drawImage(fgCanvas, 0, 0);
}
if (showPersp && resp?.grid_poly) {
drawPolyOverlay(ctx, resp.grid_poly, '#10b981', 0.9);
}
})().catch((e)=> setError(String(e)));
}, [
imgUrl,
resp,
maskUrl,
maskProc,
basePatternId,
tileDesignId,
showOriginalTiles,
overlayAlpha,
showGrout,
advancedTileRender,
groutWidthPx,
tileGroutOpacity,
tileShadowStrength,
tileSpecStrength,
rotationDeg,
offsetX,
offsetY,
occluderOn,
showPersp,
showRelight,
ensureMaskCanvas,
tileBand,
effectiveTileFeatherPx,
effectiveOccluderEdgeTrimPx,
effectiveOccluderFeatherPx,
allowTileOverlay,
showTilesRenderedOnly,
ignoreTileBandClip,
logRenderDebug,
]);
useEffect(() => {
let cancelled = false;
const run = async () => {
if (!occluderOn) return;
await ensureMaskCanvas();
if (!showOccDebug) return;
const maskCanvas = maskAlphaCanvasRef.current;
const canvas = canvasRef.current;
if (!canvas || cancelled) return;
const baseCanvas = advancedOccluderTrimCanvasRef.current ?? maskCanvas;
if (!baseCanvas) return;
const compMask = buildCompositeOccluderCanvas(
baseCanvas,
Math.max(OCCLUDER_COMPOSITE_ERODE_PX, effectiveOccluderShrinkPx),
);
await tintMaskOnCanvas(canvas, compMask);
};
run().catch((error) => console.warn('occluder debug failed', error));
return () => {
cancelled = true;
};
}, [occluderOn, showOccDebug, maskProc, maskUrl, resp?.fg_mask, resp?.mask_versions?.normalized, imgUrl, ensureMaskCanvas, effectiveOccluderShrinkPx, isOblique]);
useEffect(() => {
let cancelled = false;
async function run() {
try {
if (!showRelight) return;
const canvas = canvasRef.current;
if (!canvas || !imgUrl) return;
const response = await fetch(imgUrl, { cache: 'no-store' });
if (!response.ok) return;
const buffer = await response.arrayBuffer();
const form = new FormData();
form.append('file', new Blob([buffer], { type: 'image/jpeg' }), 'img.jpg');
const maskSource = maskProc ?? resp?.fg_mask ?? null;
if (maskSource) form.append('fg_mask', maskSource);
const lightResponse = await fetch('/nextjs/api/light-masks', { method: 'POST', body: form });
if (!lightResponse.ok) return;
const json: LightMaskResponse = await lightResponse.json();
if (cancelled) return;
await applyLightShadow(canvas, json.shadow_mask, json.light_mask, {
shadowStrength,
lightStrength,
});
} catch (error) {
console.warn('relight failed', error);
}
}
run();
return () => {
cancelled = true;
};
}, [showRelight, imgUrl, maskProc, resp?.fg_mask, shadowStrength, lightStrength]);
return (
<div className="p-4 space-y-4">
<div className="flex gap-4 items-center">
<input
type="file"
accept="image/*"
onChange={(e) => e.target.files && e.target.files[0] && onFile(e.target.files[0])}
/>
<label className="flex items-center gap-2">
<input type="checkbox" checked={showOccDebug} onChange={(e) => setShowOccDebug(e.target.checked)} />
<span>Occluder-Overlay (grün)</span>
</label>
<label className="flex items-center gap-2">
<input
type="checkbox"
checked={showTilesRenderedOnly}
onChange={(e) => setShowTilesRenderedOnly(e.target.checked)}
/>
TilesRendered only
</label>
<label className="flex items-center gap-2">
<input
type="checkbox"
checked={ignoreTileBandClip}
onChange={(e) => setIgnoreTileBandClip(e.target.checked)}
/>
Ignore tile band clip
</label>
<label className="flex items-center gap-2">
<input
type="checkbox"
checked={logRenderDebug}
onChange={(e) => setLogRenderDebug(e.target.checked)}
/>
Log render debug
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={showPersp} onChange={(e) => setShowPersp(e.target.checked)} />
<span>Perspektiv-Raster</span>
</label>
<label className="flex items-center gap-2">
<span>Pattern</span>
<select value={basePatternId} onChange={(e)=> setBasePatternId(e.target.value)}>
{PATTERNS.map(p => <option key={p.id} value={p.id}>{p.name}</option>)}
</select>
</label>
<label className="flex items-center gap-2">
<span>Tile design</span>
<select value={tileDesignId} onChange={(e)=> setTileDesignId(e.target.value)} disabled={!advancedTileRender}>
{TILE_DESIGNS.map(d => <option key={d.id} value={d.id}>{d.name}</option>)}
</select>
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={showOriginalTiles} onChange={(e)=> setShowOriginalTiles(e.target.checked)} />
<span>Show original tiles</span>
</label>
<label className="flex items-center gap-2">
<span>Alpha</span>
<input type="range" min={0} max={1} step={0.01} value={overlayAlpha} onChange={(e)=> setOverlayAlpha(Number(e.target.value))}/>
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={showGrout} onChange={(e)=> setShowGrout(e.target.checked)} />
<span>Fugen zeigen</span>
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={advancedTileRender} onChange={(e)=> setAdvancedTileRender(e.target.checked)} />
<span>Advanced render</span>
</label>
<label className="flex items-center gap-2">
<span>Grout px</span>
<input
type="range"
min={1}
max={12}
step={1}
value={groutWidthPx}
onChange={(e) => setGroutWidthPx(Math.max(1, Math.min(12, Math.round(Number(e.target.value)))))}
disabled={!advancedTileRender}
/>
<span className="text-xs tabular-nums w-7 text-right">{groutWidthPx}px</span>
</label>
<label className="flex items-center gap-2">
<span>Occ feather</span>
<input
type="range"
min={0}
max={12}
step={1}
value={occluderFeatherPx}
onChange={(e) => setOccluderFeatherPx(Math.max(0, Math.min(12, Math.round(Number(e.target.value)))))}
/>
<span className="text-xs tabular-nums w-7 text-right">{effectiveOccluderFeatherPx}px</span>
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={occluderOn} onChange={(e)=> setOccluderOn(e.target.checked)} />
<span>Occluder-Maske anwenden</span>
</label>
<label className="flex items-center gap-2">
<span>Occluder shrink</span>
<input
type="range"
min={0}
max={16}
step={1}
value={occluderShrinkPx}
onChange={(e) => setOccluderShrinkPx(Math.max(0, Math.min(16, Math.round(Number(e.target.value)))))}
/>
<span className="text-xs tabular-nums w-7 text-right">{effectiveOccluderShrinkPx}px</span>
</label>
<label className="flex items-center gap-2">
<span>Occ edge trim</span>
<input
type="range"
min={0}
max={8}
step={1}
value={occluderEdgeTrimPx}
onChange={(e) => setOccluderEdgeTrimPx(Math.max(0, Math.min(6, Math.round(Number(e.target.value)))))}
/>
<span className="text-xs tabular-nums w-7 text-right">{effectiveOccluderEdgeTrimPx}px</span>
</label>
<label className="flex items-center gap-2">
<span>Leak guard</span>
<input
type="range"
min={0}
max={6}
step={1}
value={occluderLeakGuardPx}
onChange={(e) => setOccluderLeakGuardPx(Math.max(0, Math.min(6, Math.round(Number(e.target.value)))))}
/>
<span className="text-xs tabular-nums w-7 text-right">{effectiveOccluderLeakGuardPx}px</span>
</label>
<label className="flex items-center gap-2">
<span>Tile feather</span>
<input
type="range"
min={0}
max={4}
step={1}
value={tileFeatherPx}
onChange={(e) => setTileFeatherPx(Math.max(0, Math.min(4, Math.round(Number(e.target.value)))))}
/>
<span className="text-xs tabular-nums w-7 text-right">{effectiveTileFeatherPx}px</span>
</label>
<label className="flex items-center gap-2">
<span>Masken-Preset</span>
<select
className="border rounded px-2 py-1"
value={maskPreset}
onChange={(e) => setMaskPreset(e.target.value as MaskPreset)}
>
<option value="tiles-default">Standard</option>
<option value="tiles-detail">Detail</option>
</select>
{maskAppliedPreset && maskAppliedPreset !== maskPreset && (
<span className="text-xs text-gray-500">verwendet: {maskAppliedPreset}</span>
)}
</label>
<label className="flex items-center gap-2">
<input type="checkbox" checked={showRelight} onChange={(e)=> setShowRelight(e.target.checked)} />
<span>Licht/Schatten anwenden</span>
</label>
</div>
{resp?.mask_metrics && (
<div className="flex flex-wrap items-center gap-4 text-xs text-gray-600">
<span className="font-medium">Maskenmetriken</span>
<span>FG norm: {formatPercent(resp.mask_metrics.coverageNorm ?? resp.mask_metrics.coverage)}</span>
{resp.mask_metrics.coverageRaw != null && (
<span>FG raw: {formatPercent(resp.mask_metrics.coverageRaw)}</span>
)}
{resp.mask_metrics.backgroundCoverage != null && (
<span>BG: {formatPercent(resp.mask_metrics.backgroundCoverage)}</span>
)}
{resp.mask_metrics.seedCoverage != null && (
<span>Seed: {formatPercent(resp.mask_metrics.seedCoverage)}</span>
)}
{resp.mask_metrics.latticeInteriorCoverage != null && (
<span>Interior FG: {formatPercent(resp.mask_metrics.latticeInteriorCoverage)}</span>
)}
{resp.mask_metrics.latticeLeakPct != null && (
<span className={resp.mask_metrics.latticeLeakPct > 0.8 ? 'text-red-600' : ''}>
Grout leak: {resp.mask_metrics.latticeLeakPct.toFixed(2)}%
</span>
)}
{resp.mask_metrics.autotuneMs != null && resp.mask_metrics.autotuneMs > 0 && (
<span>Autotune: {resp.mask_metrics.autotuneMs} ms</span>
)}
{resp?.depth_debug?.occluderCoverage != null && (
<span>Depth occ: {formatPercent(resp.depth_debug.occluderCoverage)}</span>
)}
{resp?.grid_poly && (
<span className={isOblique ? 'text-purple-700' : ''}>
View: {isOblique ? 'oblique' : 'frontal'}
</span>
)}
{Array.isArray((resp as any)?.depth_debug?.warnings) && (resp as any).depth_debug.warnings.length > 0 && (
<span className="text-orange-600">Depth: {(resp as any).depth_debug.warnings.join(', ')}</span>
)}
{maskAppliedPreset && (
<span>Preset aktiv: {maskAppliedPreset}</span>
)}
{resp?.mask_metrics?.source && (
<span>Occluder source: {resp.mask_metrics.source}</span>
)}
{maskWarnings.length > 0 && (
<span className="text-orange-600">Warnung: {maskWarnings.join(', ')}</span>
)}
{pipelineFallback?.usedOriginal && (
<span className="px-2 py-1 rounded bg-yellow-100 text-yellow-700">
Fallback aktiv ({fallbackReasons.join(', ') || 'auto'})
</span>
)}
</div>
)}
{showOccDebug && resp?.mask_versions && (
<div className="flex flex-wrap gap-4 text-xs text-gray-600">
{finalMaskUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={finalMaskUrl} alt="Final mask" className="h-32 border rounded" />
<figcaption>Final Mask</figcaption>
</figure>
)}
{resp.mask_versions.overlay && (
<figure className="flex flex-col gap-1 items-start">
<img src={resp.mask_versions.overlay} alt="Mask Overlay" className="h-32 border rounded" />
<figcaption>Overlay</figcaption>
</figure>
)}
{resp.mask_versions.cutout && (
<figure className="flex flex-col gap-1 items-start">
<img src={resp.mask_versions.cutout} alt="Mask Cutout" className="h-32 border rounded" />
<figcaption>Cutout</figcaption>
</figure>
)}
{resp.mask_versions.raw && (
<figure className="flex flex-col gap-1 items-start">
<img src={resp.mask_versions.raw} alt="Mask Raw" className="h-32 border rounded" />
<figcaption>Raw Mask</figcaption>
</figure>
)}
{occluderDebugSource && (
<figure className="flex flex-col gap-1 items-start">
<img src={occluderDebugSource} alt="Occluder mask" className="h-32 border rounded" />
<figcaption>Occluder</figcaption>
</figure>
)}
{renderDebug?.tilesMaskUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.tilesMaskUrl} alt="Tiles mask" className="h-32 border rounded" />
<figcaption>tilesMask</figcaption>
</figure>
)}
{renderDebug?.tileMaskFinalUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.tileMaskFinalUrl} alt="Tile mask final" className="h-32 border rounded" />
<figcaption>tileMaskFinal</figcaption>
</figure>
)}
{renderDebug?.tileAlphaFinalUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.tileAlphaFinalUrl} alt="Tile alpha final" className="h-32 border rounded" />
<figcaption>tileAlphaFinal</figcaption>
</figure>
)}
{renderDebug?.tileEdgeBandUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.tileEdgeBandUrl} alt="Tile edge band" className="h-32 border rounded" />
<figcaption>tileEdgeBand</figcaption>
</figure>
)}
{renderDebug?.groutMaskUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.groutMaskUrl} alt="Grout mask" className="h-32 border rounded" />
<figcaption>groutMask</figcaption>
</figure>
)}
{renderDebug?.lightMapUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.lightMapUrl} alt="Light map" className="h-32 border rounded" />
<figcaption>lightMap</figcaption>
</figure>
)}
{renderDebug?.shadowMapUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.shadowMapUrl} alt="Shadow map" className="h-32 border rounded" />
<figcaption>shadowMap</figcaption>
</figure>
)}
{renderDebug?.specMapUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.specMapUrl} alt="Specular map" className="h-32 border rounded" />
<figcaption>specMap</figcaption>
</figure>
)}
{renderDebug?.tilesRenderedUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.tilesRenderedUrl} alt="Tiles rendered" className="h-32 border rounded" />
<figcaption>tilesRendered</figcaption>
</figure>
)}
{renderDebug?.occluderHardUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.occluderHardUrl} alt="Occluder hard" className="h-32 border rounded" />
<figcaption>occluderHard</figcaption>
</figure>
)}
{renderDebug?.occluderFeatherUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.occluderFeatherUrl} alt="Occluder feather" className="h-32 border rounded" />
<figcaption>occluderFeather</figcaption>
</figure>
)}
{renderDebug?.occluderEdgeBandUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.occluderEdgeBandUrl} alt="Occluder edge band" className="h-32 border rounded" />
<figcaption>occluderEdgeBand</figcaption>
</figure>
)}
{renderDebug?.finalBaseUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.finalBaseUrl} alt="Final base only" className="h-32 border rounded" />
<figcaption>finalBase</figcaption>
</figure>
)}
{renderDebug?.finalTilesOnlyUrl && (
<figure className="flex flex-col gap-1 items-start">
<img src={renderDebug.finalTilesOnlyUrl} alt="Final tiles only" className="h-32 border rounded" />
<figcaption>finalTilesOnly</figcaption>
</figure>
)}
{(resp as any)?.depth_debug?.rejected && (resp as any)?.depth_debug?.occluderUrlRejected && (
<figure className="flex flex-col gap-1 items-start">
<img
src={(resp as any).depth_debug.occluderUrlRejected}
alt="Rejected depth occluder mask"
className="h-32 border rounded"
/>
<figcaption className="text-orange-700">Depth occluder (rejected)</figcaption>
</figure>
)}
</div>
)}
{showOccDebug && (resp?.mask_debug?.originMeta || resp?.fg_mask_meta) && (
<details className="text-xs text-gray-600 mt-2">
<summary className="cursor-pointer font-medium">Occluder meta</summary>
<pre className="mt-2 p-2 bg-gray-50 border rounded overflow-auto max-h-72">
{JSON.stringify({
source: resp?.mask_debug?.origin ?? null,
originMeta: resp?.mask_debug?.originMeta ?? null,
fgMaskMeta: resp?.fg_mask_meta ?? null,
}, null, 2)}
</pre>
</details>
)}
{tileMaskDisplayUrl && (
<div className="rounded-xl border p-3 bg-white space-y-3">
<div className="flex flex-wrap items-center justify-between gap-2">
<div className="text-sm font-medium text-gray-800">Tile segmentation</div>
<div className="flex items-center gap-1 text-xs">
{tileMaskViewOptions.map((opt) => {
const isActive = tileMaskViewMode === opt.mode;
const classes = [
'px-2 py-0.5 rounded border transition-colors',
isActive ? 'bg-gray-900 text-white border-gray-900' : 'bg-white text-gray-700 border-gray-300 hover:bg-gray-50',
!opt.available ? 'opacity-40 cursor-not-allowed' : '',
].join(' ');
return (
<button
key={opt.mode}
type="button"
className={classes}
onClick={() => opt.available && setTileMaskViewMode(opt.mode)}
disabled={!opt.available}
title={opt.title}
>
{opt.label}
</button>
);
})}
</div>
</div>
{(showGroutMissing || showTilesMissing) && (
<div className="text-xs text-orange-600 space-y-0.5">
{showGroutMissing && <div>Grout missing (model output below 0.5%).</div>}
{showTilesMissing && <div>Tiles missing (model output below 1%).</div>}
</div>
)}
{tileMaskMeta?.source?.includes('grout_heur') && (
<div className="text-xs text-gray-500">
Grout is heuristic (generated from tiles), not a UNet class.
</div>
)}
<img src={tileMaskDisplayUrl} alt="Tile segmentation" className="w-full rounded border" />
{tileCoverageEntries.length > 0 && (
<div className="text-xs text-gray-700 space-y-1" title={COVERAGE_DESCRIPTIONS.interior}>
<div className="font-medium">Interior coverage (occluder removed)</div>
<div className="grid grid-cols-2 gap-2">
{tileCoverageEntries.map(([name, value]) => (
<div key={name} className="flex items-center gap-2">
<span
className="inline-block w-3 h-3 rounded-sm border"
style={{ backgroundColor: tileColorForClass(name) }}
/>
<span className="capitalize">{name}</span>
<span className="ml-auto text-right">{(value * 100).toFixed(1)}%</span>
</div>
))}
</div>
</div>
)}
{tileMaskMeta?.classPixelCounts && (
<div className="text-xs text-gray-600 space-y-1">
<div className="font-medium">Class pixel counts</div>
<div className="grid grid-cols-2 gap-2">
{Object.entries(tileMaskMeta.classPixelCounts).map(([key, value]) => (
<div key={key} className="flex items-center gap-2">
<span className="font-mono">id {key}</span>
<span className="text-gray-500">
{tileMaskMeta.classIdLegend?.[key] ? `(${tileMaskMeta.classIdLegend[key]})` : ''}
</span>
<span className="ml-auto">{Number(value).toLocaleString()}</span>
</div>
))}
</div>
{tileMaskMeta.classPixelCountsRaw && (
<div className="text-gray-500">Raw UNet counts available as `classPixelCountsRaw` (legend: `classIdLegendRaw`).</div>
)}
</div>
)}
{(tileMaskMeta as any)?.unetDebug && (
<details className="text-xs text-gray-600">
<summary className="cursor-pointer font-medium">UNet debug</summary>
<pre className="mt-2 p-2 bg-gray-50 border rounded overflow-auto max-h-64">
{JSON.stringify((tileMaskMeta as any).unetDebug, null, 2)}
</pre>
</details>
)}
{tileCoverageAbsoluteEntries.length > 0 && (
<div className="text-xs text-gray-600 space-y-1" title={COVERAGE_DESCRIPTIONS.absolute}>
<div className="font-medium">Absolute coverage (full frame)</div>
<div className="grid grid-cols-2 gap-2">
{tileCoverageAbsoluteEntries.map(([name, value]) => (
<div key={name} className="flex items-center gap-2">
<span
className="inline-block w-3 h-3 rounded-sm border"
style={{ backgroundColor: tileColorForClass(name) }}
/>
<span className="capitalize">{name}</span>
<span className="ml-auto text-right">{(value * 100).toFixed(1)}%</span>
</div>
))}
</div>
</div>
)}
{tileCoverageRawEntries.length > 0 && (
<div className="text-xs text-gray-600 space-y-1">
<div className="font-medium">Raw UNet coverage</div>
<div className="grid grid-cols-2 gap-2">
{tileCoverageRawEntries.map(([name, value]) => (
<div key={name} className="flex items-center gap-2">
<span
className="inline-block w-3 h-3 rounded-sm border"
style={{ backgroundColor: tileColorForClass(name) }}
/>
<span className="capitalize">{name}</span>
<span className="ml-auto">{(value * 100).toFixed(1)}%</span>
</div>
))}
</div>
</div>
)}
{(tileOccluderCoverage != null || tileOccludedCoverage != null) && (
<div className="text-xs text-gray-600 space-y-0.5">
{tileOccluderCoverage != null && (
<div title={COVERAGE_DESCRIPTIONS.occluder}>Occluder coverage: {(tileOccluderCoverage * 100).toFixed(1)}%</div>
)}
{tileOccludedCoverage != null && (
<div title={COVERAGE_DESCRIPTIONS.occluded}>Occluded tile area: {(tileOccludedCoverage * 100).toFixed(1)}%</div>
)}
</div>
)}
{(tileOccluderPixels != null || tileOccludedPixels != null) && (
<div className="text-xs text-gray-500 space-y-0.5">
{tileOccluderPixels != null && (
<div>Occluder pixels: {tileOccluderPixels.toLocaleString()}</div>
)}
{tileOccludedPixels != null && (
<div>Pixels reclassified to occluder: {tileOccludedPixels.toLocaleString()}</div>
)}
</div>
)}
{tileMaskVersions && (tileMaskVersions.raw || tileMaskVersions.occluder) && (
<div className="flex flex-wrap gap-3 text-xs text-gray-600">
{tileMaskVersions.raw && (
<figure className="flex flex-col gap-1 items-start">
<img src={tileMaskVersions.raw} alt="Raw tile mask" className="h-28 border rounded" />
<figcaption>Raw UNet mask</figcaption>
</figure>
)}
{tileMaskVersions.occluder && (
<figure className="flex flex-col gap-1 items-start">
<img src={tileMaskVersions.occluder} alt="Occluder mask" className="h-28 border rounded" />
<figcaption>Occluder</figcaption>
</figure>
)}
</div>
)}
{tileMaskMeta?.source && (
<div className="text-xs text-gray-500">
Source: {tileMaskMeta.source}
{tileMaskMeta.model ? ` · Model: ${tileMaskMeta.model}` : null}
</div>
)}
{tileBand && (
<div className="text-xs text-gray-600">
Fliesenspiegel: y {tileBand.yMin}–{tileBand.yMax} (conf {(tileBand.confidence * 100).toFixed(1)}%)
</div>
)}
</div>
)}
<div className="flex flex-wrap gap-4 items-center text-sm">
<label className="flex items-center gap-2">
<span>Model</span>
<select value={modelVersion} onChange={(e) => setModelVersion(e.target.value as TileSegModel)}>
{TILE_MODEL_OPTIONS.map((opt) => (
<option key={opt.value} value={opt.value}>{opt.label}</option>
))}
</select>
</label>
<label className="flex items-center gap-2">
<span>U2 weight</span>
<input
type="range"
min={0.5}
max={2}
step={0.1}
value={wu}
onChange={(e) => setWu(parseFloat(e.target.value))}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
/>
<span className="w-10 text-right">{wu.toFixed(1)}</span>
</label>
<label className="flex items-center gap-2">
<span>Heur weight</span>
<input
type="range"
min={0}
max={2}
step={0.1}
value={wh}
onChange={(e) => setWh(parseFloat(e.target.value))}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
/>
<span className="w-10 text-right">{wh.toFixed(1)}</span>
</label>
<label className="flex items-center gap-2">
<span>Fusion blur</span>
<input
type="range"
min={0}
max={3}
step={1}
value={fblur}
onChange={(e) => setFblur(Number(e.target.value))}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
/>
<span className="w-8 text-right">{fblur}</span>
</label>
<label className="flex items-center gap-2">
<span>RANSAC</span>
<select
value={ransacMode}
onChange={(e) => setRansacMode(e.target.value as 'off' | 'auto' | '1')}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
>
<option value="off">off</option>
<option value="auto">auto</option>
<option value="1">force</option>
</select>
</label>
</div>
<div className="flex flex-wrap gap-4 items-center text-sm">
<span className="font-medium text-gray-700">Mask tuning</span>
<label className="flex items-center gap-2">
<span>Grow</span>
<input
type="range"
min={0}
max={20}
step={1}
value={maskGrowPx}
onChange={(e) => setMaskGrowPx(Number(e.target.value))}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
/>
<span className="w-12 text-right">{maskGrowPx.toFixed(0)}px</span>
</label>
<label className="flex items-center gap-2">
<span>Threshold</span>
<input
type="range"
min={0.2}
max={0.8}
step={0.02}
value={maskThresh}
onChange={(e) => setMaskThresh(Number(e.target.value))}
disabled={heuristicsDisabled}
title={heuristicsDisabled ? 'Disabled for UNet_latest (direct mask mode)' : undefined}
/>
<span className="w-12 text-right">{maskThresh.toFixed(2)}</span>
</label>
<label className="flex items-center gap-2">
<span>Pre-blur</span>
<input
type="range"
min={0}
max={4}
step={1}
value={maskPreBlur}
onChange={(e) => setMaskPreBlur(Number(e.target.value))}
/>
<span className="w-10 text-right">{maskPreBlur}</span>
</label>
<label className="flex items-center gap-2">
<span>Edge blur</span>
<input
type="range"
min={0}
max={4}
step={1}
value={maskFinalBlur}
onChange={(e) => setMaskFinalBlur(Number(e.target.value))}
/>
<span className="w-10 text-right">{maskFinalBlur}</span>
</label>
<label className="flex items-center gap-2">
<span>Source</span>
<select
className="border rounded px-2 py-1"
value={maskPreference}
onChange={(e) => setMaskPreference(e.target.value as MaskPreference)}
>
<option value="auto">Auto</option>
<option value="processed">Processed</option>
<option value="server">Server</option>
<option value="client">Client</option>
<option value="fused">Fused</option>
<option value="raw">Server Raw</option>
</select>
</label>
<button
type="button"
className="px-2 py-1 border rounded"
onClick={() => {
setMaskGrowPx(8)
setMaskPreBlur(1)
setMaskThresh(0.4)
setMaskFinalBlur(2)
setOccluderShrinkPx(6)
setTileFeatherPx(0)
setMaskPreference('auto')
setMaskPreset('tiles-default')
}}
>
Reset mask
</button>
</div>
<div className="flex flex-wrap gap-4 items-center text-sm">
<span className="font-medium text-gray-700">Light & shadow</span>
<label className="flex items-center gap-2">
<span>Shadow</span>
<input
type="range"
min={0}
max={1}
step={0.05}
value={shadowStrength}
onChange={(e) => setShadowStrength(Number(e.target.value))}
/>
<span className="w-12 text-right">{(shadowStrength * 100).toFixed(0)}%</span>
</label>
<label className="flex items-center gap-2">
<span>Light</span>
<input
type="range"
min={0}
max={1}
step={0.05}
value={lightStrength}
onChange={(e) => setLightStrength(Number(e.target.value))}
/>
<span className="w-12 text-right">{(lightStrength * 100).toFixed(0)}%</span>
</label>
<button
type="button"
className="px-2 py-1 border rounded"
onClick={() => {
setShadowStrength(1)
setLightStrength(1)
}}
>
Reset light
</button>
</div>
{resp && (
<div className="text-sm text-gray-800">
<div><b>Service: tile_w / tile_h / grout</b></div>
<div>{resp.tile_w_px ?? '—'} / {resp.tile_h_px ?? '—'} / {resp.grout_px ?? '—'}</div>
<div>Quelle: {resp.source ?? '—'}</div>
{resp.mix && (
<div className="space-y-1">
<div>
Mix: {resp.mix.mode ?? '—'} · wu {(resp.mix.weights?.u2 ?? resp.mix.wu ?? '—')} / wh {(resp.mix.weights?.heur ?? resp.mix.wh ?? '—')} · blur {(resp.mix.fblur_used ?? resp.mix.fblur ?? '—')}
</div>
{(resp.mix.u2_coverage != null || resp.mix.heur_coverage != null) && (
<div className="text-xs text-gray-600">
Occluder coverage · U2 {(resp.mix.u2_coverage != null ? (resp.mix.u2_coverage * 100).toFixed(1) + '%' : '—')} / Heur {(resp.mix.heur_coverage != null ? (resp.mix.heur_coverage * 100).toFixed(1) + '%' : '—')}
</div>
)}
{Array.isArray(resp.mix.weight_adjustments) && resp.mix.weight_adjustments.length > 0 && (
<div className="text-xs text-amber-600">
Weight adjustments: {resp.mix.weight_adjustments.join(', ')}
</div>
)}
</div>
)}
{resp.ransac && (
<div>
RANSAC: {resp.ransac.mode ?? '—'}
{resp.ransac.coverage != null ? ` · coverage ${(resp.ransac.coverage * 100).toFixed(1)}%` : ''}
</div>
)}
{analyzeBusy && <div className="text-xs text-blue-600">Refreshing server analysis…</div>}
{maskBusy && <div className="text-xs text-blue-600">Computing occluder mask…</div>}
{!maskBusy && maskUrl && <div className="text-xs text-green-700">Mask ready</div>}
</div>
)}
{analyzeBusy && !resp && <div className="text-sm text-blue-600">Analyzing image…</div>}
{error && <div className="text-red-600">{error}</div>}
<div className="border inline-block">
<canvas ref={canvasRef} />
</div>
</div>
);
}
// --- Client-side U2Net (WASM) helper ---
async function computeClientMask(file: File): Promise<string | null> {
const wasmConfig: OrtWasmConfig = {
wasmPaths: `${BASE_PATH}/_ort/`,
simd: true,
numThreads: 1,
};
const globalOrt = globalThis as OrtGlobal;
const globalEnv = (globalOrt.ort ??= { env: {} });
globalEnv.env ??= {};
globalEnv.env.wasm = { ...(globalEnv.env.wasm ?? {}), ...wasmConfig };
const ortModule = (await import('onnxruntime-web')) as OrtModule;
try {
const wasmEnv = ortModule.env?.wasm;
if (wasmEnv) {
Object.assign(wasmEnv, wasmConfig);
}
} catch {
// ignore configuration errors to avoid blocking mask generation
}
const img = await fileToImage(file);
const target = 320;
const cnv = document.createElement('canvas');
cnv.width = target; cnv.height = target;
const ctx = cnv.getContext('2d')!;
ctx.clearRect(0, 0, target, target);
const sourceW = Math.max(1, img.naturalWidth || img.width || target);
const sourceH = Math.max(1, img.naturalHeight || img.height || target);
const scale = target / Math.max(sourceW, sourceH);
const drawW = Math.max(1, Math.min(target, Math.round(sourceW * scale)));
const drawH = Math.max(1, Math.min(target, Math.round(sourceH * scale)));
const offsetX = Math.floor((target - drawW) / 2);
const offsetY = Math.floor((target - drawH) / 2);
ctx.fillStyle = 'black';
ctx.fillRect(0, 0, target, target);
ctx.drawImage(img, 0, 0, sourceW, sourceH, offsetX, offsetY, drawW, drawH);
const rgbaLetter = ctx.getImageData(0, 0, target, target).data;
const toCHW = (rgba: Uint8ClampedArray) => {
const out = new Float32Array(3 * target * target);
for (let i = 0, p = 0; i < rgba.length; i += 4, p++) {
const r = rgba[i] / 255;
const g = rgba[i + 1] / 255;
const b = rgba[i + 2] / 255;
out[p] = r;
out[p + target * target] = g;
out[p + 2 * target * target] = b;
}
return out;
};
const chw = toCHW(rgbaLetter);
const session = await ortModule.InferenceSession.create(`${BASE_PATH}/models/u2netp.onnx`, { executionProviders: ['wasm'] });
const inputName = session.inputNames[0];
const input = new ortModule.Tensor('float32', chw, [1, 3, target, target]);
const results = await session.run({ [inputName]: input });
const firstOut = results[Object.keys(results)[0]];
const outData = firstOut.data as Float32Array;
const mask320 = new Uint8ClampedArray(target * target * 4);
for (let i = 0; i < target * target; i++) {
let v = outData[i]; if (!Number.isFinite(v)) v = 0;
v = 1 / (1 + Math.exp(-v));
const g = Math.max(0, Math.min(255, Math.round(v * 255)));
const j = i * 4;
mask320[j] = g; mask320[j + 1] = g; mask320[j + 2] = g; mask320[j + 3] = g;
}
const mcSmall = document.createElement('canvas');
mcSmall.width = target; mcSmall.height = target;
mcSmall.getContext('2d')!.putImageData(new ImageData(mask320, target, target), 0, 0);
const mcFull = document.createElement('canvas');
mcFull.width = sourceW; mcFull.height = sourceH;
const mctx = mcFull.getContext('2d')!;
mctx.imageSmoothingEnabled = true;
const sx = Math.max(0, Math.min(target, offsetX));
const sy = Math.max(0, Math.min(target, offsetY));
const sw = Math.max(1, Math.min(target - sx, drawW));
const sh = Math.max(1, Math.min(target - sy, drawH));
mctx.clearRect(0, 0, mcFull.width, mcFull.height);
mctx.drawImage(mcSmall, sx, sy, sw, sh, 0, 0, mcFull.width, mcFull.height);
const mctxRead = mcFull.getContext('2d', { willReadFrequently: true })!;
const letterImage = mctxRead.getImageData(0, 0, sourceW, sourceH);
const letterData = letterImage.data;
const coverage = (() => {
let solid = 0;
const total = sourceW * sourceH;
for (let i = 3; i < letterData.length; i += 4) {
if (letterData[i] >= 96) solid += 1;
}
return total > 0 ? solid / total : 0;
})();
const shouldFallback = coverage < MASK_FALLBACK_THRESHOLD && Math.min(sourceW, sourceH) >= 80;
if (shouldFallback) {
const stretchCanvas = document.createElement('canvas');
stretchCanvas.width = target; stretchCanvas.height = target;
const stretchCtx = stretchCanvas.getContext('2d')!;
stretchCtx.clearRect(0, 0, target, target);
stretchCtx.drawImage(img, 0, 0, sourceW, sourceH, 0, 0, target, target);
const rgbaStretch = stretchCtx.getImageData(0, 0, target, target).data;
const chwStretch = toCHW(rgbaStretch);
const inputStretch = new ortModule.Tensor('float32', chwStretch, [1, 3, target, target]);
const stretchResults = await session.run({ [inputName]: inputStretch });
const stretchOut = stretchResults[Object.keys(stretchResults)[0]];
const stretchData = stretchOut.data as Float32Array;
const maskStretch320 = new Uint8ClampedArray(target * target * 4);
for (let i = 0; i < target * target; i++) {
let v = stretchData[i]; if (!Number.isFinite(v)) v = 0;
v = 1 / (1 + Math.exp(-v));
const g = Math.max(0, Math.min(255, Math.round(v * 255)));
const j = i * 4;
maskStretch320[j] = g; maskStretch320[j + 1] = g; maskStretch320[j + 2] = g; maskStretch320[j + 3] = g;
}
const stretchSmall = document.createElement('canvas');
stretchSmall.width = target; stretchSmall.height = target;
stretchSmall.getContext('2d')!.putImageData(new ImageData(maskStretch320, target, target), 0, 0);
const stretchFull = document.createElement('canvas');
stretchFull.width = sourceW; stretchFull.height = sourceH;
const stretchFullCtx = stretchFull.getContext('2d')!;
stretchFullCtx.imageSmoothingEnabled = true;
stretchFullCtx.drawImage(stretchSmall, 0, 0, target, target, 0, 0, sourceW, sourceH);
const stretchImage = stretchFullCtx.getImageData(0, 0, sourceW, sourceH);
const stretchAlpha = stretchImage.data;
for (let i = 3; i < letterData.length && i < stretchAlpha.length; i += 4) {
const v = Math.max(letterData[i], stretchAlpha[i]);
letterData[i] = v;
letterData[i - 1] = v;
letterData[i - 2] = v;
letterData[i - 3] = v;
}
mctxRead.putImageData(letterImage, 0, 0);
}
const finalCoverage = (() => {
let solid = 0;
const total = sourceW * sourceH;
for (let i = 3; i < letterData.length; i += 4) {
if (letterData[i] >= 96) solid += 1;
}
return total > 0 ? solid / total : 0;
})();
if (finalCoverage > 0 && finalCoverage < 0.25) {
const binary = new Uint8Array(sourceW * sourceH);
for (let i = 0, idx = 3; i < binary.length; i++, idx += 4) {
binary[i] = letterData[idx] >= 96 ? 1 : 0;
}
const dilated = new Uint8Array(binary);
const radius = finalCoverage < 0.08 ? 2 : 1;
for (let y = 0; y < sourceH; y++) {
for (let x = 0; x < sourceW; x++) {
const idx = y * sourceW + x;
if (binary[idx]) continue;
let grow = false;
for (let dy = -radius; dy <= radius && !grow; dy++) {
const ny = y + dy;
if (ny < 0 || ny >= sourceH) continue;
for (let dx = -radius; dx <= radius; dx++) {
const nx = x + dx;
if (nx < 0 || nx >= sourceW) continue;
if (Math.hypot(dx, dy) > radius + 0.01) continue;
if (binary[ny * sourceW + nx]) { grow = true; break; }
}
}
if (grow) dilated[idx] = 1;
}
}
for (let i = 0, idx = 3; i < dilated.length; i++, idx += 4) {
if (dilated[i]) {
letterData[idx] = 255;
letterData[idx - 1] = letterData[idx - 2] = letterData[idx - 3] = 255;
}
}
mctxRead.putImageData(letterImage, 0, 0);
}
return mcFull.toDataURL('image/png');
}
function fileToImage(file: File): Promise<HTMLImageElement> {
return new Promise((res, rej) => {
const img = new Image();
img.onload = () => res(img);
img.onerror = (e) => rej(e);
img.crossOrigin = 'anonymous';
img.src = URL.createObjectURL(file);
});
}
/* --- MaskLayer: Beispiel-Aufruf (aktivieren & an deine Variablen anpassen) ---
import React, { useRef } from 'react'
const bgCanvasRef = useRef<HTMLCanvasElement>(null)
const fgCanvasRef = useRef<HTMLCanvasElement>(null)
const originalImgRef = useRef<HTMLImageElement>(null)
async function runRender(resp: any, overlayAlpha = 0.6, blurPx = 2) {
const bg = bgCanvasRef.current!
const fg = fgCanvasRef.current!
const img = originalImgRef.current!
await renderAll({
bgCanvas: bg,
fgCanvas: fg,
originalImg: img,
maskDataUrl: (maskProc ?? resp.fg_mask),
drawPatternFn: (ctx, w, h) => {
// TODO: Hier dein bestehender Pattern-Renderer verwenden
// drawChecker(ctx, w, h, tileW, tileH)
},
drawGroutFn: (ctx, w, h) => {
// TODO: Deine Grid-Linien nutzen, z.B. drawGridLines(ctx, resp.grid)
},
overlayAlpha,
blurPx
})
// Optional: Maske rot debuggen
// await drawMaskTint(fg, (maskProc ?? resp.fg_mask), 'rgba(255,0,0,0.25)')
}
// --------------------------------------------------------------------------- */
/** Debug helper to tint the current mask on top of a canvas without clearing it. */
async function tintMaskOnCanvas(
canvas: HTMLCanvasElement,
maskSource: string | HTMLCanvasElement
) {
const { width: w, height: h } = canvas;
if (w === 0 || h === 0) return;
const ctx = canvas.getContext('2d');
if (!ctx) return;
let maskCanvas: HTMLCanvasElement | null = null;
if (typeof maskSource === 'string') {
maskCanvas = await prepareMaskAlphaCanvas(maskSource, w, h);
} else {
maskCanvas = maskSource;
}
if (!maskCanvas) return;
const tmp = document.createElement('canvas');
tmp.width = w; tmp.height = h;
const tmpCtx = tmp.getContext('2d', { willReadFrequently: true })!;
tmpCtx.drawImage(maskCanvas, 0, 0, w, h);
const imgData = tmpCtx.getImageData(0, 0, w, h);
const data = imgData.data;
for (let i = 0; i < data.length; i += 4) {
data[i] = 0; // R
data[i + 1] = 255; // G
data[i + 2] = 0; // B
// Alpha stays untouched so we only tint where mask has coverage
}
tmpCtx.putImageData(imgData, 0, 0);
ctx.save();
ctx.globalAlpha = 0.25;
ctx.drawImage(tmp, 0, 0, w, h);
ctx.restore();
}
/** Zeichnet shadow/light Masken über ein Ziel-Canvas.
* shadow_mask: weiß + Alpha → ctx.globalCompositeOperation='multiply'
* light_mask : weiß + Alpha → ctx.globalCompositeOperation='screen'
*/
const clamp01 = (v: number) => Math.max(0, Math.min(1, v));
async function applyLightShadow(
target: HTMLCanvasElement,
shadowDataUrl?: string | null,
lightDataUrl?: string | null,
opts?: { shadowStrength?: number; lightStrength?: number }
){
const ctx = target.getContext('2d')!
const w = target.width, h = target.height
const load = (src:string)=>new Promise<HTMLImageElement>((res,rej)=>{ const i=new Image(); i.crossOrigin='anonymous'; i.onload=()=>res(i); i.onerror=rej; i.src=src })
const shadowAlpha = clamp01(opts?.shadowStrength ?? 1)
const lightAlpha = clamp01(opts?.lightStrength ?? 1)
if (shadowDataUrl && shadowAlpha > 0){
const si = await load(shadowDataUrl)
ctx.save()
ctx.globalCompositeOperation = 'multiply'
ctx.globalAlpha = shadowAlpha
ctx.drawImage(si, 0, 0, w, h)
ctx.restore()
}
if (lightDataUrl && lightAlpha > 0){
const li = await load(lightDataUrl)
ctx.save()
ctx.globalCompositeOperation = 'screen'
ctx.globalAlpha = lightAlpha
ctx.drawImage(li, 0, 0, w, h)
ctx.restore()
}
}
function applyTileAllowedMaskFromOccluder(
tileCanvas: HTMLCanvasElement,
occluderCanvas: HTMLCanvasElement,
featherPx = 0,
tilesMaskCanvas?: HTMLCanvasElement | null,
occluderInsetPx = 0,
) {
if (tileCanvas.width === 0 || tileCanvas.height === 0) return;
const w = tileCanvas.width;
const h = tileCanvas.height;
const inset = Math.max(0, Math.round(occluderInsetPx));
// Build a tile-allowed matte: (tilesMask or full frame) minus occluder.
const allowed = document.createElement('canvas');
allowed.width = w;
allowed.height = h;
const actx = allowed.getContext('2d');
if (!actx) return;
actx.clearRect(0, 0, w, h);
if (tilesMaskCanvas) {
actx.drawImage(tilesMaskCanvas, 0, 0, w, h);
} else {
actx.fillStyle = 'rgba(255,255,255,1)';
actx.fillRect(0, 0, w, h);
}
actx.globalCompositeOperation = 'destination-out';
let occluderForMatte: HTMLCanvasElement = occluderCanvas;
if (inset > 0) {
const occBinary = extractBinaryMask(occluderCanvas, false);
if (occBinary) {
const eroded = morphErode(occBinary, w, h, inset);
occluderForMatte = canvasFromMask(eroded, w, h);
}
}
actx.drawImage(occluderForMatte, 0, 0, w, h);
actx.globalCompositeOperation = 'source-over';
// Feather ONLY the allowed matte edges, then clip back to allowed to avoid any bleed into occluders.
let matte: HTMLCanvasElement = allowed;
const fp = Math.max(0, Math.round(featherPx));
if (fp > 0) {
const blurred = applyFeather(allowed, fp);
const clipped = document.createElement('canvas');
clipped.width = w;
clipped.height = h;
const cctx = clipped.getContext('2d')!;
cctx.drawImage(blurred, 0, 0, w, h);
cctx.globalCompositeOperation = 'destination-in';
cctx.drawImage(allowed, 0, 0, w, h);
cctx.globalCompositeOperation = 'source-over';
matte = clipped;
}
const tctx = tileCanvas.getContext('2d');
if (!tctx) return;
tctx.save();
tctx.globalCompositeOperation = 'destination-in';
tctx.drawImage(matte, 0, 0, w, h);
tctx.restore();
}
function buildCompositeOccluderCanvas(
base: HTMLCanvasElement,
erodePx: number,
): HTMLCanvasElement {
const inset = Math.max(0, Math.round(erodePx));
if (!inset) return base;
const bin = extractBinaryMask(base, false);
if (!bin) return base;
const w = base.width;
const h = base.height;
const eroded = morphErode(bin, w, h, inset);
const after = coverageOfMask(eroded);
if (after <= 0.005) return base;
const canvas = canvasFromMask(eroded, w, h);
canvas.dataset.maskCoverage = after.toFixed(3);
return canvas;
}