| Server IP : 217.154.3.148 / Your IP : 216.73.216.90 Web Server : Apache System : Linux blissful-wright.217-154-3-148.plesk.page 6.8.0-136-generic #136-Ubuntu SMP PREEMPT_DYNAMIC Wed Jul 1 21:53:05 UTC 2026 x86_64 User : gracious-boyd_hfhh1h8vo9n ( 10000) PHP Version : 8.3.31 Disable Function : opcache_get_status MySQL : OFF | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : /var/www/vhosts/gracious-boyd.217-154-3-148.plesk.page/nextjs/app/api/analyze/ |
Upload File : |
import { NextRequest, NextResponse } from "next/server";
export const runtime = "nodejs";
import sharp from "sharp";
import path from "node:path";
import crypto from "node:crypto";
import fs from "node:fs/promises";
import { spawn } from "node:child_process";
import { runPeriodicAnalysis } from "@/lib/analyzers";
import { refineGridRansac } from "@/lib/ransac_grid";
import { estimateDepth, type DepthMap } from "@/lib/depth/depth-anything";
import { refineOccluderMask, type RefinedOccluderResult } from "@/lib/segmentation/occluder";
import { getOccluderDefaults } from "@/lib/occluder/defaults";
import {
normalizeWithPreset,
resolvePresetName,
type MaskPresetName,
type NormalizeMaskResponse,
type NormalizeOptions,
} from "@/lib/mask/normalize";
import { runMaskPipeline } from "@/lib/masks/pipeline";
import type { GridContext, PipelineFlags } from "@/lib/masks/types";
import { thresholdMask } from "@/lib/masks/lattice";
import {
keepSeedDrivenComponents,
suppressTileLeak,
overlapWithSeeds,
connectedComponents,
distanceToNearestLattice,
} from "@/lib/masks/components";
import { morphDilate, morphErode, maskOr, maskAnd } from "@/utils/maskOps";
import type { TileSegmentationResult } from "@/lib/segmentation/tile-unet";
import { detectTileBand, detectTileBandFromTilesBinary, type TileBand } from "@/lib/segmentation/tile-mask";
// Lazy-load segmentation only when requested to avoid failing on hosts
// without onnxruntime CPU provider. We import inside the handler.
const OCCLUDER_ENV_ALLOWLIST = new Set<string>([
"OCCLUDER_METHOD",
"OCCLUDER_SAM_MAX_LEAK",
"OCCLUDER_SAM_LEAK_WEIGHT",
"OCCLUDER_SAM_BAND_RADIUS",
"OCCLUDER_SAM_NEG_ERODE",
"OCCLUDER_SAM_NEG_Q",
"OCCLUDER_SAM_POS_GRAD_Q",
"OCCLUDER_SAM_POS_BAND_ONLY",
"OCCLUDER_SAM_MIN_OVERLAP",
"OCCLUDER_SAM_MIN_OUTSIDE",
"OCCLUDER_SAM_BOX_MARGIN",
"OCCLUDER_SAM_MAX_TILE_RATIO",
"OCCLUDER_SAM_TOP_K",
"OCCLUDER_SAM_EDGE_WEIGHT",
"OCCLUDER_SAM_EDGE_RADIUS",
"OCCLUDER_SAM_DEPTH_CLIP",
"OCCLUDER_SAM_DEPTH_CLIP_DILATE",
"OCCLUDER_SAM_DEPTH_CLIP_MIN_FRAC",
"OCCLUDER_SAM_DEPTH_CLIP_MIN_COVERAGE",
"OCCLUDER_SAM_PATCH_FNS",
"OCCLUDER_SAM_PATCH_PY",
"OCCLUDER_SAM_PATCH_MODEL",
"OCCLUDER_SAM_PATCH_BOXES",
"OCCLUDER_SAM_PATCH_MARGIN",
"OCCLUDER_SAM_PATCH_DEVICE",
"OCCLUDER_SAM_PATCH_TIMEOUT_MS",
"OCCLUDER_SAM_PATCH_TILE_BAND_ONLY",
"OCCLUDER_SAM_PATCH_TILE_BAND_PAD",
"OCCLUDER_SAM_PATCH_TILE_DILATE",
"OCCLUDER_SAM_PATCH_MIN_FN_COVERAGE",
"OCCLUDER_SAM_PATCH_REQUIRE_TILE",
"OCCLUDER_SAM_PATCH_MIN_COMP",
"OCCLUDER_SAM_PATCH_MAX_COMP",
"OCCLUDER_SAM_PATCH_MAX_COMP_FRAC",
"OCCLUDER_SAM_PATCH_USE_BACK",
"OCCLUDER_SAM_PATCH_UNION_BACK",
"OCCLUDER_SAM_PATCH_GROW",
"OCCLUDER_SAM_PATCH_MAX_COVERAGE",
"OCCLUDER_SAM_PATCH_MAX_ADD_FRAC",
"OCCLUDER_SAM_PATCH_MAX_ADD_PIXELS",
"OCCLUDER_SAM_PATCH_MAX_ADD_PIXELS_7XM",
"OCCLUDER_SAM_PATCH_ADJ_PX",
"OCCLUDER_FN_ASSIST_UNION",
"OCCLUDER_FN_ASSIST_WITHOUT_SAM",
"OCCLUDER_FN_ASSIST_MIN_COMP",
"OCCLUDER_FN_ASSIST_MAX_COMP",
"OCCLUDER_FN_ASSIST_ADJ_PX",
"OCCLUDER_FN_ASSIST_TILE_ONLY",
"OCCLUDER_DISTILL_MIN_COVERAGE",
"OCCLUDER_DISTILL_MAX_COVERAGE",
"OCCLUDER_DISTILL_MAX_TRANS_DENSITY",
"OCCLUDER_DISTILL_MAX_TILE_LEAK",
"OCCLUDER_DISTILL_BOTTOM_STRIP_FRAC",
"OCCLUDER_DISTILL_MIN_COMP_FRAC",
"OCCLUDER_DISTILL_BIG_COMP_FRAC",
"OCCLUDER_DISTILL_MIN_PROXY",
"OCCLUDER_PATCH_REFINE",
"OCCLUDER_PATCH_TOP_N",
"OCCLUDER_PATCH_MODEL",
"OCCLUDER_PATCH_DEVICE",
"OCCLUDER_PIPELINE_FALLBACK_MODE",
]);
type OccluderEnvOverrides = Record<string, string>;
function parseOccluderEnvOverrides(raw: string | null): OccluderEnvOverrides | null {
if (!raw) return null;
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
return null;
}
if (!parsed || typeof parsed !== "object") return null;
const env: OccluderEnvOverrides = {};
for (const [key, value] of Object.entries(parsed as Record<string, unknown>)) {
if (!OCCLUDER_ENV_ALLOWLIST.has(key)) continue;
if (value == null) continue;
env[key] = String(value);
}
return Object.keys(env).length ? env : null;
}
function applyOccluderEnvOverrides(overrides: OccluderEnvOverrides | null): (() => void) | null {
if (!overrides) return null;
const entries = Object.entries(overrides);
if (!entries.length) return null;
const previous = new Map<string, string | undefined>();
for (const [key, value] of entries) {
previous.set(key, process.env[key]);
process.env[key] = String(value);
}
return () => {
for (const [key, value] of previous.entries()) {
if (value == null) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
};
}
function parsePositiveIntEnv(value: string | undefined | null): number | null {
const parsed = Number(value ?? "");
if (!Number.isFinite(parsed) || parsed <= 0) return null;
return Math.max(1, Math.floor(parsed));
}
function parseTimeoutMsEnv(
key: string,
fallbackMs: number,
minMs = 1_000,
maxMs = 10 * 60_000,
): number {
const raw = Number(process.env[key] ?? "");
if (!Number.isFinite(raw) || raw <= 0) return fallbackMs;
return Math.max(minMs, Math.min(maxMs, Math.round(raw)));
}
async function withStepTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
if (!(timeoutMs > 0)) return promise;
return await new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
promise.then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(err) => {
clearTimeout(timer);
reject(err);
},
);
});
}
function resolveSamPatchMaxAddPixelsCap(sampleName: string | null | undefined): {
cap: number;
scope: "none" | "global" | "sample_7xm";
} {
const normalized = (sampleName ?? "").toLowerCase();
if (normalized.startsWith("7xm")) {
const sampleCap = parsePositiveIntEnv(process.env.OCCLUDER_SAM_PATCH_MAX_ADD_PIXELS_7XM);
if (sampleCap != null) {
return { cap: sampleCap, scope: "sample_7xm" };
}
}
const globalCap = parsePositiveIntEnv(process.env.OCCLUDER_SAM_PATCH_MAX_ADD_PIXELS);
if (globalCap != null) {
return { cap: globalCap, scope: "global" };
}
return { cap: Number.POSITIVE_INFINITY, scope: "none" };
}
function isComfyProfileUpload(name: string | null | undefined): boolean {
const normalized = (name ?? "").trim().toLowerCase();
return normalized === "comfyui_01234_.png";
}
function resolveOccluderPresetBackend(): string {
const value = (process.env.OCCLUDER_BACKEND ?? process.env.OCCLUDER_PRESET ?? "").trim().toLowerCase();
if (value === "rfdet_m3" || value === "comfy_rfdet_m3") return "rfdet_m3";
return "";
}
/** ---------- Utils ---------- */
function clamp(v:number, lo:number, hi:number){ return Math.max(lo, Math.min(hi, v)); }
function toGrayFloat32(rgba: Uint8Array, w: number, h: number): Float32Array {
const out = new Float32Array(w * h);
for (let i = 0, j = 0; i < rgba.length; i += 4, j++) {
out[j] = 0.299 * rgba[i] + 0.587 * rgba[i + 1] + 0.114 * rgba[i + 2];
}
return out;
}
function sobelMag(gray: Float32Array, w: number, h: number): Float32Array {
const out = new Float32Array(w * h);
const gxk = [-1,0,1,-2,0,2,-1,0,1];
const gyk = [-1,-2,-1,0,0,0,1,2,1];
for (let y = 1; y < h - 1; y++) {
for (let x = 1; x < w - 1; x++) {
let gx = 0, gy = 0, k = 0;
for (let dy = -1; dy <= 1; dy++) {
for (let dx = -1; dx <= 1; dx++, k++) {
const v = gray[(y + dy) * w + (x + dx)];
gx += gxk[k] * v;
gy += gyk[k] * v;
}
}
out[y * w + x] = Math.hypot(gx, gy);
}
}
return out;
}
function projectXWeighted(mag: Float32Array, weights: Float32Array|null, w: number, h: number): Float32Array {
const s = new Float32Array(w);
if (weights) {
for (let x = 0; x < w; x++) {
let acc = 0;
for (let y = 0; y < h; y++) acc += mag[y*w + x] * weights[y*w + x];
s[x] = acc;
}
} else {
for (let x = 0; x < w; x++) {
let acc = 0;
for (let y = 0; y < h; y++) acc += mag[y*w + x];
s[x] = acc;
}
}
return s;
}
function projectYWeighted(mag: Float32Array, weights: Float32Array|null, w: number, h: number): Float32Array {
const s = new Float32Array(h);
if (weights) {
for (let y = 0; y < h; y++) {
let acc = 0;
for (let x = 0; x < w; x++) acc += mag[y*w + x] * weights[y*w + x];
s[y] = acc;
}
} else {
for (let y = 0; y < h; y++) {
let acc = 0;
for (let x = 0; x < w; x++) acc += mag[y*w + x];
s[y] = acc;
}
}
return s;
}
function autoCorr1D(sig: Float32Array): Float32Array {
const n = sig.length;
const out = new Float32Array(n);
const mean = sig.reduce((a,b)=>a+b,0)/n;
for (let i=0;i<n;i++) sig[i] = sig[i] - mean;
for (let lag=1; lag<n; lag++){
let s=0;
for (let i=0; i+lag<n; i++) s += sig[i]*sig[i+lag];
out[lag] = s;
}
out[0]=0;
return out;
}
function buildTilePlaneMask(tile: TileSegmentationResult | null): Uint8Array | null {
if (!tile || !tile.ids || tile.ids.length === 0) return null;
const classes = tile.classes ?? [];
const byName = (name: string) =>
classes.find((cls) => String(cls.name).toLowerCase() === name.toLowerCase()) ?? null;
const tilesId = byName("tiles")?.id ?? (classes.length > 1 ? 1 : 0);
const groutId = byName("grout")?.id ?? null;
const out = new Uint8Array(tile.ids.length);
for (let i = 0; i < tile.ids.length; i++) {
const cls = tile.ids[i];
if (cls === tilesId || (groutId != null && cls === groutId)) {
out[i] = 255;
}
}
return out;
}
type BandSelection = {
band: TileBand;
source: string;
};
function applyBandToMask(
mask: Uint8Array,
width: number,
height: number,
yMin: number,
yMax: number,
): Uint8Array {
const out = new Uint8Array(mask);
for (let y = 0; y < height; y++) {
if (y >= yMin && y < yMax) continue;
out.fill(0, y * width, (y + 1) * width);
}
return out;
}
function normalizeBandRange(
yMin: number,
yMax: number,
height: number,
minHeight: number,
maxHeight: number,
): { yMin: number; yMax: number; adjusted: boolean } | null {
if (height <= 0) return null;
let y0 = Math.max(0, Math.min(height, Math.floor(yMin)));
let y1 = Math.max(0, Math.min(height, Math.floor(yMax)));
if (y1 <= y0) return null;
let adjusted = false;
let bandHeight = y1 - y0;
if (bandHeight < minHeight) {
const center = Math.round((y0 + y1) / 2);
y0 = Math.max(0, Math.min(height - minHeight, center - Math.floor(minHeight / 2)));
y1 = Math.min(height, y0 + minHeight);
adjusted = true;
bandHeight = y1 - y0;
}
if (bandHeight > maxHeight) {
const center = Math.round((y0 + y1) / 2);
y0 = Math.max(0, Math.min(height - maxHeight, center - Math.floor(maxHeight / 2)));
y1 = Math.min(height, y0 + maxHeight);
adjusted = true;
}
return { yMin: y0, yMax: y1, adjusted };
}
function bandFromNonZeroRows(
mask: Uint8Array,
width: number,
height: number,
): { yMin: number; yMax: number } | null {
if (!(width > 0 && height > 0) || mask.length !== width * height) return null;
let yMin = -1;
let yMax = -1;
for (let y = 0; y < height; y++) {
const row = y * width;
let hit = false;
for (let x = 0; x < width; x++) {
if (mask[row + x]) {
hit = true;
break;
}
}
if (hit) {
if (yMin < 0) yMin = y;
yMax = y;
}
}
if (yMin < 0 || yMax < yMin) return null;
return { yMin, yMax: yMax + 1 };
}
function pickTileBandFromSegmentation(
tile: TileSegmentationResult | null,
promptMask: Uint8Array | null,
width: number,
height: number,
): BandSelection | null {
if (!tile || !tile.ids || !(width > 0 && height > 0)) return null;
const classes = tile.classes ?? [];
const byName = (name: string) =>
classes.find((cls) => String(cls.name).toLowerCase() === name.toLowerCase()) ?? null;
const tilesId = byName("tiles")?.id ?? (classes.length > 1 ? 1 : 0);
const groutId = byName("grout")?.id ?? null;
if (groutId != null) {
const band = detectTileBand(tile.ids, width, height, {
minRowGrout: 0.002,
minRowTilesGrout: 0.01,
smoothWindow: 9,
excludeBottomFrac: 0.15,
groutId,
tilesId,
});
if (band) return { band, source: "tile_band_grout" };
}
if (promptMask) {
const band = detectTileBandFromTilesBinary(promptMask, null, width, height, {
smoothWindow: 11,
excludeBottomFrac: 0.15,
minStartFrac: 0.08,
minRowFracMin: 0.08,
maxRowFracToThreshFrac: 0.5,
peakThresholdFrac: 0.65,
minBandFrac: 0.12,
});
if (band) return { band, source: "tile_band_binary" };
const relaxed = detectTileBandFromTilesBinary(promptMask, null, width, height, {
smoothWindow: 9,
excludeBottomFrac: 0.15,
minStartFrac: 0.06,
minRowFracMin: 0.06,
maxRowFracToThreshFrac: 0.35,
peakThresholdFrac: 0.55,
minBandFrac: 0.1,
});
if (relaxed) return { band: relaxed, source: "tile_band_binary_relaxed" };
}
return null;
}
function bandMaskByRowDensity(
mask: Uint8Array,
width: number,
height: number,
): { mask: Uint8Array; applied: boolean; band: { yMin: number; yMax: number; threshold: number; p80: number; fallback?: boolean; } | null } {
if (width <= 0 || height <= 0 || mask.length !== width * height) {
return { mask, applied: false, band: null };
}
const rowCoverage = new Float32Array(height);
for (let y = 0; y < height; y++) {
const row = y * width;
let count = 0;
for (let x = 0; x < width; x++) {
if (mask[row + x]) count += 1;
}
rowCoverage[y] = count / Math.max(1, width);
}
const sorted = Array.from(rowCoverage).sort((a, b) => a - b);
const p80 = sorted[Math.floor(sorted.length * 0.8)] ?? 0;
const p60 = sorted[Math.floor(sorted.length * 0.6)] ?? 0;
const p50 = sorted[Math.floor(sorted.length * 0.5)] ?? 0;
const minFracEnv = Number(process.env.OCCLUDER_SAM_BAND_MIN_FRAC ?? "0.12");
const baseThreshold = Math.max(0.02, Math.min(0.6, Math.max(minFracEnv, p80 * 0.5)));
const relaxedThreshold = Math.max(0.01, Math.min(baseThreshold, Math.max(p60 * 0.35, p50 * 0.45)));
const relaxedThreshold2 = Math.max(0.005, Math.min(relaxedThreshold, p80 * 0.25));
const thresholdCandidates = [baseThreshold, relaxedThreshold, relaxedThreshold2]
.filter((value, index, arr) => value > 0 && arr.indexOf(value) === index);
let bestStart = -1;
let bestEnd = -1;
let thresholdUsed = baseThreshold;
const findLongestRun = (threshold: number) => {
let curStart = -1;
let runStart = -1;
let runEnd = -1;
for (let y = 0; y < height; y++) {
const keep = rowCoverage[y] >= threshold;
if (keep && curStart < 0) curStart = y;
if (!keep && curStart >= 0) {
const end = y - 1;
if (end - curStart > runEnd - runStart) {
runStart = curStart;
runEnd = end;
}
curStart = -1;
}
}
if (curStart >= 0) {
const end = height - 1;
if (end - curStart > runEnd - runStart) {
runStart = curStart;
runEnd = end;
}
}
if (runStart < 0 || runEnd < runStart) return null;
return { start: runStart, end: runEnd };
};
for (const candidate of thresholdCandidates) {
const run = findLongestRun(candidate);
if (run) {
bestStart = run.start;
bestEnd = run.end;
thresholdUsed = candidate;
break;
}
}
const minHeightFrac = Number(process.env.OCCLUDER_SAM_BAND_MIN_HEIGHT_FRAC ?? "0.04");
const minHeight = Math.max(4, Math.round(height * minHeightFrac));
const maxHeightFrac = Number(process.env.OCCLUDER_SAM_BAND_MAX_HEIGHT_FRAC ?? "0.6");
const maxHeight = Math.max(minHeight, Math.round(height * maxHeightFrac));
let fallbackUsed = false;
if (bestStart >= 0 && bestEnd >= bestStart) {
const bandHeight = bestEnd - bestStart + 1;
if (bandHeight < minHeight) {
const center = Math.round((bestStart + bestEnd) / 2);
const half = Math.floor(minHeight / 2);
bestStart = Math.max(0, Math.min(height - minHeight, center - half));
bestEnd = Math.min(height - 1, bestStart + minHeight - 1);
fallbackUsed = true;
}
}
if (bestStart < 0 || bestEnd < bestStart) {
const minOverall = Number(process.env.OCCLUDER_SAM_BAND_MIN_OVERALL ?? "0.01");
const overallCoverage = coverageOfMask(mask);
if (overallCoverage >= minOverall) {
let weightSum = 0;
let weightedY = 0;
for (let y = 0; y < height; y++) {
const weight = rowCoverage[y];
if (weight <= 0) continue;
weightSum += weight;
weightedY += weight * y;
}
const center = weightSum > 0 ? Math.round(weightedY / weightSum) : Math.floor(height / 2);
const targetHeightFrac = clamp(
Number(process.env.OCCLUDER_SAM_BAND_TARGET_HEIGHT_FRAC ?? "0.25"),
minHeightFrac,
maxHeightFrac,
);
const targetHeight = Math.round(height * targetHeightFrac);
const bandHeight = Math.max(minHeight, Math.min(maxHeight, targetHeight));
bestStart = Math.max(0, Math.min(height - bandHeight, center - Math.floor(bandHeight / 2)));
bestEnd = Math.min(height - 1, bestStart + bandHeight - 1);
fallbackUsed = true;
thresholdUsed = relaxedThreshold2;
}
}
if (bestStart < 0 || bestEnd < bestStart) {
return { mask, applied: false, band: null };
}
const expand = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_BAND_EXPAND ?? "6")));
let yMin = Math.max(0, bestStart - expand);
let yMax = Math.min(height - 1, bestEnd + expand);
const bandHeight = yMax - yMin + 1;
if (bandHeight > maxHeight) {
const center = Math.round((yMin + yMax) / 2);
yMin = Math.max(0, Math.min(height - maxHeight, center - Math.floor(maxHeight / 2)));
yMax = Math.min(height - 1, yMin + maxHeight - 1);
fallbackUsed = true;
}
const out = new Uint8Array(mask);
for (let y = 0; y < height; y++) {
if (y >= yMin && y <= yMax) continue;
out.fill(0, y * width, (y + 1) * width);
}
return {
mask: out,
applied: true,
band: {
yMin,
yMax: yMax + 1,
threshold: Number(thresholdUsed.toFixed(3)),
p80: Number(p80.toFixed(3)),
fallback: fallbackUsed,
},
};
}
function bestPeriod(ac: Float32Array, minP: number, maxP: number): number {
minP = Math.max(3, minP|0);
maxP = Math.max(minP+1, maxP|0);
let best = 0, bestVal = -Infinity;
for (let p=minP; p<=maxP && p<ac.length; p++) {
const v = ac[p];
if (v > bestVal) { bestVal = v; best = p; }
}
return best;
}
function phaseForPeriod(proj: Float32Array, period: number): number {
if (period <= 0) return 0;
let bestOff=0, bestScore=-Infinity;
for (let off=0; off<period; off++){
let score=0;
for (let i=off; i<proj.length; i+=period) score += proj[i];
if (score>bestScore){bestScore=score; bestOff=off;}
}
return bestOff;
}
function componentTouchesMask(
comp: { pixels: number[] },
mask: Uint8Array | null | undefined,
): boolean {
if (!mask) return false;
for (const idx of comp.pixels) {
if (mask[idx]) return true;
}
return false;
}
function componentOverlapFraction(
comp: { pixels: number[]; area: number },
mask: Uint8Array | null | undefined,
): number {
if (!mask || comp.area <= 0) return 0;
let hits = 0;
for (const idx of comp.pixels) {
if (mask[idx]) hits += 1;
}
return hits / comp.area;
}
function componentExtentX(comp: { minX: number; maxX: number }): number {
return comp.maxX - comp.minX + 1;
}
function componentExtentY(comp: { minY: number; maxY: number }): number {
return comp.maxY - comp.minY + 1;
}
function componentAspectRatio(comp: { minX: number; maxX: number; minY: number; maxY: number }): number {
const extX = componentExtentX(comp);
const extY = componentExtentY(comp);
return extX / Math.max(1, extY);
}
function componentVarianceY(
comp: { pixels: number[]; area: number; minY: number; maxY: number },
width: number,
): number {
if (comp.area <= 0 || width <= 0) return 0;
let sum = 0;
let sumSq = 0;
for (const idx of comp.pixels) {
const y = Math.floor(idx / width);
sum += y;
sumSq += y * y;
}
const mean = sum / comp.area;
const variance = Math.max(0, sumSq / comp.area - mean * mean);
const extent = Math.max(1, componentExtentY(comp));
const norm = variance / (extent * extent);
return Number.isFinite(norm) ? norm : 0;
}
function coverageOfMask(mask: Uint8Array | null | undefined): number {
if (!mask || !mask.length) return 0;
let filled = 0;
for (let i = 0; i < mask.length; i++) {
if (mask[i]) filled += 1;
}
return filled / mask.length;
}
function coverageWithinMask(mask: Uint8Array | null | undefined, region: Uint8Array | null | undefined): number {
if (!mask || !region) return 0;
const total = Math.min(mask.length, region.length);
let interior = 0;
let overlap = 0;
for (let i = 0; i < total; i++) {
if (!region[i]) continue;
interior += 1;
if (mask[i]) overlap += 1;
}
return interior > 0 ? overlap / interior : 0;
}
function sha1Hex(data: Uint8Array | Buffer | string): string {
const hash = crypto.createHash("sha1");
hash.update(data as any);
return hash.digest("hex");
}
function maskSha1(mask: Uint8Array, width: number, height: number): string {
const header = Buffer.from(`${width}x${height}\0`, "utf8");
const hash = crypto.createHash("sha1");
hash.update(header);
hash.update(Buffer.from(mask));
return hash.digest("hex");
}
function meanBinaryTransitionsPerRow(mask: Uint8Array, width: number, height: number): number {
if (!(width > 1 && height > 0) || mask.length < width * height) return 0;
let transitions = 0;
for (let y = 0; y < height; y++) {
const row = y * width;
let prev = mask[row] > 127;
for (let x = 1; x < width; x++) {
const v = mask[row + x] > 127;
if (v !== prev) transitions += 1;
prev = v;
}
}
return transitions / height;
}
function meanBinaryTransitionsPerCol(mask: Uint8Array, width: number, height: number): number {
if (!(width > 0 && height > 1) || mask.length < width * height) return 0;
let transitions = 0;
for (let x = 0; x < width; x++) {
let prev = mask[x] > 127;
for (let y = 1; y < height; y++) {
const v = mask[y * width + x] > 127;
if (v !== prev) transitions += 1;
prev = v;
}
}
return transitions / width;
}
function looksLikeGridBinaryMask(mask: Uint8Array, width: number, height: number): boolean {
const rowTransitions = meanBinaryTransitionsPerRow(mask, width, height);
const colTransitions = meanBinaryTransitionsPerCol(mask, width, height);
// Heuristic thresholds:
// - Real kitchen occluders are usually a few large blobs => low transition density.
// - Regular tile/checker patterns produce high-frequency transitions in both axes.
const rowThresh = Math.max(180, Math.round(width * 0.12));
const colThresh = Math.max(180, Math.round(height * 0.12));
return rowTransitions > rowThresh && colTransitions > colThresh;
}
async function dilateBinaryMaskViaBlur(
mask: Uint8Array,
width: number,
height: number,
radius: number,
threshold = 8,
): Promise<Uint8Array> {
const r = Math.max(0, Math.round(radius));
if (!r) return Uint8Array.from(mask);
const t = Math.max(1, Math.min(254, Math.round(threshold)));
const out = await sharp(Buffer.from(mask), { raw: { width, height, channels: 1 } })
.blur(r)
.threshold(t)
.raw()
.toBuffer();
return new Uint8Array(out);
}
type BinaryMask = { data: Uint8Array; width: number; height: number };
type DepthByteMap = { data: Uint8Array; width: number; height: number; min: number; max: number };
function normalizeDepthMap(depth: DepthMap): DepthMap {
const { data, width, height } = depth;
let min = Infinity;
let max = -Infinity;
for (const v of data) {
if (!Number.isFinite(v)) continue;
if (v < min) min = v;
if (v > max) max = v;
}
if (!Number.isFinite(min) || !Number.isFinite(max)) {
return { data: new Float32Array(data.length), width, height };
}
const range = max - min || 1;
const norm = new Float32Array(data.length);
for (let i = 0; i < data.length; i++) {
const v = Number.isFinite(data[i]) ? data[i] : min;
norm[i] = (v - min) / range;
}
return { data: norm, width, height };
}
function normalizeDepthFloatToByte(depth: DepthMap): DepthByteMap {
const { data, width, height } = depth;
let min = Infinity;
let max = -Infinity;
for (const v of data) {
if (!Number.isFinite(v)) continue;
min = Math.min(min, v);
max = Math.max(max, v);
}
const range = max - min || 1;
const out = new Uint8Array(width * height);
for (let i = 0; i < data.length; i++) {
const v = Number.isFinite(data[i]) ? data[i] : min;
out[i] = Math.max(0, Math.min(255, Math.round(((v - min) / range) * 255)));
}
return { data: out, width, height, min, max };
}
function cropDepthToAspect(
map: DepthByteMap,
targetW: number,
targetH: number,
): { map: DepthByteMap; crop: { x: number; y: number; width: number; height: number } | null } {
if (!(targetW > 0 && targetH > 0 && map.width > 0 && map.height > 0)) {
return { map, crop: null };
}
const targetRatio = targetW / targetH;
const currentRatio = map.width / map.height;
if (!Number.isFinite(targetRatio) || !Number.isFinite(currentRatio)) {
return { map, crop: null };
}
if (Math.abs(currentRatio - targetRatio) < 0.02) {
return { map, crop: null };
}
let cropW = map.width;
let cropH = map.height;
let cropX = 0;
let cropY = 0;
if (currentRatio > targetRatio) {
cropW = Math.max(1, Math.round(map.height * targetRatio));
cropX = Math.max(0, Math.floor((map.width - cropW) / 2));
} else {
cropH = Math.max(1, Math.round(map.width / targetRatio));
cropY = Math.max(0, Math.floor((map.height - cropH) / 2));
}
if (cropW === map.width && cropH === map.height) {
return { map, crop: null };
}
const cropped = new Uint8Array(cropW * cropH);
for (let y = 0; y < cropH; y++) {
const srcRow = (y + cropY) * map.width + cropX;
const dstRow = y * cropW;
cropped.set(map.data.subarray(srcRow, srcRow + cropW), dstRow);
}
return {
map: { data: cropped, width: cropW, height: cropH, min: map.min, max: map.max },
crop: { x: cropX, y: cropY, width: cropW, height: cropH },
};
}
function invertDepthBytes(map: DepthByteMap): DepthByteMap {
const out = new Uint8Array(map.data.length);
for (let i = 0; i < map.data.length; i++) out[i] = 255 - map.data[i];
return { data: out, width: map.width, height: map.height, min: 255 - map.max, max: 255 - map.min };
}
function histogram256(data: Uint8Array): Uint32Array {
const hist = new Uint32Array(256);
for (let i = 0; i < data.length; i++) hist[data[i]] += 1;
return hist;
}
function computeWallDepthFromHist(hist: Uint32Array): number {
// far range peak (0..100 bucket range) -> backsplash plane
const limit = Math.min(100, hist.length);
let peakIdx = 0;
let peakVal = -1;
for (let i = 0; i < limit; i++) {
const v = hist[i];
if (v > peakVal) {
peakVal = v;
peakIdx = i;
}
}
return peakIdx;
}
function buildDepthOccluderFromBytes(depth: DepthByteMap, cutoff: number): BinaryMask {
const { data, width, height } = depth;
const mask = new Uint8Array(data.length);
for (let i = 0; i < data.length; i++) {
mask[i] = data[i] > cutoff ? 255 : 0;
}
return { data: mask, width, height };
}
function buildDepthBackMaskFromBytes(depth: DepthByteMap, cutoff: number): BinaryMask {
const { data, width, height } = depth;
const mask = new Uint8Array(data.length);
for (let i = 0; i < data.length; i++) {
mask[i] = data[i] <= cutoff ? 255 : 0;
}
return { data: mask, width, height };
}
function morphErode(mask: Uint8Array, width: number, height: number, radius = 1): Uint8Array {
if (radius <= 0) return Uint8Array.from(mask);
const out = new Uint8Array(mask.length);
const r = Math.max(1, Math.floor(radius));
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let keep = 255;
for (let dy = -r; dy <= r && keep; dy++) {
const yy = y + dy;
if (yy < 0 || yy >= height) { keep = 0; break; }
const row = yy * width;
for (let dx = -r; dx <= r; dx++) {
const xx = x + dx;
if (xx < 0 || xx >= width) { keep = 0; break; }
if (mask[row + xx] === 0) { keep = 0; break; }
}
}
out[y * width + x] = keep ? 255 : 0;
}
}
return out;
}
function morphDilate(mask: Uint8Array, width: number, height: number, radius = 1): Uint8Array {
if (radius <= 0) return Uint8Array.from(mask);
const out = new Uint8Array(mask.length);
const r = Math.max(1, Math.floor(radius));
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let fill = 0;
for (let dy = -r; dy <= r && !fill; dy++) {
const yy = y + dy;
if (yy < 0 || yy >= height) continue;
const row = yy * width;
for (let dx = -r; dx <= r; dx++) {
const xx = x + dx;
if (xx < 0 || xx >= width) continue;
if (mask[row + xx]) { fill = 255; break; }
}
}
out[y * width + x] = fill;
}
}
return out;
}
function morphOpen(mask: Uint8Array, width: number, height: number, erodeR = 1, dilateR = 2): Uint8Array {
const eroded = morphErode(mask, width, height, erodeR);
return morphDilate(eroded, width, height, dilateR);
}
async function blurMask(mask: Uint8Array, width: number, height: number, sigma = 2): Promise<Uint8Array> {
if (sigma <= 0) return Uint8Array.from(mask);
const blurred = await sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.blur(sigma)
.toColorspace('b-w')
.raw()
.toBuffer();
return new Uint8Array(blurred);
}
function morphClose(mask: Uint8Array, width: number, height: number, radius = 3): Uint8Array {
const dilated = morphDilate(mask, width, height, radius);
return morphErode(dilated, width, height, radius);
}
function removeSmallComponents(mask: Uint8Array, width: number, height: number, minSize = 1000): Uint8Array {
const cleaned = Uint8Array.from(mask);
const components = connectedComponents(cleaned, width, height, 1);
const total = width * height;
const minPixels = Math.max(minSize, Math.floor(0.01 * total));
for (const comp of components) {
if (comp.area < minPixels) {
for (const idx of comp.pixels) cleaned[idx] = 0;
}
}
return cleaned;
}
async function smoothDepthEdges(map: DepthByteMap): Promise<DepthByteMap> {
try {
const smoothed = await sharp(Buffer.from(map.data), {
raw: { width: map.width, height: map.height, channels: 1 },
failOnError: false,
})
.median(3)
.blur(1)
.toColorspace('b-w')
.raw()
.toBuffer();
return { data: new Uint8Array(smoothed), width: map.width, height: map.height, min: map.min, max: map.max };
} catch {
return map;
}
}
function estimateWallDepth(normDepth: DepthMap): number {
const { data, width, height } = normDepth;
const x0 = Math.floor(width * 0.3);
const x1 = Math.floor(width * 0.7);
const samples: number[] = [];
for (let y = 0; y < height; y++) {
const row = y * width;
for (let x = x0; x < x1; x++) {
const v = data[row + x];
if (Number.isFinite(v)) samples.push(v);
}
}
samples.sort((a, b) => a - b);
const idx = Math.floor(samples.length * 0.8);
return samples[idx] ?? 0.5;
}
function buildDepthOccluderMask(normDepth: DepthMap, wallDepth: number, margin = 0.05): BinaryMask {
const { data, width, height } = normDepth;
const threshold = wallDepth - margin;
const mask = new Uint8Array(data.length);
for (let i = 0; i < data.length; i++) {
const v = data[i];
mask[i] = Number.isFinite(v) && v < threshold ? 255 : 0;
}
return { data: mask, width, height };
}
function buildDepthBackMask(normDepth: DepthMap, wallDepth: number, margin = 0.03): BinaryMask {
const { data, width, height } = normDepth;
const threshold = wallDepth - margin;
const mask = new Uint8Array(data.length);
for (let i = 0; i < data.length; i++) {
const v = data[i];
mask[i] = Number.isFinite(v) && v >= threshold ? 255 : 0;
}
return { data: mask, width, height };
}
async function resizeBinaryMask(mask: BinaryMask, width: number, height: number): Promise<BinaryMask> {
if (mask.width === width && mask.height === height) return mask;
const resized = await sharp(Buffer.from(mask.data), {
raw: { width: mask.width, height: mask.height, channels: 1 },
})
.resize(width, height, { kernel: "lanczos3" })
.toColorspace("b-w")
.raw()
.toBuffer();
return { data: new Uint8Array(resized), width, height };
}
function mergeOccluderMasks(base: BinaryMask, extra: BinaryMask): BinaryMask {
const merged = Uint8Array.from(base.data);
const usable = Math.min(merged.length, extra.data.length);
for (let i = 0; i < usable; i++) {
if (extra.data[i]) merged[i] = 255;
}
return { data: merged, width: base.width, height: base.height };
}
function countClassIds(ids: Uint8Array): Record<string, number> {
const counts: Record<string, number> = { "0": 0, "1": 0, "2": 0, "255": 0 };
for (let i = 0; i < ids.length; i++) {
const v = ids[i];
if (v === 0) counts["0"] += 1;
else if (v === 1) counts["1"] += 1;
else if (v === 2) counts["2"] += 1;
else if (v === 255) counts["255"] += 1;
}
return counts;
}
function buildYoloBoostMask(
base: Uint8Array,
probMap: Uint8Array | null,
width: number,
height: number,
meta: Record<string, unknown> | null,
): { mask: Uint8Array; coverage: number; applied: number } | null {
if (!meta) return null;
const yolo = (meta as any)?.yolo;
const instances = Array.isArray(yolo?.instances) ? yolo.instances as Array<any> : null;
if (!instances || instances.length === 0) return null;
const total = width * height;
const boxMask = new Uint8Array(total);
let applied = 0;
const marginBase = Math.max(3, Math.round(Math.max(width, height) * 0.004));
for (const entry of instances) {
const boxRaw = Array.isArray(entry?.bbox)
? entry.bbox
: Array.isArray(entry?.box)
? entry.box
: null;
if (!boxRaw || boxRaw.length < 4) continue;
const coords = boxRaw.map((value: unknown) => Number(value)).filter((value) => Number.isFinite(value));
if (coords.length < 4) continue;
const [bx1, by1, bx2, by2] = coords as number[];
const minX = clamp(Math.floor(Math.min(bx1, bx2)) - marginBase, 0, width - 1);
const maxX = clamp(Math.ceil(Math.max(bx1, bx2)) + marginBase, 0, width - 1);
const minY = clamp(Math.floor(Math.min(by1, by2)) - marginBase, 0, height - 1);
const maxY = clamp(Math.ceil(Math.max(by1, by2)) + marginBase, 0, height - 1);
if (maxX <= minX || maxY <= minY) continue;
applied += 1;
for (let y = minY; y <= maxY; y++) {
const row = y * width;
for (let x = minX; x <= maxX; x++) {
boxMask[row + x] = 255;
}
}
}
if (applied === 0) return null;
let support = boxMask;
if (probMap && probMap.length === total) {
const supportProb = thresholdMask(probMap, 112);
if (supportProb) {
support = maskAnd(support, supportProb);
}
}
support = morphDilate(support, width, height, 1);
const boosted = maskOr(base, support);
if (!boosted) return null;
const coverage = coverageOfMask(boosted);
return { mask: boosted, coverage, applied };
}
type GridMasks = {
interiorMask: Uint8Array;
lineMask: Uint8Array;
};
type FinalMaskMeta = {
polarity: 'white' | 'black';
coverage?: number;
preset?: MaskPresetName;
source?: string;
stats?: Record<string, unknown> | null;
};
type TileMaskMeta = {
classes: string[];
colorMap: Record<string, [number, number, number]>;
coverage?: Record<string, number>;
coverageAbsolute?: Record<string, number>;
rawCoverage?: Record<string, number>;
occluderCoverage?: number;
occludedPixels?: number;
occluderPixels?: number;
occludedCoverage?: number;
warnings?: string[];
classIdLegend?: Record<string, string>;
classIdLegendRaw?: Record<string, string>;
classPixelCountsRaw?: Record<string, number>;
classPixelCounts?: Record<string, number>;
unetDebug?: unknown;
tileBand?: { yMin: number; yMax: number; confidence: number } | null;
model?: string | null;
source?: string;
};
type TileMaskVersions = {
combined?: string | null;
raw?: string | null;
occluder?: string | null;
};
function buildGridMasks(width: number, height: number, vlines: number[], hlines: number[], bandPx: number): GridMasks {
const total = width * height;
const interior = new Uint8Array(total);
const lineMask = new Uint8Array(total);
if (!width || !height || (!vlines.length && !hlines.length) || bandPx <= 0) {
interior.fill(255);
return { interiorMask: interior, lineMask };
}
const band = Math.max(1, Math.round(bandPx));
const colFlags = new Uint8Array(width);
const rowFlags = new Uint8Array(height);
for (const raw of vlines) {
const v = Math.max(0, Math.min(width - 1, Math.round(raw)));
const start = Math.max(0, v - band);
const end = Math.min(width - 1, v + band);
for (let x = start; x <= end; x++) colFlags[x] = 1;
}
for (const raw of hlines) {
const v = Math.max(0, Math.min(height - 1, Math.round(raw)));
const start = Math.max(0, v - band);
const end = Math.min(height - 1, v + band);
for (let y = start; y <= end; y++) rowFlags[y] = 1;
}
interior.fill(255);
for (let y = 0; y < height; y++) {
const row = y * width;
const rowFlag = rowFlags[y] === 1;
for (let x = 0; x < width; x++) {
if (rowFlag || colFlags[x] === 1) {
interior[row + x] = 0;
lineMask[row + x] = 255;
}
}
}
return { interiorMask: interior, lineMask };
}
function leakRatio(mask: Uint8Array, lineMask: Uint8Array): number {
let onLine = 0;
let total = 0;
for (let i = 0; i < Math.min(mask.length, lineMask.length); i++) {
if (!lineMask[i]) continue;
total += 1;
if (mask[i]) onLine += 1;
}
if (!total) return 0;
return onLine / total;
}
type MaskBoostStats = {
changed: boolean;
coverage: number;
gamma: number;
lo: number;
hi: number;
};
type RawMaskDiagnostics = {
coverageArray: number;
coveragePng: number;
mismatch: number;
suspicious: boolean;
};
function coverageThreshold(mask: Uint8Array, threshold = 128): number {
const total = mask.length;
if (!total) return 0;
let on = 0;
for (let i = 0; i < total; i++) {
if (mask[i] >= threshold) on += 1;
}
return on / total;
}
async function maskToDataUrlWithCheck(
mask: Uint8Array,
width: number,
height: number,
): Promise<{ dataUrl: string; diagnostics: RawMaskDiagnostics }>
{
const pngBuffer = await sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.toColorspace('b-w')
.png()
.toBuffer();
const dataUrl = `data:image/png;base64,${pngBuffer.toString('base64')}`;
const arrayCoverage = coverageThreshold(mask);
const decoded = await sharp(pngBuffer)
.toColorspace('b-w')
.raw()
.toBuffer();
const pngCoverage = coverageThreshold(new Uint8Array(decoded));
const mismatch = Math.abs(pngCoverage - arrayCoverage);
return {
dataUrl,
diagnostics: {
coverageArray: Number(arrayCoverage.toFixed(6)),
coveragePng: Number(pngCoverage.toFixed(6)),
mismatch: Number(mismatch.toFixed(6)),
suspicious: mismatch > 0.02,
},
};
}
function boostSegMaskInPlace(mask: Uint8Array): MaskBoostStats {
const total = mask.length;
if (total === 0) return { changed: false, coverage: 0, gamma: 1, lo: 0, hi: 0 };
const hist = new Uint32Array(256);
let nonZero = 0;
for (let i = 0; i < total; i++) {
const v = mask[i];
hist[v] += 1;
if (v > 0) nonZero += 1;
}
if (nonZero < 32) {
return { changed: false, coverage: nonZero / total, gamma: 1, lo: 0, hi: 0 };
}
const coverage = nonZero / total;
if (coverage > 0.6) {
return { changed: false, coverage, gamma: 1, lo: 0, hi: 0 };
}
const percentile = (q: number): number => {
if (!Number.isFinite(q)) return 0;
const target = Math.max(0, Math.min(nonZero - 1, Math.round(q * (nonZero - 1))));
let acc = 0;
for (let value = 1; value < hist.length; value++) {
const count = hist[value];
if (count === 0) continue;
acc += count;
if (acc > target) return value;
}
return 255;
};
const lowQuantile = coverage < 0.05 ? 0.02 : 0.08;
const hiQuantile = coverage < 0.03 ? 0.995 : 0.97;
const lo = percentile(lowQuantile);
const hiRaw = percentile(hiQuantile);
const hi = Math.min(255, Math.max(lo + 8, hiRaw));
if (!Number.isFinite(lo) || !Number.isFinite(hi) || hi <= lo) {
return { changed: false, coverage, gamma: 1, lo, hi };
}
const gamma = coverage < 0.03 ? 0.52 : coverage < 0.08 ? 0.6 : coverage < 0.18 ? 0.68 : 0.75;
const range = hi - lo;
const invRange = range > 0 ? 1 / range : 0;
let changed = false;
for (let i = 0; i < total; i++) {
const v = mask[i];
if (v <= lo) {
if (v !== 0) { mask[i] = 0; changed = true; }
continue;
}
if (v >= hi) {
if (v !== 255) { mask[i] = 255; changed = true; }
continue;
}
const norm = (v - lo) * invRange;
const boosted = Math.pow(Math.max(0, Math.min(1, norm)), gamma);
const mapped = Math.max(0, Math.min(255, Math.round(boosted * 255)));
if (mask[i] !== mapped) {
mask[i] = mapped;
changed = true;
}
}
return { changed, coverage, gamma, lo, hi };
}
async function maskToDataUrl(mask: Uint8Array, width: number, height: number): Promise<string> {
const buffer = await sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.toColorspace("b-w")
.png()
.toBuffer();
return `data:image/png;base64,${buffer.toString('base64')}`;
}
async function maskToPngBuffer(mask: Uint8Array, width: number, height: number): Promise<Buffer> {
return sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.toColorspace("b-w")
.png()
.toBuffer();
}
async function runSamHqPatchFromBuffer(
image: Buffer,
fnMask: Uint8Array,
width: number,
height: number,
opts?: {
python?: string;
model?: string;
maxBoxes?: string;
margin?: string;
maxBoxSpanFrac?: string;
device?: string;
timeoutMs?: number;
},
): Promise<Uint8Array | null> {
const python = opts?.python ?? process.env.OCCLUDER_SAM_PATCH_PY ?? "python3";
const model = opts?.model ?? process.env.OCCLUDER_SAM_PATCH_MODEL ?? "syscv-community/sam-hq-vit-base";
const maxBoxes = opts?.maxBoxes ?? process.env.OCCLUDER_SAM_PATCH_BOXES ?? "6";
const margin = opts?.margin ?? process.env.OCCLUDER_SAM_PATCH_MARGIN ?? "24";
const maxBoxSpanFrac = opts?.maxBoxSpanFrac ?? process.env.OCCLUDER_SAM_PATCH_MAX_BOX_SPAN_FRAC ?? "0.45";
const device = opts?.device ?? process.env.OCCLUDER_SAM_PATCH_DEVICE ?? "auto";
const timeoutMs = Math.max(
60000,
Number(opts?.timeoutMs ?? process.env.OCCLUDER_SAM_PATCH_TIMEOUT_MS ?? 180000),
);
const scriptPath = path.resolve(process.cwd(), "tools", "sam_hq_patch.py");
const patchRoot = path.join(process.cwd(), "runs");
await fs.mkdir(patchRoot, { recursive: true });
const tempDir = await fs.mkdtemp(path.join(patchRoot, "sam_patch_"));
const imagePath = path.join(tempDir, "image.png");
const fnPath = path.join(tempDir, "fn_mask.png");
const outPath = path.join(tempDir, "sam_patch.png");
await fs.writeFile(imagePath, await sharp(image).resize(width, height, { fit: "fill" }).png().toBuffer());
await fs.writeFile(fnPath, await maskToPngBuffer(fnMask, width, height));
await new Promise<void>((resolve, reject) => {
const proc = spawn(
python,
[
scriptPath,
"--image",
imagePath,
"--fn-mask",
fnPath,
"--out",
outPath,
"--model",
model,
"--max-boxes",
maxBoxes,
"--box-margin",
margin,
"--max-box-span-frac",
maxBoxSpanFrac,
"--device",
device,
],
{ stdio: ["ignore", "pipe", "pipe"] },
);
let stderr = "";
let stdout = "";
const timer = setTimeout(() => {
proc.kill("SIGKILL");
reject(new Error(`sam_hq_patch timed out after ${timeoutMs}ms`));
}, timeoutMs);
proc.stdout.on("data", (chunk) => {
stdout += chunk.toString();
});
proc.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
proc.on("error", (err) => {
clearTimeout(timer);
reject(err);
});
proc.on("close", (code) => {
clearTimeout(timer);
if (code === 0) return resolve();
reject(new Error(`sam_hq_patch failed (${code}): ${stderr || stdout}`));
});
}).catch((err) => {
console.warn(err);
});
try {
const { data } = await sharp(outPath)
.resize(width, height, { fit: "fill", kernel: "nearest" })
.threshold(128)
.raw()
.toBuffer({ resolveWithObject: true });
const mask = new Uint8Array(width * height);
for (let i = 0; i < mask.length; i++) {
mask[i] = data[i] > 0 ? 255 : 0;
}
return mask;
} catch {
return null;
}
}
async function cutoutToDataUrl(source: Buffer, mask: Uint8Array, width: number, height: number): Promise<string> {
const alpha = Buffer.from(mask);
const buffer = await sharp(source)
.ensureAlpha()
.composite([
{
input: alpha,
raw: { width, height, channels: 1 },
blend: 'dest-in',
},
])
.png()
.toBuffer();
return `data:image/png;base64,${buffer.toString('base64')}`;
}
async function overlayToDataUrl(source: Buffer, mask: Uint8Array, width: number, height: number): Promise<string> {
const overlay = Buffer.alloc(width * height * 4);
for (let i = 0; i < width * height; i++) {
const alpha = mask[i];
const offset = i * 4;
overlay[offset + 0] = 255;
overlay[offset + 1] = 85;
overlay[offset + 2] = 0;
overlay[offset + 3] = Math.min(255, Math.round(alpha * 0.6));
}
const buffer = await sharp(source)
.composite([
{
input: overlay,
raw: { width, height, channels: 4 },
blend: 'over',
},
])
.png()
.toBuffer();
return `data:image/png;base64,${buffer.toString('base64')}`;
}
export async function POST(req: NextRequest) {
let restoreOccluderEnv: (() => void) | null = null;
try {
const url = new URL(req.url);
const occluderEnvOverrides = parseOccluderEnvOverrides(url.searchParams.get("occluderEnv"));
const occluderDefaults = getOccluderDefaults(process.env.OCCLUDER_DEFAULTS);
const occluderEnvDefaults = occluderDefaults?.occluderEnv ?? {};
for (const [key, value] of Object.entries(occluderEnvDefaults)) {
if (process.env[key] == null) {
process.env[key] = String(value);
}
}
restoreOccluderEnv = applyOccluderEnvOverrides(occluderEnvOverrides);
const debug = url.searchParams.get("debug") === "1";
const useSeg = (url.searchParams.get("seg") === "1") || (process.env.SEGMENT_OCCLUDERS === "1");
let occluderMethod = (process.env.OCCLUDER_METHOD ?? "").trim().toLowerCase();
let preferSegOccluder = occluderMethod === "sam_hq"
|| occluderMethod === "sam-hq"
|| occluderMethod === "sam_hq_prompted"
|| occluderMethod === "distilled_unsup"
|| occluderMethod === "distilled-unsup"
|| occluderMethod === "distilled";
const preferTileBand = (process.env.OCCLUDER_SAM_DEPTH_BAND_PREFER_TILEBAND ?? "1") !== "0";
const ransacParam = url.searchParams.get("ransac") ?? "";
const ransacForce = ransacParam === "1" || ransacParam === "true";
const ransacAuto = ransacParam === "auto";
const maskWeight = clamp(Number(url.searchParams.get("maskw") ?? process.env.MASK_WEIGHT ?? "0.85"), 0, 1);
const maskBlur = clamp(Number(url.searchParams.get("maskblur") ?? process.env.MASK_BLUR ?? "1.5"), 0, 10);
const presetParamRaw = url.searchParams.get("maskPreset") ?? process.env.MASK_PRESET_DEFAULT ?? null;
const presetRequested = resolvePresetName(presetParamRaw);
const maskDebugEnabled = debug || url.searchParams.get("maskDebug") === "1";
const fallbackThresholdEnv = Number.isFinite(Number(process.env.MASK_PRESET_FALLBACK))
? Number(process.env.MASK_PRESET_FALLBACK)
: 0;
const fallbackThreshold = Math.max(0, Math.min(0.06, fallbackThresholdEnv));
const headerFlag = (key: string) => req.headers.get(key)?.toLowerCase() ?? null;
const parseFlag = (value: string | null | undefined, defaultOn = true) => {
if (value == null) return defaultOn;
if (value === "1" || value === "true" || value === "on") return true;
if (value === "0" || value === "false" || value === "off") return false;
return defaultOn;
};
const parseNumber = (value: string | null | undefined): number | undefined => {
if (value == null) return undefined;
const num = Number(value);
return Number.isFinite(num) ? num : undefined;
};
const presetDefaults = presetRequested === 'tiles-detail'
? { lattice: true, illumination: true, autotune: true, autotuneMs: 200, thinFrac: 0.04, minAreaFrac: 0.12 }
: { lattice: true, illumination: true, autotune: true, autotuneMs: 150, thinFrac: 0.04, minAreaFrac: 0.12 };
const normalizeModelName = (value: string | null | undefined): string => {
if (!value) return 'unet_latest';
const normalized = value.toLowerCase();
if (['unet_latest', 'unet_old', 'u2net', 'none'].includes(normalized)) {
return normalized;
}
return 'unet_latest';
};
const tileModelRequested = normalizeModelName(
headerFlag('x-tile-model') ?? url.searchParams.get('model'),
);
const flags: PipelineFlags = {
enableLattice: parseFlag(
headerFlag("x-mask-lattice") ?? url.searchParams.get("maskLattice"),
presetDefaults.lattice,
),
enableIllumination: parseFlag(
headerFlag("x-mask-illumination") ?? url.searchParams.get("maskIllumination"),
presetDefaults.illumination,
),
enableAutotune: parseFlag(
headerFlag("x-mask-autotune") ?? url.searchParams.get("maskAutotune"),
presetDefaults.autotune,
),
autotuneMs: clamp(
Number(
headerFlag("x-mask-autotune-ms") ??
url.searchParams.get("maskAutotuneMs") ??
presetDefaults.autotuneMs.toString(),
),
0,
1000,
),
thinWidthFrac: (() => {
const val = parseNumber(headerFlag("x-mask-thinwidthfrac") ?? url.searchParams.get("maskThinWidthFrac"));
return Number.isFinite(val as number) ? (val as number) : presetDefaults.thinFrac;
})(),
minAreaFrac: (() => {
const val = parseNumber(headerFlag("x-mask-minareafrac") ?? url.searchParams.get("maskMinAreaFrac"));
return Number.isFinite(val as number) ? (val as number) : presetDefaults.minAreaFrac;
})(),
};
flags.thinWidthFrac = Math.max(0.0, Math.min(0.5, flags.thinWidthFrac));
flags.minAreaFrac = Math.max(0.05, Math.min(0.8, flags.minAreaFrac));
const formDataTimeoutMs = parseTimeoutMsEnv("ANALYZE_FORMDATA_TIMEOUT_MS", 20_000);
const fileBufferTimeoutMs = parseTimeoutMsEnv("ANALYZE_FILE_BUFFER_TIMEOUT_MS", 20_000);
const depthStageTimeoutMs = parseTimeoutMsEnv("ANALYZE_DEPTH_STAGE_TIMEOUT_MS", 90_000);
const form = await withStepTimeout(req.formData(), formDataTimeoutMs, "request.formData");
const file = form.get("file") as File | null;
if (!file) return NextResponse.json({ ok:false, error:"missing file" }, { status: 400 });
const fileName = typeof (file as any)?.name === "string" ? String((file as any).name) : null;
const presetBackend = resolveOccluderPresetBackend();
const presetComfyRoute = presetBackend === "rfdet_m3" && isComfyProfileUpload(fileName);
if (presetComfyRoute) {
occluderMethod = "distilled_unsup";
preferSegOccluder = true;
}
const effectiveOccluderEnvOverrides: OccluderEnvOverrides | null = (() => {
const merged: OccluderEnvOverrides = { ...(occluderEnvOverrides ?? {}) };
if (presetComfyRoute) {
merged.OCCLUDER_BACKEND = "rfdet_m3";
if (!merged.OCCLUDER_METHOD) {
merged.OCCLUDER_METHOD = "distilled_unsup";
}
}
return Object.keys(merged).length ? merged : null;
})();
const raw = Buffer.from(
await withStepTimeout(
file.arrayBuffer(),
fileBufferTimeoutMs,
"file.arrayBuffer",
),
);
const meta0 = await sharp(raw).metadata();
let W = meta0.width ?? 0, H = meta0.height ?? 0;
let tileSegmentationResult: TileSegmentationResult | null = null;
if (!(W>0 && H>0)) {
return NextResponse.json({ ok:false, error:"Decode error: image width/height invalid (w and h must be numbers)." }, { status: 422 });
}
let depthByteMap: DepthByteMap | null = null;
let depthWallDepth: number | null = null;
let depthWallThreshold: number | null = null;
let depthCutoff: number | null = null;
let depthOccluderMask: BinaryMask | null = null;
let depthBackMask: BinaryMask | null = null;
let depthRawDataUrl: string | null = null;
let depthOccluderUrl: string | null = null;
let depthMaskRawUrl: string | null = null;
let depthHistogram: number[] | null = null;
let depthInverted = false;
let depthDebugPayload: any = null;
let depthCrop: { x: number; y: number; width: number; height: number } | null = null;
try {
const depthRaw = await withStepTimeout(
estimateDepth(raw),
depthStageTimeoutMs,
"estimateDepth",
);
depthByteMap = normalizeDepthFloatToByte(depthRaw);
const targetW = meta0.width ?? depthByteMap.width;
const targetH = meta0.height ?? depthByteMap.height;
const cropResult = cropDepthToAspect(depthByteMap, targetW, targetH);
depthByteMap = cropResult.map;
depthCrop = cropResult.crop;
const depthPng = await sharp(Buffer.from(depthByteMap.data), {
raw: { width: depthByteMap.width, height: depthByteMap.height, channels: 1 },
})
.png()
.toBuffer();
const runRefine = async (buffer: Buffer, sensitivity: number) =>
refineOccluderMask(buffer, targetW, targetH, sensitivity, raw);
const scoreCandidate = (candidate: RefinedOccluderResult) => {
if (candidate.rejected) {
return { cov: 0, rowT: 0, colT: 0, gridLike: true, score: 1e9 };
}
const cov = coverageOfMask(candidate.mask);
const rowT = meanBinaryTransitionsPerRow(candidate.mask, candidate.width, candidate.height);
const colT = meanBinaryTransitionsPerCol(candidate.mask, candidate.width, candidate.height);
const gridLike = looksLikeGridBinaryMask(candidate.mask, candidate.width, candidate.height);
// Penalize extreme coverage, but do not treat it as "grid-like" by itself.
const covPenalty = cov < 0.1 ? (0.1 - cov) * 6 : cov > 0.9 ? (cov - 0.9) * 10 : 0;
const score =
(rowT / Math.max(1, candidate.width)) +
(colT / Math.max(1, candidate.height)) +
covPenalty +
(gridLike ? 2 : 0);
return { cov, rowT, colT, gridLike, score };
};
const sensitivities = [18, 22, 26, 30];
const candidates: Array<{ label: string; inverted: boolean; png: Buffer }> = [
{ label: "normal", inverted: false, png: depthPng },
];
// Only pay the cost of inverted evaluation when the normal orientation looks suspicious.
const refinedNormal = await runRefine(depthPng, sensitivities[0]);
const scoreNormal = scoreCandidate(refinedNormal);
if (scoreNormal.gridLike || scoreNormal.cov < 0.02 || scoreNormal.cov > 0.9) {
const inverted = Buffer.from(depthByteMap.data.map((v) => 255 - v));
const invertedPng = await sharp(inverted, {
raw: { width: depthByteMap.width, height: depthByteMap.height, channels: 1 },
})
.png()
.toBuffer();
candidates.push({ label: "inverted", inverted: true, png: invertedPng });
}
let refined: RefinedOccluderResult | null = null;
let chosenScore: ReturnType<typeof scoreCandidate> | null = null;
let sensitivityUsed = sensitivities[0];
let candidateLabel = candidates[0].label;
for (const candidate of candidates) {
for (const sensitivity of sensitivities) {
const candidateRefined = await runRefine(candidate.png, sensitivity);
const candidateScore = scoreCandidate(candidateRefined);
if (!refined || !chosenScore || candidateScore.score < chosenScore.score) {
refined = candidateRefined;
chosenScore = candidateScore;
depthInverted = candidate.inverted;
sensitivityUsed = sensitivity;
candidateLabel = candidate.label;
}
// Early exit on a clearly good mask.
if (!candidateScore.gridLike && candidateScore.cov >= 0.08 && candidateScore.cov <= 0.9) {
break;
}
}
}
if (!refined || !chosenScore) {
throw new Error("[depth] failed to refine occluder mask");
}
if (refined.rejected) {
depthOccluderMask = null;
depthOccluderUrl = null;
depthMaskRawUrl = null;
depthRawDataUrl = await maskToDataUrl(depthByteMap.data, depthByteMap.width, depthByteMap.height);
depthHistogram = refined.histogram ?? null;
depthWallDepth = refined.wallDepth ?? null;
depthWallThreshold = refined.threshold ?? null;
depthCutoff = refined.occluderThreshold ?? null;
depthDebugPayload = {
source: "refineOccluderMask",
rejected: true,
rejectReason: refined.rejected.reason,
rejectMetrics: refined.rejected.metrics ?? null,
inverted: depthInverted,
rawUrl: depthRawDataUrl,
resizedUrl: refined.debug?.resizedDepthUrl ?? null,
};
} else {
// Polarity: The rest of the pipeline expects 255=occluder (blocks tiles), 0=free (wall).
// Depth thresholding can sometimes yield the inverse (255=wall plane). Detect and fix via coverage heuristic.
let polarityInverted = false;
let polarityFixedMask = refined.mask;
let occCoverage = chosenScore.cov;
let autoDilateRadius = 0;
if (occCoverage > 0.65) {
const inv = new Uint8Array(refined.mask.length);
for (let i = 0; i < refined.mask.length; i++) inv[i] = 255 - refined.mask[i];
const invCoverage = coverageOfMask(inv);
if (invCoverage < occCoverage) {
polarityInverted = true;
polarityFixedMask = inv;
}
} else if (occCoverage < 0.01) {
const inv = new Uint8Array(refined.mask.length);
for (let i = 0; i < refined.mask.length; i++) inv[i] = 255 - refined.mask[i];
const invCoverage = coverageOfMask(inv);
if (invCoverage > occCoverage) {
polarityInverted = true;
polarityFixedMask = inv;
}
}
// Recompute from the final mask (source of truth) so debug/metrics match what we actually return.
occCoverage = coverageOfMask(polarityFixedMask);
// Auto-dilate depth occluder if it is too thin/fragmented.
// This stabilizes wall-adjacent objects without requiring U2/heuristics.
if (occCoverage > 0 && occCoverage < 0.20) {
for (const r of [2, 3, 4]) {
const dilated = await dilateBinaryMaskViaBlur(polarityFixedMask, refined.width, refined.height, r, 8);
const cov = coverageOfMask(dilated);
if (cov > occCoverage) {
polarityFixedMask = dilated;
occCoverage = cov;
autoDilateRadius = r;
}
if (occCoverage >= 0.26) break;
}
}
// Optional: shrink the final occluder slightly to reduce visible "halo/cutout" margins.
// This must happen after solidify/dilate so we don’t reintroduce holes.
const shrinkParam = url.searchParams.get("occluder_shrink_px") ?? "";
const requestedShrinkPx = clamp(Number.parseInt(shrinkParam || "0", 10) || 0, 0, 6);
let occluderShrinkPxApplied = 0;
let occluderUrlPreShrink: string | null = null;
if (requestedShrinkPx > 0 && polarityFixedMask.length === refined.width * refined.height) {
const before = polarityFixedMask;
const beforeCov = occCoverage;
const eroded = morphErode(before, refined.width, refined.height, requestedShrinkPx);
const afterCov = coverageOfMask(eroded);
// Guard: don’t collapse masks (e.g., on very thin occluders).
if (afterCov > 0.005 || beforeCov <= 0.02) {
occluderShrinkPxApplied = requestedShrinkPx;
occluderUrlPreShrink = await maskToDataUrl(before, refined.width, refined.height).catch(() => null);
polarityFixedMask = eroded;
occCoverage = afterCov;
}
}
depthOccluderMask = { data: polarityFixedMask, width: refined.width, height: refined.height };
// Always encode from the final mask used by the pipeline, so debug can’t drift.
const depthOccluderUrlPrePolarity = refined.debug?.processedUrl ?? null;
depthOccluderUrl = await maskToDataUrl(polarityFixedMask, refined.width, refined.height);
const depthOccluderUrlCandidate = depthOccluderUrl;
depthMaskRawUrl = refined.debug?.rawUrl ?? null;
const depthResizedUrl = refined.debug?.resizedDepthUrl ?? null;
depthRawDataUrl = await maskToDataUrl(depthByteMap.data, depthByteMap.width, depthByteMap.height);
depthHistogram = refined.histogram;
depthWallDepth = refined.wallDepth;
depthWallThreshold = refined.threshold;
depthCutoff = refined.occluderThreshold;
const occluderMaskSha1 = maskSha1(polarityFixedMask, refined.width, refined.height);
const depthMapSha1 = maskSha1(depthByteMap.data, depthByteMap.width, depthByteMap.height);
const rowTransitions = meanBinaryTransitionsPerRow(polarityFixedMask, refined.width, refined.height);
const colTransitions = meanBinaryTransitionsPerCol(polarityFixedMask, refined.width, refined.height);
let occluderPixels = 0;
for (let i = 0; i < polarityFixedMask.length; i++) {
if (polarityFixedMask[i] > 127) occluderPixels += 1;
}
// Re-check grid-likeness on the final mask (after polarity correction).
// We treat this as a warning only (not a hard rejection), because noisy occluders can have
// high transition counts without being a true tile/checker grid.
const gridLikeWarning = looksLikeGridBinaryMask(polarityFixedMask, refined.width, refined.height);
const rejectedCoverage = occCoverage > 0.98 || occCoverage < 0.002;
const rejectedSolid = occCoverage === 0 || occCoverage === 1;
const rejected = rejectedCoverage || rejectedSolid;
if (rejected) {
console.warn("[depth] rejecting depth occluder", {
reason: rejectedSolid ? "solid-mask" : "coverage-out-of-range",
coverage: Number(occCoverage.toFixed(4)),
rowTransitions: Number(rowTransitions.toFixed(2)),
colTransitions: Number(colTransitions.toFixed(2)),
candidate: candidateLabel,
sensitivityUsed,
occluderMaskSha1,
});
depthOccluderMask = null;
depthOccluderUrl = null;
}
// Build back/plane mask at target size
const resizedDepth = await sharp(Buffer.from(depthByteMap.data), {
raw: { width: depthByteMap.width, height: depthByteMap.height, channels: 1 },
})
.resize(targetW, targetH, { kernel: "lanczos3" })
.toColorspace("b-w")
.raw()
.toBuffer();
const backMask = new Uint8Array(resizedDepth.length);
const backThresh = Math.min(255, refined.threshold + 5);
for (let i = 0; i < resizedDepth.length; i++) {
backMask[i] = resizedDepth[i] <= backThresh ? 255 : 0;
}
depthBackMask = { data: backMask, width: targetW, height: targetH };
const depthOccCov = occCoverage;
const depthBackCov = coverageOfMask(backMask);
depthDebugPayload = {
source: "refineOccluderMask",
candidate: candidateLabel,
sensitivityUsed,
polarityInverted,
autoDilateRadius,
occluderShrinkPx: occluderShrinkPxApplied,
rejected,
gridLikeWarning,
warnings: refined.warnings ?? null,
rejectReason: rejected
? rejectedSolid
? "occluder is solid (all 0 or all 255)"
: "occluder coverage out of range"
: null,
occluderMaskSha1,
depthMapSha1,
occluderTransitionsPerRow: Number(rowTransitions.toFixed(2)),
occluderTransitionsPerCol: Number(colTransitions.toFixed(2)),
wallDepth: depthWallDepth,
wallThreshold: depthWallThreshold,
occlusionThreshold: depthCutoff,
histogram: depthHistogram,
inverted: depthInverted,
occluderCoverage: depthOccCov != null ? Number(depthOccCov.toFixed(4)) : null,
occluderPixels,
occluderTotalPixels: refined.width * refined.height,
backCoverage: depthBackCov != null ? Number(depthBackCov.toFixed(4)) : null,
rawUrl: depthRawDataUrl,
resizedUrl: depthResizedUrl,
maskRawUrl: depthMaskRawUrl,
occluderUrl: depthOccluderUrl,
occluderUrlPreShrink,
occluderUrlRejected: rejected ? depthOccluderUrlCandidate : null,
occluderUrlPrePolarity: depthOccluderUrlPrePolarity,
depthCrop,
};
}
} catch (depthErr) {
if (debug) {
const message =
depthErr instanceof Error
? (depthErr.stack ?? depthErr.message)
: String(depthErr);
console.warn("[depth] failed to compute depth map", depthErr);
depthDebugPayload = {
error: message,
};
}
}
const enableTileSegmentation = tileModelRequested !== 'none';
let tileCombineResult: Awaited<ReturnType<typeof combineTileMaskWithOccluder>> | null = null;
if (meta0.width && meta0.height && enableTileSegmentation) {
try {
if (tileModelRequested === 'unet_latest') {
const tileModule = await import('@/lib/segmentation/tile-unet');
if (typeof tileModule.inferTileSegmentation === 'function') {
tileSegmentationResult = await tileModule.inferTileSegmentation(raw, { debug });
}
} else {
console.warn('[tile-seg] model not implemented, skipping', tileModelRequested);
}
} catch (tileError) {
console.warn('[tile-seg] failed to compute tile mask', tileError);
}
}
// (A) Segmentierung am Original
let maskOrig: Uint8Array | null = null;
let maskOriginSource: string | null = null;
let maskOriginMeta: Record<string, unknown> | null = null;
let maskBoostStats: MaskBoostStats | null = null;
let maskPresetApplied: MaskPresetName | null = null;
let maskNormalized: Uint8Array | null = null;
let maskForWeights: Uint8Array | null = null;
let maskDebugPayload: any = null;
let maskRawDataUrl: string | null = null;
let maskNormalizedDataUrl: string | null = null;
let maskOverlayDataUrl: string | null = null;
let maskCutoutDataUrl: string | null = null;
let maskCoverage: number | null = null;
let maskBackgroundCoverage: number | null = null;
let maskOccluderCoverage: number | null = null;
let maskSeedCoverage: number | null = null;
let finalMaskDataUrl: string | null = null;
let finalMaskMeta: FinalMaskMeta | null = null;
let intensityBytes: Uint8Array | null = null;
let gridContext: GridContext | null = null;
let latticeLeak: number | null = null;
let latticeInteriorCoverage: number | null = null;
let maskWidth: number | null = null;
let maskHeight: number | null = null;
let normalizationPrimary: NormalizeMaskResponse | null = null;
let normalizationChosen: NormalizeMaskResponse | null = null;
let maskNormalizedInitial: Uint8Array | null = null;
let maskNormalizedInitialCoverage: number | null = null;
let pipelineWarnings: string[] | undefined = undefined;
let tileMaskDataUrl: string | null = null;
let tileMaskMeta: TileMaskMeta | null = null;
let tileMaskVersions: TileMaskVersions | null = null;
let samPromptedDebug: Record<string, unknown> | null = null;
let promptTileMask: Uint8Array | null = null;
let promptTileWidth: number | null = null;
let promptTileHeight: number | null = null;
let promptTileSource: string | null = null;
if (useSeg) {
const occluderDebugDir = maskDebugEnabled
? path.join(process.cwd(), "runs", "occluder-debug")
: null;
const workerUrl = process.env.OCCLUDER_WORKER_URL ?? null;
const workerTimeoutMs = Math.max(1_000, Number(process.env.OCCLUDER_WORKER_TIMEOUT ?? "20000"));
const samPromptedFlag = (process.env.OCCLUDER_SAM_PROMPTED ?? "").trim().toLowerCase();
const samAutoEnabled = samPromptedFlag === ""
? (process.env.OCCLUDER_SAM_AUTO ?? "1") !== "0"
: false;
const samPromptedEnabled = samPromptedFlag === "1"
|| (samPromptedFlag !== "0" && (Boolean(process.env.OCCLUDER_SEG_SAM_CHECKPOINT) || samAutoEnabled));
const samTimeoutMs = Math.max(30_000, Number(process.env.OCCLUDER_SEG_SAM_TIMEOUT ?? "180000"));
promptTileMask = samPromptedEnabled ? buildTilePlaneMask(tileSegmentationResult ?? null) : null;
promptTileWidth = tileSegmentationResult?.width ?? null;
promptTileHeight = tileSegmentationResult?.height ?? null;
promptTileSource = promptTileMask ? "tile_unet" : null;
const promptCoverage = coverageOfMask(promptTileMask);
const promptTooWeak = promptCoverage > 0 && promptCoverage < 0.02;
const promptTooStrong = promptCoverage > 0.95;
samPromptedDebug = samPromptedEnabled
? {
enabled: samPromptedEnabled,
mode: samPromptedFlag === "1" ? "forced" : (samPromptedFlag === "0" ? "disabled" : "auto"),
autoEnabled: samAutoEnabled,
ready: false,
source: promptTileSource ?? null,
coverage: Number((promptCoverage || 0).toFixed(4)),
tooWeak: promptTooWeak,
tooStrong: promptTooStrong,
tileMaskSize: promptTileWidth && promptTileHeight ? `${promptTileWidth}x${promptTileHeight}` : null,
}
: null;
if ((samPromptedEnabled && (!promptTileMask || promptTooWeak || promptTooStrong)) && depthBackMask) {
const targetW = meta0.width ?? depthBackMask.width;
const targetH = meta0.height ?? depthBackMask.height;
const depthAligned = await resizeBinaryMask(depthBackMask, targetW, targetH);
promptTileMask = depthAligned.data;
promptTileWidth = depthAligned.width;
promptTileHeight = depthAligned.height;
promptTileSource = "depth_back";
if (samPromptedDebug) {
samPromptedDebug.source = promptTileSource;
samPromptedDebug.coverage = Number(coverageOfMask(promptTileMask).toFixed(4));
samPromptedDebug.tileMaskSize = `${promptTileWidth}x${promptTileHeight}`;
}
}
const promptBandEnabled = (process.env.OCCLUDER_SAM_PROMPT_BAND ?? "0") === "1";
if (samPromptedEnabled && promptTileMask && promptTileWidth && promptTileHeight && promptBandEnabled) {
const banded = bandMaskByRowDensity(promptTileMask, promptTileWidth, promptTileHeight);
if (banded.applied) {
promptTileMask = banded.mask;
promptTileSource = promptTileSource ? `${promptTileSource}+band` : "banded";
if (samPromptedDebug && banded.band) {
samPromptedDebug.source = promptTileSource;
samPromptedDebug.coverage = Number(coverageOfMask(promptTileMask).toFixed(4));
samPromptedDebug.tileMaskSize = `${promptTileWidth}x${promptTileHeight}`;
samPromptedDebug.band = banded.band;
}
}
}
if (samPromptedEnabled && (!promptTileMask || promptTooWeak || promptTooStrong) && meta0.width && meta0.height) {
const topFrac = clamp(Number(process.env.OCCLUDER_SAM_FALLBACK_TOP ?? "0.35"), 0, 0.9);
const bottomFrac = clamp(Number(process.env.OCCLUDER_SAM_FALLBACK_BOTTOM ?? "0.9"), 0.1, 1);
const y0 = Math.max(0, Math.floor(meta0.height * topFrac));
const y1 = Math.max(y0 + 1, Math.floor(meta0.height * bottomFrac));
const band = new Uint8Array(meta0.width * meta0.height);
for (let y = y0; y < y1; y++) {
const row = y * meta0.width;
band.fill(255, row, row + meta0.width);
}
promptTileMask = band;
promptTileWidth = meta0.width;
promptTileHeight = meta0.height;
promptTileSource = "fallback_band";
if (samPromptedDebug) {
samPromptedDebug.source = promptTileSource;
samPromptedDebug.coverage = Number(coverageOfMask(promptTileMask).toFixed(4));
samPromptedDebug.tileMaskSize = `${promptTileWidth}x${promptTileHeight}`;
}
}
const samPromptedReady = Boolean(
samPromptedEnabled &&
promptTileMask &&
promptTileWidth &&
promptTileHeight,
);
if (samPromptedDebug) samPromptedDebug.ready = samPromptedReady;
if (samPromptedEnabled && !samPromptedReady && debug) {
console.warn("[mask] sam_prompted requested but tile mask is missing");
}
const fetchWorkerMask = async () => {
if (!workerUrl) return null;
const form = new FormData();
const blob = new Blob([raw], { type: meta0?.format ? `image/${meta0.format}` : "application/octet-stream" });
form.append("file", blob, (file as any)?.name ?? "upload");
const response = await fetch(workerUrl, {
method: "POST",
body: form,
signal: AbortSignal.timeout(workerTimeoutMs),
});
if (!response.ok) {
if (debug) console.warn(`[mask] occluder-worker responded ${response.status}`);
return null;
}
const payload = await response.json() as any;
if (!payload?.ok || typeof payload.mask !== "string") {
if (debug) console.warn("[mask] occluder-worker invalid payload", payload);
return null;
}
const maskBuffer = Buffer.from(payload.mask, "base64");
let pipeline = sharp(maskBuffer, { failOnError: false })
.ensureAlpha()
.removeAlpha()
.greyscale();
const resizeW = meta0.width ?? payload.width ?? null;
const resizeH = meta0.height ?? payload.height ?? null;
if (resizeW && resizeH) {
pipeline = pipeline.resize(resizeW, resizeH, { fit: "fill" });
}
const { data: maskData, info } = await pipeline
.raw()
.toBuffer({ resolveWithObject: true });
return {
data: new Uint8Array(maskData),
width: info.width ?? resizeW ?? meta0.width ?? 0,
height: info.height ?? resizeH ?? meta0.height ?? 0,
source: payload.source ?? "occluder-worker",
meta: payload.meta ?? null,
coverage: typeof payload.coverage === "number" ? payload.coverage : null,
};
};
if (!maskOrig && workerUrl && !samPromptedReady) {
try {
const payload = await fetchWorkerMask();
if (payload?.data && payload.data.length) {
maskOrig = payload.data;
maskWidth = payload.width || maskWidth;
maskHeight = payload.height || maskHeight;
maskOriginSource = payload.source ?? "occluder-worker";
maskOriginMeta = (payload.meta ?? null) as Record<string, unknown> | null;
if (typeof payload.coverage === "number") {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
workerCoverage: payload.coverage,
};
}
}
} catch (err) {
if (debug) console.warn("[mask] occluder-worker failed", err);
}
}
let occluderSegError: string | null = null;
if (!maskOrig) {
try {
const mod = await import("@/lib/segmentation/occluder_best");
const infer = (mod as any).inferOccluderBest as (buf: Buffer, opts?: unknown) => Promise<{ mask: Uint8Array; source: string; meta?: Record<string, unknown> | null; } | null>;
if (typeof infer === "function") {
const result = await infer(raw, {
debugDir: occluderDebugDir ?? undefined,
tileMask: samPromptedReady ? promptTileMask : undefined,
tileMaskWidth: samPromptedReady ? promptTileWidth : undefined,
tileMaskHeight: samPromptedReady ? promptTileHeight : undefined,
samPrompted: samPromptedReady,
timeoutMs: samPromptedReady ? samTimeoutMs : undefined,
env: effectiveOccluderEnvOverrides ?? undefined,
});
if (result && result.mask && result.mask.length) {
maskOrig = result.mask;
maskOriginSource = result.source ?? "yolo";
maskOriginMeta = result.meta ?? null;
if (effectiveOccluderEnvOverrides) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
occluderEnvOverrides: effectiveOccluderEnvOverrides,
};
}
if (presetComfyRoute) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
occluderPresetBackend: presetBackend,
occluderPresetApplied: true,
occluderMethodEffective: occluderMethod,
uploadFileName: fileName ?? undefined,
};
}
if (samPromptedReady) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
promptTileSource: promptTileSource ?? undefined,
};
}
} else {
occluderSegError = "inferOccluderBest returned empty result";
}
} else {
occluderSegError = "inferOccluderBest export missing";
}
} catch (err) {
occluderSegError = err instanceof Error ? err.message : String(err);
if (debug) console.warn("[mask] occluder-seg failed", err);
}
}
if (!maskOrig && workerUrl && samPromptedReady) {
try {
const payload = await fetchWorkerMask();
if (payload?.data && payload.data.length) {
maskOrig = payload.data;
maskWidth = payload.width || maskWidth;
maskHeight = payload.height || maskHeight;
maskOriginSource = payload.source ?? "occluder-worker";
maskOriginMeta = (payload.meta ?? null) as Record<string, unknown> | null;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPromptedFallback: true,
occluderSegError: occluderSegError ?? undefined,
};
if (typeof payload.coverage === "number") {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
workerCoverage: payload.coverage,
};
}
}
} catch (err) {
if (debug) console.warn("[mask] occluder-worker fallback failed", err);
}
} else if (occluderSegError) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
occluderSegError,
};
}
maskOriginMeta = {
...(maskOriginMeta ?? {}),
occluderMethodResolved: occluderMethod,
};
if (maskOrig && samPromptedReady && workerUrl && !preferSegOccluder) {
try {
const rescueEnabled = (process.env.OCCLUDER_SAM_RESCUE ?? "1") !== "0";
if (rescueEnabled && promptTileMask && promptTileWidth && promptTileHeight) {
const payload = await fetchWorkerMask();
if (payload?.data && payload.data.length) {
let workerMask = payload.data;
if (payload.width !== promptTileWidth || payload.height !== promptTileHeight) {
const aligned = await resizeBinaryMask(
{ data: workerMask, width: payload.width, height: payload.height },
promptTileWidth,
promptTileHeight,
);
workerMask = aligned.data;
}
const rescueBandPx = Math.max(0, Number(process.env.OCCLUDER_SAM_RESCUE_BAND ?? "3"));
const rescueErodePx = Math.max(0, Number(process.env.OCCLUDER_SAM_RESCUE_ERODE ?? "1"));
const rescueAdjPx = Math.max(0, Number(process.env.OCCLUDER_SAM_RESCUE_ADJ ?? "6"));
const adjacency = rescueAdjPx > 0
? morphDilate(maskOrig, promptTileWidth, promptTileHeight, rescueAdjPx)
: maskOrig;
const tileRegion = rescueBandPx > 0
? morphDilate(promptTileMask, promptTileWidth, promptTileHeight, rescueBandPx)
: promptTileMask;
let rescueRegion = tileRegion ? maskOr(tileRegion, adjacency) : adjacency;
if (depthBackMask && depthBackMask.width && depthBackMask.height) {
const aligned = await resizeBinaryMask(
{ data: depthBackMask.data, width: depthBackMask.width, height: depthBackMask.height },
promptTileWidth,
promptTileHeight,
);
const rowThresh = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_RESCUE_ROW_THRESH ?? "0.08")));
const rowMargin = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_RESCUE_ROW_MARGIN ?? "6")));
let yMin = -1;
let yMax = -1;
for (let y = 0; y < aligned.height; y++) {
let count = 0;
const row = y * aligned.width;
for (let x = 0; x < aligned.width; x++) {
if (aligned.data[row + x]) count += 1;
}
if (count / Math.max(1, aligned.width) >= rowThresh) {
if (yMin < 0) yMin = y;
yMax = y;
}
}
if (yMin >= 0 && yMax >= yMin) {
const band = new Uint8Array(aligned.width * aligned.height);
const start = Math.max(0, yMin - rowMargin);
const end = Math.min(aligned.height - 1, yMax + rowMargin);
for (let y = start; y <= end; y++) {
const row = y * aligned.width;
for (let x = 0; x < aligned.width; x++) band[row + x] = 255;
}
rescueRegion = maskOr(rescueRegion, band);
}
}
const rescueSupport = rescueErodePx > 0
? morphErode(workerMask, promptTileWidth, promptTileHeight, rescueErodePx)
: workerMask;
const merged = new Uint8Array(maskOrig);
let rescuePixels = 0;
const limit = Math.min(merged.length, rescueSupport.length, rescueRegion.length);
const minArea = Math.max(1, Number(process.env.OCCLUDER_SAM_RESCUE_MIN_AREA ?? "200"));
const components = connectedComponents(rescueSupport, promptTileWidth, promptTileHeight, minArea);
let mergedComponents = 0;
for (const comp of components) {
let touches = false;
for (const idx of comp.pixels) {
if (idx < limit && rescueRegion[idx]) {
touches = true;
break;
}
}
if (!touches) continue;
mergedComponents += 1;
for (const idx of comp.pixels) {
if (idx < limit && !merged[idx]) {
merged[idx] = 255;
rescuePixels += 1;
}
}
}
if (rescuePixels > 0) {
maskOrig = merged;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
rescue: {
pixels: rescuePixels,
coverage: Number((rescuePixels / merged.length).toFixed(6)),
bandPx: rescueBandPx,
erodePx: rescueErodePx,
adjPx: rescueAdjPx,
minArea,
components: mergedComponents,
source: payload.source ?? "occluder-worker",
},
};
if (debug) {
console.info("[mask] sam rescue merged", {
pixels: rescuePixels,
coverage: Number((rescuePixels / merged.length).toFixed(6)),
components: mergedComponents,
});
}
}
}
}
} catch (err) {
if (debug) console.warn("[mask] sam rescue failed", err);
}
}
if (!maskOrig) {
try {
const mod = await import("@/lib/segmentation/u2net");
const fn = (mod as any).inferU2Mask as (buf: Buffer) => Promise<Uint8Array>;
if (typeof fn === "function") {
maskOrig = await fn(raw);
if (maskOrig) {
maskOriginSource = maskOriginSource ?? "u2net";
}
}
} catch {
maskOrig = null;
}
}
if (!maskOrig) {
try {
const origin = new URL(req.url).origin;
const basePath = process.env.NEXT_PUBLIC_BASE_PATH || "/nextjs";
const modelUrl = `${origin}${basePath}/models/u2netp.onnx`;
const wasmCdn = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.17.1/dist/";
const modW = await import("@/lib/segmentation/u2net-wasm");
const fnW = (modW as any).inferU2MaskWasm as (buf: Buffer, mu: string, wp?: string) => Promise<Uint8Array>;
maskOrig = await fnW(raw, modelUrl, wasmCdn);
if (maskOrig) {
maskOriginSource = maskOriginSource ?? "u2net-wasm";
}
} catch {
maskOrig = null;
}
}
const depthOccCoverage = depthOccluderMask
? (depthDebugPayload?.occluderCoverage ?? coverageOfMask(depthOccluderMask.data))
: null;
const depthColTransitions = depthDebugPayload?.occluderTransitionsPerCol ?? null;
const depthGridLike = depthDebugPayload?.gridLikeWarning ?? false;
const baseOccCoverage = maskOrig ? coverageOfMask(maskOrig) : null;
const depthMaxColTrans = Number(process.env.OCCLUDER_DEPTH_MAX_COL_TRANS ?? "160");
const depthMaxCoverage = Number(process.env.OCCLUDER_DEPTH_MAX_COVERAGE ?? "0.45");
const depthMaxDelta = Number(process.env.OCCLUDER_DEPTH_MAX_DELTA ?? "0.15");
let allowDepthOccluder = Boolean(depthOccluderMask);
let depthClipApplied = false;
const depthSuppressReasons: string[] = [];
if (allowDepthOccluder && preferSegOccluder) {
allowDepthOccluder = false;
depthSuppressReasons.push("method_override");
}
if (allowDepthOccluder && maskOrig) {
if (depthGridLike) {
allowDepthOccluder = false;
depthSuppressReasons.push("grid_like");
}
if (
depthColTransitions != null &&
depthColTransitions > depthMaxColTrans &&
depthOccCoverage != null &&
depthOccCoverage > 0.18
) {
allowDepthOccluder = false;
depthSuppressReasons.push("col_transitions");
}
if (depthOccCoverage != null && depthOccCoverage > depthMaxCoverage) {
allowDepthOccluder = false;
depthSuppressReasons.push("coverage_high");
}
if (
baseOccCoverage != null &&
depthOccCoverage != null &&
depthOccCoverage > baseOccCoverage + depthMaxDelta
) {
allowDepthOccluder = false;
depthSuppressReasons.push("larger_than_base");
}
}
if (!allowDepthOccluder && depthSuppressReasons.length) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthOccluderSuppressed: {
reasons: depthSuppressReasons,
coverage: depthOccCoverage != null ? Number(depthOccCoverage.toFixed(4)) : null,
baseCoverage: baseOccCoverage != null ? Number(baseOccCoverage.toFixed(4)) : null,
colTransitions: depthColTransitions,
gridLike: depthGridLike,
},
};
}
const depthClipEnabled = (process.env.OCCLUDER_SAM_DEPTH_CLIP ?? "0") === "1";
if (depthClipEnabled && preferSegOccluder && maskOrig && depthOccluderMask) {
const targetW = maskWidth ?? depthOccluderMask.width;
const targetH = maskHeight ?? depthOccluderMask.height;
const aligned = (targetW && targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const dilatePx = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_DEPTH_CLIP_DILATE ?? "2")));
const clipMask = (dilatePx > 0 && aligned.width && aligned.height)
? morphDilate(aligned.data, aligned.width, aligned.height, dilatePx)
: aligned.data;
const clipped = maskAnd(maskOrig, clipMask);
const beforeCoverage = coverageOfMask(maskOrig);
const afterCoverage = coverageOfMask(clipped);
const minFrac = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_DEPTH_CLIP_MIN_FRAC ?? "0.4")));
const minCoverage = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_DEPTH_CLIP_MIN_COVERAGE ?? "0.05")));
if (afterCoverage >= minCoverage && (beforeCoverage <= 0 || afterCoverage / beforeCoverage >= minFrac)) {
maskOrig = clipped;
maskWidth = aligned.width;
maskHeight = aligned.height;
depthClipApplied = true;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthClip: {
applied: true,
dilatePx,
beforeCoverage: Number(beforeCoverage.toFixed(4)),
afterCoverage: Number(afterCoverage.toFixed(4)),
minFrac,
minCoverage,
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthClip: {
applied: false,
reason: "coverage_too_small",
dilatePx,
beforeCoverage: Number(beforeCoverage.toFixed(4)),
afterCoverage: Number(afterCoverage.toFixed(4)),
minFrac,
minCoverage,
},
};
}
}
const promptClipEnabled = (process.env.OCCLUDER_SAM_PROMPT_CLIP ?? "0") === "1";
if (
promptClipEnabled &&
preferSegOccluder &&
maskOrig &&
promptTileMask &&
promptTileWidth &&
promptTileHeight
) {
const targetW = maskWidth ?? promptTileWidth;
const targetH = maskHeight ?? promptTileHeight;
const promptAligned = (targetW && targetH)
? await resizeBinaryMask(
{ data: promptTileMask, width: promptTileWidth, height: promptTileHeight },
targetW,
targetH,
)
: { data: promptTileMask, width: promptTileWidth, height: promptTileHeight };
const banded = bandMaskByRowDensity(promptAligned.data, promptAligned.width, promptAligned.height);
const band = banded.applied && banded.band
? { yMin: banded.band.yMin, yMax: banded.band.yMax }
: bandFromNonZeroRows(promptAligned.data, promptAligned.width, promptAligned.height);
const padFrac = Math.max(0, Number(process.env.OCCLUDER_SAM_PROMPT_CLIP_PAD_FRAC ?? "0.03"));
const padPx = Math.max(0, Math.round(promptAligned.height * padFrac));
let clipMask = promptAligned.data;
let clipBand: { yMin: number; yMax: number } | null = null;
if (band) {
const yMin = Math.max(0, band.yMin - padPx);
const yMax = Math.min(promptAligned.height, band.yMax + padPx);
const full = new Uint8Array(promptAligned.width * promptAligned.height);
full.fill(255);
clipMask = applyBandToMask(full, promptAligned.width, promptAligned.height, yMin, yMax);
clipBand = { yMin, yMax };
}
const dilatePx = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PROMPT_CLIP_DILATE ?? "0")));
const clipMaskDilated = dilatePx > 0
? morphDilate(clipMask, promptAligned.width, promptAligned.height, dilatePx)
: clipMask;
const clipped = maskAnd(maskOrig, clipMaskDilated);
const beforeCoverage = coverageOfMask(maskOrig);
const afterCoverage = coverageOfMask(clipped);
const minFrac = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_PROMPT_CLIP_MIN_FRAC ?? "0.2")));
const minCoverage = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_PROMPT_CLIP_MIN_COVERAGE ?? "0.02")));
if (afterCoverage >= minCoverage && (beforeCoverage <= 0 || afterCoverage / beforeCoverage >= minFrac)) {
maskOrig = clipped;
maskWidth = promptAligned.width;
maskHeight = promptAligned.height;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
promptClip: {
applied: true,
dilatePx,
beforeCoverage: Number(beforeCoverage.toFixed(4)),
afterCoverage: Number(afterCoverage.toFixed(4)),
minFrac,
minCoverage,
band: clipBand,
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
promptClip: {
applied: false,
reason: "coverage_too_small",
dilatePx,
beforeCoverage: Number(beforeCoverage.toFixed(4)),
afterCoverage: Number(afterCoverage.toFixed(4)),
minFrac,
minCoverage,
band: clipBand,
},
};
}
}
const promptRescueEnabled = (process.env.OCCLUDER_SAM_PROMPT_RESCUE ?? "1") !== "0";
if (
promptRescueEnabled &&
preferSegOccluder &&
maskOrig &&
depthOccluderMask &&
promptTileMask &&
promptTileWidth &&
promptTileHeight
) {
const baseCoverage = coverageOfMask(maskOrig);
const minRescueCoverage = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_SAM_PROMPT_RESCUE_MIN_COVERAGE ?? "0.06")));
if (baseCoverage < minRescueCoverage) {
const targetW = maskWidth ?? promptTileWidth ?? depthOccluderMask.width;
const targetH = maskHeight ?? promptTileHeight ?? depthOccluderMask.height;
const promptAligned = (targetW && targetH)
? await resizeBinaryMask(
{ data: promptTileMask, width: promptTileWidth, height: promptTileHeight },
targetW,
targetH,
)
: { data: promptTileMask, width: promptTileWidth, height: promptTileHeight };
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const rowBanded = bandMaskByRowDensity(promptAligned.data, promptAligned.width, promptAligned.height);
let bandMask = rowBanded.applied ? rowBanded.mask : null;
let bandMeta: { yMin: number; yMax: number } | null = null;
if (!bandMask) {
const extent = bandFromNonZeroRows(promptAligned.data, promptAligned.width, promptAligned.height);
if (extent) {
bandMask = applyBandToMask(
new Uint8Array(promptAligned.width * promptAligned.height).fill(255),
promptAligned.width,
promptAligned.height,
extent.yMin,
extent.yMax,
);
bandMeta = { yMin: extent.yMin, yMax: extent.yMax };
}
} else if (rowBanded.band) {
bandMeta = { yMin: rowBanded.band.yMin, yMax: rowBanded.band.yMax };
}
if (bandMask) {
const bandDilate = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PROMPT_RESCUE_BAND_DILATE ?? "2")));
const bandMaskDilated = bandDilate > 0
? morphDilate(bandMask, promptAligned.width, promptAligned.height, bandDilate)
: bandMask;
const bandCoverage = coverageOfMask(bandMaskDilated);
const maxBandCoverage = clamp(
Number(process.env.OCCLUDER_SAM_PROMPT_RESCUE_MAX_BAND_COVERAGE ?? "0.5"),
0.1,
1,
);
if (bandCoverage <= maxBandCoverage) {
const clipped = maskAnd(depthAligned.data, bandMaskDilated);
const beforeCoverage = coverageOfMask(maskOrig);
const merged = maskOr(maskOrig, clipped);
const afterCoverage = coverageOfMask(merged);
maskOrig = merged;
maskWidth = targetW;
maskHeight = targetH;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
promptRescue: {
applied: true,
bandDilate,
bandCoverage: Number(bandCoverage.toFixed(4)),
maxBandCoverage: Number(maxBandCoverage.toFixed(3)),
band: bandMeta,
coverageBefore: Number(beforeCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
promptRescue: {
applied: false,
reason: "band_too_large",
bandCoverage: Number(bandCoverage.toFixed(4)),
maxBandCoverage: Number(maxBandCoverage.toFixed(3)),
band: bandMeta,
},
};
}
}
}
}
if (allowDepthOccluder && depthOccluderMask) {
if (depthClipApplied) {
// If we already clipped to depth, skip union to avoid re-expanding.
allowDepthOccluder = false;
}
}
const boardUnionMinCoverage = Math.max(
0,
Math.min(1, Number(process.env.OCCLUDER_SAM_BOARD_UNION_MIN_COVERAGE ?? "0.4")),
);
if (boardUnionMinCoverage > 0 && maskOrig && depthOccluderMask) {
const currentCoverage = coverageOfMask(maskOrig);
if (currentCoverage < boardUnionMinCoverage) {
const targetW = meta0.width ?? promptTileWidth ?? depthOccluderMask.width;
const targetH = meta0.height ?? promptTileHeight ?? depthOccluderMask.height;
if (targetW && targetH && maskOrig.length === targetW * targetH) {
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const dilatePx = Math.max(
0,
Math.round(Number(process.env.OCCLUDER_SAM_BOARD_UNION_DILATE ?? "6")),
);
const boardMask = dilatePx > 0
? morphDilate(depthAligned.data, targetW, targetH, dilatePx)
: depthAligned.data;
const merged = maskOr(maskOrig, boardMask);
const afterCoverage = coverageOfMask(merged);
const maxBoardCoverage = Math.min(
1,
Number(process.env.OCCLUDER_SAM_BOARD_UNION_MAX_COVERAGE ?? "0.9"),
);
if (afterCoverage <= maxBoardCoverage) {
maskOrig = merged;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBoardUnion: {
applied: true,
dilatePx,
coverageBefore: Number(currentCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
minCoverage: Number(boardUnionMinCoverage.toFixed(3)),
maxCoverage: Number(maxBoardCoverage.toFixed(3)),
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBoardUnion: {
applied: false,
reason: "union_too_large",
dilatePx,
coverageBefore: Number(currentCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
minCoverage: Number(boardUnionMinCoverage.toFixed(3)),
maxCoverage: Number(maxBoardCoverage.toFixed(3)),
},
};
}
}
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBoardUnion: {
applied: false,
reason: "coverage_ok",
coverage: Number(currentCoverage.toFixed(4)),
target: Number(boardUnionMinCoverage.toFixed(3)),
},
};
}
}
if (allowDepthOccluder && depthOccluderMask) {
const targetW = meta0.width ?? depthOccluderMask.width;
const targetH = meta0.height ?? depthOccluderMask.height;
const depthMaskAligned = targetW && targetH
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
if (maskOrig) {
const baseMask: BinaryMask = {
data: maskOrig,
width: maskWidth ?? depthMaskAligned.width,
height: maskHeight ?? depthMaskAligned.height,
};
const baseAligned = (baseMask.width !== depthMaskAligned.width || baseMask.height !== depthMaskAligned.height)
? await resizeBinaryMask(baseMask, depthMaskAligned.width, depthMaskAligned.height)
: baseMask;
const merged = mergeOccluderMasks(baseAligned, depthMaskAligned);
maskOrig = merged.data;
maskWidth = merged.width;
maskHeight = merged.height;
} else {
maskOrig = depthMaskAligned.data;
maskWidth = depthMaskAligned.width;
maskHeight = depthMaskAligned.height;
maskOriginSource = maskOriginSource ?? "depth";
}
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthOccluder: true,
wallDepth: depthWallDepth ?? undefined,
};
}
if (preferSegOccluder && maskOrig && depthOccluderMask && promptTileMask && promptTileWidth && promptTileHeight) {
const forceBandMinCoverage = Math.max(
0,
Math.min(1, Number(process.env.OCCLUDER_SAM_DEPTH_BAND_FORCE_MIN_COVERAGE ?? "0")),
);
const currentCoverage = coverageOfMask(maskOrig);
const forceDepthBand = forceBandMinCoverage > 0 && currentCoverage < forceBandMinCoverage;
if (preferTileBand && !forceDepthBand) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "deferred_tile_band",
...(forceBandMinCoverage > 0 ? {
forceBandMinCoverage: Number(forceBandMinCoverage.toFixed(3)),
currentCoverage: Number(currentCoverage.toFixed(4)),
} : {}),
},
};
} else {
const bandMinHeightFrac = Number(process.env.OCCLUDER_SAM_DEPTH_BAND_MIN_HEIGHT_FRAC ?? "0.25");
const bandMaxHeightFrac = Number(process.env.OCCLUDER_SAM_BAND_MAX_HEIGHT_FRAC ?? "0.6");
const bandMinConf = clamp(
Number(process.env.OCCLUDER_SAM_DEPTH_BAND_MIN_CONF ?? "0.4"),
0,
1,
);
const bandMaxCoverage = clamp(
Number(process.env.OCCLUDER_SAM_DEPTH_BAND_MAX_COVERAGE ?? "0.6"),
0.1,
1,
);
const minHeight = Math.max(4, Math.round(promptTileHeight * bandMinHeightFrac));
const maxHeight = Math.max(minHeight, Math.round(promptTileHeight * bandMaxHeightFrac));
let banded: { mask: Uint8Array; band: Record<string, unknown>; source: string } | null = null;
let rejectedBand: { source: string; confidence: number } | null = null;
const pick = pickTileBandFromSegmentation(
tileSegmentationResult,
promptTileMask,
promptTileWidth,
promptTileHeight,
);
if (pick) {
const pickConf = Number.isFinite(pick.band.confidence)
? Number(pick.band.confidence)
: 0;
if (pickConf >= bandMinConf) {
const normalized = normalizeBandRange(
pick.band.yMin,
pick.band.yMax,
promptTileHeight,
minHeight,
maxHeight,
);
if (normalized) {
banded = {
mask: applyBandToMask(
promptTileMask,
promptTileWidth,
promptTileHeight,
normalized.yMin,
normalized.yMax,
),
band: { ...pick.band, adjusted: normalized.adjusted },
source: pick.source,
};
}
} else {
rejectedBand = { source: pick.source, confidence: pickConf };
}
}
if (!banded) {
const extent = bandFromNonZeroRows(promptTileMask, promptTileWidth, promptTileHeight);
if (extent) {
const normalized = normalizeBandRange(
extent.yMin,
extent.yMax,
promptTileHeight,
minHeight,
maxHeight,
);
if (normalized) {
banded = {
mask: applyBandToMask(
promptTileMask,
promptTileWidth,
promptTileHeight,
normalized.yMin,
normalized.yMax,
),
band: {
yMin: normalized.yMin,
yMax: normalized.yMax,
adjusted: normalized.adjusted,
...(rejectedBand ? { confidence: rejectedBand.confidence } : {}),
},
source: rejectedBand ? "tile_extent_low_conf" : "tile_extent",
};
}
}
}
if (!banded) {
const rowBanded = bandMaskByRowDensity(promptTileMask, promptTileWidth, promptTileHeight);
if (rowBanded.applied && rowBanded.band) {
banded = { mask: rowBanded.mask, band: rowBanded.band, source: "row_density" };
}
}
if (!banded) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "band_unavailable",
bandRejected: rejectedBand ? {
source: rejectedBand.source,
confidence: Number(rejectedBand.confidence.toFixed(3)),
minConfidence: Number(bandMinConf.toFixed(3)),
} : null,
},
};
}
const bandBase = banded?.mask ?? null;
const targetW = meta0.width ?? promptTileWidth ?? depthOccluderMask.width;
const targetH = meta0.height ?? promptTileHeight ?? depthOccluderMask.height;
if (bandBase && targetW && targetH && maskOrig.length === targetW * targetH) {
const bandDilate = Math.max(
0,
Math.round(Number(process.env.OCCLUDER_TILE_BAND_DILATE ?? "6")),
);
const bandMask = bandDilate > 0
? morphDilate(bandBase, promptTileWidth, promptTileHeight, bandDilate)
: bandBase;
const bandCoverage = coverageOfMask(bandMask);
if (bandCoverage > bandMaxCoverage) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "band_too_large",
bandCoverage: Number(bandCoverage.toFixed(4)),
bandMaxCoverage: Number(bandMaxCoverage.toFixed(3)),
bandSource: banded?.source ?? null,
band: banded?.band ?? null,
bandRejected: rejectedBand ? {
source: rejectedBand.source,
confidence: Number(rejectedBand.confidence.toFixed(3)),
minConfidence: Number(bandMinConf.toFixed(3)),
} : null,
},
};
} else {
const depthAligned = (depthOccluderMask.width !== promptTileWidth || depthOccluderMask.height !== promptTileHeight)
? await resizeBinaryMask(depthOccluderMask, promptTileWidth, promptTileHeight)
: depthOccluderMask;
const clipped = maskAnd(depthAligned.data, bandMask);
const beforeCoverage = coverageOfMask(maskOrig);
const merged = maskOr(maskOrig, clipped);
const afterCoverage = coverageOfMask(merged);
maskOrig = merged;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: true,
bandApplied: true,
bandDilate,
bandCoverage: Number(bandCoverage.toFixed(4)),
bandMaxCoverage: Number(bandMaxCoverage.toFixed(3)),
bandSource: banded?.source ?? null,
band: banded?.band ?? null,
bandRejected: rejectedBand ? {
source: rejectedBand.source,
confidence: Number(rejectedBand.confidence.toFixed(3)),
minConfidence: Number(bandMinConf.toFixed(3)),
} : null,
coverageBefore: Number(beforeCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
},
};
if (maskDebugPayload?.originMeta) {
maskDebugPayload.originMeta = maskOriginMeta;
}
}
}
}
const tileUnionMinCoverage = Math.max(
0,
Math.min(1, Number(process.env.OCCLUDER_SAM_DEPTH_TILE_UNION_MIN_COVERAGE ?? "0")),
);
if (tileUnionMinCoverage > 0 && maskOrig && depthOccluderMask) {
const coverageNow = coverageOfMask(maskOrig);
if (coverageNow < tileUnionMinCoverage) {
const targetW = meta0.width ?? promptTileWidth ?? depthOccluderMask.width;
const targetH = meta0.height ?? promptTileHeight ?? depthOccluderMask.height;
if (targetW && targetH && maskOrig.length === targetW * targetH) {
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const tileAlignedMask = (promptTileWidth !== targetW || promptTileHeight !== targetH)
? await resizeBinaryMask(
{ data: promptTileMask, width: promptTileWidth, height: promptTileHeight },
targetW,
targetH,
)
: { data: promptTileMask, width: targetW, height: targetH };
const dilatePx = Math.max(
0,
Math.round(Number(process.env.OCCLUDER_SAM_DEPTH_TILE_UNION_DILATE ?? "2")),
);
const tileMaskDilated = dilatePx > 0
? morphDilate(tileAlignedMask.data, targetW, targetH, dilatePx)
: tileAlignedMask.data;
const clipped = maskAnd(depthAligned.data, tileMaskDilated);
const merged = maskOr(maskOrig, clipped);
const afterCoverage = coverageOfMask(merged);
const maxCoverage = Math.max(
0,
Math.min(1, Number(process.env.OCCLUDER_SAM_DEPTH_TILE_UNION_MAX_COVERAGE ?? "0.6")),
);
if (afterCoverage <= maxCoverage) {
maskOrig = merged;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthTileUnion: {
applied: true,
dilatePx,
coverageBefore: Number(coverageNow.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
minCoverage: Number(tileUnionMinCoverage.toFixed(3)),
maxCoverage: Number(maxCoverage.toFixed(3)),
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthTileUnion: {
applied: false,
reason: "union_too_large",
dilatePx,
coverageBefore: Number(coverageNow.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
minCoverage: Number(tileUnionMinCoverage.toFixed(3)),
maxCoverage: Number(maxCoverage.toFixed(3)),
},
};
}
}
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthTileUnion: {
applied: false,
reason: "coverage_ok",
coverage: Number(coverageNow.toFixed(4)),
minCoverage: Number(tileUnionMinCoverage.toFixed(3)),
},
};
}
}
}
const samPatchEnabled = Number(process.env.OCCLUDER_SAM_PATCH_FNS ?? "0") > 0;
const fnAssistEnabled = (process.env.OCCLUDER_FN_ASSIST_UNION ?? "1") !== "0";
const fnAssistWithoutSam = (process.env.OCCLUDER_FN_ASSIST_WITHOUT_SAM ?? "0") !== "0";
const shouldRunFnAssist = fnAssistEnabled && (samPatchEnabled || fnAssistWithoutSam);
if ((samPatchEnabled || shouldRunFnAssist) && maskOrig && depthOccluderMask) {
const targetW = maskWidth ?? meta0.width ?? depthOccluderMask.width;
const targetH = maskHeight ?? meta0.height ?? depthOccluderMask.height;
if (targetW && targetH && maskOrig.length === targetW * targetH) {
const baseMask: BinaryMask = { data: maskOrig, width: targetW, height: targetH };
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const tileBandOnly = (process.env.OCCLUDER_SAM_PATCH_TILE_BAND_ONLY ?? "1") !== "0";
const tileBandPad = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PATCH_TILE_BAND_PAD ?? "12")));
let tileBandMask: Uint8Array | null = null;
if (tileBandOnly && tileMaskMeta?.tileBand) {
const yMin = Math.max(0, Math.min(targetH - 1, tileMaskMeta.tileBand.yMin - tileBandPad));
const yMax = Math.max(0, Math.min(targetH - 1, tileMaskMeta.tileBand.yMax + tileBandPad));
if (yMax >= yMin) {
tileBandMask = new Uint8Array(targetW * targetH);
for (let y = yMin; y <= yMax; y++) {
const rowStart = y * targetW;
tileBandMask.fill(255, rowStart, rowStart + targetW);
}
}
}
const applyTileBandMask = (mask: Uint8Array) => (tileBandMask ? maskAnd(mask, tileBandMask) : mask);
const invertMask = (input: Uint8Array) => {
const out = new Uint8Array(input.length);
for (let i = 0; i < input.length; i++) out[i] = input[i] > 0 ? 0 : 255;
return out;
};
let fnMask = new Uint8Array(targetW * targetH);
let fnPixels = 0;
let fnSource = "depth";
for (let i = 0; i < fnMask.length; i++) {
if (depthAligned.data[i] > 0 && baseMask.data[i] === 0) {
fnMask[i] = 255;
fnPixels += 1;
}
}
if (tileBandMask) {
fnMask = applyTileBandMask(fnMask);
fnPixels = 0;
for (let i = 0; i < fnMask.length; i++) {
if (fnMask[i] > 0) fnPixels += 1;
}
}
const minFnCoverage = clamp(
Number(process.env.OCCLUDER_SAM_PATCH_MIN_FN_COVERAGE ?? "0.002"),
0,
1,
);
let fnCoverage = fnPixels / Math.max(1, fnMask.length);
const patchRequireTile = (process.env.OCCLUDER_SAM_PATCH_REQUIRE_TILE ?? "0") !== "0";
let tileDilatePx = 0;
let tileMaskDilated: Uint8Array | null = null;
if (promptTileMask && promptTileWidth && promptTileHeight) {
const tileAligned = (promptTileWidth !== targetW || promptTileHeight !== targetH)
? await resizeBinaryMask(
{ data: promptTileMask, width: promptTileWidth, height: promptTileHeight },
targetW,
targetH,
)
: { data: promptTileMask, width: targetW, height: targetH };
tileDilatePx = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PATCH_TILE_DILATE ?? "2")));
tileMaskDilated = tileDilatePx > 0
? morphDilate(tileAligned.data, targetW, targetH, tileDilatePx)
: tileAligned.data;
if (patchRequireTile) {
fnMask = maskAnd(fnMask, tileMaskDilated);
fnPixels = 0;
for (let i = 0; i < fnMask.length; i++) {
if (fnMask[i] > 0) fnPixels += 1;
}
fnCoverage = fnPixels / Math.max(1, fnMask.length);
}
}
const minCompPx = Math.max(50, Number(process.env.OCCLUDER_SAM_PATCH_MIN_COMP ?? "200"));
const maxComps = Math.max(1, Number(process.env.OCCLUDER_SAM_PATCH_MAX_COMP ?? "8"));
const maxCompFrac = clamp(
Number(process.env.OCCLUDER_SAM_PATCH_MAX_COMP_FRAC ?? "0.2"),
0,
1,
);
let baseArea = fnMask.length;
if (tileMaskDilated) {
let tileCount = 0;
for (let i = 0; i < tileMaskDilated.length; i++) {
if (tileMaskDilated[i] > 0) tileCount += 1;
}
if (tileCount > 0) baseArea = tileCount;
}
const maxCompPx = maxCompFrac > 0 ? Math.max(1, Math.round(maxCompFrac * baseArea)) : 0;
const limitFnComponents = (inputMask: Uint8Array) => {
const comps = connectedComponents(inputMask, targetW, targetH, minCompPx);
if (comps.length === 0) {
return new Uint8Array(inputMask);
}
let filtered = comps;
if (maxCompPx > 0) {
filtered = comps.filter((comp) => comp.pixels.length <= maxCompPx);
}
if (filtered.length === 0) {
return new Uint8Array(inputMask.length);
}
filtered.sort((a, b) => b.pixels.length - a.pixels.length);
const limited = filtered.slice(0, maxComps);
const limitedMask = new Uint8Array(inputMask.length);
for (const comp of limited) {
for (const idx of comp.pixels) {
limitedMask[idx] = 255;
}
}
return limitedMask;
};
fnMask = limitFnComponents(fnMask);
fnPixels = 0;
for (let i = 0; i < fnMask.length; i++) {
if (fnMask[i] > 0) fnPixels += 1;
}
fnCoverage = fnPixels / Math.max(1, fnMask.length);
const useBackFallback = (process.env.OCCLUDER_SAM_PATCH_USE_BACK ?? "1") !== "0";
const unionBack = (process.env.OCCLUDER_SAM_PATCH_UNION_BACK ?? "1") !== "0";
if (useBackFallback && depthBackMask) {
const backAligned = (depthBackMask.width !== targetW || depthBackMask.height !== targetH)
? await resizeBinaryMask(depthBackMask, targetW, targetH)
: depthBackMask;
let candidate = invertMask(backAligned.data);
if (patchRequireTile && tileMaskDilated) {
candidate = maskAnd(candidate, tileMaskDilated);
}
const baseInv = invertMask(baseMask.data);
const backCandidate = applyTileBandMask(maskAnd(candidate, baseInv));
if (unionBack) {
fnMask = maskOr(fnMask, backCandidate);
if (fnSource === "depth") fnSource = "depth+back";
} else if (fnCoverage < minFnCoverage) {
fnMask = backCandidate;
fnSource = "back";
}
fnMask = limitFnComponents(fnMask);
fnPixels = 0;
for (let i = 0; i < fnMask.length; i++) {
if (fnMask[i] > 0) fnPixels += 1;
}
fnCoverage = fnPixels / Math.max(1, fnMask.length);
}
if (shouldRunFnAssist) {
const assistMinComp = Math.max(8, Math.round(Number(process.env.OCCLUDER_FN_ASSIST_MIN_COMP ?? "30")));
const assistMaxComp = Math.max(assistMinComp, Math.round(Number(process.env.OCCLUDER_FN_ASSIST_MAX_COMP ?? "6000")));
const assistAdjPx = Math.max(0, Math.round(Number(process.env.OCCLUDER_FN_ASSIST_ADJ_PX ?? "8")));
const assistTileOnly = (process.env.OCCLUDER_FN_ASSIST_TILE_ONLY ?? "1") !== "0";
const baseAdj = assistAdjPx > 0
? morphDilate(maskOrig, targetW, targetH, assistAdjPx)
: maskOrig;
const fnComps = connectedComponents(fnMask, targetW, targetH, assistMinComp);
let assistAdded = 0;
for (const comp of fnComps) {
if (comp.pixels.length < assistMinComp || comp.pixels.length > assistMaxComp) continue;
let touchesAdj = false;
let touchesTile = false;
for (const idx of comp.pixels) {
if (!touchesAdj && baseAdj[idx] > 0) touchesAdj = true;
if (!touchesTile && tileMaskDilated && tileMaskDilated[idx] > 0) touchesTile = true;
if (touchesAdj && (touchesTile || !assistTileOnly)) break;
}
if (!touchesAdj) continue;
if (assistTileOnly && tileMaskDilated && !touchesTile) continue;
for (const idx of comp.pixels) {
if (maskOrig[idx] === 0) {
maskOrig[idx] = 255;
assistAdded += 1;
}
}
}
if (assistAdded > 0) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
fnAssist: {
applied: true,
addedPixels: assistAdded,
minComp: assistMinComp,
maxComp: assistMaxComp,
adjPx: assistAdjPx,
tileOnly: assistTileOnly,
},
};
}
}
if (samPatchEnabled && fnCoverage >= minFnCoverage) {
let patchMask = await runSamHqPatchFromBuffer(raw, fnMask, targetW, targetH);
if (patchMask && patchMask.length === maskOrig.length) {
const patchGrowPx = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PATCH_GROW ?? "10")));
if (patchGrowPx > 0) {
const fnGrow = morphDilate(fnMask, targetW, targetH, patchGrowPx);
patchMask = maskAnd(patchMask, fnGrow);
}
const adjPx = Math.max(0, Math.round(Number(process.env.OCCLUDER_SAM_PATCH_ADJ_PX ?? "12")));
if (adjPx > 0) {
const adjacency = morphDilate(maskOrig, targetW, targetH, adjPx);
const fnAdjacency = morphDilate(fnMask, targetW, targetH, Math.max(1, Math.floor(adjPx / 2)));
const allowed = patchRequireTile && tileMaskDilated
? maskOr(maskOr(adjacency, fnAdjacency), tileMaskDilated)
: maskOr(adjacency, fnAdjacency);
patchMask = maskAnd(patchMask, allowed);
}
const beforeCoverage = coverageOfMask(maskOrig);
const beforePixels = Math.round(beforeCoverage * patchMask.length);
const maxCoverage = clamp(
Number(process.env.OCCLUDER_SAM_PATCH_MAX_COVERAGE ?? "0.55"),
0.1,
1,
);
const maxAddFrac = clamp(
Number(process.env.OCCLUDER_SAM_PATCH_MAX_ADD_FRAC ?? "0.15"),
0,
1,
);
const maxAddedByCoverage = Math.max(0, Math.floor((maxCoverage * patchMask.length) - beforePixels));
const maxAddedByFrac = Math.max(0, Math.floor(maxAddFrac * patchMask.length));
const sampleName = typeof (file as any)?.name === "string"
? String((file as any).name)
: null;
const maxAddPixelsCap = resolveSamPatchMaxAddPixelsCap(sampleName);
const maxAddedPixels = Math.min(maxAddedByCoverage, maxAddedByFrac, maxAddPixelsCap.cap);
const maxAddPixelsCapValue = Number.isFinite(maxAddPixelsCap.cap)
? Math.max(0, Math.floor(maxAddPixelsCap.cap))
: null;
let addedPixels = 0;
for (let i = 0; i < patchMask.length; i++) {
if (patchMask[i] > 0 && maskOrig[i] === 0) addedPixels += 1;
}
let trimIterations = 0;
if (addedPixels > maxAddedPixels && maxAddedPixels > 0) {
let trimmed = patchMask;
let prevTrimmed = patchMask;
let nextAdded = addedPixels;
// Reduce oversized SAM patch gradually instead of hard reject.
while (nextAdded > maxAddedPixels && trimIterations < 10) {
prevTrimmed = trimmed;
trimmed = morphErode(trimmed, targetW, targetH, 1);
trimIterations += 1;
nextAdded = 0;
for (let i = 0; i < trimmed.length; i++) {
if (trimmed[i] > 0 && maskOrig[i] === 0) nextAdded += 1;
}
}
// If one erosion step undershoots heavily, add back a bounded ring so
// maxAddFrac/maxCoverage can steer the final patch size continuously.
if (nextAdded < maxAddedPixels && trimIterations > 0) {
const budget = maxAddedPixels - nextAdded;
if (budget > 0) {
const ring = new Uint8Array(trimmed.length);
for (let i = 0; i < trimmed.length; i++) {
if (prevTrimmed[i] > 0 && trimmed[i] === 0 && maskOrig[i] === 0) ring[i] = 255;
}
let restored = 0;
// Prefer ring pixels that touch trimmed mask to keep contour smooth.
for (let i = 0; i < ring.length && restored < budget; i++) {
if (ring[i] === 0) continue;
const x = i % targetW;
const y = (i / targetW) | 0;
const touching =
(x > 0 && trimmed[i - 1] > 0) ||
(x + 1 < targetW && trimmed[i + 1] > 0) ||
(y > 0 && trimmed[i - targetW] > 0) ||
(y + 1 < targetH && trimmed[i + targetW] > 0);
if (!touching) continue;
trimmed[i] = 255;
restored += 1;
}
// Fill remaining budget from ring if needed.
for (let i = 0; i < ring.length && restored < budget; i++) {
if (ring[i] === 0 || trimmed[i] > 0) continue;
trimmed[i] = 255;
restored += 1;
}
nextAdded += restored;
}
}
patchMask = trimmed;
addedPixels = nextAdded;
}
const merged = maskOr(maskOrig, patchMask);
const afterCoverage = coverageOfMask(merged);
const addFrac = Math.max(0, afterCoverage - beforeCoverage);
if (addedPixels > 0 && afterCoverage <= maxCoverage && addFrac <= maxAddFrac) {
maskOrig = merged;
maskWidth = targetW;
maskHeight = targetH;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPatch: {
applied: true,
fnSource,
fnCoverage: Number(fnCoverage.toFixed(4)),
fnPixels,
addedPixels,
coverageBefore: Number(beforeCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
tileDilatePx,
adjPx,
patchRequireTile,
tileBandOnly,
tileBandPad,
trimIterations,
maxAddedPixels,
maxAddPixelsCap: maxAddPixelsCapValue,
maxAddPixelsScope: maxAddPixelsCap.scope,
maxCoverage,
maxAddFrac: Number(maxAddFrac.toFixed(3)),
},
};
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPatch: {
applied: false,
reason: "patch_too_large",
fnSource,
fnCoverage: Number(fnCoverage.toFixed(4)),
fnPixels,
addedPixels,
coverageBefore: Number(beforeCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
tileDilatePx,
adjPx,
patchRequireTile,
tileBandOnly,
tileBandPad,
trimIterations,
maxAddedPixels,
maxAddPixelsCap: maxAddPixelsCapValue,
maxAddPixelsScope: maxAddPixelsCap.scope,
maxCoverage,
maxAddFrac: Number(maxAddFrac.toFixed(3)),
},
};
}
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPatch: {
applied: false,
reason: "patch_failed",
fnSource,
fnCoverage: Number(fnCoverage.toFixed(4)),
fnPixels,
tileDilatePx,
patchRequireTile,
tileBandOnly,
tileBandPad,
},
};
}
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPatch: {
applied: false,
reason: "fn_too_small",
fnSource,
fnCoverage: Number(fnCoverage.toFixed(4)),
fnPixels,
tileDilatePx,
patchRequireTile,
},
};
}
if (maskDebugPayload?.originMeta) {
maskDebugPayload.originMeta = maskOriginMeta;
}
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPatch: {
applied: false,
reason: "size_mismatch",
},
};
}
}
if (maskOrig && depthOccluderMask && !allowDepthOccluder && !preferSegOccluder) {
try {
const depthRescueEnabled = (process.env.OCCLUDER_DEPTH_RESCUE ?? "1") !== "0";
const onlyLargerThanBase = depthSuppressReasons.length > 0
&& depthSuppressReasons.every((reason) => reason === "larger_than_base");
const minCoverage = Math.max(0, Number(process.env.OCCLUDER_DEPTH_RESCUE_MIN_COVERAGE ?? "0.05"));
const maxCoverage = Math.min(1, Number(process.env.OCCLUDER_DEPTH_RESCUE_MAX_COVERAGE ?? "0.4"));
const depthCovOk = depthOccCoverage != null && depthOccCoverage >= minCoverage && depthOccCoverage <= maxCoverage;
if (depthRescueEnabled && onlyLargerThanBase && depthCovOk) {
const targetW = maskWidth ?? depthOccluderMask.width;
const targetH = maskHeight ?? depthOccluderMask.height;
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const rescueBandPx = Math.max(0, Number(process.env.OCCLUDER_DEPTH_RESCUE_BAND ?? "6"));
const rescueErodePx = Math.max(0, Number(process.env.OCCLUDER_DEPTH_RESCUE_ERODE ?? "1"));
const rescueAdjPx = Math.max(0, Number(process.env.OCCLUDER_DEPTH_RESCUE_ADJ ?? "8"));
const adjacency = rescueAdjPx > 0
? morphDilate(maskOrig, targetW, targetH, rescueAdjPx)
: maskOrig;
const tileRegion = promptTileMask && samPromptedReady
? morphDilate(promptTileMask, targetW, targetH, rescueBandPx)
: null;
const rescueRegion = tileRegion ? maskOr(tileRegion, adjacency) : adjacency;
const rescueSupport = rescueErodePx > 0
? morphErode(depthAligned.data, targetW, targetH, rescueErodePx)
: depthAligned.data;
const minArea = Math.max(200, Number(process.env.OCCLUDER_DEPTH_RESCUE_MIN_AREA ?? "1200"));
const components = connectedComponents(rescueSupport, targetW, targetH, minArea);
const merged = new Uint8Array(maskOrig);
let rescuePixels = 0;
let mergedComponents = 0;
const limit = Math.min(merged.length, rescueRegion.length);
for (const comp of components) {
let touches = false;
for (const idx of comp.pixels) {
if (idx < limit && rescueRegion[idx]) {
touches = true;
break;
}
}
if (!touches) continue;
mergedComponents += 1;
for (const idx of comp.pixels) {
if (idx < limit && !merged[idx]) {
merged[idx] = 255;
rescuePixels += 1;
}
}
}
if (rescuePixels > 0) {
maskOrig = merged;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthRescue: {
pixels: rescuePixels,
coverage: Number((rescuePixels / merged.length).toFixed(6)),
bandPx: rescueBandPx,
erodePx: rescueErodePx,
adjPx: rescueAdjPx,
minArea,
components: mergedComponents,
},
};
if (debug) {
console.info("[mask] depth rescue merged", {
pixels: rescuePixels,
coverage: Number((rescuePixels / merged.length).toFixed(6)),
components: mergedComponents,
});
}
}
}
} catch (err) {
if (debug) console.warn("[mask] depth rescue failed", err);
}
}
}
let rawMaskSnapshot: Uint8Array | null = null;
let rawMaskDiagnostics: RawMaskDiagnostics | null = null;
if (maskOrig) {
if (!maskOriginSource) {
maskOriginSource = 'segmentation';
}
maskBoostStats = boostSegMaskInPlace(maskOrig);
if (debug && maskBoostStats?.changed) {
console.info("[mask] boosted", {
coverage: maskBoostStats.coverage.toFixed(4),
gamma: maskBoostStats.gamma,
lo: maskBoostStats.lo,
hi: maskBoostStats.hi,
});
}
if (!maskOriginMeta && maskBoostStats) {
maskOriginMeta = {
boostCoverage: Number(maskBoostStats.coverage.toFixed(4)),
gamma: maskBoostStats.gamma,
lo: maskBoostStats.lo,
hi: maskBoostStats.hi,
};
}
maskForWeights = maskOrig;
rawMaskSnapshot = Uint8Array.from(maskOrig);
if (samPromptedDebug) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
samPrompted: samPromptedDebug,
};
}
}
let normalizationOverrides: Partial<NormalizeOptions> | undefined;
let segmentationCoverage: number | null = null;
if (maskForWeights && meta0.width && meta0.height) {
try {
const maskW = meta0.width;
const maskH = meta0.height;
const grayBytes = new Uint8Array(
await sharp(raw).removeAlpha().toColorspace('b-w').raw().toBuffer()
);
intensityBytes = grayBytes;
maskWidth = maskW;
maskHeight = maskH;
const segCoverage = coverageOfMask(maskForWeights);
segmentationCoverage = Number(segCoverage.toFixed(6));
if (maskOriginMeta) {
maskOriginMeta = {
...maskOriginMeta,
segCoverage: Number(segCoverage.toFixed(4)),
};
}
if (Number.isFinite(segCoverage) && segCoverage >= 0.02 && segCoverage <= 0.7) {
normalizationOverrides = {
invertMode: 'false',
removeBorder: false,
thinWidth: 0,
fallbackLow: 0.002,
};
}
const samSource = Boolean(
(maskOriginSource && maskOriginSource.includes("sam")) ||
(maskOriginMeta && (maskOriginMeta as any).sam)
);
if (samSource) {
normalizationOverrides = {
...(normalizationOverrides ?? {}),
invertMode: 'false',
thresholdMode: 'fixed',
thresholdValue: 1,
removeBorder: false,
thinWidth: 0,
minArea: 1,
keepLargest: 0,
gridKill: 'off',
edgeFill: { enabled: false, threshold: 0, dilate: 0 },
edgeGuard: { enabled: false, threshold: 0, dilate: 0 },
seedGrow: 0,
carveRadius: 0,
morphOps: [],
bgMode: 'classic',
fallbackLow: 0,
fallbackHigh: 1,
};
}
const primary = normalizeWithPreset({
mask: maskForWeights,
width: maskW,
height: maskH,
gray: grayBytes,
preset: presetRequested,
overrides: normalizationOverrides,
});
let chosen = primary;
const primaryInterior = primary.result.interiorCoverage ?? 0;
const primaryLeak = primary.result.leakPct ?? primary.result.leakAfter ?? 1;
const qualityGood = primaryInterior >= 0.90 && primaryLeak <= 0.10;
const primaryCov = primary.result.coverage ?? 0;
const shouldForceFallback = (fallbackThreshold > 0)
&& (primaryCov < fallbackThreshold)
&& (primaryInterior < 0.85);
if (shouldForceFallback && !qualityGood && primary.preset !== 'tiles-detail') {
const detail = normalizeWithPreset({
mask: maskForWeights,
width: maskW,
height: maskH,
gray: grayBytes,
preset: 'tiles-detail',
overrides: normalizationOverrides,
});
if (detail.result.coverage > primary.result.coverage + 0.005) {
chosen = detail;
}
}
maskPresetApplied = chosen.preset;
maskNormalized = Uint8Array.from(chosen.result.data);
maskForWeights = maskNormalized;
normalizationPrimary = primary;
normalizationChosen = chosen;
if (maskNormalized && maskWidth && maskHeight) {
const boost = buildYoloBoostMask(maskNormalized, maskOrig ?? null, maskWidth, maskHeight, maskOriginMeta);
if (boost) {
maskNormalized = boost.mask;
maskForWeights = maskNormalized;
maskOriginMeta = {
...(maskOriginMeta ?? {}),
yoloBoost: {
applied: boost.applied,
coverage: Number(boost.coverage.toFixed(6)),
},
};
if (normalizationChosen?.result) {
normalizationChosen.result.data = Uint8Array.from(maskNormalized);
normalizationChosen.result.coverage = boost.coverage;
if (normalizationChosen.result.stats?.background) {
normalizationChosen.result.stats.background.occluderCoverage = boost.coverage;
}
if (normalizationChosen.result.stats?.hybrid) {
normalizationChosen.result.stats.hybrid.seedCoverage = boost.coverage;
}
}
}
}
maskNormalizedInitial = Uint8Array.from(maskNormalized);
maskNormalizedInitialCoverage = coverageOfMask(maskNormalized);
const normalizedStats = chosen.result.stats;
const backgroundStats = normalizedStats.background ?? null;
if (maskNormalized && maskWidth && maskHeight) {
const meanAlpha = coverageOfMask(maskNormalized);
const stdEstimate = normalizedStats.hybrid?.std ?? normalizedStats.hybrid?.localStdThresh ?? 0;
if (meanAlpha < 0.22 && stdEstimate > 0 && stdEstimate < 25) {
maskNormalized = morphDilate(maskNormalized, maskWidth, maskHeight, 1);
maskForWeights = maskNormalized;
maskNormalizedInitial = Uint8Array.from(maskNormalized);
maskNormalizedInitialCoverage = coverageOfMask(maskNormalized);
}
}
maskCoverage = maskNormalized ? coverageOfMask(maskNormalized) : chosen.result.coverage;
maskBackgroundCoverage = backgroundStats?.coverage ?? null;
maskOccluderCoverage = backgroundStats?.occluderCoverage ?? null;
maskSeedCoverage = normalizedStats.hybrid?.seedCoverage ?? null;
} catch (normErr) {
if (debug) {
console.warn('[mask] normalization failed', normErr);
}
}
}
if (rawMaskSnapshot && meta0.width && meta0.height) {
const encodedRaw = await maskToDataUrlWithCheck(rawMaskSnapshot, meta0.width, meta0.height);
maskRawDataUrl = encodedRaw.dataUrl;
rawMaskDiagnostics = encodedRaw.diagnostics;
if (!finalMaskDataUrl || preferSegOccluder) {
finalMaskDataUrl = maskRawDataUrl;
finalMaskMeta = {
polarity: 'white',
source: maskOriginSource ?? undefined,
stats: maskOriginMeta ?? undefined,
};
}
}
// (B) Downscale Arbeitsbild
const maxSide = 1200;
let work = sharp(raw).removeAlpha();
if (Math.max(W,H) > maxSide) {
const scale = maxSide / Math.max(W,H);
W = Math.max(1, Math.round((meta0.width ?? W) * scale));
H = Math.max(1, Math.round((meta0.height ?? H) * scale));
work = work.resize(W, H, { fit:"inside" });
}
// (C) Maske auf Arbeitsauflösung + Feather -> Gewichte
let weights: Float32Array | null = null;
if (useSeg && (maskForWeights || maskOrig)) {
const maskSourceForWeights = maskForWeights ?? maskOrig!;
const maskResizedBuf = await sharp(Buffer.from(maskSourceForWeights), {
raw: { width: meta0.width!, height: meta0.height!, channels: 1 }
})
.resize(W, H, { kernel: "lanczos3" })
.blur(maskBlur)
.toColorspace("b-w")
.raw()
.toBuffer();
const m = new Uint8Array(maskResizedBuf);
weights = new Float32Array(W*H);
const a = maskWeight;
for (let i=0; i<m.length; i++){
const mi = m[i] / 255; // 0..1 (1=Occluder)
weights[i] = 1 - a * mi; // 1..(1-a)
}
}
// (D) Kanten + gewichtete Projektionen
const rgba = await work.raw().ensureAlpha().toBuffer({ resolveWithObject:true });
const w = rgba.info.width, h = rgba.info.height;
const gray = toGrayFloat32(rgba.data, w, h);
const mag = sobelMag(gray, w, h);
const projX = projectXWeighted(mag, weights, w, h);
const projY = projectYWeighted(mag, weights, w, h);
const acX = autoCorr1D(projX.slice());
const acY = autoCorr1D(projY.slice());
const perX = bestPeriod(acX, Math.max(12, Math.floor(w/100)), Math.max(24, Math.floor(w/3)));
const perY = bestPeriod(acY, Math.max(12, Math.floor(h/100)), Math.max(24, Math.floor(h/3)));
const phaseX = phaseForPeriod(projX, perX);
const phaseY = phaseForPeriod(projY, perY);
const grout = Math.max(1, Math.round(0.025 * ((perX + perY) / 2)));
const vlines:number[] = [];
for (let x = phaseX; x < w; x += perX || w) vlines.push(x);
const hlines:number[] = [];
for (let y = phaseY; y < h; y += perY || h) hlines.push(y);
// In Original-Pixel skalieren
const scaleBackX = (meta0.width ?? w) / w;
const scaleBackY = (meta0.height ?? h) / h;
const widthPx = meta0.width ?? w;
const heightPx = meta0.height ?? h;
const scaleLines = (values: number[], scale: number, extent: number) => {
const acc = new Set<number>();
const maxIdx = Math.max(0, extent - 1);
for (const v of values) {
const scaled = clamp(Math.round(v * scale), 0, maxIdx);
acc.add(scaled);
}
return Array.from(acc).sort((a, b) => a - b);
};
const tile_w_px = Math.round(perX * scaleBackX);
const tile_h_px = Math.round(perY * scaleBackY);
const grout_px = Math.round(grout * Math.sqrt(scaleBackX*scaleBackY));
let gridV = scaleLines(vlines, scaleBackX, widthPx);
let gridH = scaleLines(hlines, scaleBackY, heightPx);
const median = (values: number[]): number => {
if (!values.length) return 0;
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
};
const consecutiveDiffs = (values: number[]): number[] => {
const diffs: number[] = [];
for (let i = 1; i < values.length; i++) {
diffs.push(values[i] - values[i - 1]);
}
return diffs;
};
const safePeriod = (fallback: number, values: number[]): number => {
if (fallback && fallback > 0) return fallback;
const clean = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
const med = median(consecutiveDiffs(clean));
return med > 0 ? med : 0;
};
const computeCoverage = (tileW: number, tileH: number, linesV: number[], linesH: number[]) => {
const TvLocal = safePeriod(tileW, linesV);
const ThLocal = safePeriod(tileH, linesH);
const expectedVLocal = TvLocal > 0 && widthPx > 0 ? Math.max(1, Math.floor(widthPx / TvLocal) + 1) : linesV.length || 1;
const expectedHLocal = ThLocal > 0 && heightPx > 0 ? Math.max(1, Math.floor(heightPx / ThLocal) + 1) : linesH.length || 1;
const covV = Math.min(1, expectedVLocal ? linesV.length / expectedVLocal : 0);
const covH = Math.min(1, expectedHLocal ? linesH.length / expectedHLocal : 0);
return { Tv: TvLocal, Th: ThLocal, coverage: Math.min(covV, covH) };
};
const primaryCoverage = computeCoverage(tile_w_px, tile_h_px, gridV, gridH);
const coverage = primaryCoverage.coverage;
const periodicRaw = runPeriodicAnalysis(gray, w, h, { debug, weights: weights ?? null });
let periodicCandidate: null | {
tile_w_px: number;
tile_h_px: number;
grout_px: number;
gridV: number[];
gridH: number[];
coverage: number;
confidence: number;
source: string;
raw: typeof periodicRaw;
} = null;
if (periodicRaw.ok) {
const altGridV = scaleLines(periodicRaw.vlines, scaleBackX, widthPx);
const altGridH = scaleLines(periodicRaw.hlines, scaleBackY, heightPx);
const tileWAlt = Math.max(1, Math.round(periodicRaw.tile_w_px * scaleBackX));
const tileHAlt = Math.max(1, Math.round(periodicRaw.tile_h_px * scaleBackY));
const groutAlt = Math.max(1, Math.round(periodicRaw.grout_px * Math.sqrt(scaleBackX * scaleBackY)));
const coverageAltStats = computeCoverage(tileWAlt, tileHAlt, altGridV, altGridH);
const coverageAlt = coverageAltStats.coverage;
periodicCandidate = {
tile_w_px: tileWAlt,
tile_h_px: tileHAlt,
grout_px: groutAlt,
gridV: altGridV,
gridH: altGridH,
coverage: coverageAlt,
confidence: periodicRaw.confidence ?? 0,
source: periodicRaw.source ?? "cv@fft-hough",
raw: periodicRaw,
};
if (debug) {
console.log("periodic-analysis", {
tile_w_px: tileWAlt,
tile_h_px: tileHAlt,
grout_px: groutAlt,
coverage: coverageAlt,
confidence: periodicCandidate.confidence,
});
}
}
type RansacMetaType = ReturnType<typeof refineGridRansac>["meta"];
type RansacInfo = {
mode: string;
coverage: number;
v?: RansacMetaType["v"];
h?: RansacMetaType["h"];
};
type ApplyRansacResult = {
vlines: number[];
hlines: number[];
meta?: RansacMetaType;
info: RansacInfo;
coverage: number;
};
const applyRansac = (
label: string,
tileW: number,
tileH: number,
linesV: number[],
linesH: number[],
coverageSeed: number,
): ApplyRansacResult => {
let v = [...linesV];
let h = [...linesH];
let meta: RansacMetaType | undefined;
let info: RansacInfo = { mode: "off", coverage: coverageSeed };
const shouldAutoCandidate = ransacAuto && coverageSeed < 0.6;
if ((ransacForce || shouldAutoCandidate) && widthPx > 0 && heightPx > 0) {
try {
const refined = refineGridRansac(widthPx, heightPx, v, h);
const confV = refined.meta?.v?.conf ?? 0;
const confH = refined.meta?.h?.conf ?? 0;
const confOK = confV >= 0.5 && confH >= 0.5;
if (ransacForce || confOK) {
v = refined.vlines;
h = refined.hlines;
meta = refined.meta;
info = { mode: ransacForce ? "force" : "auto", coverage: coverageSeed, ...refined.meta };
} else {
info = { mode: "auto-skip", coverage: coverageSeed, ...refined.meta };
}
} catch (refineErr) {
if (debug) {
console.warn(`RANSAC ${label} failed:`, refineErr);
}
info = { mode: ransacForce ? "force-error" : "auto-error", coverage: coverageSeed };
}
}
const postCoverage = computeCoverage(tileW, tileH, v, h);
info = { ...info, coverage: postCoverage.coverage };
return { vlines: v, hlines: h, meta, info, coverage: postCoverage.coverage };
};
const currentRansac = applyRansac("projection", tile_w_px, tile_h_px, gridV, gridH, coverage);
gridV = currentRansac.vlines;
gridH = currentRansac.hlines;
type RansacMeta = RansacMetaType;
type Candidate = {
label: 'projection' | 'periodic';
tile_w_px: number;
tile_h_px: number;
grout_px: number;
grid: { vlines: number[]; hlines: number[] };
coverage: number;
confidence: number | null;
source: string;
ransac: RansacInfo;
ransacMeta?: RansacMeta;
};
const projectionCandidate: Candidate = {
label: 'projection',
tile_w_px,
tile_h_px,
grout_px,
grid: { vlines: gridV, hlines: gridH },
coverage: currentRansac.coverage,
confidence: null,
source: `${(useSeg && maskOrig) ? "seg@u2net -> weighted sobel+projection" : "sobel+projection"}${currentRansac.meta ? " + ransac1d" : ""}`,
ransac: currentRansac.info,
ransacMeta: currentRansac.meta,
};
let periodicResult: Candidate | null = null;
if (periodicCandidate) {
const periodicRansac = applyRansac("periodic", periodicCandidate.tile_w_px, periodicCandidate.tile_h_px, periodicCandidate.gridV, periodicCandidate.gridH, periodicCandidate.coverage);
periodicResult = {
label: 'periodic',
tile_w_px: periodicCandidate.tile_w_px,
tile_h_px: periodicCandidate.tile_h_px,
grout_px: periodicCandidate.grout_px,
grid: { vlines: periodicRansac.vlines, hlines: periodicRansac.hlines },
coverage: periodicRansac.coverage,
confidence: periodicCandidate.confidence ?? null,
source: `${periodicCandidate.source}${periodicRansac.meta ? " + ransac1d" : ""}`,
ransac: periodicRansac.info,
ransacMeta: periodicRansac.meta,
};
}
let best: Candidate = projectionCandidate;
let method: "projection" | "periodic" = "projection";
if (periodicResult) {
const coverageGain = periodicResult.coverage - projectionCandidate.coverage;
const periodicConf = periodicResult.confidence ?? 0;
const projectionWeak = projectionCandidate.coverage < 0.45;
const periodicStrong = periodicResult.coverage >= 0.6 && periodicConf >= 0.35;
const confidenceStrong = periodicConf >= 0.55 && coverageGain > 0.05;
if ((periodicStrong && coverageGain > 0.12) || confidenceStrong || (projectionWeak && coverageGain > 0.05)) {
best = periodicResult;
method = "periodic";
}
}
const analysis = {
method,
candidates: {
projection: {
tile_w_px: projectionCandidate.tile_w_px,
tile_h_px: projectionCandidate.tile_h_px,
grout_px: projectionCandidate.grout_px,
coverage: projectionCandidate.coverage,
confidence: projectionCandidate.confidence,
source: projectionCandidate.source,
ransac: projectionCandidate.ransac,
},
...(periodicResult ? {
periodic: {
tile_w_px: periodicResult.tile_w_px,
tile_h_px: periodicResult.tile_h_px,
grout_px: periodicResult.grout_px,
coverage: periodicResult.coverage,
confidence: periodicResult.confidence,
source: periodicResult.source,
ransac: periodicResult.ransac,
},
} : {}),
},
};
const finalTileW = best.tile_w_px;
const finalTileH = best.tile_h_px;
const finalGrout = best.grout_px;
const finalGrid = best.grid;
const source = best.source;
const ransacInfo = best.ransac;
if (
maskNormalized &&
maskPresetApplied &&
intensityBytes &&
normalizationChosen &&
normalizationChosen.result?.stats
) {
gridContext = finalGrid.vlines.length || finalGrid.hlines.length ? {
width: maskWidth ?? meta0.width ?? intensityBytes.length,
height: maskHeight ?? meta0.height ?? intensityBytes.length,
vlines: finalGrid.vlines,
hlines: finalGrid.hlines,
groutPx: finalGrout ?? 0,
tileWidthPx: finalTileW ?? 0,
tileHeightPx: finalTileH ?? 0,
} : null;
const pipelineResult = runMaskPipeline({
normalizedMask: maskNormalized,
normalizedCoverage: maskNormalizedInitialCoverage ?? maskCoverage ?? 0,
stats: normalizationChosen.result.stats,
intensity: intensityBytes,
width: maskWidth ?? meta0.width ?? intensityBytes.length,
height: maskHeight ?? meta0.height ?? intensityBytes.length,
grid: gridContext ?? undefined,
preset: maskPresetApplied,
u2BoostMask: maskOrig ? thresholdMask(maskOrig, 230) ?? undefined : undefined,
flags,
});
pipelineWarnings = Array.isArray(pipelineResult.debug?.coverage?.warnings)
? pipelineResult.debug.coverage.warnings.filter((w): w is string => typeof w === 'string')
: undefined;
const pipelineMaskBase = Uint8Array.from(pipelineResult.mask);
const pipelineCoverageAfter = pipelineResult.coverage ?? 0;
const pipelineLeakAfter = pipelineResult.leak ?? 0;
const pipelineInteriorBefore = pipelineResult.interiorCoverage ?? 0;
const coverageBefore = maskNormalizedInitialCoverage ?? maskCoverage ?? pipelineCoverageAfter;
let interiorAfter = pipelineInteriorBefore;
const leakBefore = pipelineLeakAfter;
// Apply seed-driven component filtering and lattice-aware leak suppression
let filteredMask = Uint8Array.from(pipelineResult.mask);
let seedOverlap = 0;
let u2Seeds = maskOrig && gridContext ? thresholdMask(maskOrig, 150) : null;
if (u2Seeds && maskWidth && maskHeight && maskOccluderCoverage != null && maskOccluderCoverage > 0.25) {
u2Seeds = morphDilate(u2Seeds, maskWidth, maskHeight, 1);
}
const seedCoverage = u2Seeds ? coverageOfMask(u2Seeds) : 0;
if (u2Seeds && gridContext && maskWidth && maskHeight) {
// Step 1: Keep only seed-driven components (lenient parameters)
const compFiltered = keepSeedDrivenComponents(
pipelineResult.mask,
u2Seeds,
maskWidth,
maskHeight,
0.08, // minIoU
400 // minArea
);
const compFilteredCoverage = coverageOfMask(compFiltered);
// Airbag: If component filtering is too aggressive, union with seeds
if (compFilteredCoverage < 0.75 * coverageBefore) {
// Union compFiltered with seeds to prevent collapse
for (let i = 0; i < compFiltered.length; i++) {
if (u2Seeds[i]) compFiltered[i] = 255;
}
}
// Step 2: Suppress tile-interior leak with hysteresis
filteredMask = suppressTileLeak(
compFiltered,
maskOrig, // probability map
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
gridContext.groutPx,
148 // threshold ~0.58
);
// Airbag: Pipeline must never be thinner than 60% of seeds
const filteredCoverage = coverageOfMask(filteredMask);
if (filteredCoverage < 0.60 * seedCoverage) {
// Union with seeds to guarantee >= seed coverage
for (let i = 0; i < filteredMask.length; i++) {
if (u2Seeds[i]) filteredMask[i] = 255;
}
}
}
// Measure filtered mask metrics
const gridMasks = gridContext && maskWidth && maskHeight
? buildGridMasks(maskWidth, maskHeight, gridContext.vlines, gridContext.hlines, gridContext.groutPx)
: null;
const coreBandMask = gridContext && maskWidth && maskHeight
? buildGridMasks(
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
Math.max(1, Math.round((gridContext.groutPx ?? 3) / 2)),
).lineMask
: null;
let leakAfterClamp1: number | null = null;
let leakAfterReinject: number | null = null;
let leakAfterFinal: number | null = null;
let leakAfter = gridMasks ? leakRatio(filteredMask, gridMasks.lineMask) : leakBefore;
let coverageAfter = coverageOfMask(filteredMask);
interiorAfter = gridMasks
? coverageWithinMask(filteredMask, gridMasks.interiorMask)
: pipelineInteriorBefore;
let acceptanceOverride: string | undefined;
if (leakAfter > 0.09 && maskOrig && gridContext && maskWidth && maskHeight) {
const clampBandPx = Math.max(3, (gridContext.groutPx ?? 3) + 1);
const baseMask = Uint8Array.from(filteredMask);
const tightened = suppressTileLeak(
baseMask,
maskOrig,
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
clampBandPx,
168,
);
const lineMask = gridMasks?.lineMask ?? null;
if (lineMask) {
for (let i = 0; i < tightened.length; i++) {
if (lineMask[i]) tightened[i] = 0;
}
}
const bandMask = lineMask ? morphDilate(lineMask, maskWidth, maskHeight, 1) : null;
let clampDebug: {
lostPixels: number;
restoredPixels: number;
lostComponents: number;
restoredComponents: number;
samples: Array<{ area: number; meanDist: number; detFrac: number; seedFrac: number; touchesSafe: boolean; restored: boolean }>;
} | null = null;
if (bandMask) {
const lost = new Uint8Array(tightened.length);
let lostCount = 0;
for (let i = 0; i < tightened.length; i++) {
if (baseMask[i] && !tightened[i] && bandMask[i]) {
lost[i] = 255;
lostCount += 1;
}
}
if (lostCount > 0) {
clampDebug = {
lostPixels: lostCount,
restoredPixels: 0,
lostComponents: 0,
restoredComponents: 0,
samples: [],
};
const distMap = distanceToNearestLattice(
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
clampBandPx,
);
const adjacencyMask = Uint8Array.from(tightened);
const comps = connectedComponents(lost, maskWidth, maskHeight, 24);
clampDebug.lostComponents = comps.length;
for (const comp of comps) {
const area = comp.area;
if (area < 32) continue;
const aspect = componentAspectRatio(comp);
const extentX = componentExtentX(comp);
const varianceYNorm = componentVarianceY(comp, maskWidth);
const textileWidth = Math.max(180, Math.round(maskWidth * 0.18));
const isTextileShape = aspect > 2.2 && extentX >= textileWidth && varianceYNorm < 0.6;
const coreTouch = componentTouchesMask(comp, coreBandMask);
const coreFrac = componentOverlapFraction(comp, coreBandMask);
let sumDist = 0;
let detSupport = 0;
let seedSupport = 0;
let touchesSafe = false;
for (const idx of comp.pixels) {
sumDist += distMap[idx];
if (maskOrig[idx] >= 200) detSupport += 1;
if (u2Seeds && u2Seeds[idx]) seedSupport += 1;
if (!touchesSafe) {
const x = idx % maskWidth;
const y = (idx / maskWidth) | 0;
if (
(x > 0 && adjacencyMask[idx - 1]) ||
(x + 1 < maskWidth && adjacencyMask[idx + 1]) ||
(y > 0 && adjacencyMask[idx - maskWidth]) ||
(y + 1 < maskHeight && adjacencyMask[idx + maskWidth])
) {
touchesSafe = true;
}
}
}
const meanDist = sumDist / area;
const detFrac = detSupport / area;
const seedFrac = seedSupport / Math.max(1, area);
const hasStrongSupport = detFrac >= 0.45 || seedFrac >= 0.45;
if (coreTouch && meanDist < 0.4) continue;
if (coreTouch && meanDist < 0.9 && !hasStrongSupport) continue;
if (coreFrac > 0.05) continue;
const areaThreshold = isTextileShape ? 380 : 140;
if (area < areaThreshold && !hasStrongSupport) continue;
let shouldRestore = false;
if (touchesSafe && meanDist >= 1.0) shouldRestore = true;
if (!shouldRestore && detFrac >= 0.30 && area >= 150 && meanDist >= 0.8) shouldRestore = true;
if (!shouldRestore && seedFrac >= 0.25 && area >= 150 && meanDist >= 0.8) shouldRestore = true;
if (!shouldRestore && (detFrac >= 0.55 || seedFrac >= 0.55) && meanDist >= 0.6) shouldRestore = true;
if (!shouldRestore && hasStrongSupport && area >= 800 && coreFrac <= 0.04) shouldRestore = true;
if (clampDebug && clampDebug.samples.length < 25) {
clampDebug.samples.push({
area,
meanDist: Number(meanDist.toFixed(3)),
detFrac: Number(detFrac.toFixed(3)),
seedFrac: Number(seedFrac.toFixed(3)),
touchesSafe,
restored: shouldRestore,
});
}
if (!shouldRestore) continue;
for (const idx of comp.pixels) {
const allowSeed = u2Seeds && u2Seeds[idx];
const allowProb = maskOrig[idx] >= 215;
const allow = allowProb || allowSeed;
if (distMap[idx] < 1 && !allow) continue;
const onLine = lineMask ? lineMask[idx] !== 0 : false;
if (onLine && !allow) continue;
if (!allow && distMap[idx] < 2) continue;
tightened[idx] = 255;
}
if (clampDebug) {
clampDebug.restoredComponents += 1;
clampDebug.restoredPixels += area;
}
}
}
}
if (u2Seeds && maskOrig) {
for (let i = 0; i < tightened.length; i++) {
if (!u2Seeds[i]) continue;
const allowProb = maskOrig[i] >= 210;
if (lineMask && lineMask[i] && !allowProb) continue;
tightened[i] = 255;
}
}
const tightenedCoverage = coverageOfMask(tightened);
const tightenedLeak = gridMasks ? leakRatio(tightened, gridMasks.lineMask) : leakAfter;
const tightenedInterior = gridMasks ? coverageWithinMask(tightened, gridMasks.interiorMask) : interiorAfter;
const leakThreshold = Math.min(0.08, leakAfter - 0.01);
if (tightenedLeak <= leakThreshold && tightenedCoverage >= coverageAfter * 0.6) {
filteredMask = tightened;
coverageAfter = tightenedCoverage;
leakAfter = tightenedLeak;
interiorAfter = tightenedInterior;
leakAfterClamp1 = tightenedLeak;
if (clampDebug) {
if (pipelineResult.debug?.leakClamp && typeof pipelineResult.debug.leakClamp === 'object') {
(pipelineResult.debug.leakClamp as any).occluder = clampDebug;
} else {
pipelineResult.debug.leakClamp = { enabled: true, ...clampDebug };
}
}
}
}
// Occluder reinjection using seeds / detector when clamp clears grout
if (maskOrig && gridContext && maskWidth && maskHeight && gridMasks) {
const bandPx = Math.max(3, (gridContext.groutPx ?? 3) + 1);
const bandMask = morphDilate(gridMasks.lineMask, maskWidth, maskHeight, 1);
const lostBand = new Uint8Array(filteredMask.length);
let lostBandCount = 0;
for (let i = 0; i < filteredMask.length; i++) {
if ((pipelineMaskBase[i] && !filteredMask[i]) && bandMask[i]) {
lostBand[i] = 255;
lostBandCount += 1;
}
}
if (lostBandCount > 0) {
const distMap = distanceToNearestLattice(
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
bandPx,
);
const adjacencyMask = Uint8Array.from(filteredMask);
const comps = connectedComponents(lostBand, maskWidth, maskHeight, 24);
const reinjectBase = Uint8Array.from(filteredMask);
const reinject = new Uint8Array(filteredMask);
for (const comp of comps) {
const area = comp.area;
if (area < 120) continue;
const aspect = componentAspectRatio(comp);
const extentX = componentExtentX(comp);
const varianceYNorm = componentVarianceY(comp, maskWidth);
const textileWidth = Math.max(160, Math.round(maskWidth * 0.16));
const isTextileShape = aspect > 2.2 && extentX >= textileWidth && varianceYNorm < 0.55;
const coreTouch = componentTouchesMask(comp, coreBandMask);
const coreFrac = componentOverlapFraction(comp, coreBandMask);
let sumDist = 0;
let detSupport = 0;
let seedSupport = 0;
let touchesSafe = false;
for (const idx of comp.pixels) {
sumDist += distMap[idx];
if (maskOrig[idx] >= 200) detSupport += 1;
if (u2Seeds && u2Seeds[idx]) seedSupport += 1;
if (!touchesSafe) {
const x = idx % maskWidth;
const y = (idx / maskWidth) | 0;
if (
(x > 0 && adjacencyMask[idx - 1]) ||
(x + 1 < maskWidth && adjacencyMask[idx + 1]) ||
(y > 0 && adjacencyMask[idx - maskWidth]) ||
(y + 1 < maskHeight && adjacencyMask[idx + maskWidth])
) {
touchesSafe = true;
}
}
}
const meanDist = sumDist / area;
const detFrac = detSupport / Math.max(1, area);
const seedFrac = seedSupport / Math.max(1, area);
const hasStrongSupport = detFrac >= 0.5 || seedFrac >= 0.5;
if (coreTouch && meanDist < 0.45) continue;
if (coreTouch && meanDist < 0.9 && !hasStrongSupport) continue;
if (coreFrac > 0.05) continue;
let shouldRestore = false;
const areaThreshold = isTextileShape ? 420 : 180;
if (area < areaThreshold && !hasStrongSupport) continue;
if (seedFrac >= 0.30 && meanDist >= 0.85) shouldRestore = true;
if (!shouldRestore && detFrac >= 0.24 && area >= 170 && meanDist >= 0.85) shouldRestore = true;
if (!shouldRestore && touchesSafe && meanDist >= 1.2) shouldRestore = true;
if (!shouldRestore && hasStrongSupport && meanDist >= 0.9) shouldRestore = true;
if (!shouldRestore && hasStrongSupport && area >= 800 && coreFrac <= 0.04) shouldRestore = true;
if (!shouldRestore) continue;
for (const idx of comp.pixels) {
const allowSeed = u2Seeds && u2Seeds[idx];
const allowProb = maskOrig[idx] >= 215;
const allowClose = allowProb || allowSeed;
const onLine = gridMasks.lineMask[idx] !== 0;
if (distMap[idx] < 0.5 && !allowClose) continue;
if (onLine && !allowClose) continue;
if (!allowClose && maskOrig[idx] < 200) continue;
reinject[idx] = 255;
}
}
const priorCoverage = coverageAfter;
const priorLeak = leakAfter;
const priorInterior = interiorAfter;
filteredMask = reinject;
coverageAfter = coverageOfMask(filteredMask);
leakAfter = leakRatio(filteredMask, gridMasks.lineMask);
interiorAfter = coverageWithinMask(filteredMask, gridMasks.interiorMask);
const leakGuard = Math.min(0.09, priorLeak + 0.015);
if (leakAfter > leakGuard) {
filteredMask = reinjectBase;
coverageAfter = priorCoverage;
leakAfter = priorLeak;
interiorAfter = priorInterior;
} else {
leakAfterReinject = leakAfter;
}
}
}
if (u2Seeds && maskOrig && gridContext && maskWidth && maskHeight) {
const lineMask = gridMasks?.lineMask ?? null;
const distSeeds = distanceToNearestLattice(
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
Math.max(3, (gridContext.groutPx ?? 3) + 1),
);
const priorMask = Uint8Array.from(filteredMask);
const priorCoverage = coverageAfter;
const priorLeak = leakAfter;
const priorInterior = interiorAfter;
let touched = false;
for (let i = 0; i < filteredMask.length; i++) {
if (!u2Seeds[i]) continue;
if (filteredMask[i]) continue;
const probOk = maskOrig[i] >= 230;
const distOk = distSeeds[i] >= 0.8;
if (!probOk && !distOk) continue;
if (lineMask && lineMask[i] && !probOk) continue;
filteredMask[i] = 255;
touched = true;
}
if (touched && gridMasks) {
coverageAfter = coverageOfMask(filteredMask);
leakAfter = leakRatio(filteredMask, gridMasks.lineMask);
interiorAfter = coverageWithinMask(filteredMask, gridMasks.interiorMask);
const leakGuard = Math.min(0.12, priorLeak + 0.03);
if (leakAfter > leakGuard) {
filteredMask = priorMask;
coverageAfter = priorCoverage;
leakAfter = priorLeak;
interiorAfter = priorInterior;
}
leakAfterFinal = leakAfter;
}
}
if (maskOrig && gridContext && gridMasks && maskWidth && maskHeight) {
const miniBand = Math.max(2, Math.round(Math.max(1, (gridContext.groutPx ?? 3) * 0.5)));
const miniClamp = suppressTileLeak(
Uint8Array.from(filteredMask),
maskOrig,
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
miniBand,
168,
);
const lineMask = gridMasks.lineMask;
if (lineMask) {
for (let i = 0; i < miniClamp.length; i++) {
if (!lineMask[i]) continue;
const strongProb = maskOrig ? maskOrig[i] >= 220 : false;
const strongSeed = u2Seeds ? Boolean(u2Seeds[i]) : false;
if (!strongProb && !strongSeed) {
miniClamp[i] = 0;
}
}
}
if (u2Seeds) {
for (let i = 0; i < miniClamp.length; i++) {
if (!u2Seeds[i]) continue;
const onLine = lineMask ? Boolean(lineMask[i]) : false;
if (onLine && maskOrig && maskOrig[i] < 200) continue;
miniClamp[i] = 255;
}
}
const miniLeak = leakRatio(miniClamp, gridMasks.lineMask);
const miniCoverage = coverageOfMask(miniClamp);
const miniInterior = coverageWithinMask(miniClamp, gridMasks.interiorMask);
const leakTarget = Math.min(leakAfter, 0.12);
if (miniLeak <= leakTarget && miniCoverage >= coverageAfter * 0.6) {
filteredMask = miniClamp;
coverageAfter = miniCoverage;
leakAfter = miniLeak;
interiorAfter = miniInterior;
leakAfterFinal = miniLeak;
}
}
const seedGuardShared = u2Seeds && maskWidth && maskHeight
? morphDilate(u2Seeds, maskWidth, maskHeight, 2)
: null;
if (u2Seeds && gridMasks?.lineMask && maskWidth && maskHeight) {
const lineMask = gridMasks.lineMask;
const seedGuard = seedGuardShared;
let trimmed = false;
for (let i = 0; i < filteredMask.length; i++) {
if (!filteredMask[i]) continue;
if (!lineMask[i]) continue;
if (seedGuard && seedGuard[i]) continue;
filteredMask[i] = 0;
trimmed = true;
}
if (trimmed) {
coverageAfter = coverageOfMask(filteredMask);
leakAfter = leakRatio(filteredMask, gridMasks.lineMask);
interiorAfter = coverageWithinMask(filteredMask, gridMasks.interiorMask);
leakAfterFinal = leakAfter;
}
}
if (maskNormalizedInitial && gridMasks?.lineMask && maskWidth && maskHeight) {
const union = Uint8Array.from(filteredMask);
const lineMask = gridMasks.lineMask;
let unionChanged = false;
for (let i = 0; i < union.length; i++) {
if (union[i]) continue;
if (!maskNormalizedInitial[i]) continue;
if (lineMask[i]) {
if (seedGuardShared && seedGuardShared[i]) {
// ok
} else if (!maskOrig || maskOrig[i] < 210) {
continue;
}
}
union[i] = 255;
unionChanged = true;
}
if (unionChanged) {
const unionCoverage = coverageOfMask(union);
const unionLeak = leakRatio(union, gridMasks.lineMask);
const unionInterior = coverageWithinMask(union, gridMasks.interiorMask);
const drop = coverageBefore - unionCoverage;
if (unionLeak <= 0.14 || (unionLeak <= 0.16 && drop >= 0.08)) {
filteredMask = union;
coverageAfter = unionCoverage;
leakAfter = unionLeak;
interiorAfter = unionInterior;
leakAfterFinal = unionLeak;
if (pipelineResult.debug?.leakClamp && typeof pipelineResult.debug.leakClamp === 'object') {
(pipelineResult.debug.leakClamp as any).unionApplied = {
coverage: Number(unionCoverage.toFixed(6)),
leak: Number(unionLeak.toFixed(6)),
interior: Number(unionInterior.toFixed(6)),
drop: Number(drop.toFixed(6)),
};
}
} else if (pipelineResult.debug?.leakClamp && typeof pipelineResult.debug.leakClamp === 'object') {
(pipelineResult.debug.leakClamp as any).unionRejected = {
coverage: Number(unionCoverage.toFixed(6)),
leak: Number(unionLeak.toFixed(6)),
interior: Number(unionInterior.toFixed(6)),
drop: Number(drop.toFixed(6)),
};
}
}
}
if (maskNormalizedInitial && gridMasks && maskWidth && maskHeight) {
const diff = new Uint8Array(filteredMask.length);
let diffPixels = 0;
for (let i = 0; i < filteredMask.length; i++) {
if (filteredMask[i]) continue;
if (!maskNormalizedInitial[i]) continue;
diff[i] = 255;
diffPixels += 1;
}
if (diffPixels > 0) {
const distMap = distanceToNearestLattice(
maskWidth,
maskHeight,
gridContext?.vlines ?? [],
gridContext?.hlines ?? [],
Math.max(3, (gridContext?.groutPx ?? 3) + 1),
);
const comps = connectedComponents(diff, maskWidth, maskHeight, 64);
const lineMask = gridMasks.lineMask;
const reinjected: Array<Record<string, unknown>> = [];
let reinjectGain = 0;
for (const comp of comps) {
const area = comp.area;
if (area < 80) continue;
let sumDist = 0;
let lineCount = 0;
let seedSupport = 0;
let probSupport = 0;
for (const idx of comp.pixels) {
sumDist += distMap[idx];
if (lineMask[idx]) lineCount += 1;
if (seedGuardShared && seedGuardShared[idx]) seedSupport += 1;
if (maskOrig && maskOrig[idx] >= 210) probSupport += 1;
}
const meanDist = sumDist / area;
const lineFrac = lineCount / area;
const seedFrac = seedSupport / area;
const probFrac = probSupport / area;
if (lineFrac > 0.50 && seedFrac < 0.25 && probFrac < 0.60) continue;
const allowSeedBacked = seedFrac >= 0.60;
const allowProbBacked = probFrac >= 0.80 && meanDist >= (gridContext?.groutPx ?? 3) * 2.0;
if (!allowSeedBacked && !allowProbBacked) continue;
const candidate = Uint8Array.from(filteredMask);
let applied = false;
for (const idx of comp.pixels) {
if (candidate[idx]) continue;
if (lineMask[idx] && !((seedGuardShared && seedGuardShared[idx]) || (maskOrig && maskOrig[idx] >= 220))) continue;
candidate[idx] = 255;
applied = true;
}
if (!applied) continue;
const candCoverage = coverageOfMask(candidate);
const candLeak = leakRatio(candidate, gridMasks.lineMask);
const candInterior = coverageWithinMask(candidate, gridMasks.interiorMask);
if (candLeak <= 0.14) {
const gain = candCoverage - coverageAfter;
if (gain > 0 && reinjectGain + gain > 0.06) continue;
filteredMask = candidate;
coverageAfter = candCoverage;
leakAfter = candLeak;
interiorAfter = candInterior;
leakAfterFinal = candLeak;
reinjectGain += Math.max(0, gain);
reinjected.push({
area,
meanDist: Number(meanDist.toFixed(3)),
lineFrac: Number(lineFrac.toFixed(3)),
seedFrac: Number(seedFrac.toFixed(3)),
probFrac: Number(probFrac.toFixed(3)),
coverage: Number(candCoverage.toFixed(6)),
leak: Number(candLeak.toFixed(6)),
});
}
}
if (reinjected.length && pipelineResult.debug?.leakClamp && typeof pipelineResult.debug.leakClamp === 'object') {
(pipelineResult.debug.leakClamp as any).normalizedReinject = reinjected;
}
}
}
if (leakAfterFinal == null) leakAfterFinal = leakAfter;
if (
pipelineMaskBase &&
gridMasks &&
maskOrig &&
u2Seeds &&
seedOverlap >= 0.95 &&
coverageAfter < 0.22
) {
const relaxedMask = Uint8Array.from(pipelineMaskBase);
if (gridMasks.lineMask) {
const lineMask = gridMasks.lineMask;
for (let i = 0; i < relaxedMask.length; i++) {
if (!lineMask[i]) continue;
const strong = maskOrig[i] >= 208 || Boolean(u2Seeds[i]);
if (!strong) relaxedMask[i] = 0;
}
}
for (let i = 0; i < relaxedMask.length; i++) {
if (u2Seeds[i]) relaxedMask[i] = 255;
}
const relaxedLeak = leakRatio(relaxedMask, gridMasks.lineMask);
const relaxedCoverage = coverageOfMask(relaxedMask);
const relaxedInterior = coverageWithinMask(relaxedMask, gridMasks.interiorMask);
if (
relaxedLeak <= Math.min(0.16, leakAfterFinal + 0.02) &&
relaxedCoverage >= coverageAfter * 0.9
) {
filteredMask = relaxedMask;
coverageAfter = relaxedCoverage;
leakAfter = relaxedLeak;
interiorAfter = relaxedInterior;
leakAfterFinal = relaxedLeak;
}
}
leakAfter = leakAfterFinal;
if (leakAfter > 0.12 && maskOrig && gridContext && gridMasks && maskWidth && maskHeight) {
const clampStrict = suppressTileLeak(
Uint8Array.from(filteredMask),
maskOrig,
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
Math.max(2, (gridContext.groutPx ?? 3) + 1),
172,
);
const lineMask = gridMasks.lineMask;
if (lineMask) {
for (let i = 0; i < clampStrict.length; i++) {
if (lineMask[i]) clampStrict[i] = 0;
}
}
if (u2Seeds) {
for (let i = 0; i < clampStrict.length; i++) {
if (u2Seeds[i]) clampStrict[i] = 255;
}
}
const strictLeak = leakRatio(clampStrict, gridMasks.lineMask);
if (strictLeak <= Math.min(leakAfter, 0.12)) {
filteredMask = clampStrict;
coverageAfter = coverageOfMask(filteredMask);
leakAfter = strictLeak;
interiorAfter = coverageWithinMask(filteredMask, gridMasks.interiorMask);
leakAfterFinal = strictLeak;
}
}
const bigDropSoft = coverageBefore - coverageAfter > 0.08;
const leakSafeSoft = leakAfter <= 0.08;
const interiorGood = pipelineInteriorBefore >= 0.88;
if (pipelineResult.debug?.coverage) {
pipelineResult.debug.coverage.after = Number(coverageAfter.toFixed(6));
pipelineResult.debug.coverage.leak = Number(leakAfterFinal!.toFixed(6));
if (Array.isArray(pipelineResult.debug.coverage.warnings) && leakAfter <= 0.08) {
pipelineResult.debug.coverage.warnings = pipelineResult.debug.coverage.warnings.filter((w: unknown) => w !== 'leak_warn');
}
}
if (!pipelineResult.debug) pipelineResult.debug = {} as typeof pipelineResult.debug;
if (!pipelineResult.debug.leakClamp || typeof pipelineResult.debug.leakClamp !== 'object') {
pipelineResult.debug.leakClamp = { enabled: true } as any;
}
const leakClampDebug = pipelineResult.debug.leakClamp as Record<string, unknown>;
if (leakAfterClamp1 != null) leakClampDebug.leakAfterClamp1 = Number(leakAfterClamp1.toFixed(6));
if (leakAfterReinject != null) leakClampDebug.leakAfterReinject = Number(leakAfterReinject.toFixed(6));
leakClampDebug.leakAfterFinal = Number(leakAfterFinal!.toFixed(6));
leakClampDebug.leakAfter = Number(leakAfterFinal!.toFixed(6));
if (leakAfter <= 0.08 && pipelineWarnings && pipelineWarnings.length) {
pipelineWarnings = pipelineWarnings.filter((w) => w !== 'leak_warn');
}
if (bigDropSoft && leakSafeSoft && interiorGood && maskOrig && gridContext && maskWidth && maskHeight) {
let softCandidate = suppressTileLeak(
Uint8Array.from(pipelineMaskBase),
maskOrig,
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
gridContext.groutPx,
148,
);
if (u2Seeds) {
for (let i = 0; i < softCandidate.length; i++) {
if (u2Seeds[i]) softCandidate[i] = 255;
}
}
const softMasks = gridMasks ?? buildGridMasks(
maskWidth,
maskHeight,
gridContext.vlines,
gridContext.hlines,
gridContext.groutPx,
);
const softCoverage = coverageOfMask(softCandidate);
const softLeak = leakRatio(softCandidate, softMasks.lineMask);
const softInterior = coverageWithinMask(softCandidate, softMasks.interiorMask);
if (softCoverage > coverageAfter && softLeak <= 0.08) {
filteredMask = softCandidate;
coverageAfter = softCoverage;
leakAfter = softLeak;
interiorAfter = softInterior;
acceptanceOverride = 'soft-detail';
}
}
seedOverlap = u2Seeds ? overlapWithSeeds(filteredMask, u2Seeds) : 0;
// Conditional acceptance with relaxed rules for thin occluders
const allowRelaxed = pipelineInteriorBefore >= 0.85 && seedOverlap >= 0.90;
const occluderScene = allowRelaxed;
const bigDrop = (coverageBefore - coverageAfter) >= 0.08;
const leakSafe = leakAfter <= 0.08;
let leakOk: boolean;
if (allowRelaxed) {
const drop = coverageBefore - coverageAfter;
leakOk = leakAfter <= 0.09
|| (leakAfter <= 0.14 && drop >= 0.08);
} else {
leakOk = leakAfter <= 0.06 || (leakAfter <= 0.09 && interiorAfter >= 0.80);
}
let coverageTooHigh = allowRelaxed
? coverageAfter > coverageBefore + 0.25
: coverageAfter > coverageBefore + 0.15;
let covMin = Math.max(0.16, 0.45 * seedCoverage);
if (seedOverlap >= 0.98 && interiorAfter >= 0.18) {
covMin = Math.min(covMin, 0.12);
}
const covFloorRel = Number.isFinite(coverageBefore) ? 0.35 * coverageBefore : 0;
const covFloorAbs = allowRelaxed ? Math.min(covMin, 0.10 + 0.05 * seedCoverage) : covMin;
let coverageTooLow = coverageAfter < Math.max(covFloorRel, covFloorAbs);
if (interiorAfter >= 0.90 && leakAfter <= (allowRelaxed ? 0.12 : 0.05)) {
coverageTooLow = false;
}
if (occluderScene && bigDrop && leakSafe) {
coverageTooLow = false;
}
if (seedOverlap >= 0.97 && coverageAfter >= 0.12) {
coverageTooLow = false;
}
const invalidMask = !Number.isFinite(coverageAfter) || filteredMask.length !== (maskNormalized?.length ?? 0);
const fallbackMode = (process.env.OCCLUDER_PIPELINE_FALLBACK_MODE ?? "strict").trim().toLowerCase();
const fallbackDisabled = fallbackMode === "off" || fallbackMode === "disabled" || fallbackMode === "none";
let acceptPipeline = leakOk && !coverageTooHigh && !coverageTooLow && !invalidMask;
const samPatchMeta = (maskOriginMeta as { samPatch?: { applied?: boolean } } | null | undefined)?.samPatch;
const samPatchApplied = !!samPatchMeta?.applied;
const samPatchForceAccept = (process.env.OCCLUDER_SAM_PATCH_FORCE_ACCEPT ?? "1") !== "0";
const samPatchForceMaxCoverage = clamp(
Number(process.env.OCCLUDER_SAM_PATCH_FORCE_MAX_COVERAGE ?? "0.35"),
0.05,
1,
);
const allowSamPatchOverride = samPatchForceAccept
&& samPatchApplied
&& !invalidMask
&& coverageAfter <= samPatchForceMaxCoverage
&& coverageAfter >= Math.max(0.06, covFloorRel);
const allowOverride = (acceptanceOverride === 'soft-detail' && leakSafe)
|| (occluderScene && bigDrop && leakSafe && !coverageTooHigh && !invalidMask)
|| allowSamPatchOverride;
if (!acceptPipeline && allowOverride) {
filteredMask = Uint8Array.from(pipelineMaskBase);
coverageAfter = pipelineCoverageAfter;
leakAfter = pipelineLeakAfter;
interiorAfter = pipelineInteriorBefore;
coverageTooLow = false;
coverageTooHigh = false;
leakOk = true;
acceptPipeline = true;
if (!acceptanceOverride) acceptanceOverride = allowSamPatchOverride ? 'sam-patch-force' : 'occluder-relaxed';
}
if (!acceptPipeline && fallbackDisabled && !invalidMask) {
filteredMask = Uint8Array.from(pipelineMaskBase);
coverageAfter = pipelineCoverageAfter;
leakAfter = pipelineLeakAfter;
interiorAfter = pipelineInteriorBefore;
coverageTooLow = false;
coverageTooHigh = false;
leakOk = true;
acceptPipeline = true;
if (!acceptanceOverride) acceptanceOverride = 'fallback-disabled';
}
const acceptanceDebug: Record<string, unknown> = {
coverageBefore: Number(coverageBefore.toFixed(4)),
coverageAfter: Number(coverageAfter.toFixed(4)),
coverageDelta: Number((coverageAfter - coverageBefore).toFixed(4)),
interiorAfter: Number(interiorAfter.toFixed(4)),
leakBefore: Number(leakBefore.toFixed(6)),
leakAfter: Number(leakAfter.toFixed(6)),
seedOverlap: Number(seedOverlap.toFixed(4)),
allowRelaxed,
fallbackMode,
leakOk,
coverageTooHigh,
coverageTooLow,
invalidMask,
};
if (acceptanceOverride) {
acceptanceDebug.override = acceptanceOverride;
}
pipelineResult.debug.acceptance = acceptanceDebug;
if (acceptPipeline) {
maskNormalized = filteredMask;
maskCoverage = coverageAfter;
latticeLeak = leakAfter;
latticeInteriorCoverage = interiorAfter;
pipelineResult.debug.fallback = {
reason: {
leakTooHigh: false,
coverageTooHigh: false,
coverageTooLow: false,
invalidMask: false,
},
usedOriginal: false,
originalCoverage: coverageBefore,
pipelineCoverage: coverageAfter,
leakPct: leakAfter,
};
} else {
maskNormalized = maskNormalizedInitial ?? maskNormalized;
maskCoverage = maskNormalizedInitialCoverage ?? maskCoverage;
latticeLeak = leakAfter;
latticeInteriorCoverage = interiorAfter;
pipelineResult.debug.fallback = {
reason: {
leakTooHigh: !leakOk,
coverageTooHigh,
coverageTooLow,
invalidMask,
},
usedOriginal: true,
originalCoverage: coverageBefore,
pipelineCoverage: coverageAfter,
leakPct: leakAfter,
};
}
// Final airbag: Ensure mask never falls below 60% of seed coverage
if (u2Seeds && maskNormalized && seedCoverage > 0) {
const finalCoverage = coverageOfMask(maskNormalized);
const minFrac = 0.60;
if (finalCoverage < minFrac * seedCoverage) {
// Union with seeds to guarantee minimum coverage
for (let i = 0; i < maskNormalized.length; i++) {
if (u2Seeds[i]) maskNormalized[i] = 255;
}
maskCoverage = coverageOfMask(maskNormalized);
}
}
maskDebugPayload = {
preset: maskPresetApplied,
requestedPreset: presetRequested,
fallbackApplied: normalizationChosen.preset !== (normalizationPrimary?.preset ?? normalizationChosen.preset),
normalizedCoverage: maskCoverage,
originalCoverage: normalizationChosen.result.stats.originalCoverage,
background: normalizationChosen.result.stats.background ?? null,
seedCoverage: normalizationChosen.result.stats.hybrid?.seedCoverage ?? null,
pipeline: pipelineResult.debug,
raw: rawMaskDiagnostics ?? undefined,
origin: maskOriginSource ?? undefined,
originMeta: maskOriginMeta ?? undefined,
normalizeStats: normalizationChosen.result.stats,
normalizeOverrides: normalizationOverrides ?? null,
segCoverage: segmentationCoverage,
};
if (maskNormalizedDataUrl == null && maskWidth && maskHeight) {
maskNormalizedDataUrl = await maskToDataUrl(maskNormalized, maskWidth, maskHeight);
}
if (maskDebugEnabled && maskWidth && maskHeight) {
maskOverlayDataUrl = await overlayToDataUrl(raw, maskNormalized, maskWidth, maskHeight);
maskCutoutDataUrl = await cutoutToDataUrl(raw, maskNormalized, maskWidth, maskHeight);
}
finalMaskDataUrl = maskNormalizedDataUrl;
finalMaskMeta = {
polarity: 'white',
coverage: Number(maskCoverage.toFixed(3)),
preset: maskPresetApplied,
source: maskOriginSource ?? undefined,
stats: maskOriginMeta ?? undefined,
};
}
if (!finalMaskDataUrl && maskNormalized && maskWidth && maskHeight) {
maskNormalizedDataUrl = await maskToDataUrl(maskNormalized, maskWidth, maskHeight);
finalMaskDataUrl = maskNormalizedDataUrl;
finalMaskMeta = {
polarity: 'white',
coverage: maskCoverage != null ? Number(maskCoverage.toFixed(3)) : undefined,
preset: maskPresetApplied ?? undefined,
source: maskOriginSource ?? undefined,
stats: maskOriginMeta ?? undefined,
};
if (maskDebugEnabled) {
maskOverlayDataUrl = await overlayToDataUrl(raw, maskNormalized, maskWidth, maskHeight);
maskCutoutDataUrl = await cutoutToDataUrl(raw, maskNormalized, maskWidth, maskHeight);
}
}
if (!maskDebugPayload && normalizationChosen) {
maskDebugPayload = {
preset: maskPresetApplied,
requestedPreset: presetRequested,
fallbackApplied: normalizationChosen.preset !== (normalizationPrimary?.preset ?? normalizationChosen.preset),
normalizedCoverage: maskCoverage,
originalCoverage: normalizationChosen.result.stats.originalCoverage,
background: normalizationChosen.result.stats.background ?? null,
seedCoverage: normalizationChosen.result.stats.hybrid?.seedCoverage ?? null,
pipeline: { flags },
raw: rawMaskDiagnostics ?? undefined,
origin: maskOriginSource ?? undefined,
originMeta: maskOriginMeta ?? undefined,
};
}
if (tileMaskMeta?.tileBand) {
maskDebugPayload = {
...(maskDebugPayload ?? {}),
tileBandDebug: {
enabled: true,
yMin: tileMaskMeta.tileBand.yMin,
yMax: tileMaskMeta.tileBand.yMax,
confidence: tileMaskMeta.tileBand.confidence,
},
};
}
if (depthDebugPayload) {
maskDebugPayload = { ...(maskDebugPayload ?? {}), depth: depthDebugPayload };
}
if (tileSegmentationResult) {
// Prefer depth occluder but merge with segmentation masks so farther objects are not dropped.
const baseMaskData = rawMaskSnapshot ?? maskNormalized ?? null;
const baseMaskWidth = maskWidth ?? meta0.width ?? tileSegmentationResult.width;
const baseMaskHeight = maskHeight ?? meta0.height ?? tileSegmentationResult.height;
const useDepthOccluder = Boolean(depthOccluderMask) && !preferSegOccluder;
let occluderMaskForTiles = useDepthOccluder ? depthOccluderMask?.data ?? null : baseMaskData ?? null;
let occluderWidth = useDepthOccluder ? depthOccluderMask?.width ?? baseMaskWidth : baseMaskWidth;
let occluderHeight = useDepthOccluder ? depthOccluderMask?.height ?? baseMaskHeight : baseMaskHeight;
let occluderSource = useDepthOccluder ? 'depth-anything-v2' : (maskOriginSource ?? undefined);
let occluderDataUrl = useDepthOccluder
? (depthOccluderUrl ?? finalMaskDataUrl ?? maskNormalizedDataUrl ?? null)
: (finalMaskDataUrl ?? maskNormalizedDataUrl ?? null);
if (useDepthOccluder && depthOccluderMask && baseMaskData && baseMaskWidth && baseMaskHeight) {
const baseMask: BinaryMask = {
data: baseMaskData,
width: baseMaskWidth,
height: baseMaskHeight,
};
const depthAligned = await resizeBinaryMask(depthOccluderMask, baseMask.width, baseMask.height);
const merged = mergeOccluderMasks(baseMask, depthAligned);
occluderMaskForTiles = merged.data;
occluderWidth = merged.width;
occluderHeight = merged.height;
occluderSource = 'depth+seg';
if (maskDebugEnabled) {
occluderDataUrl = await maskToDataUrl(merged.data, merged.width, merged.height).catch(() => occluderDataUrl);
}
}
const colorEntries = tileSegmentationResult.classes.map((cls) => [String(cls.id), cls.color] as const);
const classPixelCountsRaw = countClassIds(tileSegmentationResult.ids);
const rawLegend = (() => {
const names = tileSegmentationResult.classes.map((cls) => String(cls.name).toLowerCase());
// 2-class tiles model: 0=background, 1=tiles
if (names.length === 2 && names[0] === 'background' && names[1] === 'tiles') {
return { "0": "background", "1": "tiles", "255": "ignore" };
}
return { "0": "background", "1": "grout", "2": "tiles", "255": "ignore" };
})();
const effectiveClasses = rawLegend["1"] === "tiles" && Object.keys(rawLegend).length === 3
? ['background', 'grout', 'tiles']
: tileSegmentationResult.classes.map((cls) => cls.name);
const effectiveColorMap = rawLegend["1"] === "tiles" && Object.keys(rawLegend).length === 3
? ({
"0": [0, 0, 0],
"1": [52, 152, 219],
"2": [46, 204, 113],
} satisfies Record<string, [number, number, number]>)
: (Object.fromEntries(colorEntries) as Record<string, [number, number, number]>);
const applyCombinedTileMask = (combined: any, occluderPayload: { dataUrl?: string | null } | null) => {
tileCombineResult = combined;
tileMaskDataUrl = combined.combined.dataUrl;
const classPixelCountsCombined = countClassIds(combined.combined.ids);
tileMaskMeta = {
// IMPORTANT: This metadata must match the *combined* mask image.
// For the 2-class tiles model we promote to an effective 3-class mask (bg/grout/tiles),
// so the frontend can reliably extract `tilesMask` and `groutMask`.
classes: effectiveClasses,
classIdLegend: { "0": "background", "1": "grout", "2": "tiles", "255": "ignore" },
classIdLegendRaw: rawLegend,
classPixelCountsRaw,
classPixelCounts: classPixelCountsCombined,
colorMap: effectiveColorMap,
coverage: combined.combined.coverage,
coverageAbsolute: combined.combined.coverageAbsolute,
rawCoverage: tileSegmentationResult.coverage,
occluderCoverage: combined.combined.occluderCoverage,
occludedPixels: combined.combined.occludedPixels,
occluderPixels: combined.combined.occluderPixels,
occludedCoverage: combined.combined.occludedCoverage,
warnings: combined.combined.warnings,
unetDebug: tileSegmentationResult.debug ?? null,
model: tileModelRequested,
tileBand: combined.combined.tileBand ? {
yMin: combined.combined.tileBand.yMin,
yMax: combined.combined.tileBand.yMax,
confidence: combined.combined.tileBand.confidence,
} : null,
tileMaskV3: (combined.combined as any).tileMaskV3 ?? null,
tileMaskV2: combined.combined.tileMaskV2 ?? null,
source: combined.combined.source,
};
tileMaskVersions = {
combined: combined.combined.dataUrl,
raw: combined.rawDataUrl,
occluder: combined.occluderDataUrl ?? (occluderPayload?.dataUrl ?? null),
};
};
try {
const { combineTileMaskWithOccluder } = await import('@/lib/segmentation/tile-mask');
const wallMaskPayload = depthBackMask
? { data: depthBackMask.data, width: depthBackMask.width, height: depthBackMask.height }
: null;
const bandOverrideParam = url.searchParams.get('tile_band_override');
const bandOverride = (() => {
if (!bandOverrideParam) return null;
const parts = bandOverrideParam.split(',').map((p) => Number(p));
if (parts.length === 2 && parts.every((v) => Number.isFinite(v) && v >= 0 && v <= 1)) {
return { yMinNorm: parts[0], yMaxNorm: parts[1] };
}
return null;
})();
// Enable depth wall-fill when we have a wall plane. The actual fill is still constrained by
// `tileBand`/grid ROI logic inside `combineTileMaskWithOccluder` so we don't leak into upper walls.
const enableDepthWallFill = Boolean(depthBackMask);
const tileMaskV3Enabled = process.env.TILEMASK_V3 === "1";
const tileMaskV2Enabled = process.env.TILEMASK_V2 === "1";
let sourceImageRgbaCache:
| { data: Uint8Array; width: number; height: number; channels: number }
| null
| undefined = undefined;
const getSourceImageRgba = async () => {
if (!tileMaskV2Enabled && !tileMaskV3Enabled) return null;
if (sourceImageRgbaCache !== undefined) return sourceImageRgbaCache;
try {
const rgbaRaw = await sharp(raw)
.ensureAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
sourceImageRgbaCache = {
data: new Uint8Array(rgbaRaw.data),
width: rgbaRaw.info.width,
height: rgbaRaw.info.height,
channels: rgbaRaw.info.channels,
};
} catch {
sourceImageRgbaCache = null;
}
return sourceImageRgbaCache;
};
const buildCombine = async (occluderPayloadOverride: {
data: Uint8Array;
width: number;
height: number;
dataUrl?: string | null;
source?: string | null;
} | null) => {
const combined = await combineTileMaskWithOccluder(
tileSegmentationResult,
occluderPayloadOverride,
{
bandOverride,
wallMask: wallMaskPayload,
enableDepthWallFill,
grid: { vlines: finalGrid.vlines, hlines: finalGrid.hlines },
tileMaskV3Enabled,
tileMaskV2Enabled,
sourceImageRgba: await getSourceImageRgba(),
},
);
applyCombinedTileMask(combined, occluderPayloadOverride);
return combined;
};
const occluderPayload = occluderMaskForTiles && occluderWidth && occluderHeight
? {
data: occluderMaskForTiles,
width: occluderWidth,
height: occluderHeight,
dataUrl: occluderDataUrl,
source: occluderSource,
}
: null;
const combined = await buildCombine(occluderPayload);
if (
preferTileBand &&
preferSegOccluder &&
maskOrig &&
depthOccluderMask &&
combined.combined.tileBand
) {
const bandMaxCoverage = clamp(
Number(process.env.OCCLUDER_SAM_DEPTH_BAND_MAX_COVERAGE ?? "0.6"),
0.1,
1,
);
const bandDilate = Math.max(
0,
Math.round(Number(process.env.OCCLUDER_TILE_BAND_DILATE ?? "6")),
);
const band = combined.combined.tileBand;
const targetW = meta0.width ?? tileSegmentationResult?.width ?? maskWidth ?? depthOccluderMask.width;
const targetH = meta0.height ?? tileSegmentationResult?.height ?? maskHeight ?? depthOccluderMask.height;
let skipBandUnion = false;
if (!targetW || !targetH) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "tile_band_missing_dims",
},
};
if (maskDebugPayload?.originMeta) {
maskDebugPayload.originMeta = maskOriginMeta;
}
skipBandUnion = true;
}
const yMin = skipBandUnion ? 0 : clamp(Math.floor(band.yMin), 0, targetH);
const yMax = skipBandUnion ? 0 : clamp(Math.floor(band.yMax), 0, targetH);
if (!skipBandUnion && yMax <= yMin) {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "tile_band_invalid",
bandSource: "tile_band_combined",
band: {
yMin: band.yMin,
yMax: band.yMax,
confidence: band.confidence,
},
},
};
if (maskDebugPayload?.originMeta) {
maskDebugPayload.originMeta = maskOriginMeta;
}
skipBandUnion = true;
}
if (!skipBandUnion) {
const bandBase = new Uint8Array(targetW * targetH);
for (let y = yMin; y < yMax; y++) {
const row = y * targetW;
bandBase.fill(255, row, row + targetW);
}
const bandMask = bandDilate > 0
? morphDilate(bandBase, targetW, targetH, bandDilate)
: bandBase;
const bandCoverage = coverageOfMask(bandMask);
if (bandCoverage <= bandMaxCoverage) {
const depthAligned = (depthOccluderMask.width !== targetW || depthOccluderMask.height !== targetH)
? await resizeBinaryMask(depthOccluderMask, targetW, targetH)
: depthOccluderMask;
const clipped = maskAnd(depthAligned.data, bandMask);
const baseMask = (maskWidth && maskHeight && (maskWidth !== targetW || maskHeight !== targetH))
? await resizeBinaryMask({ data: maskOrig, width: maskWidth, height: maskHeight }, targetW, targetH)
: { data: maskOrig, width: targetW, height: targetH };
const beforeCoverage = coverageOfMask(baseMask.data);
const merged = maskOr(baseMask.data, clipped);
const afterCoverage = coverageOfMask(merged);
maskOrig = merged;
maskWidth = targetW;
maskHeight = targetH;
rawMaskSnapshot = Uint8Array.from(maskOrig);
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: true,
bandApplied: true,
bandDilate,
bandCoverage: Number(bandCoverage.toFixed(4)),
bandMaxCoverage: Number(bandMaxCoverage.toFixed(3)),
bandSource: "tile_band_combined",
band: {
yMin: band.yMin,
yMax: band.yMax,
confidence: band.confidence,
},
coverageBefore: Number(beforeCoverage.toFixed(4)),
coverageAfter: Number(afterCoverage.toFixed(4)),
},
};
if (maskWidth && maskHeight) {
maskRawDataUrl = await maskToDataUrl(maskOrig, maskWidth, maskHeight);
finalMaskDataUrl = maskRawDataUrl;
finalMaskMeta = {
polarity: 'white',
source: maskOriginSource ?? undefined,
stats: maskOriginMeta ?? undefined,
};
}
const updatedPayload = {
data: maskOrig,
width: maskWidth ?? promptTileWidth,
height: maskHeight ?? promptTileHeight,
dataUrl: maskRawDataUrl,
source: maskOriginSource ?? undefined,
};
await buildCombine(updatedPayload);
} else {
maskOriginMeta = {
...(maskOriginMeta ?? {}),
depthBandUnion: {
applied: false,
reason: "band_too_large",
bandCoverage: Number(bandCoverage.toFixed(4)),
bandMaxCoverage: Number(bandMaxCoverage.toFixed(3)),
bandSource: "tile_band_combined",
band: {
yMin: band.yMin,
yMax: band.yMax,
confidence: band.confidence,
},
},
};
if (maskDebugPayload?.originMeta) {
maskDebugPayload.originMeta = maskOriginMeta;
}
}
}
}
} catch (tileCombineError) {
if (debug) {
console.warn('[tile-seg] failed to combine tile mask with occluder', tileCombineError);
}
tileMaskDataUrl = tileSegmentationResult.dataUrl;
tileMaskMeta = {
classes: tileSegmentationResult.classes.map((cls) => cls.name),
classIdLegend: { "0": "background", "1": "grout", "2": "tiles", "255": "ignore" },
classIdLegendRaw: rawLegend,
classPixelCountsRaw,
classPixelCounts: classPixelCountsRaw,
colorMap: Object.fromEntries(colorEntries) as Record<string, [number, number, number]>,
coverage: tileSegmentationResult.coverage,
coverageAbsolute: tileSegmentationResult.coverage,
unetDebug: tileSegmentationResult.debug ?? null,
source: 'unet_latest',
model: tileModelRequested,
};
tileMaskVersions = {
combined: tileSegmentationResult.dataUrl,
raw: tileSegmentationResult.dataUrl,
};
}
}
if (analysis.mask && normalizationChosen) {
analysis.mask = {
original_coverage: Number(normalizationChosen.result.stats.originalCoverage.toFixed(3)),
coverage: maskCoverage != null ? Number(maskCoverage.toFixed(3)) : 0,
threshold: Number((normalizationChosen.result.threshold / 255).toFixed(3)),
inverted: normalizationChosen.result.inverted,
source: maskOriginSource ?? undefined,
};
}
// Suppress heuristic warnings when only unet_latest is requested.
if (tileModelRequested === 'unet_latest') {
pipelineWarnings = undefined;
}
const fg_mask = finalMaskDataUrl;
const fg_mask_meta = finalMaskMeta;
if (maskDebugPayload?.originMeta && fg_mask_meta?.stats) {
maskDebugPayload.originMeta = fg_mask_meta.stats;
if (fg_mask_meta.source) {
maskDebugPayload.origin = fg_mask_meta.source;
}
}
const response = {
ok: true,
image: { width: meta0.width, height: meta0.height },
tile_w_px: finalTileW,
tile_h_px: finalTileH,
grout_px: finalGrout,
tile_w_mm: null, tile_h_mm: null, grout_mm: null,
grid: {
vlines: finalGrid.vlines,
hlines: finalGrid.hlines,
},
source,
tile_model: tileModelRequested,
ransac: ransacInfo,
fg_mask, // <<<<<<<<<< neu
...(fg_mask_meta ? { fg_mask_meta } : {}),
...(tileMaskDataUrl ? { tile_mask: tileMaskDataUrl } : {}),
...(tileMaskMeta ? { tile_mask_meta: tileMaskMeta } : {}),
...(tileMaskVersions ? { tile_mask_versions: tileMaskVersions } : {}),
...(maskPresetApplied ? { mask_preset: maskPresetApplied } : {}),
...(maskDebugPayload ? { mask_debug: maskDebugPayload } : {}),
...(depthDebugPayload ? { depth_debug: depthDebugPayload } : {}),
...(tileMaskMeta?.tileBand ? {
tile_band: {
yMin: tileMaskMeta.tileBand.yMin,
yMax: tileMaskMeta.tileBand.yMax,
confidence: tileMaskMeta.tileBand.confidence,
},
} : {}),
...((maskNormalizedDataUrl || maskRawDataUrl) ? {
mask_versions: {
normalized: maskNormalizedDataUrl,
raw: maskRawDataUrl,
overlay: maskOverlayDataUrl,
cutout: maskCutoutDataUrl,
},
} : {}),
...(maskCoverage !== null ? ({
mask_metrics: {
coverage: Number(maskCoverage.toFixed(3)),
coverageNorm: Number(maskCoverage.toFixed(3)),
coverageRaw: rawMaskDiagnostics
? Number(rawMaskDiagnostics.coverageArray.toFixed(3))
: (maskNormalizedInitialCoverage != null ? Number(maskNormalizedInitialCoverage.toFixed(3)) : undefined),
backgroundCoverage: maskBackgroundCoverage != null ? Number(maskBackgroundCoverage.toFixed(3)) : undefined,
occluderCoverage: maskOccluderCoverage != null ? Number(maskOccluderCoverage.toFixed(3)) : undefined,
seedCoverage: maskSeedCoverage != null ? Number(maskSeedCoverage.toFixed(3)) : undefined,
preset: maskPresetApplied ?? undefined,
source: maskOriginSource ?? undefined,
latticeLeak: latticeLeak ?? undefined,
latticeLeakPct: latticeLeak != null ? Number((latticeLeak * 100).toFixed(3)) : undefined,
latticeInteriorCoverage: latticeInteriorCoverage != null ? Number(latticeInteriorCoverage.toFixed(3)) : undefined,
warnings: pipelineWarnings && pipelineWarnings.length ? pipelineWarnings : undefined,
autotuneMs: flags.autotuneMs,
rawMismatch: rawMaskDiagnostics ? Number(rawMaskDiagnostics.mismatch.toFixed(3)) : undefined,
rawSuspicious: rawMaskDiagnostics ? rawMaskDiagnostics.suspicious : undefined,
},
}) : {}),
name: (file as any)?.name,
type: (file as any)?.type,
} as const;
return NextResponse.json(response);
} catch (err:any) {
console.error("API /api/analyze fatal:", err);
const msg = err?.message || String(err);
const code = /width\/height invalid|missing file/i.test(msg) ? 422 : 500;
return NextResponse.json({ ok:false, error: msg }, { status: code });
} finally {
restoreOccluderEnv?.();
}
}