| Server IP : 217.154.3.148 / Your IP : 216.73.216.90 Web Server : Apache System : Linux blissful-wright.217-154-3-148.plesk.page 6.8.0-124-generic #124-Ubuntu SMP PREEMPT_DYNAMIC Tue May 26 13:00:45 UTC 2026 x86_64 User : gracious-boyd_hfhh1h8vo9n ( 10000) PHP Version : 8.3.31 Disable Function : opcache_get_status MySQL : OFF | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : /var/www/vhosts/gracious-boyd.217-154-3-148.plesk.page/nextjs/lib/segmentation/ |
Upload File : |
import sharp from 'sharp';
import type { ClassInfo, TileSegmentationResult } from './tile-unet';
import { renderTileMask } from './tile-unet';
import { groutFromTiles } from './grout_from_tiles';
import { binarize, morphDilate, morphErode } from '@/utils/maskOps';
import { solveTileMaskV2Core, tileMaskV2Bbox } from './tile-mask-v2';
import { solveTileMaskV3Core, tileMaskV3Bbox } from './tile-mask-v3';
export type BinaryMaskPayload = {
data: Uint8Array;
width: number;
height: number;
};
export type OccluderMaskPayload = BinaryMaskPayload & {
dataUrl?: string | null;
source?: string | null;
coverage?: number | null;
};
export type CombinedTileMask = {
ids: Uint8Array;
dataUrl: string;
coverage: Record<string, number>;
coverageAbsolute: Record<string, number>;
occludedCoverage: number;
occluderCoverage: number;
occludedPixels: number;
occluderPixels: number;
source: string;
tileBand: TileBand | null;
tileMaskV2?: {
applied: boolean;
reason?: string;
metrics: {
tileMaskCoverage: number;
backsplashCoverage: number;
countertopSpillProxy: number;
missingGridProxy: number;
fragmentationCount: number;
seamY: number | null;
};
} | null;
tileMaskV3?: {
applied: boolean;
reason?: string;
failureReason?: string | null;
confidence: number;
fallbackMode: 'none' | 'grid_only' | 'raw_tiles';
metrics: {
tileMaskCoverage: number;
backsplashCoverage: number;
countertopSpillProxy: number;
missingGridProxy: number;
fragmentationCount: number;
seamY: number | null;
tileMaskConfidence: number;
};
diagnostics?: {
gridReliable: boolean;
gridSparse: boolean;
gridMonotonic: boolean;
gridVCount: number;
gridHCount: number;
gridSpacingX: number | null;
gridSpacingY: number | null;
seamReliable: boolean;
seamHintY: number | null;
};
} | null;
warnings?: string[];
};
export type TileMaskCombineResult = {
combined: CombinedTileMask;
rawDataUrl: string;
occluderDataUrl?: string | null;
};
export type CombineTileMaskOptions = {
backgroundClassId?: number;
bandOverride?: { yMinNorm: number; yMaxNorm: number } | null;
wallMask?: BinaryMaskPayload | null;
enableDepthWallFill?: boolean;
grid?: { vlines: number[]; hlines: number[] } | null;
tileMaskV3Enabled?: boolean;
tileMaskV2Enabled?: boolean;
sourceImageRgba?: {
data: Uint8Array;
width: number;
height: number;
channels?: number;
} | null;
};
const DEFAULT_PRECISION = 6;
const CLASS_BACKGROUND = 0;
const CLASS_GROUT = 1;
const CLASS_TILES = 2;
const CLASS_IGNORE = 0xff;
function formatCoverage(value: number): number {
return Number(value.toFixed(DEFAULT_PRECISION));
}
function coverageFromCounts(
counts: Float32Array,
classes: ClassInfo[],
divisor: number,
): Record<string, number> {
const denom = Math.max(1, divisor);
const coverage: Record<string, number> = {};
classes.forEach((cls, idx) => {
coverage[cls.name] = formatCoverage((counts[idx] ?? 0) / denom);
});
return coverage;
}
type BinaryMaskStats = {
coverage: number;
solid: number;
total: number;
};
function countActive(mask: Uint8Array | null | undefined): number {
if (!mask || mask.length === 0) return 0;
let active = 0;
for (let i = 0; i < mask.length; i++) {
if (mask[i] > 127) active += 1;
}
return active;
}
function measureBinaryMask(mask: Uint8Array | null, region?: Uint8Array | null): BinaryMaskStats {
if (!mask || mask.length === 0) {
return { coverage: 0, solid: 0, total: 0 };
}
if (!region) {
const solid = countActive(mask);
return {
coverage: solid / mask.length,
solid,
total: mask.length,
};
}
const usable = Math.min(mask.length, region.length);
let solid = 0;
let total = 0;
for (let i = 0; i < usable; i++) {
if (!region[i]) continue;
total += 1;
if (mask[i] > 127) solid += 1;
}
return {
coverage: total > 0 ? solid / total : 0,
solid,
total,
};
}
function sumCounts(counts: Float32Array): number {
let sum = 0;
for (let i = 0; i < counts.length; i++) sum += counts[i] ?? 0;
return sum;
}
export type TileBand = {
yMin: number;
yMax: number;
confidence: number;
};
function medianSpacing(values: number[]): number {
if (values.length < 2) return 0;
const diffs: number[] = [];
for (let i = 1; i < values.length; i++) {
const d = values[i]! - values[i - 1]!;
if (Number.isFinite(d) && d > 0) diffs.push(d);
}
diffs.sort((a, b) => a - b);
return diffs.length ? diffs[Math.floor(diffs.length / 2)]! : 0;
}
function largestComponentBBox(
mask: Uint8Array,
width: number,
height: number,
minArea: number,
): { area: number; xMin: number; xMax: number; yMin: number; yMax: number } | null {
const total = width * height;
if (mask.length < total) return null;
const visited = new Uint8Array(total);
let bestArea = 0;
let bestBox: { area: number; xMin: number; xMax: number; yMin: number; yMax: number } | null = null;
const stack: number[] = [];
for (let idx = 0; idx < total; idx++) {
if (!mask[idx] || visited[idx]) continue;
let area = 0;
let xMin = width;
let xMax = -1;
let yMin = height;
let yMax = -1;
stack.length = 0;
stack.push(idx);
visited[idx] = 1;
while (stack.length) {
const cur = stack.pop()!;
area += 1;
const y = Math.floor(cur / width);
const x = cur - y * width;
if (x < xMin) xMin = x;
if (x > xMax) xMax = x;
if (y < yMin) yMin = y;
if (y > yMax) yMax = y;
if (x > 0) {
const n = cur - 1;
if (!visited[n] && mask[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (x + 1 < width) {
const n = cur + 1;
if (!visited[n] && mask[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (y > 0) {
const n = cur - width;
if (!visited[n] && mask[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (y + 1 < height) {
const n = cur + width;
if (!visited[n] && mask[n]) {
visited[n] = 1;
stack.push(n);
}
}
}
if (area >= minArea && area > bestArea) {
bestArea = area;
bestBox = { area, xMin, xMax, yMin, yMax };
}
}
return bestBox;
}
function suppressSmallComponents(
mask: Uint8Array,
width: number,
height: number,
minArea: number,
): Uint8Array {
const total = width * height;
if (mask.length < total) return mask;
const visited = new Uint8Array(total);
const out = new Uint8Array(mask);
const stack: number[] = [];
for (let idx = 0; idx < total; idx++) {
if (!out[idx] || visited[idx]) continue;
let area = 0;
stack.length = 0;
stack.push(idx);
visited[idx] = 1;
const component: number[] = [];
while (stack.length) {
const cur = stack.pop()!;
component.push(cur);
area += 1;
const y = Math.floor(cur / width);
const x = cur - y * width;
if (x > 0) {
const n = cur - 1;
if (!visited[n] && out[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (x + 1 < width) {
const n = cur + 1;
if (!visited[n] && out[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (y > 0) {
const n = cur - width;
if (!visited[n] && out[n]) {
visited[n] = 1;
stack.push(n);
}
}
if (y + 1 < height) {
const n = cur + width;
if (!visited[n] && out[n]) {
visited[n] = 1;
stack.push(n);
}
}
}
if (area < minArea) {
for (const p of component) out[p] = 0;
}
}
return out;
}
function bandFromLargestComponent(
tilesBinary: Uint8Array,
width: number,
height: number,
grid?: { vlines: number[]; hlines: number[] } | null,
): TileBand | null {
const total = width * height;
const minArea = Math.max(800, Math.floor(total * 0.002));
const comp = largestComponentBBox(tilesBinary, width, height, minArea);
if (!comp) return null;
const yTop = Math.round(height * 0.06);
const hlines = Array.isArray(grid?.hlines) ? grid!.hlines.filter((n) => Number.isFinite(n)) : [];
const tileH = hlines.length >= 2 ? medianSpacing(hlines) : Math.max(6, Math.round(height * 0.04));
const pad = Math.max(2, Math.round(tileH * 0.4));
const yMin = clampInt(Math.max(yTop, comp.yMin - pad), 0, height);
const yMax = clampInt(comp.yMax + pad, 0, height);
if (yMax - yMin < Math.round(height * 0.08)) return null;
if (yMax - yMin > Math.round(height * 0.95)) return null;
return { yMin, yMax, confidence: 0.7 };
}
function bandFromGridBelowFixture(
grid: { vlines: number[]; hlines: number[] } | null | undefined,
width: number,
height: number,
fixtureLimit: number,
): TileBand | null {
if (!grid) return null;
const yTop = Math.max(fixtureLimit, Math.round(height * 0.06));
const yBot = Math.round(height * 0.92);
const h = Array.isArray(grid.hlines)
? grid.hlines.filter((n) => Number.isFinite(n) && n >= yTop && n <= yBot)
: [];
if (h.length < 4) return null;
const run = findRegularRun(h, 4, 0.35, height);
if (run) {
const padY = Math.max(2, Math.round(run.spacing * 0.35));
const y0 = clampInt(Math.max(run.sorted[run.startIdx]! - padY, yTop), 0, height);
const y1 = clampInt(run.sorted[run.endIdx]! + padY, 0, height);
if (y1 - y0 >= Math.round(height * 0.08)) {
return { yMin: y0, yMax: y1, confidence: 1 };
}
}
const sorted = h.slice().sort((a, b) => a - b);
const spacing = medianSpacing(sorted);
if (!(spacing > 0)) return null;
const padY = Math.max(2, Math.round(spacing * 0.35));
const y0 = clampInt(Math.max(sorted[0]! - padY, yTop), 0, height);
const y1 = clampInt(sorted[sorted.length - 1]! + padY, 0, height);
if (y1 - y0 < Math.round(height * 0.08)) return null;
return { yMin: y0, yMax: y1, confidence: 0.8 };
}
async function estimateFixtureLimit(
occluder: OccluderMaskPayload,
width: number,
height: number,
): Promise<number | null> {
if (!occluder?.data || occluder.width <= 0 || occluder.height <= 0) return null;
const occ = await resizeMaskToMatch(occluder.data, occluder.width, occluder.height, width, height);
const topLimit = Math.max(1, Math.floor(height * 0.42));
const occDil = morphDilate(occ, width, height, 6);
// Row-based: find the bottom-most row where occluder covers a meaningful span.
// Ignore the center column so the hood doesn't push the fixture limit downward.
const xMin = Math.floor(width * 0.12);
const xMax = Math.ceil(width * 0.88);
const centerMin = Math.floor(width * 0.35);
const centerMax = Math.floor(width * 0.65);
let lastRow = -1;
for (let y = 0; y < topLimit; y++) {
const row = y * width;
let solid = 0;
let denom = 0;
for (let x = xMin; x < xMax; x++) {
if (x >= centerMin && x <= centerMax) continue;
if (occDil[row + x]! > 127) solid += 1;
denom += 1;
}
const frac = solid / Math.max(1, denom);
if (frac >= 0.02) lastRow = y;
}
if (lastRow >= 0) {
const margin = Math.max(2, Math.round(height * 0.015));
return clampInt(Math.max(Math.round(height * 0.06), lastRow + margin), 0, height);
}
// Fallback: column-based bottom estimate.
const stepX = Math.max(1, Math.floor(width / 220));
const bottoms: number[] = [];
for (let x = 0; x < width; x += stepX) {
if (x >= centerMin && x <= centerMax) continue;
let bottom = -1;
for (let y = 0; y < topLimit; y++) {
if (occ[y * width + x]! > 127) bottom = y;
}
if (bottom >= 0) bottoms.push(bottom);
}
if (bottoms.length < 12) return null;
bottoms.sort((a, b) => a - b);
const p10 = bottoms[Math.floor(bottoms.length * 0.1)] ?? -1;
if (p10 < 0) return null;
const margin = Math.max(2, Math.round(height * 0.015));
return clampInt(Math.max(Math.round(height * 0.06), p10 + margin), 0, height);
}
function bandFromGrid(
grid: { vlines: number[]; hlines: number[] } | null | undefined,
width: number,
height: number,
): TileBand | null {
if (!grid) return null;
const v = Array.isArray(grid.vlines) ? grid.vlines.filter((n) => Number.isFinite(n)) : [];
// Ignore top/bottom margins; stray "border lines" and countertop edges otherwise dominate.
const yTop = height * 0.06;
const yBot = height * 0.9;
const h = Array.isArray(grid.hlines)
? grid.hlines.filter((n) => Number.isFinite(n) && n >= yTop && n <= yBot)
: [];
if (h.length < 4) return null;
const run = findRegularRun(h, 4, 0.35, height);
if (!run) {
// Fallback: use min/max of detected grid lines when no clean run exists.
const sorted = h.slice().sort((a, b) => a - b);
const spacing = medianSpacing(sorted);
if (!(spacing > 0)) return null;
const padY = Math.max(2, Math.round(spacing * 0.35));
const y0 = clampInt(Math.max(sorted[0]! - padY, yTop), 0, height);
const y1 = clampInt(sorted[sorted.length - 1]! + padY, 0, height);
if (y1 - y0 < Math.round(height * 0.08)) return null;
if (y1 - y0 > Math.round(height * 0.95)) return null;
return { yMin: y0, yMax: y1, confidence: 0.35 };
}
const { sorted, startIdx, endIdx, spacing } = run;
const tileH = spacing;
// Padding should be conservative; large padding tends to leak into upper walls above the backsplash.
const padY = Math.max(2, Math.round(tileH * 0.35));
const y0 = clampInt(Math.max(sorted[startIdx]! - padY, yTop), 0, height);
const y1 = clampInt(sorted[endIdx]! + padY, 0, height);
if (y1 - y0 < Math.round(height * 0.08)) return null;
if (y1 - y0 > Math.round(height * 0.95)) return null;
void v;
return { yMin: y0, yMax: y1, confidence: 1 };
}
type GridRoi = {
xMin: number;
xMax: number;
yMin: number;
yMax: number;
tileW: number;
tileH: number;
confidence: number;
};
function clampInt(v: number, min: number, max: number): number {
return Math.max(min, Math.min(max, Math.round(v)));
}
function findRegularRun(lines: number[], minLines = 5, tolFrac = 0.3, rangeMax?: number) {
if (lines.length < minLines) return null;
const sorted = lines.slice().sort((a, b) => a - b);
const spacing = medianSpacing(sorted);
if (!(spacing > 0)) return null;
const tol = spacing * tolFrac;
let bestStart = 0;
let bestEnd = 0;
let bestScore = -Infinity;
let curStart = 0;
const scoreRun = (startIdx: number, endIdx: number) => {
const runLen = endIdx - startIdx + 1;
const mid = (sorted[startIdx]! + sorted[endIdx]!) / 2;
const denom = rangeMax && rangeMax > 0 ? rangeMax : sorted[sorted.length - 1] ?? 1;
const midFrac = denom > 0 ? Math.max(0, Math.min(1, mid / denom)) : 0.5;
// Prefer long runs, bias towards lower (larger coordinate) runs when multiple regular grids exist,
// and reject "texture line" grids (too small spacing relative to image size).
const spacingFrac = denom > 0 ? spacing / denom : 0;
const minFrac = 0.018; // ~1.8% of image dimension (filters wood/grain/texture lines)
const maxFrac = 0.25; // >25% is usually not a tile grid
const spacingScore =
spacingFrac <= minFrac
? 0
: spacingFrac >= maxFrac
? 0
: (spacingFrac - minFrac) / (maxFrac - minFrac);
return runLen * (0.5 + midFrac) * spacingScore;
};
for (let i = 1; i < sorted.length; i++) {
const d = sorted[i]! - sorted[i - 1]!;
const ok = Math.abs(d - spacing) <= tol;
if (!ok) {
const curEnd = i - 1;
const runLen = curEnd - curStart + 1;
if (runLen >= minLines) {
const s = scoreRun(curStart, curEnd);
if (s > bestScore) {
bestScore = s;
bestStart = curStart;
bestEnd = curEnd;
}
}
curStart = i;
}
}
{
const curEnd = sorted.length - 1;
const runLen = curEnd - curStart + 1;
if (runLen >= minLines) {
const s = scoreRun(curStart, curEnd);
if (s > bestScore) {
bestScore = s;
bestStart = curStart;
bestEnd = curEnd;
}
}
}
const runLen = bestEnd - bestStart + 1;
if (runLen < minLines) return null;
return { sorted, startIdx: bestStart, endIdx: bestEnd, spacing };
}
function roiFromGrid(
grid: { vlines: number[]; hlines: number[] } | null | undefined,
width: number,
height: number,
): GridRoi | null {
if (!grid) return null;
const v = Array.isArray(grid.vlines) ? grid.vlines.filter((n) => Number.isFinite(n)) : [];
const yTop = height * 0.06;
const yBot = height * 0.9;
const h = Array.isArray(grid.hlines)
? grid.hlines.filter((n) => Number.isFinite(n) && n >= yTop && n <= yBot)
: [];
if (v.length < 2 || h.length < 4) return null;
const hRun = findRegularRun(h, 4, 0.35, height);
if (!hRun) return null;
const tileH = hRun.spacing;
const padY = Math.max(2, Math.round(tileH * 0.35));
const yMin = clampInt(Math.max(hRun.sorted[hRun.startIdx]! - padY, yTop), 0, height);
const yMax = clampInt(hRun.sorted[hRun.endIdx]! + padY, 0, height);
const vSorted = v.slice().sort((a, b) => a - b);
const vRun = findRegularRun(v, 4, 0.35, width);
const tileW = vRun?.spacing ?? medianSpacing(vSorted) ?? tileH;
const padX = Math.max(2, Math.round(tileW * 0.35));
const xMin = clampInt((vRun ? vRun.sorted[vRun.startIdx]! : vSorted[0]!) - padX, 0, width);
const xMax = clampInt((vRun ? vRun.sorted[vRun.endIdx]! : vSorted[vSorted.length - 1]!) + padX, 0, width);
const conf = 1;
const roiH = yMax - yMin;
const roiW = xMax - xMin;
if (roiH < Math.round(height * 0.08) || roiW < Math.round(width * 0.08)) return null;
if (roiH > Math.round(height * 0.95) || roiW > Math.round(width * 0.98)) return null;
return { xMin, xMax, yMin, yMax, tileW, tileH, confidence: conf };
}
function smooth1D(values: Float32Array, window: number): Float32Array {
const len = values.length;
if (window <= 1 || len === 0) return values.slice(0);
const out = new Float32Array(len);
const radius = Math.max(1, Math.floor(window / 2));
let running = 0;
for (let i = 0; i < len; i++) {
running += values[i];
if (i - radius >= 0) running -= values[i - radius];
const start = Math.max(0, i - radius + 1);
const count = i - start + 1;
out[i] = count > 0 ? running / count : 0;
}
return out;
}
export function detectTileBand(
mask: Uint8Array,
width: number,
height: number,
opts?: {
minRowGrout?: number;
minRowTilesGrout?: number;
smoothWindow?: number;
excludeBottomFrac?: number;
groutId?: number | null;
tilesId?: number;
},
): TileBand | null {
if (!(width > 0 && height > 0) || mask.length < width * height) return null;
const minRowGrout = opts?.minRowGrout ?? 0.002;
const minRowTilesGrout = opts?.minRowTilesGrout ?? 0.01;
const smoothWindow = opts?.smoothWindow ?? 9;
const excludeBottomFrac = opts?.excludeBottomFrac ?? 0.15;
const groutId = opts?.groutId ?? CLASS_GROUT;
const tilesId = opts?.tilesId ?? CLASS_TILES;
const groutCounts = new Float32Array(height);
const tilesCounts = new Float32Array(height);
for (let y = 0; y < height; y++) {
const row = y * width;
for (let x = 0; x < width; x++) {
const v = mask[row + x];
if (groutId != null && v === groutId) groutCounts[y] += 1;
else if (v === tilesId) tilesCounts[y] += 1;
}
}
const rowGroutFrac = new Float32Array(height);
const rowTilesGroutFrac = new Float32Array(height);
for (let y = 0; y < height; y++) {
rowGroutFrac[y] = groutCounts[y] / Math.max(1, width);
rowTilesGroutFrac[y] = (groutCounts[y] + tilesCounts[y]) / Math.max(1, width);
}
const groutSmooth = smooth1D(rowGroutFrac, smoothWindow);
const tilesSmooth = smooth1D(rowTilesGroutFrac, smoothWindow);
type Segment = { yMin: number; yMax: number; score: number; confidence: number; center: number };
const segments: Segment[] = [];
let y = 0;
while (y < height) {
if (groutSmooth[y] < minRowGrout || tilesSmooth[y] < minRowTilesGrout) {
y += 1;
continue;
}
const start = y;
while (
y < height &&
groutSmooth[y] >= minRowGrout &&
tilesSmooth[y] >= minRowTilesGrout
) {
y += 1;
}
const end = y;
const center = (start + end - 1) / 2;
const centerFrac = center / Math.max(1, height);
if (centerFrac > 1 - excludeBottomFrac) continue;
let score = 0;
let confidence = 0;
for (let yy = start; yy < end; yy++) {
score += tilesSmooth[yy];
confidence += tilesSmooth[yy];
}
const length = Math.max(1, end - start);
confidence = confidence / length;
segments.push({ yMin: start, yMax: end, score, confidence, center });
}
if (!segments.length) return null;
segments.sort((a, b) => b.score - a.score || (b.yMax - b.yMin) - (a.yMax - a.yMin));
const best = segments[0];
return {
yMin: best.yMin,
yMax: best.yMax,
confidence: formatCoverage(best.confidence),
};
}
export function detectTileBandFromTilesBinary(
tilesBinary: Uint8Array,
validMask: Uint8Array | null,
width: number,
height: number,
opts?: {
smoothWindow?: number;
excludeBottomFrac?: number;
minStartFrac?: number;
minRowFracMin?: number;
maxRowFracToThreshFrac?: number;
peakThresholdFrac?: number;
minBandFrac?: number;
},
): TileBand | null {
if (!(width > 0 && height > 0) || tilesBinary.length < width * height) return null;
const smoothWindow = opts?.smoothWindow ?? 11;
const excludeBottomFrac = opts?.excludeBottomFrac ?? 0.15;
const minStartFrac = opts?.minStartFrac ?? 0.08;
const minRowFracMin = opts?.minRowFracMin ?? 0.12;
const maxRowFracToThreshFrac = opts?.maxRowFracToThreshFrac ?? 0.5;
const peakThresholdFrac = opts?.peakThresholdFrac ?? 0.65;
const minBandFrac = opts?.minBandFrac ?? 0.12;
const rowTilesFrac = new Float32Array(height);
for (let y = 0; y < height; y++) {
const row = y * width;
let tiles = 0;
let denom = 0;
for (let x = 0; x < width; x++) {
const idx = row + x;
if (validMask && !validMask[idx]) continue;
denom += 1;
if (tilesBinary[idx] > 127) tiles += 1;
}
rowTilesFrac[y] = tiles / Math.max(1, denom);
}
const smooth = smooth1D(rowTilesFrac, smoothWindow);
const searchStart = Math.floor(height * minStartFrac);
const searchEnd = Math.max(searchStart + 1, Math.floor(height * (1 - excludeBottomFrac)));
let peakVal = 0;
let peakIdx = -1;
for (let i = searchStart; i < Math.min(height, searchEnd); i++) {
const v = smooth[i] ?? 0;
if (v > peakVal) {
peakVal = v;
peakIdx = i;
}
}
if (!(peakVal >= minRowFracMin) || peakIdx < 0) return null;
// Expand around the densest rows (backsplash), avoiding sparse speckle regions.
// This is intentionally stricter than a simple "above threshold segment" scan.
const expandWithThreshold = (thr: number) => {
let start = peakIdx;
while (start > 0 && (smooth[start - 1] ?? 0) >= thr) start -= 1;
let end = peakIdx;
while (end + 1 < height && (smooth[end + 1] ?? 0) >= thr) end += 1;
return { start, end };
};
let thr = Math.max(minRowFracMin, peakVal * peakThresholdFrac, peakVal * maxRowFracToThreshFrac);
let { start, end } = expandWithThreshold(thr);
const bandFrac = (end - start + 1) / Math.max(1, height);
if (bandFrac < minBandFrac) {
// If the band is implausibly thin, relax slightly (common when a large foreground object
// reduces tile density in many rows, or when predictions are fragmented).
thr = Math.max(minRowFracMin, peakVal * (peakThresholdFrac * 0.85));
({ start, end } = expandWithThreshold(thr));
}
const bandFrac2 = (end - start + 1) / Math.max(1, height);
if (bandFrac2 < minBandFrac) {
thr = Math.max(minRowFracMin, peakVal * 0.45);
({ start, end } = expandWithThreshold(thr));
}
const center = (start + end) / 2;
const centerFrac = center / Math.max(1, height);
if (centerFrac > 1 - excludeBottomFrac) return null;
if (start / Math.max(1, height) < minStartFrac) return null;
let confidence = 0;
for (let yy = start; yy <= end; yy++) confidence += smooth[yy] ?? 0;
confidence = confidence / Math.max(1, end - start + 1);
return {
yMin: start,
yMax: end + 1,
confidence: formatCoverage(confidence),
};
}
async function resizeMaskToMatch(
mask: Uint8Array,
srcWidth: number,
srcHeight: number,
dstWidth: number,
dstHeight: number,
): Promise<Uint8Array> {
const threshold = 128;
if (srcWidth === dstWidth && srcHeight === dstHeight) {
return binarize(mask, threshold);
}
const resized = await sharp(Buffer.from(mask), {
raw: { width: srcWidth, height: srcHeight, channels: 1 },
})
.resize(dstWidth, dstHeight, { kernel: 'nearest' })
.threshold(threshold)
.toColorspace('b-w')
.raw()
.toBuffer();
return new Uint8Array(resized);
}
async function resizeRgbaToMatch(
rgba: Uint8Array,
srcWidth: number,
srcHeight: number,
dstWidth: number,
dstHeight: number,
channels = 4,
): Promise<Uint8Array> {
const srcChannels = Math.max(3, Math.min(4, Math.round(channels || 4)));
if (srcWidth === dstWidth && srcHeight === dstHeight && srcChannels === 4) {
return new Uint8Array(rgba);
}
let pipeline = sharp(Buffer.from(rgba), {
raw: { width: srcWidth, height: srcHeight, channels: srcChannels },
}).resize(dstWidth, dstHeight, { kernel: 'lanczos3' });
if (srcChannels !== 4) {
pipeline = pipeline.ensureAlpha();
}
const resized = await pipeline.raw().toBuffer();
return new Uint8Array(resized);
}
async function encodeMask(mask: Uint8Array, width: number, height: number): Promise<string> {
const buffer = await sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.toColorspace('b-w')
.png({ compressionLevel: 4 })
.toBuffer();
return `data:image/png;base64,${buffer.toString('base64')}`;
}
export async function combineTileMaskWithOccluder(
tile: TileSegmentationResult,
occluder: OccluderMaskPayload | null,
options: CombineTileMaskOptions = {},
): Promise<TileMaskCombineResult> {
// UNet classes are model-dependent.
// For the 2-class tiles model: 0=background, 1=tiles.
// Grout is synthesized heuristically from tile edges when needed.
// wallMask: 1 = backsplash plane; occluder masks: 1 = foreground objects
const totalPixels = tile.ids.length || 1;
const modelClasses = tile.classes;
const classByName = (name: string) =>
modelClasses.find((cls) => String(cls.name).toLowerCase() === name.toLowerCase()) ?? null;
const backgroundId = classByName('background')?.id ?? CLASS_BACKGROUND;
const tilesIdModel = classByName('tiles')?.id ?? (modelClasses.length > 1 ? 1 : 0);
const groutIdModel = classByName('grout')?.id ?? null;
const isTwoClassTilesModel = groutIdModel == null && modelClasses.length === 2;
let tilesBinaryRaw: Uint8Array | null = null;
let tilesBinaryForComponents: Uint8Array | null = null;
void Math.max(0, Math.min(modelClasses.length - 1, options.backgroundClassId ?? backgroundId));
const bandOverride = options.bandOverride;
let band: TileBand | null = null;
if (bandOverride && tile.height > 0) {
const { yMinNorm, yMaxNorm } = bandOverride;
const yMin = Math.max(0, Math.min(tile.height, Math.round(yMinNorm * tile.height)));
const yMax = Math.max(yMin, Math.min(tile.height, Math.round(yMaxNorm * tile.height)));
if (yMax > yMin) {
band = { yMin, yMax, confidence: 1 };
}
}
let wallMask: Uint8Array | null = null;
let wallPixelCount = totalPixels;
if (options.wallMask?.data && options.wallMask.width > 0 && options.wallMask.height > 0) {
wallMask = await resizeMaskToMatch(
options.wallMask.data,
options.wallMask.width,
options.wallMask.height,
tile.width,
tile.height,
);
// Depth wall masks can be speckled at low resolution; close small holes for 2-class tiles.
if (isTwoClassTilesModel && wallMask) {
const closed = morphErode(
morphDilate(wallMask, tile.width, tile.height, 2),
tile.width,
tile.height,
2,
);
wallMask = closed;
}
wallPixelCount = countActive(wallMask) || totalPixels;
}
let wallMaskForIds = isTwoClassTilesModel ? null : wallMask;
let tileMaskV3Info: CombinedTileMask['tileMaskV3'] = null;
let tileMaskV3AppliedWarning: string | null = null;
let tileMaskV2Info: CombinedTileMask['tileMaskV2'] = null;
let tileMaskV2AppliedWarning: string | null = null;
let tileMaskV2BboxOverride: { xMin: number; xMax: number; yMin: number; yMax: number } | null = null;
const gridBandCandidate = !band ? bandFromGrid(options.grid ?? null, tile.width, tile.height) : null;
const fixtureLimit = isTwoClassTilesModel && occluder?.data && occluder.width > 0 && occluder.height > 0
? await estimateFixtureLimit(occluder, tile.width, tile.height)
: null;
const gridBandBelowFixture = fixtureLimit != null
? bandFromGridBelowFixture(options.grid ?? null, tile.width, tile.height, fixtureLimit)
: null;
if (!band) {
if (isTwoClassTilesModel) {
// 2-class tiles models can produce low-density speckle noise across plain walls.
// A row-density based band finder is much more robust than the generic 3-class logic.
tilesBinaryRaw = new Uint8Array(tile.ids.length);
for (let i = 0; i < tile.ids.length; i++) tilesBinaryRaw[i] = tile.ids[i] === tilesIdModel ? 255 : 0;
// Build a "valid" region for band detection: ignore occluders (hood/shelves) so they don't
// artificially reduce row density and collapse the band.
const validMask = wallMaskForIds
? Uint8Array.from(wallMaskForIds, (v) => (v ? 1 : 0))
: new Uint8Array(tile.width * tile.height).fill(1);
let topOccluderCoverage = 0;
if (occluder?.data && occluder.width > 0 && occluder.height > 0) {
const resizedOcc = await resizeMaskToMatch(
occluder.data,
occluder.width,
occluder.height,
tile.width,
tile.height,
);
if (!tilesBinaryForComponents) {
const cleaned = new Uint8Array(tilesBinaryRaw);
const limit = Math.min(cleaned.length, resizedOcc.length);
for (let i = 0; i < limit; i++) {
if (resizedOcc[i] > 127) cleaned[i] = 0;
}
// Build a "row-dense" mask to suppress speckles on upper walls.
const rowCoverage = new Float32Array(tile.height);
for (let y = 0; y < tile.height; y++) {
const row = y * tile.width;
let tiles = 0;
let denom = 0;
for (let x = 0; x < tile.width; x++) {
const idx = row + x;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
denom += 1;
if (cleaned[idx] > 0) tiles += 1;
}
rowCoverage[y] = tiles / Math.max(1, denom);
}
const sorted = Array.from(rowCoverage).sort((a, b) => a - b);
const p80 = sorted[Math.floor(sorted.length * 0.8)] ?? 0;
const rowThreshold = Math.max(0.12, p80 * 0.5);
const dense = new Uint8Array(cleaned);
for (let y = 0; y < tile.height; y++) {
if (rowCoverage[y] < rowThreshold) {
const row = y * tile.width;
for (let x = 0; x < tile.width; x++) dense[row + x] = 0;
}
}
const minArea = Math.max(800, Math.floor(tile.width * tile.height * 0.0015));
tilesBinaryForComponents = suppressSmallComponents(dense, tile.width, tile.height, minArea);
}
const limit = Math.min(validMask.length, resizedOcc.length);
// Detect presence of "upper fixtures" (shelves/hood/cabinets) which often sit above the backsplash.
// When present, we bias the band start downward so we don't leak tiles into the upper wall region.
const topH = Math.max(1, Math.floor(tile.height * 0.28));
let topSolid = 0;
let topTotal = 0;
for (let i = 0; i < limit; i++) {
if (resizedOcc[i] > 127) validMask[i] = 0;
const y = Math.floor(i / tile.width);
if (y < topH) {
topTotal += 1;
if (resizedOcc[i] > 127) topSolid += 1;
}
}
topOccluderCoverage = topTotal > 0 ? topSolid / topTotal : 0;
}
const biasedMinStartFrac =
topOccluderCoverage > 0.08 ? 0.18 : topOccluderCoverage > 0.04 ? 0.14 : 0.08;
const bandInput = tilesBinaryForComponents ?? tilesBinaryRaw;
band = detectTileBandFromTilesBinary(bandInput, validMask, tile.width, tile.height, {
smoothWindow: 11,
excludeBottomFrac: 0.15,
minStartFrac: biasedMinStartFrac,
minRowFracMin: 0.08,
maxRowFracToThreshFrac: 0.5,
peakThresholdFrac: 0.65,
minBandFrac: 0.12,
});
if (!band) {
// Relaxed fallback (still avoids starting at y=0) so we don't return "no band" on weak predictions.
band = detectTileBandFromTilesBinary(bandInput, validMask, tile.width, tile.height, {
smoothWindow: 9,
excludeBottomFrac: 0.15,
minStartFrac: Math.max(0.06, biasedMinStartFrac * 0.85),
minRowFracMin: 0.06,
maxRowFracToThreshFrac: 0.35,
peakThresholdFrac: 0.55,
minBandFrac: 0.1,
});
}
// If the UNet band is contaminated by speckle noise above the backsplash, the grid band is usually
// more reliable. Prefer intersection (to stay conservative); otherwise fall back to the grid band.
const gridBandRef = gridBandBelowFixture ?? gridBandCandidate;
if (gridBandRef) {
if (band) {
const yMin = Math.max(band.yMin, gridBandRef.yMin);
const yMax = Math.min(band.yMax, gridBandRef.yMax);
if (yMax - yMin >= Math.round(tile.height * 0.06)) {
band = { yMin, yMax, confidence: Math.min(band.confidence, gridBandRef.confidence) };
} else {
band = gridBandRef;
}
} else {
band = gridBandRef;
}
}
if (!tilesBinaryForComponents && tilesBinaryRaw) {
const minArea = Math.max(800, Math.floor(tile.width * tile.height * 0.001));
tilesBinaryForComponents = suppressSmallComponents(tilesBinaryRaw, tile.width, tile.height, minArea);
}
}
}
// If the grid band below occluders starts significantly higher than the current band,
// prefer its start to avoid truncating the backsplash behind shelves.
if (band && gridBandBelowFixture) {
const minBandH = Math.round(tile.height * 0.08);
const raiseThreshold = Math.round(tile.height * 0.02);
if (band.yMin - gridBandBelowFixture.yMin > raiseThreshold) {
const yMin = gridBandBelowFixture.yMin;
const yMax = Math.max(band.yMax, gridBandBelowFixture.yMax);
if (yMax - yMin >= minBandH) {
band = { yMin, yMax, confidence: Math.min(band.confidence, gridBandBelowFixture.confidence) };
}
}
}
if (!band) band = gridBandCandidate;
// For 2-class models, prefer the grid-derived band (below fixtures if available).
// This avoids speckle-driven band collapse when UNet output is noisy.
if (isTwoClassTilesModel) {
const preferred = gridBandBelowFixture ?? gridBandCandidate;
if (preferred) {
const minBandH = Math.round(tile.height * 0.08);
if (preferred.yMax - preferred.yMin >= minBandH) {
band = preferred;
}
}
}
if (
isTwoClassTilesModel &&
band &&
gridBandCandidate &&
band.confidence < 0.5 &&
gridBandCandidate.yMin < band.yMin
) {
band = gridBandCandidate;
}
// For 2-class models, use the occluder row-coverage to estimate the shelf/hood bottom
// and lift the band start upward when it is clearly too low.
if (isTwoClassTilesModel && band && occluder?.data && occluder.width > 0 && occluder.height > 0) {
try {
const occ = await resizeMaskToMatch(occluder.data, occluder.width, occluder.height, tile.width, tile.height);
const topLimit = Math.max(1, Math.floor(tile.height * 0.5));
const centerMin = Math.floor(tile.width * 0.35);
const centerMax = Math.floor(tile.width * 0.65);
let lastRow = -1;
for (let y = 0; y < topLimit; y++) {
const row = y * tile.width;
let solid = 0;
for (let x = 0; x < tile.width; x++) {
if (x >= centerMin && x <= centerMax) continue;
if (occ[row + x]! > 127) solid += 1;
}
const denom = Math.max(1, tile.width - (centerMax - centerMin + 1));
const frac = solid / denom;
if (frac >= 0.01) lastRow = y;
}
if (lastRow >= 0) {
const yTop = Math.round(tile.height * 0.06);
const hlines = Array.isArray(options.grid?.hlines)
? options.grid!.hlines.filter((n) => Number.isFinite(n))
: [];
const tileH = hlines.length >= 2 ? medianSpacing(hlines) : Math.max(6, Math.round(tile.height * 0.04));
const margin = Math.max(2, Math.round(tileH * 0.4));
const candidate = clampInt(Math.max(yTop, lastRow + margin), 0, tile.height);
const minBandH = Math.round(tile.height * 0.08);
if (candidate < band.yMin - Math.round(tile.height * 0.04) && band.yMax - candidate >= minBandH) {
band = { ...band, yMin: candidate };
}
}
} catch {
// ignore
}
}
if (!band && !isTwoClassTilesModel) {
band = detectTileBand(tile.ids, tile.width, tile.height, {
minRowGrout: groutIdModel == null ? 0 : 0.002,
minRowTilesGrout: 0.01,
smoothWindow: 9,
excludeBottomFrac: 0.15,
groutId: groutIdModel,
tilesId: tilesIdModel,
});
}
const tileMaskV3Enabled = options.tileMaskV3Enabled ?? (process.env.TILEMASK_V3 === '1');
const tileMaskV2Enabled = options.tileMaskV2Enabled ?? (process.env.TILEMASK_V2 === '1');
if (
(tileMaskV3Enabled || tileMaskV2Enabled) &&
options.sourceImageRgba?.data &&
options.sourceImageRgba.width > 0 &&
options.sourceImageRgba.height > 0
) {
try {
const src = options.sourceImageRgba;
const rgbaForTile = await resizeRgbaToMatch(
src.data,
src.width,
src.height,
tile.width,
tile.height,
src.channels ?? 4,
);
const tilesBinarySupport = new Uint8Array(tile.ids.length);
for (let i = 0; i < tile.ids.length; i++) {
tilesBinarySupport[i] = tile.ids[i] === tilesIdModel ? 255 : 0;
}
let occluderResizedForV2: Uint8Array | null = null;
if (occluder?.data && occluder.width > 0 && occluder.height > 0) {
occluderResizedForV2 = await resizeMaskToMatch(
occluder.data,
occluder.width,
occluder.height,
tile.width,
tile.height,
);
}
if (tileMaskV3Enabled) {
const v3 = solveTileMaskV3Core({
rgba: rgbaForTile,
width: tile.width,
height: tile.height,
grid: options.grid ?? null,
tileBand: band,
tilesBinaryRawSupport: tilesBinarySupport,
wallPlaneMask: wallMask,
occluderBinary: occluderResizedForV2,
});
tileMaskV3Info = {
applied: v3.applied,
reason: v3.reason,
failureReason: v3.failureReason ?? null,
confidence: v3.confidence,
fallbackMode: v3.fallbackMode,
metrics: v3.metrics,
diagnostics: v3.diagnostics,
};
if (v3.applied) {
wallMaskForIds = v3.backsplashRoi;
wallPixelCount = countActive(wallMaskForIds) || totalPixels;
const bbox = tileMaskV3Bbox(v3.backsplashRoi, tile.width, tile.height);
if (bbox) {
tileMaskV2BboxOverride = bbox;
band = {
yMin: bbox.yMin,
yMax: Math.min(tile.height, bbox.yMax + 1),
confidence: 1,
};
}
tileMaskV3AppliedWarning = 'tilemask_v3_applied';
} else {
tileMaskV3AppliedWarning = `tilemask_v3_skip:${v3.failureReason ?? v3.reason ?? 'low_confidence_roi'}`;
}
}
if (!tileMaskV3Info?.applied && tileMaskV2Enabled) {
const v2 = solveTileMaskV2Core({
rgba: rgbaForTile,
width: tile.width,
height: tile.height,
grid: options.grid ?? null,
tileBand: band,
tilesBinaryRawSupport: tilesBinarySupport,
wallPlaneMask: wallMask,
occluderBinary: occluderResizedForV2,
});
tileMaskV2Info = {
applied: v2.applied,
reason: v2.reason,
metrics: v2.metrics,
};
if (v2.applied) {
wallMaskForIds = v2.backsplashRoi;
wallPixelCount = countActive(wallMaskForIds) || totalPixels;
const bbox = tileMaskV2Bbox(v2.backsplashRoi, tile.width, tile.height);
if (bbox) {
tileMaskV2BboxOverride = bbox;
band = {
yMin: bbox.yMin,
yMax: Math.min(tile.height, bbox.yMax + 1),
confidence: 1,
};
}
tileMaskV2AppliedWarning = 'tilemask_v2_applied';
} else {
tileMaskV2AppliedWarning = `tilemask_v2_skip:${v2.reason ?? 'low_confidence_roi'}`;
}
}
} catch {
if (tileMaskV3Enabled) tileMaskV3AppliedWarning = 'tilemask_v3_error';
if (tileMaskV2Enabled) tileMaskV2AppliedWarning = 'tilemask_v2_error';
}
}
const bandedIds = Uint8Array.from(tile.ids);
const gridRoi = roiFromGrid(options.grid ?? null, tile.width, tile.height);
if (gridRoi && band) {
// For 2-class tiles, prefer the broader grid ROI to avoid truncating tiles behind shelves.
const useUnion = isTwoClassTilesModel;
let yMin = useUnion ? Math.min(band.yMin, gridRoi.yMin) : Math.max(band.yMin, gridRoi.yMin);
const yMax = useUnion ? Math.max(band.yMax, gridRoi.yMax) : Math.min(band.yMax, gridRoi.yMax);
if (useUnion && fixtureLimit != null) {
yMin = Math.max(yMin, fixtureLimit);
}
if (yMax - yMin >= Math.round(tile.height * 0.06)) {
band = { yMin, yMax, confidence: band.confidence };
}
} else if (gridRoi && !band) {
band = { yMin: gridRoi.yMin, yMax: gridRoi.yMax, confidence: 1 };
}
// Final guard: if fixtures exist, never let the band start above their bottom edge.
if (band && isTwoClassTilesModel && fixtureLimit != null && fixtureLimit > band.yMin) {
const minBandH = Math.round(tile.height * 0.08);
const maxYMin = Math.max(0, band.yMax - minBandH);
const clamped = Math.min(Math.max(band.yMin, fixtureLimit), maxYMin);
if (clamped > band.yMin) {
band = { ...band, yMin: clamped };
}
}
// If upper fixtures (hood/shelves/cabinets) are present, estimate their bottom edge and use it as a
// *top boundary* for the backsplash band. This prevents tiling the plain upper wall (speckle noise),
// while still allowing us to expand the band upward to include tiles behind shelves/hood.
if (band && occluder?.data && occluder.width > 0 && occluder.height > 0) {
try {
const occ = await resizeMaskToMatch(occluder.data, occluder.width, occluder.height, tile.width, tile.height);
const limitY = Math.max(1, Math.floor(tile.height * 0.5));
const centerMin = Math.floor(tile.width * 0.35);
const centerMax = Math.floor(tile.width * 0.65);
const stepX = Math.max(1, Math.floor(tile.width / 220));
const bottoms: number[] = [];
for (let x = 0; x < tile.width; x += stepX) {
if (x >= centerMin && x <= centerMax) continue;
let bottom = -1;
let colSolid = 0;
for (let y = 0; y < limitY; y++) {
if (occ[y * tile.width + x]! > 127) {
bottom = y;
colSolid += 1;
}
}
if (bottom >= 0 && colSolid / limitY < 0.35) bottoms.push(bottom);
}
if (bottoms.length >= 12) {
bottoms.sort((a, b) => a - b);
// Use a mid-quantile (not p80) so wide hoods don't dominate over shelves.
const p10 = bottoms[Math.floor(bottoms.length * 0.1)] ?? -1;
if (p10 >= 0) {
const margin = Math.max(2, Math.round(tile.height * 0.015));
const fixtureY = clampInt(p10 + margin, 0, tile.height);
const minBandH = Math.round(tile.height * 0.08);
const yTop = Math.round(tile.height * 0.06);
const desiredYMin = clampInt(Math.max(yTop, fixtureY), 0, tile.height);
// Expand upward to include tiles behind fixtures, but never above the fixture bottom.
if (desiredYMin < band.yMin && band.yMax - desiredYMin >= minBandH) {
band = { ...band, yMin: desiredYMin };
}
}
}
} catch {
// ignore
}
}
// If we have grid hlines above the current band, extend the band upward to the topmost
// regular grid run *below the fixture bottom*. This restores backsplash coverage behind shelves
// without leaking into plain wall.
if (band && options.grid?.hlines?.length) {
const yTop = Math.round(tile.height * 0.06);
const fixtureClamp = fixtureLimit != null ? fixtureLimit : yTop;
const hlines = options.grid.hlines
.filter((n) => Number.isFinite(n) && n >= fixtureClamp && n <= band.yMin)
.sort((a, b) => a - b);
if (hlines.length >= 3) {
const spacing = medianSpacing(hlines);
if (spacing > 0) {
const tol = spacing * 0.35;
let runStart = hlines[0]!;
let runEnd = hlines[0]!;
let runLen = 1;
let bestStart = -1;
let bestEnd = -1;
let bestLen = 0;
for (let i = 1; i < hlines.length; i++) {
const d = hlines[i]! - hlines[i - 1]!;
if (Math.abs(d - spacing) <= tol) {
runEnd = hlines[i]!;
runLen += 1;
} else {
if (runLen >= 3 && bestStart < 0) {
bestStart = runStart;
bestEnd = runEnd;
bestLen = runLen;
break; // take the top-most valid run
}
runStart = hlines[i]!;
runEnd = hlines[i]!;
runLen = 1;
}
}
if (bestStart < 0 && runLen >= 3) {
bestStart = runStart;
bestEnd = runEnd;
bestLen = runLen;
}
if (bestStart >= 0 && bestLen >= 3) {
const pad = Math.max(2, Math.round(spacing * 0.35));
const newYMin = clampInt(Math.max(fixtureClamp, bestStart - pad), 0, tile.height);
const minBandH = Math.round(tile.height * 0.08);
if (newYMin < band.yMin && band.yMax - newYMin >= minBandH) {
band = { ...band, yMin: newYMin };
}
}
}
}
}
// If the grid-derived band starts noticeably higher (more coverage) than the current band,
// prefer it to avoid truncating the backsplash when occluders bias the band downward.
if (band && gridBandCandidate) {
const minBandH = Math.round(tile.height * 0.08);
const raiseThreshold = Math.round(tile.height * 0.06);
const maxRaiseY = Math.round(tile.height * 0.3);
const yTop = Math.round(tile.height * 0.06);
const fixtureClamp = fixtureLimit != null ? fixtureLimit : yTop;
if (band.yMin - gridBandCandidate.yMin > raiseThreshold && gridBandCandidate.yMin <= maxRaiseY) {
const yMin = Math.max(gridBandCandidate.yMin, fixtureClamp);
const yMax = Math.max(band.yMax, gridBandCandidate.yMax);
if (yMax - yMin >= minBandH) {
band = { yMin, yMax, confidence: Math.min(band.confidence, gridBandCandidate.confidence) };
}
}
}
if (isTwoClassTilesModel && tilesBinaryForComponents) {
const componentBand = bandFromLargestComponent(
tilesBinaryForComponents,
tile.width,
tile.height,
options.grid ?? null,
);
if (componentBand) {
const tall = band ? (band.yMax - band.yMin) > Math.round(tile.height * 0.85) : true;
const raise = band ? (componentBand.yMin - band.yMin) > Math.round(tile.height * 0.06) : true;
if (!band || tall || raise) {
band = componentBand;
}
}
}
// If the grid band is higher (more coverage) than the current band, expand upward for 2-class models.
if (isTwoClassTilesModel && band && gridBandCandidate) {
const minBandH = Math.round(tile.height * 0.08);
const raiseGap = Math.round(tile.height * 0.03);
if (band.yMin - gridBandCandidate.yMin > raiseGap) {
const yMin = Math.max(
Math.min(band.yMin, gridBandCandidate.yMin),
fixtureLimit != null ? fixtureLimit : 0,
);
const yMax = Math.max(band.yMax, gridBandCandidate.yMax);
if (yMax - yMin >= minBandH) {
band = { yMin, yMax, confidence: Math.min(band.confidence, gridBandCandidate.confidence) };
}
}
}
// Final clamp: never allow the band to start above the fixture bottom.
if (band && fixtureLimit != null && fixtureLimit > band.yMin) {
const minBandH = Math.round(tile.height * 0.08);
const maxYMin = Math.max(0, band.yMax - minBandH);
const clamped = Math.min(Math.max(band.yMin, fixtureLimit), maxYMin);
if (clamped > band.yMin) {
band = { ...band, yMin: clamped };
}
}
// If we couldn't build a full grid ROI (regular-run failure), still derive a conservative ROI from:
// - the detected tile band (y-range)
// - the min/max of vlines (x-range)
// This prevents the "speckle noise" mode where tiles are sparse and leak into upper walls.
const fallbackRoi = (() => {
if (gridRoi) return null;
if (!band) return null;
const grid = options.grid;
if (!grid) return null;
const v = Array.isArray(grid.vlines) ? grid.vlines.filter((n) => Number.isFinite(n)) : [];
if (v.length < 2) return null;
const vSorted = v.slice().sort((a, b) => a - b);
const tileW = medianSpacing(vSorted) || 0;
const padX = Math.max(2, Math.round((tileW || Math.max(8, tile.width * 0.04)) * 0.35));
const xMin = clampInt(vSorted[0]! - padX, 0, tile.width);
const xMax = clampInt(vSorted[vSorted.length - 1]! + padX, 0, tile.width);
if (xMax - xMin < Math.round(tile.width * 0.08)) return null;
return { xMin, xMax, yMin: band.yMin, yMax: band.yMax };
})();
const clipRoi = (() => {
if (tileMaskV2BboxOverride) {
return {
xMin: tileMaskV2BboxOverride.xMin,
xMax: tileMaskV2BboxOverride.xMax + 1,
yMin: tileMaskV2BboxOverride.yMin,
yMax: tileMaskV2BboxOverride.yMax + 1,
};
}
if (gridRoi) {
// Always honor the band y-range for clipping. For 2-class models, avoid
// tightening the band with grid ROI y-bounds (it can bias too low).
const yMin = band ? band.yMin : gridRoi.yMin;
const yMax = band ? band.yMax : gridRoi.yMax;
return { xMin: gridRoi.xMin, xMax: gridRoi.xMax, yMin, yMax };
}
return fallbackRoi;
})();
if (band) {
for (let y = 0; y < tile.height; y++) {
if (y < band.yMin || y >= band.yMax) {
const row = y * tile.width;
for (let x = 0; x < tile.width; x++) {
bandedIds[row + x] = backgroundId;
}
}
}
}
if (clipRoi) {
for (let y = 0; y < tile.height; y++) {
const row = y * tile.width;
if (y < clipRoi.yMin || y >= clipRoi.yMax) {
for (let x = 0; x < tile.width; x++) bandedIds[row + x] = backgroundId;
continue;
}
for (let x = 0; x < tile.width; x++) {
if (x < clipRoi.xMin || x >= clipRoi.xMax) {
bandedIds[row + x] = backgroundId;
}
}
}
}
if (wallMaskForIds) {
const limit = Math.min(bandedIds.length, wallMaskForIds.length);
for (let i = 0; i < limit; i++) {
if (!wallMaskForIds[i]) bandedIds[i] = backgroundId;
}
}
const effectiveClasses: ClassInfo[] = isTwoClassTilesModel
? [
{ id: 0, name: 'background', color: [0, 0, 0] },
{ id: 1, name: 'grout', color: [52, 152, 219] },
{ id: 2, name: 'tiles', color: [46, 204, 113] },
]
: modelClasses;
// Build combinedIds in "effective" id-space.
// - 3-class: keep as-is.
// - 2-class: promote tiles->2 and synthesize grout->1.
const combinedIds = new Uint8Array(bandedIds.length);
let filledFromGrid = false;
if (isTwoClassTilesModel) {
const tilesBin = new Uint8Array(bandedIds.length);
for (let i = 0; i < bandedIds.length; i++) tilesBin[i] = bandedIds[i] === tilesIdModel ? 255 : 0;
// If we have a stable grid ROI, we can safely "fill" the backsplash plane as tiles
// (minus occluders) so the overlay is visible even when UNet predictions are sparse/noisy.
// This also prevents grout heuristics from exploding due to speckle edges.
const fillRoi = clipRoi;
if (fillRoi) {
// Never fill outside the effective band; this is the main defense against
// "speckles above the real backsplash" when the grid ROI extends too far up.
const roiYMin = fillRoi.yMin;
const roiYMax = fillRoi.yMax;
const roiXMin = fillRoi.xMin;
const roiXMax = fillRoi.xMax;
let occResized: Uint8Array | null = null;
if (occluder?.data && occluder.width > 0 && occluder.height > 0) {
occResized = await resizeMaskToMatch(
occluder.data,
occluder.width,
occluder.height,
tile.width,
tile.height,
);
}
let roiTiles = 0;
let roiTotal = 0;
for (let y = roiYMin; y < roiYMax; y++) {
const row = y * tile.width;
for (let x = roiXMin; x < roiXMax; x++) {
const idx = row + x;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
if (occResized && occResized[idx] > 127) continue;
roiTotal += 1;
if (tilesBin[idx]) roiTiles += 1;
}
}
// For the 2-class model, fill the ROI (minus occluders) as tiles.
// This turns speckle masks into a coherent backsplash region for rendering.
if (roiTotal > 0) {
for (let y = roiYMin; y < roiYMax; y++) {
const row = y * tile.width;
for (let x = roiXMin; x < roiXMax; x++) {
const idx = row + x;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
if (occResized && occResized[idx] > 127) continue;
tilesBin[idx] = 255;
}
}
filledFromGrid = true;
}
}
const groutBin = fillRoi
? new Uint8Array(tilesBin.length)
: groutFromTiles({ data: tilesBin, width: tile.width, height: tile.height }, 2).data;
for (let i = 0; i < combinedIds.length; i++) {
if (tilesBin[i]) combinedIds[i] = 2;
else if (groutBin[i]) combinedIds[i] = 1;
else combinedIds[i] = 0;
}
} else {
combinedIds.set(bandedIds);
}
const baseCounts = new Float32Array(effectiveClasses.length);
const bandLimit = wallMaskForIds ? Math.min(combinedIds.length, wallMaskForIds.length) : combinedIds.length;
for (let i = 0; i < combinedIds.length; i++) {
if (wallMaskForIds && (i >= bandLimit || !wallMaskForIds[i])) continue;
const cls = combinedIds[i];
if (cls < baseCounts.length) baseCounts[cls] += 1;
}
let wallFillApplied = false;
const warnings: string[] = [];
if (tileMaskV3AppliedWarning) warnings.push(tileMaskV3AppliedWarning);
if (tileMaskV2AppliedWarning) warnings.push(tileMaskV2AppliedWarning);
if (filledFromGrid) warnings.push('tiles_filled_from_grid');
// Depth wall-mode fallback:
// If UNet does not predict tiles (class 2) but we have a wallMask (depth plane),
// fill the wall plane as tiles and keep grout predictions as-is.
if (options.enableDepthWallFill && wallMaskForIds) {
const denom = Math.max(1, wallPixelCount);
const tilesFrac = (baseCounts[CLASS_TILES] ?? 0) / denom;
const groutFrac = (baseCounts[CLASS_GROUT] ?? 0) / denom;
const tilesMissing = tilesFrac < 0.002;
// If grout exists but tiles are missing, the model is likely only picking grout lines.
// In that case, fill the remainder of the wall plane as tiles.
const bandFrac = band ? (band.yMax - band.yMin) / Math.max(1, tile.height) : 0;
const canFill = Boolean(band) && bandFrac >= 0.08;
if (tilesMissing && canFill && (groutFrac >= 0.001 || isTwoClassTilesModel)) {
const limit = Math.min(combinedIds.length, wallMaskForIds.length);
for (let idx = 0; idx < limit; idx++) {
const y = Math.floor(idx / tile.width);
if (band && (y < band.yMin || y >= band.yMax)) continue;
if (!wallMaskForIds[idx]) continue;
if (combinedIds[idx] === CLASS_BACKGROUND) {
combinedIds[idx] = CLASS_TILES;
}
}
wallFillApplied = true;
warnings.push('tiles_filled_from_wall');
}
}
if (!wallFillApplied) {
const denom = Math.max(1, wallMaskForIds ? wallPixelCount : totalPixels);
const tilesFrac = (baseCounts[CLASS_TILES] ?? 0) / denom;
if (tilesFrac <= 0) warnings.push('tiles_missing');
}
let occluderCoverage = 0;
let occludedPixels = 0;
let occludedCoverage = 0;
let occluderMaskPixels = 0;
let occluderDataUrl = occluder?.dataUrl ?? null;
if (occluder?.data && occluder.width > 0 && occluder.height > 0) {
const resized = await resizeMaskToMatch(
occluder.data,
occluder.width,
occluder.height,
tile.width,
tile.height,
);
const occluderMaskRaw = binarize(resized, 128);
const occluderErodePx = Math.max(
0,
Math.min(8, Math.round(Number(process.env.OCCLUDER_COMBINE_ERODE_PX ?? '2'))),
);
const occluderEroded = occluderErodePx > 0
? morphErode(occluderMaskRaw, tile.width, tile.height, occluderErodePx)
: occluderMaskRaw;
const rawStats = measureBinaryMask(occluderMaskRaw, wallMaskForIds ?? undefined);
const erodedStats = measureBinaryMask(occluderEroded, wallMaskForIds ?? undefined);
const erodeTooAggressive =
rawStats.coverage > 0.02 &&
(erodedStats.coverage < rawStats.coverage * 0.65 || erodedStats.coverage < 0.01);
const occluderMask = erodeTooAggressive ? occluderMaskRaw : occluderEroded;
if (erodeTooAggressive) warnings.push('occluder_erode_rollback');
const maskStats = erodeTooAggressive ? rawStats : erodedStats;
occluderMaskPixels = maskStats.solid;
occluderCoverage = formatCoverage(maskStats.coverage);
occluderDataUrl = await encodeMask(occluderMask, tile.width, tile.height);
const usable = Math.min(occluderMask.length, combinedIds.length);
for (let idx = 0; idx < usable; idx++) {
const y = Math.floor(idx / tile.width);
if (band && (y < band.yMin || y >= band.yMax)) continue;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
if (occluderMask[idx] > 127) {
if (combinedIds[idx] !== CLASS_BACKGROUND) {
occludedPixels += 1;
}
combinedIds[idx] = CLASS_IGNORE;
}
}
const occluderDenom = wallMaskForIds ? Math.max(1, wallPixelCount) : Math.max(1, totalPixels);
occludedCoverage = formatCoverage(occludedPixels / occluderDenom);
}
const combinedCounts = new Float32Array(effectiveClasses.length);
let minX = tile.width;
let minY = tile.height;
let maxX = -1;
let maxY = -1;
for (let y = 0; y < tile.height; y++) {
const row = y * tile.width;
for (let x = 0; x < tile.width; x++) {
const idx = row + x;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
const cls = combinedIds[idx];
if (cls < combinedCounts.length) {
combinedCounts[cls] += 1;
if (cls === CLASS_TILES || cls === CLASS_GROUT) {
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
}
}
const coverageDenom = wallMaskForIds ? Math.max(1, wallPixelCount) : totalPixels;
const coverageAbsolute = coverageFromCounts(baseCounts, effectiveClasses, coverageDenom);
let coverageInterior: Record<string, number>;
if (maxX >= minX && maxY >= minY) {
const bboxCounts = new Float32Array(effectiveClasses.length);
let denom = 0;
for (let y = minY; y <= maxY; y++) {
const row = y * tile.width;
for (let x = minX; x <= maxX; x++) {
const idx = row + x;
if (wallMaskForIds && !wallMaskForIds[idx]) continue;
const cls = combinedIds[idx];
if (cls === CLASS_IGNORE) continue;
if (cls < bboxCounts.length) bboxCounts[cls] += 1;
denom += 1;
}
}
const areaDenom = wallMaskForIds ? Math.max(1, denom) : Math.max(1, denom);
coverageInterior = coverageFromCounts(bboxCounts, effectiveClasses, areaDenom);
} else {
const interiorDenom = Math.max(1, sumCounts(combinedCounts));
coverageInterior = coverageFromCounts(combinedCounts, effectiveClasses, interiorDenom);
}
const dataUrl = await renderTileMask(combinedIds, tile.width, tile.height, effectiveClasses);
if (process.env.TILE_MASK_DEBUG === '1') {
const hist: Record<number, number> = {};
combinedIds.forEach((v) => {
hist[v] = (hist[v] ?? 0) + 1;
});
console.info('[tile-mask] histogram', hist);
}
return {
combined: {
ids: combinedIds,
dataUrl,
coverage: coverageInterior,
coverageAbsolute,
occludedCoverage,
occluderCoverage,
occludedPixels,
occluderPixels: occluderMaskPixels,
source: `${isTwoClassTilesModel ? 'unet_latest_2class' : 'unet_latest'}${occluder ? '+occluder' : ''}${wallFillApplied ? '+depth-wall-fill' : ''}${isTwoClassTilesModel ? '+grout_heur' : ''}`,
tileBand: band,
...(tileMaskV3Info ? { tileMaskV3: tileMaskV3Info } : {}),
...(tileMaskV2Info ? { tileMaskV2: tileMaskV2Info } : {}),
...(warnings.length ? { warnings } : {}),
},
rawDataUrl: tile.dataUrl,
occluderDataUrl,
};
}