| Server IP : 217.154.3.148 / Your IP : 216.73.216.90 Web Server : Apache System : Linux blissful-wright.217-154-3-148.plesk.page 6.8.0-124-generic #124-Ubuntu SMP PREEMPT_DYNAMIC Tue May 26 13:00:45 UTC 2026 x86_64 User : gracious-boyd_hfhh1h8vo9n ( 10000) PHP Version : 8.3.31 Disable Function : opcache_get_status MySQL : OFF | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : /var/www/vhosts/gracious-boyd.217-154-3-148.plesk.page/nextjs/lib/segmentation/ |
Upload File : |
import { access, readFile, stat } from 'node:fs/promises';
import path from 'node:path';
import sharp from 'sharp';
import type { InferenceSession, Tensor } from 'onnxruntime-node';
export type ClassInfo = {
id: number;
name: string;
color: [number, number, number];
};
export type TileSegmentationMeta = {
processedWidth: number;
processedHeight: number;
originalWidth: number;
originalHeight: number;
scale: number;
};
export type TileUnetChannelStats = {
min: number;
max: number;
mean: number;
};
export type TileUnetDebug = {
input: {
width: number;
height: number;
processedWidth: number;
processedHeight: number;
tile: number;
stride: number;
scale: number;
};
model?: {
path: string;
sizeBytes: number;
mtimeMs: number;
inputs?: string[];
outputs?: string[];
inputMetadata?: Record<string, unknown>;
};
inputStats?: {
normalizedMin: number;
normalizedMax: number;
normalizedMean: number;
channelStats: TileUnetChannelStats[];
samplePixels?: {
topLeft: number[];
center: number[];
};
};
output?: {
name: string;
dims: number[];
layout: 'nchw' | 'nhwc' | 'unknown';
width: number;
height: number;
classes: number;
classNames?: string[];
tileClassId?: number;
logitsStats?: TileUnetChannelStats[];
probsStats?: TileUnetChannelStats[];
};
argmaxCounts?: Record<string, number>;
};
export type TileSegmentationResult = {
ids: Uint8Array;
width: number;
height: number;
dataUrl: string;
classes: ClassInfo[];
coverage: Record<string, number>;
meta?: TileSegmentationMeta;
debug?: TileUnetDebug;
};
export type TileSegmentationOptions = {
tile?: number;
stride?: number;
mean?: [number, number, number];
std?: [number, number, number];
debug?: boolean;
};
const PROJECT_ROOT = process.cwd();
const MODEL_DIR = path.join(PROJECT_ROOT, 'models', 'unet_latest');
const MODEL_PATH = path.join(MODEL_DIR, 'unet_tiles.onnx');
const CLASS_NAMES_PATH = path.join(MODEL_DIR, 'class_names.json');
const COLOR_MAP_PATH = path.join(MODEL_DIR, 'color_map.json');
const MAX_INPUT_DIMENSION = Math.max(
0,
Number.parseInt(process.env.TILE_UNET_MAX_DIMENSION ?? '1600', 10) || 0,
);
const DEFAULT_CLASSES_2: ClassInfo[] = [
{ id: 0, name: 'background', color: [0, 0, 0] },
{ id: 1, name: 'tiles', color: [46, 204, 113] },
];
const DEFAULT_CLASSES_3: ClassInfo[] = [
{ id: 0, name: 'background', color: [0, 0, 0] },
{ id: 1, name: 'grout', color: [52, 152, 219] },
{ id: 2, name: 'tiles', color: [46, 204, 113] },
];
type OrtModule = typeof import('onnxruntime-node');
let ortModule: OrtModule | null = null;
function ensureOrt(): OrtModule {
if (!ortModule) {
// Lazy-require so Next.js doesn’t try to bundle the native addon.
ortModule = require('onnxruntime-node');
}
return ortModule;
}
let sessionPromise: Promise<InferenceSession> | null = null;
let classInfoPromise: Promise<ClassInfo[]> | null = null;
async function ensureSession(): Promise<InferenceSession> {
const ort = ensureOrt();
if (!sessionPromise) {
sessionPromise = (async () => {
await access(MODEL_PATH);
try {
const st = await stat(MODEL_PATH);
console.info('[tile-unet] initializing ONNX session', {
path: MODEL_PATH,
sizeBytes: st.size,
mtimeMs: st.mtimeMs,
});
} catch {
console.info('[tile-unet] initializing ONNX session', { path: MODEL_PATH });
}
try {
const session = await ort.InferenceSession.create(MODEL_PATH, {
graphOptimizationLevel: 'all',
});
console.info('[tile-unet] ONNX session ready');
return session;
} catch (error) {
console.warn('[tile-unet] session init failed, retrying with default settings', error);
return ort.InferenceSession.create(MODEL_PATH, {
graphOptimizationLevel: 'all',
});
}
})().catch((error) => {
sessionPromise = null;
throw new Error(
`[tile-unet] failed to load ONNX model at ${MODEL_PATH}: ${error instanceof Error ? error.message : error}`,
);
});
}
return sessionPromise;
}
async function loadClassInfo(): Promise<ClassInfo[]> {
if (!classInfoPromise) {
classInfoPromise = (async () => {
try {
const [namesRaw, colorsRaw] = await Promise.all([
access(CLASS_NAMES_PATH).then(() => importJson<string[]>(CLASS_NAMES_PATH)).catch(() => null),
access(COLOR_MAP_PATH).then(() => importJson<Record<string, [number, number, number]>>(COLOR_MAP_PATH)).catch(() => null),
]);
const names = Array.isArray(namesRaw) && namesRaw.length
? namesRaw.map((entry, idx) => String(entry ?? `class_${idx}`))
: DEFAULT_CLASSES_3.map((cls) => cls.name);
const colors = colorsRaw ?? Object.fromEntries(DEFAULT_CLASSES_3.map((cls) => [String(cls.id), cls.color]));
const mapped: ClassInfo[] = names.map((name, idx) => {
const fallback = idx < DEFAULT_CLASSES_3.length ? DEFAULT_CLASSES_3[idx].color : DEFAULT_CLASSES_3[0].color;
const color = colors[String(idx)] ?? fallback;
return { id: idx, name, color: [color[0], color[1], color[2]] } as ClassInfo;
});
if (!mapped.length) {
return DEFAULT_CLASSES_3;
}
return mapped;
} catch (error) {
console.warn('[tile-unet] falling back to default classes', error);
return DEFAULT_CLASSES_3;
}
})();
}
return classInfoPromise;
}
async function importJson<T>(filePath: string): Promise<T> {
const text = await readFile(filePath, 'utf8');
return JSON.parse(text) as T;
}
function normalizeImage(data: Buffer, width: number, height: number, mean: number[], std: number[]): Float32Array {
const total = width * height;
const out = new Float32Array(3 * total);
for (let i = 0; i < total; i++) {
const base = i * 3;
const r = data[base] / 255;
const g = data[base + 1] / 255;
const b = data[base + 2] / 255;
out[i] = (r - mean[0]) / std[0];
out[total + i] = (g - mean[1]) / std[1];
out[2 * total + i] = (b - mean[2]) / std[2];
}
return out;
}
function slidingPositions(size: number, tile: number, stride: number): number[] {
if (size <= tile) return [0];
const lastStart = Math.max(0, size - tile);
const positions: number[] = [];
for (let pos = 0; pos < lastStart; pos += stride) positions.push(pos);
positions.push(lastStart);
return Array.from(new Set(positions));
}
function fillPatch(
src: Float32Array,
dst: Float32Array,
width: number,
height: number,
tile: number,
startX: number,
startY: number,
channels: number,
): void {
const total = width * height;
const tileArea = tile * tile;
for (let c = 0; c < channels; c++) {
const srcOffset = c * total;
const dstOffset = c * tileArea;
let dstIdx = 0;
for (let py = 0; py < tile; py++) {
const yy = clamp(startY + py, 0, height - 1);
const row = yy * width;
for (let px = 0; px < tile; px++, dstIdx++) {
const xx = clamp(startX + px, 0, width - 1);
dst[dstOffset + dstIdx] = src[srcOffset + row + xx];
}
}
}
}
function clamp(value: number, min: number, max: number): number {
if (value < min) return min;
if (value > max) return max;
return value;
}
function resolveProcessingSize(width: number, height: number) {
if (!(width > 0 && height > 0) || MAX_INPUT_DIMENSION <= 0) {
return { width, height, scale: 1 };
}
const longest = Math.max(width, height);
if (longest <= MAX_INPUT_DIMENSION) {
return { width, height, scale: 1 };
}
const scale = MAX_INPUT_DIMENSION / longest;
const nextWidth = Math.max(32, Math.round(width * scale));
const nextHeight = Math.max(32, Math.round(height * scale));
return { width: nextWidth, height: nextHeight, scale };
}
async function upscaleMask(
ids: Uint8Array,
fromWidth: number,
fromHeight: number,
toWidth: number,
toHeight: number,
): Promise<Uint8Array> {
if (fromWidth === toWidth && fromHeight === toHeight) {
return ids;
}
const buffer = await sharp(Buffer.from(ids), {
raw: { width: fromWidth, height: fromHeight, channels: 1 },
})
.resize(toWidth, toHeight, { kernel: sharp.kernel.nearest })
// Ensure we keep a 1-channel (class-id) mask; sharp may otherwise expand to RGB.
.toColorspace('b-w')
.raw()
.toBuffer();
return new Uint8Array(buffer);
}
function computeCoverageFromIds(
ids: Uint8Array,
classes: ClassInfo[],
width: number,
height: number,
): Record<string, number> {
const totals = new Array<number>(classes.length).fill(0);
for (let idx = 0; idx < ids.length; idx++) {
const cls = ids[idx];
if (cls < totals.length) totals[cls] += 1;
}
const denom = Math.max(1, width * height);
const coverage: Record<string, number> = {};
classes.forEach((cls, idx) => {
coverage[cls.name] = Number((totals[idx] / denom).toFixed(6));
});
// Return raw fractions of the full image (not normalized to sum=1).
return coverage;
}
function normalizeCoverageMap(map: Record<string, number>): Record<string, number> {
const entries = Object.entries(map);
const sum = entries.reduce((acc, [, value]) => acc + (Number.isFinite(value) ? value : 0), 0);
if (!(sum > 0)) return map;
const normalized: Record<string, number> = {};
entries.forEach(([key, value]) => {
normalized[key] = Number(((value ?? 0) / sum).toFixed(6));
});
return normalized;
}
function resolveInputSize(
session: InferenceSession,
fallback: number,
providedInputName: string,
metadata: Record<string, { dimensions?: number[] }>,
): { inputName: string; tile: number; inputChannels: number } {
const inputName = providedInputName;
const meta = metadata[inputName];
const dims = Array.isArray(meta?.dimensions) ? meta.dimensions : null;
let tile = fallback;
let inputChannels = 3;
if (Array.isArray(dims) && dims.length >= 4) {
const h = dims[dims.length - 2];
const w = dims[dims.length - 1];
if (typeof h === 'number' && h > 0) tile = h;
if (typeof w === 'number' && w > 0) tile = Math.min(tile, w);
const cDim = dims[1];
if (typeof cDim === 'number' && cDim > 0) inputChannels = cDim;
}
tile = Math.max(32, tile);
return { inputName, tile, inputChannels };
}
type LogitLayout = 'nchw' | 'nhwc' | 'unknown';
function channelStats(
data: Float32Array,
channels: number,
area: number,
layout: LogitLayout,
): TileUnetChannelStats[] {
const stats: TileUnetChannelStats[] = [];
for (let c = 0; c < channels; c++) {
let min = Infinity;
let max = -Infinity;
let sum = 0;
for (let idx = 0; idx < area; idx++) {
const v =
layout === 'nhwc'
? data[idx * channels + c]
: data[c * area + idx];
if (v < min) min = v;
if (v > max) max = v;
sum += v;
}
stats.push({
min: Number(min.toFixed(4)),
max: Number(max.toFixed(4)),
mean: Number((sum / Math.max(1, area)).toFixed(6)),
});
}
return stats;
}
function flatStats(values: Float32Array): { min: number; max: number; mean: number } {
if (!values.length) return { min: 0, max: 0, mean: 0 };
let min = Infinity;
let max = -Infinity;
let sum = 0;
for (let i = 0; i < values.length; i++) {
const v = values[i]!;
if (v < min) min = v;
if (v > max) max = v;
sum += v;
}
return {
min: Number(min.toFixed(6)),
max: Number(max.toFixed(6)),
mean: Number((sum / Math.max(1, values.length)).toFixed(6)),
};
}
function sampleNormalizedPixels(
normalized: Float32Array,
width: number,
height: number,
): { topLeft: number[]; center: number[] } {
const total = width * height;
const sampleAt = (x: number, y: number) => {
const idx = Math.max(0, Math.min(total - 1, y * width + x));
const r = normalized[idx];
const g = normalized[total + idx];
const b = normalized[2 * total + idx];
return [Number(r.toFixed(4)), Number(g.toFixed(4)), Number(b.toFixed(4))];
};
return {
topLeft: sampleAt(0, 0),
center: sampleAt(Math.floor(width / 2), Math.floor(height / 2)),
};
}
function softmaxChannels(logits: Float32Array, channels: number, area: number, layout: LogitLayout): Float32Array {
const probs = new Float32Array(logits.length);
for (let idx = 0; idx < area; idx++) {
let maxLogit = -Infinity;
for (let c = 0; c < channels; c++) {
const value = layout === 'nhwc'
? logits[idx * channels + c]
: logits[c * area + idx];
if (value > maxLogit) maxLogit = value;
}
let sum = 0;
for (let c = 0; c < channels; c++) {
const src = layout === 'nhwc'
? logits[idx * channels + c]
: logits[c * area + idx];
const expVal = Math.exp(src - maxLogit);
// Store probs in NCHW layout for downstream code.
probs[c * area + idx] = expVal;
sum += expVal;
}
const inv = sum > 0 ? 1 / sum : 1;
for (let c = 0; c < channels; c++) {
probs[c * area + idx] *= inv;
}
}
return probs;
}
export async function renderTileMask(
ids: Uint8Array,
width: number,
height: number,
classes: ClassInfo[],
): Promise<string> {
const lut = new Map<number, [number, number, number]>();
classes.forEach((cls) => lut.set(cls.id, cls.color));
const rgba = Buffer.alloc(width * height * 4);
for (let i = 0; i < ids.length; i++) {
const color = lut.get(ids[i]) ?? [0, 0, 0];
const offset = i * 4;
rgba[offset] = color[0];
rgba[offset + 1] = color[1];
rgba[offset + 2] = color[2];
rgba[offset + 3] = 255;
}
const png = await sharp(rgba, { raw: { width, height, channels: 4 } }).png().toBuffer();
return `data:image/png;base64,${png.toString('base64')}`;
}
export async function inferTileSegmentation(
buffer: Buffer,
options: TileSegmentationOptions = {},
): Promise<TileSegmentationResult> {
const debugEnabled = Boolean(options.debug || process.env.TILE_UNET_DEBUG === '1');
const base = sharp(buffer, { failOnError: false });
const metadata = await base.metadata();
const originalWidth = metadata.width ?? 0;
const originalHeight = metadata.height ?? 0;
if (!(originalWidth > 0 && originalHeight > 0)) {
throw new Error('[tile-unet] failed to decode image dimensions');
}
const { width: targetWidth, height: targetHeight, scale } = resolveProcessingSize(
originalWidth,
originalHeight,
);
let pipeline = base.clone().ensureAlpha().removeAlpha();
if (scale !== 1 || targetWidth !== originalWidth || targetHeight !== originalHeight) {
pipeline = pipeline.resize(targetWidth, targetHeight, {
kernel: sharp.kernel.lanczos3,
});
}
const { data, info } = await pipeline
.raw()
.toBuffer({ resolveWithObject: true });
const width = info.width ?? targetWidth;
const height = info.height ?? targetHeight;
const session = await ensureSession();
const classes = await loadClassInfo();
const fallbackTile = Math.max(64, options.tile ?? 512);
const stride = Math.max(16, options.stride ?? 256);
const mean = options.mean ?? [0.485, 0.456, 0.406];
const std = options.std ?? [0.229, 0.224, 0.225];
const inputNames = Array.isArray(session.inputNames) ? session.inputNames : [];
const inputMetadata = (session as unknown as { inputMetadata?: Record<string, { dimensions?: number[] }> }).inputMetadata ?? {};
const resolvedInputName = inputNames[0] ?? Object.keys(inputMetadata)[0];
if (!resolvedInputName) {
throw new Error('[tile-unet] model does not expose any input tensor');
}
const { inputName, tile, inputChannels } = resolveInputSize(
session,
fallbackTile,
resolvedInputName,
inputMetadata,
);
const positionsY = slidingPositions(height, tile, stride);
const positionsX = slidingPositions(width, tile, stride);
const debugInfo: TileUnetDebug | undefined = debugEnabled
? {
input: {
width: originalWidth,
height: originalHeight,
processedWidth: width,
processedHeight: height,
tile,
stride,
scale,
},
}
: undefined;
const normalized = normalizeImage(data, width, height, mean, std);
const pixels = width * height;
const patchArea = tile * tile;
let outputLayout: LogitLayout = 'unknown';
let outputWidth = tile;
let outputHeight = tile;
let outputClasses = 0;
const traceEnabled = process.env.TILE_SEG_TRACE === '1';
if (traceEnabled) {
console.info('[tile-unet] session info', {
inputs: session.inputNames,
outputs: session.outputNames,
inputName,
tile,
classCount,
});
}
if (debugInfo) {
const overall = flatStats(normalized);
debugInfo.inputStats = {
normalizedMin: overall.min,
normalizedMax: overall.max,
normalizedMean: overall.mean,
channelStats: channelStats(normalized, 3, pixels, 'nchw'),
samplePixels: sampleNormalizedPixels(normalized, width, height),
};
debugInfo.model = {
path: MODEL_PATH,
sizeBytes: 0,
mtimeMs: 0,
inputs: session.inputNames,
outputs: session.outputNames,
inputMetadata: inputMetadata as unknown as Record<string, unknown>,
};
try {
const st = await stat(MODEL_PATH);
debugInfo.model.sizeBytes = st.size;
debugInfo.model.mtimeMs = st.mtimeMs;
} catch {
/* ignore */
}
}
const { Tensor } = ensureOrt();
const runPatch = async (startX: number, startY: number) => {
if (inputChannels !== 3) {
throw new Error(`[tile-unet] unexpected inputChannels=${inputChannels} (expected 3 RGB channels)`);
}
const patch = new Float32Array(inputChannels * patchArea);
fillPatch(normalized, patch, width, height, tile, startX, startY, inputChannels);
const feeds: Record<string, Tensor> = {
[inputName]: new Tensor('float32', patch, [1, inputChannels, tile, tile]),
};
let result;
try {
result = await session.run(feeds);
} catch (runError) {
console.error('[tile-unet] session.run failed', {
inputName,
tile,
inputChannels,
startX,
startY,
}, runError);
throw runError;
}
const outputName = session.outputNames[0] ?? Object.keys(result)[0];
if (!outputName || !result[outputName]) {
throw new Error('[tile-unet] model response missing output tensor');
}
const outputTensor = result[outputName] as Tensor;
const logits = outputTensor.data as Float32Array;
return { outputName, outputTensor, logits };
};
// Probe the first patch to determine output shape/classes/layout.
const firstY = positionsY[0] ?? 0;
const firstX = positionsX[0] ?? 0;
const first = await runPatch(firstX, firstY);
{
const dims = Array.isArray(first.outputTensor.dims) ? first.outputTensor.dims.map((v) => Number(v)) : [];
const last4 = dims.length >= 4 ? dims.slice(dims.length - 4) : dims;
if (last4.length === 4) {
const d1 = last4[1];
const d2 = last4[2];
const d3 = last4[3];
if (typeof d1 === 'number' && d1 > 0 && d2 === tile && d3 === tile) {
outputLayout = 'nchw';
outputClasses = d1;
outputHeight = d2;
outputWidth = d3;
} else if (typeof d3 === 'number' && d3 > 0 && d1 === tile && d2 === tile) {
outputLayout = 'nhwc';
outputClasses = d3;
outputHeight = d1;
outputWidth = d2;
} else {
outputLayout = 'unknown';
}
}
if (outputLayout === 'unknown' || !(outputClasses > 0)) {
throw new Error(`[tile-unet] unable to determine output layout/classes from dims=${JSON.stringify(dims)}`);
}
if (outputWidth * outputHeight !== patchArea) {
throw new Error(`[tile-unet] unexpected output spatial size ${outputWidth}x${outputHeight} (expected ${tile}x${tile})`);
}
if (debugInfo) {
debugInfo.output = {
name: first.outputName,
dims,
layout: outputLayout,
width: outputWidth,
height: outputHeight,
classes: outputClasses,
logitsStats: channelStats(first.logits, outputClasses, patchArea, outputLayout),
};
}
}
const probAccum = new Float32Array(outputClasses * pixels);
const countAccum = new Uint32Array(pixels);
const accumulate = (probs: Float32Array, startX: number, startY: number) => {
for (let py = 0; py < tile; py++) {
const yy = clamp(startY + py, 0, height - 1);
const row = yy * width;
for (let px = 0; px < tile; px++) {
const xx = clamp(startX + px, 0, width - 1);
const dstIdx = row + xx;
const pixelOffset = py * tile + px;
for (let c = 0; c < outputClasses; c++) {
probAccum[c * pixels + dstIdx] += probs[c * patchArea + pixelOffset];
}
countAccum[dstIdx] += 1;
}
}
};
const firstProbs = softmaxChannels(first.logits, outputClasses, patchArea, outputLayout);
if (debugInfo?.output && !debugInfo.output.probsStats) {
debugInfo.output.probsStats = channelStats(firstProbs, outputClasses, patchArea, 'nchw');
}
accumulate(firstProbs, firstX, firstY);
for (const startY of positionsY) {
for (const startX of positionsX) {
if (startX === firstX && startY === firstY) continue;
const patch = await runPatch(startX, startY);
const probs = softmaxChannels(patch.logits, outputClasses, patchArea, outputLayout);
accumulate(probs, startX, startY);
}
}
const ids = new Uint8Array(pixels);
for (let idx = 0; idx < pixels; idx++) {
const count = countAccum[idx] > 0 ? countAccum[idx] : 1;
let bestClass = 0;
let bestScore = -Infinity;
for (let c = 0; c < outputClasses; c++) {
const score = probAccum[c * pixels + idx] / count;
if (score > bestScore) {
bestScore = score;
bestClass = c;
}
}
ids[idx] = bestClass;
}
let finalIds = ids;
let finalWidth = width;
let finalHeight = height;
if (finalWidth !== originalWidth || finalHeight !== originalHeight) {
finalIds = await upscaleMask(ids, width, height, originalWidth, originalHeight);
finalWidth = originalWidth;
finalHeight = originalHeight;
}
if (debugInfo) {
const counts: Record<string, number> = {};
for (let i = 0; i < outputClasses; i++) counts[String(i)] = 0;
for (let idx = 0; idx < finalIds.length; idx++) {
const cls = finalIds[idx];
if (cls < outputClasses) counts[String(cls)] = (counts[String(cls)] ?? 0) + 1;
}
debugInfo.argmaxCounts = counts;
}
const classSubsetRaw = classes.slice(0, outputClasses);
let classSubset: ClassInfo[];
if (classSubsetRaw.length === outputClasses) {
classSubset = classSubsetRaw;
} else if (outputClasses === 2) {
classSubset = DEFAULT_CLASSES_2;
} else if (outputClasses === 3) {
classSubset = DEFAULT_CLASSES_3;
} else {
classSubset = Array.from({ length: outputClasses }, (_, idx) => ({
id: idx,
name: idx === 0 ? 'background' : `class_${idx}`,
color: idx === 0 ? [0, 0, 0] : [255, 255, 255],
})) as ClassInfo[];
}
const coverage = computeCoverageFromIds(finalIds, classSubset, finalWidth, finalHeight);
const dataUrl = await renderTileMask(finalIds, finalWidth, finalHeight, classSubset);
if (debugInfo?.output) {
debugInfo.output.classNames = classSubset.map((cls) => cls.name);
const tileCls = classSubset.find((cls) => String(cls.name).toLowerCase() === 'tiles');
debugInfo.output.tileClassId = tileCls ? tileCls.id : undefined;
}
return {
ids: finalIds,
width: finalWidth,
height: finalHeight,
dataUrl,
classes: classSubset,
coverage,
meta: {
processedWidth: width,
processedHeight: height,
originalWidth,
originalHeight,
scale,
},
...(debugInfo ? { debug: debugInfo } : {}),
};
}