| Server IP : 217.154.3.148 / Your IP : 216.73.216.90 Web Server : Apache System : Linux blissful-wright.217-154-3-148.plesk.page 6.8.0-124-generic #124-Ubuntu SMP PREEMPT_DYNAMIC Tue May 26 13:00:45 UTC 2026 x86_64 User : gracious-boyd_hfhh1h8vo9n ( 10000) PHP Version : 8.3.31 Disable Function : opcache_get_status MySQL : OFF | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : /var/www/vhosts/gracious-boyd.217-154-3-148.plesk.page/nextjs/lib/segmentation/ |
Upload File : |
import fs from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import { spawn, spawnSync } from "node:child_process";
import sharp from "sharp";
import { connectedComponents } from "@/lib/masks/components";
export type OccluderSegOptions = {
debugDir?: string | null;
timeoutMs?: number;
env?: NodeJS.ProcessEnv;
minCoverage?: number;
tileMask?: Uint8Array | null;
tileMaskWidth?: number | null;
tileMaskHeight?: number | null;
samPrompted?: boolean;
};
export type OccluderSegResult = {
mask: Uint8Array;
width: number;
height: number;
source: string;
meta?: Record<string, unknown> | null;
};
export type RefinedOccluderResult = {
mask: Uint8Array;
width: number;
height: number;
// Threshold used to build the wall/backsplash plane (far cluster).
threshold: number;
// Threshold used to build the occluder/object mask (near pixels).
occluderThreshold: number;
wallDepth: number;
histogram: number[];
warnings?: string[] | null;
rejected?: {
reason: string;
metrics?: Record<string, number>;
} | null;
debug?: {
resizedDepthUrl?: string | null;
rawUrl?: string | null;
processedUrl?: string | null;
};
};
const DEFAULT_TIMEOUT_MS = Math.max(5_000, Number(process.env.OCCLUDER_SEG_TIMEOUT ?? "90000"));
const MIN_COVERAGE_DEFAULT = Math.max(0.0, Math.min(0.25, Number(process.env.OCCLUDER_SEG_MIN_COVERAGE ?? "0.02")));
const GUIDED_DEFAULT_TIMEOUT_MS = 20_000;
function projectRootFromLib(dirname: string): string {
return path.resolve(dirname, "..", "..");
}
async function resolveScriptPath(scriptFile: string): Promise<{ scriptPath: string; projectRoot: string; }> {
const roots = [projectRootFromLib(__dirname), process.cwd()];
for (const root of roots) {
const scriptPath = path.join(root, "scripts", scriptFile);
try {
await fs.access(scriptPath);
return { scriptPath, projectRoot: root };
} catch {
/* try next */
}
}
let cursor = __dirname;
for (let i = 0; i < 6; i++) {
const parent = path.resolve(cursor, "..");
if (parent === cursor) break;
const scriptPath = path.join(parent, "scripts", scriptFile);
try {
await fs.access(scriptPath);
return { scriptPath, projectRoot: parent };
} catch {
/* continue */
}
cursor = parent;
}
const fallbackRoot = projectRootFromLib(__dirname);
return { scriptPath: path.join(fallbackRoot, "scripts", scriptFile), projectRoot: fallbackRoot };
}
function pythonCandidates(): string[] {
const explicit = process.env.OCCLUDER_SEG_PY ?? process.env.PYTHON_EXEC;
if (explicit) {
return explicit
.split(/[,\s]+/g)
.map((entry) => entry.trim())
.filter(Boolean);
}
return ["python3", "python"];
}
function resolvePython(): string | null {
const candidates = pythonCandidates();
for (const candidate of candidates) {
const check = spawnSync(candidate, ["--version"], { stdio: "ignore" });
if (check.status === 0) {
return candidate;
}
}
return null;
}
async function ensureSharpPng(input: Buffer, outPath: string): Promise<void> {
await sharp(input, { failOnError: false })
.withMetadata()
.png({ compressionLevel: 3, adaptiveFiltering: false })
.toFile(outPath);
}
async function loadMask(maskPath: string): Promise<{ data: Uint8Array; width: number; height: number; }> {
const { data, info } = await sharp(maskPath, { failOnError: false })
.ensureAlpha()
.removeAlpha()
.greyscale()
.raw()
.toBuffer({ resolveWithObject: true });
const width = info.width ?? 0;
const height = info.height ?? 0;
if (!(width > 0 && height > 0)) {
throw new Error(`occluder mask returned invalid size (${width}x${height})`);
}
return { data: new Uint8Array(data), width, height };
}
function coverageOf(mask: Uint8Array): number {
if (!mask || mask.length === 0) return 0;
let solid = 0;
for (let i = 0; i < mask.length; i++) {
if (mask[i] > 127) solid += 1;
}
return solid / mask.length;
}
async function toDataUrlPng(mask: Uint8Array, width: number, height: number): Promise<string> {
const buffer = await sharp(Buffer.from(mask), {
raw: { width, height, channels: 1 },
})
.toColorspace("b-w")
.png({ compressionLevel: 4 })
.toBuffer();
return `data:image/png;base64,${buffer.toString("base64")}`;
}
function histogram256(data: Uint8Array): Uint32Array {
const hist = new Uint32Array(256);
for (let i = 0; i < data.length; i++) hist[data[i]] += 1;
return hist;
}
type GuidedMaskResult = {
mask: Uint8Array;
width: number;
height: number;
meta?: Record<string, unknown> | null;
};
async function runGuidedOccluder(
depthBuffer: Buffer,
guideBuffer: Buffer,
): Promise<GuidedMaskResult> {
const { scriptPath, projectRoot } = await resolveScriptPath("occluder_guided.py");
const python = resolvePython();
if (!python) {
throw new Error("no python interpreter available (set OCCLUDER_SEG_PY)");
}
const workdir = await fs.mkdtemp(path.join(os.tmpdir(), "occluder-guided-"));
const keepTemp = (process.env.OCCLUDER_GUIDED_KEEP_TEMP ?? "0") === "1";
const cleanup = async () => {
if (!keepTemp) {
try {
await fs.rm(workdir, { recursive: true, force: true });
} catch {
/* ignore */
}
}
};
const depthPath = path.join(workdir, "depth.png");
const imagePath = path.join(workdir, "image.png");
const maskPath = path.join(workdir, "mask.png");
const jsonPath = path.join(workdir, "mask.json");
try {
await ensureSharpPng(depthBuffer, depthPath);
await ensureSharpPng(guideBuffer, imagePath);
const args = [
scriptPath,
"--depth",
depthPath,
"--image",
imagePath,
"--output",
maskPath,
"--json",
jsonPath,
];
const radiusRaw = Number(process.env.OCCLUDER_GUIDED_RADIUS ?? "");
if (Number.isFinite(radiusRaw) && radiusRaw > 0) {
args.push("--radius", String(Math.round(radiusRaw)));
}
const epsRaw = Number(process.env.OCCLUDER_GUIDED_EPS ?? "");
if (Number.isFinite(epsRaw) && epsRaw > 0) {
args.push("--eps", String(epsRaw));
}
const erodeRaw = Number(process.env.OCCLUDER_GUIDED_ERODE ?? "");
if (Number.isFinite(erodeRaw) && erodeRaw >= 0) {
args.push("--erode", String(Math.round(erodeRaw)));
}
const invertFlag = (process.env.OCCLUDER_GUIDED_INVERT ?? "").toLowerCase();
if (invertFlag === "1" || invertFlag === "true" || invertFlag === "on") {
args.push("--invert");
}
const timeoutRaw = Number(process.env.OCCLUDER_GUIDED_TIMEOUT ?? "");
const timeoutMs = Number.isFinite(timeoutRaw)
? Math.max(1_000, timeoutRaw)
: Math.max(GUIDED_DEFAULT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS);
const stdoutChunks: Buffer[] = [];
const stderrChunks: Buffer[] = [];
const child = spawn(python, args, {
cwd: projectRoot,
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout?.on("data", (chunk) => stdoutChunks.push(Buffer.from(chunk)));
child.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
const exitCode: number = await new Promise((resolve, reject) => {
let settled = false;
const timer = timeoutMs > 0 ? setTimeout(() => {
if (!settled) {
settled = true;
child.kill("SIGKILL");
reject(new Error(`occluder guided filter timed out after ${timeoutMs}ms`));
}
}, timeoutMs) : null;
child.on("error", (err) => {
if (!settled) {
settled = true;
if (timer) clearTimeout(timer);
reject(err);
}
});
child.on("close", (code) => {
if (!settled) {
settled = true;
if (timer) clearTimeout(timer);
resolve(code ?? -1);
}
});
});
if (exitCode !== 0) {
const stdoutText = Buffer.concat(stdoutChunks).toString("utf8");
const stderrText = Buffer.concat(stderrChunks).toString("utf8");
throw new Error(`occluder guided filter failed (code ${exitCode}):\nstdout: ${stdoutText}\nstderr: ${stderrText}`);
}
const { data, width, height } = await loadMask(maskPath);
const meta = await readJson(jsonPath);
return { mask: data, width, height, meta };
} finally {
await cleanup();
}
}
function entropyFromHist(hist: Uint32Array, total: number): number {
if (!(total > 0)) return 0;
let entropy = 0;
const inv = 1 / total;
for (let i = 0; i < hist.length; i++) {
const count = hist[i] ?? 0;
if (!count) continue;
const p = count * inv;
entropy -= p * Math.log2(p);
}
return entropy;
}
function countNonZeroBins(hist: Uint32Array): number {
let bins = 0;
for (let i = 0; i < hist.length; i++) if (hist[i]) bins += 1;
return bins;
}
function meanBinaryTransitionsPerRow(mask: Uint8Array, width: number, height: number, threshold = 127): number {
if (!(width > 1 && height > 0)) return 0;
let sum = 0;
for (let y = 0; y < height; y++) {
const row = y * width;
let prev = mask[row] > threshold ? 1 : 0;
let transitions = 0;
for (let x = 1; x < width; x++) {
const next = mask[row + x] > threshold ? 1 : 0;
if (next !== prev) transitions += 1;
prev = next;
}
sum += transitions;
}
return sum / height;
}
function meanBinaryTransitionsPerCol(mask: Uint8Array, width: number, height: number, threshold = 127): number {
if (!(width > 0 && height > 1)) return 0;
let sum = 0;
for (let x = 0; x < width; x++) {
let prev = mask[x] > threshold ? 1 : 0;
let transitions = 0;
for (let y = 1; y < height; y++) {
const next = mask[y * width + x] > threshold ? 1 : 0;
if (next !== prev) transitions += 1;
prev = next;
}
sum += transitions;
}
return sum / width;
}
async function validateDepthLikeInput(depthBuffer: Buffer): Promise<{
ok: boolean;
reason?: string;
metrics: Record<string, number>;
preview?: { data: Uint8Array; width: number; height: number } | null;
}> {
// Cheap rejection heuristic to prevent tile/checker buffers leaking into depth processing.
// Depth maps tend to have many gray levels and relatively low "perfect grid" transition density.
const sampleSize = 256;
const { data, info } = await sharp(depthBuffer, { failOnError: false })
.greyscale()
.resize({
width: sampleSize,
height: sampleSize,
fit: "inside",
withoutEnlargement: true,
kernel: "nearest",
})
.raw()
.toBuffer({ resolveWithObject: true });
const w = info.width ?? 0;
const h = info.height ?? 0;
const pixels = new Uint8Array(data);
const total = pixels.length;
const hist = histogram256(pixels);
const bins = countNonZeroBins(hist);
const entropy = entropyFromHist(hist, total);
const rowT = meanBinaryTransitionsPerRow(pixels, w, h);
const colT = meanBinaryTransitionsPerCol(pixels, w, h);
const metrics = {
sampleW: w,
sampleH: h,
bins,
entropy: Number(entropy.toFixed(3)),
rowTransitions: Number(rowT.toFixed(2)),
colTransitions: Number(colT.toFixed(2)),
rowTransitionsFrac: w > 0 ? Number((rowT / w).toFixed(3)) : 0,
colTransitionsFrac: h > 0 ? Number((colT / h).toFixed(3)) : 0,
};
// Conservative rejection: require BOTH low grayscale diversity and high transition density.
const lowDiversity = bins <= 12 && entropy < 2.6;
const gridish = (w > 0 && h > 0) && (rowT / Math.max(1, w) > 0.35) && (colT / Math.max(1, h) > 0.35);
const ok = !(lowDiversity && gridish);
return {
ok,
reason: ok ? undefined : "input_not_depth",
metrics,
preview: w > 0 && h > 0 ? { data: pixels, width: w, height: h } : null,
};
}
function wallPeakFromHist(hist: Uint32Array): number {
// Search the "far" cluster in the darker part of the histogram.
// Using a wider window than 0–80 makes the method robust across images with different depth scaling.
const limit = Math.min(Math.floor(hist.length * 0.6), hist.length);
let peak = 0;
let idx = 0;
for (let i = 0; i < limit; i++) {
if (hist[i] > peak) {
peak = hist[i];
idx = i;
}
}
return idx;
}
function holeFill(mask: Uint8Array, width: number, height: number, radius = 2, minNeighbors = 10): Uint8Array {
const out = Uint8Array.from(mask);
for (let y = radius; y < height - radius; y++) {
for (let x = radius; x < width - radius; x++) {
const idx = y * width + x;
if (mask[idx]) continue;
let white = 0;
for (let dy = -radius; dy <= radius; dy++) {
const row = (y + dy) * width;
for (let dx = -radius; dx <= radius; dx++) {
if (mask[row + (x + dx)]) white++;
}
}
if (white > minNeighbors) out[idx] = 255;
}
}
return out;
}
function erodeBinary(mask: Uint8Array, width: number, height: number, radius = 1): Uint8Array {
const r = Math.max(0, Math.round(radius));
if (!r) return Uint8Array.from(mask);
const out = new Uint8Array(mask.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let all = 1;
for (let dy = -r; dy <= r && all; dy++) {
const ny = y + dy;
if (ny < 0 || ny >= height) {
all = 0;
break;
}
for (let dx = -r; dx <= r; dx++) {
const nx = x + dx;
if (nx < 0 || nx >= width) {
all = 0;
break;
}
if (!mask[ny * width + nx]) {
all = 0;
break;
}
}
}
out[y * width + x] = all ? 255 : 0;
}
}
return out;
}
function dilateBinary(mask: Uint8Array, width: number, height: number, radius = 1): Uint8Array {
const r = Math.max(0, Math.round(radius));
if (!r) return Uint8Array.from(mask);
const out = new Uint8Array(mask.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let solid = false;
for (let dy = -r; dy <= r; dy++) {
const ny = y + dy;
if (ny < 0 || ny >= height) continue;
const row = ny * width;
for (let dx = -r; dx <= r; dx++) {
const nx = x + dx;
if (nx < 0 || nx >= width) continue;
if (mask[row + nx]) {
solid = true;
break;
}
}
if (solid) break;
}
out[y * width + x] = solid ? 255 : 0;
}
}
return out;
}
function removeSmallBlobs(mask: Uint8Array, width: number, height: number, minPixels = 1000): Uint8Array {
const cleaned = Uint8Array.from(mask);
const comps = connectedComponents(cleaned, width, height, 1);
const total = width * height;
// 1% of the frame is far too aggressive for thin shelves/boards.
// Use a small adaptive floor instead (0.05%).
const minArea = Math.max(minPixels, Math.floor(total * 0.0005));
for (const comp of comps) {
if (comp.area < minArea) {
for (const idx of comp.pixels) cleaned[idx] = 0;
}
}
return cleaned;
}
/**
* High-quality refinement for depth-based occluder masks.
* Expects an encoded grayscale depth image (e.g., PNG from Depth Anything) and outputs a crisp 0/255 mask.
*/
export async function refineOccluderMask(
depthBuffer: Buffer,
targetWidth: number,
targetHeight: number,
sensitivity = 18,
guideBuffer?: Buffer,
): Promise<RefinedOccluderResult> {
const validation = await validateDepthLikeInput(depthBuffer).catch(() => null);
if (validation && !validation.ok) {
const empty = new Uint8Array(Math.max(1, targetWidth) * Math.max(1, targetHeight));
const resizedDepthUrl = validation.preview
? await toDataUrlPng(validation.preview.data, validation.preview.width, validation.preview.height).catch(() => null)
: null;
return {
mask: empty,
width: targetWidth,
height: targetHeight,
threshold: 0,
occluderThreshold: 0,
wallDepth: 0,
histogram: new Array(256).fill(0),
warnings: null,
rejected: { reason: validation.reason ?? "input_not_depth", metrics: validation.metrics },
debug: {
resizedDepthUrl,
rawUrl: null,
processedUrl: null,
},
};
}
const warnings: string[] = [];
const processingSize = 1500;
const depthImage = sharp(depthBuffer, { failOnError: false })
.resize({
width: processingSize,
height: processingSize,
fit: "inside",
withoutEnlargement: false,
kernel: "lanczos3",
})
.toColorspace("b-w")
.blur(0.8);
const { data: pixels, info } = await depthImage
.raw()
.toBuffer({ resolveWithObject: true });
const w = info.width ?? processingSize;
const h = info.height ?? processingSize;
const hist = histogram256(pixels);
let maxCount = 0;
let wallDepth = 0;
for (let i = 0; i < 80; i++) {
if (hist[i] > maxCount) {
maxCount = hist[i];
wallDepth = i;
}
}
const baseThreshold = Math.max(0, Math.min(255, wallDepth + Math.max(8, Math.round(sensitivity * 0.55))));
const wallThreshold = baseThreshold;
let occluderThreshold = baseThreshold;
const debugResizedDepthUrl = await toDataUrlPng(new Uint8Array(pixels), w, h).catch(() => null);
const erosionStrength = 2;
const buildMask = (t: number) => {
const temp = new Uint8Array(pixels.length);
for (let i = 0; i < pixels.length; i++) {
temp[i] = pixels[i] > t ? 255 : 0;
}
const eroded = new Uint8Array(pixels.length);
for (let y = erosionStrength; y < h - erosionStrength; y++) {
const row = y * w;
for (let x = erosionStrength; x < w - erosionStrength; x++) {
let isSolid = true;
for (let dy = -erosionStrength; dy <= erosionStrength; dy++) {
const nrow = (y + dy) * w;
for (let dx = -erosionStrength; dx <= erosionStrength; dx++) {
if (!temp[nrow + (x + dx)]) {
isSolid = false;
break;
}
}
if (!isSolid) break;
}
eroded[row + x] = isSolid ? 255 : 0;
}
}
return { temp, eroded };
};
const outputThreshold = Math.max(96, Math.min(160, 118 + Math.round(sensitivity * 0.2)));
const guidedFlag = (process.env.OCCLUDER_GUIDED ?? "1").toLowerCase();
const guidedEnabled = !!guideBuffer && guidedFlag !== "0" && guidedFlag !== "false" && guidedFlag !== "off";
if (guidedEnabled) {
try {
const guided = await runGuidedOccluder(depthBuffer, guideBuffer);
let guidedMask = guided.mask;
let guidedW = guided.width;
let guidedH = guided.height;
if (guidedW !== targetWidth || guidedH !== targetHeight) {
const resized = await sharp(Buffer.from(guidedMask), {
raw: { width: guidedW, height: guidedH, channels: 1 },
})
.resize(targetWidth, targetHeight, { kernel: "lanczos3" })
.threshold(128)
.raw()
.toBuffer();
guidedMask = new Uint8Array(resized);
guidedW = targetWidth;
guidedH = targetHeight;
}
const rawMask = Uint8Array.from(guidedMask);
const tightenPx = Math.max(0, Math.round(Number(process.env.OCCLUDER_TIGHTEN_PX ?? "0")));
if (tightenPx > 0) {
const erodedFinal = erodeBinary(guidedMask, guidedW, guidedH, tightenPx);
if (coverageOf(erodedFinal) > 0.01) guidedMask = erodedFinal;
}
let guidedCoverage = coverageOf(guidedMask);
const minCoverage = Math.max(0, Math.min(1, Number(process.env.OCCLUDER_GUIDED_MIN_COVERAGE ?? "0.32")));
const mergeFlag = (process.env.OCCLUDER_GUIDED_MERGE_COARSE ?? "1").toLowerCase();
const mergeEnabled = mergeFlag !== "0" && mergeFlag !== "false" && mergeFlag !== "off";
if (mergeEnabled && guidedCoverage < minCoverage) {
const looseDelta = Math.max(6, Math.round(sensitivity * 0.3));
const looseThreshold = Math.max(0, occluderThreshold - looseDelta);
const { temp: looseTemp } = buildMask(looseThreshold);
const coarse = await sharp(Buffer.from(looseTemp), {
raw: { width: w, height: h, channels: 1 },
})
.blur(0.4)
.resize(targetWidth, targetHeight, { kernel: "lanczos3" })
.threshold(outputThreshold)
.raw()
.toBuffer();
const merged = new Uint8Array(guidedMask.length);
const coarseMask = new Uint8Array(coarse);
for (let i = 0; i < merged.length; i++) {
merged[i] = guidedMask[i] || coarseMask[i] ? 255 : 0;
}
guidedMask = merged;
guidedCoverage = coverageOf(guidedMask);
if (guidedCoverage < minCoverage) {
const dilated = dilateBinary(guidedMask, guidedW, guidedH, 1);
if (coverageOf(dilated) > guidedCoverage) {
guidedMask = dilated;
guidedCoverage = coverageOf(guidedMask);
}
}
}
if (guidedCoverage < 0.15) warnings.push("occluder_undersegmented");
const debugRawUrl = await toDataUrlPng(rawMask, guidedW, guidedH).catch(() => null);
const debugProcessedUrl = await toDataUrlPng(guidedMask, guidedW, guidedH).catch(() => null);
return {
mask: guidedMask,
width: guidedW,
height: guidedH,
threshold: wallThreshold,
occluderThreshold,
wallDepth,
histogram: Array.from(hist),
warnings: warnings.length ? warnings : null,
rejected: null,
debug: {
resizedDepthUrl: debugResizedDepthUrl,
rawUrl: debugRawUrl,
processedUrl: debugProcessedUrl,
},
};
} catch {
warnings.push("guided_failed");
}
}
let { temp: tempMask, eroded } = buildMask(occluderThreshold);
let erodedCoverage = coverageOf(eroded);
if (erodedCoverage < 0.12) {
occluderThreshold = Math.max(0, occluderThreshold - 6);
({ temp: tempMask, eroded } = buildMask(occluderThreshold));
erodedCoverage = coverageOf(eroded);
} else if (erodedCoverage > 0.6) {
occluderThreshold = Math.min(255, occluderThreshold + 6);
({ temp: tempMask, eroded } = buildMask(occluderThreshold));
erodedCoverage = coverageOf(eroded);
}
const processed = await sharp(Buffer.from(eroded), {
raw: { width: w, height: h, channels: 1 },
})
.blur(1.0)
.resize(targetWidth, targetHeight, { kernel: "lanczos3" })
.threshold(outputThreshold)
.toFormat("png")
.toBuffer();
const { data: finalMask, info: finalInfo } = await sharp(processed, { failOnError: false })
.greyscale()
.raw()
.toBuffer({ resolveWithObject: true });
let finalMaskU8 = new Uint8Array(finalMask);
const tightenPx = Math.max(0, Math.round(Number(process.env.OCCLUDER_TIGHTEN_PX ?? "0")));
if (tightenPx > 0) {
const erodedFinal = erodeBinary(finalMaskU8, finalInfo.width ?? targetWidth, finalInfo.height ?? targetHeight, tightenPx);
if (coverageOf(erodedFinal) > 0.01) finalMaskU8 = erodedFinal;
}
const finalCoverage = coverageOf(finalMaskU8);
if (finalCoverage < 0.15) warnings.push("occluder_undersegmented");
const debugRawUrl = await toDataUrlPng(tempMask, w, h).catch(() => null);
const debugProcessedUrl = await toDataUrlPng(finalMaskU8, finalInfo.width ?? targetWidth, finalInfo.height ?? targetHeight).catch(() => null);
return {
mask: finalMaskU8,
width: finalInfo.width ?? targetWidth,
height: finalInfo.height ?? targetHeight,
threshold: wallThreshold,
occluderThreshold,
wallDepth,
histogram: Array.from(hist),
warnings: warnings.length ? warnings : null,
rejected: null,
debug: {
resizedDepthUrl: debugResizedDepthUrl,
rawUrl: debugRawUrl,
processedUrl: debugProcessedUrl,
},
};
}
async function readJson(optionalPath: string | null): Promise<Record<string, unknown> | null> {
if (!optionalPath) return null;
try {
const raw = await fs.readFile(optionalPath, "utf8");
if (!raw.trim()) return null;
return JSON.parse(raw);
} catch {
return null;
}
}
export async function inferOccluderMask(
raw: Buffer,
options: OccluderSegOptions = {},
): Promise<OccluderSegResult> {
const { scriptPath, projectRoot } = await resolveScriptPath("occluder_segment.py");
const python = resolvePython();
if (!python) {
throw new Error("no python interpreter available (set OCCLUDER_SEG_PY)");
}
const workdir = await fs.mkdtemp(path.join(os.tmpdir(), "occluder-seg-"));
const keepTemp = (process.env.OCCLUDER_SEG_KEEP_TEMP ?? "0") === "1";
const cleanup = async () => {
if (!keepTemp) {
try {
await fs.rm(workdir, { recursive: true, force: true });
} catch {
/* ignore */
}
}
};
const inputPath = path.join(workdir, "input.png");
const maskPath = path.join(workdir, "mask.png");
const jsonPath = path.join(workdir, "mask.json");
const tileMaskPath = path.join(workdir, "tile_mask.png");
try {
await ensureSharpPng(raw, inputPath);
const args = [
scriptPath,
"--image",
inputPath,
"--output",
maskPath,
"--json",
jsonPath,
];
const samPrompted = Boolean(
options.samPrompted &&
options.tileMask &&
options.tileMaskWidth &&
options.tileMaskHeight,
);
if (samPrompted && options.tileMask && options.tileMaskWidth && options.tileMaskHeight) {
await sharp(Buffer.from(options.tileMask), {
raw: { width: options.tileMaskWidth, height: options.tileMaskHeight, channels: 1 },
})
.toColorspace("b-w")
.png({ compressionLevel: 4 })
.toFile(tileMaskPath);
args.push("--tile-mask", tileMaskPath, "--sam-prompted");
}
if (options.debugDir) {
args.push("--debug-dir", options.debugDir);
}
const defaultModel = process.env.OCCLUDER_SEG_MODEL;
if (defaultModel) {
args.push("--model", defaultModel);
}
const samCkpt = process.env.OCCLUDER_SEG_SAM_CHECKPOINT;
if (samCkpt) {
args.push("--sam-checkpoint", samCkpt);
}
const samVariant = process.env.OCCLUDER_SEG_SAM_MODEL;
if (samVariant) {
args.push("--sam-model", samVariant);
}
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const env = { ...process.env, ...(options.env ?? {}) };
const stdoutChunks: Buffer[] = [];
const stderrChunks: Buffer[] = [];
const child = spawn(python, args, {
cwd: projectRoot,
env,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout?.on("data", (chunk) => stdoutChunks.push(Buffer.from(chunk)));
child.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
const exitCode: number = await new Promise((resolve, reject) => {
let settled = false;
const timer = timeoutMs > 0 ? setTimeout(() => {
if (!settled) {
settled = true;
child.kill("SIGKILL");
reject(new Error(`occluder segmentation timed out after ${timeoutMs}ms`));
}
}, timeoutMs) : null;
child.on("error", (err) => {
if (!settled) {
settled = true;
if (timer) clearTimeout(timer);
reject(err);
}
});
child.on("close", (code) => {
if (!settled) {
settled = true;
if (timer) clearTimeout(timer);
resolve(code ?? -1);
}
});
});
if (exitCode !== 0) {
const stdoutText = Buffer.concat(stdoutChunks).toString("utf8");
const stderrText = Buffer.concat(stderrChunks).toString("utf8");
throw new Error(`occluder segmentation failed (code ${exitCode}):\nstdout: ${stdoutText}\nstderr: ${stderrText}`);
}
const meta = await readJson(jsonPath);
const source = typeof meta?.source === "string" && meta.source
? meta.source
: (samCkpt ? "yolo+sam" : "yolo");
const { data, width, height } = await loadMask(maskPath);
const coverage = coverageOf(data);
const minCoverage = options.minCoverage ?? MIN_COVERAGE_DEFAULT;
const isSamSource = source.includes("sam");
if (coverage <= 0) {
throw new Error("occluder segmentation returned empty mask");
}
if (coverage < minCoverage && !isSamSource) {
throw new Error(`occluder segmentation coverage ${coverage.toFixed(4)} below minimum ${minCoverage}`);
}
if (coverage < minCoverage && isSamSource && meta) {
meta.coverageBelowMin = Number(coverage.toFixed(6));
meta.minCoverage = Number(minCoverage.toFixed(6));
}
if (options.debugDir) {
try {
await fs.mkdir(options.debugDir, { recursive: true });
await fs.copyFile(maskPath, path.join(options.debugDir, path.basename(maskPath)));
if (meta) {
await fs.writeFile(
path.join(options.debugDir, "mask.meta.json"),
JSON.stringify(meta, null, 2),
"utf8",
);
}
} catch {
/* ignore */
}
}
return {
mask: data,
width,
height,
source,
meta,
};
} finally {
await cleanup();
}
}