From f86cb8c2f4e5f0cd1ad1becbf492a9ef4714bb06 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Wed, 5 Aug 2026 11:25:01 +0200 Subject: [PATCH 1/7] Add DA2 sensor depth upscale workflow Just faster and using much less memory. Implements https://github.com/SpexAI/PointClouds/issues/12 --- CMakeLists.txt | 5 + README.md | 29 +++++ examples/cli/main.cpp | 48 ++++++++ src/cli.cpp | 19 +++ src/cli.hpp | 5 +- src/depth_upscale.cpp | 217 +++++++++++++++++++++++++++++++++++ src/depth_upscale.hpp | 27 +++++ src/tiff_io.cpp | 143 +++++++++++++++++++++++ src/tiff_io.hpp | 23 ++++ tests/CMakeLists.txt | 1 + tests/test_depth_upscale.cpp | 119 +++++++++++++++++++ 11 files changed, 635 insertions(+), 1 deletion(-) create mode 100644 src/depth_upscale.cpp create mode 100644 src/depth_upscale.hpp create mode 100644 src/tiff_io.cpp create mode 100644 src/tiff_io.hpp create mode 100644 tests/test_depth_upscale.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 874b64c..f20365f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,8 @@ set(GGML_VULKAN ${DA_GGML_VULKAN} CACHE BOOL "" FORCE) # and a GPU present at configure time, so it breaks cross/CI builds (e.g. the # l4t/Jetson image on CMake 3.22 yields an empty "compute_" arch). For a local # GPU-targeted build pass -DCMAKE_CUDA_ARCHITECTURES=native explicitly. +find_package(TIFF REQUIRED) + add_subdirectory(third_party/ggml EXCLUDE_FROM_ALL) set(DA_SOURCES @@ -47,6 +49,8 @@ set(DA_SOURCES src/backend.cpp src/image_io.cpp src/preprocess.cpp + src/tiff_io.cpp + src/depth_upscale.cpp src/rope2d.cpp src/uv_posembed.cpp src/attention.cpp @@ -108,6 +112,7 @@ else() endif() endif() target_link_libraries(depthanything PUBLIC ggml) +target_link_libraries(depthanything PUBLIC TIFF::TIFF) # OpenMP for the single-threaded CPU geometry post-processing (surface fusion, # per-seam ICP, normal estimation). ggml already pulls OpenMP into the toolchain; diff --git a/README.md b/README.md index 520d23e..d252ee1 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Given an image it recovers a dense **metric depth** map, per-pixel **confidence* - **Quantization** to f16 / q8_0 / q6_k / q5_k / q4_k - q4_k is **99 MB** (0.25x the f32) and near-lossless. - **CPU-first, GPU-ready.** Tuned CPU path (tinyBLAS, Winograd, flash-attention) plus CUDA / Metal / Vulkan ggml backends. - **Flat C API** (`include/da_capi.h`) - embed from C, C++, Go, or Rust. Powers the [LocalAI](#use-it-from-localai) backend. +- **SpexAI sensor-calibrated TIFF depth.** Run a DA2 relative-depth model on an RGB TIFF, fit it to a two-channel 16-bit sensor-depth TIFF, and emit a full-resolution uint16 TIFF. - **Parity-first.** Every component is gated against PyTorch-dumped reference tensors; the end-to-end depth matches the real `net()` at correlation 1.0. --- @@ -80,6 +81,27 @@ On **GPU** (NVIDIA GB10, via `-DDA_GGML_CUDA=ON`) the ggml CUDA path with flash ![inference speed](benchmarks/media/infer_speed.png) ![peak memory](benchmarks/media/memory.png) +### SpexAI depth-upscale capacity + +`depth-upscale` is the SpexAI-specific capture workflow: it combines a DA2 relative-depth +prediction from an RGB TIFF with a paired sensor TIFF. The sensor image stores each uint16 +depth sample in its first two interleaved uint8 channels (high byte, then low byte); the command +resizes and smooths the DA2 map, polynomial-fits it to valid sensor pixels, and writes an +uncompressed single-channel uint16 TIFF. + +Measured as one cold task on the supplied SpexAI captures (3072×2048 RGB → 1280×720 depth) +with `depth-anything2-large-q4_k.gguf` (289.1 MiB), four CPU threads: + +| Device | Wall time | Peak host RSS | Peak GPU memory | Capacity guidance | +|---------------|----------:|--------------:|----------------:|-----------------------| +| CPU | 31.14 s | 833.1 MiB | — | 1 GiB RAM per worker | +| NVIDIA M2000M | **9.73 s**| **661.9 MiB** | 2,473 MiB task allocation; 2,806 MiB global peak (333 MiB baseline) | 1 GiB RAM + 3 GiB VRAM per worker; one worker on a 4 GiB card | +| old SciPy CPU | 4.79 s | 883.6 MiB | | | + +The CUDA run (`-DDA_GGML_CUDA=ON`, `DA_DEVICE=CUDA0`) offloaded 403 weights and used +flash attention. GPU memory was sampled every 100 ms; its ≈2.8 GiB global plateau persisted +for several seconds. The CUDA output was validated as a 1280×720, single-channel uint16 TIFF. + ### See it run Real photos through the actual CLI, input next to the colorized depth (turbo): @@ -100,6 +122,8 @@ cmake --build build -j # -> build/examples/cli/da3-cli ``` +`depth-upscale` uses libtiff for TIFF decode/encode; install its development package before configuring (for example, `libtiff-dev` on Debian/Ubuntu). + ### CMake options | Option | Default | Effect | @@ -181,6 +205,11 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - # Nested metric-scale depth (two GGUFs) $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm +# Sensor-calibrated, full-resolution depth TIFF. The sensor input stores each +# 16-bit depth sample in its first two 8-bit channels (high byte, low byte). +$CLI depth-upscale --model models/depth-anything2-large-f32.gguf \ + --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff + # Multi-view depth + pose $CLI depth --model $M --input a.jpg --input b.jpg --out-prefix scene diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 2ee5b0e..1e26481 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -7,6 +7,8 @@ #include "preprocess.hpp" #include "glb_export.hpp" #include "colmap_export.hpp" +#include "depth_upscale.hpp" +#include "tiff_io.hpp" #include #include #include @@ -152,6 +154,51 @@ static int cmd_depth_export(const da::cli::Parsed& p, da::Engine& eng){ if (!p.output_png.empty()) da::write_depth_png(p.output_png, depth, H, W, p.invert); return 0; } +static int cmd_depth_upscale(const da::cli::Parsed& p){ + std::string error; + da::Image image; + if (!da::load_tiff_rgb(p.input, image, &error)){ + std::fprintf(stderr, "error: load RGB TIFF failed: %s\n", error.c_str()); + return 1; + } + std::vector sensor_depth; int sensor_h = 0, sensor_w = 0; + if (!da::load_packed_depth_tiff(p.sensor_depth_tiff, sensor_depth, sensor_h, sensor_w, &error)){ + std::fprintf(stderr, "error: load packed sensor depth TIFF failed: %s\n", error.c_str()); + return 1; + } + auto eng = da::Engine::load(p.model, p.n_threads); + if (!eng){ std::fprintf(stderr, "error: load failed\n"); return 1; } + if (!eng->is_da2()){ + std::fprintf(stderr, "error: depth-upscale requires a Depth Anything V2 GGUF\n"); + return 1; + } + std::vector predicted; int prediction_h = 0, prediction_w = 0; + if (!eng->depth_relative(image, predicted, prediction_h, prediction_w)){ + std::fprintf(stderr, "error: Depth Anything V2 inference failed\n"); + return 1; + } + std::vector normalized; + if (!da::normalize_depth_u8(predicted, normalized, &error)){ + std::fprintf(stderr, "error: normalize model depth failed: %s\n", error.c_str()); + return 1; + } + da::DepthUpscaleOptions options; + options.degree = p.upscale_degree; + std::vector output; + if (!da::upscale_depth_map(sensor_depth, sensor_h, sensor_w, normalized, prediction_h, prediction_w, + output, options, &error)){ + std::fprintf(stderr, "error: depth calibration failed: %s\n", error.c_str()); + return 1; + } + if (!da::write_depth_tiff_u16(p.output_depth_tiff, output, sensor_h, sensor_w, &error)){ + std::fprintf(stderr, "error: write depth TIFF failed: %s\n", error.c_str()); + return 1; + } + std::printf("upscaled depth %dx%d from DA2 %dx%d -> %s\n", sensor_w, sensor_h, prediction_w, + prediction_h, p.output_depth_tiff.c_str()); + return 0; +} + static int cmd_depth(const da::cli::Parsed& p){ if (!p.metric_model.empty()) return cmd_depth_metric(p); if (p.repeat > 0 && p.inputs.size() <= 1) return cmd_depth_bench(p); @@ -243,6 +290,7 @@ int main(int argc, char** argv){ case S::Depth: return cmd_depth(p); case S::Reconstruct: return cmd_reconstruct(p); case S::Quantize: return cmd_quantize(p); + case S::DepthUpscale: return cmd_depth_upscale(p); case S::Help: da::cli::print_help(); return 0; default: da::cli::print_help(); return 1; } diff --git a/src/cli.cpp b/src/cli.cpp index 3593bd4..269e5c1 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -11,6 +11,7 @@ void print_help(){ " da3-cli depth --model --input [--glb ] [--colmap ] [--colmap-txt ] (single-image 3D export)\n" " da3-cli depth --model --metric-model --input [--pfm ] (nested metric-scale depth)\n" " da3-cli depth --model --input a.png --input b.png [--out-prefix out] [--no-invert] (multi-view)\n" + " da3-cli depth-upscale --model --input --sensor-depth --tiff [--degree N] [--threads N]\n" " da3-cli reconstruct --model --input --ply [--pose ]\n" " da3-cli quantize (type: f16|q8_0|q6_k|q5_k|q4_k)\n"); } @@ -54,6 +55,24 @@ Parsed parse(int argc, char** argv){ else if (r.inputs.empty()) r.error = "depth: --input required"; return r; } + if (first == "depth-upscale"){ + r.sub = Sub::DepthUpscale; + for (int i=2;i #include namespace da { namespace cli { -enum class Sub { Info, Depth, Reconstruct, Quantize, Help, None }; +enum class Sub { Info, Depth, DepthUpscale, Reconstruct, Quantize, Help, None }; struct Parsed { Sub sub = Sub::None; std::string model; @@ -16,9 +16,12 @@ struct Parsed { bool colmap_binary = true; // --colmap => bin; --colmap-txt => txt variant std::vector inputs; // accumulates repeated --input; >1 => multi-view mode std::string out_prefix; // --out-prefix for multi-view outputs + std::string sensor_depth_tiff; // depth-upscale: packed two-channel sensor TIFF + std::string output_depth_tiff; // depth-upscale: calibrated uint16 TIFF std::string q_in, q_out, q_type; // quantize: int n_threads = 0; // --threads N (0 => engine default of 1) int repeat = 0; // --repeat N (bench hook: time N inferences, then exit) + int upscale_degree = 2; // depth-upscale: polynomial calibration degree bool invert = true; // Single-image depth uses the REAL DA3 native-resolution resize by default. // --legacy-resize forces the old floor-to-patch path (fixture parity only). diff --git a/src/depth_upscale.cpp b/src/depth_upscale.cpp new file mode 100644 index 0000000..a356ae1 --- /dev/null +++ b/src/depth_upscale.cpp @@ -0,0 +1,217 @@ +#include "depth_upscale.hpp" + +#include +#include +#include + +namespace da { +namespace { + +void set_error(std::string* error, const char* message) { + if (error) *error = message; +} + +bool valid_shape(size_t count, int h, int w) { + return h > 0 && w > 0 && count == static_cast(h) * w; +} + +int reflect_index(int index, int size) { + while (index < 0 || index >= size) { + index = index < 0 ? -index - 1 : 2 * size - index - 1; + } + return index; +} + +std::vector resize_bilinear_reflect(const std::vector& image, int in_h, int in_w, + int out_h, int out_w) { + std::vector out(static_cast(out_h) * out_w); + for (int y = 0; y < out_h; ++y) { + const float source_y = (static_cast(y) + 0.5f) * in_h / out_h - 0.5f; + const int y0 = static_cast(std::floor(source_y)); + const float fy = source_y - y0; + const int y1 = y0 + 1; + for (int x = 0; x < out_w; ++x) { + const float source_x = (static_cast(x) + 0.5f) * in_w / out_w - 0.5f; + const int x0 = static_cast(std::floor(source_x)); + const float fx = source_x - x0; + const int x1 = x0 + 1; + const float a = image[static_cast(reflect_index(y0, in_h)) * in_w + reflect_index(x0, in_w)]; + const float b = image[static_cast(reflect_index(y0, in_h)) * in_w + reflect_index(x1, in_w)]; + const float c = image[static_cast(reflect_index(y1, in_h)) * in_w + reflect_index(x0, in_w)]; + const float d = image[static_cast(reflect_index(y1, in_h)) * in_w + reflect_index(x1, in_w)]; + out[static_cast(y) * out_w + x] = + (a + (b - a) * fx) + ((c + (d - c) * fx) - (a + (b - a) * fx)) * fy; + } + } + return out; +} + +void gaussian_blur_reflect(std::vector& image, int h, int w, float sigma) { + if (sigma <= 0.0f) return; + const int radius = static_cast(std::ceil(4.0f * sigma)); + std::vector kernel(static_cast(2 * radius + 1)); + float sum = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + const float value = std::exp(-0.5f * offset * offset / (sigma * sigma)); + kernel[static_cast(offset + radius)] = value; + sum += value; + } + for (float& value : kernel) value /= sum; + + std::vector tmp(image.size()); + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + float value = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + value += kernel[static_cast(offset + radius)] * + image[static_cast(y) * w + reflect_index(x + offset, w)]; + } + tmp[static_cast(y) * w + x] = value; + } + } + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + float value = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + value += kernel[static_cast(offset + radius)] * + tmp[static_cast(reflect_index(y + offset, h)) * w + x]; + } + image[static_cast(y) * w + x] = value; + } + } +} + +bool solve_linear_system(std::vector& matrix, std::vector& rhs, int n) { + for (int column = 0; column < n; ++column) { + int pivot = column; + for (int row = column + 1; row < n; ++row) { + if (std::abs(matrix[static_cast(row) * n + column]) > + std::abs(matrix[static_cast(pivot) * n + column])) { + pivot = row; + } + } + const double pivot_value = matrix[static_cast(pivot) * n + column]; + if (!std::isfinite(pivot_value) || std::abs(pivot_value) < 1e-12) return false; + if (pivot != column) { + for (int j = column; j < n; ++j) { + std::swap(matrix[static_cast(pivot) * n + j], matrix[static_cast(column) * n + j]); + } + std::swap(rhs[pivot], rhs[column]); + } + for (int row = column + 1; row < n; ++row) { + const double factor = matrix[static_cast(row) * n + column] / pivot_value; + for (int j = column; j < n; ++j) { + matrix[static_cast(row) * n + j] -= factor * matrix[static_cast(column) * n + j]; + } + rhs[row] -= factor * rhs[column]; + } + } + for (int row = n - 1; row >= 0; --row) { + double value = rhs[row]; + for (int column = row + 1; column < n; ++column) { + value -= matrix[static_cast(row) * n + column] * rhs[column]; + } + rhs[row] = value / matrix[static_cast(row) * n + row]; + if (!std::isfinite(rhs[row])) return false; + } + return true; +} + +} // namespace + +bool normalize_depth_u8(const std::vector& depth, std::vector& normalized, + std::string* error) { + if (depth.empty()) { + set_error(error, "model depth is empty"); + return false; + } + auto range = std::minmax_element(depth.begin(), depth.end()); + if (!std::isfinite(*range.first) || !std::isfinite(*range.second) || *range.first >= *range.second) { + set_error(error, "model depth must contain a finite non-constant range"); + return false; + } + const float scale = 255.0f / (*range.second - *range.first); + normalized.resize(depth.size()); + for (size_t i = 0; i < depth.size(); ++i) { + normalized[i] = static_cast(std::clamp((depth[i] - *range.first) * scale, 0.0f, 255.0f)); + } + return true; +} + +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + std::string* error) { + if (!valid_shape(sensor_depth.size(), sensor_h, sensor_w) || + !valid_shape(relative_depth.size(), relative_h, relative_w)) { + set_error(error, "depth dimensions do not match pixel counts"); + return false; + } + if (options.degree < 0 || options.degree > 8 || !std::isfinite(options.gaussian_sigma) || + options.gaussian_sigma < 0.0f) { + set_error(error, "invalid polynomial degree or Gaussian sigma"); + return false; + } + + std::vector predictor = resize_bilinear_reflect(relative_depth, relative_h, relative_w, sensor_h, sensor_w); + gaussian_blur_reflect(predictor, sensor_h, sensor_w, options.gaussian_sigma); + const float predictor_max = *std::max_element(predictor.begin(), predictor.end()); + const uint16_t sensor_max = *std::max_element(sensor_depth.begin(), sensor_depth.end()); + if (!std::isfinite(predictor_max) || predictor_max <= 0.0f || sensor_max == 0) { + set_error(error, "sensor and relative depth must have positive maxima"); + return false; + } + + const int coefficient_count = options.degree + 1; + std::vector normal(static_cast(coefficient_count) * coefficient_count, 0.0); + std::vector rhs(coefficient_count, 0.0); + size_t valid_count = 0; + for (size_t i = 0; i < predictor.size(); ++i) { + if (sensor_depth[i] == 0 || predictor[i] <= 0.0f || !std::isfinite(predictor[i])) continue; + const double x = predictor[i] / predictor_max; + const double y = sensor_depth[i]; + double powers[17] = {1.0}; + for (int p = 1; p <= 2 * options.degree; ++p) powers[p] = powers[p - 1] * x; + for (int row = 0; row < coefficient_count; ++row) { + rhs[row] += powers[row] * y; + for (int column = 0; column < coefficient_count; ++column) { + normal[static_cast(row) * coefficient_count + column] += powers[row + column]; + } + } + ++valid_count; + } + if (valid_count < static_cast(coefficient_count) || !solve_linear_system(normal, rhs, coefficient_count)) { + set_error(error, "insufficient non-degenerate overlapping depth samples for polynomial fit"); + return false; + } + + // The original workflow clips to uint16 before its final full-range + // normalization. Preserve that order so the TIFF spans exactly [0, 65535]. + std::vector scaled(predictor.size()); + uint16_t output_min = std::numeric_limits::max(); + uint16_t output_max = 0; + for (size_t i = 0; i < predictor.size(); ++i) { + const double x = predictor[i] / predictor_max; + double value = 0.0; + for (int coefficient = coefficient_count - 1; coefficient >= 0; --coefficient) { + value = value * x + rhs[coefficient]; + } + const uint16_t clipped = static_cast(std::clamp(value, 0.0, 65535.0)); + scaled[i] = clipped; + output_min = std::min(output_min, clipped); + output_max = std::max(output_max, clipped); + } + if (output_min >= output_max) { + set_error(error, "polynomial fit produced a constant or invalid depth map"); + return false; + } + + const double normalization = 65535.0 / (output_max - output_min); + output.resize(scaled.size()); + for (size_t i = 0; i < scaled.size(); ++i) { + output[i] = static_cast(std::clamp((scaled[i] - output_min) * normalization, 0.0, 65535.0)); + } + return true; +} + +} // namespace da diff --git a/src/depth_upscale.hpp b/src/depth_upscale.hpp new file mode 100644 index 0000000..866875e --- /dev/null +++ b/src/depth_upscale.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + +namespace da { + +struct DepthUpscaleOptions { + int degree = 2; + float gaussian_sigma = 1.0f; +}; + +// Calibrates a relative Depth Anything map against packed sensor depth. The +// predictor is resized, Gaussian-smoothed, polynomial-fitted over pixels where +// both maps are valid, then normalized to the full uint16 output range. +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options = {}, + std::string* error = nullptr); + +// Matches the former service boundary: render a model depth field to the +// normalized 8-bit image that feeds upscale_depth_map. +bool normalize_depth_u8(const std::vector& depth, std::vector& normalized, + std::string* error = nullptr); + +} // namespace da diff --git a/src/tiff_io.cpp b/src/tiff_io.cpp new file mode 100644 index 0000000..0ce6e1a --- /dev/null +++ b/src/tiff_io.cpp @@ -0,0 +1,143 @@ +#include "tiff_io.hpp" + +#include +#include + +namespace da { +namespace { + +void set_error(std::string* error, const char* message) { + if (error) *error = message; +} + +bool dimensions_fit_int(uint32_t width, uint32_t height) { + return width > 0 && height > 0 && + width <= static_cast(std::numeric_limits::max()) && + height <= static_cast(std::numeric_limits::max()); +} + +} // namespace + +bool load_tiff_rgb(const std::string& path, Image& out, std::string* error) { + TIFF* tiff = TIFFOpen(path.c_str(), "r"); + if (!tiff) { + set_error(error, "cannot open TIFF"); + return false; + } + + uint32_t width = 0, height = 0; + const bool valid = TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &width) && + TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &height) && + dimensions_fit_int(width, height); + if (!valid || static_cast(width) > std::numeric_limits::max() / height) { + TIFFClose(tiff); + set_error(error, "invalid TIFF dimensions"); + return false; + } + + std::vector rgba(static_cast(width) * height); + if (!TIFFReadRGBAImageOriented(tiff, width, height, rgba.data(), ORIENTATION_TOPLEFT, 0)) { + TIFFClose(tiff); + set_error(error, "failed to decode TIFF pixels"); + return false; + } + TIFFClose(tiff); + + out.w = static_cast(width); + out.h = static_cast(height); + out.rgb.resize(rgba.size() * 3); + for (size_t i = 0; i < rgba.size(); ++i) { + out.rgb[3 * i + 0] = TIFFGetR(rgba[i]); + out.rgb[3 * i + 1] = TIFFGetG(rgba[i]); + out.rgb[3 * i + 2] = TIFFGetB(rgba[i]); + } + return true; +} + +bool load_packed_depth_tiff(const std::string& path, std::vector& depth, + int& h, int& w, std::string* error) { + TIFF* tiff = TIFFOpen(path.c_str(), "r"); + if (!tiff) { + set_error(error, "cannot open TIFF"); + return false; + } + + uint32_t width = 0, height = 0; + uint16_t bits = 0, samples = 0, planar = PLANARCONFIG_CONTIG; + const bool valid = TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &width) && + TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &height) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_BITSPERSAMPLE, &bits) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_SAMPLESPERPIXEL, &samples) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_PLANARCONFIG, &planar) && + dimensions_fit_int(width, height) && bits == 8 && samples >= 2 && + planar == PLANARCONFIG_CONTIG; + if (!valid || static_cast(width) > std::numeric_limits::max() / height) { + TIFFClose(tiff); + set_error(error, "expected an interleaved two-channel 8-bit depth TIFF"); + return false; + } + + const tsize_t scanline_size = TIFFScanlineSize(tiff); + if (scanline_size < static_cast(width) * samples) { + TIFFClose(tiff); + set_error(error, "invalid TIFF scanline size"); + return false; + } + std::vector row(static_cast(scanline_size)); + depth.resize(static_cast(width) * height); + for (uint32_t y = 0; y < height; ++y) { + if (TIFFReadScanline(tiff, row.data(), y, 0) < 0) { + TIFFClose(tiff); + depth.clear(); + set_error(error, "failed to decode depth TIFF scanline"); + return false; + } + for (uint32_t x = 0; x < width; ++x) { + const size_t src = static_cast(x) * samples; + depth[static_cast(y) * width + x] = + static_cast((static_cast(row[src]) << 8) | row[src + 1]); + } + } + TIFFClose(tiff); + h = static_cast(height); + w = static_cast(width); + return true; +} + +bool write_depth_tiff_u16(const std::string& path, const std::vector& depth, + int h, int w, std::string* error) { + if (h <= 0 || w <= 0 || depth.size() != static_cast(h) * w) { + set_error(error, "depth dimensions do not match pixel count"); + return false; + } + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) { + set_error(error, "cannot create TIFF"); + return false; + } + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, static_cast(w)) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, static_cast(h)) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 16) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 1) && + TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_UINT) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_MINISBLACK) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE) && + TIFFSetField(tiff, TIFFTAG_ROWSPERSTRIP, static_cast(h)); + if (!configured) { + TIFFClose(tiff); + set_error(error, "failed to configure output TIFF"); + return false; + } + for (int y = 0; y < h; ++y) { + if (TIFFWriteScanline(tiff, const_cast(depth.data() + static_cast(y) * w), y, 0) < 0) { + TIFFClose(tiff); + set_error(error, "failed to write depth TIFF scanline"); + return false; + } + } + TIFFClose(tiff); + return true; +} + +} // namespace da diff --git a/src/tiff_io.hpp b/src/tiff_io.hpp new file mode 100644 index 0000000..a6ee3f9 --- /dev/null +++ b/src/tiff_io.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "image_io.hpp" +#include +#include +#include + +namespace da { + +// Reads the first TIFF image directory as an 8-bit RGB image. Supports the +// compressed RGB TIFF captures used by the depth-upscale workflow. +bool load_tiff_rgb(const std::string& path, Image& out, std::string* error = nullptr); + +// Reads a depth sensor TIFF whose first two 8-bit interleaved samples encode a +// big-endian unsigned 16-bit depth value. +bool load_packed_depth_tiff(const std::string& path, std::vector& depth, + int& h, int& w, std::string* error = nullptr); + +// Writes one uncompressed, single-channel, unsigned 16-bit depth TIFF. +bool write_depth_tiff_u16(const std::string& path, const std::vector& depth, + int h, int w, std::string* error = nullptr); + +} // namespace da diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e53d0f..39e5972 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,7 @@ da_add_test(test_engine_pose) da_add_test(test_engine_mv) da_add_test(test_quantize) da_add_test(test_quantize_accuracy) +da_add_test(test_depth_upscale) # Standalone harness for scripts/oracle_fuse.py (takes CLI args; not a ctest). add_executable(fuse_dump fuse_dump.cpp) diff --git a/tests/test_depth_upscale.cpp b/tests/test_depth_upscale.cpp new file mode 100644 index 0000000..25a45c0 --- /dev/null +++ b/tests/test_depth_upscale.cpp @@ -0,0 +1,119 @@ +#include "depth_upscale.hpp" +#include "tiff_io.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace da; + +static int failures = 0; +static void check(bool ok, const char* message) { + std::fprintf(stderr, "%s: %s\n", ok ? "ok" : "FAIL", message); + if (!ok) ++failures; +} + +static std::string temporary_path(const char* suffix) { + return std::string("/tmp/da-depth-upscale-") + std::to_string(getpid()) + suffix; +} + +static bool write_packed_sensor(const std::string& path, const std::vector& values, int h, int w) { + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) return false; + uint16_t extra_sample = EXTRASAMPLE_UNASSALPHA; + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, static_cast(w)) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, static_cast(h)) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 8) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 2) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_MINISBLACK) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE) && + TIFFSetField(tiff, TIFFTAG_EXTRASAMPLES, 1, &extra_sample); + std::vector row(static_cast(w) * 2); + bool written = configured; + for (int y = 0; written && y < h; ++y) { + for (int x = 0; x < w; ++x) { + const uint16_t value = values[static_cast(y) * w + x]; + row[2 * x] = static_cast(value >> 8); + row[2 * x + 1] = static_cast(value); + } + written = TIFFWriteScanline(tiff, row.data(), y, 0) >= 0; + } + TIFFClose(tiff); + return written; +} + +static bool write_rgb_fixture(const std::string& path) { + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) return false; + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, 2) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, 1) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 8) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 3) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE); + const std::vector pixels { 1, 2, 3, 250, 251, 252 }; + const bool written = configured && TIFFWriteScanline(tiff, const_cast(pixels.data()), 0, 0) >= 0; + TIFFClose(tiff); + return written; +} +int main() { + + std::vector normalized; + std::string error; + check(normalize_depth_u8({-2.0f, 0.0f, 2.0f}, normalized, &error), "normalizes finite model depth"); + check(normalized == std::vector({0, 127, 255}), "normalization matches uint8 truncation semantics"); + check(!normalize_depth_u8({1.0f, 1.0f}, normalized, &error), "rejects constant model depth"); + + constexpr int h = 4, w = 5; + std::vector relative(h * w); + std::vector sensor(h * w); + for (int i = 0; i < h * w; ++i) { + relative[i] = static_cast(i + 1); + sensor[i] = static_cast(1000 + 200 * relative[i]); + } + DepthUpscaleOptions options; + options.degree = 1; + options.gaussian_sigma = 0.0f; + std::vector output; + check(upscale_depth_map(sensor, h, w, relative, h, w, output, options, &error), "fits linear overlapping depth"); + check(output.front() == 0 && output.back() == 65535, "normalizes calibrated map to full uint16 range"); + bool monotonic = true; + for (size_t i = 1; i < output.size(); ++i) monotonic = monotonic && output[i - 1] <= output[i]; + check(monotonic, "preserves relative-depth ordering after calibration"); + check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), + "rejects degenerate predictor"); + + const std::string packed_path = temporary_path("-packed.tiff"); + const std::string output_path = temporary_path("-output.tiff"); + const std::string rgb_path = temporary_path("-rgb.tiff"); + check(write_rgb_fixture(rgb_path), "writes RGB TIFF fixture"); + Image rgb; + check(load_tiff_rgb(rgb_path, rgb, &error), "reads RGB TIFF"); + check(rgb.w == 2 && rgb.h == 1 && rgb.rgb == std::vector({1, 2, 3, 250, 251, 252}), + "preserves RGB TIFF pixels"); + + check(write_packed_sensor(packed_path, sensor, h, w), "writes packed sensor fixture"); + std::vector loaded; int loaded_h = 0, loaded_w = 0; + check(load_packed_depth_tiff(packed_path, loaded, loaded_h, loaded_w, &error), "reads packed sensor TIFF"); + check(loaded_h == h && loaded_w == w && loaded == sensor, "decodes big-endian two-channel sensor depth"); + check(write_depth_tiff_u16(output_path, output, h, w, &error), "writes uint16 depth TIFF"); + + TIFF* tiff = TIFFOpen(output_path.c_str(), "r"); + uint16_t bits = 0, samples = 0, format = 0; + const bool metadata_ok = tiff && TIFFGetField(tiff, TIFFTAG_BITSPERSAMPLE, &bits) && + TIFFGetField(tiff, TIFFTAG_SAMPLESPERPIXEL, &samples) && + TIFFGetField(tiff, TIFFTAG_SAMPLEFORMAT, &format); + check(metadata_ok && bits == 16 && samples == 1 && format == SAMPLEFORMAT_UINT, + "output TIFF has uint16 single-channel metadata"); + if (tiff) TIFFClose(tiff); + std::remove(packed_path.c_str()); + std::remove(output_path.c_str()); + std::remove(rgb_path.c_str()); + std::fprintf(stderr, "%s (%d failures)\n", failures ? "FAILED" : "PASSED", failures); + return failures ? 1 : 0; +} From d03c6ecef5ede134fea8f0cd7d9cbb32c274121f Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Wed, 16 Sep 2026 09:40:40 +0200 Subject: [PATCH 2/7] Add DA3 range upscale library API --- README.md | 21 ++++++---- examples/cli/main.cpp | 17 ++------ include/da_capi.h | 12 +++++- server/engine.go | 2 +- src/cli.cpp | 2 +- src/da_capi.cpp | 46 +++++++++++++++++++- src/depth_upscale.cpp | 81 ++++++++++++++++++++++++++---------- src/depth_upscale.hpp | 22 ++++++++-- tests/test_capi.cpp | 2 +- tests/test_depth_upscale.cpp | 15 +++++-- 10 files changed, 165 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d252ee1..eb70282 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Given an image it recovers a dense **metric depth** map, per-pixel **confidence* - **Quantization** to f16 / q8_0 / q6_k / q5_k / q4_k - q4_k is **99 MB** (0.25x the f32) and near-lossless. - **CPU-first, GPU-ready.** Tuned CPU path (tinyBLAS, Winograd, flash-attention) plus CUDA / Metal / Vulkan ggml backends. - **Flat C API** (`include/da_capi.h`) - embed from C, C++, Go, or Rust. Powers the [LocalAI](#use-it-from-localai) backend. -- **SpexAI sensor-calibrated TIFF depth.** Run a DA2 relative-depth model on an RGB TIFF, fit it to a two-channel 16-bit sensor-depth TIFF, and emit a full-resolution uint16 TIFF. +- **SpexAI sensor-calibrated depth.** Run DA2 or single-file DA3 on a main-camera image, fit it to projected sensor ranges, and emit calibrated uint16 millimetres. - **Parity-first.** Every component is gated against PyTorch-dumped reference tensors; the end-to-end depth matches the real `net()` at correlation 1.0. --- @@ -83,11 +83,13 @@ On **GPU** (NVIDIA GB10, via `-DDA_GGML_CUDA=ON`) the ggml CUDA path with flash ### SpexAI depth-upscale capacity -`depth-upscale` is the SpexAI-specific capture workflow: it combines a DA2 relative-depth -prediction from an RGB TIFF with a paired sensor TIFF. The sensor image stores each uint16 +`depth-upscale` is the SpexAI-specific capture workflow: it combines a DA2 or single-file +DA3 prediction from an RGB TIFF with a paired sensor TIFF. The sensor image stores each uint16 depth sample in its first two interleaved uint8 channels (high byte, then low byte); the command -resizes and smooths the DA2 map, polynomial-fits it to valid sensor pixels, and writes an -uncompressed single-channel uint16 TIFF. +resizes and smooths the model map, polynomial-fits it to valid sensor pixels, and writes an +uncompressed single-channel uint16 TIFF in calibrated sensor units. The library API also accepts +a sparse range image already projected into the main camera by PointClouds, preserving its +millimetre scale rather than stretching each result to 0..65535. Measured as one cold task on the supplied SpexAI captures (3072×2048 RGB → 1280×720 depth) with `depth-anything2-large-q4_k.gguf` (289.1 MiB), four CPU threads: @@ -205,7 +207,7 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - # Nested metric-scale depth (two GGUFs) $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm -# Sensor-calibrated, full-resolution depth TIFF. The sensor input stores each +# Sensor-calibrated DA2/DA3 depth TIFF. The sensor input stores each # 16-bit depth sample in its first two 8-bit channels (high byte, low byte). $CLI depth-upscale --model models/depth-anything2-large-f32.gguf \ --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff @@ -249,7 +251,7 @@ Gallery entries cover base (q4_k/q8_0/f16/f32), small, large, giant, and mono-la ## C API -A flat C ABI (`include/da_capi.h`, `abi_version` 4) over `libdepthanything.so`: +A flat C ABI (`include/da_capi.h`, `abi_version` 12) over `libdepthanything.so`: ```c da_ctx* ctx = da_capi_load("model.gguf", /*threads*/ 8); @@ -263,6 +265,11 @@ da_capi_free_floats(depth); int n; float *xyz; unsigned char *rgb; da_capi_points(ctx, "photo.jpg", /*conf_thresh*/ 1.0f, &n, &xyz, &rgb); // 3D cloud da_capi_export_glb(ctx, "photo.jpg", "scene.glb"); + +// PCL projects sensor points into the main-camera grid first. The result and +// output are caller-owned uint16 millimetre buffers of size range_h*range_w. +da_capi_depth_upscale(ctx, "main.tiff", projected_range_mm, range_h, range_w, + /*degree*/ 2, /*gaussian_sigma*/ 1.0f, dense_range_mm); da_capi_free(ctx); ``` diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 1e26481..d65c7e5 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -168,24 +168,15 @@ static int cmd_depth_upscale(const da::cli::Parsed& p){ } auto eng = da::Engine::load(p.model, p.n_threads); if (!eng){ std::fprintf(stderr, "error: load failed\n"); return 1; } - if (!eng->is_da2()){ - std::fprintf(stderr, "error: depth-upscale requires a Depth Anything V2 GGUF\n"); - return 1; - } std::vector predicted; int prediction_h = 0, prediction_w = 0; - if (!eng->depth_relative(image, predicted, prediction_h, prediction_w)){ - std::fprintf(stderr, "error: Depth Anything V2 inference failed\n"); - return 1; - } - std::vector normalized; - if (!da::normalize_depth_u8(predicted, normalized, &error)){ - std::fprintf(stderr, "error: normalize model depth failed: %s\n", error.c_str()); + if (!da::predict_depth_for_upscale(*eng, image, predicted, prediction_h, prediction_w, &error)){ + std::fprintf(stderr, "error: model inference failed: %s\n", error.c_str()); return 1; } da::DepthUpscaleOptions options; options.degree = p.upscale_degree; std::vector output; - if (!da::upscale_depth_map(sensor_depth, sensor_h, sensor_w, normalized, prediction_h, prediction_w, + if (!da::upscale_depth_map(sensor_depth, sensor_h, sensor_w, predicted, prediction_h, prediction_w, output, options, &error)){ std::fprintf(stderr, "error: depth calibration failed: %s\n", error.c_str()); return 1; @@ -194,7 +185,7 @@ static int cmd_depth_upscale(const da::cli::Parsed& p){ std::fprintf(stderr, "error: write depth TIFF failed: %s\n", error.c_str()); return 1; } - std::printf("upscaled depth %dx%d from DA2 %dx%d -> %s\n", sensor_w, sensor_h, prediction_w, + std::printf("upscaled depth %dx%d from model %dx%d -> %s\n", sensor_w, sensor_h, prediction_w, prediction_h, p.output_depth_tiff.c_str()); return 0; } diff --git a/include/da_capi.h b/include/da_capi.h index f3ebb53..3d53f1e 100644 --- a/include/da_capi.h +++ b/include/da_capi.h @@ -1,6 +1,7 @@ #ifndef DA_CAPI_H #define DA_CAPI_H #include +#include #ifdef __cplusplus extern "C" { #endif @@ -19,7 +20,8 @@ typedef struct da_ctx da_ctx; 10: added da_capi_set_fuse_params — scene-relative TSDF voxel/truncation knobs for the next streamed fuse (exposed as viewer sliders). 11: added temporal exposed-face voxel mesh generation and retrieval for streamed - fused scenes. */ + fused scenes. + 12: added sensor-calibrated DA2/DA3 range upscaling. */ int da_capi_abi_version(void); /* Set scene-relative TSDF surface-fusion knobs consumed by the NEXT da_capi_points_stream whose fuse flag is set. voxel_frac: voxel edge as a fraction @@ -43,6 +45,14 @@ void da_capi_free(da_ctx* ctx); /* safe on NULL char* da_capi_info_json(da_ctx* ctx); void da_capi_free_string(char* s); const char* da_capi_last_error(da_ctx* ctx); /* owned by ctx, "" if none */ +/* Infer DA2 or single-file DA3 depth for image_path and calibrate it against a + sparse projected range image. projected_range_mm and out_range_mm are caller- + owned [range_h*range_w] buffers. The output remains in calibrated millimetres; + it is not normalized to the full uint16 range. Returns 0 or -1 on error. */ +int da_capi_depth_upscale(da_ctx* ctx, const char* image_path, + const uint16_t* projected_range_mm, int range_h, int range_w, + int polynomial_degree, float gaussian_sigma, + uint16_t* out_range_mm); /* Run depth on an image file. On success writes *out_h,*out_w and returns a malloc'd float[H*W] depth map (row-major); caller frees via da_capi_free_floats. NULL on error. */ float* da_capi_depth_path(da_ctx* ctx, const char* image_path, int* out_h, int* out_w); diff --git a/server/engine.go b/server/engine.go index 98b60a6..b4e37b4 100644 --- a/server/engine.go +++ b/server/engine.go @@ -1,5 +1,5 @@ // engine.go — purego bindings to libdepthanything (the flat C API in -// include/da_capi.h, ABI 11). No cgo: the shared library is dlopen'd once and each +// include/da_capi.h, ABI 12). No cgo: the shared library is dlopen'd once and each // model is loaded as its own da_ctx. A context is NOT thread-safe (one ggml // backend + compute graph), so all inference is serialized by the caller. package main diff --git a/src/cli.cpp b/src/cli.cpp index 269e5c1..0f022e0 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -11,7 +11,7 @@ void print_help(){ " da3-cli depth --model --input [--glb ] [--colmap ] [--colmap-txt ] (single-image 3D export)\n" " da3-cli depth --model --metric-model --input [--pfm ] (nested metric-scale depth)\n" " da3-cli depth --model --input a.png --input b.png [--out-prefix out] [--no-invert] (multi-view)\n" - " da3-cli depth-upscale --model --input --sensor-depth --tiff [--degree N] [--threads N]\n" + " da3-cli depth-upscale --model --input --sensor-depth --tiff [--degree N] [--threads N]\n" " da3-cli reconstruct --model --input --ply [--pose ]\n" " da3-cli quantize (type: f16|q8_0|q6_k|q5_k|q4_k)\n"); } diff --git a/src/da_capi.cpp b/src/da_capi.cpp index d14123c..0a23bce 100644 --- a/src/da_capi.cpp +++ b/src/da_capi.cpp @@ -1,5 +1,7 @@ #include "da_capi.h" #include "engine.hpp" +#include "depth_upscale.hpp" +#include "tiff_io.hpp" #include "preprocess.hpp" #include "image_io.hpp" #include "glb_export.hpp" @@ -90,7 +92,7 @@ static bool capi_run_nested(da_ctx* c, const char* image_path, } extern "C" { -int da_capi_abi_version(void){ return 11; } +int da_capi_abi_version(void){ return 12; } // Scene-relative TSDF fusion knobs applied by the NEXT da_capi_points_stream when // its fuse flag is set. voxel_frac = voxel edge as a fraction of the bbox diagonal @@ -129,6 +131,48 @@ char* da_capi_info_json(da_ctx* c){ } void da_capi_free_string(char* s){ std::free(s); } const char* da_capi_last_error(da_ctx* c){ return c ? c->last_error.c_str() : ""; } +int da_capi_depth_upscale(da_ctx* c, const char* image_path, + const uint16_t* projected_range_mm, int range_h, int range_w, + int polynomial_degree, float gaussian_sigma, + uint16_t* out_range_mm){ + if (!c || !c->engine || !image_path || !projected_range_mm || !out_range_mm || + range_h <= 0 || range_w <= 0){ + if (c) c->last_error = "depth_upscale: bad args"; + return -1; + } + da::Image image; + std::string path(image_path), error; + std::string lower = path; + for (char& ch : lower) ch = (char)std::tolower((unsigned char)ch); + const bool is_tiff = lower.size() >= 4 && + (lower.compare(lower.size() - 4, 4, ".tif") == 0 || + (lower.size() >= 5 && lower.compare(lower.size() - 5, 5, ".tiff") == 0)); + if (is_tiff ? !da::load_tiff_rgb(path, image, &error) : !da::load_image_rgb(path, image)){ + c->last_error = "depth_upscale: load image failed"; + if (!error.empty()) c->last_error += ": " + error; + return -1; + } + std::vector predicted; + int prediction_h = 0, prediction_w = 0; + if (!da::predict_depth_for_upscale(*c->engine, image, predicted, + prediction_h, prediction_w, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + const size_t count = (size_t)range_h * (size_t)range_w; + std::vector projected(projected_range_mm, projected_range_mm + count), output; + da::DepthUpscaleOptions options; + options.degree = polynomial_degree; + options.gaussian_sigma = gaussian_sigma; + if (!da::upscale_depth_map(projected, range_h, range_w, predicted, + prediction_h, prediction_w, output, options, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + std::memcpy(out_range_mm, output.data(), count * sizeof(uint16_t)); + c->last_error.clear(); + return 0; +} float* da_capi_depth_path(da_ctx* c, const char* image_path, int* out_h, int* out_w){ if (!c || !c->engine || !image_path){ if (c) c->last_error = "depth: bad args"; return nullptr; } std::vector depth, conf; int H = 0, W = 0; diff --git a/src/depth_upscale.cpp b/src/depth_upscale.cpp index a356ae1..dec5280 100644 --- a/src/depth_upscale.cpp +++ b/src/depth_upscale.cpp @@ -1,8 +1,9 @@ #include "depth_upscale.hpp" +#include "engine.hpp" +#include "image_io.hpp" #include #include -#include namespace da { namespace { @@ -22,7 +23,7 @@ int reflect_index(int index, int size) { return index; } -std::vector resize_bilinear_reflect(const std::vector& image, int in_h, int in_w, +std::vector resize_bilinear_reflect(const std::vector& image, int in_h, int in_w, int out_h, int out_w) { std::vector out(static_cast(out_h) * out_w); for (int y = 0; y < out_h; ++y) { @@ -119,6 +120,30 @@ bool solve_linear_system(std::vector& matrix, std::vector& rhs, } // namespace +bool predict_depth_for_upscale(Engine& engine, const Image& image, + std::vector& depth, int& h, int& w, + std::string* error) { + if (engine.is_nested()) { + set_error(error, "two-file nested DA3 models are not supported for range upscaling"); + return false; + } + if (engine.is_da2()) { + if (engine.depth_relative(image, depth, h, w)) return true; + set_error(error, "Depth Anything V2 inference failed"); + return false; + } + if (engine.is_mono()) { + std::vector sky; + if (engine.depth_mono(image, depth, sky, h, w)) return true; + set_error(error, "Depth Anything V3 mono inference failed"); + return false; + } + std::vector confidence; + if (engine.depth_native_image(image, depth, confidence, h, w)) return true; + set_error(error, "Depth Anything V3 inference failed"); + return false; +} + bool normalize_depth_u8(const std::vector& depth, std::vector& normalized, std::string* error) { if (depth.empty()) { @@ -139,7 +164,7 @@ bool normalize_depth_u8(const std::vector& depth, std::vector& n } bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, - const std::vector& relative_depth, int relative_h, int relative_w, + const std::vector& relative_depth, int relative_h, int relative_w, std::vector& output, const DepthUpscaleOptions& options, std::string* error) { if (!valid_shape(sensor_depth.size(), sensor_h, sensor_w) || @@ -153,7 +178,25 @@ bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, return false; } - std::vector predictor = resize_bilinear_reflect(relative_depth, relative_h, relative_w, sensor_h, sensor_w); + auto relative_range = std::minmax_element(relative_depth.begin(), relative_depth.end()); + if (!std::isfinite(*relative_range.first) || !std::isfinite(*relative_range.second) || + *relative_range.first >= *relative_range.second) { + set_error(error, "model depth must contain a finite non-constant range"); + return false; + } + const float relative_scale = 1.0f / (*relative_range.second - *relative_range.first); + std::vector normalized(relative_depth.size()); + for (size_t i = 0; i < relative_depth.size(); ++i) { + if (!std::isfinite(relative_depth[i])) { + set_error(error, "model depth must be finite"); + return false; + } + normalized[i] = std::clamp((relative_depth[i] - *relative_range.first) * relative_scale, + 0.0f, 1.0f); + } + + std::vector predictor = resize_bilinear_reflect(normalized, relative_h, relative_w, + sensor_h, sensor_w); gaussian_blur_reflect(predictor, sensor_h, sensor_w, options.gaussian_sigma); const float predictor_max = *std::max_element(predictor.begin(), predictor.end()); const uint16_t sensor_max = *std::max_element(sensor_depth.begin(), sensor_depth.end()); @@ -185,33 +228,25 @@ bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, return false; } - // The original workflow clips to uint16 before its final full-range - // normalization. Preserve that order so the TIFF spans exactly [0, 65535]. - std::vector scaled(predictor.size()); - uint16_t output_min = std::numeric_limits::max(); - uint16_t output_max = 0; + output.resize(predictor.size()); for (size_t i = 0; i < predictor.size(); ++i) { const double x = predictor[i] / predictor_max; double value = 0.0; for (int coefficient = coefficient_count - 1; coefficient >= 0; --coefficient) { value = value * x + rhs[coefficient]; } - const uint16_t clipped = static_cast(std::clamp(value, 0.0, 65535.0)); - scaled[i] = clipped; - output_min = std::min(output_min, clipped); - output_max = std::max(output_max, clipped); - } - if (output_min >= output_max) { - set_error(error, "polynomial fit produced a constant or invalid depth map"); - return false; - } - - const double normalization = 65535.0 / (output_max - output_min); - output.resize(scaled.size()); - for (size_t i = 0; i < scaled.size(); ++i) { - output[i] = static_cast(std::clamp((scaled[i] - output_min) * normalization, 0.0, 65535.0)); + output[i] = static_cast(std::lround(std::clamp(value, 0.0, 65535.0))); } return true; } +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + std::string* error) { + std::vector converted(relative_depth.begin(), relative_depth.end()); + return upscale_depth_map(sensor_depth, sensor_h, sensor_w, converted, relative_h, relative_w, + output, options, error); +} + } // namespace da diff --git a/src/depth_upscale.hpp b/src/depth_upscale.hpp index 866875e..a766f17 100644 --- a/src/depth_upscale.hpp +++ b/src/depth_upscale.hpp @@ -6,14 +6,30 @@ namespace da { +class Engine; +struct Image; + struct DepthUpscaleOptions { int degree = 2; float gaussian_sigma = 1.0f; }; -// Calibrates a relative Depth Anything map against packed sensor depth. The -// predictor is resized, Gaussian-smoothed, polynomial-fitted over pixels where -// both maps are valid, then normalized to the full uint16 output range. +// Runs the depth-only inference path suitable for range calibration. Supports +// DA2, DA3MONO, and the single-file DA3 DualDPT models without computing pose. +bool predict_depth_for_upscale(Engine& engine, const Image& image, + std::vector& depth, int& h, int& w, + std::string* error = nullptr); + +// Calibrates a Depth Anything map against projected sensor ranges. The +// predictor is normalized, resized, Gaussian-smoothed, and polynomial-fitted +// over pixels where sensor samples exist. Output remains calibrated uint16 +// sensor units (normally millimetres); it is not stretched to the full range. +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options = {}, + std::string* error = nullptr); + +// Compatibility overload for the former normalized-uint8 service boundary. bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, const std::vector& relative_depth, int relative_h, int relative_w, std::vector& output, const DepthUpscaleOptions& options = {}, diff --git a/tests/test_capi.cpp b/tests/test_capi.cpp index 2934912..f2a40df 100644 --- a/tests/test_capi.cpp +++ b/tests/test_capi.cpp @@ -3,7 +3,7 @@ #include #include int main(){ - if (da_capi_abi_version() != 11) return 1; + if (da_capi_abi_version() != 12) return 1; const char* gguf = std::getenv("DA_TEST_GGUF"); if (!gguf) return 77; da_ctx* c = da_capi_load(gguf, 1); diff --git a/tests/test_depth_upscale.cpp b/tests/test_depth_upscale.cpp index 25a45c0..912a1ab 100644 --- a/tests/test_depth_upscale.cpp +++ b/tests/test_depth_upscale.cpp @@ -1,5 +1,6 @@ #include "depth_upscale.hpp" #include "tiff_io.hpp" +#include "da_capi.h" #include #include @@ -63,6 +64,11 @@ static bool write_rgb_fixture(const std::string& path) { } int main() { + check(da_capi_abi_version() == 12, "reports C API version 12"); + uint16_t one = 1; + check(da_capi_depth_upscale(nullptr, "missing.tiff", &one, 1, 1, 2, 1.0f, &one) == -1, + "C API rejects a null context"); + std::vector normalized; std::string error; check(normalize_depth_u8({-2.0f, 0.0f, 2.0f}, normalized, &error), "normalizes finite model depth"); @@ -70,10 +76,10 @@ int main() { check(!normalize_depth_u8({1.0f, 1.0f}, normalized, &error), "rejects constant model depth"); constexpr int h = 4, w = 5; - std::vector relative(h * w); + std::vector relative(h * w); std::vector sensor(h * w); for (int i = 0; i < h * w; ++i) { - relative[i] = static_cast(i + 1); + relative[i] = static_cast(i + 1); sensor[i] = static_cast(1000 + 200 * relative[i]); } DepthUpscaleOptions options; @@ -81,11 +87,12 @@ int main() { options.gaussian_sigma = 0.0f; std::vector output; check(upscale_depth_map(sensor, h, w, relative, h, w, output, options, &error), "fits linear overlapping depth"); - check(output.front() == 0 && output.back() == 65535, "normalizes calibrated map to full uint16 range"); + check(output.front() == sensor.front() && output.back() == sensor.back(), + "preserves calibrated sensor units without full-range stretching"); bool monotonic = true; for (size_t i = 1; i < output.size(); ++i) monotonic = monotonic && output[i - 1] <= output[i]; check(monotonic, "preserves relative-depth ordering after calibration"); - check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), + check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), "rejects degenerate predictor"); const std::string packed_path = temporary_path("-packed.tiff"); From 25b50b748a8e29247fad6df29c48f7b191ec0cf9 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 21 Sep 2026 10:49:01 +0200 Subject: [PATCH 3/7] Add DA2 ONNX depth upscale default --- README.md | 15 ++- make.sh | 2 + requirements.txt | 1 + scripts/convert_da2_onnx_to_gguf.py | 150 ++++++++++++++++++++++++++++ scripts/fetch_models.sh | 3 +- src/cli.cpp | 6 +- tests/CMakeLists.txt | 1 + tests/test_cli.cpp | 34 +++++++ 8 files changed, 204 insertions(+), 8 deletions(-) create mode 100755 make.sh create mode 100755 scripts/convert_da2_onnx_to_gguf.py create mode 100644 tests/test_cli.cpp diff --git a/README.md b/README.md index eb70282..20464c8 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,12 @@ python scripts/convert_nested_to_gguf.py --model models/DA3NESTED-GIANT-LARGE -- python scripts/convert_da2_to_gguf.py --encoder vitl --ckpt models/depth_anything_v2_vitl.pth \ --output models/depth-anything2-large-f32.gguf --name Depth-Anything-V2-Large +# Legacy depth-upscale model: yuvraj108c's static ViT-B ONNX export. +# This writes the depth-upscale default and needs only onnx, numpy, and gguf. +python scripts/convert_da2_onnx_to_gguf.py \ + --onnx ../depth-upscale/models/depth_anything_v2/1/model.onnx \ + --output models/depth-anything2-base-f32.gguf + # Depth Anything V2 — metric (add --max-depth: 20 for Hypersim/indoor, 80 for VKITTI/outdoor) python scripts/convert_da2_to_gguf.py --encoder vits --ckpt models/depth_anything_v2_metric_hypersim_vits.pth \ --output models/depth-anything2-metric-hypersim-small-f32.gguf \ @@ -207,10 +213,11 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - # Nested metric-scale depth (two GGUFs) $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm -# Sensor-calibrated DA2/DA3 depth TIFF. The sensor input stores each -# 16-bit depth sample in its first two 8-bit channels (high byte, low byte). -$CLI depth-upscale --model models/depth-anything2-large-f32.gguf \ - --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff +# Sensor-calibrated DA2/DA3 depth TIFF. depth-upscale defaults to the converted +# yuvraj108c Depth Anything V2 ViT-B model at models/depth-anything2-base-f32.gguf. +# The sensor input stores each 16-bit depth sample in its first two 8-bit channels +# (high byte, low byte); pass --model to override the default. +$CLI depth-upscale --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff # Multi-view depth + pose $CLI depth --model $M --input a.jpg --input b.jpg --out-prefix scene diff --git a/make.sh b/make.sh new file mode 100755 index 0000000..3d77084 --- /dev/null +++ b/make.sh @@ -0,0 +1,2 @@ +cmake -B build -DDA_BUILD_CLI=ON -DCMAKE_CUDA_ARCHITECTURES=native -DDA_GGML_CUDA=ON +cmake --build build -j diff --git a/requirements.txt b/requirements.txt index 5880269..c994131 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ torch>=2.4 # CPU build is fine: torch==2.x+cpu numpy>=1.26 gguf>=0.19.0 # GGUF writer/reader +onnx>=1.16 # scripts/convert_da2_onnx_to_gguf.py huggingface_hub>=0.24 # scripts/download_model.py safetensors>=0.4 einops>=0.8 diff --git a/scripts/convert_da2_onnx_to_gguf.py b/scripts/convert_da2_onnx_to_gguf.py new file mode 100755 index 0000000..17bbf9b --- /dev/null +++ b/scripts/convert_da2_onnx_to_gguf.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Convert the yuvraj108c Depth Anything V2 ONNX exports to GGUF. + +The legacy depth-upscale service uses ONNX Runtime directly. depth-anything.cpp +uses the same parameter tensors through its DA2 GGUF route, so this converter +copies ONNX initializers into the existing tensor/metadata schema without +requiring PyTorch or the upstream Depth Anything source tree. +""" +import argparse +import math +import os +import sys + +import numpy as np +import onnx +from onnx import numpy_helper + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import gguf +import scripts.gguf_keys as K + + +ENCODERS = { + "vits": (384, 12, 6, [2, 5, 8, 11]), + "vitb": (768, 12, 12, [2, 5, 8, 11]), + "vitl": (1024, 24, 16, [4, 11, 17, 23]), + "vitg": (1536, 40, 24, [9, 19, 29, 39]), +} + + +def infer_encoder(initializers): + patch = initializers.get("pretrained.patch_embed.proj.weight") + if patch is None or len(patch.dims) != 4 or patch.dims[1:] != [3, 14, 14]: + raise ValueError("expected a Depth Anything V2 [embed,3,14,14] patch embedding") + embed_dim = int(patch.dims[0]) + block_count = 0 + while f"pretrained.blocks.{block_count}.attn.qkv.weight" in initializers: + block_count += 1 + for name, (expected_embed, expected_blocks, _, _) in ENCODERS.items(): + if (embed_dim, block_count) == (expected_embed, expected_blocks): + return name + raise ValueError(f"unsupported DA2 backbone: embed_dim={embed_dim}, blocks={block_count}") + + +def inferred_max_depth(path): + name = os.path.basename(path).lower() + if "metric_hypersim" in name: + return 20.0 + if "metric_vkitti" in name: + return 80.0 + return 0.0 + + +def validate_io(model): + if len(model.graph.input) != 1 or len(model.graph.output) != 1: + raise ValueError("expected one ONNX input and one output") + def dims(value): + return [int(d.dim_value) for d in value.type.tensor_type.shape.dim] + if dims(model.graph.input[0]) != [1, 3, 518, 518]: + raise ValueError(f"expected input [1,3,518,518], got {dims(model.graph.input[0])}") + if dims(model.graph.output[0]) != [1, 518, 518]: + raise ValueError(f"expected output [1,518,518], got {dims(model.graph.output[0])}") + + +def write_da2_onnx_gguf(onnx_path, output, checkpoint_name, max_depth): + model = onnx.load(onnx_path, load_external_data=False) + validate_io(model) + initializers = {tensor.name: tensor for tensor in model.graph.initializer} + encoder = infer_encoder(initializers) + embed_dim, depth, num_heads, out_layers = ENCODERS[encoder] + pos_embed = initializers.get("pretrained.pos_embed") + if pos_embed is None or list(pos_embed.dims) != [1, 1369 + 1, embed_dim]: + raise ValueError("invalid DA2 positional embedding") + head_features = int(initializers["depth_head.scratch.layer1_rn.weight"].dims[0]) + head_out_channels = [int(initializers[f"depth_head.projects.{i}.weight"].dims[0]) for i in range(4)] + qkv_bias = "pretrained.blocks.0.attn.qkv.bias" in initializers + mlp_hidden = int(initializers["pretrained.blocks.0.mlp.fc1.weight"].dims[0]) + pos_grid = math.isqrt(int(pos_embed.dims[1]) - 1) + + writer = gguf.GGUFWriter(output, K.ARCH) + writer.add_string(K.KV["arch"], "depthanything2") + writer.add_string(K.KV["checkpoint_name"], checkpoint_name) + writer.add_uint32(K.KV["patch_size"], 14) + writer.add_uint32(K.KV["vit.embed_dim"], embed_dim) + writer.add_uint32(K.KV["vit.depth"], depth) + writer.add_uint32(K.KV["vit.num_heads"], num_heads) + writer.add_uint32(K.KV["vit.head_dim"], embed_dim // num_heads) + writer.add_uint32(K.KV["vit.mlp_hidden"], mlp_hidden) + writer.add_string(K.KV["vit.ffn_type"], "mlp") + writer.add_uint32(K.KV["vit.num_register"], 0) + writer.add_float32(K.KV["vit.init_values"], 1.0) + writer.add_int32(K.KV["vit.alt_start"], -1) + writer.add_int32(K.KV["vit.rope_start"], -1) + writer.add_int32(K.KV["vit.qknorm_start"], -1) + writer.add_float32(K.KV["vit.rope_freq"], 100.0) + writer.add_bool(K.KV["vit.cat_token"], False) + writer.add_bool(K.KV["vit.qkv_bias"], qkv_bias) + writer.add_float32(K.KV["vit.ln_eps"], 1e-6) + writer.add_float32(K.KV["vit.interp_offset"], 0.1) + writer.add_bool(K.KV["vit.interp_antialias"], False) + writer.add_uint32(K.KV["vit.pos_embed_grid"], pos_grid) + writer.add_array(K.KV["vit.out_layers"], out_layers) + writer.add_array(K.KV["img.mean"], [0.485, 0.456, 0.406]) + writer.add_array(K.KV["img.std"], [0.229, 0.224, 0.225]) + writer.add_string(K.KV["img.resize_mode"], "lower_bound") + writer.add_uint32(K.KV["img.resize_target"], 518) + writer.add_uint32(K.KV["head.features"], head_features) + writer.add_array(K.KV["head.out_channels"], head_out_channels) + writer.add_uint32(K.KV["head.output_dim"], 1) + writer.add_bool(K.KV["head.pos_embed"], False) + writer.add_string(K.KV["head.activation"], "sigmoid" if max_depth > 0 else "relu") + writer.add_string(K.KV["head.norm_type"], "idt") + if max_depth > 0: + writer.add_float32(K.KV["head.max_depth"], max_depth) + + written = 0 + for name, tensor in initializers.items(): + if name.startswith("pretrained."): + gguf_name = K.rename_backbone(name.removeprefix("pretrained.")) + elif name.startswith("depth_head."): + gguf_name = K.rename_head(name.removeprefix("depth_head.")) + else: + raise ValueError(f"unexpected ONNX initializer {name}") + if gguf_name is None: + raise ValueError(f"unmapped ONNX initializer {name}") + values = np.ascontiguousarray(numpy_helper.to_array(tensor), dtype=np.float32) + writer.add_tensor(gguf_name, values) + written += 1 + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + print(f"wrote {output}: tensors={written} arch=depthanything2 encoder={encoder} max_depth={max_depth:g}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--onnx", required=True, help="Depth Anything V2 ONNX model") + parser.add_argument("--output", required=True, help="destination GGUF path") + parser.add_argument("--name", default="yuvraj108c/Depth-Anything-2-Onnx") + parser.add_argument("--max-depth", type=float, default=None, + help="metric range in metres; inferred from metric model filenames") + args = parser.parse_args() + max_depth = inferred_max_depth(args.onnx) if args.max_depth is None else args.max_depth + write_da2_onnx_gguf(args.onnx, args.output, args.name, max_depth) + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch_models.sh b/scripts/fetch_models.sh index 6908f6a..e08ed9c 100755 --- a/scripts/fetch_models.sh +++ b/scripts/fetch_models.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Pre-download the curated DA3 model set for the demo app (~13 GB). +# Pre-download the curated DA3 model set plus the DA2 depth-upscale default (~13 GB). # One weight per capability, not the full 31 GB of redundant quants. # Idempotent: skips files already present at the right size; resumes partials. set -uo pipefail @@ -16,6 +16,7 @@ CURATED=( "relative+pose+gaussians depth-anything-giant-f32.gguf" "metric+pose (anyview branch) depth-anything-nested-anyview.gguf" "metric+pose (metric branch) depth-anything-nested-metric.gguf" + "relative depth (upscale default) depth-anything2-base-f32.gguf" ) echo "Downloading curated DA3 models into $DIR" diff --git a/src/cli.cpp b/src/cli.cpp index 0f022e0..675031f 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -11,7 +11,7 @@ void print_help(){ " da3-cli depth --model --input [--glb ] [--colmap ] [--colmap-txt ] (single-image 3D export)\n" " da3-cli depth --model --metric-model --input [--pfm ] (nested metric-scale depth)\n" " da3-cli depth --model --input a.png --input b.png [--out-prefix out] [--no-invert] (multi-view)\n" - " da3-cli depth-upscale --model --input --sensor-depth --tiff [--degree N] [--threads N]\n" + " da3-cli depth-upscale [--model ] --input --sensor-depth --tiff [--degree N] [--threads N]\n" " da3-cli reconstruct --model --input --ply [--pose ]\n" " da3-cli quantize (type: f16|q8_0|q6_k|q5_k|q4_k)\n"); } @@ -67,8 +67,8 @@ Parsed parse(int argc, char** argv){ else if (a == "--threads" && i+1 +#include +#include + +static int failures = 0; + +static void check(bool ok, const char* message) { + std::fprintf(stderr, "%s: %s\n", ok ? "ok" : "FAIL", message); + if (!ok) ++failures; +} + +static da::cli::Parsed parse(std::vector args) { + std::vector argv; + argv.reserve(args.size()); + for (std::string& arg : args) argv.push_back(arg.data()); + return da::cli::parse(static_cast(argv.size()), argv.data()); +} + +int main() { + auto defaults = parse({"da3-cli", "depth-upscale", "--input", "rgb.tiff", + "--sensor-depth", "sensor.tiff", "--tiff", "depth.tiff"}); + check(defaults.error.empty(), "depth-upscale accepts required arguments without --model"); + check(defaults.model == "models/depth-anything2-base-f32.gguf", + "depth-upscale uses the converted DA2 ONNX model by default"); + + auto explicit_model = parse({"da3-cli", "depth-upscale", "--model", "custom.gguf", + "--input", "rgb.tiff", "--sensor-depth", "sensor.tiff", + "--tiff", "depth.tiff"}); + check(explicit_model.error.empty(), "depth-upscale accepts an explicit model"); + check(explicit_model.model == "custom.gguf", "explicit depth-upscale model overrides default"); + return failures == 0 ? 0 : 1; +} From 2877eb84d50017ec5d0b323197f5346efa6e8f60 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 21 Sep 2026 14:42:10 +0200 Subject: [PATCH 4/7] Harden depth upscale C API --- include/da_capi.h | 5 +-- src/da_capi.cpp | 67 +++++++++++++++++++++--------------- src/depth_upscale.cpp | 50 +++++++++++++++++---------- src/depth_upscale.hpp | 2 ++ tests/test_capi_da2.cpp | 23 ++++++++++++- tests/test_depth_upscale.cpp | 11 ++++++ 6 files changed, 108 insertions(+), 50 deletions(-) diff --git a/include/da_capi.h b/include/da_capi.h index 3d53f1e..3766b95 100644 --- a/include/da_capi.h +++ b/include/da_capi.h @@ -47,8 +47,9 @@ void da_capi_free_string(char* s); const char* da_capi_last_error(da_ctx* ctx); /* owned by ctx, "" if none */ /* Infer DA2 or single-file DA3 depth for image_path and calibrate it against a sparse projected range image. projected_range_mm and out_range_mm are caller- - owned [range_h*range_w] buffers. The output remains in calibrated millimetres; - it is not normalized to the full uint16 range. Returns 0 or -1 on error. */ + owned [range_h*range_w] buffers. gaussian_sigma must be finite in [0, 64]. + The output remains in calibrated millimetres; it is not normalized to the full + uint16 range. Returns 0 or -1 on error. */ int da_capi_depth_upscale(da_ctx* ctx, const char* image_path, const uint16_t* projected_range_mm, int range_h, int range_w, int polynomial_degree, float gaussian_sigma, diff --git a/src/da_capi.cpp b/src/da_capi.cpp index 0a23bce..0a321cb 100644 --- a/src/da_capi.cpp +++ b/src/da_capi.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -140,38 +141,48 @@ int da_capi_depth_upscale(da_ctx* c, const char* image_path, if (c) c->last_error = "depth_upscale: bad args"; return -1; } - da::Image image; - std::string path(image_path), error; - std::string lower = path; - for (char& ch : lower) ch = (char)std::tolower((unsigned char)ch); - const bool is_tiff = lower.size() >= 4 && - (lower.compare(lower.size() - 4, 4, ".tif") == 0 || - (lower.size() >= 5 && lower.compare(lower.size() - 5, 5, ".tiff") == 0)); - if (is_tiff ? !da::load_tiff_rgb(path, image, &error) : !da::load_image_rgb(path, image)){ - c->last_error = "depth_upscale: load image failed"; - if (!error.empty()) c->last_error += ": " + error; + if (!std::isfinite(gaussian_sigma) || gaussian_sigma < 0.0f || + gaussian_sigma > da::kMaxDepthUpscaleGaussianSigma) { + c->last_error = "depth_upscale: invalid Gaussian sigma"; return -1; } - std::vector predicted; - int prediction_h = 0, prediction_w = 0; - if (!da::predict_depth_for_upscale(*c->engine, image, predicted, - prediction_h, prediction_w, &error)){ - c->last_error = "depth_upscale: " + error; - return -1; - } - const size_t count = (size_t)range_h * (size_t)range_w; - std::vector projected(projected_range_mm, projected_range_mm + count), output; - da::DepthUpscaleOptions options; - options.degree = polynomial_degree; - options.gaussian_sigma = gaussian_sigma; - if (!da::upscale_depth_map(projected, range_h, range_w, predicted, - prediction_h, prediction_w, output, options, &error)){ - c->last_error = "depth_upscale: " + error; + try { + da::Image image; + std::string path(image_path), error; + std::string lower = path; + for (char& ch : lower) ch = (char)std::tolower((unsigned char)ch); + const bool is_tiff = lower.size() >= 4 && + (lower.compare(lower.size() - 4, 4, ".tif") == 0 || + (lower.size() >= 5 && lower.compare(lower.size() - 5, 5, ".tiff") == 0)); + if (is_tiff ? !da::load_tiff_rgb(path, image, &error) : !da::load_image_rgb(path, image)){ + c->last_error = "depth_upscale: load image failed"; + if (!error.empty()) c->last_error += ": " + error; + return -1; + } + std::vector predicted; + int prediction_h = 0, prediction_w = 0; + if (!da::predict_depth_for_upscale(*c->engine, image, predicted, + prediction_h, prediction_w, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + const size_t count = (size_t)range_h * (size_t)range_w; + std::vector projected(projected_range_mm, projected_range_mm + count), output; + da::DepthUpscaleOptions options; + options.degree = polynomial_degree; + options.gaussian_sigma = gaussian_sigma; + if (!da::upscale_depth_map(projected, range_h, range_w, predicted, + prediction_h, prediction_w, output, options, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + std::memcpy(out_range_mm, output.data(), count * sizeof(uint16_t)); + c->last_error.clear(); + return 0; + } catch (const std::bad_alloc&) { + c->last_error = "depth_upscale: allocation failed"; return -1; } - std::memcpy(out_range_mm, output.data(), count * sizeof(uint16_t)); - c->last_error.clear(); - return 0; } float* da_capi_depth_path(da_ctx* c, const char* image_path, int* out_h, int* out_w){ if (!c || !c->engine || !image_path){ if (c) c->last_error = "depth: bad args"; return nullptr; } diff --git a/src/depth_upscale.cpp b/src/depth_upscale.cpp index dec5280..c036a9c 100644 --- a/src/depth_upscale.cpp +++ b/src/depth_upscale.cpp @@ -163,36 +163,40 @@ bool normalize_depth_u8(const std::vector& depth, std::vector& n return true; } -bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, - const std::vector& relative_depth, int relative_h, int relative_w, - std::vector& output, const DepthUpscaleOptions& options, - std::string* error) { +static bool upscale_depth_map_impl(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + bool normalize_relative, std::string* error) { if (!valid_shape(sensor_depth.size(), sensor_h, sensor_w) || !valid_shape(relative_depth.size(), relative_h, relative_w)) { set_error(error, "depth dimensions do not match pixel counts"); return false; } if (options.degree < 0 || options.degree > 8 || !std::isfinite(options.gaussian_sigma) || - options.gaussian_sigma < 0.0f) { + options.gaussian_sigma < 0.0f || options.gaussian_sigma > kMaxDepthUpscaleGaussianSigma) { set_error(error, "invalid polynomial degree or Gaussian sigma"); return false; } - auto relative_range = std::minmax_element(relative_depth.begin(), relative_depth.end()); - if (!std::isfinite(*relative_range.first) || !std::isfinite(*relative_range.second) || - *relative_range.first >= *relative_range.second) { - set_error(error, "model depth must contain a finite non-constant range"); - return false; - } - const float relative_scale = 1.0f / (*relative_range.second - *relative_range.first); std::vector normalized(relative_depth.size()); - for (size_t i = 0; i < relative_depth.size(); ++i) { - if (!std::isfinite(relative_depth[i])) { - set_error(error, "model depth must be finite"); + if (normalize_relative) { + auto relative_range = std::minmax_element(relative_depth.begin(), relative_depth.end()); + if (!std::isfinite(*relative_range.first) || !std::isfinite(*relative_range.second) || + *relative_range.first >= *relative_range.second) { + set_error(error, "model depth must contain a finite non-constant range"); return false; } - normalized[i] = std::clamp((relative_depth[i] - *relative_range.first) * relative_scale, - 0.0f, 1.0f); + const float relative_scale = 1.0f / (*relative_range.second - *relative_range.first); + for (size_t i = 0; i < relative_depth.size(); ++i) { + if (!std::isfinite(relative_depth[i])) { + set_error(error, "model depth must be finite"); + return false; + } + normalized[i] = std::clamp((relative_depth[i] - *relative_range.first) * relative_scale, + 0.0f, 1.0f); + } + } else { + normalized = relative_depth; } std::vector predictor = resize_bilinear_reflect(normalized, relative_h, relative_w, @@ -240,13 +244,21 @@ bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, return true; } +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + std::string* error) { + return upscale_depth_map_impl(sensor_depth, sensor_h, sensor_w, relative_depth, relative_h, relative_w, + output, options, true, error); +} + bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, const std::vector& relative_depth, int relative_h, int relative_w, std::vector& output, const DepthUpscaleOptions& options, std::string* error) { std::vector converted(relative_depth.begin(), relative_depth.end()); - return upscale_depth_map(sensor_depth, sensor_h, sensor_w, converted, relative_h, relative_w, - output, options, error); + return upscale_depth_map_impl(sensor_depth, sensor_h, sensor_w, converted, relative_h, relative_w, + output, options, false, error); } } // namespace da diff --git a/src/depth_upscale.hpp b/src/depth_upscale.hpp index a766f17..3873ef2 100644 --- a/src/depth_upscale.hpp +++ b/src/depth_upscale.hpp @@ -8,6 +8,8 @@ namespace da { class Engine; struct Image; +inline constexpr float kMaxDepthUpscaleGaussianSigma = 64.0f; + struct DepthUpscaleOptions { int degree = 2; diff --git a/tests/test_capi_da2.cpp b/tests/test_capi_da2.cpp index e8ac79e..46b849b 100644 --- a/tests/test_capi_da2.cpp +++ b/tests/test_capi_da2.cpp @@ -4,6 +4,8 @@ #include "da_capi.h" #include #include +#include +#include #include static bool finite_all(const float* p, int n){ @@ -22,6 +24,25 @@ int main(){ da_ctx* c = da_capi_load(gguf, 1); if (!c){ std::fprintf(stderr, "da2: load failed\n"); return 1; } + const uint16_t projected_range[] = { + 1000, 1100, 1200, 1300, + 1150, 1250, 1350, 1450, + 1300, 1400, 1500, 1600, + 1450, 1550, 1650, 1750, + }; + uint16_t upscaled[16] = {}; + const int invalid_sigma = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 1, std::numeric_limits::max(), upscaled); + const int invalid_image = da_capi_depth_upscale( + c, "missing-image.jpg", projected_range, 4, 4, 1, 0.0f, upscaled); + const int upscale = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 1, 0.0f, upscaled); + bool upscale_ok = invalid_sigma == -1 && invalid_image == -1 && upscale == 0; + for (uint16_t value : upscaled) upscale_ok = upscale_ok && value > 0; + std::fprintf(stderr, "da2 upscale: invalid_sigma=%d invalid_image=%d result=%d -> %s\n", + invalid_sigma, invalid_image, upscale, upscale_ok ? "OK" : "FAIL"); + + int H=0, W=0, is_metric=-1; float *depth=nullptr, *conf=nullptr, *sky=nullptr; float ext[12], intr[9]; int r = da_capi_depth_dense(c, png, &H, &W, &depth, &conf, &sky, ext, intr, &is_metric); @@ -39,5 +60,5 @@ int main(){ rp, da_capi_last_error(c), okp?"OK":"FAIL"); da_capi_free(c); - return (ok && okp) ? 0 : 1; + return (ok && okp && upscale_ok) ? 0 : 1; } diff --git a/tests/test_depth_upscale.cpp b/tests/test_depth_upscale.cpp index 912a1ab..ab0890b 100644 --- a/tests/test_depth_upscale.cpp +++ b/tests/test_depth_upscale.cpp @@ -75,6 +75,7 @@ int main() { check(normalized == std::vector({0, 127, 255}), "normalization matches uint8 truncation semantics"); check(!normalize_depth_u8({1.0f, 1.0f}, normalized, &error), "rejects constant model depth"); + constexpr int h = 4, w = 5; std::vector relative(h * w); std::vector sensor(h * w); @@ -94,6 +95,16 @@ int main() { check(monotonic, "preserves relative-depth ordering after calibration"); check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), "rejects degenerate predictor"); + // The uint8 compatibility overload must retain the supplied nonzero range: + // normalizing it first would turn 10 into zero and leave only three samples + // for this cubic fit. + std::vector legacy_sensor {1000, 2000, 3000, 4000}; + std::vector legacy_relative {10, 20, 30, 40}; + std::vector legacy_output; + options.degree = 3; + check(upscale_depth_map(legacy_sensor, 2, 2, legacy_relative, 2, 2, legacy_output, options, &error), + "preserves legacy uint8 predictor values during calibration"); + options.degree = 1; const std::string packed_path = temporary_path("-packed.tiff"); const std::string output_path = temporary_path("-output.tiff"); From b14d9a347a99bec62364f5c35c1e5325baa4fdf5 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 21 Sep 2026 18:35:35 +0200 Subject: [PATCH 5/7] Cover depth upscale C API routes --- src/da_capi.cpp | 21 ++++++++++--- tests/CMakeLists.txt | 1 + tests/test_capi_da2.cpp | 8 +++-- tests/test_capi_upscale.cpp | 63 +++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 tests/test_capi_upscale.cpp diff --git a/src/da_capi.cpp b/src/da_capi.cpp index 0a321cb..70120cc 100644 --- a/src/da_capi.cpp +++ b/src/da_capi.cpp @@ -41,6 +41,15 @@ struct da_ctx { std::vector temporal_voxel_mesh; }; +static void capi_set_error(da_ctx* c, const char* message) noexcept { + if (!c) return; + try { + c->last_error = message; + } catch (...) { + c->last_error.clear(); + } +} + static char* dup_cstr(const std::string& s){ char* p = (char*)std::malloc(s.size()+1); if (p) std::memcpy(p, s.c_str(), s.size()+1); @@ -138,12 +147,13 @@ int da_capi_depth_upscale(da_ctx* c, const char* image_path, uint16_t* out_range_mm){ if (!c || !c->engine || !image_path || !projected_range_mm || !out_range_mm || range_h <= 0 || range_w <= 0){ - if (c) c->last_error = "depth_upscale: bad args"; + capi_set_error(c, "depth_upscale: bad args"); return -1; } - if (!std::isfinite(gaussian_sigma) || gaussian_sigma < 0.0f || + if (polynomial_degree < 0 || polynomial_degree > 8 || + !std::isfinite(gaussian_sigma) || gaussian_sigma < 0.0f || gaussian_sigma > da::kMaxDepthUpscaleGaussianSigma) { - c->last_error = "depth_upscale: invalid Gaussian sigma"; + capi_set_error(c, "depth_upscale: invalid polynomial degree or Gaussian sigma"); return -1; } try { @@ -180,7 +190,10 @@ int da_capi_depth_upscale(da_ctx* c, const char* image_path, c->last_error.clear(); return 0; } catch (const std::bad_alloc&) { - c->last_error = "depth_upscale: allocation failed"; + capi_set_error(c, "depth_upscale: allocation failed"); + return -1; + } catch (...) { + capi_set_error(c, "depth_upscale: failed"); return -1; } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6615655..10ca7a0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,7 @@ da_add_test(test_ggml_extend) da_add_test(test_capi) da_add_test(test_capi_dense) da_add_test(test_capi_da2) +da_add_test(test_capi_upscale) da_add_test(test_preprocess) da_add_test(test_preprocess_real) da_add_test(test_rope2d) diff --git a/tests/test_capi_da2.cpp b/tests/test_capi_da2.cpp index 46b849b..2d3d7fc 100644 --- a/tests/test_capi_da2.cpp +++ b/tests/test_capi_da2.cpp @@ -33,14 +33,16 @@ int main(){ uint16_t upscaled[16] = {}; const int invalid_sigma = da_capi_depth_upscale( c, png, projected_range, 4, 4, 1, std::numeric_limits::max(), upscaled); + const int invalid_degree = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 9, 0.0f, upscaled); const int invalid_image = da_capi_depth_upscale( c, "missing-image.jpg", projected_range, 4, 4, 1, 0.0f, upscaled); const int upscale = da_capi_depth_upscale( c, png, projected_range, 4, 4, 1, 0.0f, upscaled); - bool upscale_ok = invalid_sigma == -1 && invalid_image == -1 && upscale == 0; + bool upscale_ok = invalid_sigma == -1 && invalid_degree == -1 && invalid_image == -1 && upscale == 0; for (uint16_t value : upscaled) upscale_ok = upscale_ok && value > 0; - std::fprintf(stderr, "da2 upscale: invalid_sigma=%d invalid_image=%d result=%d -> %s\n", - invalid_sigma, invalid_image, upscale, upscale_ok ? "OK" : "FAIL"); + std::fprintf(stderr, "da2 upscale: invalid_sigma=%d invalid_degree=%d invalid_image=%d result=%d -> %s\n", + invalid_sigma, invalid_degree, invalid_image, upscale, upscale_ok ? "OK" : "FAIL"); int H=0, W=0, is_metric=-1; float *depth=nullptr, *conf=nullptr, *sky=nullptr; diff --git a/tests/test_capi_upscale.cpp b/tests/test_capi_upscale.cpp new file mode 100644 index 0000000..6dc70a1 --- /dev/null +++ b/tests/test_capi_upscale.cpp @@ -0,0 +1,63 @@ +#include "da_capi.h" + +#include +#include +#include + +struct Fixture { + const char* label; + const char* gguf_var; + const char* image_var; +}; + +static bool exists(const char* path) { + if (!path) return false; + FILE* file = std::fopen(path, "rb"); + if (!file) return false; + std::fclose(file); + return true; +} + +static bool run_upscale(const char* label, const char* gguf_var, const char* image_var) { + const char* gguf = std::getenv(gguf_var); + const char* image = std::getenv(image_var); + if (!exists(gguf) || !exists(image)) return false; + + da_ctx* ctx = da_capi_load(gguf, 1); + if (!ctx) { + std::fprintf(stderr, "%s: failed to load model\n", label); + return false; + } + const uint16_t projected_range[] = { + 1000, 1100, 1200, 1300, + 1150, 1250, 1350, 1450, + 1300, 1400, 1500, 1600, + 1450, 1550, 1650, 1750, + }; + uint16_t output[16] = {}; + const int result = da_capi_depth_upscale(ctx, image, projected_range, 4, 4, 1, 0.0f, output); + bool ok = result == 0; + for (uint16_t value : output) ok = ok && value > 0; + std::fprintf(stderr, "%s: upscale=%d -> %s\n", label, result, ok ? "OK" : "FAIL"); + da_capi_free(ctx); + return ok; +} + +int main() { + int available = 0; + bool ok = true; + const Fixture fixtures[] = { + {"DualDPT DA3", "DA_TEST_GGUF", "DA_TEST_NATIVE_PNG"}, + {"DA3 mono", "DA_TEST_GGUF_MONO", "DA_TEST_MONO_PNG"}, + }; + for (const Fixture& fixture : fixtures) { + if (!exists(std::getenv(fixture.gguf_var)) || !exists(std::getenv(fixture.image_var))) continue; + ++available; + ok = run_upscale(fixture.label, fixture.gguf_var, fixture.image_var) && ok; + } + if (available == 0) { + std::fprintf(stderr, "DA3 upscale fixtures absent, skipping\n"); + return 77; + } + return ok ? 0 : 1; +} From 87575efadd39ff816711896b7cb2d60dada5785a Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Tue, 22 Sep 2026 12:38:23 +0200 Subject: [PATCH 6/7] Address pull request review findings --- CMakeLists.txt | 13 +++++++++---- README.md | 8 +++++--- examples/cli/main.cpp | 12 +++++++++++- include/da_capi.h | 3 ++- scripts/convert_da2_onnx_to_gguf.py | 13 +++++++++++-- src/cli.cpp | 2 +- src/da_capi.cpp | 15 +++++++++++++-- src/depth_upscale.cpp | 3 ++- tests/test_depth_upscale.cpp | 21 ++++++++++++++++++--- 9 files changed, 72 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f20365f..301e5fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ option(DA_SHARED "Build libdepthanything as a shared lib" OFF) option(DA_GGML_CUDA "ggml CUDA backend" OFF) option(DA_GGML_METAL "ggml Metal backend" OFF) option(DA_GGML_VULKAN "ggml Vulkan backend" OFF) +option(DA_ENABLE_TIFF "Enable libtiff-backed TIFF I/O" OFF) set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) @@ -40,8 +41,6 @@ set(GGML_VULKAN ${DA_GGML_VULKAN} CACHE BOOL "" FORCE) # and a GPU present at configure time, so it breaks cross/CI builds (e.g. the # l4t/Jetson image on CMake 3.22 yields an empty "compute_" arch). For a local # GPU-targeted build pass -DCMAKE_CUDA_ARCHITECTURES=native explicitly. -find_package(TIFF REQUIRED) - add_subdirectory(third_party/ggml EXCLUDE_FROM_ALL) set(DA_SOURCES @@ -49,7 +48,6 @@ set(DA_SOURCES src/backend.cpp src/image_io.cpp src/preprocess.cpp - src/tiff_io.cpp src/depth_upscale.cpp src/rope2d.cpp src/uv_posembed.cpp @@ -83,6 +81,10 @@ set(DA_SOURCES src/cli.cpp src/quantize.cpp ) +if(DA_ENABLE_TIFF) + find_package(TIFF REQUIRED) + list(APPEND DA_SOURCES src/tiff_io.cpp) +endif() if(DA_SHARED) add_library(depthanything SHARED ${DA_SOURCES}) else() @@ -112,7 +114,10 @@ else() endif() endif() target_link_libraries(depthanything PUBLIC ggml) -target_link_libraries(depthanything PUBLIC TIFF::TIFF) +if(DA_ENABLE_TIFF) + target_compile_definitions(depthanything PUBLIC DA_ENABLE_TIFF=1) + target_link_libraries(depthanything PRIVATE TIFF::TIFF) +endif() # OpenMP for the single-threaded CPU geometry post-processing (surface fusion, # per-seam ICP, normal estimation). ggml already pulls OpenMP into the toolchain; diff --git a/README.md b/README.md index 20464c8..25717e1 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ cmake --build build -j # -> build/examples/cli/da3-cli ``` -`depth-upscale` uses libtiff for TIFF decode/encode; install its development package before configuring (for example, `libtiff-dev` on Debian/Ubuntu). +The default build has no libtiff dependency. To build the TIFF-based `depth-upscale` workflow, install libtiff's development package (for example, `libtiff-dev` on Debian/Ubuntu) and configure with `-DDA_ENABLE_TIFF=ON`. ### CMake options @@ -132,6 +132,7 @@ cmake --build build -j |--------|---------|--------| | `DA_BUILD_CLI` | ON | build the `da3-cli` tool | | `DA_BUILD_TESTS` | OFF | build the ctest parity suite | +| `DA_ENABLE_TIFF` | OFF | enable libtiff-backed TIFF I/O and the `depth-upscale` CLI workflow | | `DA_SHARED` | OFF | build `libdepthanything.so` (static ggml, PIC) for embedding | | `DA_GGML_LLAMAFILE` | ON | tinyBLAS AVX-512/AVX2 matmul kernels (faster CPU) | | `DA_GGML_CUDA` | OFF | CUDA backend (`-DCMAKE_CUDA_ARCHITECTURES=native` auto) | @@ -213,7 +214,8 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - # Nested metric-scale depth (two GGUFs) $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm -# Sensor-calibrated DA2/DA3 depth TIFF. depth-upscale defaults to the converted +# Sensor-calibrated DA2/DA3 depth TIFF (build with -DDA_ENABLE_TIFF=ON). +# depth-upscale defaults to the converted # yuvraj108c Depth Anything V2 ViT-B model at models/depth-anything2-base-f32.gguf. # The sensor input stores each 16-bit depth sample in its first two 8-bit channels # (high byte, low byte); pass --model to override the default. @@ -280,7 +282,7 @@ da_capi_depth_upscale(ctx, "main.tiff", projected_range_mm, range_h, range_w, da_capi_free(ctx); ``` -Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`. +Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`; add `-DDA_ENABLE_TIFF=ON` when `image_path` can be a TIFF. --- diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index d65c7e5..b23f1bd 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -7,8 +7,10 @@ #include "preprocess.hpp" #include "glb_export.hpp" #include "colmap_export.hpp" +#ifdef DA_ENABLE_TIFF #include "depth_upscale.hpp" #include "tiff_io.hpp" +#endif #include #include #include @@ -154,6 +156,7 @@ static int cmd_depth_export(const da::cli::Parsed& p, da::Engine& eng){ if (!p.output_png.empty()) da::write_depth_png(p.output_png, depth, H, W, p.invert); return 0; } +#ifdef DA_ENABLE_TIFF static int cmd_depth_upscale(const da::cli::Parsed& p){ std::string error; da::Image image; @@ -189,6 +192,7 @@ static int cmd_depth_upscale(const da::cli::Parsed& p){ prediction_h, p.output_depth_tiff.c_str()); return 0; } +#endif static int cmd_depth(const da::cli::Parsed& p){ if (!p.metric_model.empty()) return cmd_depth_metric(p); @@ -281,7 +285,13 @@ int main(int argc, char** argv){ case S::Depth: return cmd_depth(p); case S::Reconstruct: return cmd_reconstruct(p); case S::Quantize: return cmd_quantize(p); - case S::DepthUpscale: return cmd_depth_upscale(p); + case S::DepthUpscale: +#ifdef DA_ENABLE_TIFF + return cmd_depth_upscale(p); +#else + std::fprintf(stderr, "error: depth-upscale requires a build with -DDA_ENABLE_TIFF=ON\n"); + return 1; +#endif case S::Help: da::cli::print_help(); return 0; default: da::cli::print_help(); return 1; } diff --git a/include/da_capi.h b/include/da_capi.h index 3766b95..325dfb2 100644 --- a/include/da_capi.h +++ b/include/da_capi.h @@ -49,7 +49,8 @@ const char* da_capi_last_error(da_ctx* ctx); /* owned by ctx sparse projected range image. projected_range_mm and out_range_mm are caller- owned [range_h*range_w] buffers. gaussian_sigma must be finite in [0, 64]. The output remains in calibrated millimetres; it is not normalized to the full - uint16 range. Returns 0 or -1 on error. */ + uint16 range. TIFF image input requires a build with DA_ENABLE_TIFF. Returns 0 + or -1 on error. */ int da_capi_depth_upscale(da_ctx* ctx, const char* image_path, const uint16_t* projected_range_mm, int range_h, int range_w, int polynomial_degree, float gaussian_sigma, diff --git a/scripts/convert_da2_onnx_to_gguf.py b/scripts/convert_da2_onnx_to_gguf.py index 17bbf9b..ba73085 100755 --- a/scripts/convert_da2_onnx_to_gguf.py +++ b/scripts/convert_da2_onnx_to_gguf.py @@ -74,7 +74,16 @@ def write_da2_onnx_gguf(onnx_path, output, checkpoint_name, max_depth): head_features = int(initializers["depth_head.scratch.layer1_rn.weight"].dims[0]) head_out_channels = [int(initializers[f"depth_head.projects.{i}.weight"].dims[0]) for i in range(4)] qkv_bias = "pretrained.blocks.0.attn.qkv.bias" in initializers - mlp_hidden = int(initializers["pretrained.blocks.0.mlp.fc1.weight"].dims[0]) + fc1 = initializers.get("pretrained.blocks.0.mlp.fc1.weight") + w12 = initializers.get("pretrained.blocks.0.mlp.w12.weight") + if fc1 is not None: + mlp_hidden = int(fc1.dims[0]) + ffn_type = "mlp" + elif w12 is not None: + mlp_hidden = int(w12.dims[0]) // 2 + ffn_type = "swiglu" + else: + raise ValueError("unsupported DA2 MLP tensors") pos_grid = math.isqrt(int(pos_embed.dims[1]) - 1) writer = gguf.GGUFWriter(output, K.ARCH) @@ -86,7 +95,7 @@ def write_da2_onnx_gguf(onnx_path, output, checkpoint_name, max_depth): writer.add_uint32(K.KV["vit.num_heads"], num_heads) writer.add_uint32(K.KV["vit.head_dim"], embed_dim // num_heads) writer.add_uint32(K.KV["vit.mlp_hidden"], mlp_hidden) - writer.add_string(K.KV["vit.ffn_type"], "mlp") + writer.add_string(K.KV["vit.ffn_type"], ffn_type) writer.add_uint32(K.KV["vit.num_register"], 0) writer.add_float32(K.KV["vit.init_values"], 1.0) writer.add_int32(K.KV["vit.alt_start"], -1) diff --git a/src/cli.cpp b/src/cli.cpp index 675031f..07aed5d 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -11,7 +11,7 @@ void print_help(){ " da3-cli depth --model --input [--glb ] [--colmap ] [--colmap-txt ] (single-image 3D export)\n" " da3-cli depth --model --metric-model --input [--pfm ] (nested metric-scale depth)\n" " da3-cli depth --model --input a.png --input b.png [--out-prefix out] [--no-invert] (multi-view)\n" - " da3-cli depth-upscale [--model ] --input --sensor-depth --tiff [--degree N] [--threads N]\n" + " da3-cli depth-upscale [--model ] --input --sensor-depth --tiff [--degree N] [--threads N] (requires -DDA_ENABLE_TIFF=ON)\n" " da3-cli reconstruct --model --input --ply [--pose ]\n" " da3-cli quantize (type: f16|q8_0|q6_k|q5_k|q4_k)\n"); } diff --git a/src/da_capi.cpp b/src/da_capi.cpp index 70120cc..6d64ef8 100644 --- a/src/da_capi.cpp +++ b/src/da_capi.cpp @@ -1,7 +1,9 @@ #include "da_capi.h" #include "engine.hpp" #include "depth_upscale.hpp" +#ifdef DA_ENABLE_TIFF #include "tiff_io.hpp" +#endif #include "preprocess.hpp" #include "image_io.hpp" #include "glb_export.hpp" @@ -164,9 +166,18 @@ int da_capi_depth_upscale(da_ctx* c, const char* image_path, const bool is_tiff = lower.size() >= 4 && (lower.compare(lower.size() - 4, 4, ".tif") == 0 || (lower.size() >= 5 && lower.compare(lower.size() - 5, 5, ".tiff") == 0)); - if (is_tiff ? !da::load_tiff_rgb(path, image, &error) : !da::load_image_rgb(path, image)){ + if (is_tiff) { +#ifdef DA_ENABLE_TIFF + if (!da::load_tiff_rgb(path, image, &error)) { + c->last_error = "depth_upscale: load image failed: " + error; + return -1; + } +#else + c->last_error = "depth_upscale: TIFF input requires a build with -DDA_ENABLE_TIFF=ON"; + return -1; +#endif + } else if (!da::load_image_rgb(path, image)) { c->last_error = "depth_upscale: load image failed"; - if (!error.empty()) c->last_error += ": " + error; return -1; } std::vector predicted; diff --git a/src/depth_upscale.cpp b/src/depth_upscale.cpp index c036a9c..5fa1178 100644 --- a/src/depth_upscale.cpp +++ b/src/depth_upscale.cpp @@ -214,7 +214,8 @@ static bool upscale_depth_map_impl(const std::vector& sensor_depth, in std::vector rhs(coefficient_count, 0.0); size_t valid_count = 0; for (size_t i = 0; i < predictor.size(); ++i) { - if (sensor_depth[i] == 0 || predictor[i] <= 0.0f || !std::isfinite(predictor[i])) continue; + const bool missing_legacy_predictor = !normalize_relative && predictor[i] <= 0.0f; + if (sensor_depth[i] == 0 || missing_legacy_predictor || !std::isfinite(predictor[i])) continue; const double x = predictor[i] / predictor_max; const double y = sensor_depth[i]; double powers[17] = {1.0}; diff --git a/tests/test_depth_upscale.cpp b/tests/test_depth_upscale.cpp index ab0890b..e9ae7c4 100644 --- a/tests/test_depth_upscale.cpp +++ b/tests/test_depth_upscale.cpp @@ -1,8 +1,9 @@ #include "depth_upscale.hpp" -#include "tiff_io.hpp" #include "da_capi.h" - +#ifdef DA_ENABLE_TIFF +#include "tiff_io.hpp" #include +#endif #include #include #include @@ -17,6 +18,7 @@ static void check(bool ok, const char* message) { if (!ok) ++failures; } +#ifdef DA_ENABLE_TIFF static std::string temporary_path(const char* suffix) { return std::string("/tmp/da-depth-upscale-") + std::to_string(getpid()) + suffix; } @@ -62,6 +64,8 @@ static bool write_rgb_fixture(const std::string& path) { TIFFClose(tiff); return written; } +#endif + int main() { check(da_capi_abi_version() == 12, "reports C API version 12"); @@ -95,6 +99,14 @@ int main() { check(monotonic, "preserves relative-depth ordering after calibration"); check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), "rejects degenerate predictor"); + + std::vector zero_sensor {1000, 2000, 3000}; + std::vector zero_relative {1.0f, 2.0f, 3.0f}; + options.degree = 2; + check(upscale_depth_map(zero_sensor, 1, 3, zero_relative, 1, 3, output, options, &error), + "fits normalized zero-valued predictor samples"); + check(output == zero_sensor, "calibrates the full normalized predictor range"); + // The uint8 compatibility overload must retain the supplied nonzero range: // normalizing it first would turn 10 into zero and leave only three samples // for this cubic fit. @@ -106,6 +118,7 @@ int main() { "preserves legacy uint8 predictor values during calibration"); options.degree = 1; +#ifdef DA_ENABLE_TIFF const std::string packed_path = temporary_path("-packed.tiff"); const std::string output_path = temporary_path("-output.tiff"); const std::string rgb_path = temporary_path("-rgb.tiff"); @@ -119,7 +132,7 @@ int main() { std::vector loaded; int loaded_h = 0, loaded_w = 0; check(load_packed_depth_tiff(packed_path, loaded, loaded_h, loaded_w, &error), "reads packed sensor TIFF"); check(loaded_h == h && loaded_w == w && loaded == sensor, "decodes big-endian two-channel sensor depth"); - check(write_depth_tiff_u16(output_path, output, h, w, &error), "writes uint16 depth TIFF"); + check(write_depth_tiff_u16(output_path, sensor, h, w, &error), "writes uint16 depth TIFF"); TIFF* tiff = TIFFOpen(output_path.c_str(), "r"); uint16_t bits = 0, samples = 0, format = 0; @@ -132,6 +145,8 @@ int main() { std::remove(packed_path.c_str()); std::remove(output_path.c_str()); std::remove(rgb_path.c_str()); +#endif + std::fprintf(stderr, "%s (%d failures)\n", failures ? "FAILED" : "PASSED", failures); return failures ? 1 : 0; } From 1053c705b7a6cd0a09788365c7adf6522f285d71 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 26 Oct 2026 11:27:52 +0100 Subject: [PATCH 7/7] Use DA3 base F32 as depth-upscale default --- README.md | 6 +++--- scripts/fetch_models.sh | 4 ++-- src/cli.cpp | 2 +- tests/test_cli.cpp | 5 ++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 25717e1..3584c31 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,8 @@ python scripts/convert_da2_to_gguf.py --encoder vitl --ckpt models/depth_anythin --output models/depth-anything2-large-f32.gguf --name Depth-Anything-V2-Large # Legacy depth-upscale model: yuvraj108c's static ViT-B ONNX export. -# This writes the depth-upscale default and needs only onnx, numpy, and gguf. +# Optional DA2 alternative; pass --model to depth-upscale to use it. +# Conversion needs only onnx, numpy, and gguf. python scripts/convert_da2_onnx_to_gguf.py \ --onnx ../depth-upscale/models/depth_anything_v2/1/model.onnx \ --output models/depth-anything2-base-f32.gguf @@ -215,8 +216,7 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm # Sensor-calibrated DA2/DA3 depth TIFF (build with -DDA_ENABLE_TIFF=ON). -# depth-upscale defaults to the converted -# yuvraj108c Depth Anything V2 ViT-B model at models/depth-anything2-base-f32.gguf. +# depth-upscale defaults to DA3-BASE (F32) at models/depth-anything-base-f32.gguf. # The sensor input stores each 16-bit depth sample in its first two 8-bit channels # (high byte, low byte); pass --model to override the default. $CLI depth-upscale --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff diff --git a/scripts/fetch_models.sh b/scripts/fetch_models.sh index e08ed9c..6fefb94 100755 --- a/scripts/fetch_models.sh +++ b/scripts/fetch_models.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Pre-download the curated DA3 model set plus the DA2 depth-upscale default (~13 GB). +# Pre-download the curated DA3 model set plus the F32 depth-upscale default (~13 GB). # One weight per capability, not the full 31 GB of redundant quants. # Idempotent: skips files already present at the right size; resumes partials. set -uo pipefail @@ -16,7 +16,7 @@ CURATED=( "relative+pose+gaussians depth-anything-giant-f32.gguf" "metric+pose (anyview branch) depth-anything-nested-anyview.gguf" "metric+pose (metric branch) depth-anything-nested-metric.gguf" - "relative depth (upscale default) depth-anything2-base-f32.gguf" + "relative depth (upscale default) depth-anything-base-f32.gguf" ) echo "Downloading curated DA3 models into $DIR" diff --git a/src/cli.cpp b/src/cli.cpp index 07aed5d..f14159a 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -67,7 +67,7 @@ Parsed parse(int argc, char** argv){ else if (a == "--threads" && i+1