Add DA2 and DA3 sensor depth upscale workflow

CMakeLists.txt

@@ -21,6 +21,7 @@ option(DA_SHARED       "Build libdepthanything as a shared lib"   OFF)
 option(DA_GGML_CUDA    "ggml CUDA backend"                        OFF)
 option(DA_GGML_METAL   "ggml Metal backend"                       OFF)
 option(DA_GGML_VULKAN  "ggml Vulkan backend"                      OFF)
+option(DA_ENABLE_TIFF   "Enable libtiff-backed TIFF I/O"            OFF)
 
 set(GGML_BUILD_TESTS    OFF CACHE BOOL "" FORCE)
 set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
@@ -47,6 +48,7 @@ set(DA_SOURCES
     src/backend.cpp
     src/image_io.cpp
     src/preprocess.cpp
+    src/depth_upscale.cpp
     src/rope2d.cpp
     src/uv_posembed.cpp
     src/attention.cpp
@@ -79,6 +81,10 @@ set(DA_SOURCES
     src/cli.cpp
     src/quantize.cpp
 )
+if(DA_ENABLE_TIFF)
+    find_package(TIFF REQUIRED)
+    list(APPEND DA_SOURCES src/tiff_io.cpp)
+endif()
 if(DA_SHARED)
     add_library(depthanything SHARED ${DA_SOURCES})
 else()
@@ -108,6 +114,10 @@ else()
     endif()
 endif()
 target_link_libraries(depthanything PUBLIC ggml)
+if(DA_ENABLE_TIFF)
+    target_compile_definitions(depthanything PUBLIC DA_ENABLE_TIFF=1)
+    target_link_libraries(depthanything PRIVATE TIFF::TIFF)
+endif()
 
 # OpenMP for the single-threaded CPU geometry post-processing (surface fusion,
 # per-seam ICP, normal estimation). ggml already pulls OpenMP into the toolchain;

README.md

@@ -25,6 +25,7 @@ Given an image it recovers a dense **metric depth** map, per-pixel **confidence*
 - **Quantization** to f16 / q8_0 / q6_k / q5_k / q4_k - q4_k is **99 MB** (0.25x the f32) and near-lossless.
 - **CPU-first, GPU-ready.** Tuned CPU path (tinyBLAS, Winograd, flash-attention) plus CUDA / Metal / Vulkan ggml backends.
 - **Flat C API** (`include/da_capi.h`) - embed from C, C++, Go, or Rust. Powers the [LocalAI](#use-it-from-localai) backend.
+- **SpexAI sensor-calibrated depth.** Run DA2 or single-file DA3 on a main-camera image, fit it to projected sensor ranges, and emit calibrated uint16 millimetres.
 - **Parity-first.** Every component is gated against PyTorch-dumped reference tensors; the end-to-end depth matches the real `net()` at correlation 1.0.
 
 ---
@@ -80,6 +81,29 @@ On **GPU** (NVIDIA GB10, via `-DDA_GGML_CUDA=ON`) the ggml CUDA path with flash
 
 ![inference speed](benchmarks/media/infer_speed.png) ![peak memory](benchmarks/media/memory.png)
 
+### SpexAI depth-upscale capacity
+
+`depth-upscale` is the SpexAI-specific capture workflow: it combines a DA2 or single-file
+DA3 prediction from an RGB TIFF with a paired sensor TIFF. The sensor image stores each uint16
+depth sample in its first two interleaved uint8 channels (high byte, then low byte); the command
+resizes and smooths the model map, polynomial-fits it to valid sensor pixels, and writes an
+uncompressed single-channel uint16 TIFF in calibrated sensor units. The library API also accepts
+a sparse range image already projected into the main camera by PointClouds, preserving its
+millimetre scale rather than stretching each result to 0..65535.
+
+Measured as one cold task on the supplied SpexAI captures (3072×2048 RGB → 1280×720 depth)
+with `depth-anything2-large-q4_k.gguf` (289.1 MiB), four CPU threads:
+
+| Device        | Wall time | Peak host RSS | Peak GPU memory |   Capacity guidance   |
+|---------------|----------:|--------------:|----------------:|-----------------------|
+| CPU           | 31.14 s   | 833.1 MiB     |              —  | 1 GiB RAM per worker  |
+| NVIDIA M2000M | **9.73 s**| **661.9 MiB** | 2,473 MiB task allocation; 2,806 MiB global peak (333 MiB baseline) | 1 GiB RAM + 3 GiB VRAM per worker; one worker on a 4 GiB card |
+| old SciPy CPU | 4.79 s    | 883.6 MiB     |                 |                       |
+
+The CUDA run (`-DDA_GGML_CUDA=ON`, `DA_DEVICE=CUDA0`) offloaded 403 weights and used
+flash attention. GPU memory was sampled every 100 ms; its ≈2.8 GiB global plateau persisted
+for several seconds. The CUDA output was validated as a 1280×720, single-channel uint16 TIFF.
+
 ### See it run
 
 Real photos through the actual CLI, input next to the colorized depth (turbo):
@@ -100,12 +124,15 @@ cmake --build build -j
 # -> build/examples/cli/da3-cli
 ```
 
+The default build has no libtiff dependency. To build the TIFF-based `depth-upscale` workflow, install libtiff's development package (for example, `libtiff-dev` on Debian/Ubuntu) and configure with `-DDA_ENABLE_TIFF=ON`.
+
 ### CMake options
 
 | Option | Default | Effect |
 |--------|---------|--------|
 | `DA_BUILD_CLI` | ON | build the `da3-cli` tool |
 | `DA_BUILD_TESTS` | OFF | build the ctest parity suite |
+| `DA_ENABLE_TIFF` | OFF | enable libtiff-backed TIFF I/O and the `depth-upscale` CLI workflow |
 | `DA_SHARED` | OFF | build `libdepthanything.so` (static ggml, PIC) for embedding |
 | `DA_GGML_LLAMAFILE` | ON | tinyBLAS AVX-512/AVX2 matmul kernels (faster CPU) |
 | `DA_GGML_CUDA` | OFF | CUDA backend (`-DCMAKE_CUDA_ARCHITECTURES=native` auto) |
@@ -142,6 +169,13 @@ python scripts/convert_nested_to_gguf.py --model models/DA3NESTED-GIANT-LARGE --
 python scripts/convert_da2_to_gguf.py --encoder vitl --ckpt models/depth_anything_v2_vitl.pth \
     --output models/depth-anything2-large-f32.gguf --name Depth-Anything-V2-Large
 
+# Legacy depth-upscale model: yuvraj108c's static ViT-B ONNX export.
+# Optional DA2 alternative; pass --model to depth-upscale to use it.
+# Conversion needs only onnx, numpy, and gguf.
+python scripts/convert_da2_onnx_to_gguf.py \
+    --onnx ../depth-upscale/models/depth_anything_v2/1/model.onnx \
+    --output models/depth-anything2-base-f32.gguf
+
 # Depth Anything V2 — metric (add --max-depth: 20 for Hypersim/indoor, 80 for VKITTI/outdoor)
 python scripts/convert_da2_to_gguf.py --encoder vits --ckpt models/depth_anything_v2_metric_hypersim_vits.pth \
     --output models/depth-anything2-metric-hypersim-small-f32.gguf \
@@ -181,6 +215,12 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg -
 # Nested metric-scale depth (two GGUFs)
 $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm
 
+# Sensor-calibrated DA2/DA3 depth TIFF (build with -DDA_ENABLE_TIFF=ON).
+# depth-upscale defaults to DA3-BASE (F32) at models/depth-anything-base-f32.gguf.
+# The sensor input stores each 16-bit depth sample in its first two 8-bit channels
+# (high byte, low byte); pass --model to override the default.
+$CLI depth-upscale --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff
+
 # Multi-view depth + pose
 $CLI depth --model $M --input a.jpg --input b.jpg --out-prefix scene
 
@@ -220,7 +260,7 @@ Gallery entries cover base (q4_k/q8_0/f16/f32), small, large, giant, and mono-la
 
 ## C API
 
-A flat C ABI (`include/da_capi.h`, `abi_version` 4) over `libdepthanything.so`:
+A flat C ABI (`include/da_capi.h`, `abi_version` 12) over `libdepthanything.so`:
 
 ```c
 da_ctx* ctx = da_capi_load("model.gguf", /*threads*/ 8);
@@ -234,10 +274,15 @@ da_capi_free_floats(depth);
 int n; float *xyz; unsigned char *rgb;
 da_capi_points(ctx, "photo.jpg", /*conf_thresh*/ 1.0f, &n, &xyz, &rgb);   // 3D cloud
 da_capi_export_glb(ctx, "photo.jpg", "scene.glb");
+
+// PCL projects sensor points into the main-camera grid first. The result and
+// output are caller-owned uint16 millimetre buffers of size range_h*range_w.
+da_capi_depth_upscale(ctx, "main.tiff", projected_range_mm, range_h, range_w,
+                      /*degree*/ 2, /*gaussian_sigma*/ 1.0f, dense_range_mm);
 da_capi_free(ctx);
 ```
 
-Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`.
+Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`; add `-DDA_ENABLE_TIFF=ON` when `image_path` can be a TIFF.
 
 ---
 

examples/cli/main.cpp

@@ -7,6 +7,10 @@
 #include "preprocess.hpp"
 #include "glb_export.hpp"
 #include "colmap_export.hpp"
+#ifdef DA_ENABLE_TIFF
+#include "depth_upscale.hpp"
+#include "tiff_io.hpp"
+#endif
 #include <cstdio>
 #include <algorithm>
 #include <array>
@@ -152,6 +156,44 @@ static int cmd_depth_export(const da::cli::Parsed& p, da::Engine& eng){
     if (!p.output_png.empty()) da::write_depth_png(p.output_png, depth, H, W, p.invert);
     return 0;
 }
+#ifdef DA_ENABLE_TIFF
+static int cmd_depth_upscale(const da::cli::Parsed& p){
+    std::string error;
+    da::Image image;
+    if (!da::load_tiff_rgb(p.input, image, &error)){
+        std::fprintf(stderr, "error: load RGB TIFF failed: %s\n", error.c_str());
+        return 1;
+    }
+    std::vector<uint16_t> sensor_depth; int sensor_h = 0, sensor_w = 0;
+    if (!da::load_packed_depth_tiff(p.sensor_depth_tiff, sensor_depth, sensor_h, sensor_w, &error)){
+        std::fprintf(stderr, "error: load packed sensor depth TIFF failed: %s\n", error.c_str());
+        return 1;
+    }
+    auto eng = da::Engine::load(p.model, p.n_threads);
+    if (!eng){ std::fprintf(stderr, "error: load failed\n"); return 1; }
+    std::vector<float> predicted; int prediction_h = 0, prediction_w = 0;
+    if (!da::predict_depth_for_upscale(*eng, image, predicted, prediction_h, prediction_w, &error)){
+        std::fprintf(stderr, "error: model inference failed: %s\n", error.c_str());
+        return 1;
+    }
+    da::DepthUpscaleOptions options;
+    options.degree = p.upscale_degree;
+    std::vector<uint16_t> output;
+    if (!da::upscale_depth_map(sensor_depth, sensor_h, sensor_w, predicted, prediction_h, prediction_w,
+                               output, options, &error)){
+        std::fprintf(stderr, "error: depth calibration failed: %s\n", error.c_str());
+        return 1;
+    }
+    if (!da::write_depth_tiff_u16(p.output_depth_tiff, output, sensor_h, sensor_w, &error)){
+        std::fprintf(stderr, "error: write depth TIFF failed: %s\n", error.c_str());
+        return 1;
+    }
+    std::printf("upscaled depth %dx%d from model %dx%d -> %s\n", sensor_w, sensor_h, prediction_w,
+                prediction_h, p.output_depth_tiff.c_str());
+    return 0;
+}
+#endif
+
 static int cmd_depth(const da::cli::Parsed& p){
     if (!p.metric_model.empty()) return cmd_depth_metric(p);
     if (p.repeat > 0 && p.inputs.size() <= 1) return cmd_depth_bench(p);
@@ -243,6 +285,13 @@ int main(int argc, char** argv){
         case S::Depth: return cmd_depth(p);
         case S::Reconstruct: return cmd_reconstruct(p);
         case S::Quantize: return cmd_quantize(p);
+        case S::DepthUpscale:
+#ifdef DA_ENABLE_TIFF
+            return cmd_depth_upscale(p);
+#else
+            std::fprintf(stderr, "error: depth-upscale requires a build with -DDA_ENABLE_TIFF=ON\n");
+            return 1;
+#endif
         case S::Help: da::cli::print_help(); return 0;
         default: da::cli::print_help(); return 1;
     }

include/da_capi.h

@@ -1,6 +1,7 @@
 #ifndef DA_CAPI_H
 #define DA_CAPI_H
 #include <stddef.h>
+#include <stdint.h>
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -19,7 +20,8 @@ typedef struct da_ctx da_ctx;
    10: added da_capi_set_fuse_params — scene-relative TSDF voxel/truncation knobs for
       the next streamed fuse (exposed as viewer sliders).
    11: added temporal exposed-face voxel mesh generation and retrieval for streamed
-      fused scenes. */
+      fused scenes.
+   12: added sensor-calibrated DA2/DA3 range upscaling. */
 int         da_capi_abi_version(void);
 /* Set scene-relative TSDF surface-fusion knobs consumed by the NEXT
    da_capi_points_stream whose fuse flag is set. voxel_frac: voxel edge as a fraction
@@ -43,6 +45,16 @@ void        da_capi_free(da_ctx* ctx);                           /* safe on NULL
 char*       da_capi_info_json(da_ctx* ctx);
 void        da_capi_free_string(char* s);
 const char* da_capi_last_error(da_ctx* ctx);                     /* owned by ctx, "" if none */
+/* Infer DA2 or single-file DA3 depth for image_path and calibrate it against a
+   sparse projected range image. projected_range_mm and out_range_mm are caller-
+   owned [range_h*range_w] buffers. gaussian_sigma must be finite in [0, 64].
+   The output remains in calibrated millimetres; it is not normalized to the full
+   uint16 range. TIFF image input requires a build with DA_ENABLE_TIFF. Returns 0
+   or -1 on error. */
+int da_capi_depth_upscale(da_ctx* ctx, const char* image_path,
+                          const uint16_t* projected_range_mm, int range_h, int range_w,
+                          int polynomial_degree, float gaussian_sigma,
+                          uint16_t* out_range_mm);
 /* Run depth on an image file. On success writes *out_h,*out_w and returns a malloc'd
    float[H*W] depth map (row-major); caller frees via da_capi_free_floats. NULL on error. */
 float* da_capi_depth_path(da_ctx* ctx, const char* image_path, int* out_h, int* out_w);

make.sh

@@ -0,0 +1,2 @@
+cmake -B build -DDA_BUILD_CLI=ON -DCMAKE_CUDA_ARCHITECTURES=native -DDA_GGML_CUDA=ON
+cmake --build build -j

requirements.txt

@@ -8,6 +8,7 @@
 torch>=2.4            # CPU build is fine: torch==2.x+cpu
 numpy>=1.26
 gguf>=0.19.0          # GGUF writer/reader
+onnx>=1.16           # scripts/convert_da2_onnx_to_gguf.py
 huggingface_hub>=0.24 # scripts/download_model.py
 safetensors>=0.4
 einops>=0.8