diff --git a/CMakeLists.txt b/CMakeLists.txt index 874b64c..301e5fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ option(DA_SHARED "Build libdepthanything as a shared lib" OFF) option(DA_GGML_CUDA "ggml CUDA backend" OFF) option(DA_GGML_METAL "ggml Metal backend" OFF) option(DA_GGML_VULKAN "ggml Vulkan backend" OFF) +option(DA_ENABLE_TIFF "Enable libtiff-backed TIFF I/O" OFF) set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) @@ -47,6 +48,7 @@ set(DA_SOURCES src/backend.cpp src/image_io.cpp src/preprocess.cpp + src/depth_upscale.cpp src/rope2d.cpp src/uv_posembed.cpp src/attention.cpp @@ -79,6 +81,10 @@ set(DA_SOURCES src/cli.cpp src/quantize.cpp ) +if(DA_ENABLE_TIFF) + find_package(TIFF REQUIRED) + list(APPEND DA_SOURCES src/tiff_io.cpp) +endif() if(DA_SHARED) add_library(depthanything SHARED ${DA_SOURCES}) else() @@ -108,6 +114,10 @@ else() endif() endif() target_link_libraries(depthanything PUBLIC ggml) +if(DA_ENABLE_TIFF) + target_compile_definitions(depthanything PUBLIC DA_ENABLE_TIFF=1) + target_link_libraries(depthanything PRIVATE TIFF::TIFF) +endif() # OpenMP for the single-threaded CPU geometry post-processing (surface fusion, # per-seam ICP, normal estimation). ggml already pulls OpenMP into the toolchain; diff --git a/README.md b/README.md index 520d23e..3584c31 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Given an image it recovers a dense **metric depth** map, per-pixel **confidence* - **Quantization** to f16 / q8_0 / q6_k / q5_k / q4_k - q4_k is **99 MB** (0.25x the f32) and near-lossless. - **CPU-first, GPU-ready.** Tuned CPU path (tinyBLAS, Winograd, flash-attention) plus CUDA / Metal / Vulkan ggml backends. - **Flat C API** (`include/da_capi.h`) - embed from C, C++, Go, or Rust. Powers the [LocalAI](#use-it-from-localai) backend. +- **SpexAI sensor-calibrated depth.** Run DA2 or single-file DA3 on a main-camera image, fit it to projected sensor ranges, and emit calibrated uint16 millimetres. - **Parity-first.** Every component is gated against PyTorch-dumped reference tensors; the end-to-end depth matches the real `net()` at correlation 1.0. --- @@ -80,6 +81,29 @@ On **GPU** (NVIDIA GB10, via `-DDA_GGML_CUDA=ON`) the ggml CUDA path with flash ![inference speed](benchmarks/media/infer_speed.png) ![peak memory](benchmarks/media/memory.png) +### SpexAI depth-upscale capacity + +`depth-upscale` is the SpexAI-specific capture workflow: it combines a DA2 or single-file +DA3 prediction from an RGB TIFF with a paired sensor TIFF. The sensor image stores each uint16 +depth sample in its first two interleaved uint8 channels (high byte, then low byte); the command +resizes and smooths the model map, polynomial-fits it to valid sensor pixels, and writes an +uncompressed single-channel uint16 TIFF in calibrated sensor units. The library API also accepts +a sparse range image already projected into the main camera by PointClouds, preserving its +millimetre scale rather than stretching each result to 0..65535. + +Measured as one cold task on the supplied SpexAI captures (3072×2048 RGB → 1280×720 depth) +with `depth-anything2-large-q4_k.gguf` (289.1 MiB), four CPU threads: + +| Device | Wall time | Peak host RSS | Peak GPU memory | Capacity guidance | +|---------------|----------:|--------------:|----------------:|-----------------------| +| CPU | 31.14 s | 833.1 MiB | — | 1 GiB RAM per worker | +| NVIDIA M2000M | **9.73 s**| **661.9 MiB** | 2,473 MiB task allocation; 2,806 MiB global peak (333 MiB baseline) | 1 GiB RAM + 3 GiB VRAM per worker; one worker on a 4 GiB card | +| old SciPy CPU | 4.79 s | 883.6 MiB | | | + +The CUDA run (`-DDA_GGML_CUDA=ON`, `DA_DEVICE=CUDA0`) offloaded 403 weights and used +flash attention. GPU memory was sampled every 100 ms; its ≈2.8 GiB global plateau persisted +for several seconds. The CUDA output was validated as a 1280×720, single-channel uint16 TIFF. + ### See it run Real photos through the actual CLI, input next to the colorized depth (turbo): @@ -100,12 +124,15 @@ cmake --build build -j # -> build/examples/cli/da3-cli ``` +The default build has no libtiff dependency. To build the TIFF-based `depth-upscale` workflow, install libtiff's development package (for example, `libtiff-dev` on Debian/Ubuntu) and configure with `-DDA_ENABLE_TIFF=ON`. + ### CMake options | Option | Default | Effect | |--------|---------|--------| | `DA_BUILD_CLI` | ON | build the `da3-cli` tool | | `DA_BUILD_TESTS` | OFF | build the ctest parity suite | +| `DA_ENABLE_TIFF` | OFF | enable libtiff-backed TIFF I/O and the `depth-upscale` CLI workflow | | `DA_SHARED` | OFF | build `libdepthanything.so` (static ggml, PIC) for embedding | | `DA_GGML_LLAMAFILE` | ON | tinyBLAS AVX-512/AVX2 matmul kernels (faster CPU) | | `DA_GGML_CUDA` | OFF | CUDA backend (`-DCMAKE_CUDA_ARCHITECTURES=native` auto) | @@ -142,6 +169,13 @@ python scripts/convert_nested_to_gguf.py --model models/DA3NESTED-GIANT-LARGE -- python scripts/convert_da2_to_gguf.py --encoder vitl --ckpt models/depth_anything_v2_vitl.pth \ --output models/depth-anything2-large-f32.gguf --name Depth-Anything-V2-Large +# Legacy depth-upscale model: yuvraj108c's static ViT-B ONNX export. +# Optional DA2 alternative; pass --model to depth-upscale to use it. +# Conversion needs only onnx, numpy, and gguf. +python scripts/convert_da2_onnx_to_gguf.py \ + --onnx ../depth-upscale/models/depth_anything_v2/1/model.onnx \ + --output models/depth-anything2-base-f32.gguf + # Depth Anything V2 — metric (add --max-depth: 20 for Hypersim/indoor, 80 for VKITTI/outdoor) python scripts/convert_da2_to_gguf.py --encoder vits --ckpt models/depth_anything_v2_metric_hypersim_vits.pth \ --output models/depth-anything2-metric-hypersim-small-f32.gguf \ @@ -181,6 +215,12 @@ $CLI depth --model models/depth-anything-mono-large-f32.gguf --input photo.jpg - # Nested metric-scale depth (two GGUFs) $CLI depth --model nested-anyview.gguf --metric-model nested-metric.gguf --input photo.jpg --pfm metric.pfm +# Sensor-calibrated DA2/DA3 depth TIFF (build with -DDA_ENABLE_TIFF=ON). +# depth-upscale defaults to DA3-BASE (F32) at models/depth-anything-base-f32.gguf. +# The sensor input stores each 16-bit depth sample in its first two 8-bit channels +# (high byte, low byte); pass --model to override the default. +$CLI depth-upscale --input rgb_capture.tiff --sensor-depth sensor_depth.tiff --tiff calibrated_depth.tiff + # Multi-view depth + pose $CLI depth --model $M --input a.jpg --input b.jpg --out-prefix scene @@ -220,7 +260,7 @@ Gallery entries cover base (q4_k/q8_0/f16/f32), small, large, giant, and mono-la ## C API -A flat C ABI (`include/da_capi.h`, `abi_version` 4) over `libdepthanything.so`: +A flat C ABI (`include/da_capi.h`, `abi_version` 12) over `libdepthanything.so`: ```c da_ctx* ctx = da_capi_load("model.gguf", /*threads*/ 8); @@ -234,10 +274,15 @@ da_capi_free_floats(depth); int n; float *xyz; unsigned char *rgb; da_capi_points(ctx, "photo.jpg", /*conf_thresh*/ 1.0f, &n, &xyz, &rgb); // 3D cloud da_capi_export_glb(ctx, "photo.jpg", "scene.glb"); + +// PCL projects sensor points into the main-camera grid first. The result and +// output are caller-owned uint16 millimetre buffers of size range_h*range_w. +da_capi_depth_upscale(ctx, "main.tiff", projected_range_mm, range_h, range_w, + /*degree*/ 2, /*gaussian_sigma*/ 1.0f, dense_range_mm); da_capi_free(ctx); ``` -Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`. +Opaque handles, C types only, `da_capi_last_error` for diagnostics. Build it with `-DDA_SHARED=ON`; add `-DDA_ENABLE_TIFF=ON` when `image_path` can be a TIFF. --- diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 2ee5b0e..b23f1bd 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -7,6 +7,10 @@ #include "preprocess.hpp" #include "glb_export.hpp" #include "colmap_export.hpp" +#ifdef DA_ENABLE_TIFF +#include "depth_upscale.hpp" +#include "tiff_io.hpp" +#endif #include #include #include @@ -152,6 +156,44 @@ static int cmd_depth_export(const da::cli::Parsed& p, da::Engine& eng){ if (!p.output_png.empty()) da::write_depth_png(p.output_png, depth, H, W, p.invert); return 0; } +#ifdef DA_ENABLE_TIFF +static int cmd_depth_upscale(const da::cli::Parsed& p){ + std::string error; + da::Image image; + if (!da::load_tiff_rgb(p.input, image, &error)){ + std::fprintf(stderr, "error: load RGB TIFF failed: %s\n", error.c_str()); + return 1; + } + std::vector sensor_depth; int sensor_h = 0, sensor_w = 0; + if (!da::load_packed_depth_tiff(p.sensor_depth_tiff, sensor_depth, sensor_h, sensor_w, &error)){ + std::fprintf(stderr, "error: load packed sensor depth TIFF failed: %s\n", error.c_str()); + return 1; + } + auto eng = da::Engine::load(p.model, p.n_threads); + if (!eng){ std::fprintf(stderr, "error: load failed\n"); return 1; } + std::vector predicted; int prediction_h = 0, prediction_w = 0; + if (!da::predict_depth_for_upscale(*eng, image, predicted, prediction_h, prediction_w, &error)){ + std::fprintf(stderr, "error: model inference failed: %s\n", error.c_str()); + return 1; + } + da::DepthUpscaleOptions options; + options.degree = p.upscale_degree; + std::vector output; + if (!da::upscale_depth_map(sensor_depth, sensor_h, sensor_w, predicted, prediction_h, prediction_w, + output, options, &error)){ + std::fprintf(stderr, "error: depth calibration failed: %s\n", error.c_str()); + return 1; + } + if (!da::write_depth_tiff_u16(p.output_depth_tiff, output, sensor_h, sensor_w, &error)){ + std::fprintf(stderr, "error: write depth TIFF failed: %s\n", error.c_str()); + return 1; + } + std::printf("upscaled depth %dx%d from model %dx%d -> %s\n", sensor_w, sensor_h, prediction_w, + prediction_h, p.output_depth_tiff.c_str()); + return 0; +} +#endif + static int cmd_depth(const da::cli::Parsed& p){ if (!p.metric_model.empty()) return cmd_depth_metric(p); if (p.repeat > 0 && p.inputs.size() <= 1) return cmd_depth_bench(p); @@ -243,6 +285,13 @@ int main(int argc, char** argv){ case S::Depth: return cmd_depth(p); case S::Reconstruct: return cmd_reconstruct(p); case S::Quantize: return cmd_quantize(p); + case S::DepthUpscale: +#ifdef DA_ENABLE_TIFF + return cmd_depth_upscale(p); +#else + std::fprintf(stderr, "error: depth-upscale requires a build with -DDA_ENABLE_TIFF=ON\n"); + return 1; +#endif case S::Help: da::cli::print_help(); return 0; default: da::cli::print_help(); return 1; } diff --git a/include/da_capi.h b/include/da_capi.h index f3ebb53..325dfb2 100644 --- a/include/da_capi.h +++ b/include/da_capi.h @@ -1,6 +1,7 @@ #ifndef DA_CAPI_H #define DA_CAPI_H #include +#include #ifdef __cplusplus extern "C" { #endif @@ -19,7 +20,8 @@ typedef struct da_ctx da_ctx; 10: added da_capi_set_fuse_params — scene-relative TSDF voxel/truncation knobs for the next streamed fuse (exposed as viewer sliders). 11: added temporal exposed-face voxel mesh generation and retrieval for streamed - fused scenes. */ + fused scenes. + 12: added sensor-calibrated DA2/DA3 range upscaling. */ int da_capi_abi_version(void); /* Set scene-relative TSDF surface-fusion knobs consumed by the NEXT da_capi_points_stream whose fuse flag is set. voxel_frac: voxel edge as a fraction @@ -43,6 +45,16 @@ void da_capi_free(da_ctx* ctx); /* safe on NULL char* da_capi_info_json(da_ctx* ctx); void da_capi_free_string(char* s); const char* da_capi_last_error(da_ctx* ctx); /* owned by ctx, "" if none */ +/* Infer DA2 or single-file DA3 depth for image_path and calibrate it against a + sparse projected range image. projected_range_mm and out_range_mm are caller- + owned [range_h*range_w] buffers. gaussian_sigma must be finite in [0, 64]. + The output remains in calibrated millimetres; it is not normalized to the full + uint16 range. TIFF image input requires a build with DA_ENABLE_TIFF. Returns 0 + or -1 on error. */ +int da_capi_depth_upscale(da_ctx* ctx, const char* image_path, + const uint16_t* projected_range_mm, int range_h, int range_w, + int polynomial_degree, float gaussian_sigma, + uint16_t* out_range_mm); /* Run depth on an image file. On success writes *out_h,*out_w and returns a malloc'd float[H*W] depth map (row-major); caller frees via da_capi_free_floats. NULL on error. */ float* da_capi_depth_path(da_ctx* ctx, const char* image_path, int* out_h, int* out_w); diff --git a/make.sh b/make.sh new file mode 100755 index 0000000..3d77084 --- /dev/null +++ b/make.sh @@ -0,0 +1,2 @@ +cmake -B build -DDA_BUILD_CLI=ON -DCMAKE_CUDA_ARCHITECTURES=native -DDA_GGML_CUDA=ON +cmake --build build -j diff --git a/requirements.txt b/requirements.txt index 5880269..c994131 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ torch>=2.4 # CPU build is fine: torch==2.x+cpu numpy>=1.26 gguf>=0.19.0 # GGUF writer/reader +onnx>=1.16 # scripts/convert_da2_onnx_to_gguf.py huggingface_hub>=0.24 # scripts/download_model.py safetensors>=0.4 einops>=0.8 diff --git a/scripts/convert_da2_onnx_to_gguf.py b/scripts/convert_da2_onnx_to_gguf.py new file mode 100755 index 0000000..ba73085 --- /dev/null +++ b/scripts/convert_da2_onnx_to_gguf.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Convert the yuvraj108c Depth Anything V2 ONNX exports to GGUF. + +The legacy depth-upscale service uses ONNX Runtime directly. depth-anything.cpp +uses the same parameter tensors through its DA2 GGUF route, so this converter +copies ONNX initializers into the existing tensor/metadata schema without +requiring PyTorch or the upstream Depth Anything source tree. +""" +import argparse +import math +import os +import sys + +import numpy as np +import onnx +from onnx import numpy_helper + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import gguf +import scripts.gguf_keys as K + + +ENCODERS = { + "vits": (384, 12, 6, [2, 5, 8, 11]), + "vitb": (768, 12, 12, [2, 5, 8, 11]), + "vitl": (1024, 24, 16, [4, 11, 17, 23]), + "vitg": (1536, 40, 24, [9, 19, 29, 39]), +} + + +def infer_encoder(initializers): + patch = initializers.get("pretrained.patch_embed.proj.weight") + if patch is None or len(patch.dims) != 4 or patch.dims[1:] != [3, 14, 14]: + raise ValueError("expected a Depth Anything V2 [embed,3,14,14] patch embedding") + embed_dim = int(patch.dims[0]) + block_count = 0 + while f"pretrained.blocks.{block_count}.attn.qkv.weight" in initializers: + block_count += 1 + for name, (expected_embed, expected_blocks, _, _) in ENCODERS.items(): + if (embed_dim, block_count) == (expected_embed, expected_blocks): + return name + raise ValueError(f"unsupported DA2 backbone: embed_dim={embed_dim}, blocks={block_count}") + + +def inferred_max_depth(path): + name = os.path.basename(path).lower() + if "metric_hypersim" in name: + return 20.0 + if "metric_vkitti" in name: + return 80.0 + return 0.0 + + +def validate_io(model): + if len(model.graph.input) != 1 or len(model.graph.output) != 1: + raise ValueError("expected one ONNX input and one output") + def dims(value): + return [int(d.dim_value) for d in value.type.tensor_type.shape.dim] + if dims(model.graph.input[0]) != [1, 3, 518, 518]: + raise ValueError(f"expected input [1,3,518,518], got {dims(model.graph.input[0])}") + if dims(model.graph.output[0]) != [1, 518, 518]: + raise ValueError(f"expected output [1,518,518], got {dims(model.graph.output[0])}") + + +def write_da2_onnx_gguf(onnx_path, output, checkpoint_name, max_depth): + model = onnx.load(onnx_path, load_external_data=False) + validate_io(model) + initializers = {tensor.name: tensor for tensor in model.graph.initializer} + encoder = infer_encoder(initializers) + embed_dim, depth, num_heads, out_layers = ENCODERS[encoder] + pos_embed = initializers.get("pretrained.pos_embed") + if pos_embed is None or list(pos_embed.dims) != [1, 1369 + 1, embed_dim]: + raise ValueError("invalid DA2 positional embedding") + head_features = int(initializers["depth_head.scratch.layer1_rn.weight"].dims[0]) + head_out_channels = [int(initializers[f"depth_head.projects.{i}.weight"].dims[0]) for i in range(4)] + qkv_bias = "pretrained.blocks.0.attn.qkv.bias" in initializers + fc1 = initializers.get("pretrained.blocks.0.mlp.fc1.weight") + w12 = initializers.get("pretrained.blocks.0.mlp.w12.weight") + if fc1 is not None: + mlp_hidden = int(fc1.dims[0]) + ffn_type = "mlp" + elif w12 is not None: + mlp_hidden = int(w12.dims[0]) // 2 + ffn_type = "swiglu" + else: + raise ValueError("unsupported DA2 MLP tensors") + pos_grid = math.isqrt(int(pos_embed.dims[1]) - 1) + + writer = gguf.GGUFWriter(output, K.ARCH) + writer.add_string(K.KV["arch"], "depthanything2") + writer.add_string(K.KV["checkpoint_name"], checkpoint_name) + writer.add_uint32(K.KV["patch_size"], 14) + writer.add_uint32(K.KV["vit.embed_dim"], embed_dim) + writer.add_uint32(K.KV["vit.depth"], depth) + writer.add_uint32(K.KV["vit.num_heads"], num_heads) + writer.add_uint32(K.KV["vit.head_dim"], embed_dim // num_heads) + writer.add_uint32(K.KV["vit.mlp_hidden"], mlp_hidden) + writer.add_string(K.KV["vit.ffn_type"], ffn_type) + writer.add_uint32(K.KV["vit.num_register"], 0) + writer.add_float32(K.KV["vit.init_values"], 1.0) + writer.add_int32(K.KV["vit.alt_start"], -1) + writer.add_int32(K.KV["vit.rope_start"], -1) + writer.add_int32(K.KV["vit.qknorm_start"], -1) + writer.add_float32(K.KV["vit.rope_freq"], 100.0) + writer.add_bool(K.KV["vit.cat_token"], False) + writer.add_bool(K.KV["vit.qkv_bias"], qkv_bias) + writer.add_float32(K.KV["vit.ln_eps"], 1e-6) + writer.add_float32(K.KV["vit.interp_offset"], 0.1) + writer.add_bool(K.KV["vit.interp_antialias"], False) + writer.add_uint32(K.KV["vit.pos_embed_grid"], pos_grid) + writer.add_array(K.KV["vit.out_layers"], out_layers) + writer.add_array(K.KV["img.mean"], [0.485, 0.456, 0.406]) + writer.add_array(K.KV["img.std"], [0.229, 0.224, 0.225]) + writer.add_string(K.KV["img.resize_mode"], "lower_bound") + writer.add_uint32(K.KV["img.resize_target"], 518) + writer.add_uint32(K.KV["head.features"], head_features) + writer.add_array(K.KV["head.out_channels"], head_out_channels) + writer.add_uint32(K.KV["head.output_dim"], 1) + writer.add_bool(K.KV["head.pos_embed"], False) + writer.add_string(K.KV["head.activation"], "sigmoid" if max_depth > 0 else "relu") + writer.add_string(K.KV["head.norm_type"], "idt") + if max_depth > 0: + writer.add_float32(K.KV["head.max_depth"], max_depth) + + written = 0 + for name, tensor in initializers.items(): + if name.startswith("pretrained."): + gguf_name = K.rename_backbone(name.removeprefix("pretrained.")) + elif name.startswith("depth_head."): + gguf_name = K.rename_head(name.removeprefix("depth_head.")) + else: + raise ValueError(f"unexpected ONNX initializer {name}") + if gguf_name is None: + raise ValueError(f"unmapped ONNX initializer {name}") + values = np.ascontiguousarray(numpy_helper.to_array(tensor), dtype=np.float32) + writer.add_tensor(gguf_name, values) + written += 1 + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + print(f"wrote {output}: tensors={written} arch=depthanything2 encoder={encoder} max_depth={max_depth:g}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--onnx", required=True, help="Depth Anything V2 ONNX model") + parser.add_argument("--output", required=True, help="destination GGUF path") + parser.add_argument("--name", default="yuvraj108c/Depth-Anything-2-Onnx") + parser.add_argument("--max-depth", type=float, default=None, + help="metric range in metres; inferred from metric model filenames") + args = parser.parse_args() + max_depth = inferred_max_depth(args.onnx) if args.max_depth is None else args.max_depth + write_da2_onnx_gguf(args.onnx, args.output, args.name, max_depth) + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch_models.sh b/scripts/fetch_models.sh index 6908f6a..6fefb94 100755 --- a/scripts/fetch_models.sh +++ b/scripts/fetch_models.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Pre-download the curated DA3 model set for the demo app (~13 GB). +# Pre-download the curated DA3 model set plus the F32 depth-upscale default (~13 GB). # One weight per capability, not the full 31 GB of redundant quants. # Idempotent: skips files already present at the right size; resumes partials. set -uo pipefail @@ -16,6 +16,7 @@ CURATED=( "relative+pose+gaussians depth-anything-giant-f32.gguf" "metric+pose (anyview branch) depth-anything-nested-anyview.gguf" "metric+pose (metric branch) depth-anything-nested-metric.gguf" + "relative depth (upscale default) depth-anything-base-f32.gguf" ) echo "Downloading curated DA3 models into $DIR" diff --git a/server/engine.go b/server/engine.go index 98b60a6..b4e37b4 100644 --- a/server/engine.go +++ b/server/engine.go @@ -1,5 +1,5 @@ // engine.go — purego bindings to libdepthanything (the flat C API in -// include/da_capi.h, ABI 11). No cgo: the shared library is dlopen'd once and each +// include/da_capi.h, ABI 12). No cgo: the shared library is dlopen'd once and each // model is loaded as its own da_ctx. A context is NOT thread-safe (one ggml // backend + compute graph), so all inference is serialized by the caller. package main diff --git a/src/cli.cpp b/src/cli.cpp index 3593bd4..f14159a 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -11,6 +11,7 @@ void print_help(){ " da3-cli depth --model --input [--glb ] [--colmap ] [--colmap-txt ] (single-image 3D export)\n" " da3-cli depth --model --metric-model --input [--pfm ] (nested metric-scale depth)\n" " da3-cli depth --model --input a.png --input b.png [--out-prefix out] [--no-invert] (multi-view)\n" + " da3-cli depth-upscale [--model ] --input --sensor-depth --tiff [--degree N] [--threads N] (requires -DDA_ENABLE_TIFF=ON)\n" " da3-cli reconstruct --model --input --ply [--pose ]\n" " da3-cli quantize (type: f16|q8_0|q6_k|q5_k|q4_k)\n"); } @@ -54,6 +55,24 @@ Parsed parse(int argc, char** argv){ else if (r.inputs.empty()) r.error = "depth: --input required"; return r; } + if (first == "depth-upscale"){ + r.sub = Sub::DepthUpscale; + for (int i=2;i #include namespace da { namespace cli { -enum class Sub { Info, Depth, Reconstruct, Quantize, Help, None }; +enum class Sub { Info, Depth, DepthUpscale, Reconstruct, Quantize, Help, None }; struct Parsed { Sub sub = Sub::None; std::string model; @@ -16,9 +16,12 @@ struct Parsed { bool colmap_binary = true; // --colmap => bin; --colmap-txt => txt variant std::vector inputs; // accumulates repeated --input; >1 => multi-view mode std::string out_prefix; // --out-prefix for multi-view outputs + std::string sensor_depth_tiff; // depth-upscale: packed two-channel sensor TIFF + std::string output_depth_tiff; // depth-upscale: calibrated uint16 TIFF std::string q_in, q_out, q_type; // quantize: int n_threads = 0; // --threads N (0 => engine default of 1) int repeat = 0; // --repeat N (bench hook: time N inferences, then exit) + int upscale_degree = 2; // depth-upscale: polynomial calibration degree bool invert = true; // Single-image depth uses the REAL DA3 native-resolution resize by default. // --legacy-resize forces the old floor-to-patch path (fixture parity only). diff --git a/src/da_capi.cpp b/src/da_capi.cpp index d14123c..6d64ef8 100644 --- a/src/da_capi.cpp +++ b/src/da_capi.cpp @@ -1,5 +1,9 @@ #include "da_capi.h" #include "engine.hpp" +#include "depth_upscale.hpp" +#ifdef DA_ENABLE_TIFF +#include "tiff_io.hpp" +#endif #include "preprocess.hpp" #include "image_io.hpp" #include "glb_export.hpp" @@ -17,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +43,15 @@ struct da_ctx { std::vector temporal_voxel_mesh; }; +static void capi_set_error(da_ctx* c, const char* message) noexcept { + if (!c) return; + try { + c->last_error = message; + } catch (...) { + c->last_error.clear(); + } +} + static char* dup_cstr(const std::string& s){ char* p = (char*)std::malloc(s.size()+1); if (p) std::memcpy(p, s.c_str(), s.size()+1); @@ -90,7 +104,7 @@ static bool capi_run_nested(da_ctx* c, const char* image_path, } extern "C" { -int da_capi_abi_version(void){ return 11; } +int da_capi_abi_version(void){ return 12; } // Scene-relative TSDF fusion knobs applied by the NEXT da_capi_points_stream when // its fuse flag is set. voxel_frac = voxel edge as a fraction of the bbox diagonal @@ -129,6 +143,71 @@ char* da_capi_info_json(da_ctx* c){ } void da_capi_free_string(char* s){ std::free(s); } const char* da_capi_last_error(da_ctx* c){ return c ? c->last_error.c_str() : ""; } +int da_capi_depth_upscale(da_ctx* c, const char* image_path, + const uint16_t* projected_range_mm, int range_h, int range_w, + int polynomial_degree, float gaussian_sigma, + uint16_t* out_range_mm){ + if (!c || !c->engine || !image_path || !projected_range_mm || !out_range_mm || + range_h <= 0 || range_w <= 0){ + capi_set_error(c, "depth_upscale: bad args"); + return -1; + } + if (polynomial_degree < 0 || polynomial_degree > 8 || + !std::isfinite(gaussian_sigma) || gaussian_sigma < 0.0f || + gaussian_sigma > da::kMaxDepthUpscaleGaussianSigma) { + capi_set_error(c, "depth_upscale: invalid polynomial degree or Gaussian sigma"); + return -1; + } + try { + da::Image image; + std::string path(image_path), error; + std::string lower = path; + for (char& ch : lower) ch = (char)std::tolower((unsigned char)ch); + const bool is_tiff = lower.size() >= 4 && + (lower.compare(lower.size() - 4, 4, ".tif") == 0 || + (lower.size() >= 5 && lower.compare(lower.size() - 5, 5, ".tiff") == 0)); + if (is_tiff) { +#ifdef DA_ENABLE_TIFF + if (!da::load_tiff_rgb(path, image, &error)) { + c->last_error = "depth_upscale: load image failed: " + error; + return -1; + } +#else + c->last_error = "depth_upscale: TIFF input requires a build with -DDA_ENABLE_TIFF=ON"; + return -1; +#endif + } else if (!da::load_image_rgb(path, image)) { + c->last_error = "depth_upscale: load image failed"; + return -1; + } + std::vector predicted; + int prediction_h = 0, prediction_w = 0; + if (!da::predict_depth_for_upscale(*c->engine, image, predicted, + prediction_h, prediction_w, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + const size_t count = (size_t)range_h * (size_t)range_w; + std::vector projected(projected_range_mm, projected_range_mm + count), output; + da::DepthUpscaleOptions options; + options.degree = polynomial_degree; + options.gaussian_sigma = gaussian_sigma; + if (!da::upscale_depth_map(projected, range_h, range_w, predicted, + prediction_h, prediction_w, output, options, &error)){ + c->last_error = "depth_upscale: " + error; + return -1; + } + std::memcpy(out_range_mm, output.data(), count * sizeof(uint16_t)); + c->last_error.clear(); + return 0; + } catch (const std::bad_alloc&) { + capi_set_error(c, "depth_upscale: allocation failed"); + return -1; + } catch (...) { + capi_set_error(c, "depth_upscale: failed"); + return -1; + } +} float* da_capi_depth_path(da_ctx* c, const char* image_path, int* out_h, int* out_w){ if (!c || !c->engine || !image_path){ if (c) c->last_error = "depth: bad args"; return nullptr; } std::vector depth, conf; int H = 0, W = 0; diff --git a/src/depth_upscale.cpp b/src/depth_upscale.cpp new file mode 100644 index 0000000..5fa1178 --- /dev/null +++ b/src/depth_upscale.cpp @@ -0,0 +1,265 @@ +#include "depth_upscale.hpp" +#include "engine.hpp" +#include "image_io.hpp" + +#include +#include + +namespace da { +namespace { + +void set_error(std::string* error, const char* message) { + if (error) *error = message; +} + +bool valid_shape(size_t count, int h, int w) { + return h > 0 && w > 0 && count == static_cast(h) * w; +} + +int reflect_index(int index, int size) { + while (index < 0 || index >= size) { + index = index < 0 ? -index - 1 : 2 * size - index - 1; + } + return index; +} + +std::vector resize_bilinear_reflect(const std::vector& image, int in_h, int in_w, + int out_h, int out_w) { + std::vector out(static_cast(out_h) * out_w); + for (int y = 0; y < out_h; ++y) { + const float source_y = (static_cast(y) + 0.5f) * in_h / out_h - 0.5f; + const int y0 = static_cast(std::floor(source_y)); + const float fy = source_y - y0; + const int y1 = y0 + 1; + for (int x = 0; x < out_w; ++x) { + const float source_x = (static_cast(x) + 0.5f) * in_w / out_w - 0.5f; + const int x0 = static_cast(std::floor(source_x)); + const float fx = source_x - x0; + const int x1 = x0 + 1; + const float a = image[static_cast(reflect_index(y0, in_h)) * in_w + reflect_index(x0, in_w)]; + const float b = image[static_cast(reflect_index(y0, in_h)) * in_w + reflect_index(x1, in_w)]; + const float c = image[static_cast(reflect_index(y1, in_h)) * in_w + reflect_index(x0, in_w)]; + const float d = image[static_cast(reflect_index(y1, in_h)) * in_w + reflect_index(x1, in_w)]; + out[static_cast(y) * out_w + x] = + (a + (b - a) * fx) + ((c + (d - c) * fx) - (a + (b - a) * fx)) * fy; + } + } + return out; +} + +void gaussian_blur_reflect(std::vector& image, int h, int w, float sigma) { + if (sigma <= 0.0f) return; + const int radius = static_cast(std::ceil(4.0f * sigma)); + std::vector kernel(static_cast(2 * radius + 1)); + float sum = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + const float value = std::exp(-0.5f * offset * offset / (sigma * sigma)); + kernel[static_cast(offset + radius)] = value; + sum += value; + } + for (float& value : kernel) value /= sum; + + std::vector tmp(image.size()); + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + float value = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + value += kernel[static_cast(offset + radius)] * + image[static_cast(y) * w + reflect_index(x + offset, w)]; + } + tmp[static_cast(y) * w + x] = value; + } + } + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + float value = 0.0f; + for (int offset = -radius; offset <= radius; ++offset) { + value += kernel[static_cast(offset + radius)] * + tmp[static_cast(reflect_index(y + offset, h)) * w + x]; + } + image[static_cast(y) * w + x] = value; + } + } +} + +bool solve_linear_system(std::vector& matrix, std::vector& rhs, int n) { + for (int column = 0; column < n; ++column) { + int pivot = column; + for (int row = column + 1; row < n; ++row) { + if (std::abs(matrix[static_cast(row) * n + column]) > + std::abs(matrix[static_cast(pivot) * n + column])) { + pivot = row; + } + } + const double pivot_value = matrix[static_cast(pivot) * n + column]; + if (!std::isfinite(pivot_value) || std::abs(pivot_value) < 1e-12) return false; + if (pivot != column) { + for (int j = column; j < n; ++j) { + std::swap(matrix[static_cast(pivot) * n + j], matrix[static_cast(column) * n + j]); + } + std::swap(rhs[pivot], rhs[column]); + } + for (int row = column + 1; row < n; ++row) { + const double factor = matrix[static_cast(row) * n + column] / pivot_value; + for (int j = column; j < n; ++j) { + matrix[static_cast(row) * n + j] -= factor * matrix[static_cast(column) * n + j]; + } + rhs[row] -= factor * rhs[column]; + } + } + for (int row = n - 1; row >= 0; --row) { + double value = rhs[row]; + for (int column = row + 1; column < n; ++column) { + value -= matrix[static_cast(row) * n + column] * rhs[column]; + } + rhs[row] = value / matrix[static_cast(row) * n + row]; + if (!std::isfinite(rhs[row])) return false; + } + return true; +} + +} // namespace + +bool predict_depth_for_upscale(Engine& engine, const Image& image, + std::vector& depth, int& h, int& w, + std::string* error) { + if (engine.is_nested()) { + set_error(error, "two-file nested DA3 models are not supported for range upscaling"); + return false; + } + if (engine.is_da2()) { + if (engine.depth_relative(image, depth, h, w)) return true; + set_error(error, "Depth Anything V2 inference failed"); + return false; + } + if (engine.is_mono()) { + std::vector sky; + if (engine.depth_mono(image, depth, sky, h, w)) return true; + set_error(error, "Depth Anything V3 mono inference failed"); + return false; + } + std::vector confidence; + if (engine.depth_native_image(image, depth, confidence, h, w)) return true; + set_error(error, "Depth Anything V3 inference failed"); + return false; +} + +bool normalize_depth_u8(const std::vector& depth, std::vector& normalized, + std::string* error) { + if (depth.empty()) { + set_error(error, "model depth is empty"); + return false; + } + auto range = std::minmax_element(depth.begin(), depth.end()); + if (!std::isfinite(*range.first) || !std::isfinite(*range.second) || *range.first >= *range.second) { + set_error(error, "model depth must contain a finite non-constant range"); + return false; + } + const float scale = 255.0f / (*range.second - *range.first); + normalized.resize(depth.size()); + for (size_t i = 0; i < depth.size(); ++i) { + normalized[i] = static_cast(std::clamp((depth[i] - *range.first) * scale, 0.0f, 255.0f)); + } + return true; +} + +static bool upscale_depth_map_impl(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + bool normalize_relative, std::string* error) { + if (!valid_shape(sensor_depth.size(), sensor_h, sensor_w) || + !valid_shape(relative_depth.size(), relative_h, relative_w)) { + set_error(error, "depth dimensions do not match pixel counts"); + return false; + } + if (options.degree < 0 || options.degree > 8 || !std::isfinite(options.gaussian_sigma) || + options.gaussian_sigma < 0.0f || options.gaussian_sigma > kMaxDepthUpscaleGaussianSigma) { + set_error(error, "invalid polynomial degree or Gaussian sigma"); + return false; + } + + std::vector normalized(relative_depth.size()); + if (normalize_relative) { + auto relative_range = std::minmax_element(relative_depth.begin(), relative_depth.end()); + if (!std::isfinite(*relative_range.first) || !std::isfinite(*relative_range.second) || + *relative_range.first >= *relative_range.second) { + set_error(error, "model depth must contain a finite non-constant range"); + return false; + } + const float relative_scale = 1.0f / (*relative_range.second - *relative_range.first); + for (size_t i = 0; i < relative_depth.size(); ++i) { + if (!std::isfinite(relative_depth[i])) { + set_error(error, "model depth must be finite"); + return false; + } + normalized[i] = std::clamp((relative_depth[i] - *relative_range.first) * relative_scale, + 0.0f, 1.0f); + } + } else { + normalized = relative_depth; + } + + std::vector predictor = resize_bilinear_reflect(normalized, relative_h, relative_w, + sensor_h, sensor_w); + gaussian_blur_reflect(predictor, sensor_h, sensor_w, options.gaussian_sigma); + const float predictor_max = *std::max_element(predictor.begin(), predictor.end()); + const uint16_t sensor_max = *std::max_element(sensor_depth.begin(), sensor_depth.end()); + if (!std::isfinite(predictor_max) || predictor_max <= 0.0f || sensor_max == 0) { + set_error(error, "sensor and relative depth must have positive maxima"); + return false; + } + + const int coefficient_count = options.degree + 1; + std::vector normal(static_cast(coefficient_count) * coefficient_count, 0.0); + std::vector rhs(coefficient_count, 0.0); + size_t valid_count = 0; + for (size_t i = 0; i < predictor.size(); ++i) { + const bool missing_legacy_predictor = !normalize_relative && predictor[i] <= 0.0f; + if (sensor_depth[i] == 0 || missing_legacy_predictor || !std::isfinite(predictor[i])) continue; + const double x = predictor[i] / predictor_max; + const double y = sensor_depth[i]; + double powers[17] = {1.0}; + for (int p = 1; p <= 2 * options.degree; ++p) powers[p] = powers[p - 1] * x; + for (int row = 0; row < coefficient_count; ++row) { + rhs[row] += powers[row] * y; + for (int column = 0; column < coefficient_count; ++column) { + normal[static_cast(row) * coefficient_count + column] += powers[row + column]; + } + } + ++valid_count; + } + if (valid_count < static_cast(coefficient_count) || !solve_linear_system(normal, rhs, coefficient_count)) { + set_error(error, "insufficient non-degenerate overlapping depth samples for polynomial fit"); + return false; + } + + output.resize(predictor.size()); + for (size_t i = 0; i < predictor.size(); ++i) { + const double x = predictor[i] / predictor_max; + double value = 0.0; + for (int coefficient = coefficient_count - 1; coefficient >= 0; --coefficient) { + value = value * x + rhs[coefficient]; + } + output[i] = static_cast(std::lround(std::clamp(value, 0.0, 65535.0))); + } + return true; +} + +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + std::string* error) { + return upscale_depth_map_impl(sensor_depth, sensor_h, sensor_w, relative_depth, relative_h, relative_w, + output, options, true, error); +} + +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options, + std::string* error) { + std::vector converted(relative_depth.begin(), relative_depth.end()); + return upscale_depth_map_impl(sensor_depth, sensor_h, sensor_w, converted, relative_h, relative_w, + output, options, false, error); +} + +} // namespace da diff --git a/src/depth_upscale.hpp b/src/depth_upscale.hpp new file mode 100644 index 0000000..3873ef2 --- /dev/null +++ b/src/depth_upscale.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +namespace da { + +class Engine; +struct Image; +inline constexpr float kMaxDepthUpscaleGaussianSigma = 64.0f; + + +struct DepthUpscaleOptions { + int degree = 2; + float gaussian_sigma = 1.0f; +}; + +// Runs the depth-only inference path suitable for range calibration. Supports +// DA2, DA3MONO, and the single-file DA3 DualDPT models without computing pose. +bool predict_depth_for_upscale(Engine& engine, const Image& image, + std::vector& depth, int& h, int& w, + std::string* error = nullptr); + +// Calibrates a Depth Anything map against projected sensor ranges. The +// predictor is normalized, resized, Gaussian-smoothed, and polynomial-fitted +// over pixels where sensor samples exist. Output remains calibrated uint16 +// sensor units (normally millimetres); it is not stretched to the full range. +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options = {}, + std::string* error = nullptr); + +// Compatibility overload for the former normalized-uint8 service boundary. +bool upscale_depth_map(const std::vector& sensor_depth, int sensor_h, int sensor_w, + const std::vector& relative_depth, int relative_h, int relative_w, + std::vector& output, const DepthUpscaleOptions& options = {}, + std::string* error = nullptr); + +// Matches the former service boundary: render a model depth field to the +// normalized 8-bit image that feeds upscale_depth_map. +bool normalize_depth_u8(const std::vector& depth, std::vector& normalized, + std::string* error = nullptr); + +} // namespace da diff --git a/src/tiff_io.cpp b/src/tiff_io.cpp new file mode 100644 index 0000000..0ce6e1a --- /dev/null +++ b/src/tiff_io.cpp @@ -0,0 +1,143 @@ +#include "tiff_io.hpp" + +#include +#include + +namespace da { +namespace { + +void set_error(std::string* error, const char* message) { + if (error) *error = message; +} + +bool dimensions_fit_int(uint32_t width, uint32_t height) { + return width > 0 && height > 0 && + width <= static_cast(std::numeric_limits::max()) && + height <= static_cast(std::numeric_limits::max()); +} + +} // namespace + +bool load_tiff_rgb(const std::string& path, Image& out, std::string* error) { + TIFF* tiff = TIFFOpen(path.c_str(), "r"); + if (!tiff) { + set_error(error, "cannot open TIFF"); + return false; + } + + uint32_t width = 0, height = 0; + const bool valid = TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &width) && + TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &height) && + dimensions_fit_int(width, height); + if (!valid || static_cast(width) > std::numeric_limits::max() / height) { + TIFFClose(tiff); + set_error(error, "invalid TIFF dimensions"); + return false; + } + + std::vector rgba(static_cast(width) * height); + if (!TIFFReadRGBAImageOriented(tiff, width, height, rgba.data(), ORIENTATION_TOPLEFT, 0)) { + TIFFClose(tiff); + set_error(error, "failed to decode TIFF pixels"); + return false; + } + TIFFClose(tiff); + + out.w = static_cast(width); + out.h = static_cast(height); + out.rgb.resize(rgba.size() * 3); + for (size_t i = 0; i < rgba.size(); ++i) { + out.rgb[3 * i + 0] = TIFFGetR(rgba[i]); + out.rgb[3 * i + 1] = TIFFGetG(rgba[i]); + out.rgb[3 * i + 2] = TIFFGetB(rgba[i]); + } + return true; +} + +bool load_packed_depth_tiff(const std::string& path, std::vector& depth, + int& h, int& w, std::string* error) { + TIFF* tiff = TIFFOpen(path.c_str(), "r"); + if (!tiff) { + set_error(error, "cannot open TIFF"); + return false; + } + + uint32_t width = 0, height = 0; + uint16_t bits = 0, samples = 0, planar = PLANARCONFIG_CONTIG; + const bool valid = TIFFGetField(tiff, TIFFTAG_IMAGEWIDTH, &width) && + TIFFGetField(tiff, TIFFTAG_IMAGELENGTH, &height) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_BITSPERSAMPLE, &bits) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_SAMPLESPERPIXEL, &samples) && + TIFFGetFieldDefaulted(tiff, TIFFTAG_PLANARCONFIG, &planar) && + dimensions_fit_int(width, height) && bits == 8 && samples >= 2 && + planar == PLANARCONFIG_CONTIG; + if (!valid || static_cast(width) > std::numeric_limits::max() / height) { + TIFFClose(tiff); + set_error(error, "expected an interleaved two-channel 8-bit depth TIFF"); + return false; + } + + const tsize_t scanline_size = TIFFScanlineSize(tiff); + if (scanline_size < static_cast(width) * samples) { + TIFFClose(tiff); + set_error(error, "invalid TIFF scanline size"); + return false; + } + std::vector row(static_cast(scanline_size)); + depth.resize(static_cast(width) * height); + for (uint32_t y = 0; y < height; ++y) { + if (TIFFReadScanline(tiff, row.data(), y, 0) < 0) { + TIFFClose(tiff); + depth.clear(); + set_error(error, "failed to decode depth TIFF scanline"); + return false; + } + for (uint32_t x = 0; x < width; ++x) { + const size_t src = static_cast(x) * samples; + depth[static_cast(y) * width + x] = + static_cast((static_cast(row[src]) << 8) | row[src + 1]); + } + } + TIFFClose(tiff); + h = static_cast(height); + w = static_cast(width); + return true; +} + +bool write_depth_tiff_u16(const std::string& path, const std::vector& depth, + int h, int w, std::string* error) { + if (h <= 0 || w <= 0 || depth.size() != static_cast(h) * w) { + set_error(error, "depth dimensions do not match pixel count"); + return false; + } + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) { + set_error(error, "cannot create TIFF"); + return false; + } + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, static_cast(w)) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, static_cast(h)) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 16) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 1) && + TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_UINT) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_MINISBLACK) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE) && + TIFFSetField(tiff, TIFFTAG_ROWSPERSTRIP, static_cast(h)); + if (!configured) { + TIFFClose(tiff); + set_error(error, "failed to configure output TIFF"); + return false; + } + for (int y = 0; y < h; ++y) { + if (TIFFWriteScanline(tiff, const_cast(depth.data() + static_cast(y) * w), y, 0) < 0) { + TIFFClose(tiff); + set_error(error, "failed to write depth TIFF scanline"); + return false; + } + } + TIFFClose(tiff); + return true; +} + +} // namespace da diff --git a/src/tiff_io.hpp b/src/tiff_io.hpp new file mode 100644 index 0000000..a6ee3f9 --- /dev/null +++ b/src/tiff_io.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "image_io.hpp" +#include +#include +#include + +namespace da { + +// Reads the first TIFF image directory as an 8-bit RGB image. Supports the +// compressed RGB TIFF captures used by the depth-upscale workflow. +bool load_tiff_rgb(const std::string& path, Image& out, std::string* error = nullptr); + +// Reads a depth sensor TIFF whose first two 8-bit interleaved samples encode a +// big-endian unsigned 16-bit depth value. +bool load_packed_depth_tiff(const std::string& path, std::vector& depth, + int& h, int& w, std::string* error = nullptr); + +// Writes one uncompressed, single-channel, unsigned 16-bit depth TIFF. +bool write_depth_tiff_u16(const std::string& path, const std::vector& depth, + int h, int w, std::string* error = nullptr); + +} // namespace da diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e53d0f..10ca7a0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,7 @@ da_add_test(test_ggml_extend) da_add_test(test_capi) da_add_test(test_capi_dense) da_add_test(test_capi_da2) +da_add_test(test_capi_upscale) da_add_test(test_preprocess) da_add_test(test_preprocess_real) da_add_test(test_rope2d) @@ -62,6 +63,8 @@ da_add_test(test_engine_pose) da_add_test(test_engine_mv) da_add_test(test_quantize) da_add_test(test_quantize_accuracy) +da_add_test(test_depth_upscale) +da_add_test(test_cli) # Standalone harness for scripts/oracle_fuse.py (takes CLI args; not a ctest). add_executable(fuse_dump fuse_dump.cpp) diff --git a/tests/test_capi.cpp b/tests/test_capi.cpp index 2934912..f2a40df 100644 --- a/tests/test_capi.cpp +++ b/tests/test_capi.cpp @@ -3,7 +3,7 @@ #include #include int main(){ - if (da_capi_abi_version() != 11) return 1; + if (da_capi_abi_version() != 12) return 1; const char* gguf = std::getenv("DA_TEST_GGUF"); if (!gguf) return 77; da_ctx* c = da_capi_load(gguf, 1); diff --git a/tests/test_capi_da2.cpp b/tests/test_capi_da2.cpp index e8ac79e..2d3d7fc 100644 --- a/tests/test_capi_da2.cpp +++ b/tests/test_capi_da2.cpp @@ -4,6 +4,8 @@ #include "da_capi.h" #include #include +#include +#include #include static bool finite_all(const float* p, int n){ @@ -22,6 +24,27 @@ int main(){ da_ctx* c = da_capi_load(gguf, 1); if (!c){ std::fprintf(stderr, "da2: load failed\n"); return 1; } + const uint16_t projected_range[] = { + 1000, 1100, 1200, 1300, + 1150, 1250, 1350, 1450, + 1300, 1400, 1500, 1600, + 1450, 1550, 1650, 1750, + }; + uint16_t upscaled[16] = {}; + const int invalid_sigma = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 1, std::numeric_limits::max(), upscaled); + const int invalid_degree = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 9, 0.0f, upscaled); + const int invalid_image = da_capi_depth_upscale( + c, "missing-image.jpg", projected_range, 4, 4, 1, 0.0f, upscaled); + const int upscale = da_capi_depth_upscale( + c, png, projected_range, 4, 4, 1, 0.0f, upscaled); + bool upscale_ok = invalid_sigma == -1 && invalid_degree == -1 && invalid_image == -1 && upscale == 0; + for (uint16_t value : upscaled) upscale_ok = upscale_ok && value > 0; + std::fprintf(stderr, "da2 upscale: invalid_sigma=%d invalid_degree=%d invalid_image=%d result=%d -> %s\n", + invalid_sigma, invalid_degree, invalid_image, upscale, upscale_ok ? "OK" : "FAIL"); + + int H=0, W=0, is_metric=-1; float *depth=nullptr, *conf=nullptr, *sky=nullptr; float ext[12], intr[9]; int r = da_capi_depth_dense(c, png, &H, &W, &depth, &conf, &sky, ext, intr, &is_metric); @@ -39,5 +62,5 @@ int main(){ rp, da_capi_last_error(c), okp?"OK":"FAIL"); da_capi_free(c); - return (ok && okp) ? 0 : 1; + return (ok && okp && upscale_ok) ? 0 : 1; } diff --git a/tests/test_capi_upscale.cpp b/tests/test_capi_upscale.cpp new file mode 100644 index 0000000..6dc70a1 --- /dev/null +++ b/tests/test_capi_upscale.cpp @@ -0,0 +1,63 @@ +#include "da_capi.h" + +#include +#include +#include + +struct Fixture { + const char* label; + const char* gguf_var; + const char* image_var; +}; + +static bool exists(const char* path) { + if (!path) return false; + FILE* file = std::fopen(path, "rb"); + if (!file) return false; + std::fclose(file); + return true; +} + +static bool run_upscale(const char* label, const char* gguf_var, const char* image_var) { + const char* gguf = std::getenv(gguf_var); + const char* image = std::getenv(image_var); + if (!exists(gguf) || !exists(image)) return false; + + da_ctx* ctx = da_capi_load(gguf, 1); + if (!ctx) { + std::fprintf(stderr, "%s: failed to load model\n", label); + return false; + } + const uint16_t projected_range[] = { + 1000, 1100, 1200, 1300, + 1150, 1250, 1350, 1450, + 1300, 1400, 1500, 1600, + 1450, 1550, 1650, 1750, + }; + uint16_t output[16] = {}; + const int result = da_capi_depth_upscale(ctx, image, projected_range, 4, 4, 1, 0.0f, output); + bool ok = result == 0; + for (uint16_t value : output) ok = ok && value > 0; + std::fprintf(stderr, "%s: upscale=%d -> %s\n", label, result, ok ? "OK" : "FAIL"); + da_capi_free(ctx); + return ok; +} + +int main() { + int available = 0; + bool ok = true; + const Fixture fixtures[] = { + {"DualDPT DA3", "DA_TEST_GGUF", "DA_TEST_NATIVE_PNG"}, + {"DA3 mono", "DA_TEST_GGUF_MONO", "DA_TEST_MONO_PNG"}, + }; + for (const Fixture& fixture : fixtures) { + if (!exists(std::getenv(fixture.gguf_var)) || !exists(std::getenv(fixture.image_var))) continue; + ++available; + ok = run_upscale(fixture.label, fixture.gguf_var, fixture.image_var) && ok; + } + if (available == 0) { + std::fprintf(stderr, "DA3 upscale fixtures absent, skipping\n"); + return 77; + } + return ok ? 0 : 1; +} diff --git a/tests/test_cli.cpp b/tests/test_cli.cpp new file mode 100644 index 0000000..8bf135e --- /dev/null +++ b/tests/test_cli.cpp @@ -0,0 +1,33 @@ +#include "cli.hpp" + +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const char* message) { + std::fprintf(stderr, "%s: %s\n", ok ? "ok" : "FAIL", message); + if (!ok) ++failures; +} + +static da::cli::Parsed parse(std::vector args) { + std::vector argv; + argv.reserve(args.size()); + for (std::string& arg : args) argv.push_back(arg.data()); + return da::cli::parse(static_cast(argv.size()), argv.data()); +} + +int main() { + auto defaults = parse({"da3-cli", "depth-upscale", "--input", "rgb.tiff", + "--sensor-depth", "sensor.tiff", "--tiff", "depth.tiff"}); + check(defaults.error.empty(), "depth-upscale accepts required arguments without --model"); + check(defaults.model == "models/depth-anything-base-f32.gguf", + "depth-upscale uses the DA3_BASE f32 model by default"); + auto explicit_model = parse({"da3-cli", "depth-upscale", "--model", "custom.gguf", + "--input", "rgb.tiff", "--sensor-depth", "sensor.tiff", + "--tiff", "depth.tiff"}); + check(explicit_model.error.empty(), "depth-upscale accepts an explicit model"); + check(explicit_model.model == "custom.gguf", "explicit depth-upscale model overrides default"); + return failures == 0 ? 0 : 1; +} diff --git a/tests/test_depth_upscale.cpp b/tests/test_depth_upscale.cpp new file mode 100644 index 0000000..e9ae7c4 --- /dev/null +++ b/tests/test_depth_upscale.cpp @@ -0,0 +1,152 @@ +#include "depth_upscale.hpp" +#include "da_capi.h" +#ifdef DA_ENABLE_TIFF +#include "tiff_io.hpp" +#include +#endif +#include +#include +#include +#include +#include + +using namespace da; + +static int failures = 0; +static void check(bool ok, const char* message) { + std::fprintf(stderr, "%s: %s\n", ok ? "ok" : "FAIL", message); + if (!ok) ++failures; +} + +#ifdef DA_ENABLE_TIFF +static std::string temporary_path(const char* suffix) { + return std::string("/tmp/da-depth-upscale-") + std::to_string(getpid()) + suffix; +} + +static bool write_packed_sensor(const std::string& path, const std::vector& values, int h, int w) { + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) return false; + uint16_t extra_sample = EXTRASAMPLE_UNASSALPHA; + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, static_cast(w)) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, static_cast(h)) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 8) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 2) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_MINISBLACK) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE) && + TIFFSetField(tiff, TIFFTAG_EXTRASAMPLES, 1, &extra_sample); + std::vector row(static_cast(w) * 2); + bool written = configured; + for (int y = 0; written && y < h; ++y) { + for (int x = 0; x < w; ++x) { + const uint16_t value = values[static_cast(y) * w + x]; + row[2 * x] = static_cast(value >> 8); + row[2 * x + 1] = static_cast(value); + } + written = TIFFWriteScanline(tiff, row.data(), y, 0) >= 0; + } + TIFFClose(tiff); + return written; +} + +static bool write_rgb_fixture(const std::string& path) { + TIFF* tiff = TIFFOpen(path.c_str(), "w"); + if (!tiff) return false; + const bool configured = TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, 2) && + TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, 1) && + TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, 8) && + TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 3) && + TIFFSetField(tiff, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG) && + TIFFSetField(tiff, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB) && + TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_NONE); + const std::vector pixels { 1, 2, 3, 250, 251, 252 }; + const bool written = configured && TIFFWriteScanline(tiff, const_cast(pixels.data()), 0, 0) >= 0; + TIFFClose(tiff); + return written; +} +#endif + +int main() { + + check(da_capi_abi_version() == 12, "reports C API version 12"); + uint16_t one = 1; + check(da_capi_depth_upscale(nullptr, "missing.tiff", &one, 1, 1, 2, 1.0f, &one) == -1, + "C API rejects a null context"); + + std::vector normalized; + std::string error; + check(normalize_depth_u8({-2.0f, 0.0f, 2.0f}, normalized, &error), "normalizes finite model depth"); + check(normalized == std::vector({0, 127, 255}), "normalization matches uint8 truncation semantics"); + check(!normalize_depth_u8({1.0f, 1.0f}, normalized, &error), "rejects constant model depth"); + + + constexpr int h = 4, w = 5; + std::vector relative(h * w); + std::vector sensor(h * w); + for (int i = 0; i < h * w; ++i) { + relative[i] = static_cast(i + 1); + sensor[i] = static_cast(1000 + 200 * relative[i]); + } + DepthUpscaleOptions options; + options.degree = 1; + options.gaussian_sigma = 0.0f; + std::vector output; + check(upscale_depth_map(sensor, h, w, relative, h, w, output, options, &error), "fits linear overlapping depth"); + check(output.front() == sensor.front() && output.back() == sensor.back(), + "preserves calibrated sensor units without full-range stretching"); + bool monotonic = true; + for (size_t i = 1; i < output.size(); ++i) monotonic = monotonic && output[i - 1] <= output[i]; + check(monotonic, "preserves relative-depth ordering after calibration"); + check(!upscale_depth_map(sensor, h, w, std::vector(h * w, 1), h, w, output, options, &error), + "rejects degenerate predictor"); + + std::vector zero_sensor {1000, 2000, 3000}; + std::vector zero_relative {1.0f, 2.0f, 3.0f}; + options.degree = 2; + check(upscale_depth_map(zero_sensor, 1, 3, zero_relative, 1, 3, output, options, &error), + "fits normalized zero-valued predictor samples"); + check(output == zero_sensor, "calibrates the full normalized predictor range"); + + // The uint8 compatibility overload must retain the supplied nonzero range: + // normalizing it first would turn 10 into zero and leave only three samples + // for this cubic fit. + std::vector legacy_sensor {1000, 2000, 3000, 4000}; + std::vector legacy_relative {10, 20, 30, 40}; + std::vector legacy_output; + options.degree = 3; + check(upscale_depth_map(legacy_sensor, 2, 2, legacy_relative, 2, 2, legacy_output, options, &error), + "preserves legacy uint8 predictor values during calibration"); + options.degree = 1; + +#ifdef DA_ENABLE_TIFF + const std::string packed_path = temporary_path("-packed.tiff"); + const std::string output_path = temporary_path("-output.tiff"); + const std::string rgb_path = temporary_path("-rgb.tiff"); + check(write_rgb_fixture(rgb_path), "writes RGB TIFF fixture"); + Image rgb; + check(load_tiff_rgb(rgb_path, rgb, &error), "reads RGB TIFF"); + check(rgb.w == 2 && rgb.h == 1 && rgb.rgb == std::vector({1, 2, 3, 250, 251, 252}), + "preserves RGB TIFF pixels"); + + check(write_packed_sensor(packed_path, sensor, h, w), "writes packed sensor fixture"); + std::vector loaded; int loaded_h = 0, loaded_w = 0; + check(load_packed_depth_tiff(packed_path, loaded, loaded_h, loaded_w, &error), "reads packed sensor TIFF"); + check(loaded_h == h && loaded_w == w && loaded == sensor, "decodes big-endian two-channel sensor depth"); + check(write_depth_tiff_u16(output_path, sensor, h, w, &error), "writes uint16 depth TIFF"); + + TIFF* tiff = TIFFOpen(output_path.c_str(), "r"); + uint16_t bits = 0, samples = 0, format = 0; + const bool metadata_ok = tiff && TIFFGetField(tiff, TIFFTAG_BITSPERSAMPLE, &bits) && + TIFFGetField(tiff, TIFFTAG_SAMPLESPERPIXEL, &samples) && + TIFFGetField(tiff, TIFFTAG_SAMPLEFORMAT, &format); + check(metadata_ok && bits == 16 && samples == 1 && format == SAMPLEFORMAT_UINT, + "output TIFF has uint16 single-channel metadata"); + if (tiff) TIFFClose(tiff); + std::remove(packed_path.c_str()); + std::remove(output_path.c_str()); + std::remove(rgb_path.c_str()); +#endif + + std::fprintf(stderr, "%s (%d failures)\n", failures ? "FAILED" : "PASSED", failures); + return failures ? 1 : 0; +}