feat(image): ROS compressedDepth rendering (v1.6.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ioai/rosview",
-  "version": "1.5.4",
+  "version": "1.6.0",
   "description": "High-performance robotics data visualization for MCAP, ROS bag, ROS2 db3, HDF5 and BVH — embeddable React component and standalone SPA",
   "keywords": [
     "ros",

scripts/gen-e2e-fixtures.mjs

@@ -29,6 +29,7 @@ runNode('gen-test-mcap.mjs');
 runNode('gen-test-mcap-pose.mjs');
 runNode('gen-test-mcap-3cam.mjs');
 runNode('gen-test-mcap-h264.mjs');
+runNode('gen-test-mcap-compressed-depth.mjs');
 runPython('gen-test-hdf5.py');
 runNode('gen-test-bvh.mjs');
 

scripts/gen-test-mcap-compressed-depth.mjs

@@ -0,0 +1,42 @@
+/**
+ * Minimal MCAP with 16UC1 compressedDepth CompressedImage messages for E2E.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  createIndexedMcapWriter,
+  encodeCompressedImageCdr,
+  readFixture,
+  registerCompressedImageChannel,
+  writeExample,
+} from './mcap-fixture-utils.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const payload = readFixture('image/compressed-depth-16uc1.bin');
+const format = fs.readFileSync(
+  path.join(__dirname, '../test-fixtures/image/compressed-depth-16uc1.format.txt'),
+  'utf8',
+).trim();
+
+const { writer, writable } = await createIndexedMcapWriter();
+
+const channelId = await registerCompressedImageChannel(
+  '/camera/realsense2_camera/aligned_depth_to_color/image_raw/compressed_depth',
+  writer,
+);
+
+const frames = [1_000_000_000n, 2_000_000_000n, 3_000_000_000n];
+for (const [idx, ts] of frames.entries()) {
+  const stamp = { sec: Number(ts / 1_000_000_000n), nsec: Number(ts % 1_000_000_000n) };
+  await writer.addMessage({
+    channelId,
+    sequence: idx + 1,
+    logTime: ts,
+    publishTime: ts,
+    data: encodeCompressedImageCdr(stamp, format, payload),
+  });
+}
+
+await writer.end();
+writeExample('test_compressed_depth.mcap', writable.getBuffer());

src/features/panels/Image/ImagePanel.tsx

@@ -16,6 +16,7 @@ import {
 } from './core/imageTypes';
 import { repairH264Seek } from './core/h264SeekRepair';
 import { isH264MessageEvent, toWorkerFrame } from './core/messageFrameAdapter';
+import { applyDepthTopicPreset } from './core/depthColorDefaults';
 import type { ImageConfig } from './defaults';
 import { TopicQuickPicker } from '../framework/TopicQuickPicker';
 import ImageRenderWorkerClass from './core/ImageRender.worker.ts?worker&inline';
@@ -306,7 +307,7 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
       <div className="flex shrink-0 items-center gap-2 border-b border-zinc-800 bg-zinc-950">
         <TopicQuickPicker
           value={topic}
-          onChange={(nextTopic) => setConfig((prev) => ({ ...prev, topic: nextTopic }))}
+          onChange={(nextTopic) => setConfig((prev) => applyDepthTopicPreset(nextTopic, prev))}
           typeIncludes={[...IMAGE_PANEL_TOPIC_INCLUDES]}
           placeholder={formatMessage({ id: 'panels.framework.topicPicker.imagePlaceholder' })}
           className="min-w-0 flex-1"

src/features/panels/Image/ImagePanelSettings.tsx

@@ -13,7 +13,8 @@ import {
 } from '../framework/settings';
 import { messageBus } from '@/core/pipeline/messageBus';
 import { useTopicSeq } from '@/core/pipeline/useMessageBus';
-import { isRawImageMessage, isRawImageTopicSchema, IMAGE_PANEL_TOPIC_INCLUDES } from './core/imageTypes';
+import { isRawImageMessage, isRawImageTopicSchema, isCompressedImageMessage, depthEncodingFromFormat, IMAGE_PANEL_TOPIC_INCLUDES } from './core/imageTypes';
+import { applyDepthTopicPreset, defaultDepthMaxValue, defaultDepthMinValue } from './core/depthColorDefaults';
 import type { ImageConfig } from './defaults';
 
 const DEPTH_ENCODINGS = new Set(['mono16', '16uc1', '32fc1']);
@@ -44,6 +45,9 @@ function useLastFrameEncoding(topic: string): string | null {
   if (isRawImageMessage(msg)) {
     return msg.encoding.trim().toLowerCase();
   }
+  if (isCompressedImageMessage(msg)) {
+    return depthEncodingFromFormat(msg.format);
+  }
   return null;
 }
 
@@ -99,6 +103,18 @@ export function ImagePanelSettings({
     encodingForDepthSliders != null && DEPTH_ENCODINGS.has(encodingForDepthSliders)
       ? depthSliderBounds(encodingForDepthSliders)
       : { min: 0, max: 65535 };
+  const sliderDefaultMax =
+    encodingForDepthSliders != null
+      ? defaultDepthMaxValue(encodingForDepthSliders, config.topic)
+      : sliderBounds.max;
+  const sliderDefaultMin =
+    encodingForDepthSliders != null
+      ? defaultDepthMinValue(encodingForDepthSliders, config.topic)
+      : sliderBounds.min;
+
+  const applyTopicChange = (topic: string) => {
+    setConfig(applyDepthTopicPreset(topic, config));
+  };
 
   return (
     <div className="space-y-2">
@@ -109,7 +125,7 @@ export function ImagePanelSettings({
           >
             <TopicAutocomplete
               value={config.topic}
-              onChange={(topic) => setConfig({ ...config, topic })}
+              onChange={applyTopicChange}
               topics={topics}
               typeIncludes={[...IMAGE_PANEL_TOPIC_INCLUDES]}
               placeholder={formatMessage({ id: 'panels.image.settings.field.topic.placeholder' })}
@@ -295,7 +311,7 @@ export function ImagePanelSettings({
                       )}
                     >
                       <SettingsSlider
-                        value={config.minValue ?? sliderBounds.min}
+                        value={config.minValue ?? sliderDefaultMin}
                         onChange={(minValue) => setConfig({ ...config, minValue })}
                         min={sliderBounds.min}
                         max={sliderBounds.max}
@@ -310,7 +326,7 @@ export function ImagePanelSettings({
                       )}
                     >
                       <SettingsSlider
-                        value={config.maxValue ?? sliderBounds.max}
+                        value={config.maxValue ?? sliderDefaultMax}
                         onChange={(maxValue) => setConfig({ ...config, maxValue })}
                         min={sliderBounds.min}
                         max={sliderBounds.max}

src/features/panels/Image/core/ImageRender.worker.ts

@@ -1,11 +1,13 @@
 /// <reference lib="webworker" />
 
 import type { Time } from '@/core/types/ros';
+import { decodeCompressedDepth } from './compressedDepthDecoder';
 import { decodeRawImage } from './rawDecoders';
 import { getH264ChunkType } from './h264';
 import { withTimeout } from './asyncTimeout';
 import {
   getCompressedKind,
+  isCompressedDepthFormat,
   normalizeCompressedMime,
   type ImageSurfaceStatus,
 } from './imageTypes';
@@ -396,11 +398,27 @@ class ImageRenderWorkerRuntime {
     this.#emitStatus({ phase: 'decoding', receiveTime: frame.receiveTime });
     try {
       if (frame.kind === 'compressed') {
-        const kind = getCompressedKind(frame.format);
         const bytes = ensureOwnedBytes(frame.data);
         if (bytes.byteLength === 0) {
           throw new Error(`Compressed image payload is empty: ${frame.format}`);
         }
+
+        // ROS compressedDepth: PNG → 16UC1/32FC1, then same colormap path as RawImage.
+        if (isCompressedDepthFormat(frame.format)) {
+          const decoded = await decodeCompressedDepth(bytes, frame.format);
+          this.#renderRawFrame({
+            receiveTime: frame.receiveTime,
+            encoding: decoded.encoding,
+            width: decoded.width,
+            height: decoded.height,
+            step: decoded.step,
+            isBigEndian: decoded.isBigEndian,
+            data: ensureOwnedBytes(decoded.data),
+          });
+          return;
+        }
+
+        const kind = getCompressedKind(frame.format);
         const sortKey = timeToKey(frame.receiveTime);
 
         if (kind === 'h264') {
@@ -471,52 +489,14 @@ class ImageRenderWorkerRuntime {
 
       // Raw frame
       const bytes = ensureOwnedBytes(frame.data);
-      const pixelBytes = frame.width * frame.height * 4;
-      let rgba = this.#rawRgba;
-      if (!rgba || rgba.length !== pixelBytes) {
-        rgba = new Uint8ClampedArray(pixelBytes);
-        this.#rawRgba = rgba;
-      }
-      if (!this.#rawImageData || this.#rawImageData.width !== frame.width || this.#rawImageData.height !== frame.height) {
-        this.#rawImageData = new ImageData(rgba, frame.width, frame.height);
-      }
-
-      const step = frame.step ?? (frame.width * bytesPerPixel(frame.encoding));
-      const isBigEndian = frame.isBigEndian ?? false;
-
-      decodeRawImage(
-        {
-          encoding: frame.encoding,
-          width: frame.width,
-          height: frame.height,
-          step,
-          is_bigendian: isBigEndian,
-          data: bytes,
-        },
-        rgba,
-        this.#rawDecodeOptions,
-      );
-
-      // Store source bytes so rawDecodeOptions changes can re-decode immediately.
-      this.#disposeCachedBitmap();
-      this.#cachedFrame = {
-        kind: 'raw',
-        width: frame.width,
-        height: frame.height,
-        encoding: frame.encoding,
-        step,
-        isBigEndian,
-        data: bytes,
+      this.#renderRawFrame({
         receiveTime: frame.receiveTime,
-      };
-
-      this.#drawRawImageData(frame.width, frame.height);
-      this.#emitStatus({
-        phase: 'ready',
+        encoding: frame.encoding,
         width: frame.width,
         height: frame.height,
-        encoding: frame.encoding,
-        receiveTime: frame.receiveTime,
+        step: frame.step ?? (frame.width * bytesPerPixel(frame.encoding)),
+        isBigEndian: frame.isBigEndian ?? false,
+        data: bytes,
       });
     } catch (error) {
       this.#haltUntilReset = true;
@@ -527,6 +507,60 @@ class ImageRenderWorkerRuntime {
     }
   }
 
+  #renderRawFrame(frame: {
+    receiveTime: Time;
+    encoding: string;
+    width: number;
+    height: number;
+    step: number;
+    isBigEndian: boolean;
+    data: Uint8Array<ArrayBuffer>;
+  }): void {
+    const pixelBytes = frame.width * frame.height * 4;
+    let rgba = this.#rawRgba;
+    if (!rgba || rgba.length !== pixelBytes) {
+      rgba = new Uint8ClampedArray(pixelBytes);
+      this.#rawRgba = rgba;
+    }
+    if (!this.#rawImageData || this.#rawImageData.width !== frame.width || this.#rawImageData.height !== frame.height) {
+      this.#rawImageData = new ImageData(rgba, frame.width, frame.height);
+    }
+
+    decodeRawImage(
+      {
+        encoding: frame.encoding,
+        width: frame.width,
+        height: frame.height,
+        step: frame.step,
+        is_bigendian: frame.isBigEndian,
+        data: frame.data,
+      },
+      rgba,
+      this.#rawDecodeOptions,
+    );
+
+    this.#disposeCachedBitmap();
+    this.#cachedFrame = {
+      kind: 'raw',
+      width: frame.width,
+      height: frame.height,
+      encoding: frame.encoding,
+      step: frame.step,
+      isBigEndian: frame.isBigEndian,
+      data: frame.data,
+      receiveTime: frame.receiveTime,
+    };
+
+    this.#drawRawImageData(frame.width, frame.height);
+    this.#emitStatus({
+      phase: 'ready',
+      width: frame.width,
+      height: frame.height,
+      encoding: frame.encoding,
+      receiveTime: frame.receiveTime,
+    });
+  }
+
   /** Re-decode the last raw frame with current rawDecodeOptions and redraw. */
   #redrawRawCached(): void {
     const cached = this.#cachedFrame;
@@ -611,7 +645,7 @@ class ImageRenderWorkerRuntime {
     data: Uint8Array<ArrayBuffer>,
     format: string,
   ): Promise<ImageBitmap | VideoFrame> {
-    const mime = normalizeCompressedMime(format);
+    const mime = normalizeCompressedMime(format, data);
     if (typeof ImageDecoder !== 'undefined') {
       let supported = this.#imageDecoderMimeSupported.get(mime);
       if (supported === undefined) {

src/features/panels/Image/core/compressedDepthDecoder.test.ts

@@ -0,0 +1,56 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import {
+  decodeCompressedDepth,
+  findPngOffset,
+  parseCompressedDepthHeader,
+} from './compressedDepthDecoder';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const FIXTURE_16UC1 = path.join(__dirname, '../../../../../test-fixtures/image/compressed-depth-16uc1.bin');
+const FIXTURE_32FC1 = path.join(__dirname, '../../../../../test-fixtures/image/compressed-depth-32fc1.bin');
+const FIXTURE_FORMAT_16UC1 = '16UC1; compressedDepth';
+
+describe('compressedDepthDecoder', () => {
+  it('finds the PNG signature after the ROS header', () => {
+    const fixture = fs.readFileSync(FIXTURE_16UC1);
+    expect(findPngOffset(fixture)).toBe(12);
+  });
+
+  it('decodes a RealSense 16UC1 compressed depth fixture', async () => {
+    const fixture = fs.readFileSync(FIXTURE_16UC1);
+    const decoded = await decodeCompressedDepth(fixture, FIXTURE_FORMAT_16UC1);
+
+    expect(decoded.encoding).toBe('16uc1');
+    expect(decoded.width).toBe(640);
+    expect(decoded.height).toBe(480);
+    expect(decoded.step).toBe(640 * 2);
+    expect(decoded.isBigEndian).toBe(false);
+
+    const view = new DataView(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
+    expect(view.getUint16(0, true)).toBe(3524);
+    expect(view.getUint16(2, true)).toBe(3565);
+  });
+
+  it('dequantizes 32FC1 compressed depth PNG payloads', async () => {
+    const payload = fs.readFileSync(FIXTURE_32FC1);
+    const header = parseCompressedDepthHeader(payload, findPngOffset(payload));
+    expect(header.depthParam).toEqual([10, 5]);
+
+    const decoded = await decodeCompressedDepth(payload, '32FC1; compressedDepth');
+    expect(decoded.encoding).toBe('32fc1');
+    expect(decoded.width).toBe(2);
+    expect(decoded.height).toBe(1);
+
+    const view = new DataView(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength);
+    expect(view.getFloat32(0, true)).toBeCloseTo(10 / (100 - 5), 5);
+    expect(view.getFloat32(4, true)).toBeCloseTo(10 / (200 - 5), 5);
+  });
+
+  it('rejects unsupported RVL compressed depth formats', async () => {
+    const fixture = fs.readFileSync(FIXTURE_16UC1);
+    await expect(decodeCompressedDepth(fixture, '16UC1; compressedDepth rvl')).rejects.toThrow(/RVL/);
+  });
+});