Judging Distance From One Camera

# glados/depth.py
"""Distance from one camera: centimetres where the size is known, ranking everywhere else."""
import numpy as np

FOCAL_PX = 605.0    # this webcam at 640x480. Yours will be a different number.


def focal_length_px(real_height_cm: float, pixel_height: float,
                    distance_cm: float) -> float:
    """Focal length in pixels, from one object of known height at a measured distance."""
    if real_height_cm <= 0 or distance_cm <= 0:
        raise ValueError("calibration needs a positive height and a positive distance")
    return pixel_height * distance_cm / real_height_cm


if __name__ == "__main__":
    focal = focal_length_px(real_height_cm=20.0, pixel_height=121, distance_cm=100.0)
    print(f"focal length: {focal:.1f} px")

$ uv run python -m glados.depth   # measured on the bench: yours will vary
focal length: 605.0 px

Stand a 20 cm ruler upright exactly one metre from the camera and count how many rows of pixels it covers. On the bench that was 121, so 121 times 100 divided by 20 is 605, and that number now describes the camera, not the ruler or the metre. It holds until the hardware changes. It will change if you switch resolution, because 640 columns spread across the same view are twice as wide as 1280, so calibrate at the resolution you intend to run and record both numbers together.

Two habits keep the measurement honest. Run the tape from the sensor, not the front rim of the housing: the equation counts distance to the lens, and a couple of centimetres of plastic is pure error at close range. And stand the object square to the camera, because a ruler leaning away covers fewer rows than an upright one, so a tilt reads as extra distance and never announces itself.

def distance_cm(real_height_cm: float, pixel_height: float,
                focal_px: float = FOCAL_PX) -> float:
    """How far away an object of known real height is, from how tall it looks."""
    if pixel_height <= 0:
        raise ValueError(f"pixel height must be positive, got {pixel_height}")
    return real_height_cm * focal_px / pixel_height


if __name__ == "__main__":
    focal = focal_length_px(real_height_cm=20.0, pixel_height=121, distance_cm=100.0)
    for tall in (242, 121, 81, 61, 40):
        near = distance_cm(20.0, tall, focal)
        far = distance_cm(20.0, tall + 1, focal)
        print(f"  {tall:3d} px -> {near:6.1f} cm   one pixel of measurement error: "
              f"{near - far:5.2f} cm")

$ uv run python -m glados.depth
  242 px ->   50.0 cm   one pixel of measurement error:  0.21 cm
  121 px ->  100.0 cm   one pixel of measurement error:  0.82 cm
   81 px ->  149.4 cm   one pixel of measurement error:  1.82 cm
   61 px ->  198.4 cm   one pixel of measurement error:  3.20 cm
   40 px ->  302.5 cm   one pixel of measurement error:  7.38 cm

Check any row by hand. 20 times 605 is 12,100, and 12,100 divided by 242 is 50.0 exactly. Divided by 40 it is 302.5. Nothing in that column came from a model or a calibration file; it is one multiplication and one division, and if the answer is wrong you can find out why.

The right-hand column decides where this stops being useful. Pixel height sits on the bottom of the fraction, so losing one costs little up close and a lot far away: a fifth of a centimetre at half a metre, over seven at three. Miscount by three pixels across the room and you are a hand's width out. Use it in the near field where the arm works, and treat far readings as rough.

The other limit is the one in the rule: you need the real height, and it has to be fixed. A mug qualifies, a printed marker qualifies, a box you measured qualifies. A hand does not, a cat does not, and a person only qualifies if you accept being wrong by the gap between the tallest and shortest people who walk in. The object also has to be fully in frame, top and bottom, or the pixel height belongs to something other than what you meant.

import cv2
import torch

MODEL_TYPE = "MiDaS_small"


def load_midas(model_type: str = MODEL_TYPE):
    """Fetch the model and its input transform. Returns (model, transform, device)."""
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = torch.hub.load("intel-isl/MiDaS", model_type, trust_repo=True)
    model.to(device).eval()
    hub = torch.hub.load("intel-isl/MiDaS", "transforms", trust_repo=True)
    transform = hub.small_transform if model_type == "MiDaS_small" else hub.dpt_transform
    return model, transform, device


def normalize(raw: np.ndarray) -> np.ndarray:
    """Rescale to 0.0 farthest, 1.0 nearest. A flat map stays flat instead of dividing by zero."""
    low, high = float(raw.min()), float(raw.max())
    if high - low < 1e-6:
        return np.zeros_like(raw, dtype=np.float32)
    return np.clip((raw - low) / (high - low), 0.0, 1.0).astype(np.float32)


def estimate_depth(frame_bgr: np.ndarray, model, transform, device) -> np.ndarray:
    """One forward pass. Returns a float32 map the same height and width as the frame."""
    rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
    with torch.no_grad():
        prediction = model(transform(rgb).to(device))
        prediction = torch.nn.functional.interpolate(
            prediction.unsqueeze(1),
            size=frame_bgr.shape[:2],
            mode="bicubic",
            align_corners=False,
        ).squeeze()
    return normalize(prediction.cpu().numpy().astype(np.float32))


if __name__ == "__main__":
    from glados.vision import capture_frame

    model, transform, device = load_midas()
    frame = capture_frame()
    depth = estimate_depth(frame, model, transform, device)
    print(f"device: {device}")
    print(f"frame:  {frame.shape} {frame.dtype}")
    print(f"depth:  {depth.shape} {depth.dtype}")
    print(f"min {depth.min():.3f}  max {depth.max():.3f}  mean {depth.mean():.3f}")

$ uv run python -m glados.depth   # measured on the bench: yours will vary
Using cache found in /home/glados/.cache/torch/hub/intel-isl_MiDaS_master
device: cuda
frame:  (480, 640, 3) uint8
depth:  (480, 640) float32
min 0.000  max 1.000  mean 0.374

Four lines there are doing work you cannot skip. The colour conversion comes first because MiDaS was trained on red-green-blue while OpenCV hands you blue-green-red, and the wrong order returns a worse map with no error, the same silent swap chapter 86 warned about at the JPEG boundary. interpolate matters because the model predicts at its own input size, 256 pixels a side for the small variant, and resizing back to 480 by 640 is what makes depth[y, x] describe the same spot as frame[y, x]. no_grad switches off the bookkeeping PyTorch keeps for training. normalize clips into 0 to 1 so every map, from any source, arrives on one scale.

Now the sentence this chapter exists to make you believe. MiDaS returns relative depth: a ranking of the pixels from far to near, with no unit attached. A value of 0.71 says this pixel sits near the front of this frame. It does not say 71 cm, or 71% of anything, and no factor you apply afterwards turns it into centimetres. The arithmetic in stage 2 measures. This ranks.

    model, transform, device = load_midas()
    input("aim at the mug on the desk, then press enter: ")
    before = estimate_depth(capture_frame(), model, transform, device)
    input("roll a chair into the foreground, do not touch the mug, press enter: ")
    after = estimate_depth(capture_frame(), model, transform, device)
    for label, depth in (("before", before), ("after ", after)):
        print(f"{label}  mug centre (300, 240): {depth[240, 300]:.2f}   "
              f"frame mean: {depth.mean():.3f}")

$ uv run python -m glados.depth   # measured on the bench: yours will vary
aim at the mug on the desk, then press enter:
roll a chair into the foreground, do not touch the mug, press enter:
before  mug centre (300, 240): 0.71   frame mean: 0.374
after   mug centre (300, 240): 0.52   frame mean: 0.408

The mug did not move a millimetre and its number fell by nearly two tenths. Two things caused that, and both are the ranking working correctly. The chair is now the nearest object, so it takes the top of the range and everything else slides beneath it. The model's own output shifts as well, because it may place the whole scene anywhere on an arbitrary scale, and the chair changed what the scene is.

So: compare pixels inside one frame, never the same pixel across two. A threshold written against a fixed value like 0.6 will fire and stop firing as objects wander in and out of the picture, and it will look like a sensor fault when it is the definition of the output.

# labs/depth_cost.py
import time

import numpy as np

from glados.depth import MODEL_TYPE, estimate_depth, load_midas


def used_gb() -> float:
    """Memory in use across the Jetson's one shared pool, straight from /proc/meminfo."""
    field: dict[str, int] = {}
    with open("/proc/meminfo", encoding="ascii") as meminfo:
        for line in meminfo:
            key, value = line.split(":", 1)
            field[key] = int(value.split()[0])          # kilobytes
    return (field["MemTotal"] - field["MemAvailable"]) / 1024 / 1024


def pool_gb() -> float:
    """Total pool Linux can see; MemTotal is the first line of the file."""
    with open("/proc/meminfo", encoding="ascii") as meminfo:
        return int(meminfo.readline().split()[1]) / 1024 / 1024


if __name__ == "__main__":
    frame = np.random.default_rng(0).integers(0, 255, (480, 640, 3), dtype=np.uint8)
    print(f"pool:            {pool_gb():.2f} GB usable")
    print(f"before torch:    {used_gb():.2f} GB in use")
    model, transform, device = load_midas(MODEL_TYPE)
    print(f"model loaded:    {used_gb():.2f} GB in use")
    started = time.monotonic()
    estimate_depth(frame, model, transform, device)
    print(f"first frame:     {used_gb():.2f} GB in use, {time.monotonic() - started:.1f} s")
    started = time.monotonic()
    for _ in range(10):
        estimate_depth(frame, model, transform, device)
    print(f"ten more:        {(time.monotonic() - started) / 10:.2f} s each")

$ uv run python -m labs.depth_cost   # measured on the bench: yours will vary
pool:            7.44 GB usable
before torch:    3.62 GB in use
model loaded:    5.21 GB in use
first frame:     5.88 GB in use, 1.9 s
ten more:        0.09 s each

The weights are 80 MB and the process grew by 2.26 GB. Almost all of that is PyTorch itself plus the CUDA context the driver builds the first time a tensor touches the GPU, paid once per process and not once per frame. So keep the depth source alive inside her running service: a script that starts Python, loads the model, answers once and exits pays the whole bill for every single look.

The first frame took 1.9 seconds and the next ten took under a tenth each, which is the GPU compiling its kernels on the way through. If you want a fair number for any accelerator, throw the first result away.

Put that beside the pool. Eight gigabytes are fitted, 7.44 reach Linux because the firmware reserves the rest before boot, and CPU and GPU share every one of them. Her chat model already held 3.6, this takes 2.3 more, and about 1.6 are left. Chapter 86's ollama ps showed llava:7b resident at 6.0 GB, so the describing model and the ranking model cannot both be up on this board. Keep one loaded and let the other pay a reload, or drop to a smaller vision model. The measurement turns a crash into a decision.

NEAR_PERCENTILE = 95.0
ALERT_LEVEL = 0.75


def near_level(depth: np.ndarray, percentile: float = NEAR_PERCENTILE) -> float:
    """How near the nearest slice of the frame is, on this frame's own scale."""
    return float(np.percentile(depth, percentile))


def something_close(depth: np.ndarray, level: float = ALERT_LEVEL) -> bool:
    return near_level(depth) >= level


def nearest_pixel(depth: np.ndarray) -> tuple[int, int]:
    """(x, y) of the nearest region. Column first, the order OpenCV draws in."""
    row, col = np.unravel_index(int(np.argmax(depth)), depth.shape)
    return int(col), int(row)


if __name__ == "__main__":
    far = np.full((100, 100), 0.20, dtype=np.float32)
    speck = far.copy()
    speck[50, 50] = 1.0                 # one pixel in ten thousand
    blob = far.copy()
    blob[20:60, 20:60] = 0.95           # 1,600 pixels, 16% of the frame
    for name, depth in (("empty", far), ("one speck", speck), ("a real blob", blob)):
        print(f"{name:12s} near {near_level(depth):.2f}   close? {something_close(depth)}")
    print(f"nearest pixel (x, y): {nearest_pixel(blob)}")

$ uv run python -m glados.depth
empty        near 0.20   close? False
one speck    near 0.20   close? False
a real blob  near 0.95   close? True
nearest pixel (x, y): (20, 20)

The middle row is the reason for the percentile. One pixel at 1.0 is the brightest thing in the frame, and the maximum would report it as an object arriving; the 95th percentile ignores it, since one pixel is nowhere near the top five per cent of ten thousand. Moving that number takes a region of more than 500 pixels, which is about what a hand at arm's length covers and nothing like a speck of sensor noise. The percentile is as much a tuning knob as the level.

These three functions cover what a ranking honestly supports. Is anything close enough that the arm should hold still? Which of the two people in the doorway is nearer? Where is the closest thing, so the eye servos from volume 7 can turn toward it? Every one is a comparison inside one frame and none needs a unit. What none of them answers is the question chapter 94 asks, which wants a target in centimetres before it can solve for shoulder and elbow angles. When she has to reach, the two halves of this chapter pair up: the ranking picks which object, and the known-size arithmetic says how far.

from typing import Protocol


class DepthSource(Protocol):
    def depth(self, frame_bgr: np.ndarray) -> np.ndarray: ...


class MidasDepth:
    """The model. Needs torch, weights on disk, and about 2 GB of the pool."""

    def __init__(self, model_type: str = MODEL_TYPE) -> None:
        self.model, self.transform, self.device = load_midas(model_type)

    def depth(self, frame_bgr: np.ndarray) -> np.ndarray:
        return estimate_depth(frame_bgr, self.model, self.transform, self.device)


class SharpnessDepth:
    """No model at all: crisp regions read as near. Cheap, and easily fooled."""

    def __init__(self, blur: int = 31) -> None:
        self.blur = blur

    def depth(self, frame_bgr: np.ndarray) -> np.ndarray:
        gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
        edges = np.abs(cv2.Laplacian(gray, cv2.CV_64F)).astype(np.float32)
        return normalize(cv2.GaussianBlur(edges, (self.blur, self.blur), 0))


def open_depth_source() -> DepthSource:
    try:
        return MidasDepth()
    except Exception as err:
        print(f"depth model unavailable ({err}); falling back to sharpness")
        return SharpnessDepth()


if __name__ == "__main__":
    rng = np.random.default_rng(0)
    frame = np.full((240, 320, 3), 40, dtype=np.uint8)                    # a blank wall
    frame[80:160, 100:220] = rng.integers(0, 255, (80, 120, 3), dtype=np.uint8)
    depth = SharpnessDepth().depth(frame)
    print(f"{depth.shape} {depth.dtype}  min {depth.min():.3f}  max {depth.max():.3f}")
    print(f"textured patch {depth[120, 160]:.2f}   flat corner {depth[10, 10]:.2f}")

$ uv run python -m glados.depth
(240, 320) float32  min 0.000  max 1.000
textured patch 0.96   flat corner 0.00

Both classes take a BGR frame and return a normalized float32 map the size of that frame. Nothing downstream, not near_level, not nearest_pixel, not the eye, learns which one it holds. On the Jetson you get the model; on a board with no torch and no room for one you get a Laplacian sharpness estimate, which measures how fast brightness changes from pixel to pixel, blurs that into regions, and calls the crisp ones near. It needs a millisecond of CPU and no network.

Be clear-eyed about how much worse the fallback is. It assumes the lens focused on something near, so anything soft must be far, and a poster on a wall breaks that instantly. Treat it as a stand-in for "is there anything at all in front of me". The interface keeps the honesty in one place: the class says what it can do, and the rest of her code stays put when you upgrade the board.