CUDA and GPU Verification

# labs/verify_cuda.py
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path


@dataclass
class Ran:
    """What one command did, including never having been there to run."""
    argv: list[str]
    found: bool = False
    code: int = -1
    out: str = ""
    err: str = ""


def run(argv: list[str], timeout: float = 15.0) -> Ran:
    """Look the binary up first, then execute it. Absence returns; it never raises."""
    ran = Ran(argv)
    binary = shutil.which(argv[0])
    if binary is None:
        ran.err = f"{argv[0]}: not found on PATH"
        return ran
    ran.found = True
    try:
        done = subprocess.run([binary, *argv[1:]], capture_output=True, text=True, timeout=timeout)
    except subprocess.TimeoutExpired:
        ran.err = f"{argv[0]} printed nothing in {timeout:.0f}s"
        return ran
    ran.code, ran.out, ran.err = done.returncode, done.stdout.strip(), done.stderr.strip()
    return ran


if __name__ == "__main__":
    for argv in (["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], ["nvcc", "--version"]):
        ran = run(argv)
        print(f"{argv[0]:<11} found={ran.found} code={ran.code} out={ran.out!r} err={ran.err!r}")

$ uv run python -m labs.verify_cuda   # on the Jetson, before any PATH edits
nvidia-smi  found=True code=0 out='Orin (nvgpu)' err=''
nvcc        found=False code=-1 out='' err="nvcc: not found on PATH"

found and code are deliberately separate. A tool that is absent and a tool that ran and failed need different responses from you, and collapsing both into "non-zero" throws away which one happened. The starting code of -1 is a value no process can exit with, so a row still showing it means nothing ever ran. shutil.which does the same search the shell does, over the PATH this Python process inherited, and it takes an absolute path too: given one, it checks that the file exists and is executable and hands it straight back. One function then serves both kinds of lookup. The timeout is there because nvidia-smi on a wedged driver does not fail, it hangs, and a diagnostic that hangs is worse than one that reports trouble.

# labs/verify_cuda.py — continued
from labs.jetpack_verify import Check, run_check


@dataclass
class Layer(Check):
    """A Check that also carries the command that would fix it."""
    fix: str = ""


def check_driver() -> Layer:
    layer = Layer("driver", "a GPU nvidia-smi can talk to")
    ran = run(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"])
    if not ran.found:
        layer.detected = "nvidia-smi is not installed"
        layer.fix = "reflash: the driver comes with the L4T image, so its absence is a flash fault"
        return layer
    if ran.code != 0:
        layer.detected = ran.err or f"nvidia-smi exited {ran.code} silently"
        return layer
    name, memory, driver = (f.strip() for f in ran.out.splitlines()[0].split(","))
    layer.detected = f"{name}, driver {driver}, memory field {memory}"
    layer.ok = True
    return layer

$ uv run python -m labs.verify_cuda   # measured on the bench — yours will vary
[  OK  ] driver   Orin (nvgpu), driver 540.4.0, memory field [N/A]

--format=csv,noheader turns a human dashboard into one comma-separated line, so three fields come out of one split with no parsing to maintain. Two of them read oddly on this board and both readings are right. The name is Orin (nvgpu) because the GPU is a block inside the same chip as the CPU and not a card on a bus. The memory total is [N/A] for the same reason: there is no separate pool of video memory to report, since the GPU and the CPU share the board's LPDDR5. A parser written on a desktop and moved here will happily turn [N/A] into a megabyte figure of zero and then decide the GPU is full. Print the field as it came.

# labs/verify_cuda.py — continued
import re

CUDA_ROOT = Path("/usr/local/cuda")


def check_toolkit() -> Layer:
    layer = Layer("toolkit", "nvcc the shell can find")
    on_path = shutil.which("nvcc")
    on_disk = CUDA_ROOT / "bin" / "nvcc"
    if on_path is None and not on_disk.is_file():
        layer.detected = f"no nvcc on PATH and no file at {on_disk}"
        layer.fix = "sudo apt install cuda-toolkit-12-6"
        return layer

    ran = run([on_path or str(on_disk), "--version"])
    release = re.search(r"release (\d+\.\d+), (V\S+)", ran.out)
    if release is None:
        layer.detected = f"nvcc ran, exit {ran.code}, no release line in its output"
        return layer

    layer.detected = f"CUDA {release.group(1)} ({release.group(2)}) at {ran.argv[0]}"
    layer.ok = on_path is not None
    if on_path is None:
        layer.fix = "echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc"
    return layer

$ uv run python -m labs.verify_cuda   # measured on the bench — your build number will vary
[  OK  ] driver   Orin (nvgpu), driver 540.4.0, memory field [N/A]
[ FAIL ] toolkit  CUDA 12.6 (V12.6.68) at /usr/local/cuda/bin/nvcc
                  fix: echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc

Read that failing row for what it says, because it is more useful than a plain "not found". The compiler exists, it ran, it identified itself as CUDA 12.6, and the row still fails, since the question this layer asks is whether the shell can find it. The two version numbers in the output belong to different counters: 12.6 is the release that packages are built against, and V12.6.68 is the compiler's own build. Compare the first and print the second, the same split Chapter 76 made for the SDK manifest.

Why is CUDA off the PATH at all? The driver's tools live in /usr/bin, which every shell searches by default. The toolkit installs under /usr/local/cuda, a versioned tree that lets several CUDAs sit side by side, and no installer can add one of them to your PATH without picking a winner for you. So it picks none, and the choice is left in a file you edit.

# labs/verify_cuda.py — continued
import ctypes


def mapped_file(fragment: str) -> str:
    """The file behind a library this process has already loaded, per /proc/self/maps."""
    with open("/proc/self/maps") as handle:
        for line in handle:
            path = line.rsplit(" ", 1)[-1].strip()
            if fragment in path:
                return path
    return "loaded, but no mapping names it"


def check_runtime() -> Layer:
    layer = Layer("runtime", "libcudart loadable by this process")
    try:
        ctypes.CDLL("libcudart.so.12")
    except OSError as exc:
        layer.detected = str(exc).strip()
        layer.fix = ("echo /usr/local/cuda/lib64 | sudo tee /etc/ld.so.conf.d/cuda.conf "
                     "&& sudo ldconfig")
        return layer
    layer.detected = mapped_file("libcudart")
    layer.ok = True
    return layer

$ uv run python -m labs.verify_cuda   # before the ldconfig fix
[ FAIL ] runtime  libcudart.so.12: cannot open shared object file: No such file or directory
                  fix: echo /usr/local/cuda/lib64 | sudo tee /etc/ld.so.conf.d/cuda.conf && sudo ldconfig

$ uv run python -m labs.verify_cuda   # after it
[  OK  ] runtime  /usr/local/cuda-12.6/targets/aarch64-linux/lib/libcudart.so.12.6.68

This is the layer neither of the first two can see. When a Python extension module starts, the dynamic loader resolves its library dependencies against a compiled cache built by ldconfig, plus anything in LD_LIBRARY_PATH and any search path baked into the binary. PATH plays no part in it. That is why fixing the compiler lookup does nothing for a library import, and why the repair here is a file under /etc/ld.so.conf.d/ instead of a line in a shell profile: the cache is system-wide, so a service starting at boot with no shell and no profile gets the same answer you do.

ctypes.CDLL performs that resolution on demand and raises OSError with the loader's own message when it fails, which makes it the cheapest honest test available. The successful row then reads the answer back out of /proc/self/maps, the kernel's list of what this process has mapped into memory. Notice the path is not the one added to the config: it is a real file at the end of a chain of symlinks, with the full build number in its name. Where two CUDAs are installed, that line is what tells you which one won.

# labs/verify_cuda.py — continued
import sys


def check_engine() -> Layer:
    layer = Layer("engine", "at least one CUDA device visible to ctranslate2")
    try:
        import ctranslate2
    except ImportError as exc:
        layer.detected = f"ctranslate2 will not import: {exc}"
        layer.fix = "uv sync"
        return layer
    count = ctranslate2.get_cuda_device_count()
    layer.detected = f"{count} CUDA device(s), ctranslate2 {ctranslate2.__version__}"
    layer.ok = count > 0
    if count == 0:
        layer.fix = 'never set device="auto" here: with 0 devices it picks CPU and says nothing'
    return layer


LAYERS = [check_driver, check_toolkit, check_runtime, check_engine]


def report(layers: list[Check]) -> bool:
    width = max(len(layer.name) for layer in layers)
    for layer in layers:
        print(f"[{'  OK  ' if layer.ok else ' FAIL '}] {layer.name:<{width}}  {layer.detected}")
        fix = getattr(layer, "fix", "")
        if fix and not layer.ok:
            print(f"{'':<{width + 11}}fix: {fix}")
    usable = sum(1 for layer in layers if layer.ok)
    print(f"{usable}/{len(layers)} layers usable by the process that asked.")
    return usable == len(layers)


if __name__ == "__main__":
    sys.exit(0 if report([run_check(fn) for fn in LAYERS]) else 1)

$ uv run python -m labs.verify_cuda; echo "exit=$?"   # measured on the bench — yours will vary
[  OK  ] driver   Orin (nvgpu), driver 540.4.0, memory field [N/A]
[  OK  ] toolkit  CUDA 12.6 (V12.6.68) at /usr/local/cuda/bin/nvcc
[  OK  ] runtime  /usr/local/cuda-12.6/targets/aarch64-linux/lib/libcudart.so.12.6.68
[  OK  ] engine   1 CUDA device(s), ctranslate2 4.5.0
4/4 layers usable by the process that asked.
exit=0

The last layer is the only one that asks the software you actually run. CTranslate2 is the engine underneath faster-whisper, and get_cuda_device_count reports how many devices this build of it can see, which is a narrower claim than the driver's. A wheel compiled without CUDA support returns 0 on a board with a perfect driver, and so does a build whose cuDNN requirement the loader cannot satisfy. Volume 1 constructed WhisperModel with device="cpu", since a laptop had nothing else to offer. The tempting edit on this board is device="auto", and a 0 here is exactly the case where "auto" quietly resolves to CPU, transcribes correctly, and takes many times longer with no message anywhere. Name the device you demand and let it fail loudly.

run_check converts a raised exception into a plain Check, not a Layer, so the printer reaches for fix with getattr and a default. A check that died has no repair to suggest anyway; it has a traceback in its detected field, and that is the more useful thing to read.