Running Larger Models

# labs/model_catalog.py — the candidates, written as numbers instead of adjectives
BITS_PER_WEIGHT = {"q4_K_M": 4.6, "q5_K_M": 5.6, "q8_0": 8.5, "f16": 16.0}

MODELS: list[dict] = [
    {"name": "qwen2.5:0.5b", "params_b": 0.5, "quant": "q4_K_M", "disk_gb": 0.4,
     "layers": 24, "kv_heads": 2, "head_dim": 64},
    {"name": "gemma2:2b", "params_b": 2.6, "quant": "q4_K_M", "disk_gb": 1.6,
     "layers": 26, "kv_heads": 4, "head_dim": 256},
    {"name": "llama3.2:3b", "params_b": 3.2, "quant": "q4_K_M", "disk_gb": 2.0,
     "layers": 28, "kv_heads": 8, "head_dim": 128},
    {"name": "llama3.1:8b", "params_b": 8.0, "quant": "q4_K_M", "disk_gb": 4.9,
     "layers": 32, "kv_heads": 8, "head_dim": 128},
    {"name": "qwen2.5:14b", "params_b": 14.8, "quant": "q4_K_M", "disk_gb": 9.0,
     "layers": 48, "kv_heads": 8, "head_dim": 128},
]


def weight_gb(params_b: float, quant: str) -> float:
    """Billions of parameters at so many bits each, in gigabytes of file."""
    return params_b * 1e9 * BITS_PER_WEIGHT[quant] / 8 / 1e9


if __name__ == "__main__":
    print(f"{'model':<14}{'params':>8}{'quant':>9}{'predicted':>11}{'pulled':>9}")
    for m in MODELS:
        print(f"{m['name']:<14}{m['params_b']:>7.1f}B{m['quant']:>9}"
              f"{weight_gb(m['params_b'], m['quant']):>10.1f}G{m['disk_gb']:>8.1f}G")

$ uv run python -m labs.model_catalog
model           params    quant  predicted   pulled
qwen2.5:0.5b      0.5B   q4_K_M       0.3G     0.4G
gemma2:2b         2.6B   q4_K_M       1.5G     1.6G
llama3.2:3b       3.2B   q4_K_M       1.8G     2.0G
llama3.1:8b       8.0B   q4_K_M       4.6G     4.9G
qwen2.5:14b      14.8B   q4_K_M       8.5G     9.0G

That one function is the bridge between the number on the box and the number that matters. A parameter is a stored value, and how many bytes it occupies is a separate decision made when the file was built. Ollama serves 4-bit K-quantised weights by default, and the constant here says 4.6 rather than 4.0 because the K schemes are mixed: most of the network is packed into 4-bit blocks, while the embedding and output tensors, the ones the answer quality is most sensitive to, are kept at higher precision. Averaged over the whole file that lands near 4.6 bits per weight.

Read the two right-hand columns as a check on the arithmetic, not as a table of facts. From 2.6B upwards the prediction sits within about ten percent of the real file and is always the smaller of the two, because a downloaded model also carries a tokeniser, a vocabulary and metadata. On the 0.5B row the prediction is out by a third: the fixed tensors are a large fraction of a tiny model, so a flat rate per weight has nothing left to average over. Trust the pulled figure for anything on disk, and use weight_gb to sketch a model you have not downloaded yet.

# labs/model_catalog.py — continued
CONTEXT_TOKENS = 4096      # the window Ollama opens by default on this board
KV_BYTES = 2               # the cache is kept in float16
POOL_GB = 8.0
RESIDENT_GB = 1.9          # the OS and her own stack, measured with the loop idle
RESERVE_GB = 1.0           # held back so a burst never meets the OOM killer
HEADROOM_GB = POOL_GB - RESIDENT_GB
BUDGET_GB = HEADROOM_GB - RESERVE_GB


def kv_cache_gb(model: dict, context_tokens: int = CONTEXT_TOKENS) -> float:
    """One key and one value per head, per layer, per token in the window."""
    per_token = 2 * model["layers"] * model["kv_heads"] * model["head_dim"] * KV_BYTES
    return per_token * context_tokens / 1e9


def footprint_gb(model: dict, context_tokens: int = CONTEXT_TOKENS) -> float:
    return model["disk_gb"] + kv_cache_gb(model, context_tokens)


def fit(model: dict, context_tokens: int = CONTEXT_TOKENS) -> tuple[str, float]:
    """Where this model lands against the budget, and by how much."""
    used = footprint_gb(model, context_tokens)
    if used <= BUDGET_GB:
        return "fits", BUDGET_GB - used
    if used <= HEADROOM_GB:
        return "tight", HEADROOM_GB - used
    return "over", used - HEADROOM_GB


VERDICTS = {"fits": "fits, {:.2f} GB under budget",
            "tight": "tight, {:.2f} GB left above the OS",
            "over": "over by {:.2f} GB"}

if __name__ == "__main__":
    print(f"budget {BUDGET_GB:.1f} GB: an {POOL_GB:.1f} GB pool, "
          f"{RESIDENT_GB:.1f} GB resident, {RESERVE_GB:.1f} GB held back")
    print(f"{'model':<14}{'weights':>9}{'kv@' + str(CONTEXT_TOKENS):>9}{'total':>8}   verdict")
    for m in MODELS:
        verdict, margin = fit(m)
        print(f"{m['name']:<14}{m['disk_gb']:>9.1f}{kv_cache_gb(m):>9.2f}"
              f"{footprint_gb(m):>8.2f}   {VERDICTS[verdict].format(margin)}")

$ uv run python -m labs.model_catalog
budget 5.1 GB: an 8.0 GB pool, 1.9 GB resident, 1.0 GB held back
model           weights  kv@4096   total   verdict
qwen2.5:0.5b        0.4     0.05    0.45   fits, 4.65 GB under budget
gemma2:2b           1.6     0.44    2.04   fits, 3.06 GB under budget
llama3.2:3b         2.0     0.47    2.47   fits, 2.63 GB under budget
llama3.1:8b         4.9     0.54    5.44   tight, 0.66 GB left above the OS
qwen2.5:14b         9.0     0.81    9.81   over by 3.71 GB

The cache calculation is worth walking through once, because every term in it is something you can look up. Attention works by comparing the token being generated against a key and a value for every token before it, and recomputing those from scratch each time would make a long conversation quadratically slow, so they are computed once and kept. Two tensors, per attention head that stores them, per layer, per token in the window, at two bytes each. For llama3.2:3b that is 2 × 28 × 8 × 128 × 2, or 114,688 bytes for every token the window can hold, and a full 4,096-token window therefore costs 0.47 GB on top of the weights.

Notice that the cache column does not track the weights column. gemma2:2b is a fifth smaller on disk than llama3.2:3b and its cache is almost the same size, because it stores 256-wide heads where the Llama model stores 128-wide ones. Nothing on a download page says so.

The reserve is the other correction to chapter 75. That chapter asked whether the weights fit and answered yes with 1.2 GB spare. Count the window, hold a gigabyte back for the transcription and speech models that both spike during a turn, and the 8B model has 0.66 GB standing between it and the kernel's out-of-memory killer, which does not negotiate or warn. The three verdicts exist so that condition has a name instead of being a positive margin nobody looked at twice.

# labs/model_rates.py — the other half of the trade: what each row costs in time
import glob
import json

from labs.model_catalog import BUDGET_GB, MODELS, fit, footprint_gb

BANDWIDTH_GB_S = 102.0          # this board's peak, off the spec sheet
EFFICIENCY = 0.36               # measured decode over ceiling, on this board
FITTED_BAND_GB = (2.0, 4.9)     # the two weight sizes that efficiency came from
BOARD = "jetson-orin-nano-super"
RESULTS_DIR = "glados/data/benchmarks"

REPLY_TOKENS = 27               # decode tokens implied by the benchmark's stage mean
STAGE_CEILING_MS = 3000         # the llm slice of the turn budget
PREFILL_ALLOWANCE_MS = 350      # 96 prompt tokens at the measured 287.67 t/s, rounded up
FLOOR_TPS = REPLY_TOKENS / ((STAGE_CEILING_MS - PREFILL_ALLOWANCE_MS) / 1000)


def ceiling_tps(disk_gb: float) -> float:
    """Every weight is read once per token: bandwidth over weight bytes."""
    return BANDWIDTH_GB_S / disk_gb


def measured_tps(pattern: str = f"{RESULTS_DIR}/*.json") -> dict[str, float]:
    """Decode means for this board out of the benchmark archive, keyed by model."""
    rates: dict[str, float] = {}
    for path in sorted(glob.glob(pattern)):
        with open(path) as handle:
            record = json.load(handle)
        if record["board"] == BOARD:
            rates[record["model"]] = record["decode_tps"]["mean"]
    return rates


def rate_of(model: dict, rates: dict[str, float]) -> tuple[float, str]:
    """The best rate available for this model, and where it came from."""
    if model["name"] in rates:
        return rates[model["name"]], "measured"
    low, high = FITTED_BAND_GB
    estimate = ceiling_tps(model["disk_gb"]) * EFFICIENCY
    if low / 2 <= model["disk_gb"] <= high * 2:
        return estimate, "estimate"
    return estimate, "extrapolated"

Three words in the last column of the finished table, and they are not decoration. A measured rate came off this board and is in a file with a date on it. An estimate is the ceiling from chapter 75, bandwidth divided by weight bytes, multiplied by the 0.36 of peak this board actually reached. An extrapolated rate is the same arithmetic applied outside the range it was fitted in, which is a different kind of claim and deserves a different word. The band rule is the ordinary one for a fitted constant: trust it within a factor of two of the data it came from, and stop there.

# labs/model_rates.py — continued
def speed_of(model: dict, rates: dict[str, float]) -> str:
    """One word for the rate column: a floor comparison, or an admission."""
    tps, source = rate_of(model, rates)
    if source == "extrapolated":
        return "unknown"
    return "ok" if tps >= FLOOR_TPS else "slow"


def recommend(rates: dict[str, float]) -> dict:
    """The most capable row that clears both budgets, capability read as parameters."""
    ok = [m for m in MODELS
          if fit(m)[0] == "fits" and speed_of(m, rates) == "ok"]
    return max(ok, key=lambda m: m["params_b"])


def dashboard() -> None:
    rates = measured_tps()
    print("=" * 70)
    print(f"  Model budget {BUDGET_GB:.1f} GB   floor {FLOOR_TPS:.1f} t/s   "
          f"peak {BANDWIDTH_GB_S:.0f} GB/s")
    print("=" * 70)
    print(f"  {'model':<14}{'total GB':>9}  {'memory':<8}{'ceiling':>8}{'t/s':>7}  "
          f"{'source':<14}speed")
    for m in MODELS:
        tps, source = rate_of(m, rates)
        print(f"  {m['name']:<14}{footprint_gb(m):>9.2f}  {fit(m)[0]:<8}"
              f"{ceiling_tps(m['disk_gb']):>8.1f}{tps:>7.1f}  {source:<14}"
              f"{speed_of(m, rates)}")
    print("-" * 70)
    print(f"  recommended: {recommend(rates)['name']}")
    print("=" * 70)


if __name__ == "__main__":
    dashboard()

$ uv run python -m labs.model_rates   # two rates from the bench archive — yours will vary
======================================================================
  Model budget 5.1 GB   floor 10.2 t/s   peak 102 GB/s
======================================================================
  model          total GB  memory   ceiling    t/s  source        speed
  qwen2.5:0.5b       0.45  fits       255.0   91.8  extrapolated  unknown
  gemma2:2b          2.04  fits        63.8   22.9  estimate      ok
  llama3.2:3b        2.47  fits        51.0   18.4  measured      ok
  llama3.1:8b        5.44  tight       20.8    7.4  measured      slow
  qwen2.5:14b        9.81  over        11.3    4.1  estimate      slow
----------------------------------------------------------------------
  recommended: llama3.2:3b
======================================================================

The floor is the one number here that could have been an opinion and is not. Every term in it was measured somewhere earlier in the book. Chapter 61 gave the language-model stage 3,000 milliseconds of the turn. The benchmark in chapter 80 timed that stage at a mean of 1,807.6 milliseconds while decoding at 18.4 tokens a second, and its 96-token prompt costs 334 milliseconds at the prefill rate this board measured, so the decoding part was 1,473.6 milliseconds and her typical reply is 27 tokens long. Twenty-seven tokens in the 2.65 seconds left after prefill is 10.2 tokens a second. Change the turn budget or the length of her replies and the floor moves on its own.

Two of those columns deserve an argument. The 91.8 in the top row is arithmetic, not a prediction: at that rate a token would take eleven milliseconds, and the fixed costs per token, sampling, the HTTP round trip, Python itself, stop being negligible long before then. The word unknown beside it is the honest output, and it keeps that row out of the recommendation until somebody benchmarks it. And llama3.1:8b is the row this whole chapter exists for: it loads, it answers well, it is the most capable thing on the board, and at 7.4 tokens a second it puts the language stage 1,471 milliseconds over its budget. Capability and latency are separate purchases and this board cannot afford both.

The recommendation falls out of the data with no branching in it: among the rows whose memory verdict is fits and whose speed is ok, take the one with the most parameters. It picks the model she is already running. That result is worth more now than it was in volume 1, because the reason has changed from "it was all that fit" to a floor, a budget and two measurements.

# labs/pull_model.py — the menu, and the checks standing between it and a download
import subprocess

from labs.model_catalog import MODELS, VERDICTS, fit
from labs.model_rates import FLOOR_TPS, dashboard, measured_tps, rate_of


def check_choice(raw: str, rates: dict[str, float]) -> tuple[int, list[str]]:
    """Validate what the reader typed, returning every reason to refuse it."""
    try:
        number = int(raw)
    except ValueError:
        return 0, [f"{raw!r} is not a number"]
    if not 1 <= number <= len(MODELS):
        return 0, [f"{number} is not on the menu (1 to {len(MODELS)})"]

    model, reasons = MODELS[number - 1], []
    verdict, margin = fit(model)
    if verdict == "over":
        reasons.append(f"{model['name']} is {VERDICTS['over'].format(margin)}")
    tps, source = rate_of(model, rates)
    if source == "extrapolated":
        reasons.append(f"{model['name']} has no rate this board has measured")
    elif tps < FLOOR_TPS:
        reasons.append(f"{model['name']} decodes at {tps:.1f} t/s, under the "
                       f"{FLOOR_TPS:.1f} t/s floor")
    return number, reasons


def main() -> None:
    rates = measured_tps()
    dashboard()
    for index, model in enumerate(MODELS, 1):
        print(f"  {index}. {model['name']}")
    try:
        raw = input("\n  pull which model? (q to quit) ").strip()
    except (EOFError, KeyboardInterrupt):
        print("\n  nothing pulled.")
        return
    if raw.lower() == "q":
        return

    number, reasons = check_choice(raw, rates)
    for reason in reasons:
        print(f"  refused: {reason}")
    if reasons:
        return
    name = MODELS[number - 1]["name"]
    print(f"\n  pulling {name}")
    subprocess.run(["ollama", "pull", name], check=True)


if __name__ == "__main__":
    main()

$ ssh -t glados-jetson uv run python -m labs.pull_model
  1. qwen2.5:0.5b
  2. gemma2:2b
  3. llama3.2:3b
  4. llama3.1:8b
  5. qwen2.5:14b

  pull which model? (q to quit) 5
  refused: qwen2.5:14b is over by 3.71 GB
  refused: qwen2.5:14b decodes at 4.1 t/s, under the 10.2 t/s floor

Both reasons print. That is the same decision the config validator made in chapter 34, for the same reason: a function that stops at the first problem makes the reader fix one thing, run again, and discover the next one, and a function that returns a list tells them everything wrong with the choice in a single pass. An empty list is the only unambiguous permission to download nine gigabytes.

subprocess.run gets the list form, so the model name is handed to the program as one argument and never reaches a shell that could split it on the colon. check=True is the other half: a pull that fails halfway through, and they do, on a board whose Wi-Fi is a USB adapter, raises instead of letting the script print a success line over a partial download.