Hearing Tone, Not Just Words

# glados/prosody.py
"""How something was said: four numbers off the buffer Whisper reads the words from."""
import math

import numpy as np

SAMPLE_RATE = 16000                              # the capture rate since chapter 4
FRAME_MS = 30
FRAME_SAMPLES = SAMPLE_RATE * FRAME_MS // 1000   # 480
FRAME_BYTES = FRAME_SAMPLES * 2                  # int16 audio: two bytes a sample
FULL_SCALE = 32768.0


def to_float(pcm: bytes) -> np.ndarray:
    """int16 bytes from the recorder, as floats between -1.0 and 1.0."""
    return np.frombuffer(pcm, dtype=np.int16).astype(np.float64) / FULL_SCALE


def rms(samples: np.ndarray) -> float:
    """Square every sample, take the mean, take the root."""
    if samples.size == 0:
        return 0.0
    return float(np.sqrt(np.mean(samples ** 2)))


def dbfs(level: float) -> float:
    """That level in decibels, 0 dB at full scale. Digital silence floors at -100."""
    if level <= 0.0:
        return -100.0
    return 20.0 * math.log10(level)


if __name__ == "__main__":
    n = np.arange(FRAME_SAMPLES)
    silence = np.zeros(FRAME_SAMPLES)
    square = np.where((n // 40) % 2 == 0, 0.1, -0.1)             # 200 Hz, always ±0.1
    sine = 0.1 * np.sin(2 * np.pi * 200 * n / SAMPLE_RATE + 0.5)  # 200 Hz, peak 0.1
    for name, frame in (("silence", silence), ("square 0.1", square), ("sine 0.1", sine)):
        level = rms(frame)
        print(f"{name:11s} rms {level:.4f}   {dbfs(level):7.2f} dBFS")

$ uv run python -m glados.prosody
silence     rms 0.0000   -100.00 dBFS
square 0.1  rms 0.1000    -20.00 dBFS
sine 0.1    rms 0.0707    -23.01 dBFS

Every row is checkable without running anything. The square wave sits at plus or minus 0.1 for all 480 samples, so every square is 0.01, the mean of 480 copies of 0.01 is 0.01, and its root is 0.1. The sine touches 0.1 only at its peaks, and the mean of a squared sine over whole cycles is exactly one half, so its RMS is 0.1 divided by the square root of two. Same peak, less energy.

The decibel column says the same thing more usefully. Half the power is always 3.01 decibels down, whatever the signal, because ten times the logarithm of two is 3.0103. That fixed step is why loudness gets compared in decibels here and never in raw RMS: a person who leans back from the microphone and doubles their distance loses about six decibels, and six is a number you can hold in your head next to the numbers a mood produces. Stage 1 hands the rest of the chapter one honest measurement and no interpretation at all.

FMIN, FMAX = 65.0, 400.0    # the floor and ceiling of a speaking voice, in Hz
CONFIDENCE = 0.3            # how much of the frame's own energy the repeat must recover


def zero_crossings(samples: np.ndarray) -> int:
    """How many times the waveform changes sign inside this frame."""
    if samples.size < 2:
        return 0
    return int(np.count_nonzero(np.diff(np.signbit(samples))))


def pitch_from_crossings(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> float:
    """A tone crosses zero twice a cycle, so half the crossing rate is its frequency."""
    seconds = samples.size / sample_rate
    if seconds <= 0.0:
        return 0.0
    return zero_crossings(samples) / seconds / 2.0


def pitch_from_autocorr(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> float:
    """Frequency from the lag at which the frame most resembles itself. 0.0 if none does."""
    x = samples - samples.mean()
    if not np.any(x):
        return 0.0
    corr = np.correlate(x, x, mode="full")[x.size - 1:]
    lo, hi = int(sample_rate / FMAX), min(int(sample_rate / FMIN), corr.size - 1)
    if hi <= lo:
        return 0.0
    lag = lo + int(np.argmax(corr[lo:hi]))
    if corr[lag] < CONFIDENCE * corr[0]:
        return 0.0
    return sample_rate / lag


if __name__ == "__main__":
    for name, frame in (("silence", silence), ("sine 200 Hz", sine)):
        print(f"{name:11s} crossings {zero_crossings(frame):3d}   "
              f"by crossings {pitch_from_crossings(frame):6.1f} Hz   "
              f"by autocorrelation {pitch_from_autocorr(frame):6.1f} Hz")

$ uv run python -m glados.prosody
silence     crossings   0   by crossings    0.0 Hz   by autocorrelation    0.0 Hz
sine 200 Hz crossings  12   by crossings  200.0 Hz   by autocorrelation  200.0 Hz

Two routes to 200.0, both arithmetic. A 200 hertz wave completes six cycles in 30 milliseconds and crosses zero twice a cycle, so twelve crossings in 0.03 seconds is 400 crossings a second, and half of that is the frequency. The autocorrelation takes the other road: at 16,000 samples a second one cycle spans 80 samples, the delayed copy lines up exactly there, and 16,000 divided by 80 is 200. The confidence test is what makes the second one able to abstain. Silence and hiss never recover 30 per cent of the frame's energy at any lag in range, so the function returns 0.0 to mean "no pitch here" instead of returning a number nobody should use.

Then point both at a person and watch them part company. The lab below slices one recorded sentence into 30 millisecond frames and prints two of them, a vowel and a fricative.

# labs/prosody_frames.py
"""The two pitch estimators, side by side, on frames of one recorded sentence."""
import wave

import numpy as np

from glados.prosody import (FRAME_SAMPLES, pitch_from_autocorr, pitch_from_crossings,
                            to_float, zero_crossings)

UTTERANCE = "glados/data/utterance.wav"


def frames(path: str = UTTERANCE) -> list[np.ndarray]:
    """The clip cut into whole frames; a trailing partial frame is dropped."""
    with wave.open(path) as wf:
        samples = to_float(wf.readframes(wf.getnframes()))
    whole = samples.size // FRAME_SAMPLES * FRAME_SAMPLES
    return list(samples[:whole].reshape(-1, FRAME_SAMPLES))


if __name__ == "__main__":
    picked = {31: 'vowel in "afternoon"', 58: 'the "s" in "sensor"'}
    for i, frame in enumerate(frames()):
        if i in picked:
            print(f"frame {i:3d}  {picked[i]:22s} crossings {zero_crossings(frame):4d}   "
                  f"by crossings {pitch_from_crossings(frame):7.1f} Hz   "
                  f"by autocorrelation {pitch_from_autocorr(frame):6.1f} Hz")

$ uv run python -m labs.prosody_frames   # measured on the bench: yours will vary
frame  31  vowel in "afternoon"   crossings   14   by crossings   233.3 Hz   by autocorrelation  110.3 Hz
frame  58  the "s" in "sensor"    crossings  214   by crossings  3566.7 Hz   by autocorrelation    0.0 Hz

The vowel is the interesting failure. Fourteen crossings in 30 milliseconds gives 233.3 hertz by the cheap method, and the speaker's pitch is nowhere near that: a vowel is a buzz at 110 hertz plus a stack of harmonics on top, and those harmonics drag the waveform across zero extra times inside every cycle. Autocorrelation is unmoved, because it asks when the whole pattern repeats and the whole pattern repeats at the fundamental, lag 145, which is 110.3 hertz. The fricative settles it. An "s" is filtered noise with no repeat in it at all; crossings report a confident 3,566.7 hertz, and the honest answer is the 0.0 the other estimator gives.

import librosa

HOP = 256    # 16 ms between pitch estimates at 16 kHz


def pitch_track(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
    """Pitch per frame across the whole clip, with the unvoiced frames removed."""
    f0, _flag, _prob = librosa.pyin(samples, fmin=FMIN, fmax=FMAX, sr=sample_rate,
                                    frame_length=1024, hop_length=HOP)
    return f0[~np.isnan(f0)]


def semitones_from(track: np.ndarray, reference: float) -> np.ndarray:
    """The track rewritten as musical distance from a reference pitch."""
    return 12.0 * np.log2(track / reference)


def pitch_spread(track: np.ndarray) -> float:
    """How far the pitch wandered, in semitones, so two different voices compare."""
    if track.size < 2:
        return 0.0
    return float(np.std(semitones_from(track, float(np.median(track)))))


if __name__ == "__main__":
    octave = np.array([100.0, 200.0])
    print(f"an octave apart: median {np.median(octave):.1f} Hz   "
          f"spread {pitch_spread(octave):.2f} semitones")

$ uv run python -m glados.prosody
an octave apart: median 150.0 Hz   spread 6.00 semitones

librosa.pyin runs a smarter version of stage 2's comparison over overlapping windows and returns three arrays; the first is the pitch track, and every frame it judges unvoiced holds NaN instead of a number. f0[~np.isnan(f0)] keeps the frames where a person was actually voicing. Leave that filter out and a single silent frame poisons every statistic downstream, which is this chapter's worked failure and a mistake that never announces itself.

Measuring the wander in semitones instead of hertz is the choice that makes this number portable between people. Two frames an octave apart give a spread of 6.00 semitones whatever the pitch they sit at, since the standard deviation of two values is half the gap between them and an octave is twelve semitones by definition. In hertz an octave is 100 wide starting from 100 and 200 wide starting from 200, so a spread measured in hertz mostly reports how deep a voice is. Equal semitone steps are equal frequency ratios, and ratios are what a listener hears.

from dataclasses import dataclass

import webrtcvad

AGGRESSIVENESS = 2    # the setting the recorder from chapter 87 already listens with


def speech_frames(pcm: bytes, aggressiveness: int = AGGRESSIVENESS) -> tuple[int, int]:
    """(frames carrying voice, frames in the clip), counted by the same detector."""
    vad = webrtcvad.Vad(aggressiveness)
    total = len(pcm) // FRAME_BYTES
    voiced = sum(1 for i in range(total)
                 if vad.is_speech(pcm[i * FRAME_BYTES:(i + 1) * FRAME_BYTES], SAMPLE_RATE))
    return voiced, total


@dataclass(frozen=True)
class Prosody:
    """How an utterance was delivered. Nothing in here says what was said."""
    level_db: float     # loudness of the whole clip, dBFS
    pitch_hz: float     # median pitch of the voiced frames
    spread_st: float    # how far the pitch wandered, semitones
    density: float      # fraction of frames carrying voice
    frames: int         # length of the clip, in 30 ms frames


def measure(pcm: bytes) -> Prosody:
    """One pass over one recording. No decisions are made here."""
    samples = to_float(pcm)
    track = pitch_track(samples)
    voiced, total = speech_frames(pcm)
    return Prosody(
        level_db=round(dbfs(rms(samples)), 1),
        pitch_hz=round(float(np.median(track)), 1) if track.size else 0.0,
        spread_st=round(pitch_spread(track), 1),
        density=round(voiced / total, 2) if total else 0.0,
        frames=total,
    )


if __name__ == "__main__":
    from labs.vad_listen import record_utterance

    for delivery in ("flat", "ordinary", "agitated"):
        input(f"say the same sentence {delivery}, then press enter: ")
        p = measure(record_utterance())
        print(f"{delivery:9s} {p.level_db:7.1f} dB {p.pitch_hz:7.1f} Hz "
              f"{p.spread_st:5.1f} st {p.density:6.2f} {p.frames:5d} frames")

$ uv run python -m glados.prosody   # measured on the bench: yours will vary
flat        -31.2 dB   108.6 Hz   1.9 st   0.55   121 frames
ordinary    -27.4 dB   114.2 Hz   3.1 st   0.66   108 frames
agitated    -21.8 dB   139.7 Hz   4.4 st   0.87    96 frames

One sentence, three deliveries, and all four numbers move together in the direction you would guess. Density is the one that needs its definition stated carefully, since it is voiced frames divided by total frames and nothing more. It rises when somebody talks without pausing and it falls when they leave gaps, so it does not measure syllables a second: a fast talker who breathes between phrases and a slow one who never stops can land on the same 0.66. Call it how densely packed the clip is, write that in the docstring, and never let a later reader think it is a speaking rate.

Two practical details keep it comparable between clips. The recorder proves you have finished by waiting through a second and a half of quiet, and its trim leaves 300 milliseconds of that on the end, so every clip carries the same ten silent frames and the ratio is not skewed by a tail that varies. And the whole measurement runs on the buffer that already exists, so hearing tone costs one more pass over samples the pipeline was holding anyway.

$ uv run python -m glados.prosody   # the same script, a different speaker: yours will vary
flat        -29.8 dB   188.4 Hz   2.6 st   0.58   118 frames
ordinary    -25.9 dB   196.3 Hz   3.6 st   0.69   104 frames
agitated    -20.6 dB   214.8 Hz   5.1 st   0.88    93 frames

# labs/prosody_calibrate.py
"""Record a handful of ordinary sentences and write down your own middle."""
import json
import statistics
from pathlib import Path

from glados.prosody import Prosody, measure
from labs.vad_listen import record_utterance

BASELINE_PATH = Path("glados/data/prosody_baseline.json")
FIELDS = ("level_db", "pitch_hz", "spread_st", "density")
CLIPS = 8


def mad(values: list[float]) -> float:
    """Median absolute deviation: the typical distance from the middle."""
    middle = statistics.median(values)
    return statistics.median([abs(v - middle) for v in values])


def build_baseline(clips: list[Prosody]) -> dict:
    """Middle and spread per feature, from your own voice on an ordinary day."""
    out: dict = {}
    for field in FIELDS:
        values = [getattr(c, field) for c in clips]
        out[field] = {"middle": round(statistics.median(values), 2),
                      "mad": round(mad(values), 2)}
    return out


def distance(value: float, stat: dict) -> float:
    """How far from your middle, counted in your own typical deviations."""
    if stat["mad"] <= 0.0:
        return 0.0
    return (value - stat["middle"]) / stat["mad"]


def load_baseline(path: Path = BASELINE_PATH) -> dict | None:
    """None means this reader has not calibrated, and prosody stays switched off."""
    if not path.exists():
        return None
    data = json.loads(path.read_text())
    return data if all(f in data for f in FIELDS) else None


if __name__ == "__main__":
    print("mad([1, 2, 3, 4, 100]) =", mad([1.0, 2.0, 3.0, 4.0, 100.0]))
    clips = []
    for i in range(CLIPS):
        input(f"clip {i + 1}/{CLIPS}: say an ordinary sentence, then press enter: ")
        clips.append(measure(record_utterance()))
        p = clips[-1]
        print(f"          {p.level_db:7.1f} dB {p.pitch_hz:7.1f} Hz "
              f"{p.spread_st:5.1f} st {p.density:6.2f}")
    baseline = build_baseline(clips)
    BASELINE_PATH.parent.mkdir(parents=True, exist_ok=True)
    BASELINE_PATH.write_text(json.dumps(baseline, indent=2))
    for field, stat in baseline.items():
        print(f"  {field:10s} middle {stat['middle']:8.2f}   mad {stat['mad']:5.2f}")

$ uv run python -m labs.prosody_calibrate   # measured on the bench: yours will vary
mad([1, 2, 3, 4, 100]) = 1.0
clip 1/8: say an ordinary sentence, then press enter:
            -27.9 dB   113.2 Hz   2.8 st   0.64
clip 2/8: say an ordinary sentence, then press enter:
            -26.1 dB   117.3 Hz   3.4 st   0.72
clip 3/8: say an ordinary sentence, then press enter:
            -30.7 dB   105.8 Hz   1.7 st   0.51
clip 4/8: say an ordinary sentence, then press enter:
            -27.3 dB   114.6 Hz   3.0 st   0.66
clip 5/8: say an ordinary sentence, then press enter:
            -25.2 dB   120.1 Hz   3.8 st   0.76
clip 6/8: say an ordinary sentence, then press enter:
            -29.4 dB   109.3 Hz   2.3 st   0.57
clip 7/8: say an ordinary sentence, then press enter:
            -26.4 dB   116.9 Hz   3.2 st   0.70
clip 8/8: say an ordinary sentence, then press enter:
            -28.5 dB   111.8 Hz   2.6 st   0.61
  level_db   middle   -27.60   mad  1.35
  pitch_hz   middle   113.90   mad  3.20
  spread_st  middle     2.90   mad  0.40
  density    middle     0.65   mad  0.06

The first line is the argument for the median absolute deviation, and you can verify it in your head. The middle of 1, 2, 3, 4, 100 is 3; the distances from it are 2, 1, 0, 1 and 97; and the middle of those is 1. One clip where the dog barked contributes a 97 that changes nothing. A standard deviation over the same five numbers is about 39, and every threshold built on it would be nonsense.

The summary at the bottom is checkable off the eight rows above it in the same way. Sort the level column and the fourth and fifth entries are -27.9 and -27.3, whose average is the -27.60 printed; take each row's distance from that middle and the fourth and fifth of those are 1.2 and 1.5, whose average is the 1.35. Eight clips of ordinary speech vary by five and a half decibels and fourteen hertz, which is the first useful thing calibration tells you: your own ordinary is a range, not a point.

distance is the whole payoff. It converts a measurement into a count of your own deviations: the agitated clip's -21.8 decibels is 4.3 of this reader's deviations above their middle, its 139.7 hertz is 8.1 above, its density is 3.7 above. Those three counts mean the same thing for the second speaker, whose middles are nowhere near these ones. And load_baseline returning None is a design decision rather than an error path: a reader who has not calibrated gets no prosody at all, and she behaves exactly as she did before this chapter.

from labs.mood_state import STATE_PATH, MoodState

AROUSAL_FIELDS = ("level_db", "pitch_hz", "density")   # spread is its own axis
MAX_NUDGE = 0.15      # the most one utterance's delivery may move the dial
AROUSAL_FULL = 3.0    # deviations at which that maximum is reached
HINT_LEVEL = 1.5      # deviations below which she is told nothing at all


def arousal(p: Prosody, baseline: dict) -> float:
    """One score: how far this delivery sits from your own ordinary."""
    return statistics.mean(distance(getattr(p, f), baseline[f]) for f in AROUSAL_FIELDS)


def style_hint(score: float) -> str:
    """One advisory line for the system prompt. Advice about manner, never an instruction."""
    if score >= HINT_LEVEL:
        return "The speaker sounds more keyed up than is usual for them. Answer briefly."
    if score <= -HINT_LEVEL:
        return "The speaker sounds flatter than is usual for them. Do not press them."
    return ""


def apply_prosody(state: MoodState, p: Prosody, baseline: dict) -> tuple[float, str]:
    """Nudge the intensity, return the hint. The mood name is never touched here."""
    score = arousal(p, baseline)
    delta = MAX_NUDGE * max(-1.0, min(1.0, score / AROUSAL_FULL))
    state.nudge(delta)
    return score, style_hint(score)


if __name__ == "__main__":
    baseline = load_baseline()
    state = MoodState.load(STATE_PATH)
    for delivery in ("agitated", "ordinary", "flat"):
        input(f"say the same sentence {delivery}, then press enter: ")
        before = state.intensity
        score, hint = apply_prosody(state, measure(record_utterance()), baseline)
        print(f"{delivery:9s} arousal {score:+5.1f}   nudge {state.intensity - before:+.2f}   "
              f"{state.mood} {before:.2f} -> {state.intensity:.2f}")
        print(f"          hint: {hint!r}")

$ uv run python -m labs.prosody_calibrate   # measured on the bench: yours will vary
say the same sentence agitated, then press enter:
agitated  arousal  +5.3   nudge +0.15   neutral 0.50 -> 0.65
          hint: 'The speaker sounds more keyed up than is usual for them. Answer briefly.'
say the same sentence ordinary, then press enter:
ordinary  arousal  +0.1   nudge +0.01   neutral 0.65 -> 0.66
          hint: ''
say the same sentence flat, then press enter:
flat      arousal  -2.0   nudge -0.10   neutral 0.66 -> 0.56
          hint: 'The speaker sounds flatter than is usual for them. Do not press them.'

The mood column never changes, and that is the design, not a coincidence in the data. Chapter 47 split her mood into two axes: a name drawn from a fixed set, and a number between 0 and 1 saying how much of it she is feeling. Chapter 24's transition table owns the name, moving her between neutral and hostile and curious from the sentiment of the words. Prosody has no business there, because how loudly a sentence arrived says nothing about whether it was kind. It owns a slice of the other axis instead, capped at 0.15 a turn, so ten agitated sentences in a row can carry her from calm to fully felt and one cannot.

The clamp inside nudge keeps the result in range without any checking here, and the cap on delta bounds how wrong this can be. That matters because being wrong is the normal case. If the reading is nonsense, she is at most fifteen per cent more intense about a mood the words already justified. Note also what the hint is allowed to say: it describes manner and asks for brevity. It does not name an emotion, it does not reach the permission layer from volume 6, and it never gets repeated to the speaker, since being told by a machine that you sound stressed is unpleasant when it is right and infuriating when it is wrong.