Where the Weights Come From

// Command brains is chapter 54's bench: where a controller's 378
// numbers come from, how big each one is allowed to be, what it costs
// to write them down, and why nothing here computes a gradient. Every
// mode is arithmetic on drawn numbers; no valley is read and no gram
// moves.
package main

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"fmt"
	"math"
	"os"

	"theworld/internal/mind"
)

// layout prints where each of the four blocks starts, which run of
// numbers each neuron owns, and what one thought costs.
func layout(in, hid, out, creatures int) {
	n := mind.New(in, hid, out)
	fmt.Printf("brains: one %d-%d-%d controller as a flat row of numbers\n\n", in, hid, out)
	fmt.Printf("  %-16s %7s %8s %8s   %s\n", "part", "count", "from", "to", "arithmetic")
	fmt.Printf("  %-16s %7d %8d %8d   %d inputs x %d hidden\n",
		"hidden weights", n.HidB()-n.HidW(), n.HidW(), n.HidB(), in, hid)
	fmt.Printf("  %-16s %7d %8d %8d   one per hidden neuron\n",
		"hidden biases", n.OutW()-n.HidB(), n.HidB(), n.OutW())
	fmt.Printf("  %-16s %7d %8d %8d   %d hidden x %d out\n",
		"output weights", n.OutB()-n.OutW(), n.OutW(), n.OutB(), hid, out)
	fmt.Printf("  %-16s %7d %8d %8d   one per output neuron\n",
		"output biases", len(n.W)-n.OutB(), n.OutB(), len(n.W))
	fmt.Printf("  %-16s %7d\n", "the whole brain", len(n.W))

	fmt.Printf("\n  hidden neuron j reads W[%d + j*%d : %d + j*%d + %d]\n",
		n.HidW(), in, n.HidW(), in, in)
	fmt.Printf("  output neuron k reads W[%d + k*%d : %d + k*%d + %d]\n",
		n.OutW(), hid, n.OutW(), hid, hid)
	for _, j := range []int{0, 1, hid - 1} {
		fmt.Printf("    hidden %2d: W[%3d:%3d]   bias W[%d]\n",
			j, n.HidW()+j*in, n.HidW()+(j+1)*in, n.HidB()+j)
	}
	for _, k := range []int{0, out - 1} {
		fmt.Printf("    output %2d: W[%3d:%3d]   bias W[%d]\n",
			k, n.OutW()+k*hid, n.OutW()+(k+1)*hid, n.OutB()+k)
	}

	muls := in*hid + hid*out
	fmt.Printf("\n  one tick of one brain: %d multiplications, %d additions, %d tanh\n",
		muls, muls+hid+out, hid)
	fmt.Printf("  %d creatures, all thinking once: %d multiplications a tick\n",
		creatures, muls*creatures)
}

$ go run ./cmd/brains -mode row
brains: one 24-12-6 controller as a flat row of numbers

  part               count     from       to   arithmetic
  hidden weights       288        0      288   24 inputs x 12 hidden
  hidden biases         12      288      300   one per hidden neuron
  output weights        72      300      372   12 hidden x 6 out
  output biases          6      372      378   one per output neuron
  the whole brain      378

  hidden neuron j reads W[0 + j*24 : 0 + j*24 + 24]
  output neuron k reads W[300 + k*12 : 300 + k*12 + 12]
    hidden  0: W[  0: 24]   bias W[288]
    hidden  1: W[ 24: 48]   bias W[289]
    hidden 11: W[264:288]   bias W[299]
    output  0: W[300:312]   bias W[372]
    output  5: W[360:372]   bias W[377]

  one tick of one brain: 360 multiplications, 378 additions, 12 tanh
  3000 creatures, all thinking once: 1080000 multiplications a tick

Read the printing code against the table it printed. Not one from and not one to in that output is arithmetic this file did: they are n.HidW(), n.HidB(), n.OutW() and n.OutB(), asked and printed. The counts are subtractions between two of them. Change the middle layer from twelve neurons to twenty and every number on the page moves: the first block grows to 480, the biases start at 480, the whole brain becomes 626. Nothing here has to be edited for that to come out right, and nothing here can drift away from what the forward pass believes, because it is asking the forward pass what it believes.

Note the last two lines as well. The multiplications are the entire arithmetic cost of a brain thinking once, and three thousand of them come to a little over a million a tick. That is a number to hold on to.

// internal/mind/weights.go
package mind

import (
	"encoding/binary"
	"math"
	"math/rand/v2"
)

// Fan is how many numbers a neuron in each row adds up: a hidden neuron
// reads all In of the inputs, an output neuron reads all Hid of the
// middle values. It is the only thing the size of a drawn weight
// depends on, and it is read off the sizes rather than stated anywhere.
func (n Net) Fan() (hid, out int) { return n.In, n.Hid }

// StreamBrain is where a brain's numbers come from: stream 14 on the
// world's seed, continuing the registry the terrarium opened. Nothing
// else in the world draws from it, so a valley that gains a creature
// does not move a plant's numbers.
const StreamBrain = 14

// Draw fills a brain's whole row from one stream, in flat-slice order,
// one number per draw. Every number lands between -1 and 1 and is then
// divided by the square root of the fan its neuron adds up, so a
// hidden neuron reading twenty-four inputs gets smaller weights than an
// output neuron reading twelve. The order is the contract: a brain is a
// fixed number of steps down the stream, and two runs of one seed hand
// the same creature the same brain.
func (n Net) Draw(r *rand.Rand) {
	hid, out := n.Fan()
	small := [2]float64{1 / math.Sqrt(float64(hid)), 1 / math.Sqrt(float64(out))}
	split := n.OutW()
	for i := range n.W {
		layer := 0
		if i >= split {
			layer = 1
		}
		n.W[i] = (r.Float64()*2 - 1) * small[layer]
	}
}

// Brains opens the one generator a valley draws its brains from.
func Brains(seed uint64) *rand.Rand {
	return rand.New(rand.NewPCG(seed, StreamBrain))
}

// Bits is the row as raw bytes, every number in flat-slice order,
// eight bytes each, most significant first. It is what a digest is
// taken over: two brains agree here or they are two different brains.
func (n Net) Bits() []byte {
	out := make([]byte, 8*len(n.W))
	for i, w := range n.W {
		binary.BigEndian.PutUint64(out[8*i:], math.Float64bits(w))
	}
	return out
}

The single comparison i >= split is what splits the row into the two fans, and split is n.OutW(), the same offset the forward pass slices at. Everything before it belongs to a middle neuron reading twenty-four inputs; everything from it onward belongs to an output neuron reading twelve, biases included, since a bias is added into the same sum as the weights beside it and has to arrive on the same scale. Note the loop draws once for every position, all 378 of them, biases and all. That is not an accident of writing: a brain has to be a fixed number of steps down the stream, because otherwise the tenth creature born in a run gets different numbers depending on what the ninth one turned out to be.

// loud is the noisiest row a layer's inputs allow: every entry at the
// far end of its range, alternating in sign so nothing cancels for a
// tidy reason.
func loud(n int) []float64 {
	x := make([]float64, n)
	for i := range x {
		x[i] = 1
		if i%2 == 1 {
			x[i] = -1
		}
	}
	return x
}

// spread is what a whole layer of sums came to: the usual size of one
// of them, measured the way the interlude defines it, the average size
// of what tanh made of it, and the share of the layer tanh answered
// past the pin with.
func spread(sums []float64) (usual, squashed, pinned float64) {
	for _, z := range sums {
		usual += z * z
		t := math.Abs(mind.Tanh(z))
		squashed += t
		if t > 0.99 {
			pinned++
		}
	}
	n := float64(len(sums))
	return math.Sqrt(usual / n), squashed / n, pinned / n
}

$ go run ./cmd/brains -mode fan
brains: a layer of 4000 neurons fed the loudest row its inputs allow,
  drawn off stream 14 and used twice: as drawn, then divided by the root of the fan

  inputs    root    predict    usual   |tanh|  pinned    predict    usual   |tanh|  pinned
       4   2.000      1.291    1.306   0.6372      4%      0.645    0.653   0.4300      0%
      12   3.464      2.082    2.098   0.7525     21%      0.601    0.606   0.4050      0%
      24   4.899      2.887    2.877   0.8156     36%      0.589    0.587   0.3957      0%
      96   9.798      5.686    5.743   0.9046     64%      0.580    0.586   0.3967      0%
     384  19.596     11.328   11.422   0.9520     82%      0.578    0.583   0.3937      0%

  left half: every number between -1 and 1. right half: the same numbers over the root of the fan.
  predict is sqrt((inputs+1)/3) times the size the numbers were drawn at.
  usual is the measured one: every sum squared, averaged, and the root taken back off.
  pinned is the share of the layer whose tanh came back past 0.99.

Read the two predict columns against the two usual columns first, because that is the interlude being checked: 2.887 predicted and 2.877 measured at twenty-four inputs, 11.328 predicted and 11.422 measured at three hundred and eighty-four. The arithmetic was done on paper and the run agrees with it to about a percent, which is what four thousand samples buys.

Now read the pinned columns, which say what it costs. Left, with the division left out, more than a third of the layer is past 0.99 at twenty-four inputs and four fifths of it is past 0.99 at three hundred and eighty-four. Every pinned neuron has stopped reporting: it will say 0.995 to this sensor row and 0.996 to a completely different one, and its contribution to the six scores is very nearly a constant. Right, with the division in, the pinned column reads 0% at every size on the table and the average middle value sits near 0.40 whatever the fan. This valley's own controller is the 24 line, so it is the mild case, and the rule is what keeps it mild.

// digest is the sixteen hex digits this bench compares brains by,
// folded the way the valley folds its own state: sha256 over the raw
// bytes of every weight in order, cut to the first sixteen. Two brains
// agree here only if every one of their numbers is the same number
// down to the last bit.
func digest(n mind.Net) string {
	sum := sha256.Sum256(n.Bits())
	return hex.EncodeToString(sum[:])[:16]
}

// brainAt is the k'th brain a valley draws: open stream 14 on the world
// seed, throw away the brains born before this one, and take the next
// row of numbers off it.
func brainAt(seed uint64, k, in, hid, out int) mind.Net {
	r := mind.Brains(seed)
	n := mind.New(in, hid, out)
	n.Squash = mind.Tanh
	for range k {
		n.Draw(r)
	}
	n.Draw(r)
	return n
}

$ go run ./cmd/brains -mode draw
brains: 4 of them drawn off stream 14 on world seed 5

  brain         W[0]         W[1]       W[377]   digest
      0   0.11373768   0.15953022   0.06711743   a978c376398aa14e
      1   0.12787853  -0.08717750  -0.15217619   a6ce5d024e4641ad
      2   0.13539839   0.03748572   0.24593342   c90a2c7c021463d2
      3  -0.00090827   0.11102333  -0.08975907   36797ad6205e4801

  the same seed again, from a generator that has never been used:
      0   0.11373768   0.15953022   0.06711743   a978c376398aa14e
      1   0.12787853  -0.08717750  -0.15217619   a6ce5d024e4641ad
      2   0.13539839   0.03748572   0.24593342   c90a2c7c021463d2
      3  -0.00090827   0.11102333  -0.08975907   36797ad6205e4801

  4 of 4 brains came back identical, bit for bit
  brain 3 skipped to directly: 36797ad6205e4801

Four brains, four digests, no two alike, and the whole set repeats exactly when a fresh generator is opened on the same seed. The last line is the property that matters most and is easiest to lose. It was produced by brainAt: open the stream, throw away three brains' worth of draws without keeping a single number, then draw the fourth. The digest is 36797ad6205e4801, the same as the fourth brain in the table above. Because every brain costs exactly 378 draws, position in the stream is arithmetic, and a creature born fourth gets the fourth brain whatever happened to the three before it.

// internal/mind/file.go
package mind

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
)

// Brain is one controller written down: the name it answers to, the
// three sizes it was built at, and its whole row of numbers. Nothing
// else is stored, because nothing else is the brain.
type Brain struct {
	ID  int       `json:"id"`
	In  int       `json:"in"`
	Hid int       `json:"hid"`
	Out int       `json:"out"`
	W   []float64 `json:"w"`
}

// Save writes brains one JSON object to a line, in the order they are
// handed over. A row whose length disagrees with the sizes beside it is
// refused here, where the caller still knows which brain it was.
func Save(w io.Writer, brains []Brain) error {
	enc := json.NewEncoder(w)
	for _, b := range brains {
		if want := Weights(b.In, b.Hid, b.Out); len(b.W) != want {
			return fmt.Errorf("save brain %d: %d numbers for a %d-%d-%d net, which needs %d",
				b.ID, len(b.W), b.In, b.Hid, b.Out, want)
		}
		if err := enc.Encode(b); err != nil {
			return fmt.Errorf("save brain %d: %w", b.ID, err)
		}
	}
	return nil
}

// Load reads back what Save wrote, one line at a time, and checks every
// row against the sizes written beside it. The scanner is given a
// megabyte because one line here is thousands of numbers long and the
// default would stop reading in the middle of a brain.
func Load(r io.Reader) ([]Brain, error) {
	var out []Brain
	sc := bufio.NewScanner(r)
	sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
	for line := 1; sc.Scan(); line++ {
		var b Brain
		if err := json.Unmarshal(sc.Bytes(), &b); err != nil {
			return nil, fmt.Errorf("brain file line %d: %w", line, err)
		}
		if want := Weights(b.In, b.Hid, b.Out); len(b.W) != want {
			return nil, fmt.Errorf("brain file line %d: %d numbers for a %d-%d-%d net, which needs %d",
				line, len(b.W), b.In, b.Hid, b.Out, want)
		}
		out = append(out, b)
	}
	if err := sc.Err(); err != nil {
		return nil, fmt.Errorf("brain file: %w", err)
	}
	return out, nil
}

$ go run ./cmd/brains -mode file
brains: 4 of them written to brains.jsonl, 31040 bytes

  the first line, cut off at 96 characters:
  {"id":0,"in":24,"hid":12,"out":6,"w":[0.11373767798729006,0.15953021871777115,0.1650562778221637...

  brain                drawn            read back  same
      0     a978c376398aa14e     a978c376398aa14e  true
      1     a6ce5d024e4641ad     a6ce5d024e4641ad  true
      2     c90a2c7c021463d2     c90a2c7c021463d2  true
      3     36797ad6205e4801     36797ad6205e4801  true

  4 of 4 rows survived the trip to disk with every bit intact
  W[0] drawn      0.11373767798729006
  W[0] read back  0.11373767798729006

The digests are the claim. They are taken over the raw eight bytes of every number in order, not over the printed text, so two rows agreeing here have agreed bit for bit. All four survived, and the reason is the seventeen digits in the last two lines: encoding/json writes the shortest run of decimal digits that reads back as exactly the same float64, and 0.11373767798729006 is that run for this number. It is not a rounded version of the weight. It is the weight.

The size is honest about what the format costs. Four brains take 31,040 bytes, so one brain is 7,760, against the 3,024 bytes its 378 numbers occupy in memory. Text is about two and a half times the price of the numbers, and it buys a file a person can open, search, and compare line by line without a tool. For thousands of brains a run that trade stays reasonable; for millions it would not, and the digest is what would tell you when to change it.

// Where each reading sits in the sensor row, in the order the senses
// were fixed in: nine rays, two numbers to a ray, then the two slopes,
// then the store and the speed. This bench never reads a valley, so
// these are the only positions it needs.
const (
	rays    = 9              // vision rays in the fan
	senses  = 2*rays + 4 + 2 // and the whole row they open
	wetX    = 2 * rays       // the moisture slope, going east
	wetY    = wetX + 1       // and going south
	foodX   = wetY + 1       // the standing-tissue slope, going east
	foodY   = foodX + 1      // and going south
	storeAt = foodY + 1      // the store, as a share of full
	speedAt = storeAt + 1    // the speed, as a share of top
)

// eye is where ray i's two numbers sit: how far it went at eye(i), and
// what it met at eye(i)+1.
func eye(i int) int { return 2 * i }

// The action table, in the order the six output scores are read in.
// The two turns are shortened here to keep a column narrow.
var actions = [6]string{"rest", "walk", "sprint", "left", "right", "bite"}

// The row every swept question below is asked with: twenty-four numbers
// typed here rather than read off a valley. A creature standing still
// with a quarter of a store, on ground that gets wetter and greener to
// the east, with nine rays that have met nothing at all.
func row() []float64 {
	in := make([]float64, senses)
	for i := range rays {
		in[eye(i)] = 0.5
		in[eye(i)+1] = 0.0
	}
	in[wetX], in[wetY] = 0.10, -0.05
	in[foodX], in[foodY] = 0.30, 0.15
	in[storeAt] = 0.24
	in[speedAt] = 0.00
	return in
}

// scores runs one sensor row through a brain and hands back the six
// numbers, the action that wins, and how far ahead of the runner-up it
// finished.
func scores(n mind.Net, in, hid, out []float64) (best int, margin float64) {
	n.Forward(in, hid, out)
	best, second := 0, 1
	if out[1] > out[0] {
		best, second = 1, 0
	}
	for i := 2; i < len(out); i++ {
		switch {
		case out[i] > out[best]:
			best, second = i, best
		case out[i] > out[second]:
			second = i
		}
	}
	return best, out[best] - out[second]
}

$ go run ./cmd/brains -mode sweep -brain 11 -what rays
brains: brain 11 off stream 14, digest 0a9391b5cbf1a897
  one sensor row held still but for "rays", swept from 0.00 to 1.00

     rays      rest      walk    sprint      left     right      bite   picks     margin
     0.00   -0.1599    0.3069   -0.0597   -0.0350    0.1138   -0.1192   walk      0.1931
     0.10   -0.1164    0.3108   -0.0695   -0.0617    0.1401   -0.1396   walk      0.1706
     0.20   -0.0732    0.3139   -0.0783   -0.0887    0.1664   -0.1599   walk      0.1475
     0.30   -0.0304    0.3164   -0.0862   -0.1158    0.1924   -0.1799   walk      0.1240
     0.40    0.0116    0.3182   -0.0932   -0.1427    0.2177   -0.1995   walk      0.1006
     0.50    0.0527    0.3195   -0.0994   -0.1692    0.2420   -0.2184   walk      0.0775
     0.60    0.0926    0.3204   -0.1048   -0.1948    0.2651   -0.2365   walk      0.0552
     0.70    0.1311    0.3208   -0.1095   -0.2195    0.2868   -0.2537   walk      0.0340
     0.80    0.1681    0.3209   -0.1136   -0.2430    0.3069   -0.2698   walk      0.0140
     0.90    0.2035    0.3207   -0.1172   -0.2652    0.3252   -0.2848   right     0.0045
     1.00    0.2373    0.3204   -0.1203   -0.2859    0.3418   -0.2987   right     0.0214

  2 of the 6 actions won a step, and the winner changed hands once

Every one of the six columns moves smoothly. Not one of them jumps, and the column that wins most of the table barely moves at all: walk travels from 0.3069 to 0.3209 and turns back down again, a span of fourteen thousandths across the whole sweep. The picks column does something else. It holds at walk for nine rows, and on the tenth it becomes right and stays. Look at the margin beside it. At 0.80 the winner is ahead by 0.0140; one row later the winner is a different action, ahead by 0.0045. The behaviour did not bend. It switched.