Monorepo for Aesthetic.Computer
aesthetic.computer
1import { createSeededRandom, hashString } from "./runtime.mjs";
2
3const GLOBAL_GRID_X = 4;
4const GLOBAL_GRID_Y = 4;
5const GLOBAL_FEATURE_COUNT = 13 + GLOBAL_GRID_X * GLOBAL_GRID_Y;
6const STATE_LATENT_SIZE = 8;
7const CELL_COUNT = 64;
8const TILE_GRID_X = 8;
9const TILE_GRID_Y = 8;
10const TILE_FEATURE_COUNT = 12;
11const LATENT_FIELD_CHANNELS = 12;
12const OUTPUT_SIZE = 40;
13const ADDITIVE_PARTIALS = 6;
14const FORMANT_COUNT = 3;
15const EXPERT_NAMES = ["tonal", "vocal", "table", "living"];
16const TAU = Math.PI * 2;
17
18export const DECODER_NAMES = [...EXPERT_NAMES];
19
20const SOUND_STYLES = {
21 default: {
22 latentBlend: 0.24,
23 fieldBlend: 0.68,
24 fieldMemory: 0.64,
25 byteMixScale: 0.9,
26 petriMixScale: 0.82,
27 tonalMixScale: 0.98,
28 vocalMixScale: 0.96,
29 tableMixScale: 1.08,
30 livingMixScale: 0.78,
31 byteSoftness: 0.18,
32 byteLowpass: 0.66,
33 outputSmoothing: 0.58,
34 gainScale: 0.92,
35 panScale: 1.0,
36 exciteScale: 1.0,
37 couplingScale: 1.0,
38 growScale: 1.0,
39 diffuseScale: 1.0,
40 sparkleScale: 1.0,
41 byteHarmonicsScale: 1.0,
42 formantWarmth: 1.0,
43 tableWarpScale: 1.0,
44 },
45 soft: {
46 latentBlend: 0.18,
47 fieldBlend: 0.6,
48 fieldMemory: 0.74,
49 byteMixScale: 0.34,
50 petriMixScale: 0.74,
51 tonalMixScale: 0.88,
52 vocalMixScale: 1.12,
53 tableMixScale: 0.82,
54 livingMixScale: 0.28,
55 byteSoftness: 0.78,
56 byteLowpass: 0.9,
57 outputSmoothing: 0.88,
58 gainScale: 0.82,
59 panScale: 0.68,
60 exciteScale: 0.54,
61 couplingScale: 0.68,
62 growScale: 0.76,
63 diffuseScale: 1.18,
64 sparkleScale: 0.72,
65 byteHarmonicsScale: 0.58,
66 formantWarmth: 1.18,
67 tableWarpScale: 0.74,
68 },
69};
70
71function clamp(value, low, high) {
72 return Math.max(low, Math.min(high, value));
73}
74
75function lerp(a, b, t) {
76 return a + (b - a) * t;
77}
78
79function mapSigned(value, low, high) {
80 return low + ((value + 1) * 0.5) * (high - low);
81}
82
83function tanh(value) {
84 return Math.tanh(value);
85}
86
87function normalize01(value) {
88 return clamp(value, 0, 1) * 2 - 1;
89}
90
91function wrap01(value) {
92 const wrapped = value % 1;
93 return wrapped < 0 ? wrapped + 1 : wrapped;
94}
95
96function buildNetwork(seed, inputSize, hiddenSizes, outputSize) {
97 const prng = createSeededRandom(seed);
98 const sizes = [inputSize, ...hiddenSizes, outputSize];
99 const layers = [];
100
101 for (let layerIndex = 0; layerIndex < sizes.length - 1; layerIndex += 1) {
102 const inSize = sizes[layerIndex];
103 const outSize = sizes[layerIndex + 1];
104 const scale = 1 / Math.sqrt(inSize);
105 const weights = new Float32Array(inSize * outSize);
106 const biases = new Float32Array(outSize);
107
108 for (let i = 0; i < weights.length; i += 1) {
109 weights[i] = (prng() * 2 - 1) * scale;
110 }
111
112 for (let i = 0; i < biases.length; i += 1) {
113 biases[i] = (prng() * 2 - 1) * scale;
114 }
115
116 layers.push({ inSize, outSize, weights, biases });
117 }
118
119 return { layers };
120}
121
122function forwardNetwork(network, input) {
123 let activations = input;
124
125 for (let layerIndex = 0; layerIndex < network.layers.length; layerIndex += 1) {
126 const layer = network.layers[layerIndex];
127 const next = new Float32Array(layer.outSize);
128 for (let out = 0; out < layer.outSize; out += 1) {
129 let sum = layer.biases[out];
130 const weightOffset = out * layer.inSize;
131 for (let i = 0; i < layer.inSize; i += 1) {
132 sum += layer.weights[weightOffset + i] * activations[i];
133 }
134 next[out] = tanh(sum);
135 }
136 activations = next;
137 }
138
139 return activations;
140}
141
142function colorPolar(r, g, b) {
143 const hueAngle = Math.atan2(Math.sqrt(3) * (g - b), 2 * r - g - b);
144 const max = Math.max(r, g, b);
145 const min = Math.min(r, g, b);
146 return {
147 hueSin: Math.sin(hueAngle),
148 hueCos: Math.cos(hueAngle),
149 saturation: max - min,
150 value: max,
151 };
152}
153
154function extractFramebufferFeatures(rgba, width, height) {
155 const totalPixels = Math.max(1, width * height);
156 const invW = width > 1 ? 1 / (width - 1) : 0;
157 const invH = height > 1 ? 1 / (height - 1) : 0;
158 const prevRow = new Float32Array(width);
159 const gridSums = new Float32Array(GLOBAL_GRID_X * GLOBAL_GRID_Y);
160 const gridCounts = new Float32Array(GLOBAL_GRID_X * GLOBAL_GRID_Y);
161
162 let rSum = 0;
163 let gSum = 0;
164 let bSum = 0;
165 let lumSum = 0;
166 let lumSqSum = 0;
167 let edgeX = 0;
168 let edgeY = 0;
169 let centroidX = 0;
170 let centroidY = 0;
171 let diagMain = 0;
172 let diagCross = 0;
173
174 for (let y = 0; y < height; y += 1) {
175 let leftLum = 0;
176 for (let x = 0; x < width; x += 1) {
177 const offset = (y * width + x) * 4;
178 const r = rgba[offset] / 255;
179 const g = rgba[offset + 1] / 255;
180 const b = rgba[offset + 2] / 255;
181 const lum = r * 0.299 + g * 0.587 + b * 0.114;
182
183 rSum += r;
184 gSum += g;
185 bSum += b;
186 lumSum += lum;
187 lumSqSum += lum * lum;
188 centroidX += x * invW * lum;
189 centroidY += y * invH * lum;
190
191 if (x > 0) edgeX += Math.abs(lum - leftLum);
192 if (y > 0) edgeY += Math.abs(lum - prevRow[x]);
193 leftLum = lum;
194 prevRow[x] = lum;
195
196 if (x <= y * (width / Math.max(1, height))) diagMain += lum;
197 else diagCross += lum;
198
199 const cellX = Math.min(GLOBAL_GRID_X - 1, Math.floor((x / Math.max(1, width)) * GLOBAL_GRID_X));
200 const cellY = Math.min(GLOBAL_GRID_Y - 1, Math.floor((y / Math.max(1, height)) * GLOBAL_GRID_Y));
201 const cellIndex = cellY * GLOBAL_GRID_X + cellX;
202 gridSums[cellIndex] += lum;
203 gridCounts[cellIndex] += 1;
204 }
205 }
206
207 const meanR = rSum / totalPixels;
208 const meanG = gSum / totalPixels;
209 const meanB = bSum / totalPixels;
210 const meanLum = lumSum / totalPixels;
211 const varianceLum = Math.max(0, lumSqSum / totalPixels - meanLum * meanLum);
212 const edgeNormX = edgeX / totalPixels;
213 const edgeNormY = edgeY / totalPixels;
214 const centroidNormX = lumSum > 1e-6 ? centroidX / lumSum : 0.5;
215 const centroidNormY = lumSum > 1e-6 ? centroidY / lumSum : 0.5;
216 const colorSpread = (Math.abs(meanR - meanG) + Math.abs(meanG - meanB) + Math.abs(meanB - meanR)) / 3;
217 const diagonalBias = (diagMain - diagCross) / Math.max(1e-6, diagMain + diagCross);
218
219 const features = new Float32Array(GLOBAL_FEATURE_COUNT);
220 features[0] = normalize01(meanLum);
221 features[1] = normalize01(meanR);
222 features[2] = normalize01(meanG);
223 features[3] = normalize01(meanB);
224 features[4] = clamp(varianceLum * 10 - 1, -1, 1);
225 features[5] = clamp(edgeNormX * 5 - 1, -1, 1);
226 features[6] = clamp(edgeNormY * 5 - 1, -1, 1);
227 features[7] = centroidNormX * 2 - 1;
228 features[8] = centroidNormY * 2 - 1;
229 features[9] = clamp(meanR - meanG, -1, 1);
230 features[10] = clamp(meanG - meanB, -1, 1);
231 features[11] = clamp(colorSpread * 4 - 1, -1, 1);
232 features[12] = clamp(diagonalBias, -1, 1);
233
234 for (let i = 0; i < gridSums.length; i += 1) {
235 const average = gridSums[i] / Math.max(1, gridCounts[i]);
236 features[13 + i] = average * 2 - 1;
237 }
238
239 return features;
240}
241
242function extractTileFeatures(rgba, width, height, tileX, tileY) {
243 const startX = Math.floor((tileX * width) / TILE_GRID_X);
244 const endX = Math.max(startX + 1, Math.floor(((tileX + 1) * width) / TILE_GRID_X));
245 const startY = Math.floor((tileY * height) / TILE_GRID_Y);
246 const endY = Math.max(startY + 1, Math.floor(((tileY + 1) * height) / TILE_GRID_Y));
247 const tileWidth = Math.max(1, endX - startX);
248 const tileHeight = Math.max(1, endY - startY);
249 const totalPixels = tileWidth * tileHeight;
250 const prevRow = new Float32Array(tileWidth);
251
252 let rSum = 0;
253 let gSum = 0;
254 let bSum = 0;
255 let lumSum = 0;
256 let lumSqSum = 0;
257 let edgeX = 0;
258 let edgeY = 0;
259 let hueSinSum = 0;
260 let hueCosSum = 0;
261 let satSum = 0;
262
263 for (let y = startY; y < endY; y += 1) {
264 let leftLum = 0;
265 for (let x = startX; x < endX; x += 1) {
266 const localX = x - startX;
267 const offset = (y * width + x) * 4;
268 const r = rgba[offset] / 255;
269 const g = rgba[offset + 1] / 255;
270 const b = rgba[offset + 2] / 255;
271 const lum = r * 0.299 + g * 0.587 + b * 0.114;
272 const polar = colorPolar(r, g, b);
273
274 rSum += r;
275 gSum += g;
276 bSum += b;
277 lumSum += lum;
278 lumSqSum += lum * lum;
279 hueSinSum += polar.hueSin;
280 hueCosSum += polar.hueCos;
281 satSum += polar.saturation;
282
283 if (x > startX) edgeX += Math.abs(lum - leftLum);
284 if (y > startY) edgeY += Math.abs(lum - prevRow[localX]);
285 leftLum = lum;
286 prevRow[localX] = lum;
287 }
288 }
289
290 const meanR = rSum / totalPixels;
291 const meanG = gSum / totalPixels;
292 const meanB = bSum / totalPixels;
293 const meanLum = lumSum / totalPixels;
294 const varianceLum = Math.max(0, lumSqSum / totalPixels - meanLum * meanLum);
295 const features = new Float32Array(TILE_FEATURE_COUNT);
296
297 features[0] = normalize01(meanLum);
298 features[1] = normalize01(meanR);
299 features[2] = normalize01(meanG);
300 features[3] = normalize01(meanB);
301 features[4] = clamp(varianceLum * 10 - 1, -1, 1);
302 features[5] = clamp(edgeX / totalPixels * 6 - 1, -1, 1);
303 features[6] = clamp(edgeY / totalPixels * 6 - 1, -1, 1);
304 features[7] = clamp(hueSinSum / totalPixels, -1, 1);
305 features[8] = clamp(hueCosSum / totalPixels, -1, 1);
306 features[9] = clamp(satSum / totalPixels * 2 - 1, -1, 1);
307 features[10] = tileX / Math.max(1, TILE_GRID_X - 1) * 2 - 1;
308 features[11] = tileY / Math.max(1, TILE_GRID_Y - 1) * 2 - 1;
309
310 return features;
311}
312
313function buildPcmField(rgba, width, height) {
314 const totalPixels = Math.max(1, width * height);
315 const field = new Float32Array(totalPixels * 3);
316 let energy = 0;
317
318 for (let pixelIndex = 0; pixelIndex < totalPixels; pixelIndex += 1) {
319 const rgbaOffset = pixelIndex * 4;
320 const writeOffset = pixelIndex * 3;
321 const r = rgba[rgbaOffset] / 127.5 - 1;
322 const g = rgba[rgbaOffset + 1] / 127.5 - 1;
323 const b = rgba[rgbaOffset + 2] / 127.5 - 1;
324 field[writeOffset] = r;
325 field[writeOffset + 1] = g;
326 field[writeOffset + 2] = b;
327 energy += Math.abs(r) + Math.abs(g) + Math.abs(b);
328 }
329
330 return {
331 field,
332 energy: energy / field.length,
333 };
334}
335
336function samplePcmField(field, phase) {
337 const scaled = wrap01(phase) * field.length;
338 const index = Math.floor(scaled);
339 const nextIndex = (index + 1) % field.length;
340 const frac = scaled - index;
341 return lerp(field[index], field[nextIndex], frac);
342}
343
344function encodeLatentField(rgba, width, height, previousField, globalLatent, encoderNetwork, style) {
345 const input = new Float32Array(TILE_FEATURE_COUNT + STATE_LATENT_SIZE + LATENT_FIELD_CHANNELS);
346 const nextField = new Float32Array(previousField.length);
347 const summary = new Float32Array(LATENT_FIELD_CHANNELS);
348 let flux = 0;
349 let energy = 0;
350
351 for (let tileY = 0; tileY < TILE_GRID_Y; tileY += 1) {
352 for (let tileX = 0; tileX < TILE_GRID_X; tileX += 1) {
353 const tileFeatures = extractTileFeatures(rgba, width, height, tileX, tileY);
354 const tileIndex = tileY * TILE_GRID_X + tileX;
355 const latentOffset = tileIndex * LATENT_FIELD_CHANNELS;
356 const previousLatent = previousField.subarray(latentOffset, latentOffset + LATENT_FIELD_CHANNELS);
357 input.set(tileFeatures, 0);
358 input.set(globalLatent, TILE_FEATURE_COUNT);
359 input.set(previousLatent, TILE_FEATURE_COUNT + STATE_LATENT_SIZE);
360 const encoded = forwardNetwork(encoderNetwork, input);
361
362 for (let channel = 0; channel < LATENT_FIELD_CHANNELS; channel += 1) {
363 const directFeature = tileFeatures[channel % TILE_FEATURE_COUNT];
364 const encodedValue = lerp(directFeature, encoded[channel], style.fieldBlend);
365 const nextValue = clamp(previousLatent[channel] * style.fieldMemory + encodedValue * (1 - style.fieldMemory), -1, 1);
366 nextField[latentOffset + channel] = nextValue;
367 summary[channel] += nextValue;
368 flux += Math.abs(nextValue - previousLatent[channel]);
369 energy += Math.abs(nextValue);
370 }
371 }
372 }
373
374 const divisor = TILE_GRID_X * TILE_GRID_Y;
375 for (let channel = 0; channel < summary.length; channel += 1) {
376 summary[channel] /= divisor;
377 }
378
379 return {
380 field: nextField,
381 summary,
382 flux: flux / nextField.length,
383 energy: energy / nextField.length,
384 };
385}
386
387function sampleLatentField(field, x, y, out) {
388 const fx = wrap01(x) * TILE_GRID_X;
389 const fy = wrap01(y) * TILE_GRID_Y;
390 const x0 = Math.floor(fx) % TILE_GRID_X;
391 const y0 = Math.floor(fy) % TILE_GRID_Y;
392 const x1 = (x0 + 1) % TILE_GRID_X;
393 const y1 = (y0 + 1) % TILE_GRID_Y;
394 const tx = fx - Math.floor(fx);
395 const ty = fy - Math.floor(fy);
396
397 const index00 = (y0 * TILE_GRID_X + x0) * LATENT_FIELD_CHANNELS;
398 const index10 = (y0 * TILE_GRID_X + x1) * LATENT_FIELD_CHANNELS;
399 const index01 = (y1 * TILE_GRID_X + x0) * LATENT_FIELD_CHANNELS;
400 const index11 = (y1 * TILE_GRID_X + x1) * LATENT_FIELD_CHANNELS;
401
402 for (let channel = 0; channel < LATENT_FIELD_CHANNELS; channel += 1) {
403 const a = lerp(field[index00 + channel], field[index10 + channel], tx);
404 const b = lerp(field[index01 + channel], field[index11 + channel], tx);
405 out[channel] = lerp(a, b, ty);
406 }
407
408 return out;
409}
410
411function interpretRules(output, features, fieldSummary) {
412 const brightness = (features[0] + 1) * 0.5;
413 const edge = ((features[5] + 1) * 0.5 + (features[6] + 1) * 0.5) * 0.5;
414 const spread = (features[11] + 1) * 0.5;
415 const fieldColor = (Math.abs(fieldSummary[1]) + Math.abs(fieldSummary[2]) + Math.abs(fieldSummary[3])) / 3;
416 const fieldMotion = (Math.abs(fieldSummary[4]) + Math.abs(fieldSummary[5])) * 0.5;
417
418 return {
419 grow: mapSigned(output[8], 0.02, 0.2) * (0.7 + brightness * 0.6),
420 diffuse: mapSigned(output[9], 0.01, 0.26) * (0.7 + edge * 0.5),
421 decay: mapSigned(output[10], 0.004, 0.07),
422 excite: mapSigned(output[11], 0.04, 0.92),
423 coupling: mapSigned(output[12], 0.08, 0.88),
424 strideA: Math.max(1, Math.round(mapSigned(output[13], 1, 19))),
425 strideB: Math.max(1, Math.round(mapSigned(output[14], 3, 29))),
426 shiftA: Math.max(1, Math.round(mapSigned(output[15], 2, 9))),
427 shiftB: Math.max(1, Math.round(mapSigned(output[16], 3, 13))),
428 shiftC: Math.max(1, Math.round(mapSigned(output[17], 4, 17))),
429 mulA: Math.max(1, Math.round(mapSigned(output[18], 3, 61))),
430 mulB: Math.max(1, Math.round(mapSigned(output[19], 5, 83))),
431 mask: Math.max(31, Math.round(mapSigned(output[20], 31, 255))),
432 byteMix: mapSigned(output[21], 0.08, 0.8),
433 petriMix: mapSigned(output[22], 0.06, 0.76),
434 panSkew: clamp(output[23] + features[7] * 0.25, -1, 1),
435 sparkle: clamp(spread * 0.55 + brightness * 0.25 + fieldColor * 0.2, 0, 1),
436 tonalMix: mapSigned(output[24], 0.16, 0.96),
437 vocalMix: mapSigned(output[25], 0.1, 0.92),
438 tableMix: mapSigned(output[26], 0.22, 1.05),
439 livingMix: mapSigned(output[27], 0.08, 0.88),
440 scanRateX: mapSigned(output[28], -0.42, 0.42) * (0.45 + edge * 0.5),
441 scanRateY: mapSigned(output[29], -0.42, 0.42) * (0.45 + spread * 0.5),
442 scanWarp: mapSigned(output[30], 0.08, 2.4),
443 scanOrbit: mapSigned(output[31], 0.02, 0.34),
444 basePitch: mapSigned(output[32], 42, 420) * (0.72 + brightness * 0.42 + fieldColor * 0.12),
445 pitchSpread: mapSigned(output[33], 0.2, 2.8),
446 breath: mapSigned(output[34], 0.04, 0.88),
447 formantShift: mapSigned(output[35], 0.78, 1.44),
448 tableRate: mapSigned(output[36], 0.24, 2.8),
449 tableWarp: mapSigned(output[37], 0.08, 3.2),
450 latentDrift: mapSigned(output[38], 0.02, 0.28) * (0.6 + fieldMotion * 0.5),
451 stereoDrift: clamp(output[39], -1, 1),
452 };
453}
454
455function seedPetriDish(cells, features, latent) {
456 for (let i = 0; i < cells.length; i += 1) {
457 const feature = features[i % features.length];
458 const memory = latent[i % latent.length];
459 cells[i] = clamp(cells[i] * 0.7 + feature * 0.2 + memory * 0.1, -1, 1);
460 }
461}
462
463function evolvePetriDish(state, features, rules, sampleIndex) {
464 const current = state.cells;
465 const next = state.nextCells;
466 const latent = state.latent;
467 const featureOffset = sampleIndex % features.length;
468
469 for (let i = 0; i < current.length; i += 1) {
470 const left = current[(i + current.length - 1) % current.length];
471 const center = current[i];
472 const right = current[(i + 1) % current.length];
473 const feature = features[(featureOffset + i * 3) % features.length];
474 const memory = latent[i % latent.length];
475 const reagent = feature * rules.excite + memory * rules.coupling;
476 const growth = tanh(left * 0.9 + center * (0.4 + rules.sparkle) + right * 0.9 + reagent);
477 const diffusion = (left + right - 2 * center) * rules.diffuse;
478 next[i] = clamp(center * (1 - rules.decay) + growth * rules.grow + diffusion, -1, 1);
479 }
480
481 state.cells = next;
482 state.nextCells = current;
483}
484
485function bytebeatSample(t, rules, petriByteA, petriByteB) {
486 return (
487 (((t * rules.mulA) & ((t >> rules.shiftA) | petriByteA)) ^
488 ((t * rules.mulB) & (t >> rules.shiftB)) ^
489 ((t + petriByteB) >> rules.shiftC)) & rules.mask
490 ) & 255;
491}
492
493function blendRules(a, b, mix, style) {
494 return {
495 grow: lerp(a.grow, b.grow, mix) * style.growScale,
496 diffuse: lerp(a.diffuse, b.diffuse, mix) * style.diffuseScale,
497 decay: lerp(a.decay, b.decay, mix),
498 excite: lerp(a.excite, b.excite, mix) * style.exciteScale,
499 coupling: lerp(a.coupling, b.coupling, mix) * style.couplingScale,
500 strideA: Math.round(lerp(a.strideA, b.strideA, mix)),
501 strideB: Math.round(lerp(a.strideB, b.strideB, mix)),
502 shiftA: Math.round(lerp(a.shiftA, b.shiftA, mix)),
503 shiftB: Math.round(lerp(a.shiftB, b.shiftB, mix)),
504 shiftC: Math.round(lerp(a.shiftC, b.shiftC, mix)),
505 mulA: Math.round(lerp(a.mulA, b.mulA, mix)),
506 mulB: Math.round(lerp(a.mulB, b.mulB, mix)),
507 mask: Math.round(lerp(a.mask, b.mask, mix)),
508 byteMix: clamp(lerp(a.byteMix, b.byteMix, mix), 0.05, 1.2),
509 petriMix: clamp(lerp(a.petriMix, b.petriMix, mix), 0.05, 1.25),
510 panSkew: lerp(a.panSkew, b.panSkew, mix),
511 sparkle: lerp(a.sparkle, b.sparkle, mix),
512 tonalMix: clamp(lerp(a.tonalMix, b.tonalMix, mix), 0.02, 1.2),
513 vocalMix: clamp(lerp(a.vocalMix, b.vocalMix, mix), 0.02, 1.2),
514 tableMix: clamp(lerp(a.tableMix, b.tableMix, mix), 0.02, 1.2),
515 livingMix: clamp(lerp(a.livingMix, b.livingMix, mix), 0.02, 1.2),
516 scanRateX: lerp(a.scanRateX, b.scanRateX, mix),
517 scanRateY: lerp(a.scanRateY, b.scanRateY, mix),
518 scanWarp: lerp(a.scanWarp, b.scanWarp, mix),
519 scanOrbit: lerp(a.scanOrbit, b.scanOrbit, mix),
520 basePitch: lerp(a.basePitch, b.basePitch, mix),
521 pitchSpread: lerp(a.pitchSpread, b.pitchSpread, mix),
522 breath: lerp(a.breath, b.breath, mix),
523 formantShift: lerp(a.formantShift, b.formantShift, mix),
524 tableRate: lerp(a.tableRate, b.tableRate, mix),
525 tableWarp: lerp(a.tableWarp, b.tableWarp, mix),
526 latentDrift: lerp(a.latentDrift, b.latentDrift, mix),
527 stereoDrift: lerp(a.stereoDrift, b.stereoDrift, mix),
528 };
529}
530
531function normalizeWeights(values) {
532 const output = new Float32Array(values.length);
533 let sum = 0;
534
535 for (let i = 0; i < values.length; i += 1) {
536 const value = Math.max(0.0001, values[i]);
537 output[i] = value;
538 sum += value;
539 }
540
541 for (let i = 0; i < output.length; i += 1) {
542 output[i] /= sum;
543 }
544
545 return output;
546}
547
548function deriveExpertWeights(rules, latentVec, style) {
549 return normalizeWeights([
550 rules.tonalMix * style.tonalMixScale * (0.52 + (latentVec[0] + 1) * 0.2 + Math.abs(latentVec[6]) * 0.12),
551 rules.vocalMix * style.vocalMixScale * (0.48 + (latentVec[1] + 1) * 0.18 + rules.breath * 0.2),
552 rules.tableMix * style.tableMixScale * (0.55 + (latentVec[2] + 1) * 0.18 + Math.abs(latentVec[7]) * 0.14),
553 rules.livingMix * style.livingMixScale * (0.42 + (latentVec[3] + 1) * 0.18 + rules.sparkle * 0.2),
554 ]);
555}
556
557function stepAdditive(channelState, latentVec, rules, sampleRate, stereoOffset) {
558 const basePitch = clamp(
559 rules.basePitch * Math.pow(2, latentVec[0] * rules.pitchSpread * 0.3) * (1 + stereoOffset * 0.015),
560 24,
561 sampleRate * 0.45,
562 );
563
564 let sum = 0;
565 let ampSum = 0;
566
567 for (let partial = 0; partial < ADDITIVE_PARTIALS; partial += 1) {
568 const ratio = 1 + partial * (0.78 + (latentVec[(partial + 2) % latentVec.length] + 1) * 0.22);
569 const detune = 1 + stereoOffset * 0.012 * (partial + 1) + latentVec[(partial + 5) % latentVec.length] * 0.004;
570 const frequency = clamp(basePitch * ratio * detune, 24, sampleRate * 0.45);
571 channelState.phases[partial] = (channelState.phases[partial] + TAU * frequency / sampleRate) % TAU;
572 const amplitude = (0.28 + (latentVec[(partial + 7) % latentVec.length] + 1) * 0.18) / (partial + 1);
573 sum += Math.sin(channelState.phases[partial]) * amplitude;
574 ampSum += amplitude;
575 }
576
577 return ampSum > 0 ? sum / ampSum : 0;
578}
579
580function resonatorStep(frequency, bandwidth, input, state, offset, sampleRate) {
581 const clampedFrequency = clamp(frequency, 40, sampleRate * 0.45);
582 const radius = clamp(Math.exp(-Math.PI * bandwidth / sampleRate), 0.7, 0.9995);
583 const coefficient = 2 * radius * Math.cos(TAU * clampedFrequency / sampleRate);
584 const output = input + coefficient * state[offset] - radius * radius * state[offset + 1];
585 state[offset + 1] = state[offset];
586 state[offset] = output;
587 return output;
588}
589
590function stepVocal(channelState, latentVec, rules, sampleRate, noiseValue, style, stereoOffset) {
591 const basePitch = clamp(
592 rules.basePitch * (0.45 + (latentVec[4] + 1) * 0.18) * (1 + stereoOffset * 0.02),
593 55,
594 720,
595 );
596 channelState.phase = (channelState.phase + TAU * basePitch / sampleRate) % TAU;
597
598 const voiced =
599 Math.sin(channelState.phase) * 0.78 +
600 Math.sin(channelState.phase * 2 + latentVec[5] * 0.8) * 0.26 +
601 Math.sin(channelState.phase * 3 + latentVec[6] * 0.4) * 0.12;
602 const aspiration = noiseValue * (0.12 + rules.breath * 0.42) + voiced * (0.86 - rules.breath * 0.34);
603 const formantShift = rules.formantShift * style.formantWarmth * (1 + latentVec[7] * 0.08);
604 const bandwidthTilt = 1 + Math.abs(latentVec[8]) * 0.5 + rules.breath * 0.35;
605 const formants = [
606 mapSigned(latentVec[1], 260, 880) * formantShift,
607 mapSigned(latentVec[2], 900, 2400) * formantShift,
608 mapSigned(latentVec[3], 1800, 3600) * formantShift,
609 ];
610 const bandwidths = [90, 140, 200].map((value) => value * bandwidthTilt);
611 let output = 0;
612
613 for (let index = 0; index < FORMANT_COUNT; index += 1) {
614 output += resonatorStep(
615 formants[index],
616 bandwidths[index],
617 aspiration * (0.45 - index * 0.08),
618 channelState.resonators,
619 index * 2,
620 sampleRate,
621 );
622 }
623
624 return clamp(output * 0.08, -1, 1);
625}
626
627function stepTable(channelState, pcmField, latentVec, rules, sampleRate, headX, headY, style, stereoOffset) {
628 const playbackHz = clamp(
629 rules.basePitch * rules.tableRate * (0.3 + (latentVec[0] + 1) * 0.24) * (1 + stereoOffset * 0.02),
630 18,
631 sampleRate * 0.45,
632 );
633 channelState.phase = wrap01(channelState.phase + playbackHz / sampleRate);
634 const warpAmount = rules.tableWarp * style.tableWarpScale;
635 const warpedPhase = wrap01(
636 channelState.phase +
637 Math.sin(channelState.phase * TAU * (1.1 + Math.abs(latentVec[3]) * 1.8) + headY * TAU) * 0.025 * warpAmount +
638 headX * 0.17 +
639 headY * 0.09 +
640 latentVec[4] * 0.04,
641 );
642 const primary = samplePcmField(pcmField, warpedPhase);
643 const secondary = samplePcmField(
644 pcmField,
645 wrap01(warpedPhase * (1.01 + latentVec[5] * 0.03) + latentVec[6] * 0.05 + stereoOffset * 0.01),
646 );
647 return clamp(primary * 0.72 + secondary * 0.28, -1, 1);
648}
649
650function writeAscii(view, offset, text) {
651 for (let i = 0; i < text.length; i += 1) {
652 view.setUint8(offset + i, text.charCodeAt(i));
653 }
654}
655
656export function createSonicFrameEngine(options = {}) {
657 const source = options.source || "";
658 const fps = options.fps || 30;
659 const sampleRate = options.sampleRate || 48000;
660 const width = options.width || 128;
661 const height = options.height || 128;
662 const seed = options.seed ?? hashString(source || "kidlisp-wasm-sonic-frame");
663 const style = SOUND_STYLES[options.style] || SOUND_STYLES.default;
664 const controlNetwork = buildNetwork(seed ^ 0x9e3779b9, GLOBAL_FEATURE_COUNT + STATE_LATENT_SIZE, [32, 32], OUTPUT_SIZE);
665 const encoderNetwork = buildNetwork(
666 seed ^ 0x85ebca6b,
667 TILE_FEATURE_COUNT + STATE_LATENT_SIZE + LATENT_FIELD_CHANNELS,
668 [24, 24],
669 LATENT_FIELD_CHANNELS,
670 );
671 const jitter = createSeededRandom(seed ^ 0xc2b2ae35);
672 const noise = createSeededRandom(seed ^ 0x27d4eb2f);
673
674 let cells = new Float32Array(CELL_COUNT);
675 let nextCells = new Float32Array(CELL_COUNT);
676 let globalLatent = new Float32Array(STATE_LATENT_SIZE);
677 let latentField = new Float32Array(TILE_GRID_X * TILE_GRID_Y * LATENT_FIELD_CHANNELS);
678 let sampleClock = 0;
679 let byteLeftState = 0;
680 let byteRightState = 0;
681 let smoothLeft = 0;
682 let smoothRight = 0;
683 let previousRules = null;
684 const tonalLeftState = { phases: new Float32Array(ADDITIVE_PARTIALS) };
685 const tonalRightState = { phases: new Float32Array(ADDITIVE_PARTIALS) };
686 const vocalLeftState = { phase: 0, resonators: new Float32Array(FORMANT_COUNT * 2) };
687 const vocalRightState = { phase: 0, resonators: new Float32Array(FORMANT_COUNT * 2) };
688 const tableLeftState = { phase: jitter() };
689 const tableRightState = { phase: jitter() };
690
691 for (let i = 0; i < cells.length; i += 1) {
692 cells[i] = jitter() * 2 - 1;
693 }
694
695 for (let i = 0; i < globalLatent.length; i += 1) {
696 globalLatent[i] = jitter() * 2 - 1;
697 }
698
699 for (let i = 0; i < latentField.length; i += 1) {
700 latentField[i] = jitter() * 2 - 1;
701 }
702
703 return {
704 synthesizeFrame(rgba, frameIndex) {
705 const features = extractFramebufferFeatures(rgba, width, height);
706 const controlInput = new Float32Array(GLOBAL_FEATURE_COUNT + STATE_LATENT_SIZE);
707 controlInput.set(features, 0);
708 controlInput.set(globalLatent, GLOBAL_FEATURE_COUNT);
709
710 const controlOutput = forwardNetwork(controlNetwork, controlInput);
711 const nextGlobalLatent = new Float32Array(STATE_LATENT_SIZE);
712 for (let i = 0; i < STATE_LATENT_SIZE; i += 1) {
713 nextGlobalLatent[i] = clamp(lerp(globalLatent[i], controlOutput[i], style.latentBlend), -1, 1);
714 }
715 globalLatent = nextGlobalLatent;
716
717 const { field: nextField, summary: fieldSummary, flux: fieldFlux, energy: fieldEnergy } = encodeLatentField(
718 rgba,
719 width,
720 height,
721 latentField,
722 globalLatent,
723 encoderNetwork,
724 style,
725 );
726 latentField = nextField;
727
728 const rules = interpretRules(controlOutput, features, fieldSummary);
729 seedPetriDish(cells, features, globalLatent);
730 const pcm = buildPcmField(rgba, width, height);
731
732 const frameStart = Math.round(frameIndex * sampleRate / fps);
733 const frameEnd = Math.round((frameIndex + 1) * sampleRate / fps);
734 const sampleCount = Math.max(1, frameEnd - frameStart);
735 const left = new Float32Array(sampleCount);
736 const right = new Float32Array(sampleCount);
737 const lastRules = previousRules || rules;
738 const state = { cells, nextCells, latent: globalLatent };
739 const latentLeft = new Float32Array(LATENT_FIELD_CHANNELS);
740 const latentRight = new Float32Array(LATENT_FIELD_CHANNELS);
741 const latentMix = new Float32Array(LATENT_FIELD_CHANNELS);
742 const expertSums = new Float32Array(EXPERT_NAMES.length);
743 let leftPower = 0;
744 let rightPower = 0;
745 let stereoDiff = 0;
746 let motionAccumulator = 0;
747
748 for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex += 1) {
749 const mix = sampleCount === 1 ? 1 : sampleIndex / (sampleCount - 1);
750 const blendedRules = blendRules(lastRules, rules, mix, style);
751 evolvePetriDish(state, features, blendedRules, sampleIndex);
752 cells = state.cells;
753 nextCells = state.nextCells;
754
755 const absoluteTime = sampleClock / sampleRate;
756 const scanDrift = frameIndex / Math.max(1, fps) * blendedRules.latentDrift;
757 const orbitPhase = absoluteTime * (0.35 + blendedRules.scanWarp * 0.3) + globalLatent[0];
758 const orbitX = Math.sin(orbitPhase + globalLatent[1] * 0.7) * blendedRules.scanOrbit;
759 const orbitY = Math.cos(orbitPhase * 1.17 + globalLatent[2] * 0.6) * blendedRules.scanOrbit;
760 const headX = wrap01((features[7] * 0.5 + 0.5) + scanDrift + absoluteTime * blendedRules.scanRateX + orbitX);
761 const headY = wrap01((features[8] * 0.5 + 0.5) - scanDrift + absoluteTime * blendedRules.scanRateY + orbitY);
762 const stereoSpread = 0.04 + Math.abs(blendedRules.stereoDrift) * 0.1;
763 const leftX = wrap01(headX - stereoSpread + globalLatent[3] * 0.03);
764 const leftY = wrap01(headY + stereoSpread * 0.5 + globalLatent[4] * 0.03);
765 const rightX = wrap01(headX + stereoSpread + globalLatent[5] * 0.03);
766 const rightY = wrap01(headY - stereoSpread * 0.5 + globalLatent[6] * 0.03);
767
768 sampleLatentField(latentField, leftX, leftY, latentLeft);
769 sampleLatentField(latentField, rightX, rightY, latentRight);
770 for (let i = 0; i < LATENT_FIELD_CHANNELS; i += 1) {
771 latentMix[i] = (latentLeft[i] + latentRight[i]) * 0.5;
772 }
773
774 const expertWeights = deriveExpertWeights(blendedRules, latentMix, style);
775 for (let i = 0; i < expertWeights.length; i += 1) {
776 expertSums[i] += expertWeights[i];
777 }
778
779 const tonalLeft = stepAdditive(tonalLeftState, latentLeft, blendedRules, sampleRate, -1);
780 const tonalRight = stepAdditive(tonalRightState, latentRight, blendedRules, sampleRate, 1);
781 const noiseLeft = noise() * 2 - 1;
782 const noiseRight = noise() * 2 - 1;
783 const vocalLeft = stepVocal(vocalLeftState, latentLeft, blendedRules, sampleRate, noiseLeft, style, -1);
784 const vocalRight = stepVocal(vocalRightState, latentRight, blendedRules, sampleRate, noiseRight, style, 1);
785 const tableLeft = stepTable(tableLeftState, pcm.field, latentLeft, blendedRules, sampleRate, leftX, leftY, style, -1);
786 const tableRight = stepTable(tableRightState, pcm.field, latentRight, blendedRules, sampleRate, rightX, rightY, style, 1);
787
788 const t = sampleClock;
789 const petriIndexA = (t * blendedRules.strideA + sampleIndex) % cells.length;
790 const petriIndexB = (t * blendedRules.strideB + sampleIndex * 3) % cells.length;
791 const petriA = cells[petriIndexA];
792 const petriB = cells[petriIndexB];
793 const petriByteA = Math.floor((petriA * 0.5 + 0.5) * 255) & 255;
794 const petriByteB = Math.floor((petriB * 0.5 + 0.5) * 255) & 255;
795 const byteLeftRaw = bytebeatSample(t, blendedRules, petriByteA, petriByteB) / 127.5 - 1;
796 const byteRightRaw = bytebeatSample(t + 17, blendedRules, petriByteB, petriByteA) / 127.5 - 1;
797 const byteLeftShaped = lerp(byteLeftRaw, Math.sin(byteLeftRaw * Math.PI * 0.5), style.byteSoftness);
798 const byteRightShaped = lerp(byteRightRaw, Math.sin(byteRightRaw * Math.PI * 0.5), style.byteSoftness);
799
800 byteLeftState = byteLeftState * style.byteLowpass + byteLeftShaped * (1 - style.byteLowpass);
801 byteRightState = byteRightState * style.byteLowpass + byteRightShaped * (1 - style.byteLowpass);
802
803 const livingLeft = clamp(
804 byteLeftState * blendedRules.byteMix * style.byteMixScale * style.byteHarmonicsScale +
805 petriA * blendedRules.petriMix * style.petriMixScale,
806 -1,
807 1,
808 );
809 const livingRight = clamp(
810 byteRightState * blendedRules.byteMix * style.byteMixScale * style.byteHarmonicsScale +
811 petriB * blendedRules.petriMix * style.petriMixScale,
812 -1,
813 1,
814 );
815
816 const rawLeft =
817 tonalLeft * expertWeights[0] * 0.86 +
818 vocalLeft * expertWeights[1] * 0.96 +
819 tableLeft * expertWeights[2] * 0.92 +
820 livingLeft * expertWeights[3] * 0.84;
821 const rawRight =
822 tonalRight * expertWeights[0] * 0.86 +
823 vocalRight * expertWeights[1] * 0.96 +
824 tableRight * expertWeights[2] * 0.92 +
825 livingRight * expertWeights[3] * 0.84;
826
827 const pan = clamp(0.5 + blendedRules.panSkew * 0.32 * style.panScale, 0.12, 0.88);
828 const mixedLeft = rawLeft * (1 - pan * 0.18) + rawRight * pan * 0.08 + petriB * 0.04;
829 const mixedRight = rawRight * (0.82 + pan * 0.18) + rawLeft * (1 - pan) * 0.08 + petriA * 0.04;
830 const gain = (0.34 + blendedRules.sparkle * 0.06 * style.sparkleScale) * style.gainScale;
831
832 smoothLeft = smoothLeft * style.outputSmoothing + mixedLeft * (1 - style.outputSmoothing);
833 smoothRight = smoothRight * style.outputSmoothing + mixedRight * (1 - style.outputSmoothing);
834
835 left[sampleIndex] = clamp(tanh(smoothLeft * gain), -1, 1);
836 right[sampleIndex] = clamp(tanh(smoothRight * gain), -1, 1);
837 leftPower += left[sampleIndex] * left[sampleIndex];
838 rightPower += right[sampleIndex] * right[sampleIndex];
839 stereoDiff += Math.abs(left[sampleIndex] - right[sampleIndex]);
840 motionAccumulator += Math.abs(orbitX) + Math.abs(orbitY);
841 sampleClock += 1;
842 }
843
844 previousRules = rules;
845 return {
846 left,
847 right,
848 rules,
849 features,
850 analysis: {
851 expertNames: DECODER_NAMES,
852 expertMix: Array.from(expertSums, (sum) => sum / sampleCount),
853 rmsLeft: Math.sqrt(leftPower / sampleCount),
854 rmsRight: Math.sqrt(rightPower / sampleCount),
855 stereoSpread: stereoDiff / sampleCount,
856 latentFlux: fieldFlux,
857 latentEnergy: fieldEnergy,
858 pcmEnergy: pcm.energy,
859 motionSpread: motionAccumulator / sampleCount,
860 },
861 };
862 },
863 };
864}
865
866export function encodeStereoWav(leftChunks, rightChunks, sampleRate = 48000) {
867 const totalSamples = leftChunks.reduce((sum, chunk) => sum + chunk.length, 0);
868 const bytesPerSample = 2;
869 const numChannels = 2;
870 const dataSize = totalSamples * numChannels * bytesPerSample;
871 const buffer = new ArrayBuffer(44 + dataSize);
872 const view = new DataView(buffer);
873
874 writeAscii(view, 0, "RIFF");
875 view.setUint32(4, 36 + dataSize, true);
876 writeAscii(view, 8, "WAVE");
877 writeAscii(view, 12, "fmt ");
878 view.setUint32(16, 16, true);
879 view.setUint16(20, 1, true);
880 view.setUint16(22, numChannels, true);
881 view.setUint32(24, sampleRate, true);
882 view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
883 view.setUint16(32, numChannels * bytesPerSample, true);
884 view.setUint16(34, 16, true);
885 writeAscii(view, 36, "data");
886 view.setUint32(40, dataSize, true);
887
888 let offset = 44;
889 for (let i = 0; i < leftChunks.length; i += 1) {
890 const left = leftChunks[i];
891 const right = rightChunks[i];
892 for (let sample = 0; sample < left.length; sample += 1) {
893 view.setInt16(offset, clamp(left[sample], -1, 1) * 0x7fff, true);
894 offset += 2;
895 view.setInt16(offset, clamp(right[sample], -1, 1) * 0x7fff, true);
896 offset += 2;
897 }
898 }
899
900 return Buffer.from(buffer);
901}