+18

ac-m4l/README.md

reviewed

··· 42 42 } 43 43 ``` 44 44 45 45 + ### Remote Relay Playback 46 46 + 47 47 + `notepat` now understands relay subscription parameters for live note playback from `ac-native` sources: 48 48 + 49 49 + ```text 50 50 + https://localhost:8888/notepat?daw=1&relayHandle=jeffrey 51 51 + https://localhost:8888/notepat?daw=1&relayHandle=jeffrey&relayMachine=ac-1234abcd 52 52 + https://localhost:8888/notepat?daw=1&relayAll=1 53 53 + ``` 54 54 + 55 55 + You can also retarget a running `jweb~` instance with `postMessage`: 56 56 + 57 57 + ```js 58 58 + window.postMessage({ type: "notepat:midi:set-source", handle: "jeffrey" }, "*"); 59 59 + window.postMessage({ type: "notepat:midi:set-source", handle: "jeffrey", machineId: "ac-1234abcd" }, "*"); 60 60 + window.postMessage({ type: "notepat:midi:sources" }, "*"); 61 61 + ``` 62 62 + 45 63 ### Requirements 46 64 47 65 - Ableton Live with Max for Live

+1 -1

fedac/native/pieces/list.mjs

reviewed

··· 29 29 { name: "bye", desc: "log out" }, 30 30 { name: "version", desc: "OS hash" }, 31 31 { name: "ssh", desc: "SSH server" }, 32 32 - { name: "midi", desc: "usb midi gadget" }, 32 32 + { name: "midi", desc: "usb midi + udp relay" }, 33 33 { name: "clear", desc: "clear history" }, 34 34 { name: "help", desc: "quick help" }, 35 35 ];

+52

fedac/native/pieces/notepat.mjs

reviewed

··· 110 110 let usbMidiRecent = []; 111 111 let usbMidiNextRefreshFrame = 0; 112 112 let usbMidiTypecSignature = ""; 113 113 + let udpMidiBroadcast = false; 114 114 + let udpMidiNextHeartbeatFrame = 0; 113 115 114 116 115 117 // OS update panel state machine ··· 286 288 return "USB MIDI OFF"; 287 289 } 288 290 291 291 + function loadUdpMidiConfig(system) { 292 292 + try { 293 293 + const raw = system?.readFile?.("/mnt/config.json"); 294 294 + if (!raw) { 295 295 + udpMidiBroadcast = false; 296 296 + return; 297 297 + } 298 298 + const cfg = JSON.parse(raw); 299 299 + udpMidiBroadcast = cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; 300 300 + } catch (_) { 301 301 + udpMidiBroadcast = false; 302 302 + } 303 303 + } 304 304 + 305 305 + function sendUdpMidiEvent(system, event, midiNote, velocity, channel = 0) { 306 306 + if (!udpMidiBroadcast || !system?.udp?.connected) return; 307 307 + system?.udp?.sendMidi?.(event, midiNote, velocity, channel, "notepat"); 308 308 + } 309 309 + 310 310 + function maybeSendUdpMidiHeartbeat(system) { 311 311 + if (!udpMidiBroadcast || !system?.udp?.connected) return; 312 312 + if (frame < udpMidiNextHeartbeatFrame) return; 313 313 + system?.udp?.sendMidiHeartbeat?.("notepat"); 314 314 + udpMidiNextHeartbeatFrame = frame + 300; 315 315 + } 316 316 + 317 317 + function udpMidiRelayStatusText(system) { 318 318 + if (!udpMidiBroadcast) return ""; 319 319 + const handle = system?.udp?.handle || system?.config?.handle || ""; 320 320 + if (system?.udp?.connected) return handle ? "relay @" + handle : "relay on"; 321 321 + return handle ? "relay ...@" + handle : "relay ..."; 322 322 + } 323 323 + 289 324 function rememberSound(key, entry, system, velocity = 1) { 290 325 if (!entry) return; 291 326 entry.midiNote = noteToMidiNumber(entry.note, entry.octave); 292 327 entry.midiChannel = 0; 293 328 sounds[key] = entry; 294 329 system?.usbMidi?.noteOn?.(entry.midiNote, velocityToMidi(velocity), entry.midiChannel); 330 330 + sendUdpMidiEvent(system, "note_on", entry.midiNote, velocityToMidi(velocity), entry.midiChannel); 295 331 pushUsbMidiRecent(">", entry.note, entry.octave); 296 332 } 297 333 ··· 305 341 } 306 342 if (entry.midiNote !== undefined) { 307 343 system?.usbMidi?.noteOff?.(entry.midiNote, 0, entry.midiChannel || 0); 344 344 + sendUdpMidiEvent(system, "note_off", entry.midiNote, 0, entry.midiChannel || 0); 308 345 pushUsbMidiRecent("<", entry.note, entry.octave); 309 346 } 310 347 delete sounds[key]; ··· 441 478 wipe(0); 442 479 soundAPI = sound; 443 480 systemAPI = system; 481 481 + loadUdpMidiConfig(system); 482 482 + udpMidiNextHeartbeatFrame = 0; 444 483 const mic = sound?.microphone || null; 445 484 if (mic && (mic.sampleLength || 0) > 0) { 446 485 sampleLoaded = true; ··· 1266 1305 statusWrite(usbMidiText, FG_DIM, FG_DIM, FG_DIM, 200); 1267 1306 } 1268 1307 1308 1308 + const relayText = udpMidiRelayStatusText(system); 1309 1309 + if (relayText) { 1310 1310 + statusWrite( 1311 1311 + relayText, 1312 1312 + system?.udp?.connected ? 80 : 255, 1313 1313 + system?.udp?.connected ? 180 : 180, 1314 1314 + system?.udp?.connected ? 255 : 90, 1315 1315 + 210 1316 1316 + ); 1317 1317 + } 1318 1318 + 1269 1319 // Metronome indicator (pendulum) in status bar — shown when enabled 1270 1320 if (metronomeEnabled) { 1271 1321 const bpmLabel = metronomeBPM + "b"; ··· 1437 1487 system.ws?.connect("wss://chat-system.aesthetic.computer/"); 1438 1488 // Connect raw UDP for fairy co-presence 1439 1489 system.udp?.connect("session-server.aesthetic.computer", 10010); 1490 1490 + udpMidiNextHeartbeatFrame = frame + 30; 1440 1491 if (system?.writeFile && savedCreds.length > 0) { 1441 1492 system.writeFile(CREDS_PATH, JSON.stringify(savedCreds)); 1442 1493 } ··· 2508 2559 2509 2560 // 🧚 Fairy co-presence — send cursor + paint received fireflies 2510 2561 if (system.udp?.connected) { 2562 2562 + maybeSendUdpMidiHeartbeat(system); 2511 2563 // Send current cursor/touch position as fairy point (~60Hz, throttled by UDP thread) 2512 2564 if (hoverX >= 0 && hoverY >= 0) { 2513 2565 system.udp.sendFairy(hoverX / w, hoverY / h);

+48 -3

fedac/native/pieces/prompt.mjs

reviewed

··· 56 56 "theme": "prompt theme", 57 57 "voice": "system voice", 58 58 "login": "switch identity", 59 59 - "midi": "usb midi gadget", 59 59 + "midi": "usb midi + udp relay", 60 60 "dark": "dark mode", 61 61 "light": "light mode", 62 62 "auto": "auto dark/light", ··· 138 138 return "usb midi off (" + status.reason + ")"; 139 139 } 140 140 return "usb midi off"; 141 141 + } 142 142 + 143 143 + function readConfig(system) { 144 144 + try { 145 145 + const raw = system?.readFile?.("/mnt/config.json"); 146 146 + return raw ? JSON.parse(raw) : {}; 147 147 + } catch (_) { 148 148 + return {}; 149 149 + } 150 150 + } 151 151 + 152 152 + function udpMidiRelayEnabled(system) { 153 153 + const cfg = readConfig(system); 154 154 + return cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; 155 155 + } 156 156 + 157 157 + function formatUdpMidiRelayStatus(system) { 158 158 + const cfg = readConfig(system); 159 159 + const enabled = cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; 160 160 + if (!enabled) return "udp midi relay off"; 161 161 + if (cfg.handle) return "udp midi relay on @" + cfg.handle; 162 162 + return "udp midi relay on"; 141 163 } 142 164 143 165 function boot({ system }) { ··· 331 353 } 332 354 if (baseWord === "midi") { 333 355 const arg = spaceIdx >= 0 ? lower.slice(spaceIdx + 1).trim() : "status"; 356 356 + if (arg === "relay" || arg.startsWith("relay ")) { 357 357 + const relayArg = arg === "relay" ? "status" : arg.slice("relay ".length).trim(); 358 358 + if (!relayArg || relayArg === "status") { 359 359 + message = formatUdpMidiRelayStatus(system); 360 360 + messageFrame = 0; 361 361 + return; 362 362 + } 363 363 + if (relayArg === "on" || relayArg === "enable") { 364 364 + system?.saveConfig?.("udpMidiBroadcast", "true"); 365 365 + message = formatUdpMidiRelayStatus(system); 366 366 + messageFrame = 0; 367 367 + return; 368 368 + } 369 369 + if (relayArg === "off" || relayArg === "disable") { 370 370 + system?.saveConfig?.("udpMidiBroadcast", "false"); 371 371 + message = formatUdpMidiRelayStatus(system); 372 372 + messageFrame = 0; 373 373 + return; 374 374 + } 375 375 + message = "usage: midi relay [status|on|off]"; 376 376 + messageFrame = 0; 377 377 + return; 378 378 + } 334 379 if (!arg || arg === "status") { 335 335 - message = formatUsbMidiStatus(readUsbMidiStatus(system)); 380 380 + message = formatUsbMidiStatus(readUsbMidiStatus(system)) + " | " + formatUdpMidiRelayStatus(system); 336 381 messageFrame = 0; 337 382 return; 338 383 } ··· 356 401 messageFrame = 0; 357 402 return; 358 403 } 359 359 - message = "usage: midi [status|on|off|refresh]"; 404 404 + message = "usage: midi [status|on|off|refresh|relay]"; 360 405 messageFrame = 0; 361 406 return; 362 407 }

+62 -7

fedac/native/src/js-bindings.c

reviewed

··· 2567 2567 // system.udp — Raw UDP fairy point co-presence 2568 2568 // --------------------------------------------------------------------------- 2569 2569 2570 2570 + static void sync_udp_identity(void) { 2571 2571 + extern char g_machine_id[64]; 2572 2572 + 2573 2573 + if (!current_rt || !current_rt->udp) return; 2574 2574 + udp_set_identity( 2575 2575 + current_rt->udp, 2576 2576 + current_rt->handle[0] ? current_rt->handle : "", 2577 2577 + g_machine_id[0] ? g_machine_id : "unknown" 2578 2578 + ); 2579 2579 + } 2580 2580 + 2570 2581 static JSValue js_udp_connect(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { 2571 2582 (void)this_val; 2572 2583 if (!current_rt || !current_rt->udp) return JS_UNDEFINED; ··· 2574 2585 if (!host) host = "session-server.aesthetic.computer"; 2575 2586 int port = UDP_FAIRY_PORT; 2576 2587 if (argc > 1) JS_ToInt32(ctx, &port, argv[1]); 2577 2577 - // Set handle for identity 2578 2578 - if (current_rt->handle[0]) { 2579 2579 - pthread_mutex_lock(&current_rt->udp->mu); 2580 2580 - strncpy(current_rt->udp->handle, current_rt->handle, 63); 2581 2581 - current_rt->udp->handle[63] = 0; 2582 2582 - pthread_mutex_unlock(&current_rt->udp->mu); 2583 2583 - } 2588 2588 + sync_udp_identity(); 2584 2589 udp_connect(current_rt->udp, host, port); 2585 2590 if (argc > 0) JS_FreeCString(ctx, host); 2586 2591 return JS_UNDEFINED; ··· 2596 2601 return JS_UNDEFINED; 2597 2602 } 2598 2603 2604 2604 + static JSValue js_udp_send_midi(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { 2605 2605 + (void)this_val; 2606 2606 + if (!current_rt || !current_rt->udp || argc < 4) return JS_UNDEFINED; 2607 2607 + 2608 2608 + const char *event = JS_ToCString(ctx, argv[0]); 2609 2609 + int32_t note = 0; 2610 2610 + int32_t velocity = 0; 2611 2611 + int32_t channel = 0; 2612 2612 + const char *piece = argc > 4 ? JS_ToCString(ctx, argv[4]) : NULL; 2613 2613 + 2614 2614 + JS_ToInt32(ctx, &note, argv[1]); 2615 2615 + JS_ToInt32(ctx, &velocity, argv[2]); 2616 2616 + JS_ToInt32(ctx, &channel, argv[3]); 2617 2617 + 2618 2618 + if (!event) { 2619 2619 + if (piece) JS_FreeCString(ctx, piece); 2620 2620 + return JS_UNDEFINED; 2621 2621 + } 2622 2622 + 2623 2623 + sync_udp_identity(); 2624 2624 + udp_send_midi( 2625 2625 + current_rt->udp, 2626 2626 + event, 2627 2627 + (int)note, 2628 2628 + (int)velocity, 2629 2629 + (int)channel, 2630 2630 + piece ? piece : "notepat" 2631 2631 + ); 2632 2632 + 2633 2633 + JS_FreeCString(ctx, event); 2634 2634 + if (piece) JS_FreeCString(ctx, piece); 2635 2635 + return JS_UNDEFINED; 2636 2636 + } 2637 2637 + 2638 2638 + static JSValue js_udp_send_midi_heartbeat(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { 2639 2639 + (void)this_val; 2640 2640 + if (!current_rt || !current_rt->udp) return JS_UNDEFINED; 2641 2641 + 2642 2642 + const char *piece = argc > 0 ? JS_ToCString(ctx, argv[0]) : NULL; 2643 2643 + sync_udp_identity(); 2644 2644 + udp_send_midi_heartbeat(current_rt->udp, piece ? piece : "notepat"); 2645 2645 + if (piece) JS_FreeCString(ctx, piece); 2646 2646 + return JS_UNDEFINED; 2647 2647 + } 2648 2648 + 2599 2649 static JSValue build_udp_obj(JSContext *ctx, const char *phase) { 2600 2650 JSValue obj = JS_NewObject(ctx); 2601 2651 JS_SetPropertyStr(ctx, obj, "connect", JS_NewCFunction(ctx, js_udp_connect, "connect", 2)); 2602 2652 JS_SetPropertyStr(ctx, obj, "sendFairy", JS_NewCFunction(ctx, js_udp_send_fairy, "sendFairy", 2)); 2653 2653 + JS_SetPropertyStr(ctx, obj, "sendMidi", JS_NewCFunction(ctx, js_udp_send_midi, "sendMidi", 5)); 2654 2654 + JS_SetPropertyStr(ctx, obj, "sendMidiHeartbeat", JS_NewCFunction(ctx, js_udp_send_midi_heartbeat, "sendMidiHeartbeat", 1)); 2603 2655 2604 2656 ACUdp *udp = current_rt ? current_rt->udp : NULL; 2605 2657 if (udp) { 2658 2658 + sync_udp_identity(); 2606 2659 JS_SetPropertyStr(ctx, obj, "connected", JS_NewBool(ctx, udp->connected)); 2660 2660 + JS_SetPropertyStr(ctx, obj, "handle", 2661 2661 + JS_NewString(ctx, current_rt && current_rt->handle[0] ? current_rt->handle : "")); 2607 2662 2608 2663 // Only deliver fairies during paint phase 2609 2664 if (strcmp(phase, "paint") == 0) {

+159

fedac/native/src/udp-client.c

reviewed

··· 1 1 #include "udp-client.h" 2 2 3 3 + #include <stdint.h> 3 4 #include <stdio.h> 4 5 #include <stdlib.h> 5 6 #include <string.h> ··· 10 11 #include <arpa/inet.h> 11 12 #include <fcntl.h> 12 13 #include <poll.h> 14 14 + #include <time.h> 13 15 14 16 extern void ac_log(const char *fmt, ...); 15 17 ··· 23 25 #define PKT_FAIRY_SEND 0x01 24 26 #define PKT_FAIRY_RECV 0x02 25 27 28 28 + static uint64_t udp_now_ms(void) { 29 29 + struct timespec ts; 30 30 + clock_gettime(CLOCK_REALTIME, &ts); 31 31 + return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL; 32 32 + } 33 33 + 34 34 + static void json_escape_copy(char *dst, size_t dst_size, const char *src) { 35 35 + size_t out = 0; 36 36 + 37 37 + if (!dst || dst_size == 0) return; 38 38 + if (!src) src = ""; 39 39 + 40 40 + while (*src && out + 1 < dst_size) { 41 41 + unsigned char c = (unsigned char)*src++; 42 42 + if (c == '"' || c == '\\') { 43 43 + if (out + 2 >= dst_size) break; 44 44 + dst[out++] = '\\'; 45 45 + dst[out++] = (char)c; 46 46 + } else if (c >= 32 && c < 127) { 47 47 + dst[out++] = (char)c; 48 48 + } 49 49 + } 50 50 + 51 51 + dst[out] = '\0'; 52 52 + } 53 53 + 54 54 + static int udp_snapshot_identity( 55 55 + ACUdp *udp, 56 56 + int *sock_out, 57 57 + struct sockaddr_in *addr_out, 58 58 + char *handle_out, 59 59 + size_t handle_out_size, 60 60 + char *machine_out, 61 61 + size_t machine_out_size 62 62 + ) { 63 63 + int sock = -1; 64 64 + int connected = 0; 65 65 + 66 66 + if (!udp) return 0; 67 67 + 68 68 + pthread_mutex_lock(&udp->mu); 69 69 + sock = udp->sock; 70 70 + connected = udp->connected; 71 71 + if (addr_out) *addr_out = udp->server_addr; 72 72 + if (handle_out && handle_out_size > 0) { 73 73 + strncpy(handle_out, udp->handle, handle_out_size - 1); 74 74 + handle_out[handle_out_size - 1] = 0; 75 75 + } 76 76 + if (machine_out && machine_out_size > 0) { 77 77 + strncpy(machine_out, udp->machine_id, machine_out_size - 1); 78 78 + machine_out[machine_out_size - 1] = 0; 79 79 + } 80 80 + pthread_mutex_unlock(&udp->mu); 81 81 + 82 82 + if (sock_out) *sock_out = sock; 83 83 + return connected && sock >= 0; 84 84 + } 85 85 + 86 86 + static void udp_send_json(ACUdp *udp, const char *json) { 87 87 + int sock = -1; 88 88 + struct sockaddr_in server_addr; 89 89 + 90 90 + if (!udp || !json || !json[0]) return; 91 91 + if (!udp_snapshot_identity(udp, &sock, &server_addr, NULL, 0, NULL, 0)) return; 92 92 + 93 93 + ssize_t sent = sendto( 94 94 + sock, 95 95 + json, 96 96 + strlen(json), 97 97 + 0, 98 98 + (struct sockaddr *)&server_addr, 99 99 + sizeof(server_addr) 100 100 + ); 101 101 + 102 102 + if (sent < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { 103 103 + ac_log("[udp] json send failed: %s\n", strerror(errno)); 104 104 + } 105 105 + } 106 106 + 26 107 static void *udp_thread(void *arg) { 27 108 ACUdp *udp = (ACUdp *)arg; 28 109 unsigned char buf[256]; ··· 139 220 udp->send_y = y; 140 221 udp->send_pending = 1; 141 222 pthread_mutex_unlock(&udp->mu); 223 223 + } 224 224 + 225 225 + void udp_set_identity(ACUdp *udp, const char *handle, const char *machine_id) { 226 226 + if (!udp) return; 227 227 + pthread_mutex_lock(&udp->mu); 228 228 + if (handle) { 229 229 + strncpy(udp->handle, handle, sizeof(udp->handle) - 1); 230 230 + udp->handle[sizeof(udp->handle) - 1] = 0; 231 231 + } 232 232 + if (machine_id) { 233 233 + strncpy(udp->machine_id, machine_id, sizeof(udp->machine_id) - 1); 234 234 + udp->machine_id[sizeof(udp->machine_id) - 1] = 0; 235 235 + } 236 236 + pthread_mutex_unlock(&udp->mu); 237 237 + } 238 238 + 239 239 + void udp_send_midi(ACUdp *udp, const char *event, int note, int velocity, int channel, const char *piece) { 240 240 + char handle[64] = ""; 241 241 + char machine_id[64] = ""; 242 242 + char handle_json[128]; 243 243 + char machine_json[128]; 244 244 + char piece_json[64]; 245 245 + char event_json[32]; 246 246 + char json[512]; 247 247 + 248 248 + if (!udp || !event || !piece) return; 249 249 + if (!udp_snapshot_identity(udp, NULL, NULL, handle, sizeof(handle), machine_id, sizeof(machine_id))) return; 250 250 + 251 251 + json_escape_copy(handle_json, sizeof(handle_json), handle); 252 252 + json_escape_copy(machine_json, sizeof(machine_json), machine_id[0] ? machine_id : "unknown"); 253 253 + json_escape_copy(piece_json, sizeof(piece_json), piece); 254 254 + json_escape_copy(event_json, sizeof(event_json), event); 255 255 + 256 256 + snprintf( 257 257 + json, 258 258 + sizeof(json), 259 259 + "{\"type\":\"notepat:midi\",\"event\":\"%s\",\"note\":%d,\"velocity\":%d," 260 260 + "\"channel\":%d,\"handle\":\"%s\",\"machineId\":\"%s\",\"piece\":\"%s\",\"ts\":%llu}", 261 261 + event_json, 262 262 + note, 263 263 + velocity, 264 264 + channel, 265 265 + handle_json, 266 266 + machine_json, 267 267 + piece_json, 268 268 + (unsigned long long)udp_now_ms() 269 269 + ); 270 270 + 271 271 + udp_send_json(udp, json); 272 272 + } 273 273 + 274 274 + void udp_send_midi_heartbeat(ACUdp *udp, const char *piece) { 275 275 + char handle[64] = ""; 276 276 + char machine_id[64] = ""; 277 277 + char handle_json[128]; 278 278 + char machine_json[128]; 279 279 + char piece_json[64]; 280 280 + char json[512]; 281 281 + 282 282 + if (!udp || !piece) return; 283 283 + if (!udp_snapshot_identity(udp, NULL, NULL, handle, sizeof(handle), machine_id, sizeof(machine_id))) return; 284 284 + 285 285 + json_escape_copy(handle_json, sizeof(handle_json), handle); 286 286 + json_escape_copy(machine_json, sizeof(machine_json), machine_id[0] ? machine_id : "unknown"); 287 287 + json_escape_copy(piece_json, sizeof(piece_json), piece); 288 288 + 289 289 + snprintf( 290 290 + json, 291 291 + sizeof(json), 292 292 + "{\"type\":\"notepat:midi:heartbeat\",\"handle\":\"%s\",\"machineId\":\"%s\"," 293 293 + "\"piece\":\"%s\",\"broadcast\":true,\"ts\":%llu}", 294 294 + handle_json, 295 295 + machine_json, 296 296 + piece_json, 297 297 + (unsigned long long)udp_now_ms() 298 298 + ); 299 299 + 300 300 + udp_send_json(udp, json); 142 301 } 143 302 144 303 int udp_poll_fairies(ACUdp *udp, UDPFairy *out, int max) {

+8

fedac/native/src/udp-client.h

reviewed

··· 34 34 35 35 // Identity 36 36 char handle[64]; 37 37 + char machine_id[64]; 37 38 38 39 pthread_mutex_t mu; 39 40 } ACUdp; ··· 46 47 47 48 // Queue a fairy point to send (non-blocking, main thread) 48 49 void udp_send_fairy(ACUdp *udp, float x, float y); 50 50 + 51 51 + // Update identifying metadata used by outgoing packets. 52 52 + void udp_set_identity(ACUdp *udp, const char *handle, const char *machine_id); 53 53 + 54 54 + // Send note relay events to the session server over UDP. 55 55 + void udp_send_midi(ACUdp *udp, const char *event, int note, int velocity, int channel, const char *piece); 56 56 + void udp_send_midi_heartbeat(ACUdp *udp, const char *piece); 49 57 50 58 // Poll received fairies (returns count, fills out[] up to max) 51 59 // Clears the buffer after reading.

+512

plans/notepat-udp-midi-relay.md

reviewed

··· 1 1 + # Notepat UDP MIDI Relay Plan 2 2 + 3 3 + ## Overview 4 4 + 5 5 + Let running `ac-native` `notepat` sessions automatically publish note events to the Aesthetic Computer network, so a Max for Live device or `notepat.com`-based device can subscribe to those events and play them back live. 6 6 + 7 7 + The important twist is identity: 8 8 + 9 9 + - every note stream should be attributable to a `@handle` 10 10 + - we should also preserve `machineId` so multiple devices owned by the same person can be distinguished 11 11 + - subscribers should be able to filter by `@handle`, by `machineId`, or by "everyone" 12 12 + 13 13 + This would make an `ac-native` machine feel like a network instrument source, while Ableton / Max acts like a remote listener, mirror, recorder, or re-player. 14 14 + 15 15 + ## Why This Shape 16 16 + 17 17 + We already have the right footholds: 18 18 + 19 19 + - `fedac/native/src/udp-client.c` already sends compact UDP packets from `ac-native` 20 20 + - `fedac/native/src/js-bindings.c` already injects the current `handle` into the UDP client 21 21 + - `system.machineId` already exists in native 22 22 + - `notepat.mjs` already has a clean note-on / note-off seam for outgoing MIDI events 23 23 + - `ac-m4l/AC Notepat.amxd` and related M4L assets already exist 24 24 + 25 25 + The part that does not exist yet is a note-event relay path and a subscription model on the receiver side. 26 26 + 27 27 + ## Recommended Architecture 28 28 + 29 29 + Use a hybrid model: 30 30 + 31 31 + 1. `ac-native` sends note events to a UDP relay endpoint for low-latency ingest. 32 32 + 2. The relay normalizes those events and republishes them to WebSocket subscribers. 33 33 + 3. Max for Live / `notepat.com` devices subscribe over WebSocket by `@handle` or `machineId`. 34 34 + 35 35 + This is better than raw UDP end-to-end because: 36 36 + 37 37 + - browser-based receivers do not want to deal with raw UDP directly 38 38 + - WebSocket subscription/filtering is much easier to debug in M4L and web UIs 39 39 + - the relay can enrich events with metadata, timestamps, and presence 40 40 + - we can keep the native sender lightweight 41 41 + 42 42 + ## Architecture 43 43 + 44 44 + ```text 45 45 + ┌──────────────────────┐ 46 46 + │ ac-native / notepat │ 47 47 + │ on Yoga / AC machine │ 48 48 + └──────────┬───────────┘ 49 49 + │ 50 50 + │ UDP note events 51 51 + │ handle + machineId + midi 52 52 + ▼ 53 53 + ┌──────────────────────────────┐ 54 54 + │ session-server MIDI relay │ 55 55 + │ - UDP ingest │ 56 56 + │ - presence / last-seen map │ 57 57 + │ - WS fanout │ 58 58 + │ - handle + machine filters │ 59 59 + └──────────┬───────────────────┘ 60 60 + │ 61 61 + │ WebSocket subscribe: 62 62 + │ notepat:midi:subscribe 63 63 + ▼ 64 64 + ┌──────────────────────────────┐ 65 65 + │ Max for Live / AC Notepat │ 66 66 + │ or notepat.com device │ 67 67 + │ - choose source handle │ 68 68 + │ - optional machine filter │ 69 69 + │ - playback / monitor UI │ 70 70 + └──────────────────────────────┘ 71 71 + ``` 72 72 + 73 73 + ## Current Repo Footholds 74 74 + 75 75 + ### Native sender 76 76 + 77 77 + - `fedac/native/pieces/notepat.mjs` 78 78 + - already computes note-on / note-off centrally 79 79 + - already knows `system.usbMidi`, `system.udp`, and `system.machineId` 80 80 + - `fedac/native/src/js-bindings.c` 81 81 + - already exposes `system.udp` 82 82 + - already exposes `system.machineId` 83 83 + - already sets UDP identity from the config `handle` 84 84 + - `fedac/native/src/udp-client.c` 85 85 + - currently supports fairy packets only 86 86 + - already has background send / recv thread 87 87 + 88 88 + ### Receiver side 89 89 + 90 90 + - `ac-m4l/AC Notepat.amxd` 91 91 + - `ac-m4l/AC-Notepat.maxpat` 92 92 + - `ac-m4l/ac-ws-server.js` 93 93 + - `plans/kidlisp-m4l-integration.md` 94 94 + 95 95 + These give us a plausible place for a handle-selectable network note receiver. 96 96 + 97 97 + ## Event Model 98 98 + 99 99 + ### Minimum event payload 100 100 + 101 101 + Each note event should carry: 102 102 + 103 103 + - `type`: `note_on` or `note_off` 104 104 + - `note`: MIDI note number 105 105 + - `velocity`: 0-127 106 106 + - `channel`: MIDI channel 107 107 + - `handle`: performer handle without `@`, if known 108 108 + - `machineId`: native machine identity 109 109 + - `piece`: usually `notepat` 110 110 + - `ts`: sender timestamp in ms 111 111 + 112 112 + ### Suggested normalized relay JSON 113 113 + 114 114 + ```json 115 115 + { 116 116 + "type": "notepat:midi", 117 117 + "event": "note_on", 118 118 + "note": 60, 119 119 + "velocity": 109, 120 120 + "channel": 0, 121 121 + "handle": "jeffrey", 122 122 + "machineId": "ac-1234abcd", 123 123 + "piece": "notepat", 124 124 + "ts": 1710000000000 125 125 + } 126 126 + ``` 127 127 + 128 128 + ### Optional later fields 129 129 + 130 130 + - `source`: `ac-native` 131 131 + - `wave`: current notepat wave mode 132 132 + - `usbMidi`: whether it was also forwarded to a local USB MIDI host 133 133 + - `session`: optional room / channel 134 134 + - `pressure`: for aftertouch-like experiments 135 135 + 136 136 + ## UDP Protocol Plan 137 137 + 138 138 + ### Phase 1 139 139 + 140 140 + Add new packet types to `fedac/native/src/udp-client.c`: 141 141 + 142 142 + - `0x10` = note on 143 143 + - `0x11` = note off 144 144 + - `0x12` = heartbeat / source status 145 145 + 146 146 + Suggested binary shape: 147 147 + 148 148 + ```text 149 149 + [1 type] 150 150 + [1 note] 151 151 + [1 velocity] 152 152 + [1 channel] 153 153 + [1 handle_len][N handle_bytes] 154 154 + [1 machine_len][N machine_bytes] 155 155 + [4 timestamp_ms_u32] 156 156 + ``` 157 157 + 158 158 + This keeps the packet compact while still carrying identity. 159 159 + 160 160 + ### Alternative 161 161 + 162 162 + If we want to move faster and accept a slightly fatter payload, we could send short JSON strings over UDP first, then tighten to binary later. That would reduce implementation friction in the relay. 163 163 + 164 164 + Recommendation: 165 165 + 166 166 + - binary if we want to keep it native-first and stable long-term 167 167 + - JSON if we want the fastest prototype 168 168 + 169 169 + My recommendation is: 170 170 + 171 171 + - prototype with JSON-over-UDP 172 172 + - switch to binary once the event schema settles 173 173 + 174 174 + ## Relay / Server Plan 175 175 + 176 176 + The relay should do four jobs: 177 177 + 178 178 + 1. accept note events from native senders 179 179 + 2. maintain a live source index 180 180 + 3. expose source discovery to listeners 181 181 + 4. fan events out to subscribers 182 182 + 183 183 + ### Source index 184 184 + 185 185 + Maintain a map keyed by `handle + machineId`: 186 186 + 187 187 + ```js 188 188 + { 189 189 + "jeffrey:ac-1234abcd": { 190 190 + handle: "jeffrey", 191 191 + machineId: "ac-1234abcd", 192 192 + lastSeen: 1710000000000, 193 193 + lastEvent: "note_on", 194 194 + piece: "notepat" 195 195 + } 196 196 + } 197 197 + ``` 198 198 + 199 199 + ### WebSocket API 200 200 + 201 201 + Add a small family of messages: 202 202 + 203 203 + ```json 204 204 + { "type": "notepat:midi:sources" } 205 205 + ``` 206 206 + 207 207 + Response: 208 208 + 209 209 + ```json 210 210 + { 211 211 + "type": "notepat:midi:sources", 212 212 + "sources": [ 213 213 + { "handle": "jeffrey", "machineId": "ac-1234abcd", "piece": "notepat", "lastSeen": 1710000000000 } 214 214 + ] 215 215 + } 216 216 + ``` 217 217 + 218 218 + Subscribe: 219 219 + 220 220 + ```json 221 221 + { "type": "notepat:midi:subscribe", "handle": "jeffrey" } 222 222 + ``` 223 223 + 224 224 + Or: 225 225 + 226 226 + ```json 227 227 + { "type": "notepat:midi:subscribe", "handle": "jeffrey", "machineId": "ac-1234abcd" } 228 228 + ``` 229 229 + 230 230 + Or: 231 231 + 232 232 + ```json 233 233 + { "type": "notepat:midi:subscribe", "all": true } 234 234 + ``` 235 235 + 236 236 + Broadcast event: 237 237 + 238 238 + ```json 239 239 + { 240 240 + "type": "notepat:midi", 241 241 + "event": "note_on", 242 242 + "handle": "jeffrey", 243 243 + "machineId": "ac-1234abcd", 244 244 + "note": 60, 245 245 + "velocity": 96, 246 246 + "channel": 0, 247 247 + "ts": 1710000000000 248 248 + } 249 249 + ``` 250 250 + 251 251 + ## Native Sender Changes 252 252 + 253 253 + ### `notepat.mjs` 254 254 + 255 255 + Extend the current note send helpers so note events can also go to the network: 256 256 + 257 257 + - note on -> UDP relay send 258 258 + - note off -> UDP relay send 259 259 + - all notes off -> optional cleanup event 260 260 + 261 261 + Suggested behavior: 262 262 + 263 263 + - default off until user opts in 264 264 + - per-boot toggle in prompt or `notepat` 265 265 + - persist in `/mnt/config.json` 266 266 + 267 267 + Suggested config key: 268 268 + 269 269 + ```json 270 270 + { "udpMidiBroadcast": true } 271 271 + ``` 272 272 + 273 273 + ### `js-bindings.c` 274 274 + 275 275 + Expose one small higher-level helper on `system.udp`, for example: 276 276 + 277 277 + - `system.udp.sendMidi(event, note, velocity, channel, machineId, piece)` 278 278 + 279 279 + That keeps the piece code clean and avoids reimplementing packet shape in JS. 280 280 + 281 281 + ### `udp-client.c` 282 282 + 283 283 + Add: 284 284 + 285 285 + - note packet encoding 286 286 + - light send queue / coalescing 287 287 + - heartbeat support 288 288 + 289 289 + The heartbeat should announce: 290 290 + 291 291 + - `handle` 292 292 + - `machineId` 293 293 + - `piece=notepat` 294 294 + - `broadcast=true` 295 295 + 296 296 + every few seconds while the piece is active. 297 297 + 298 298 + ## Max for Live / notepat.com Receiver Plan 299 299 + 300 300 + ### UX 301 301 + 302 302 + The receiver device should have: 303 303 + 304 304 + - source mode: `all`, `handle`, or `handle + machine` 305 305 + - handle selector 306 306 + - machine selector 307 307 + - monitor list showing live note events 308 308 + - armed / muted toggle 309 309 + - optional MIDI passthrough to Live track 310 310 + 311 311 + ### Playback behavior 312 312 + 313 313 + For the first version: 314 314 + 315 315 + - incoming `note_on` -> emit MIDI note on in Max 316 316 + - incoming `note_off` -> emit MIDI note off in Max 317 317 + 318 318 + Later: 319 319 + 320 320 + - optional quantization 321 321 + - optional octave remap 322 322 + - optional channel remap 323 323 + - optional monitor-only mode 324 324 + 325 325 + ### Good first target 326 326 + 327 327 + Build this into `AC Notepat.amxd` or a sibling device such as: 328 328 + 329 329 + - `AC Notepat Relay.amxd` 330 330 + 331 331 + That keeps the "instrument" device separate from the "remote listener" device until the UI settles. 332 332 + 333 333 + ## Filtering By Handle 334 334 + 335 335 + This is the core product behavior. 336 336 + 337 337 + ### Rules 338 338 + 339 339 + - if `handle` is selected, accept events from all machines owned by that handle 340 340 + - if `handle + machineId` is selected, accept only that machine 341 341 + - if `all` is selected, accept all public broadcasts 342 342 + 343 343 + ### Why both handle and machineId matter 344 344 + 345 345 + `@handle` alone is not enough if: 346 346 + 347 347 + - someone has two AC machines running at once 348 348 + - one machine is a practice keyboard and one is a stage performer 349 349 + - we want to mirror a specific machine into a specific Ableton set 350 350 + 351 351 + So `handle` should be the human-facing grouping key, while `machineId` is the precise routing key. 352 352 + 353 353 + ## Presence and Discovery 354 354 + 355 355 + The receiver should be able to see active sources without typing blindly. 356 356 + 357 357 + Recommended behavior: 358 358 + 359 359 + - native sender heartbeats every 5-10 seconds 360 360 + - relay expires sources after 15-20 seconds of silence 361 361 + - M4L receiver requests source list on load and periodically refreshes 362 362 + 363 363 + Displayed label: 364 364 + 365 365 + ```text 366 366 + @jeffrey ac-1234abcd notepat seen 2s ago 367 367 + ``` 368 368 + 369 369 + ## Logging and Observability 370 370 + 371 371 + We should keep a paper trail because this will be hard to debug live. 372 372 + 373 373 + ### Native 374 374 + 375 375 + Log to `/mnt/ac-native.log`: 376 376 + 377 377 + - UDP relay connected 378 378 + - UDP note-on / note-off send errors 379 379 + - broadcast enabled / disabled 380 380 + 381 381 + ### Relay 382 382 + 383 383 + Log: 384 384 + 385 385 + - source registration 386 386 + - malformed packets 387 387 + - subscriber counts 388 388 + - fanout counts by handle 389 389 + 390 390 + ### M4L / receiver 391 391 + 392 392 + Expose a small event monitor: 393 393 + 394 394 + - last event time 395 395 + - last source handle 396 396 + - last note 397 397 + - dropped note count if any 398 398 + 399 399 + ## Privacy / Safety 400 400 + 401 401 + Because note streams can become ambient network presence, this should be opt-in. 402 402 + 403 403 + ### Recommendation 404 404 + 405 405 + - sender broadcasting is off by default 406 406 + - enabling it is explicit in prompt / config 407 407 + - receiver subscribes explicitly; nothing auto-plays by surprise 408 408 + 409 409 + Optional later: 410 410 + 411 411 + - `public` vs `friends` vs `private` 412 412 + - signed session tokens 413 413 + - authenticated per-handle subscriptions 414 414 + 415 415 + ## Phased Implementation 416 416 + 417 417 + ### Phase 1: Native -> relay prototype 418 418 + 419 419 + - [ ] Add `udpMidiBroadcast` config flag 420 420 + - [ ] Add note send hook in `fedac/native/pieces/notepat.mjs` 421 421 + - [ ] Add UDP note packet support in `fedac/native/src/udp-client.c` 422 422 + - [ ] Add relay listener endpoint 423 423 + - [ ] Log received events server-side 424 424 + 425 425 + Success condition: 426 426 + 427 427 + - notes from one `ac-native` machine appear in relay logs with `handle` and `machineId` 428 428 + 429 429 + ### Phase 2: Relay -> WebSocket subscribers 430 430 + 431 431 + - [ ] Add `notepat:midi:sources` 432 432 + - [ ] Add `notepat:midi:subscribe` 433 433 + - [ ] Add source expiry / heartbeat tracking 434 434 + - [ ] Broadcast normalized note events over WebSocket 435 435 + 436 436 + Success condition: 437 437 + 438 438 + - a browser console can subscribe to `@handle` and print incoming notes 439 439 + 440 440 + ### Phase 3: Max for Live receiver 441 441 + 442 442 + - [ ] Create or fork `AC Notepat` receiver device 443 443 + - [ ] Subscribe by `handle` 444 444 + - [ ] Emit Live MIDI notes 445 445 + - [ ] Show source / last-note monitor 446 446 + 447 447 + Success condition: 448 448 + 449 449 + - notes played on `ac-native` trigger a Live instrument from the selected handle 450 450 + 451 451 + ### Phase 4: Refinement 452 452 + 453 453 + - [ ] machine-level filtering 454 454 + - [ ] monitor-only mode 455 455 + - [ ] quantize / transform options 456 456 + - [ ] optional recording into Live clips 457 457 + - [ ] note-pressure / CC extension 458 458 + 459 459 + ## Open Questions 460 460 + 461 461 + ### 1. Should the relay live inside the existing session server? 462 462 + 463 463 + Recommendation: 464 464 + 465 465 + - yes for the first version 466 466 + 467 467 + Reason: 468 468 + 469 469 + - it already handles realtime identity-oriented traffic 470 470 + - it already has WebSocket clients 471 471 + - it is the easiest place to hang `handle`-filtered subscriptions 472 472 + 473 473 + ### 2. Should the receiver use raw UDP too? 474 474 + 475 475 + Recommendation: 476 476 + 477 477 + - no for the first version 478 478 + 479 479 + Reason: 480 480 + 481 481 + - browser / M4L environments are easier over WebSocket 482 482 + - UDP ingest + WS fanout already gets most of the benefit 483 483 + 484 484 + ### 3. Should we mirror every note or only notes that go to USB MIDI? 485 485 + 486 486 + Recommendation: 487 487 + 488 488 + - every note played in `notepat`, independent of USB MIDI state 489 489 + 490 490 + Reason: 491 491 + 492 492 + - network relay and local USB MIDI solve different problems 493 493 + - it is more musically useful if network relay does not depend on a cable 494 494 + 495 495 + ### 4. Should this be public performance presence? 496 496 + 497 497 + Recommendation: 498 498 + 499 499 + - start opt-in and semi-private 500 500 + - add public browsing later if it feels magical 501 501 + 502 502 + ## First Concrete Build Slice 503 503 + 504 504 + If we want the smallest useful first slice: 505 505 + 506 506 + 1. add `udpMidiBroadcast` toggle in native prompt 507 507 + 2. send JSON note events from `notepat` over UDP 508 508 + 3. add relay logging + source list in session server 509 509 + 4. build a tiny browser subscriber page that filters by `@handle` 510 510 + 5. only then move into Max for Live playback 511 511 + 512 512 + That gets us from idea to audible proof fast, without overcommitting to a packet format or Max UI too early.

+309 -79

session-server/session.mjs

reviewed

··· 73 73 const fairyThrottle = new Map(); // channelId -> last publish timestamp 74 74 const FAIRY_THROTTLE_MS = 100; // 10Hz max per connection 75 75 76 76 - // Raw UDP fairy relay (for native bare-metal clients) 77 77 - const udpRelay = dgram.createSocket("udp4"); 78 78 - const udpClients = new Map(); // key "ip:port" → { address, port, handle, lastSeen } 76 76 + // Raw UDP fairy relay (for native bare-metal clients) 77 77 + const udpRelay = dgram.createSocket("udp4"); 78 78 + const udpClients = new Map(); // key "ip:port" → { address, port, handle, lastSeen } 79 79 + const UDP_MIDI_SOURCE_TTL_MS = 20000; 80 80 + const notepatMidiSources = new Map(); // key "@handle:machine" -> source metadata 81 81 + const notepatMidiSubscribers = new Map(); // connection id -> { ws, all, handle, machineId } 79 82 80 83 // Error logging ring buffer (for dashboard display) 81 84 const errorLog = []; ··· 2325 2328 return; 2326 2329 } 2327 2330 2328 2328 - if (msg.type === "daw:code") { 2329 2329 - // IDE sending code to all connected devices 2330 2330 - log(`🎹 DAW code broadcast from ${id} to ${dawDevices.size} devices`); 2331 2331 - const codeMsg = JSON.stringify({ 2332 2332 - type: "daw:code", 2331 2331 + if (msg.type === "daw:code") { 2332 2332 + // IDE sending code to all connected devices 2333 2333 + log(`🎹 DAW code broadcast from ${id} to ${dawDevices.size} devices`); 2334 2334 + const codeMsg = JSON.stringify({ 2335 2335 + type: "daw:code", 2333 2336 content: msg.content, 2334 2337 from: id 2335 2338 }); ··· 2341 2344 deviceWs.send(codeMsg); 2342 2345 log(`🎹 Sent code to device ${deviceId}`); 2343 2346 } 2344 2344 - } 2345 2345 - return; 2346 2346 - } 2347 2347 - 2348 2348 - msg.id = id; // TODO: When sending a server generated message, use a special id. 2347 2347 + } 2348 2348 + return; 2349 2349 + } 2350 2350 + 2351 2351 + if (msg.type === "notepat:midi:sources") { 2352 2352 + sendNotepatMidiSources(ws); 2353 2353 + return; 2354 2354 + } 2355 2355 + 2356 2356 + if (msg.type === "notepat:midi:subscribe") { 2357 2357 + const filter = msg.content || {}; 2358 2358 + addNotepatMidiSubscriber(id, ws, filter); 2359 2359 + return; 2360 2360 + } 2361 2361 + 2362 2362 + if (msg.type === "notepat:midi:unsubscribe") { 2363 2363 + removeNotepatMidiSubscriber(id); 2364 2364 + if (ws.readyState === WebSocket.OPEN) { 2365 2365 + ws.send(pack("notepat:midi:unsubscribed", true, "midi-relay")); 2366 2366 + } 2367 2367 + return; 2368 2368 + } 2369 2369 + 2370 2370 + msg.id = id; // TODO: When sending a server generated message, use a special id. 2349 2371 2350 2372 // Extract user identity and handle from ANY message that contains it 2351 2373 if (msg.content?.user?.sub) { ··· 2821 2843 }); 2822 2844 2823 2845 // More info: https://stackoverflow.com/a/49791634/8146077 2824 2824 - ws.on("close", () => { 2825 2825 - log("🚪 Someone left:", id, "Online:", wss.clients.size, "🫂"); 2826 2826 - const departingHandle = normalizeProfileHandle(clients?.[id]?.handle); 2827 2827 - 2828 2828 - // Remove from VSCode clients if present 2829 2829 - vscodeClients.delete(ws); 2846 2846 + ws.on("close", () => { 2847 2847 + log("🚪 Someone left:", id, "Online:", wss.clients.size, "🫂"); 2848 2848 + const departingHandle = normalizeProfileHandle(clients?.[id]?.handle); 2849 2849 + removeNotepatMidiSubscriber(id); 2850 2850 + 2851 2851 + // Remove from VSCode clients if present 2852 2852 + vscodeClients.delete(ws); 2830 2853 2831 2854 // Remove from DAW devices if present 2832 2855 if (dawDevices.has(id)) { ··· 3001 3024 // Track VSCode extension clients for direct jump message routing 3002 3025 const vscodeClients = new Set(); 3003 3026 3004 3004 - function normalizeProfileHandle(handle) { 3005 3005 - if (!handle) return null; 3006 3006 - const raw = `${handle}`.trim(); 3007 3007 - if (!raw) return null; 3008 3008 - return `@${raw.replace(/^@+/, "").toLowerCase()}`; 3009 3009 - } 3027 3027 + function normalizeProfileHandle(handle) { 3028 3028 + if (!handle) return null; 3029 3029 + const raw = `${handle}`.trim(); 3030 3030 + if (!raw) return null; 3031 3031 + return `@${raw.replace(/^@+/, "").toLowerCase()}`; 3032 3032 + } 3033 3033 + 3034 3034 + function normalizeMidiHandle(handle) { 3035 3035 + const normalized = normalizeProfileHandle(handle); 3036 3036 + return normalized ? normalized.slice(1) : ""; 3037 3037 + } 3038 3038 + 3039 3039 + function notepatMidiSourceKey(handle, machineId) { 3040 3040 + const handleKey = normalizeProfileHandle(handle) || "@unknown"; 3041 3041 + const machineKey = `${machineId || "unknown"}`.trim() || "unknown"; 3042 3042 + return `${handleKey}:${machineKey}`; 3043 3043 + } 3044 3044 + 3045 3045 + function listNotepatMidiSources() { 3046 3046 + return [...notepatMidiSources.values()] 3047 3047 + .sort((a, b) => (b.lastSeen || 0) - (a.lastSeen || 0)) 3048 3048 + .map((source) => ({ 3049 3049 + handle: source.handle || null, 3050 3050 + machineId: source.machineId, 3051 3051 + piece: source.piece || "notepat", 3052 3052 + lastSeen: source.lastSeen || 0, 3053 3053 + lastEvent: source.lastEvent || null, 3054 3054 + })); 3055 3055 + } 3056 3056 + 3057 3057 + function sendNotepatMidiSources(ws) { 3058 3058 + if (!ws || ws.readyState !== WebSocket.OPEN) return; 3059 3059 + try { 3060 3060 + ws.send(pack("notepat:midi:sources", { sources: listNotepatMidiSources() }, "midi-relay")); 3061 3061 + } catch (err) { 3062 3062 + error("🎹 Failed to send notepat midi sources:", err); 3063 3063 + } 3064 3064 + } 3065 3065 + 3066 3066 + function removeNotepatMidiSubscriber(id) { 3067 3067 + if (id === undefined || id === null) return; 3068 3068 + notepatMidiSubscribers.delete(id); 3069 3069 + } 3070 3070 + 3071 3071 + function addNotepatMidiSubscriber(id, ws, filter = {}) { 3072 3072 + if (id === undefined || id === null || !ws) return; 3073 3073 + 3074 3074 + notepatMidiSubscribers.set(id, { 3075 3075 + ws, 3076 3076 + all: filter.all === true, 3077 3077 + handle: normalizeMidiHandle(filter.handle), 3078 3078 + machineId: filter.machineId ? `${filter.machineId}`.trim() : "", 3079 3079 + }); 3080 3080 + 3081 3081 + if (ws.readyState === WebSocket.OPEN) { 3082 3082 + ws.send(pack("notepat:midi:subscribed", { 3083 3083 + all: filter.all === true, 3084 3084 + handle: normalizeMidiHandle(filter.handle) || null, 3085 3085 + machineId: filter.machineId ? `${filter.machineId}`.trim() : null, 3086 3086 + }, "midi-relay")); 3087 3087 + } 3088 3088 + 3089 3089 + sendNotepatMidiSources(ws); 3090 3090 + } 3091 3091 + 3092 3092 + function broadcastNotepatMidiSources() { 3093 3093 + for (const [id, sub] of notepatMidiSubscribers) { 3094 3094 + if (!sub?.ws || sub.ws.readyState !== WebSocket.OPEN) { 3095 3095 + notepatMidiSubscribers.delete(id); 3096 3096 + continue; 3097 3097 + } 3098 3098 + sendNotepatMidiSources(sub.ws); 3099 3099 + } 3100 3100 + } 3101 3101 + 3102 3102 + function notepatMidiSubscriberMatches(sub, event) { 3103 3103 + if (!sub) return false; 3104 3104 + if (sub.all) return true; 3105 3105 + 3106 3106 + const eventHandle = normalizeMidiHandle(event?.handle); 3107 3107 + const eventMachine = event?.machineId ? `${event.machineId}`.trim() : ""; 3108 3108 + 3109 3109 + if (sub.handle && sub.handle !== eventHandle) return false; 3110 3110 + if (sub.machineId && sub.machineId !== eventMachine) return false; 3111 3111 + 3112 3112 + return !!(sub.handle || sub.machineId); 3113 3113 + } 3114 3114 + 3115 3115 + function broadcastNotepatMidiEvent(event) { 3116 3116 + for (const [id, sub] of notepatMidiSubscribers) { 3117 3117 + if (!sub?.ws || sub.ws.readyState !== WebSocket.OPEN) { 3118 3118 + notepatMidiSubscribers.delete(id); 3119 3119 + continue; 3120 3120 + } 3121 3121 + if (!notepatMidiSubscriberMatches(sub, event)) continue; 3122 3122 + try { 3123 3123 + sub.ws.send(pack("notepat:midi", event, "midi-relay")); 3124 3124 + } catch (err) { 3125 3125 + error("🎹 Failed to fan out notepat midi event:", err); 3126 3126 + } 3127 3127 + } 3128 3128 + } 3129 3129 + 3130 3130 + function upsertNotepatMidiSource({ handle, machineId, piece, lastEvent, ts, address, port }) { 3131 3131 + const cleanHandle = normalizeMidiHandle(handle); 3132 3132 + const cleanMachineId = `${machineId || "unknown"}`.trim() || "unknown"; 3133 3133 + const key = notepatMidiSourceKey(cleanHandle, cleanMachineId); 3134 3134 + const previous = notepatMidiSources.get(key); 3135 3135 + const next = { 3136 3136 + handle: cleanHandle || null, 3137 3137 + machineId: cleanMachineId, 3138 3138 + piece: piece || "notepat", 3139 3139 + lastSeen: ts || Date.now(), 3140 3140 + lastEvent: lastEvent || previous?.lastEvent || null, 3141 3141 + address: address || previous?.address || null, 3142 3142 + port: port || previous?.port || null, 3143 3143 + }; 3144 3144 + 3145 3145 + notepatMidiSources.set(key, next); 3146 3146 + 3147 3147 + if (!previous) { 3148 3148 + log(`🎹 Notepat MIDI source online: ${next.handle ? "@" + next.handle : "@unknown"} ${next.machineId}`); 3149 3149 + } 3150 3150 + 3151 3151 + if ( 3152 3152 + !previous || 3153 3153 + previous.handle !== next.handle || 3154 3154 + previous.machineId !== next.machineId || 3155 3155 + previous.piece !== next.piece 3156 3156 + ) { 3157 3157 + broadcastNotepatMidiSources(); 3158 3158 + } 3159 3159 + 3160 3160 + return next; 3161 3161 + } 3010 3162 3011 3163 function compactProfileText(value) { 3012 3164 return `${value || ""}`.replace(/\s+/g, " ").trim(); ··· 3430 3582 3431 3583 // #endregion 3432 3584 3433 3433 - // --------------------------------------------------------------------------- 3434 3434 - // 🧚 Raw UDP fairy relay (port 10010) — for native bare-metal clients 3435 3435 - // Binary packet format: 3436 3436 - // [1 byte type] [4 float x LE] [4 float y LE] [1 handle_len] [N handle] 3437 3437 - // Type 0x01 = client→server, 0x02 = server→client broadcast 3438 3438 - // --------------------------------------------------------------------------- 3439 3439 - const UDP_FAIRY_PORT = 10010; 3440 3440 - 3441 3441 - udpRelay.on("message", (msg, rinfo) => { 3442 3442 - if (msg.length < 10 || msg[0] !== 0x01) return; 3443 3443 - 3444 3444 - const key = `${rinfo.address}:${rinfo.port}`; 3445 3445 - const x = msg.readFloatLE(1); 3446 3446 - const y = msg.readFloatLE(5); 3447 3447 - const hlen = msg[9]; 3448 3448 - const handle = msg.slice(10, 10 + hlen).toString("utf8"); 3449 3449 - 3450 3450 - udpClients.set(key, { address: rinfo.address, port: rinfo.port, handle, lastSeen: Date.now() }); 3451 3451 - 3452 3452 - // Build broadcast packet (type 0x02) 3453 3453 - const bcast = Buffer.alloc(msg.length); 3454 3454 - msg.copy(bcast); 3455 3455 - bcast[0] = 0x02; 3456 3456 - 3457 3457 - // Broadcast to all other UDP clients 3458 3458 - for (const [k, client] of udpClients) { 3459 3459 - if (k !== key) { 3460 3460 - udpRelay.send(bcast, client.port, client.address); 3461 3461 - } 3462 3462 - } 3463 3463 - 3464 3464 - // Also broadcast to Geckos.io WebRTC clients as fairy:point 3465 3465 - const fairyData = JSON.stringify({ x, y, handle }); 3466 3466 - try { 3467 3467 - // Emit to all geckos channels 3468 3468 - io.room().emit("fairy:point", fairyData); 3469 3469 - } catch (e) { /* ignore */ } 3470 3470 - 3471 3471 - // Publish to Redis for silo firehose (throttled) 3472 3472 - const now = Date.now(); 3473 3473 - const lastFairy = fairyThrottle.get(key) || 0; 3474 3474 - if (now - lastFairy >= FAIRY_THROTTLE_MS) { 3475 3475 - fairyThrottle.set(key, now); 3476 3476 - pub.publish("fairy:point", fairyData).catch(() => {}); 3477 3477 - } 3478 3478 - }); 3479 3479 - 3480 3480 - // Clean up stale UDP clients every 30s 3481 3481 - setInterval(() => { 3482 3482 - const now = Date.now(); 3483 3483 - for (const [key, client] of udpClients) { 3484 3484 - if (now - client.lastSeen > 30000) udpClients.delete(key); 3485 3485 - } 3486 3486 - }, 30000); 3585 3585 + // --------------------------------------------------------------------------- 3586 3586 + // 🧚 Raw UDP fairy relay (port 10010) — for native bare-metal clients 3587 3587 + // Binary packet format: 3588 3588 + // [1 byte type] [4 float x LE] [4 float y LE] [1 handle_len] [N handle] 3589 3589 + // Type 0x01 = client→server, 0x02 = server→client broadcast 3590 3590 + // --------------------------------------------------------------------------- 3591 3591 + const UDP_FAIRY_PORT = 10010; 3592 3592 + 3593 3593 + function handleNotepatMidiUdpPacket(payload, rinfo) { 3594 3594 + if (!payload || (payload.type !== "notepat:midi" && payload.type !== "notepat:midi:heartbeat")) { 3595 3595 + return false; 3596 3596 + } 3597 3597 + 3598 3598 + const now = Date.now(); 3599 3599 + const source = upsertNotepatMidiSource({ 3600 3600 + handle: payload.handle, 3601 3601 + machineId: payload.machineId, 3602 3602 + piece: payload.piece || "notepat", 3603 3603 + lastEvent: payload.type === "notepat:midi" ? payload.event : "heartbeat", 3604 3604 + ts: now, 3605 3605 + address: rinfo.address, 3606 3606 + port: rinfo.port, 3607 3607 + }); 3608 3608 + 3609 3609 + if (!source.handle && !source.machineId) { 3610 3610 + return true; 3611 3611 + } 3612 3612 + 3613 3613 + if (payload.type === "notepat:midi:heartbeat") { 3614 3614 + return true; 3615 3615 + } 3616 3616 + 3617 3617 + const rawNote = Number(payload.note); 3618 3618 + const rawVelocity = Number(payload.velocity); 3619 3619 + const rawChannel = Number(payload.channel); 3620 3620 + if (!Number.isFinite(rawNote) || !Number.isFinite(rawVelocity) || !Number.isFinite(rawChannel)) { 3621 3621 + log("🎹 Invalid notepat midi UDP payload:", payload); 3622 3622 + return true; 3623 3623 + } 3624 3624 + 3625 3625 + let event = payload.event === "note_off" ? "note_off" : "note_on"; 3626 3626 + const note = Math.max(0, Math.min(127, Math.round(rawNote))); 3627 3627 + const velocity = Math.max(0, Math.min(127, Math.round(rawVelocity))); 3628 3628 + const channel = Math.max(0, Math.min(15, Math.round(rawChannel))); 3629 3629 + if (event === "note_on" && velocity === 0) event = "note_off"; 3630 3630 + 3631 3631 + broadcastNotepatMidiEvent({ 3632 3632 + type: "notepat:midi", 3633 3633 + event, 3634 3634 + note, 3635 3635 + velocity, 3636 3636 + channel, 3637 3637 + handle: source.handle, 3638 3638 + machineId: source.machineId, 3639 3639 + piece: source.piece || "notepat", 3640 3640 + ts: Number.isFinite(Number(payload.ts)) ? Number(payload.ts) : now, 3641 3641 + }); 3642 3642 + 3643 3643 + return true; 3644 3644 + } 3645 3645 + 3646 3646 + function pruneNotepatMidiSources() { 3647 3647 + const now = Date.now(); 3648 3648 + let changed = false; 3649 3649 + 3650 3650 + for (const [key, source] of notepatMidiSources) { 3651 3651 + if (now - (source.lastSeen || 0) > UDP_MIDI_SOURCE_TTL_MS) { 3652 3652 + notepatMidiSources.delete(key); 3653 3653 + changed = true; 3654 3654 + } 3655 3655 + } 3656 3656 + 3657 3657 + if (changed) broadcastNotepatMidiSources(); 3658 3658 + } 3659 3659 + 3660 3660 + udpRelay.on("message", (msg, rinfo) => { 3661 3661 + if (msg.length > 0 && msg[0] === 0x01 && msg.length >= 10) { 3662 3662 + const key = `${rinfo.address}:${rinfo.port}`; 3663 3663 + const x = msg.readFloatLE(1); 3664 3664 + const y = msg.readFloatLE(5); 3665 3665 + const hlen = msg[9]; 3666 3666 + const handle = msg.slice(10, 10 + hlen).toString("utf8"); 3667 3667 + 3668 3668 + udpClients.set(key, { address: rinfo.address, port: rinfo.port, handle, lastSeen: Date.now() }); 3669 3669 + 3670 3670 + // Build broadcast packet (type 0x02) 3671 3671 + const bcast = Buffer.alloc(msg.length); 3672 3672 + msg.copy(bcast); 3673 3673 + bcast[0] = 0x02; 3674 3674 + 3675 3675 + // Broadcast to all other UDP clients 3676 3676 + for (const [k, client] of udpClients) { 3677 3677 + if (k !== key) { 3678 3678 + udpRelay.send(bcast, client.port, client.address); 3679 3679 + } 3680 3680 + } 3681 3681 + 3682 3682 + // Also broadcast to Geckos.io WebRTC clients as fairy:point 3683 3683 + const fairyData = JSON.stringify({ x, y, handle }); 3684 3684 + try { 3685 3685 + // Emit to all geckos channels 3686 3686 + io.room().emit("fairy:point", fairyData); 3687 3687 + } catch (e) { /* ignore */ } 3688 3688 + 3689 3689 + // Publish to Redis for silo firehose (throttled) 3690 3690 + const now = Date.now(); 3691 3691 + const lastFairy = fairyThrottle.get(key) || 0; 3692 3692 + if (now - lastFairy >= FAIRY_THROTTLE_MS) { 3693 3693 + fairyThrottle.set(key, now); 3694 3694 + pub.publish("fairy:point", fairyData).catch(() => {}); 3695 3695 + } 3696 3696 + return; 3697 3697 + } 3698 3698 + 3699 3699 + if (msg.length > 0 && msg[0] === 0x7b) { 3700 3700 + try { 3701 3701 + const payload = JSON.parse(msg.toString("utf8")); 3702 3702 + if (handleNotepatMidiUdpPacket(payload, rinfo)) return; 3703 3703 + } catch (err) { 3704 3704 + log("🎹 Failed to parse UDP JSON packet:", err?.message || err); 3705 3705 + } 3706 3706 + } 3707 3707 + }); 3708 3708 + 3709 3709 + // Clean up stale UDP clients every 30s 3710 3710 + setInterval(() => { 3711 3711 + const now = Date.now(); 3712 3712 + for (const [key, client] of udpClients) { 3713 3713 + if (now - client.lastSeen > 30000) udpClients.delete(key); 3714 3714 + } 3715 3715 + pruneNotepatMidiSources(); 3716 3716 + }, 30000); 3487 3717 3488 3718 udpRelay.bind(UDP_FAIRY_PORT, () => { 3489 3719 console.log(`🧚 Raw UDP fairy relay listening on port ${UDP_FAIRY_PORT}`);

+417 -39

system/public/aesthetic.computer/disks/notepat.mjs

reviewed

··· 894 894 "+b", 895 895 ]; 896 896 897 897 - const buttonNoteLookup = new Set(buttonNotes); 898 898 - 899 899 - const midiActiveNotes = new Map(); 897 897 + const buttonNoteLookup = new Set(buttonNotes); 898 898 + const MIDI_NOTE_NAMES = ["c", "c#", "d", "d#", "e", "f", "f#", "g", "g#", "a", "a#", "b"]; 899 899 + 900 900 + const midiActiveNotes = new Map(); 900 901 901 902 const MIDI_PITCH_BEND_RANGE = 2; // Semitones up/down for pitch wheel. 902 903 let midiPitchBendValue = 0; // Normalized -1..1 position of the wheel. ··· 1250 1251 1251 1252 // song = parseSong(rawSong); 1252 1253 1253 1253 - let startupSfx; 1254 1254 - let udpServer; 1255 1255 - 1256 1256 - let stampleSampleId = null; 1257 1257 - let stampleSampleData = null; 1258 1258 - let stampleSampleRate = null; 1254 1254 + let startupSfx; 1255 1255 + let udpServer; 1256 1256 + let relaySocket = null; 1257 1257 + let relaySourceHandle = ""; 1258 1258 + let relaySourceMachineId = ""; 1259 1259 + let relaySubscribeAll = false; 1260 1260 + let relaySources = []; 1261 1261 + let relayStatusText = "relay off"; 1262 1262 + let relayLastEventText = ""; 1263 1263 + let relayLastEventAt = 0; 1264 1264 + let relayNeedsPanic = false; 1265 1265 + let relayMessageHandler = null; 1266 1266 + const relayMidiQueue = []; 1267 1267 + const relayActiveNotes = new Map(); 1268 1268 + 1269 1269 + let stampleSampleId = null; 1270 1270 + let stampleSampleData = null; 1271 1271 + let stampleSampleRate = null; 1259 1272 let stampleNeedleProgress = 0; 1260 1273 let stampleNeedleNote = null; 1261 1274 let stampleProgressTick = 0; 1262 1275 let kidlispSampleRefreshTick = 0; 1263 1276 1264 1264 - let autopatHud = null; 1265 1265 - let autopatTypeface = null; 1266 1266 - 1267 1267 - let picture; 1268 1268 - let matrixFont; // MatrixChunky8 font for note letters 1277 1277 + let autopatHud = null; 1278 1278 + let autopatTypeface = null; 1279 1279 + 1280 1280 + let picture; 1281 1281 + let matrixFont; // MatrixChunky8 font for note letters 1282 1282 + 1283 1283 + function normalizeRelayHandle(handle) { 1284 1284 + return `${handle || ""}`.trim().replace(/^@+/, "").toLowerCase(); 1285 1285 + } 1286 1286 + 1287 1287 + function relayTargetLabel() { 1288 1288 + if (relaySubscribeAll) return "all"; 1289 1289 + if (relaySourceHandle && relaySourceMachineId) return `@${relaySourceHandle} ${relaySourceMachineId}`; 1290 1290 + if (relaySourceHandle) return `@${relaySourceHandle}`; 1291 1291 + if (relaySourceMachineId) return relaySourceMachineId; 1292 1292 + return "off"; 1293 1293 + } 1294 1294 + 1295 1295 + function relaySubscriptionPayload() { 1296 1296 + if (relaySubscribeAll) return { all: true }; 1297 1297 + if (relaySourceHandle || relaySourceMachineId) { 1298 1298 + return { 1299 1299 + handle: relaySourceHandle || undefined, 1300 1300 + machineId: relaySourceMachineId || undefined, 1301 1301 + }; 1302 1302 + } 1303 1303 + return null; 1304 1304 + } 1305 1305 + 1306 1306 + function updateRelayBridgeState() { 1307 1307 + if (typeof window === "undefined") return; 1308 1308 + window.acNotepatRelay = { 1309 1309 + status: relayStatusText, 1310 1310 + target: relayTargetLabel(), 1311 1311 + sources: relaySources, 1312 1312 + lastEvent: relayLastEventText, 1313 1313 + setSource(next = {}) { 1314 1314 + setRelaySource(next); 1315 1315 + }, 1316 1316 + }; 1317 1317 + } 1318 1318 + 1319 1319 + function requestRelaySources() { 1320 1320 + relaySocket?.send?.("notepat:midi:sources"); 1321 1321 + } 1322 1322 + 1323 1323 + function sendRelaySubscription() { 1324 1324 + const payload = relaySubscriptionPayload(); 1325 1325 + if (!relaySocket?.send) return; 1326 1326 + if (!payload) { 1327 1327 + relaySocket.send("notepat:midi:unsubscribe", true); 1328 1328 + relayStatusText = "relay off"; 1329 1329 + updateRelayBridgeState(); 1330 1330 + return; 1331 1331 + } 1332 1332 + relaySocket.send("notepat:midi:subscribe", payload); 1333 1333 + relayStatusText = `relay ${relayTargetLabel()}`; 1334 1334 + updateRelayBridgeState(); 1335 1335 + } 1336 1336 + 1337 1337 + function setRelaySource(next = {}) { 1338 1338 + const hasHandle = Object.prototype.hasOwnProperty.call(next, "handle"); 1339 1339 + const hasMachineId = Object.prototype.hasOwnProperty.call(next, "machineId"); 1340 1340 + const hasAll = Object.prototype.hasOwnProperty.call(next, "all"); 1341 1341 + 1342 1342 + if (hasHandle) { 1343 1343 + relaySourceHandle = normalizeRelayHandle(next.handle); 1344 1344 + } 1345 1345 + if (hasMachineId) { 1346 1346 + relaySourceMachineId = `${next.machineId || ""}`.trim(); 1347 1347 + } else if (hasHandle) { 1348 1348 + relaySourceMachineId = ""; 1349 1349 + } 1350 1350 + if (hasAll) { 1351 1351 + relaySubscribeAll = next.all === true; 1352 1352 + } 1353 1353 + if (relaySubscribeAll) { 1354 1354 + relaySourceHandle = ""; 1355 1355 + relaySourceMachineId = ""; 1356 1356 + } 1357 1357 + relayNeedsPanic = true; 1358 1358 + requestRelaySources(); 1359 1359 + sendRelaySubscription(); 1360 1360 + } 1361 1361 + 1362 1362 + function handleRelaySocketMessage(id, type, content) { 1363 1363 + if (type === "connected" || type === "connected:already") { 1364 1364 + requestRelaySources(); 1365 1365 + sendRelaySubscription(); 1366 1366 + return; 1367 1367 + } 1368 1368 + 1369 1369 + if (type === "notepat:midi:sources") { 1370 1370 + relaySources = Array.isArray(content?.sources) ? content.sources : []; 1371 1371 + updateRelayBridgeState(); 1372 1372 + return; 1373 1373 + } 1374 1374 + 1375 1375 + if (type === "notepat:midi:subscribed") { 1376 1376 + relayStatusText = `relay ${relayTargetLabel()}`; 1377 1377 + updateRelayBridgeState(); 1378 1378 + return; 1379 1379 + } 1380 1380 + 1381 1381 + if (type === "notepat:midi:unsubscribed") { 1382 1382 + relayStatusText = "relay off"; 1383 1383 + relayNeedsPanic = true; 1384 1384 + updateRelayBridgeState(); 1385 1385 + return; 1386 1386 + } 1387 1387 + 1388 1388 + if (type === "notepat:midi") { 1389 1389 + relayMidiQueue.push(content); 1390 1390 + relayLastEventText = `${content?.handle ? "@" + content.handle + " " : ""}${content?.event || "event"} ${content?.note ?? ""}`; 1391 1391 + relayLastEventAt = Date.now(); 1392 1392 + updateRelayBridgeState(); 1393 1393 + } 1394 1394 + } 1269 1395 1270 1270 - async function boot({ 1396 1396 + async function boot({ 1271 1397 params, 1272 1398 api, 1273 1399 colon, ··· 1282 1408 sound, 1283 1409 clock, 1284 1410 query, 1285 1285 - }) { 1286 1286 - autopatApi = api; 1287 1287 - autopatHud = hud; 1288 1288 - autopatTypeface = typeface; 1411 1411 + }) { 1412 1412 + autopatApi = api; 1413 1413 + autopatHud = hud; 1414 1414 + autopatTypeface = typeface; 1415 1415 + setSoundContext({ 1416 1416 + synth: sound?.synth, 1417 1417 + play: sound?.play, 1418 1418 + freq: sound?.freq, 1419 1419 + }); 1289 1420 1290 1421 // ✨ Show ".com" superscript in the HUD corner label (notepat.com branding) 1291 1422 hud.superscript(".com"); 1292 1423 dotComMode = true; 1293 1424 1294 1425 // 🎹 Check if we're in DAW mode (loaded from Ableton M4L) 1295 1295 - dawMode = query?.daw === "1" || query?.daw === 1 || query?.daw === true; 1296 1296 - console.log("🎹 Notepat: dawMode =", dawMode, "query.daw =", query?.daw, typeof query?.daw); 1426 1426 + dawMode = query?.daw === "1" || query?.daw === 1 || query?.daw === true; 1427 1427 + console.log("🎹 Notepat: dawMode =", dawMode, "query.daw =", query?.daw, typeof query?.daw); 1297 1428 1298 1429 // Also check if we already have DAW data (survives hot reload) 1299 1430 if (!dawMode && sound.daw?.bpm) { ··· 1349 1480 // Disabled: dynamic audio reinit was breaking audio - now using 48kHz globally 1350 1481 pendingAudioReinit = false; 1351 1482 1352 1352 - // fps(4); 1353 1353 - udpServer = net.udp(); // For sending messages to `tv`. 1354 1354 - 1355 1355 - // Create picture buffer at quarter resolution (quarter width, quarter height) 1356 1356 - const pictureWidth = Math.max(1, Math.floor(screen.width / 4)); 1483 1483 + // fps(4); 1484 1484 + udpServer = net.udp(); // For sending messages to `tv`. 1485 1485 + relaySourceHandle = normalizeRelayHandle(query?.relayHandle); 1486 1486 + relaySourceMachineId = `${query?.relayMachine || ""}`.trim(); 1487 1487 + relaySubscribeAll = query?.relayAll === "1" || query?.relay === "all"; 1488 1488 + relayStatusText = relaySubscriptionPayload() ? `relay ${relayTargetLabel()}` : "relay off"; 1489 1489 + relaySocket = net.socket(handleRelaySocketMessage); 1490 1490 + requestRelaySources(); 1491 1491 + sendRelaySubscription(); 1492 1492 + if (typeof window !== "undefined") { 1493 1493 + if (relayMessageHandler) window.removeEventListener("message", relayMessageHandler); 1494 1494 + relayMessageHandler = (event) => { 1495 1495 + const data = event?.data; 1496 1496 + if (!data || typeof data !== "object") return; 1497 1497 + if (data.type === "notepat:midi:set-source" || data.type === "notepat:midi:subscribe") { 1498 1498 + setRelaySource({ 1499 1499 + handle: data.handle, 1500 1500 + machineId: data.machineId, 1501 1501 + all: data.all === true, 1502 1502 + }); 1503 1503 + } else if (data.type === "notepat:midi:unsubscribe") { 1504 1504 + setRelaySource({ handle: "", machineId: "", all: false }); 1505 1505 + } else if (data.type === "notepat:midi:sources") { 1506 1506 + requestRelaySources(); 1507 1507 + } 1508 1508 + }; 1509 1509 + window.addEventListener("message", relayMessageHandler); 1510 1510 + updateRelayBridgeState(); 1511 1511 + } 1512 1512 + 1513 1513 + // Create picture buffer at quarter resolution (quarter width, quarter height) 1514 1514 + const pictureWidth = Math.max(1, Math.floor(screen.width / 4)); 1357 1515 const pictureHeight = Math.max(1, Math.floor(screen.height / 4)); 1358 1516 picture = painting(pictureWidth, pictureHeight, ({ wipe }) => { 1359 1517 wipe("gray"); ··· 1526 1684 setupButtons(api); 1527 1685 } 1528 1686 1529 1529 - function sim({ sound, simCount, num, clock, painting }) { 1530 1530 - const simTick = typeof simCount === "bigint" ? Number(simCount) : simCount; 1531 1531 - 1532 1532 - if (lowLatencyMode) { 1687 1687 + function sim({ sound, simCount, num, clock, painting }) { 1688 1688 + const simTick = typeof simCount === "bigint" ? Number(simCount) : simCount; 1689 1689 + setSoundContext({ 1690 1690 + synth: sound?.synth, 1691 1691 + play: sound?.play, 1692 1692 + freq: sound?.freq, 1693 1693 + num, 1694 1694 + }); 1695 1695 + flushRelayMidiQueue(); 1696 1696 + 1697 1697 + if (lowLatencyMode) { 1533 1698 if (simTick % 3 === 0) sound.speaker?.poll(); 1534 1699 } else { 1535 1700 sound.speaker?.poll(); ··· 5320 5485 const connectedGamepads = {}; 5321 5486 let miniMapActiveNote = null; 5322 5487 let miniMapActiveKey = null; 5323 5323 - let topBarPianoActiveNote = null; // Track active note from top bar piano 5324 5324 - let soundContext = null; 5325 5325 - 5326 5326 - function setSoundContext(ctx) { 5327 5327 - soundContext = ctx; 5328 5328 - } 5329 5329 - 5330 5330 - function makeNoteSound(tone, velocity = 127, pan = 0) { 5331 5331 - const synth = soundContext?.synth; 5488 5488 + let topBarPianoActiveNote = null; // Track active note from top bar piano 5489 5489 + let soundContext = null; 5490 5490 + 5491 5491 + function setSoundContext(ctx) { 5492 5492 + soundContext = ctx; 5493 5493 + } 5494 5494 + 5495 5495 + function lowerBaseOctave() { 5496 5496 + return parseInt(octave) + lowerOctaveShift; 5497 5497 + } 5498 5498 + 5499 5499 + function upperBaseOctave() { 5500 5500 + return parseInt(octave) + 1 + upperOctaveShift; 5501 5501 + } 5502 5502 + 5503 5503 + function midiNoteToRelayButton(noteNumber) { 5504 5504 + if (typeof noteNumber !== "number") return null; 5505 5505 + 5506 5506 + const normalizedNote = ((noteNumber % 12) + 12) % 12; 5507 5507 + const noteName = MIDI_NOTE_NAMES[normalizedNote]; 5508 5508 + const noteOctave = Math.floor(noteNumber / 12) - 1; 5509 5509 + const lowerOct = lowerBaseOctave(); 5510 5510 + const upperOct = upperBaseOctave(); 5511 5511 + 5512 5512 + if (noteOctave === lowerOct && buttonNoteLookup.has(noteName)) { 5513 5513 + return noteName; 5514 5514 + } 5515 5515 + 5516 5516 + if (noteOctave === upperOct) { 5517 5517 + const upperNote = `+${noteName}`; 5518 5518 + if (buttonNoteLookup.has(upperNote)) return upperNote; 5519 5519 + } 5520 5520 + 5521 5521 + if (noteOctave < lowerOct) { 5522 5522 + const baseNote = noteOctave < lowerOct - 1 ? noteName : `+${noteName}`; 5523 5523 + if (buttonNoteLookup.has(baseNote)) return baseNote; 5524 5524 + } 5525 5525 + 5526 5526 + if (noteOctave > upperOct) { 5527 5527 + const baseNote = noteOctave > upperOct + 1 ? noteName : `+${noteName}`; 5528 5528 + if (buttonNoteLookup.has(baseNote)) return baseNote; 5529 5529 + } 5530 5530 + 5531 5531 + if (buttonNoteLookup.has(noteName)) return noteName; 5532 5532 + if (buttonNoteLookup.has(`+${noteName}`)) return `+${noteName}`; 5533 5533 + return null; 5534 5534 + } 5535 5535 + 5536 5536 + function startRelayButtonNote(buttonNote, velocity = 127) { 5537 5537 + if (!buttonNote) return false; 5538 5538 + 5539 5539 + const num = soundContext?.num; 5540 5540 + const synth = soundContext?.synth; 5541 5541 + const freq = soundContext?.freq; 5542 5542 + 5543 5543 + if (song && buttonNote.toUpperCase() !== song?.[songIndex]?.[0]) { 5544 5544 + synth?.({ 5545 5545 + type: "noise-white", 5546 5546 + tone: 1000, 5547 5547 + duration: 0.05, 5548 5548 + volume: 0.3, 5549 5549 + attack: 0, 5550 5550 + }); 5551 5551 + return false; 5552 5552 + } 5553 5553 + 5554 5554 + anyDown = true; 5555 5555 + noteShake[buttonNote] = 3; 5556 5556 + 5557 5557 + let noteName = buttonNote; 5558 5558 + let targetOctave = lowerBaseOctave(); 5559 5559 + if (buttonNote.startsWith("+")) { 5560 5560 + noteName = buttonNote.slice(1); 5561 5561 + targetOctave = upperBaseOctave(); 5562 5562 + } 5563 5563 + 5564 5564 + const tone = `${targetOctave}${noteName.toUpperCase()}`; 5565 5565 + const active = orderedByCount(sounds); 5566 5566 + 5567 5567 + if (slide && active.length > 0) { 5568 5568 + sounds[active[0]]?.sound?.update({ tone, duration: 0.1 }); 5569 5569 + tonestack[buttonNote] = { 5570 5570 + count: Object.keys(tonestack).length, 5571 5571 + tone, 5572 5572 + }; 5573 5573 + sounds[buttonNote] = sounds[active[0]]; 5574 5574 + if (sounds[buttonNote]) sounds[buttonNote].note = buttonNote; 5575 5575 + delete sounds[active[0]]; 5576 5576 + applyPitchBendToNotes([buttonNote], { immediate: true }); 5577 5577 + } else { 5578 5578 + tonestack[buttonNote] = { 5579 5579 + count: Object.keys(tonestack).length, 5580 5580 + tone, 5581 5581 + }; 5582 5582 + 5583 5583 + const pan = getPanForButtonNote(buttonNote); 5584 5584 + let soundHandle = makeNoteSound(tone, velocity, pan); 5585 5585 + 5586 5586 + if (!soundHandle || typeof soundHandle.kill !== "function") { 5587 5587 + const velocityRatio = velocity === undefined ? 1 : velocity / 127; 5588 5588 + const clampedRatio = num?.clamp 5589 5589 + ? num.clamp(velocityRatio, 0, 1) 5590 5590 + : Math.max(0, Math.min(1, velocityRatio)); 5591 5591 + const minVelocityVolume = 0.05; 5592 5592 + const fallbackVolume = 5593 5593 + toneVolume * (minVelocityVolume + (1 - minVelocityVolume) * clampedRatio); 5594 5594 + soundHandle = synth?.({ 5595 5595 + type: "sine", 5596 5596 + tone: freq?.(tone), 5597 5597 + attack: quickFade ? 0.0015 : attack, 5598 5598 + decay: 0.9, 5599 5599 + duration: 0.4, 5600 5600 + volume: fallbackVolume, 5601 5601 + pan, 5602 5602 + }); 5603 5603 + } 5604 5604 + 5605 5605 + sounds[buttonNote] = { 5606 5606 + note: buttonNote, 5607 5607 + count: active.length + 1, 5608 5608 + sound: soundHandle, 5609 5609 + }; 5610 5610 + 5611 5611 + applyPitchBendToNotes([buttonNote], { immediate: true }); 5612 5612 + 5613 5613 + if (buttonNote.toUpperCase() === song?.[songIndex]?.[0]) { 5614 5614 + songNoteDown = true; 5615 5615 + } 5616 5616 + 5617 5617 + delete trail[buttonNote]; 5618 5618 + 5619 5619 + if (autopatApi) pictureAdd(autopatApi, tone); 5620 5620 + } 5621 5621 + 5622 5622 + if (buttons[buttonNote]) { 5623 5623 + buttons[buttonNote].down = true; 5624 5624 + buttons[buttonNote].over = true; 5625 5625 + } 5626 5626 + 5627 5627 + return true; 5628 5628 + } 5629 5629 + 5630 5630 + function stopRelayButtonNote(buttonNote) { 5631 5631 + if (!buttonNote) return; 5632 5632 + 5633 5633 + const orderedTones = orderedByCount(tonestack); 5634 5634 + 5635 5635 + if (slide && orderedTones.length > 1 && sounds[buttonNote]) { 5636 5636 + const previousKey = orderedTones[orderedTones.length - 2]; 5637 5637 + const previousTone = tonestack[previousKey]?.tone; 5638 5638 + if (previousTone) { 5639 5639 + sounds[buttonNote]?.sound?.update({ tone: previousTone, duration: 0.1 }); 5640 5640 + sounds[previousKey] = sounds[buttonNote]; 5641 5641 + if (sounds[previousKey]) sounds[previousKey].note = previousKey; 5642 5642 + applyPitchBendToNotes([previousKey], { immediate: true }); 5643 5643 + } 5644 5644 + } else if (sounds[buttonNote]?.sound) { 5645 5645 + const soundEntry = sounds[buttonNote]; 5646 5646 + const lifespan = soundEntry.sound?.startedAt 5647 5647 + ? performance.now() / 1000 - soundEntry.sound.startedAt 5648 5648 + : 0.1; 5649 5649 + const fade = max(0.075, min(lifespan, 0.15)); 5650 5650 + soundEntry.sound.kill(quickFade ? fastFade : fade); 5651 5651 + } 5652 5652 + 5653 5653 + if (buttonNote.toUpperCase() === song?.[songIndex]?.[0]) { 5654 5654 + songIndex = (songIndex + 1) % song.length; 5655 5655 + songNoteDown = false; 5656 5656 + songShifting = true; 5657 5657 + } 5658 5658 + 5659 5659 + delete tonestack[buttonNote]; 5660 5660 + delete sounds[buttonNote]; 5661 5661 + trail[buttonNote] = 1; 5662 5662 + 5663 5663 + if (buttons[buttonNote]) { 5664 5664 + buttons[buttonNote].down = false; 5665 5665 + buttons[buttonNote].over = false; 5666 5666 + } 5667 5667 + } 5668 5668 + 5669 5669 + function flushRelayMidiQueue() { 5670 5670 + if (relayNeedsPanic) { 5671 5671 + for (const buttonNote of relayActiveNotes.values()) { 5672 5672 + stopRelayButtonNote(buttonNote); 5673 5673 + } 5674 5674 + relayActiveNotes.clear(); 5675 5675 + relayNeedsPanic = false; 5676 5676 + } 5677 5677 + 5678 5678 + while (relayMidiQueue.length > 0) { 5679 5679 + const event = relayMidiQueue.shift(); 5680 5680 + const noteNumber = Number(event?.note); 5681 5681 + if (!Number.isFinite(noteNumber)) continue; 5682 5682 + 5683 5683 + const key = [ 5684 5684 + normalizeRelayHandle(event?.handle), 5685 5685 + `${event?.machineId || "unknown"}`, 5686 5686 + `${event?.channel ?? 0}`, 5687 5687 + `${Math.round(noteNumber)}`, 5688 5688 + ].join(":"); 5689 5689 + 5690 5690 + if (event?.event === "note_off" || Number(event?.velocity) === 0) { 5691 5691 + const activeButtonNote = relayActiveNotes.get(key) || midiNoteToRelayButton(Math.round(noteNumber)); 5692 5692 + if (activeButtonNote) stopRelayButtonNote(activeButtonNote); 5693 5693 + relayActiveNotes.delete(key); 5694 5694 + continue; 5695 5695 + } 5696 5696 + 5697 5697 + const buttonNote = midiNoteToRelayButton(Math.round(noteNumber)); 5698 5698 + if (!buttonNote) continue; 5699 5699 + if (relayActiveNotes.has(key)) { 5700 5700 + stopRelayButtonNote(relayActiveNotes.get(key)); 5701 5701 + } 5702 5702 + if (startRelayButtonNote(buttonNote, Number(event?.velocity) || 127)) { 5703 5703 + relayActiveNotes.set(key, buttonNote); 5704 5704 + } 5705 5705 + } 5706 5706 + } 5707 5707 + 5708 5708 + function makeNoteSound(tone, velocity = 127, pan = 0) { 5709 5709 + const synth = soundContext?.synth; 5332 5710 const play = soundContext?.play; 5333 5711 const freq = soundContext?.freq; 5334 5712 const num = soundContext?.num;