this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

no HPS

alice 08b6eb1f fc801525

+34 -262
-206
CLAUDE.md
··· 231 231 - `vqtos(octave, note)`: Smoothed VQT by musical note 232 232 233 233 ### Signal Processing Enhancements 234 - - **Harmonic-Percussive Separation**: Isolate tonal content from drums 235 234 - **Adaptive Thresholding**: Dynamic noise floor removal 236 - - **Note Onset Enhancement**: Make note attacks more visible 237 - - **Enhanced Variable-Q**: Support for 16K FFT for better bass resolution 238 235 239 236 ### Platform Support 240 237 - Add VQT to other language bindings (currently Lua only) ··· 248 245 - The studio uses immediate mode GUI principles 249 246 - Cartridge format is documented in wiki and src/studio/project.c 250 247 - PRO version enables additional features like extra memory banks 251 - 252 - ## Harmonic-Percussive Separation (HPS) Plan 253 - 254 - ### Overview 255 - 256 - Harmonic-Percussive Separation isolates tonal (harmonic) content from rhythmic (percussive) content in audio signals. This enhancement will apply HPS exclusively to VQT data, providing better separation of musical elements like sustained notes from drum hits. 257 - 258 - ### Algorithm Design 259 - 260 - **Median Filtering Approach (Fitzgerald 2010):** 261 - - Apply median filters on VQT magnitude spectrogram 262 - - Horizontal (time) median filter → captures harmonic content (sustained tones) 263 - - Vertical (frequency) median filter → captures percussive content (transients) 264 - - Generate masks by comparing filtered outputs 265 - - Apply masks to separate harmonic and percussive components 266 - 267 - ### Architecture 268 - 269 - #### New Files 270 - - **src/ext/hps.c**: HPS processing implementation 271 - - **src/ext/hps.h**: HPS interface and function declarations 272 - - **src/hpsdata.h/c**: HPS data structures and storage 273 - 274 - #### Data Flow 275 - ``` 276 - Audio Buffer → VQT Processing → VQT Magnitude → HPS Processing → Separated Components 277 - 278 - Harmonic & Percussive VQT 279 - ``` 280 - 281 - #### Integration in tic_core_tick 282 - ```c 283 - if (fftEnabled) { 284 - FFT_GetFFT(fftData); // Regular FFT processing (unchanged) 285 - 286 - if (vqtEnabled) { 287 - VQT_ProcessAudio(); 288 - 289 - if (hpsEnabled) { 290 - HPS_ProcessVQT(vqtData); // Apply HPS to VQT only 291 - } 292 - } 293 - } 294 - ``` 295 - 296 - ### Data Structures 297 - 298 - ```c 299 - // In hpsdata.h 300 - #define HPS_HISTORY_SIZE 32 // Frames of history for median filtering 301 - #define HPS_MEDIAN_SIZE_H 17 // Horizontal filter size (0.8-1.5s @ 5.4fps) 302 - #define HPS_MEDIAN_SIZE_P 17 // Vertical filter size (1.4 octaves) 303 - 304 - typedef struct { 305 - // VQT magnitude history buffer 306 - float vqtHistory[HPS_HISTORY_SIZE][120]; 307 - int historyIndex; 308 - 309 - // Separated VQT components 310 - float harmonicVQT[120]; // Sustained tonal content 311 - float percussiveVQT[120]; // Transient/rhythmic content 312 - 313 - // Smoothed outputs 314 - float harmonicSmoothing[120]; 315 - float percussiveSmoothing[120]; 316 - 317 - // Normalization data 318 - float harmonicNormalized[120]; 319 - float percussiveNormalized[120]; 320 - 321 - // Configuration 322 - float separationStrength; // 0.0-1.0, controls mask hardness 323 - float harmonicGain; // Gain for harmonic component 324 - float percussiveGain; // Gain for percussive component 325 - bool enabled; 326 - } HpsData; 327 - ``` 328 - 329 - ### API Functions 330 - 331 - ```lua 332 - -- VQT-based HPS functions (update rate ~5.4 fps) 333 - hpsh(bin) -- Get harmonic component at VQT bin (0-119) 334 - hpsp(bin) -- Get percussive component at VQT bin (0-119) 335 - hpshs(bin) -- Get smoothed harmonic component 336 - hpsps(bin) -- Get smoothed percussive component 337 - 338 - -- Musical note access (octave 0-9, note 0-11) 339 - hpsho(octave, note) -- Harmonic by musical note 340 - hpspo(octave, note) -- Percussive by musical note 341 - 342 - -- Configuration 343 - hpscfg(param, value) -- Configure HPS parameters 344 - -- param: "strength" (0.0-1.0), "hgain" (0.0-2.0), "pgain" (0.0-2.0) 345 - ``` 346 - 347 - ### Implementation Details 348 - 349 - #### Median Filtering 350 - - Use efficient sliding window median algorithm 351 - - Handle VQT's logarithmic frequency spacing 352 - - Circular buffer for time history 353 - - Optimize for 120 bins and 32 frame history 354 - 355 - #### Masking Strategy 356 - 1. Compute harmonic emphasis: `H = median_h(|VQT|²)` 357 - 2. Compute percussive emphasis: `P = median_p(|VQT|²)` 358 - 3. Generate binary masks: 359 - - Harmonic mask: `M_h = H >= P` 360 - - Percussive mask: `M_p = P > H` 361 - 4. Optional soft masking using Wiener filtering 362 - 363 - #### Memory Usage 364 - - History buffer: 32 × 120 × 4 bytes = 15KB 365 - - Processing buffers: ~10KB 366 - - Total additional memory: ~25KB 367 - 368 - ### Performance Considerations 369 - 370 - - Process only when VQT is updated (~5.4 fps) 371 - - Reuse existing VQT magnitude calculations 372 - - Optimize median filters for small kernel sizes 373 - - Cache-friendly memory access patterns 374 - - Optional SIMD for median operations 375 - 376 - ### Example Usage 377 - 378 - ```lua 379 - -- Visualize harmonic vs percussive content 380 - function TIC() 381 - cls(0) 382 - 383 - -- Draw harmonic content (sustained notes) 384 - for oct=2,6 do 385 - for note=0,11 do 386 - local h = hpsh(oct*12+note) * 50 387 - local x = note * 20 388 - local y = 100 - oct * 15 389 - rect(x, y-h, 18, h, note+1) 390 - end 391 - end 392 - 393 - -- Draw percussive hits 394 - for i=0,119 do 395 - local p = hpsp(i) 396 - if p > 0.1 then 397 - local x = (i % 12) * 20 398 - local y = 120 - (i // 12) * 10 399 - circ(x+9, y, p*20, 15) 400 - end 401 - end 402 - end 403 - 404 - -- React to bass drum vs bass note 405 - function TIC() 406 - local bassDrum = 0 407 - local bassNote = 0 408 - 409 - -- Sum percussive energy in bass range (20-80 Hz) 410 - for i=0,35 do -- First 3 octaves 411 - bassDrum = bassDrum + hpsp(i) 412 - end 413 - 414 - -- Find strongest harmonic in bass range 415 - for i=24,35 do -- 2nd octave 416 - if hpsh(i) > hpsh(bassNote) then 417 - bassNote = i 418 - end 419 - end 420 - 421 - -- Visualize separately 422 - cls(0) 423 - circ(120, 68, bassDrum * 30, 8) -- Drum pulse 424 - print("Note: " .. (bassNote % 12), 100, 60, (bassNote % 12) + 1) 425 - end 426 - ``` 427 - 428 - ### Implementation Phases 429 - 430 - **Phase 1: Core HPS Algorithm** 431 - - Implement circular buffer for VQT history 432 - - Create median filtering functions 433 - - Implement basic binary masking 434 - - Add core API functions 435 - 436 - **Phase 2: Enhancements** 437 - - Add Wiener soft masking option 438 - - Implement smoothing and normalization 439 - - Add gain controls 440 - - Optimize performance 441 - 442 - **Phase 3: Extended API** 443 - - Add musical note access functions 444 - - Implement configuration system 445 - - Create demo scripts 446 - - Documentation 447 - 448 - ### Benefits 449 - 450 - - **Better Bass Separation**: Distinguish bass drums from bass notes 451 - - **Cleaner Visualizations**: Separate melody from rhythm 452 - - **Musical Analysis**: Identify chord progressions without drum interference 453 - - **Creative Effects**: Process harmonic and percussive content differently
+30 -3
src/api/luaapi.c
··· 1611 1611 return 0; 1612 1612 } 1613 1613 1614 - // Return normalized VQT data 1615 - lua_pushnumber(lua, vqtNormalizedData[bin]); 1614 + // Return raw VQT data (normalized but unsmoothed) 1615 + lua_pushnumber(lua, vqtData[bin] / vqtPeakSmoothValue); 1616 1616 return 1; 1617 1617 } 1618 1618 1619 1619 luaL_error(lua, "invalid params, vqt(bin)\n"); 1620 1620 return 0; 1621 1621 } 1622 + 1623 + static s32 lua_vqts(lua_State* lua) 1624 + { 1625 + s32 top = lua_gettop(lua); 1626 + 1627 + if (top >= 1) 1628 + { 1629 + s32 bin = getLuaNumber(lua, 1); 1630 + 1631 + // Validate bin range 1632 + if (bin < 0 || bin >= VQT_BINS) 1633 + { 1634 + luaL_error(lua, "vqts bin out of range (0-%d)\n", VQT_BINS - 1); 1635 + return 0; 1636 + } 1637 + 1638 + // Return smoothed VQT data (smoothed + normalized) 1639 + // This gives more stable values 1640 + lua_pushnumber(lua, vqtNormalizedData[bin]); 1641 + return 1; 1642 + } 1643 + 1644 + luaL_error(lua, "invalid params, vqts(bin)\n"); 1645 + return 0; 1646 + } 1647 + 1622 1648 1623 1649 static int lua_dofile(lua_State *lua) 1624 1650 { ··· 1674 1700 registerLuaFunction(core, lua_dofile, "dofile"); 1675 1701 registerLuaFunction(core, lua_loadfile, "loadfile"); 1676 1702 1677 - // Register VQT function 1703 + // Register VQT functions 1678 1704 registerLuaFunction(core, lua_vqt, "vqt"); 1705 + registerLuaFunction(core, lua_vqts, "vqts"); 1679 1706 } 1680 1707 1681 1708 void luaapi_close(tic_mem* tic)
+1 -29
src/ext/vqt.c
··· 343 343 } 344 344 #endif 345 345 346 - // Apply spectral whitening if enabled 347 - #if VQT_SPECTRAL_WHITENING_ENABLED 348 - for (int i = 0; i < VQT_BINS; i++) 349 - { 350 - // Update running average for this bin 351 - vqtBinAverages[i] = vqtBinAverages[i] * VQT_WHITENING_DECAY + 352 - vqtData[i] * (1.0f - VQT_WHITENING_DECAY); 353 - 354 - // Ensure average doesn't go below floor 355 - if (vqtBinAverages[i] < VQT_WHITENING_FLOOR) 356 - vqtBinAverages[i] = VQT_WHITENING_FLOOR; 357 - 358 - // Apply whitening by dividing by the average 359 - vqtWhitenedData[i] = vqtData[i] / vqtBinAverages[i]; 360 - 361 - // Check for NaN or Inf 362 - if (!isfinite(vqtWhitenedData[i])) 363 - vqtWhitenedData[i] = 0.0f; 364 - } 365 - 366 - // Apply smoothing to whitened data 367 - for (int i = 0; i < VQT_BINS; i++) 368 - { 369 - vqtSmoothingData[i] = vqtSmoothingData[i] * VQT_SMOOTHING_FACTOR + 370 - vqtWhitenedData[i] * (1.0f - VQT_SMOOTHING_FACTOR); 371 - } 372 - #else 373 - // Apply smoothing to raw data (spectral whitening disabled) 346 + // Apply smoothing to raw data 374 347 for (int i = 0; i < VQT_BINS; i++) 375 348 { 376 349 vqtSmoothingData[i] = vqtSmoothingData[i] * VQT_SMOOTHING_FACTOR + 377 350 vqtData[i] * (1.0f - VQT_SMOOTHING_FACTOR); 378 351 } 379 - #endif 380 352 381 353 // Find peak for normalization 382 354 float currentPeak = 0.0f;
+1 -13
src/vqtdata.c
··· 1 1 #include "vqtdata.h" 2 2 #include <string.h> 3 3 #include <stdlib.h> 4 + #include <math.h> 4 5 5 6 // Global VQT data arrays 6 7 float vqtData[VQT_BINS]; ··· 17 18 // Array of kernels, one per VQT bin 18 19 VqtKernel vqtKernels[VQT_BINS]; 19 20 20 - // Spectral whitening data 21 - float vqtBinAverages[VQT_BINS]; 22 - float vqtWhitenedData[VQT_BINS]; 23 21 24 22 void VQT_Init(void) 25 23 { ··· 34 32 35 33 // Zero kernel pointers 36 34 memset(vqtKernels, 0, sizeof(vqtKernels)); 37 - 38 - // Initialize spectral whitening arrays 39 - memset(vqtBinAverages, 0, sizeof(vqtBinAverages)); 40 - memset(vqtWhitenedData, 0, sizeof(vqtWhitenedData)); 41 - 42 - // Start with small initial averages to avoid divide-by-zero 43 - for (int i = 0; i < VQT_BINS; i++) 44 - { 45 - vqtBinAverages[i] = VQT_WHITENING_FLOOR; 46 - } 47 35 } 48 36 49 37 void VQT_Cleanup(void)
+2 -11
src/vqtdata.h
··· 21 21 #define VQT_BASS_Q_MAX 17.0f // Full Q achieved at 80+ Hz 22 22 #define VQT_TREBLE_Q_FACTOR 11.0f // Smoother for high frequencies 23 23 24 - // Spectral whitening toggle (set to 0 to disable) 25 - #define VQT_SPECTRAL_WHITENING_ENABLED 1 26 - 27 - // Spectral whitening parameters 28 - #define VQT_WHITENING_DECAY 0.99f // Running average decay (0.98-0.995 for 1-2 second adaptation) 29 - #define VQT_WHITENING_FLOOR 0.001f // Minimum average to prevent divide-by-zero 24 + // Spectral whitening removed - caused spreading issues with VQT's inherent spectral leakage 30 25 31 26 // Raw VQT magnitude data 32 27 extern float vqtData[VQT_BINS]; ··· 44 39 // Enable flag (tied to fftEnabled initially) 45 40 extern bool vqtEnabled; 46 41 47 - // Spectral whitening data 48 - extern float vqtBinAverages[VQT_BINS]; // Long-term running averages per bin 49 - extern float vqtWhitenedData[VQT_BINS]; // Whitened VQT data 50 - 51 42 // Sparse kernel storage structures 52 43 typedef struct { 53 44 float* real; // Real parts of kernel (sparse) ··· 63 54 void VQT_Init(void); 64 55 65 56 // Free VQT kernel memory 66 - void VQT_Cleanup(void); 57 + void VQT_Cleanup(void);