Overhaul pitch detection and note matching

- Pick detector per instrument range: ACF2PLUS (robust down to ~40 Hz) for
  normal instruments, YIN only for the whistle register. pitchfinder's YIN
  latches onto tau=2 (~20 kHz garbage) below ~100 Hz and for quiet input;
  the old fixed 4096 window masked this, the smaller window exposed it.
- RMS-normalize analysis input for the YIN path only; ACF2PLUS keeps the raw
  signal so its internal unpitched gate rejects quiet decayed audio instead
  of amplifying noise into confident wrong pitches (tuner stuck on 'a note').
- Adaptive analysis window sized to the instrument's lowest note (~2 periods):
  half the latency, a quarter of the CPU for guitar/piano/voice; bass keeps
  the full window for E1.
- Reject detections implausibly below the instrument floor (mains hum).
- Clear stale pitch readings after 200 ms instead of keeping them forever.
- Shared AudioEngine singleton for all playback (was: two AudioContexts);
  tracks audibility so the app stops hearing its own speaker (ear-training
  self-scoring) and gates matching plus display during playback.
- MatchTracker: grace-period based note hold instead of reset-on-every-flicker
  (YIN flickers during attacks/decays; the old hard reset made the 50 ms
  threshold fire unreliably).
- Display damping in the producer (EMA for cents/level, 3-frame note-name
  hysteresis); display components render directly without CSS transitions.
- Regression tests: synthetic signals through the exact pipeline helpers
  (MatchTracker + pitchDetection test suites).
This commit is contained in:
2026-09-13 18:52:21 +02:00
parent d8602e0bd8
commit 079d574b66
12 changed files with 873 additions and 182 deletions

84
src/audio/AudioEngine.ts Normal file
View File

@@ -0,0 +1,84 @@
import { midiToFrequency } from '../music/NoteUtils';
import { PLAYBACK_GATE_TAIL_MS } from '../AppConfig';
/**
* Create an AudioContext with a typed webkit fallback (older Safari).
*/
export function createAudioContext(): AudioContext {
const Ctor =
window.AudioContext ??
(window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
return new Ctor({ latencyHint: 'interactive' });
}
/**
* Shared audio playback engine (singleton).
*
* All speaker output goes through a single AudioContext (previously App and
* useGameLogic each created their own via separate useAudioPlayer instances).
* Playback is tracked so pitch detection can ignore the microphone while the
* app is audible — otherwise the app hears its own reference notes through
* the speaker and scores them as correct (ear-training feedback loop).
*/
class AudioEngine {
private ctx: AudioContext | null = null;
/** ctx.currentTime (seconds) until which output may still be audible, incl. gate tail */
private audibleUntil = 0;
private ensureContext(): AudioContext {
if (!this.ctx) {
this.ctx = createAudioContext();
}
if (this.ctx.state === 'suspended') {
void this.ctx.resume();
}
return this.ctx;
}
playNote(midi: number, duration = 0.5, volume = 0.3): void {
const ctx = this.ensureContext();
const osc = ctx.createOscillator();
const gainNode = ctx.createGain();
osc.frequency.value = midiToFrequency(midi);
// Triangle wave is often nicer than sine for music training
osc.type = 'triangle';
osc.connect(gainNode);
gainNode.connect(ctx.destination);
const now = ctx.currentTime;
// Attack
gainNode.gain.setValueAtTime(0, now);
gainNode.gain.linearRampToValueAtTime(volume, now + 0.02);
// Sustain -> Release
gainNode.gain.exponentialRampToValueAtTime(0.001, now + duration);
osc.start(now);
osc.stop(now + duration + 0.1); // Stop slightly after fade out
// The note decays to -60dB at `now + duration`; keep the mic gated for
// an additional tail so room reverb / speaker bleed can't self-trigger.
this.audibleUntil = Math.max(
this.audibleUntil,
now + duration + PLAYBACK_GATE_TAIL_MS / 1000
);
}
/**
* True while speaker output (plus gate tail) may still be audible at the
* microphone. Mic-based note matching should be ignored during this time.
*
* A suspended AudioContext (autoplay policy, no user gesture yet) reports
* currentTime = 0 forever — without the state check the gate would be
* stuck "audible" permanently and matching would never engage.
*/
isAudible(): boolean {
if (!this.ctx || this.ctx.state !== 'running') return false;
return this.ctx.currentTime < this.audibleUntil;
}
}
export const audioEngine = new AudioEngine();

View File

@@ -1,5 +1,15 @@
import { YIN } from "pitchfinder";
import { MIC_SENSITIVITY_DB_RANGE, AUDIO_CONSTRAINTS } from "../AppConfig";
import { YIN, ACF2PLUS } from "pitchfinder";
import {
MIC_SENSITIVITY_DB_RANGE,
AUDIO_CONSTRAINTS,
PITCH_BUFFER_SIZE,
PITCH_MIN_WINDOW,
PITCH_DEFAULT_LOWEST_HZ,
PITCH_HIGH_REGISTER_HZ,
PITCH_NORMALIZE_TARGET_RMS,
PITCH_BELOW_FLOOR_REJECT_FACTOR,
} from "../AppConfig";
import { createAudioContext } from "./AudioEngine";
export interface MicrophoneDebugInfo {
audioContextState: AudioContextState | 'inactive';
@@ -12,13 +22,85 @@ export interface MicrophoneDebugInfo {
isCapturing: boolean;
}
export type PitchDetectorFn = (buffer: Float32Array) => number | null;
export interface DetectorSpec {
detector: PitchDetectorFn;
/** Whether the analysis buffer must be RMS-normalized before detection */
normalize: boolean;
}
/**
* Smallest power-of-two analysis window that still fits ~2 periods of the
* lowest required frequency. Smaller window = lower latency and ~4x less CPU
* per analysis frame. Clamped to [PITCH_MIN_WINDOW, PITCH_BUFFER_SIZE].
*/
export function chooseAnalysisWindowSize(sampleRate: number, lowestHz: number): number {
const needed = Math.ceil((2 * sampleRate) / lowestHz);
const pow2 = 2 ** Math.ceil(Math.log2(Math.max(1, needed)));
return Math.max(PITCH_MIN_WINDOW, Math.min(PITCH_BUFFER_SIZE, pow2));
}
/**
* Pick a detector for the instrument's range.
*
* pitchfinder's YIN force-sets yinBuffer[1] = 1 before cumulative
* normalization, which makes its absolute-threshold search latch onto tau=2
* (~20 kHz garbage) for low fundamentals — below ~100 Hz it is unreliable no
* matter the window size. ACF2PLUS is robust from ~40 Hz up to ~2 kHz, but
* tends to pick subharmonics above that. So: ACF2PLUS for normal instruments,
* YIN only for high registers (e.g. whistle) where its low-end weakness is
* irrelevant.
*
* Only the YIN path gets normalized input: ACF2PLUS has an internal
* "rms < 0.01 -> unpitched" gate that must see the RAW signal — normalizing
* first would amplify room hum/noise floors into confident wrong pitches
* (the tuner then appears "stuck" on a note nobody played).
*/
export function selectDetector(sampleRate: number, lowestHz: number): DetectorSpec {
if (lowestHz >= PITCH_HIGH_REGISTER_HZ) {
return { detector: YIN({ sampleRate }), normalize: true };
}
return { detector: ACF2PLUS({ sampleRate }), normalize: false };
}
/** Kept for backwards compatibility with existing tests. */
export function createPitchDetector(sampleRate: number, lowestHz: number): PitchDetectorFn {
return selectDetector(sampleRate, lowestHz).detector;
}
/**
* Scale the buffer in place to a constant RMS operating level (clamped to
* [-1, 1]). Detectors are sensitive to absolute amplitude; without this,
* quiet low-frequency input makes YIN return ~20 kHz nonsense.
*/
export function normalizeSignal(buffer: Float32Array, targetRms?: number): void {
let rms = 0;
for (let i = 0; i < buffer.length; i++) rms += buffer[i] * buffer[i];
rms = Math.sqrt(rms / buffer.length);
if (rms < 1e-6) return;
const target = targetRms ?? PITCH_NORMALIZE_TARGET_RMS;
const gain = target / rms;
for (let i = 0; i < buffer.length; i++) {
const v = buffer[i] * gain;
buffer[i] = v > 1 ? 1 : v < -1 ? -1 : v;
}
}
export class PitchAnalyzer {
private detector: (buffer: Float32Array) => number | null;
private detector: PitchDetectorFn | null = null;
private normalizeInput = false;
private audioContext: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private mediaStream: MediaStream | null = null;
private source: MediaStreamAudioSourceNode | null = null;
private buffer: Float32Array;
/** Full-size backing buffer (fftSize samples) */
private buffer = new Float32Array(PITCH_BUFFER_SIZE);
/** Tail-view into `buffer`; only the most recent windowSize samples are analyzed */
private window: Float32Array<ArrayBuffer> = this.buffer;
private windowSize = PITCH_BUFFER_SIZE;
private lowestFrequency = PITCH_DEFAULT_LOWEST_HZ;
// Debug info
private inputDeviceLabel: string = 'Not detected';
@@ -26,11 +108,6 @@ export class PitchAnalyzer {
private permissionState: PermissionState | 'unknown' = 'unknown';
private currentRmsLevel: number = 0;
constructor() {
this.detector = YIN({ sampleRate: 44100 }); // Default, will update on start
this.buffer = new Float32Array(2048); // Standard size
}
private sensitivityThreshold = 0.03; // Default
/**
@@ -49,6 +126,43 @@ export class PitchAnalyzer {
this.sensitivityThreshold = Math.pow(10, db / 20);
}
/**
* Configure the lowest frequency that must be detectable (Hz) and shrink
* the analysis window accordingly.
*
* Pitch detection needs ~2 periods of the target frequency in the window.
* A fixed 4096-sample window always analyzes ~90ms of audio history, which
* makes the tuner feel sluggish. Guitar/piano/voice (>= ~65 Hz) only need
* 2048 samples (~45ms) — half the latency at a quarter of the CPU cost.
* Bass (E1 ~ 41 Hz) keeps the full 4096 window.
*/
setLowestFrequency(hz: number) {
if (hz > 0) {
this.lowestFrequency = hz;
this.updateWindowSize();
this.rebuildDetector();
}
}
private updateWindowSize() {
if (!this.audioContext) return; // applied on start()
const size = chooseAnalysisWindowSize(this.audioContext.sampleRate, this.lowestFrequency);
if (size !== this.windowSize) {
this.windowSize = size;
// View over the tail of the backing buffer: getFloatTimeDomainData
// fills it with the most recent `size` samples.
const byteOffset = (PITCH_BUFFER_SIZE - size) * Float32Array.BYTES_PER_ELEMENT;
this.window = new Float32Array(this.buffer.buffer, byteOffset, size);
}
}
private rebuildDetector() {
if (!this.audioContext) return; // applied on start()
const spec = selectDetector(this.audioContext.sampleRate, this.lowestFrequency);
this.detector = spec.detector;
this.normalizeInput = spec.normalize;
}
async start(): Promise<void> {
if (this.audioContext) return;
@@ -63,8 +177,9 @@ export class PitchAnalyzer {
this.permissionState = 'unknown';
}
this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)();
this.detector = YIN({ sampleRate: this.audioContext.sampleRate });
this.audioContext = createAudioContext();
this.updateWindowSize();
this.rebuildDetector();
try {
// Use configured audio constraints
@@ -98,8 +213,7 @@ export class PitchAnalyzer {
this.source = this.audioContext.createMediaStreamSource(this.mediaStream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 4096; // Higher FFT size for better resolution at low frequencies
this.buffer = new Float32Array(this.analyser.fftSize);
this.analyser.fftSize = PITCH_BUFFER_SIZE; // Large buffer; analyzed window adapts per instrument
this.source.connect(this.analyser);
@@ -123,6 +237,9 @@ export class PitchAnalyzer {
this.audioContext.close();
this.audioContext = null;
}
this.detector = null;
this.analyser = null;
this.source = null;
this.currentRmsLevel = 0;
this.inputDeviceLabel = 'Not detected';
this.inputDeviceId = '';
@@ -159,26 +276,43 @@ export class PitchAnalyzer {
* getPitch returns the detected frequency in Hz, or null if no pitch detected
*/
getPitch(): number | null {
if (!this.analyser) return null;
this.analyser.getFloatTimeDomainData(this.buffer as any);
if (!this.analyser || !this.detector) return null;
// Always copy the full fftSize buffer (universally supported usage),
// then analyze only the tail view = most recent windowSize samples.
// (Passing an undersized array directly relies on browser-specific
// behavior of getFloatTimeDomainData.)
this.analyser.getFloatTimeDomainData(this.buffer);
// Simple RMS volume check to avoid noise
// Simple RMS volume check to avoid noise (on the analyzed window)
let rms = 0;
for (let i = 0; i < this.buffer.length; i++) {
rms += this.buffer[i] * this.buffer[i];
for (let i = 0; i < this.window.length; i++) {
rms += this.window[i] * this.window[i];
}
rms = Math.sqrt(rms / this.buffer.length);
rms = Math.sqrt(rms / this.window.length);
// Store RMS for level meter
this.currentRmsLevel = rms;
if (rms < this.sensitivityThreshold) return null;
const pitch = this.detector(this.buffer);
// Normalize only on the YIN/high-register path. ACF2PLUS must see the
// raw signal so its internal "rms < 0.01 -> unpitched" gate keeps
// rejecting quiet decayed audio instead of amplifying it into a
// confident wrong pitch.
if (this.normalizeInput) {
normalizeSignal(this.window);
}
const pitch = this.detector(this.window);
// Widen range for Bass (E1 ~41Hz, but A0 is 27.5Hz) and Whistle (C8 ~4186Hz, Harmonics go higher)
// Range 25Hz - 8000Hz covers Piano A0 to well above highest fundamental
if (pitch && (pitch < 25 || pitch > 8000)) return null;
// (Also filters out the -1 some detectors return for "unpitched".)
if (pitch === null || pitch < 25 || pitch > 8000) return null;
// Implausible for the current instrument (e.g. 50/60 Hz mains hum
// while playing guitar): reject readings far below the configured floor.
if (pitch < this.lowestFrequency * PITCH_BELOW_FLOOR_REJECT_FACTOR) return null;
return pitch;
}

View File

@@ -0,0 +1,156 @@
import { describe, it, expect } from 'vitest';
import {
chooseAnalysisWindowSize,
selectDetector,
normalizeSignal,
} from './PitchAnalyzer';
import {
PITCH_BUFFER_SIZE,
PITCH_MIN_WINDOW,
PITCH_BELOW_FLOOR_REJECT_FACTOR,
} from '../AppConfig';
/**
* Regression tests for the pitch detection pipeline.
*
* pitchfinder's YIN returns ~20 kHz garbage for low fundamentals (its
* forced yinBuffer[1] = 1 breaks the absolute-threshold search below
* ~100 Hz) and for quiet input. The app therefore uses ACF2PLUS for normal
* instruments, YIN only for high registers, and normalizes the analysis
* buffer to a constant RMS. These tests synthesize instrument-like signals
* (fundamental + harmonics + pluck decay, deterministic) and run them
* through the exact helpers PitchAnalyzer uses.
*/
/** Deterministic instrument-like test signal (no RNG → no flaky tests). */
function synth(freq: number, sampleRate: number, length: number, amp: number): Float32Array {
const buf = new Float32Array(length);
for (let i = 0; i < length; i++) {
const t = i / sampleRate;
const env = Math.exp(-t * 3); // pluck-like decay
buf[i] = amp * env * (
Math.sin(2 * Math.PI * freq * t) +
0.5 * Math.sin(2 * Math.PI * freq * 2 * t) +
0.25 * Math.sin(2 * Math.PI * freq * 3 * t) +
0.1 * Math.sin(2 * Math.PI * freq * 4 * t) +
0.02 * Math.sin(2 * Math.PI * 7.13 * i / length) // deterministic "noise" dither
);
}
return buf;
}
/** Mirror of PitchAnalyzer.getPitch, minus microphone plumbing. */
function detect(freq: number, sampleRate: number, lowestHz: number, amp = 0.3): number | null {
const size = chooseAnalysisWindowSize(sampleRate, lowestHz);
const buffer = synth(freq, sampleRate, size, amp);
const { detector, normalize } = selectDetector(sampleRate, lowestHz);
if (normalize) normalizeSignal(buffer);
const pitch = detector(buffer);
if (pitch === null || pitch < 25 || pitch > 8000) return null;
if (pitch < lowestHz * PITCH_BELOW_FLOOR_REJECT_FACTOR) return null;
return pitch;
}
function expectCloseTo(actual: number | null, expected: number) {
expect(actual).not.toBeNull();
expect(Math.abs(actual! - expected) / expected).toBeLessThan(0.03);
}
describe('chooseAnalysisWindowSize', () => {
it('adapts to the instrument floor', () => {
// Guitar low E2 (~78 Hz w/ margin) -> half window at 48k
expect(chooseAnalysisWindowSize(48000, 77.8)).toBe(2048);
// Bass low E1 (~39 Hz w/ margin) -> full window
expect(chooseAnalysisWindowSize(48000, 38.9)).toBe(PITCH_BUFFER_SIZE);
// Whistle register -> minimum window
expect(chooseAnalysisWindowSize(48000, 622)).toBe(PITCH_MIN_WINDOW);
});
});
describe('pitch detection across instruments (sample rate 48000)', () => {
const SR = 48000;
it('guitar (low E2)', () => {
const floor = 77.8; // E2 minus one semitone
expectCloseTo(detect(82.4, SR, floor), 82.4); // E2 — the regression case
expectCloseTo(detect(110.0, SR, floor), 110); // A2
expectCloseTo(detect(329.6, SR, floor), 329.6); // E4
});
it('guitar at low input level (quiet pluck tail)', () => {
expectCloseTo(detect(82.4, SR, 77.8, 0.05), 82.4);
expectCloseTo(detect(440, SR, 77.8, 0.05), 440);
});
it('bass (low E1)', () => {
const floor = 38.9;
expectCloseTo(detect(41.2, SR, floor), 41.2); // E1 — broken even before the rewrite
expectCloseTo(detect(55.0, SR, floor), 55); // A1
});
it('piano', () => {
const floor = 61.7; // C2 minus one semitone
expectCloseTo(detect(65.4, SR, floor), 65.4); // C2
expectCloseTo(detect(261.6, SR, floor), 261.6); // C4
expectCloseTo(detect(1046.5, SR, floor), 1046.5); // C6
});
it('whistle register (uses YIN, small window)', () => {
const floor = 622; // E5 minus one semitone
expectCloseTo(detect(659.3, SR, floor), 659.3); // E5
expectCloseTo(detect(2093.0, SR, floor), 2093); // C7
expectCloseTo(detect(4186.0, SR, floor), 4186); // C8
});
});
describe('pitch detection (sample rate 44100)', () => {
it('still works', () => {
expectCloseTo(detect(82.4, 44100, 77.8), 82.4);
expectCloseTo(detect(41.2, 44100, 38.9), 41.2);
expectCloseTo(detect(440, 44100, 77.8), 440);
});
});
describe('normalizeSignal', () => {
it('scales to the target RMS', () => {
const buf = synth(440, 48000, 2048, 0.02);
normalizeSignal(buf, 0.3);
let rms = 0;
for (const v of buf) rms += v * v;
rms = Math.sqrt(rms / buf.length);
expect(rms).toBeGreaterThan(0.2); // ~0.3 modulo decay envelope shape
});
it('leaves silence untouched', () => {
const buf = new Float32Array(1024);
normalizeSignal(buf);
expect(buf.every(v => v === 0)).toBe(true);
});
});
describe('noise-floor rejection (tuner freeze regression)', () => {
const SR = 48000;
it('rejects 50 Hz mains hum for guitar (floor ~78 Hz)', () => {
expect(detect(50, SR, 77.8)).toBeNull();
});
it('rejects 60 Hz mains hum for guitar', () => {
expect(detect(60, SR, 77.8)).toBeNull();
});
it('rejects hum for the whistle register too', () => {
expect(detect(50, SR, 622)).toBeNull();
expect(detect(120, SR, 622)).toBeNull();
});
it('returns null for very quiet decayed signal instead of a confident wrong pitch', () => {
// Raw RMS ~0.003: below ACF2PLUS's internal unpitched gate.
// (Normalization must NOT rescue this into a bogus reading.)
expect(detect(82.4, SR, 77.8, 0.005)).toBeNull();
});
it('still detects the string when it rings at moderate level', () => {
expectCloseTo(detect(82.4, SR, 77.8, 0.05), 82.4);
});
});