jens/sheetmusictrainer
1
0
Fork 0
mirror of https://github.com/9x/sheetmusictrainer.git synced 2026-11-03 22:03:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

Overhaul pitch detection and note matching

Browse Source 
- Pick detector per instrument range: ACF2PLUS (robust down to ~40 Hz) for
  normal instruments, YIN only for the whistle register. pitchfinder's YIN
  latches onto tau=2 (~20 kHz garbage) below ~100 Hz and for quiet input;
  the old fixed 4096 window masked this, the smaller window exposed it.
- RMS-normalize analysis input for the YIN path only; ACF2PLUS keeps the raw
  signal so its internal unpitched gate rejects quiet decayed audio instead
  of amplifying noise into confident wrong pitches (tuner stuck on 'a note').
- Adaptive analysis window sized to the instrument's lowest note (~2 periods):
  half the latency, a quarter of the CPU for guitar/piano/voice; bass keeps
  the full window for E1.
- Reject detections implausibly below the instrument floor (mains hum).
- Clear stale pitch readings after 200 ms instead of keeping them forever.
- Shared AudioEngine singleton for all playback (was: two AudioContexts);
  tracks audibility so the app stops hearing its own speaker (ear-training
  self-scoring) and gates matching plus display during playback.
- MatchTracker: grace-period based note hold instead of reset-on-every-flicker
  (YIN flickers during attacks/decays; the old hard reset made the 50 ms
  threshold fire unreliably).
- Display damping in the producer (EMA for cents/level, 3-frame note-name
  hysteresis); display components render directly without CSS transitions.
- Regression tests: synthetic signals through the exact pipeline helpers
  (MatchTracker + pitchDetection test suites).
This commit is contained in:
Jens Mohrmann
2026-09-13 18:52:21 +02:00
parent d8602e0bd8
commit 079d574b66
12 changed files with 873 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/hooks/useAudioPlayer.ts
View File

@@ -1,61 +1,18 @@
import { useRef, useEffect, useCallback } from 'react';
import { midiToFrequency } from '../music/NoteUtils';
import { useCallback } from 'react';
import { audioEngine } from '../audio/AudioEngine';
interface AudioPlayerHandle {
playNote: (midi: number, duration?: number, volume?: number) => void;
}
/**
* Thin hook wrapper around the shared AudioEngine singleton.
* All playback goes through one AudioContext and is tracked there, so pitch
* detection can gate itself while the app's own speaker output is audible.
*/
export function useAudioPlayer(): AudioPlayerHandle {
const audioContext = useRef<AudioContext | null>(null);
// Initialize AudioContext lazily or on user interaction if possible,
// but here we just ensure it exists when we try to play.
const playNote = useCallback((midi: number, duration: number = 0.5, volume: number = 0.3) => {
if (!audioContext.current) {
audioContext.current = new (window.AudioContext || (window as any).webkitAudioContext)();
}
const ctx = audioContext.current;
if (ctx.state === 'suspended') {
ctx.resume();
}
const osc = ctx.createOscillator();
const gainNode = ctx.createGain();
const freq = midiToFrequency(midi);
osc.frequency.value = freq;
// Triangle wave is often nicer than sine for music training
osc.type = 'triangle';
osc.connect(gainNode);
gainNode.connect(ctx.destination);
const now = ctx.currentTime;
// Attack
gainNode.gain.setValueAtTime(0, now);
gainNode.gain.linearRampToValueAtTime(volume, now + 0.02);
// Sustain -> Release
// Note: For a "pluck" or single hit, we just ramp down.
gainNode.gain.exponentialRampToValueAtTime(0.001, now + duration);
osc.start(now);
osc.stop(now + duration + 0.1); // Stop slightly after fade out
}, []);
// Cleanup
useEffect(() => {
return () => {
if (audioContext.current) {
audioContext.current.close();
audioContext.current = null;
}
};
audioEngine.playNote(midi, duration, volume);
}, []);
return { playNote };

112
src/hooks/useGameLogic.ts
View File

@@ -1,7 +1,9 @@
import { useState, useEffect, useCallback, useMemo, useRef } from 'react';
import { useSettings } from '../context/SettingsContext';
import { useSettings } from '../context/useSettings';
import { useAudioPlayer } from './useAudioPlayer';
import { useMetronome } from './useMetronome';
import { audioEngine } from '../audio/AudioEngine';
import { MatchTracker } from '../game/MatchTracker';
import {
getRandomNote,
getNoteDetails
@@ -12,7 +14,11 @@ import {
getFirstPositionNotes,
} from '../music/Tunings';
import { INSTRUMENT_DEFINITIONS } from '../music/InstrumentConfigs';
import { NOTE_MATCH_THRESHOLD_MS } from '../AppConfig';
import {
NOTE_MATCH_THRESHOLD_MS,
NOTE_MATCH_GRACE_MS,
REFERENCE_QUIET_MAX_WAIT_MS,
} from '../AppConfig';
export const useGameLogic = (
pitchData: { midi: number; note: string; cents: number; frequency: number } | null
@@ -21,12 +27,14 @@ export const useGameLogic = (
const { playNote } = useAudioPlayer();
const [targetMidi, setTargetMidi] = useState<number>(60);
const [matchStartTime, setMatchStartTime] = useState<number | null>(null);
const [feedbackMessage, setFeedbackMessage] = useState<string>("");
const [revealed, setRevealed] = useState(false);
const [virtualNote, setVirtualNote] = useState<number | null>(null);
const feedbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
// Flicker-tolerant match timer: brief detection dropouts don't reset progress
const [tracker] = useState(() => new MatchTracker(NOTE_MATCH_THRESHOLD_MS, NOTE_MATCH_GRACE_MS));
// --- Valid Notes Calculation ---
const currentTuning = TUNINGS[settings.tuningId];
const currentInstrumentDef = INSTRUMENT_DEFINITIONS[settings.instrument];
@@ -94,19 +102,18 @@ export const useGameLogic = (
const min = validNotes[0];
const max = validNotes[validNotes.length - 1];
const newNote = getRandomNote(min, max, validNotes);
if (newNote === targetMidi && validNotes.length > 1) {
const retry = getRandomNote(min, max, validNotes);
setTargetMidi(retry);
} else {
setTargetMidi(newNote);
}
setMatchStartTime(null);
// Avoid repeating the previous note (a single retry could still land on it)
const candidates = validNotes.length > 1
? validNotes.filter(n => n !== targetMidi)
: validNotes;
setTargetMidi(getRandomNote(min, max, candidates));
tracker.reset();
if (!keepFeedback) {
setFeedbackMessage("");
}
setRevealed(false);
}, [validNotes, targetMidi]);
}, [validNotes, targetMidi, tracker]);
// --- Metronome Logic ---
const effectiveBpm = useMemo(() => {
@@ -129,16 +136,54 @@ export const useGameLogic = (
});
// --- Audio Auto-Play ---
useEffect(() => {
const shouldAutoPlay = settings.gameMode === 'ear_training' || (settings.gameMode === 'sight_reading' && settings.autoPlaySightReading);
// Boolean summary of "is the mic currently detecting a sounding note".
// The effect below depends on this BOOLEAN rather than on `pitchData`
// itself (which is a new object every frame — it would reset the 100ms
// play timer 60 times per second and the note would never play).
const isMicQuiet = pitchData === null;
if (shouldAutoPlay && !revealed) {
const timer = setTimeout(() => {
playNote(targetMidi, 1.0, settings.autoPlayVolume ?? 0.5);
}, 100);
return () => clearTimeout(timer);
// Remembers which target the reference note was already played for,
// so re-running the effect (e.g. the quiet-state flipping) can't play it twice.
const autoPlayedForRef = useRef<number | null>(null);
useEffect(() => {
const shouldAutoPlay =
settings.gameMode === 'ear_training' ||
(settings.gameMode === 'sight_reading' && settings.autoPlaySightReading);
if (!shouldAutoPlay || revealed) return;
if (autoPlayedForRef.current === targetMidi) return; // already played
const duration = settings.referenceNoteDuration ?? 1.5;
const volume = settings.autoPlayVolume ?? 0.5;
// "Wait for silence": hold the next reference note while the user's
// instrument is still ringing, so it can't mask the new note.
if (settings.waitForQuiet && !isMicQuiet) {
const cap = setTimeout(() => {
if (autoPlayedForRef.current === targetMidi) return;
autoPlayedForRef.current = targetMidi;
playNote(targetMidi, duration, volume);
}, REFERENCE_QUIET_MAX_WAIT_MS);
return () => clearTimeout(cap);
}
}, [targetMidi, settings.gameMode, settings.autoPlaySightReading, revealed, playNote, settings.autoPlayVolume]);
const timer = setTimeout(() => {
autoPlayedForRef.current = targetMidi;
playNote(targetMidi, duration, volume);
}, 100);
return () => clearTimeout(timer);
}, [
targetMidi,
revealed,
isMicQuiet,
settings.gameMode,
settings.autoPlaySightReading,
settings.referenceNoteDuration,
settings.autoPlayVolume,
settings.waitForQuiet,
playNote,
]);
// --- Init / Reset ---
useEffect(() => {
@@ -156,7 +201,6 @@ export const useGameLogic = (
const noteDetails = getNoteDetails(targetMidi);
setFeedbackMessage(`Good! ${noteDetails.name}`);
setRevealed(true);
setMatchStartTime(null);
const isRhythmActive = settings.rhythm.active && settings.rhythm.autoAdvance;
const isTimerMode = settings.rhythm.mode === 'seconds';
@@ -178,7 +222,6 @@ export const useGameLogic = (
feedbackTimeoutRef.current = setTimeout(() => {
setFeedbackMessage("");
}, 1500);
setMatchStartTime(null);
}
}
}, [targetMidi, settings.rhythm, settings.disableAnimation, restartMetronome, generateNewNote]);
@@ -201,25 +244,22 @@ export const useGameLogic = (
// --- Match Checking Loop ---
// Tolerant of flicker: MatchTracker keeps accumulating hold time across
// brief dropouts instead of resetting on every single bad frame.
useEffect(() => {
if (!pitchData) {
setMatchStartTime(null);
// Ignore the microphone while the app itself is making sound — the
// speaker output would otherwise be picked up and scored as correct
// (ear-training auto-play, Play Note button, virtual instruments).
if (audioEngine.isAudible()) {
tracker.reset();
return;
}
if (pitchData.midi === targetMidi) {
if (matchStartTime === null) {
setMatchStartTime(Date.now());
} else {
const duration = Date.now() - matchStartTime;
if (duration > NOTE_MATCH_THRESHOLD_MS) {
handleMatchSuccess();
}
}
} else {
setMatchStartTime(null);
const matching = pitchData !== null && pitchData.midi === targetMidi;
if (tracker.update(matching, Date.now())) {
handleMatchSuccess();
}
}, [pitchData, targetMidi, matchStartTime, handleMatchSuccess]);
}, [pitchData, targetMidi, handleMatchSuccess, tracker]);
return {
targetMidi,

13
src/hooks/useMetronome.ts
View File

@@ -1,4 +1,5 @@
import { useRef, useEffect, useCallback } from 'react';
import { createAudioContext } from '../audio/AudioEngine';
interface MetronomeOptions {
bpm: number;
@@ -53,6 +54,10 @@ export function useMetronome({ bpm, volume, playing, onTick }: MetronomeOptions)
onTickRef.current(currentBeat.current);
}, [volume]);
// Indirection so the scheduler can reschedule itself without referencing
// its own variable during initialization
const schedulerRef = useRef<() => void>(() => { });
const scheduler = useCallback(() => {
if (!audioContext.current) return;
@@ -62,9 +67,13 @@ export function useMetronome({ bpm, volume, playing, onTick }: MetronomeOptions)
playClick(nextNoteTime.current);
nextNote();
}
timerID.current = window.setTimeout(scheduler, lookahead);
timerID.current = window.setTimeout(() => schedulerRef.current(), lookahead);
}, [nextNote, playClick]);
useEffect(() => {
schedulerRef.current = scheduler;
}, [scheduler]);
const restart = useCallback(() => {
if (audioContext.current) {
// Reset to play one beat interval from NOW
@@ -78,7 +87,7 @@ export function useMetronome({ bpm, volume, playing, onTick }: MetronomeOptions)
useEffect(() => {
if (playing) {
if (!audioContext.current) {
audioContext.current = new (window.AudioContext || (window as any).webkitAudioContext)();
audioContext.current = createAudioContext();
}
// Resume if suspended (browser autoplay policy)

232
src/hooks/usePitchDetector.ts
View File

@@ -1,6 +1,12 @@
import { useEffect, useRef, useState, useCallback } from 'react';
import { useEffect, useRef, useState } from 'react';
import { PitchAnalyzer, type MicrophoneDebugInfo } from '../audio/PitchAnalyzer';
import { frequencyToMidi, getNoteDetails, getCentDifference } from '../music/NoteUtils';
import {
MIC_DEFAULT_SENSITIVITY,
PITCH_READING_HOLD_MS,
DISPLAY_EMA_FACTOR,
NOTE_DISPLAY_STABLE_FRAMES,
} from '../AppConfig';
interface PitchData {
frequency: number;
@@ -10,98 +16,186 @@ interface PitchData {
clarity: number; // Placeholder for now, maybe uses probability if YIN exposes it
}
import { MIC_DEFAULT_SENSITIVITY } from '../AppConfig';
export type { MicrophoneDebugInfo };
export function usePitchDetector(active: boolean, sensitivity: number = MIC_DEFAULT_SENSITIVITY) {
export function usePitchDetector(
active: boolean,
sensitivity: number = MIC_DEFAULT_SENSITIVITY,
minFrequencyHz?: number
) {
const analyzerRef = useRef<PitchAnalyzer | null>(null);
const [pitchData, setPitchData] = useState<PitchData | null>(null);
const [isListening, setIsListening] = useState(false);
const [error, setError] = useState<string | null>(null);
const [audioLevel, setAudioLevel] = useState<number>(0);
const [debugInfo, setDebugInfo] = useState<MicrophoneDebugInfo | null>(null);
const animationRef = useRef<number | null>(null);
// Update sensitivity when it changes
// Latest values for the capture effect (refs keep its dep list minimal so
// changing them does not restart the microphone)
const sensitivityRef = useRef(sensitivity);
const minFrequencyRef = useRef(minFrequencyHz);
// Apply sensitivity live without restarting capture
useEffect(() => {
if (analyzerRef.current) {
analyzerRef.current.setSensitivity(sensitivity);
}
sensitivityRef.current = sensitivity;
analyzerRef.current?.setSensitivity(sensitivity);
}, [sensitivity]);
const updatePitch = useCallback(() => {
if (!analyzerRef.current) return;
const freq = analyzerRef.current.getPitch();
if (freq) {
const midi = frequencyToMidi(freq);
const { scientific } = getNoteDetails(midi);
const cents = getCentDifference(freq, midi);
setPitchData({
frequency: freq,
midi,
note: scientific,
cents,
clarity: 1
});
// Adapt the analysis window to the instrument's lowest note, live
useEffect(() => {
minFrequencyRef.current = minFrequencyHz;
if (minFrequencyHz) {
analyzerRef.current?.setLowestFrequency(minFrequencyHz);
}
// Always update level and debug info
setAudioLevel(analyzerRef.current.getCurrentLevel());
setDebugInfo(analyzerRef.current.getDebugInfo());
animationRef.current = requestAnimationFrame(updatePitch);
}, []);
}, [minFrequencyHz]);
useEffect(() => {
if (active) {
if (!analyzerRef.current) {
analyzerRef.current = new PitchAnalyzer();
analyzerRef.current.setSensitivity(sensitivity);
if (!active) {
setIsListening(false);
setPitchData(null);
setAudioLevel(0);
setDebugInfo(null);
return;
}
let cancelled = false;
let rafId = 0;
let frameCount = 0;
let lastPitchAt = 0;
// Display smoothing: pitch/cents/level arrive on every detection frame
// (~60 fps) and jitter slightly. A short exponential moving average keeps
// the tuner needle and level meter readable. Done here (in the producer)
// so the display components can render directly without CSS transitions —
// transitions on values that change every frame get interrupted 60x/sec,
// which some engines (Safari/WebKit) punish with a visually stuck needle.
let smoothedCents: number | null = null;
let smoothedLevel = 0;
// Note-name stability: the displayed note only switches after the raw
// detection reports a different note for several consecutive frames.
// Prevents D#2/E2-style flicker at semitone boundaries. The raw midi
// (used for matching) is unaffected.
let stableMidi: number | null = null;
let candidateMidi: number | null = null;
let candidateCount = 0;
const analyzer = new PitchAnalyzer();
analyzer.setSensitivity(sensitivityRef.current);
if (minFrequencyRef.current) {
analyzer.setLowestFrequency(minFrequencyRef.current);
}
analyzerRef.current = analyzer;
function tick() {
if (cancelled) return;
const freq = analyzer.getPitch();
const now = performance.now();
if (freq) {
lastPitchAt = now;
const midi = frequencyToMidi(freq);
// Note-name hysteresis: only switch the displayed note after
// the new one persists for a few consecutive frames.
if (midi === stableMidi) {
candidateMidi = null;
candidateCount = 0;
} else if (midi === candidateMidi) {
if (++candidateCount >= NOTE_DISPLAY_STABLE_FRAMES) {
stableMidi = midi;
candidateMidi = null;
candidateCount = 0;
// Re-anchor the cents average: a new note means a new
// reference (the needle must not sweep across the scale).
smoothedCents = null;
}
} else {
candidateMidi = midi;
candidateCount = 1;
}
// Cents are displayed relative to the STABLE note (like real
// tuners): the needle sweeps to ±50 and only then the note flips.
const displayMidi = stableMidi ?? midi;
const { scientific } = getNoteDetails(displayMidi);
const cents = getCentDifference(freq, displayMidi);
smoothedCents = smoothedCents === null
? cents
: smoothedCents + (cents - smoothedCents) * DISPLAY_EMA_FACTOR;
setPitchData({
frequency: freq,
midi, // raw — matching uses this and must stay unsmoothed
note: scientific,
cents: smoothedCents, // smoothed — display only
clarity: 1
});
} else if (lastPitchAt !== 0 && now - lastPitchAt > PITCH_READING_HOLD_MS) {
// Clear stale readings so the UI doesn't keep displaying
// (and matching against) a note that stopped sounding long ago.
lastPitchAt = 0;
smoothedCents = null;
stableMidi = null;
candidateMidi = null;
candidateCount = 0;
setPitchData(null);
}
analyzerRef.current.start()
.then(() => {
setIsListening(true);
setError(null);
updatePitch();
})
.catch(err => {
console.error(err);
// Provide more specific error messages
if (err.name === 'NotAllowedError') {
setError("Microphone access denied. Please allow microphone access.");
} else if (err.name === 'NotFoundError') {
setError("No microphone found. Please connect a microphone.");
} else if (err.name === 'NotReadableError') {
setError("Microphone is in use by another app.");
} else {
setError("Could not access microphone.");
}
setIsListening(false);
});
} else {
if (analyzerRef.current) {
analyzerRef.current.stop();
// Always update level (smoothed)
smoothedLevel += (analyzer.getCurrentLevel() - smoothedLevel) * DISPLAY_EMA_FACTOR;
setAudioLevel(smoothedLevel);
// Debug info is only shown in the settings modal; ~4 Hz is plenty
// and avoids a third state update (full re-render) every frame.
if (++frameCount % 15 === 0) {
setDebugInfo(analyzer.getDebugInfo());
}
rafId = requestAnimationFrame(tick);
}
analyzer.start()
.then(() => {
if (cancelled) {
// StrictMode double-mount or quick toggle-off while
// getUserMedia was pending: don't leave the mic running.
analyzer.stop();
return;
}
setIsListening(true);
setError(null);
tick();
})
.catch((err: Error) => {
if (cancelled) return;
console.error(err);
// Provide more specific error messages
if (err.name === 'NotAllowedError') {
setError("Microphone access denied. Please allow microphone access.");
} else if (err.name === 'NotFoundError') {
setError("No microphone found. Please connect a microphone.");
} else if (err.name === 'NotReadableError') {
setError("Microphone is in use by another app.");
} else {
setError("Could not access microphone.");
}
setIsListening(false);
});
return () => {
cancelled = true;
cancelAnimationFrame(rafId);
analyzer.stop();
if (analyzerRef.current === analyzer) {
analyzerRef.current = null;
}
setIsListening(false);
setPitchData(null);
setAudioLevel(0);
setDebugInfo(null);
if (animationRef.current) cancelAnimationFrame(animationRef.current);
}
return () => {
if (animationRef.current) cancelAnimationFrame(animationRef.current);
if (analyzerRef.current) {
analyzerRef.current.stop();
}
};
}, [active, updatePitch]);
}, [active]);
return { pitchData, isListening, error, audioLevel, debugInfo };
}