Attempt to fix microphone on android

This commit is contained in:
2025-12-30 19:41:21 +01:00
parent b27c24cfd0
commit be7d994ca5
4 changed files with 301 additions and 15 deletions

View File

@@ -64,7 +64,7 @@ function App() {
disableAnimation: false disableAnimation: false
}); });
const { pitchData, error } = usePitchDetector(listening, settings.micSensitivity); const { pitchData, error, audioLevel, debugInfo, isListening } = usePitchDetector(listening, settings.micSensitivity);
const [matchStartTime, setMatchStartTime] = useState<number | null>(null); const [matchStartTime, setMatchStartTime] = useState<number | null>(null);
const [feedbackMessage, setFeedbackMessage] = useState<string>(""); const [feedbackMessage, setFeedbackMessage] = useState<string>("");
@@ -705,6 +705,9 @@ function App() {
onClose={() => setIsSettingsOpen(false)} onClose={() => setIsSettingsOpen(false)}
settings={settings} settings={settings}
onUpdateSettings={setSettings} onUpdateSettings={setSettings}
audioLevel={audioLevel}
debugInfo={debugInfo}
isListening={isListening}
/> />
<OpenSourceModal <OpenSourceModal

View File

@@ -1,5 +1,16 @@
import { YIN } from "pitchfinder"; import { YIN } from "pitchfinder";
export interface MicrophoneDebugInfo {
audioContextState: AudioContextState | 'inactive';
sampleRate: number | null;
inputDeviceLabel: string;
inputDeviceId: string;
permissionState: PermissionState | 'unknown';
currentRmsLevel: number;
currentRmsDb: number;
isCapturing: boolean;
}
export class PitchAnalyzer { export class PitchAnalyzer {
private detector: (buffer: Float32Array) => number | null; private detector: (buffer: Float32Array) => number | null;
private audioContext: AudioContext | null = null; private audioContext: AudioContext | null = null;
@@ -8,6 +19,12 @@ export class PitchAnalyzer {
private source: MediaStreamAudioSourceNode | null = null; private source: MediaStreamAudioSourceNode | null = null;
private buffer: Float32Array; private buffer: Float32Array;
// Debug info
private inputDeviceLabel: string = 'Not detected';
private inputDeviceId: string = '';
private permissionState: PermissionState | 'unknown' = 'unknown';
private currentRmsLevel: number = 0;
constructor() { constructor() {
this.detector = YIN({ sampleRate: 44100 }); // Default, will update on start this.detector = YIN({ sampleRate: 44100 }); // Default, will update on start
this.buffer = new Float32Array(2048); // Standard size this.buffer = new Float32Array(2048); // Standard size
@@ -36,17 +53,66 @@ export class PitchAnalyzer {
async start(): Promise<void> { async start(): Promise<void> {
if (this.audioContext) return; if (this.audioContext) return;
// Check permission state if available
try {
if (navigator.permissions && navigator.permissions.query) {
const result = await navigator.permissions.query({ name: 'microphone' as PermissionName });
this.permissionState = result.state;
}
} catch {
// permissions API not available on all browsers
this.permissionState = 'unknown';
}
this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)(); this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)();
this.detector = YIN({ sampleRate: this.audioContext.sampleRate }); this.detector = YIN({ sampleRate: this.audioContext.sampleRate });
try { try {
this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); // Use more explicit audio constraints for better Android compatibility
const constraints: MediaStreamConstraints = {
audio: {
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
}
};
this.mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
// Get device info
const audioTracks = this.mediaStream.getAudioTracks();
if (audioTracks.length > 0) {
const track = audioTracks[0];
const settings = track.getSettings();
this.inputDeviceId = settings.deviceId || '';
this.inputDeviceLabel = track.label || 'Unknown Device';
// If label is empty, try to get it from enumerateDevices
if (!track.label && settings.deviceId) {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const matchedDevice = devices.find(d => d.deviceId === settings.deviceId);
if (matchedDevice && matchedDevice.label) {
this.inputDeviceLabel = matchedDevice.label;
}
} catch {
// enumerateDevices not available
}
}
}
this.source = this.audioContext.createMediaStreamSource(this.mediaStream); this.source = this.audioContext.createMediaStreamSource(this.mediaStream);
this.analyser = this.audioContext.createAnalyser(); this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 4096; // Higher FFT size for better resolution at low frequencies this.analyser.fftSize = 4096; // Higher FFT size for better resolution at low frequencies
this.buffer = new Float32Array(this.analyser.fftSize); this.buffer = new Float32Array(this.analyser.fftSize);
this.source.connect(this.analyser); this.source.connect(this.analyser);
// CRITICAL for Android: Resume AudioContext after user interaction
// Android Chrome suspends AudioContext by default
if (this.audioContext.state === 'suspended') {
await this.audioContext.resume();
}
} catch (e) { } catch (e) {
console.error("Error accessing microphone:", e); console.error("Error accessing microphone:", e);
throw e; throw e;
@@ -62,6 +128,36 @@ export class PitchAnalyzer {
this.audioContext.close(); this.audioContext.close();
this.audioContext = null; this.audioContext = null;
} }
this.currentRmsLevel = 0;
this.inputDeviceLabel = 'Not detected';
this.inputDeviceId = '';
}
/**
* Get current debug/diagnostic information
*/
getDebugInfo(): MicrophoneDebugInfo {
const rmsDb = this.currentRmsLevel > 0
? 20 * Math.log10(this.currentRmsLevel)
: -Infinity;
return {
audioContextState: this.audioContext?.state || 'inactive',
sampleRate: this.audioContext?.sampleRate || null,
inputDeviceLabel: this.inputDeviceLabel,
inputDeviceId: this.inputDeviceId,
permissionState: this.permissionState,
currentRmsLevel: this.currentRmsLevel,
currentRmsDb: isFinite(rmsDb) ? rmsDb : -100,
isCapturing: this.analyser !== null && this.audioContext?.state === 'running',
};
}
/**
* Get current RMS level (0-1 range, useful for level meters)
*/
getCurrentLevel(): number {
return this.currentRmsLevel;
} }
/** /**
@@ -78,6 +174,9 @@ export class PitchAnalyzer {
} }
rms = Math.sqrt(rms / this.buffer.length); rms = Math.sqrt(rms / this.buffer.length);
// Store RMS for level meter
this.currentRmsLevel = rms;
if (rms < this.sensitivityThreshold) return null; if (rms < this.sensitivityThreshold) return null;
const pitch = this.detector(this.buffer); const pitch = this.detector(this.buffer);

View File

@@ -1,15 +1,67 @@
import React from 'react'; import React, { useState } from 'react';
import { X, Volume2, VolumeX } from 'lucide-react'; import { X, Volume2, VolumeX, ChevronDown, ChevronUp } from 'lucide-react';
import type { AppSettings } from './Controls'; import type { AppSettings } from './Controls';
import type { MicrophoneDebugInfo } from '../hooks/usePitchDetector';
interface SettingsModalProps { interface SettingsModalProps {
isOpen: boolean; isOpen: boolean;
onClose: () => void; onClose: () => void;
settings: AppSettings; settings: AppSettings;
onUpdateSettings: (s: AppSettings) => void; onUpdateSettings: (s: AppSettings) => void;
audioLevel?: number;
debugInfo?: MicrophoneDebugInfo | null;
isListening?: boolean;
} }
export const SettingsModal: React.FC<SettingsModalProps> = ({ isOpen, onClose, settings, onUpdateSettings }) => { // Level meter component
const LevelMeter: React.FC<{ level: number; isActive: boolean }> = ({ level, isActive }) => {
// Convert RMS to dB for display, then normalize to 0-100%
// RMS of 0.001 = -60dB, RMS of 1.0 = 0dB
const db = level > 0 ? 20 * Math.log10(level) : -100;
// Map -60dB to 0dB => 0% to 100%
const percentage = Math.max(0, Math.min(100, ((db + 60) / 60) * 100));
return (
<div style={{
display: 'flex',
alignItems: 'center',
gap: '8px',
opacity: isActive ? 1 : 0.5
}}>
<div style={{
flex: 1,
height: '12px',
background: 'rgba(0,0,0,0.3)',
borderRadius: '6px',
overflow: 'hidden',
border: '1px solid rgba(255,255,255,0.1)'
}}>
<div style={{
height: '100%',
width: `${percentage}%`,
background: percentage > 80 ? '#ef4444' : percentage > 50 ? '#f59e0b' : '#22c55e',
borderRadius: '6px',
transition: 'width 0.05s ease-out'
}} />
</div>
<span style={{ fontSize: '11px', opacity: 0.7, minWidth: '45px', textAlign: 'right' }}>
{isActive ? `${Math.round(db)} dB` : '— dB'}
</span>
</div>
);
};
export const SettingsModal: React.FC<SettingsModalProps> = ({
isOpen,
onClose,
settings,
onUpdateSettings,
audioLevel = 0,
debugInfo,
isListening = false
}) => {
const [showDebugInfo, setShowDebugInfo] = useState(false);
if (!isOpen) return null; if (!isOpen) return null;
return ( return (
@@ -141,6 +193,125 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({ isOpen, onClose, s
Adjust if notes are not detected (increase/right) or if background noise triggers notes (decrease/left). Adjust if notes are not detected (increase/right) or if background noise triggers notes (decrease/left).
</p> </p>
</div> </div>
{/* Microphone Level Meter */}
<div className="control-group" style={{ marginTop: '16px' }}>
<label className="control-label" style={{ marginBottom: '8px', fontSize: '14px' }}>
<span>Microphone Level</span>
<span style={{ fontSize: '11px', opacity: 0.6, marginLeft: '8px' }}>
{isListening ? '(active)' : '(inactive)'}
</span>
</label>
<LevelMeter level={audioLevel} isActive={isListening} />
{!isListening && (
<p style={{ fontSize: '11px', opacity: 0.5, margin: '4px 0 0 0' }}>
Enable microphone to see audio levels
</p>
)}
</div>
{/* Debug Info Toggle */}
<button
onClick={() => setShowDebugInfo(!showDebugInfo)}
style={{
marginTop: '16px',
background: 'transparent',
border: '1px solid rgba(255,255,255,0.2)',
borderRadius: '4px',
padding: '8px 12px',
color: 'inherit',
cursor: 'pointer',
display: 'flex',
alignItems: 'center',
gap: '8px',
fontSize: '12px',
opacity: 0.7,
width: '100%',
justifyContent: 'center'
}}
>
{showDebugInfo ? <ChevronUp size={14} /> : <ChevronDown size={14} />}
Microphone Debug Info
</button>
{/* Debug Info Panel */}
{showDebugInfo && (
<div style={{
marginTop: '12px',
padding: '12px',
background: 'rgba(0,0,0,0.3)',
borderRadius: '8px',
fontSize: '11px',
fontFamily: 'monospace'
}}>
<div style={{ display: 'grid', gridTemplateColumns: 'auto 1fr', gap: '4px 12px' }}>
<span style={{ opacity: 0.6 }}>Status:</span>
<span style={{
color: debugInfo?.isCapturing ? '#22c55e' :
debugInfo?.audioContextState === 'suspended' ? '#f59e0b' : '#ef4444'
}}>
{debugInfo?.isCapturing ? '✓ Capturing' :
debugInfo?.audioContextState === 'suspended' ? '⚠ Suspended' :
isListening ? '✗ Not Capturing' : '○ Inactive'}
</span>
<span style={{ opacity: 0.6 }}>AudioContext:</span>
<span>{debugInfo?.audioContextState || 'N/A'}</span>
<span style={{ opacity: 0.6 }}>Sample Rate:</span>
<span>{debugInfo?.sampleRate ? `${debugInfo.sampleRate} Hz` : 'N/A'}</span>
<span style={{ opacity: 0.6 }}>Input Device:</span>
<span style={{ wordBreak: 'break-word' }}>
{debugInfo?.inputDeviceLabel || 'N/A'}
</span>
<span style={{ opacity: 0.6 }}>Permission:</span>
<span style={{
color: debugInfo?.permissionState === 'granted' ? '#22c55e' :
debugInfo?.permissionState === 'denied' ? '#ef4444' : '#f59e0b'
}}>
{debugInfo?.permissionState || 'unknown'}
</span>
<span style={{ opacity: 0.6 }}>RMS Level:</span>
<span>
{debugInfo?.currentRmsLevel !== undefined
? `${debugInfo.currentRmsLevel.toFixed(4)} (${Math.round(debugInfo.currentRmsDb)} dB)`
: 'N/A'}
</span>
<span style={{ opacity: 0.6 }}>Browser:</span>
<span style={{ wordBreak: 'break-word', fontSize: '10px' }}>
{typeof navigator !== 'undefined' ? navigator.userAgent.slice(0, 60) + '...' : 'N/A'}
</span>
</div>
{debugInfo?.audioContextState === 'suspended' && (
<div style={{
marginTop: '12px',
padding: '8px',
background: 'rgba(245, 158, 11, 0.2)',
borderRadius: '4px',
color: '#f59e0b'
}}>
AudioContext is suspended. Try tapping the mic button again or interacting with the page.
</div>
)}
{isListening && !debugInfo?.isCapturing && debugInfo?.audioContextState === 'running' && (
<div style={{
marginTop: '12px',
padding: '8px',
background: 'rgba(239, 68, 68, 0.2)',
borderRadius: '4px',
color: '#ef4444'
}}>
AudioContext is running but no audio is being captured. Check if another app is using the microphone.
</div>
)}
</div>
)}
</div> </div>
</div> </div>

View File

@@ -1,5 +1,5 @@
import { useEffect, useRef, useState, useCallback } from 'react'; import { useEffect, useRef, useState, useCallback } from 'react';
import { PitchAnalyzer } from '../audio/PitchAnalyzer'; import { PitchAnalyzer, type MicrophoneDebugInfo } from '../audio/PitchAnalyzer';
import { frequencyToMidi, getNoteDetails, getCentDifference } from '../music/NoteUtils'; import { frequencyToMidi, getNoteDetails, getCentDifference } from '../music/NoteUtils';
interface PitchData { interface PitchData {
@@ -10,11 +10,15 @@ interface PitchData {
clarity: number; // Placeholder for now, maybe uses probability if YIN exposes it clarity: number; // Placeholder for now, maybe uses probability if YIN exposes it
} }
export type { MicrophoneDebugInfo };
export function usePitchDetector(active: boolean, sensitivity: number = 0.5) { export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
const analyzerRef = useRef<PitchAnalyzer | null>(null); const analyzerRef = useRef<PitchAnalyzer | null>(null);
const [pitchData, setPitchData] = useState<PitchData | null>(null); const [pitchData, setPitchData] = useState<PitchData | null>(null);
const [isListening, setIsListening] = useState(false); const [isListening, setIsListening] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [audioLevel, setAudioLevel] = useState<number>(0);
const [debugInfo, setDebugInfo] = useState<MicrophoneDebugInfo | null>(null);
const animationRef = useRef<number | null>(null); const animationRef = useRef<number | null>(null);
// Update sensitivity when it changes // Update sensitivity when it changes
@@ -27,18 +31,11 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
const updatePitch = useCallback(() => { const updatePitch = useCallback(() => {
if (!analyzerRef.current) return; if (!analyzerRef.current) return;
// Ensure sensitivity is set on start
// (Though the effect above handles updates, safely re-asserting here doesn't hurt,
// but let's trust the effect and the init sequence).
const freq = analyzerRef.current.getPitch(); const freq = analyzerRef.current.getPitch();
if (freq) { if (freq) {
const midi = frequencyToMidi(freq); const midi = frequencyToMidi(freq);
const { scientific } = getNoteDetails(midi); const { scientific } = getNoteDetails(midi);
// Calculate cents off purely for display if needed,
// but for training we usually care if midi matches.
const cents = getCentDifference(freq, midi); const cents = getCentDifference(freq, midi);
setPitchData({ setPitchData({
@@ -50,6 +47,10 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
}); });
} }
// Always update level and debug info
setAudioLevel(analyzerRef.current.getCurrentLevel());
setDebugInfo(analyzerRef.current.getDebugInfo());
animationRef.current = requestAnimationFrame(updatePitch); animationRef.current = requestAnimationFrame(updatePitch);
}, []); }, []);
@@ -63,11 +64,21 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
analyzerRef.current.start() analyzerRef.current.start()
.then(() => { .then(() => {
setIsListening(true); setIsListening(true);
setError(null);
updatePitch(); updatePitch();
}) })
.catch(err => { .catch(err => {
console.error(err); console.error(err);
setError("Could not access microphone."); // Provide more specific error messages
if (err.name === 'NotAllowedError') {
setError("Microphone access denied. Please allow microphone access.");
} else if (err.name === 'NotFoundError') {
setError("No microphone found. Please connect a microphone.");
} else if (err.name === 'NotReadableError') {
setError("Microphone is in use by another app.");
} else {
setError("Could not access microphone.");
}
setIsListening(false); setIsListening(false);
}); });
} else { } else {
@@ -77,6 +88,8 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
} }
setIsListening(false); setIsListening(false);
setPitchData(null); setPitchData(null);
setAudioLevel(0);
setDebugInfo(null);
if (animationRef.current) cancelAnimationFrame(animationRef.current); if (animationRef.current) cancelAnimationFrame(animationRef.current);
} }
@@ -88,5 +101,5 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
}; };
}, [active, updatePitch]); }, [active, updatePitch]);
return { pitchData, isListening, error }; return { pitchData, isListening, error, audioLevel, debugInfo };
} }