mirror of
https://github.com/9x/sheetmusictrainer.git
synced 2026-09-02 09:34:33 +02:00
Attempt to fix microphone on android
This commit is contained in:
@@ -64,7 +64,7 @@ function App() {
|
||||
disableAnimation: false
|
||||
});
|
||||
|
||||
const { pitchData, error } = usePitchDetector(listening, settings.micSensitivity);
|
||||
const { pitchData, error, audioLevel, debugInfo, isListening } = usePitchDetector(listening, settings.micSensitivity);
|
||||
|
||||
const [matchStartTime, setMatchStartTime] = useState<number | null>(null);
|
||||
const [feedbackMessage, setFeedbackMessage] = useState<string>("");
|
||||
@@ -705,6 +705,9 @@ function App() {
|
||||
onClose={() => setIsSettingsOpen(false)}
|
||||
settings={settings}
|
||||
onUpdateSettings={setSettings}
|
||||
audioLevel={audioLevel}
|
||||
debugInfo={debugInfo}
|
||||
isListening={isListening}
|
||||
/>
|
||||
|
||||
<OpenSourceModal
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import { YIN } from "pitchfinder";
|
||||
|
||||
export interface MicrophoneDebugInfo {
|
||||
audioContextState: AudioContextState | 'inactive';
|
||||
sampleRate: number | null;
|
||||
inputDeviceLabel: string;
|
||||
inputDeviceId: string;
|
||||
permissionState: PermissionState | 'unknown';
|
||||
currentRmsLevel: number;
|
||||
currentRmsDb: number;
|
||||
isCapturing: boolean;
|
||||
}
|
||||
|
||||
export class PitchAnalyzer {
|
||||
private detector: (buffer: Float32Array) => number | null;
|
||||
private audioContext: AudioContext | null = null;
|
||||
@@ -8,6 +19,12 @@ export class PitchAnalyzer {
|
||||
private source: MediaStreamAudioSourceNode | null = null;
|
||||
private buffer: Float32Array;
|
||||
|
||||
// Debug info
|
||||
private inputDeviceLabel: string = 'Not detected';
|
||||
private inputDeviceId: string = '';
|
||||
private permissionState: PermissionState | 'unknown' = 'unknown';
|
||||
private currentRmsLevel: number = 0;
|
||||
|
||||
constructor() {
|
||||
this.detector = YIN({ sampleRate: 44100 }); // Default, will update on start
|
||||
this.buffer = new Float32Array(2048); // Standard size
|
||||
@@ -36,17 +53,66 @@ export class PitchAnalyzer {
|
||||
async start(): Promise<void> {
|
||||
if (this.audioContext) return;
|
||||
|
||||
// Check permission state if available
|
||||
try {
|
||||
if (navigator.permissions && navigator.permissions.query) {
|
||||
const result = await navigator.permissions.query({ name: 'microphone' as PermissionName });
|
||||
this.permissionState = result.state;
|
||||
}
|
||||
} catch {
|
||||
// permissions API not available on all browsers
|
||||
this.permissionState = 'unknown';
|
||||
}
|
||||
|
||||
this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)();
|
||||
this.detector = YIN({ sampleRate: this.audioContext.sampleRate });
|
||||
|
||||
try {
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
// Use more explicit audio constraints for better Android compatibility
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: {
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
|
||||
// Get device info
|
||||
const audioTracks = this.mediaStream.getAudioTracks();
|
||||
if (audioTracks.length > 0) {
|
||||
const track = audioTracks[0];
|
||||
const settings = track.getSettings();
|
||||
this.inputDeviceId = settings.deviceId || '';
|
||||
this.inputDeviceLabel = track.label || 'Unknown Device';
|
||||
|
||||
// If label is empty, try to get it from enumerateDevices
|
||||
if (!track.label && settings.deviceId) {
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
const matchedDevice = devices.find(d => d.deviceId === settings.deviceId);
|
||||
if (matchedDevice && matchedDevice.label) {
|
||||
this.inputDeviceLabel = matchedDevice.label;
|
||||
}
|
||||
} catch {
|
||||
// enumerateDevices not available
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.source = this.audioContext.createMediaStreamSource(this.mediaStream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 4096; // Higher FFT size for better resolution at low frequencies
|
||||
this.buffer = new Float32Array(this.analyser.fftSize);
|
||||
|
||||
this.source.connect(this.analyser);
|
||||
|
||||
// CRITICAL for Android: Resume AudioContext after user interaction
|
||||
// Android Chrome suspends AudioContext by default
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error accessing microphone:", e);
|
||||
throw e;
|
||||
@@ -62,6 +128,36 @@ export class PitchAnalyzer {
|
||||
this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
}
|
||||
this.currentRmsLevel = 0;
|
||||
this.inputDeviceLabel = 'Not detected';
|
||||
this.inputDeviceId = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current debug/diagnostic information
|
||||
*/
|
||||
getDebugInfo(): MicrophoneDebugInfo {
|
||||
const rmsDb = this.currentRmsLevel > 0
|
||||
? 20 * Math.log10(this.currentRmsLevel)
|
||||
: -Infinity;
|
||||
|
||||
return {
|
||||
audioContextState: this.audioContext?.state || 'inactive',
|
||||
sampleRate: this.audioContext?.sampleRate || null,
|
||||
inputDeviceLabel: this.inputDeviceLabel,
|
||||
inputDeviceId: this.inputDeviceId,
|
||||
permissionState: this.permissionState,
|
||||
currentRmsLevel: this.currentRmsLevel,
|
||||
currentRmsDb: isFinite(rmsDb) ? rmsDb : -100,
|
||||
isCapturing: this.analyser !== null && this.audioContext?.state === 'running',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current RMS level (0-1 range, useful for level meters)
|
||||
*/
|
||||
getCurrentLevel(): number {
|
||||
return this.currentRmsLevel;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,6 +174,9 @@ export class PitchAnalyzer {
|
||||
}
|
||||
rms = Math.sqrt(rms / this.buffer.length);
|
||||
|
||||
// Store RMS for level meter
|
||||
this.currentRmsLevel = rms;
|
||||
|
||||
if (rms < this.sensitivityThreshold) return null;
|
||||
|
||||
const pitch = this.detector(this.buffer);
|
||||
|
||||
@@ -1,15 +1,67 @@
|
||||
import React from 'react';
|
||||
import { X, Volume2, VolumeX } from 'lucide-react';
|
||||
import React, { useState } from 'react';
|
||||
import { X, Volume2, VolumeX, ChevronDown, ChevronUp } from 'lucide-react';
|
||||
import type { AppSettings } from './Controls';
|
||||
import type { MicrophoneDebugInfo } from '../hooks/usePitchDetector';
|
||||
|
||||
interface SettingsModalProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
settings: AppSettings;
|
||||
onUpdateSettings: (s: AppSettings) => void;
|
||||
audioLevel?: number;
|
||||
debugInfo?: MicrophoneDebugInfo | null;
|
||||
isListening?: boolean;
|
||||
}
|
||||
|
||||
export const SettingsModal: React.FC<SettingsModalProps> = ({ isOpen, onClose, settings, onUpdateSettings }) => {
|
||||
// Level meter component
|
||||
const LevelMeter: React.FC<{ level: number; isActive: boolean }> = ({ level, isActive }) => {
|
||||
// Convert RMS to dB for display, then normalize to 0-100%
|
||||
// RMS of 0.001 = -60dB, RMS of 1.0 = 0dB
|
||||
const db = level > 0 ? 20 * Math.log10(level) : -100;
|
||||
// Map -60dB to 0dB => 0% to 100%
|
||||
const percentage = Math.max(0, Math.min(100, ((db + 60) / 60) * 100));
|
||||
|
||||
return (
|
||||
<div style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '8px',
|
||||
opacity: isActive ? 1 : 0.5
|
||||
}}>
|
||||
<div style={{
|
||||
flex: 1,
|
||||
height: '12px',
|
||||
background: 'rgba(0,0,0,0.3)',
|
||||
borderRadius: '6px',
|
||||
overflow: 'hidden',
|
||||
border: '1px solid rgba(255,255,255,0.1)'
|
||||
}}>
|
||||
<div style={{
|
||||
height: '100%',
|
||||
width: `${percentage}%`,
|
||||
background: percentage > 80 ? '#ef4444' : percentage > 50 ? '#f59e0b' : '#22c55e',
|
||||
borderRadius: '6px',
|
||||
transition: 'width 0.05s ease-out'
|
||||
}} />
|
||||
</div>
|
||||
<span style={{ fontSize: '11px', opacity: 0.7, minWidth: '45px', textAlign: 'right' }}>
|
||||
{isActive ? `${Math.round(db)} dB` : '— dB'}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const SettingsModal: React.FC<SettingsModalProps> = ({
|
||||
isOpen,
|
||||
onClose,
|
||||
settings,
|
||||
onUpdateSettings,
|
||||
audioLevel = 0,
|
||||
debugInfo,
|
||||
isListening = false
|
||||
}) => {
|
||||
const [showDebugInfo, setShowDebugInfo] = useState(false);
|
||||
|
||||
if (!isOpen) return null;
|
||||
|
||||
return (
|
||||
@@ -141,6 +193,125 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({ isOpen, onClose, s
|
||||
Adjust if notes are not detected (increase/right) or if background noise triggers notes (decrease/left).
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Microphone Level Meter */}
|
||||
<div className="control-group" style={{ marginTop: '16px' }}>
|
||||
<label className="control-label" style={{ marginBottom: '8px', fontSize: '14px' }}>
|
||||
<span>Microphone Level</span>
|
||||
<span style={{ fontSize: '11px', opacity: 0.6, marginLeft: '8px' }}>
|
||||
{isListening ? '(active)' : '(inactive)'}
|
||||
</span>
|
||||
</label>
|
||||
<LevelMeter level={audioLevel} isActive={isListening} />
|
||||
{!isListening && (
|
||||
<p style={{ fontSize: '11px', opacity: 0.5, margin: '4px 0 0 0' }}>
|
||||
Enable microphone to see audio levels
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Debug Info Toggle */}
|
||||
<button
|
||||
onClick={() => setShowDebugInfo(!showDebugInfo)}
|
||||
style={{
|
||||
marginTop: '16px',
|
||||
background: 'transparent',
|
||||
border: '1px solid rgba(255,255,255,0.2)',
|
||||
borderRadius: '4px',
|
||||
padding: '8px 12px',
|
||||
color: 'inherit',
|
||||
cursor: 'pointer',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '8px',
|
||||
fontSize: '12px',
|
||||
opacity: 0.7,
|
||||
width: '100%',
|
||||
justifyContent: 'center'
|
||||
}}
|
||||
>
|
||||
{showDebugInfo ? <ChevronUp size={14} /> : <ChevronDown size={14} />}
|
||||
Microphone Debug Info
|
||||
</button>
|
||||
|
||||
{/* Debug Info Panel */}
|
||||
{showDebugInfo && (
|
||||
<div style={{
|
||||
marginTop: '12px',
|
||||
padding: '12px',
|
||||
background: 'rgba(0,0,0,0.3)',
|
||||
borderRadius: '8px',
|
||||
fontSize: '11px',
|
||||
fontFamily: 'monospace'
|
||||
}}>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'auto 1fr', gap: '4px 12px' }}>
|
||||
<span style={{ opacity: 0.6 }}>Status:</span>
|
||||
<span style={{
|
||||
color: debugInfo?.isCapturing ? '#22c55e' :
|
||||
debugInfo?.audioContextState === 'suspended' ? '#f59e0b' : '#ef4444'
|
||||
}}>
|
||||
{debugInfo?.isCapturing ? '✓ Capturing' :
|
||||
debugInfo?.audioContextState === 'suspended' ? '⚠ Suspended' :
|
||||
isListening ? '✗ Not Capturing' : '○ Inactive'}
|
||||
</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>AudioContext:</span>
|
||||
<span>{debugInfo?.audioContextState || 'N/A'}</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>Sample Rate:</span>
|
||||
<span>{debugInfo?.sampleRate ? `${debugInfo.sampleRate} Hz` : 'N/A'}</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>Input Device:</span>
|
||||
<span style={{ wordBreak: 'break-word' }}>
|
||||
{debugInfo?.inputDeviceLabel || 'N/A'}
|
||||
</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>Permission:</span>
|
||||
<span style={{
|
||||
color: debugInfo?.permissionState === 'granted' ? '#22c55e' :
|
||||
debugInfo?.permissionState === 'denied' ? '#ef4444' : '#f59e0b'
|
||||
}}>
|
||||
{debugInfo?.permissionState || 'unknown'}
|
||||
</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>RMS Level:</span>
|
||||
<span>
|
||||
{debugInfo?.currentRmsLevel !== undefined
|
||||
? `${debugInfo.currentRmsLevel.toFixed(4)} (${Math.round(debugInfo.currentRmsDb)} dB)`
|
||||
: 'N/A'}
|
||||
</span>
|
||||
|
||||
<span style={{ opacity: 0.6 }}>Browser:</span>
|
||||
<span style={{ wordBreak: 'break-word', fontSize: '10px' }}>
|
||||
{typeof navigator !== 'undefined' ? navigator.userAgent.slice(0, 60) + '...' : 'N/A'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{debugInfo?.audioContextState === 'suspended' && (
|
||||
<div style={{
|
||||
marginTop: '12px',
|
||||
padding: '8px',
|
||||
background: 'rgba(245, 158, 11, 0.2)',
|
||||
borderRadius: '4px',
|
||||
color: '#f59e0b'
|
||||
}}>
|
||||
⚠ AudioContext is suspended. Try tapping the mic button again or interacting with the page.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isListening && !debugInfo?.isCapturing && debugInfo?.audioContextState === 'running' && (
|
||||
<div style={{
|
||||
marginTop: '12px',
|
||||
padding: '8px',
|
||||
background: 'rgba(239, 68, 68, 0.2)',
|
||||
borderRadius: '4px',
|
||||
color: '#ef4444'
|
||||
}}>
|
||||
✗ AudioContext is running but no audio is being captured. Check if another app is using the microphone.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
import { PitchAnalyzer } from '../audio/PitchAnalyzer';
|
||||
import { PitchAnalyzer, type MicrophoneDebugInfo } from '../audio/PitchAnalyzer';
|
||||
import { frequencyToMidi, getNoteDetails, getCentDifference } from '../music/NoteUtils';
|
||||
|
||||
interface PitchData {
|
||||
@@ -10,11 +10,15 @@ interface PitchData {
|
||||
clarity: number; // Placeholder for now, maybe uses probability if YIN exposes it
|
||||
}
|
||||
|
||||
export type { MicrophoneDebugInfo };
|
||||
|
||||
export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
const analyzerRef = useRef<PitchAnalyzer | null>(null);
|
||||
const [pitchData, setPitchData] = useState<PitchData | null>(null);
|
||||
const [isListening, setIsListening] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [audioLevel, setAudioLevel] = useState<number>(0);
|
||||
const [debugInfo, setDebugInfo] = useState<MicrophoneDebugInfo | null>(null);
|
||||
const animationRef = useRef<number | null>(null);
|
||||
|
||||
// Update sensitivity when it changes
|
||||
@@ -27,18 +31,11 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
const updatePitch = useCallback(() => {
|
||||
if (!analyzerRef.current) return;
|
||||
|
||||
// Ensure sensitivity is set on start
|
||||
// (Though the effect above handles updates, safely re-asserting here doesn't hurt,
|
||||
// but let's trust the effect and the init sequence).
|
||||
|
||||
const freq = analyzerRef.current.getPitch();
|
||||
if (freq) {
|
||||
const midi = frequencyToMidi(freq);
|
||||
const { scientific } = getNoteDetails(midi);
|
||||
|
||||
// Calculate cents off purely for display if needed,
|
||||
// but for training we usually care if midi matches.
|
||||
|
||||
const cents = getCentDifference(freq, midi);
|
||||
|
||||
setPitchData({
|
||||
@@ -50,6 +47,10 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
});
|
||||
}
|
||||
|
||||
// Always update level and debug info
|
||||
setAudioLevel(analyzerRef.current.getCurrentLevel());
|
||||
setDebugInfo(analyzerRef.current.getDebugInfo());
|
||||
|
||||
animationRef.current = requestAnimationFrame(updatePitch);
|
||||
}, []);
|
||||
|
||||
@@ -63,11 +64,21 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
analyzerRef.current.start()
|
||||
.then(() => {
|
||||
setIsListening(true);
|
||||
setError(null);
|
||||
updatePitch();
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
// Provide more specific error messages
|
||||
if (err.name === 'NotAllowedError') {
|
||||
setError("Microphone access denied. Please allow microphone access.");
|
||||
} else if (err.name === 'NotFoundError') {
|
||||
setError("No microphone found. Please connect a microphone.");
|
||||
} else if (err.name === 'NotReadableError') {
|
||||
setError("Microphone is in use by another app.");
|
||||
} else {
|
||||
setError("Could not access microphone.");
|
||||
}
|
||||
setIsListening(false);
|
||||
});
|
||||
} else {
|
||||
@@ -77,6 +88,8 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
}
|
||||
setIsListening(false);
|
||||
setPitchData(null);
|
||||
setAudioLevel(0);
|
||||
setDebugInfo(null);
|
||||
if (animationRef.current) cancelAnimationFrame(animationRef.current);
|
||||
}
|
||||
|
||||
@@ -88,5 +101,5 @@ export function usePitchDetector(active: boolean, sensitivity: number = 0.5) {
|
||||
};
|
||||
}, [active, updatePitch]);
|
||||
|
||||
return { pitchData, isListening, error };
|
||||
return { pitchData, isListening, error, audioLevel, debugInfo };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user