Before attempting to integrate osciloscope properely. STILL BROKEN NOW
This commit is contained in:
@@ -0,0 +1,246 @@
|
||||
import { useState, useRef, useCallback, useEffect } from 'react';
|
||||
|
||||
interface AudioAnalyzerState {
|
||||
isActive: boolean;
|
||||
error: string | null;
|
||||
source: 'microphone' | 'file' | null;
|
||||
fileName: string | null;
|
||||
isPlaying: boolean;
|
||||
}
|
||||
|
||||
export const useAudioAnalyzer = () => {
|
||||
const [state, setState] = useState<AudioAnalyzerState>({
|
||||
isActive: false,
|
||||
error: null,
|
||||
source: null,
|
||||
fileName: null,
|
||||
isPlaying: false,
|
||||
});
|
||||
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const analyzerLeftRef = useRef<AnalyserNode | null>(null);
|
||||
const analyzerRightRef = useRef<AnalyserNode | null>(null);
|
||||
const sourceRef = useRef<MediaStreamAudioSourceNode | MediaElementAudioSourceNode | null>(null);
|
||||
const splitterRef = useRef<ChannelSplitterNode | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const analysisGainNodeRef = useRef<GainNode | null>(null);
|
||||
const audioElementRef = useRef<HTMLAudioElement | null>(null);
|
||||
const gainValueRef = useRef<number>(3); // Default higher gain for analysis sensitivity only
|
||||
|
||||
const getTimeDomainData = useCallback(() => {
|
||||
if (!analyzerLeftRef.current) return null;
|
||||
|
||||
const bufferLength = analyzerLeftRef.current.fftSize;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
analyzerLeftRef.current.getByteTimeDomainData(dataArray);
|
||||
|
||||
return dataArray;
|
||||
}, []);
|
||||
|
||||
const getStereoData = useCallback(() => {
|
||||
if (!analyzerLeftRef.current || !analyzerRightRef.current) return null;
|
||||
|
||||
const bufferLength = analyzerLeftRef.current.fftSize;
|
||||
const leftData = new Uint8Array(bufferLength);
|
||||
const rightData = new Uint8Array(bufferLength);
|
||||
|
||||
analyzerLeftRef.current.getByteTimeDomainData(leftData);
|
||||
analyzerRightRef.current.getByteTimeDomainData(rightData);
|
||||
|
||||
return { left: leftData, right: rightData };
|
||||
}, []);
|
||||
|
||||
const setGain = useCallback((value: number) => {
|
||||
gainValueRef.current = value;
|
||||
if (analysisGainNodeRef.current) {
|
||||
analysisGainNodeRef.current.gain.value = value;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const setupAnalyzers = useCallback((audioContext: AudioContext) => {
|
||||
// Create gain node for analysis sensitivity (does NOT affect audio output)
|
||||
analysisGainNodeRef.current = audioContext.createGain();
|
||||
analysisGainNodeRef.current.gain.value = gainValueRef.current;
|
||||
|
||||
// Create channel splitter for stereo
|
||||
splitterRef.current = audioContext.createChannelSplitter(2);
|
||||
|
||||
// Create analyzers for each channel
|
||||
analyzerLeftRef.current = audioContext.createAnalyser();
|
||||
analyzerRightRef.current = audioContext.createAnalyser();
|
||||
|
||||
// Configure analyzers for higher sensitivity
|
||||
const fftSize = 2048;
|
||||
analyzerLeftRef.current.fftSize = fftSize;
|
||||
analyzerRightRef.current.fftSize = fftSize;
|
||||
analyzerLeftRef.current.smoothingTimeConstant = 0.5;
|
||||
analyzerRightRef.current.smoothingTimeConstant = 0.5;
|
||||
analyzerLeftRef.current.minDecibels = -90;
|
||||
analyzerRightRef.current.minDecibels = -90;
|
||||
analyzerLeftRef.current.maxDecibels = -10;
|
||||
analyzerRightRef.current.maxDecibels = -10;
|
||||
}, []);
|
||||
|
||||
const startMicrophone = useCallback(async () => {
|
||||
try {
|
||||
setState(prev => ({ ...prev, isActive: false, error: null }));
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
},
|
||||
});
|
||||
|
||||
streamRef.current = stream;
|
||||
audioContextRef.current = new AudioContext();
|
||||
|
||||
setupAnalyzers(audioContextRef.current);
|
||||
|
||||
// Create source from microphone
|
||||
const micSource = audioContextRef.current.createMediaStreamSource(stream);
|
||||
sourceRef.current = micSource;
|
||||
|
||||
// Connect: source -> analysisGain -> splitter -> analyzers
|
||||
// (microphone doesn't need output, just analysis)
|
||||
micSource.connect(analysisGainNodeRef.current!);
|
||||
analysisGainNodeRef.current!.connect(splitterRef.current!);
|
||||
splitterRef.current!.connect(analyzerLeftRef.current!, 0);
|
||||
splitterRef.current!.connect(analyzerRightRef.current!, 1);
|
||||
|
||||
setState({
|
||||
isActive: true,
|
||||
error: null,
|
||||
source: 'microphone',
|
||||
fileName: null,
|
||||
isPlaying: true
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to access microphone';
|
||||
setState(prev => ({ ...prev, isActive: false, error: message }));
|
||||
}
|
||||
}, [setupAnalyzers]);
|
||||
|
||||
const loadAudioFile = useCallback(async (file: File) => {
|
||||
try {
|
||||
// Stop any existing audio
|
||||
stop();
|
||||
|
||||
setState(prev => ({ ...prev, isActive: false, error: null }));
|
||||
|
||||
// Create audio element
|
||||
const audioElement = new Audio();
|
||||
audioElement.src = URL.createObjectURL(file);
|
||||
audioElement.loop = true;
|
||||
audioElementRef.current = audioElement;
|
||||
|
||||
audioContextRef.current = new AudioContext();
|
||||
setupAnalyzers(audioContextRef.current);
|
||||
|
||||
// Create source from audio element
|
||||
const audioSource = audioContextRef.current.createMediaElementSource(audioElement);
|
||||
sourceRef.current = audioSource;
|
||||
|
||||
// For files: source -> destination (clean audio output)
|
||||
// source -> analysisGain -> splitter -> analyzers (boosted for visualization)
|
||||
audioSource.connect(audioContextRef.current.destination);
|
||||
audioSource.connect(analysisGainNodeRef.current!);
|
||||
analysisGainNodeRef.current!.connect(splitterRef.current!);
|
||||
splitterRef.current!.connect(analyzerLeftRef.current!, 0);
|
||||
splitterRef.current!.connect(analyzerRightRef.current!, 1);
|
||||
|
||||
// Start playing
|
||||
await audioElement.play();
|
||||
|
||||
setState({
|
||||
isActive: true,
|
||||
error: null,
|
||||
source: 'file',
|
||||
fileName: file.name,
|
||||
isPlaying: true
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to load audio file';
|
||||
setState(prev => ({ ...prev, isActive: false, error: message }));
|
||||
}
|
||||
}, [setupAnalyzers]);
|
||||
|
||||
const togglePlayPause = useCallback(() => {
|
||||
if (!audioElementRef.current) return;
|
||||
|
||||
if (audioElementRef.current.paused) {
|
||||
audioElementRef.current.play();
|
||||
setState(prev => ({ ...prev, isPlaying: true }));
|
||||
} else {
|
||||
audioElementRef.current.pause();
|
||||
setState(prev => ({ ...prev, isPlaying: false }));
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach(track => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
|
||||
if (audioElementRef.current) {
|
||||
audioElementRef.current.pause();
|
||||
audioElementRef.current.src = '';
|
||||
audioElementRef.current = null;
|
||||
}
|
||||
|
||||
if (sourceRef.current) {
|
||||
sourceRef.current.disconnect();
|
||||
sourceRef.current = null;
|
||||
}
|
||||
|
||||
if (analysisGainNodeRef.current) {
|
||||
analysisGainNodeRef.current.disconnect();
|
||||
analysisGainNodeRef.current = null;
|
||||
}
|
||||
|
||||
if (splitterRef.current) {
|
||||
splitterRef.current.disconnect();
|
||||
splitterRef.current = null;
|
||||
}
|
||||
|
||||
if (audioContextRef.current) {
|
||||
audioContextRef.current.close();
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
|
||||
analyzerLeftRef.current = null;
|
||||
analyzerRightRef.current = null;
|
||||
|
||||
setState({
|
||||
isActive: false,
|
||||
error: null,
|
||||
source: null,
|
||||
fileName: null,
|
||||
isPlaying: false
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
const getAudioElement = useCallback(() => {
|
||||
return audioElementRef.current;
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...state,
|
||||
startMicrophone,
|
||||
loadAudioFile,
|
||||
togglePlayPause,
|
||||
stop,
|
||||
setGain,
|
||||
getTimeDomainData,
|
||||
getStereoData,
|
||||
getAudioElement,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,452 @@
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
|
||||
export type ExportStage = 'idle' | 'preparing' | 'rendering' | 'encoding' | 'complete';
|
||||
|
||||
interface ExportState {
|
||||
isExporting: boolean;
|
||||
progress: number;
|
||||
error: string | null;
|
||||
stage: ExportStage;
|
||||
fps: number;
|
||||
}
|
||||
|
||||
interface ExportOptions {
|
||||
fps: number;
|
||||
format: 'webm' | 'mp4';
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
interface WavHeader {
|
||||
sampleRate: number;
|
||||
numChannels: number;
|
||||
bitsPerSample: number;
|
||||
dataOffset: number;
|
||||
dataSize: number;
|
||||
}
|
||||
|
||||
// Parse WAV header without loading entire file
|
||||
async function parseWavHeader(file: File): Promise<WavHeader> {
|
||||
const headerBuffer = await file.slice(0, 44).arrayBuffer();
|
||||
const view = new DataView(headerBuffer);
|
||||
|
||||
// Verify RIFF header
|
||||
const riff = String.fromCharCode(view.getUint8(0), view.getUint8(1), view.getUint8(2), view.getUint8(3));
|
||||
if (riff !== 'RIFF') throw new Error('Not a valid WAV file');
|
||||
|
||||
const wave = String.fromCharCode(view.getUint8(8), view.getUint8(9), view.getUint8(10), view.getUint8(11));
|
||||
if (wave !== 'WAVE') throw new Error('Not a valid WAV file');
|
||||
|
||||
// Find fmt chunk
|
||||
const numChannels = view.getUint16(22, true);
|
||||
const sampleRate = view.getUint32(24, true);
|
||||
const bitsPerSample = view.getUint16(34, true);
|
||||
|
||||
// Find data chunk - scan for 'data' marker
|
||||
let dataOffset = 36;
|
||||
let dataSize = 0;
|
||||
|
||||
// Read more bytes to find data chunk
|
||||
const extendedBuffer = await file.slice(0, Math.min(1024, file.size)).arrayBuffer();
|
||||
const extendedView = new DataView(extendedBuffer);
|
||||
|
||||
for (let i = 36; i < extendedBuffer.byteLength - 8; i++) {
|
||||
const marker = String.fromCharCode(
|
||||
extendedView.getUint8(i),
|
||||
extendedView.getUint8(i + 1),
|
||||
extendedView.getUint8(i + 2),
|
||||
extendedView.getUint8(i + 3)
|
||||
);
|
||||
if (marker === 'data') {
|
||||
dataOffset = i + 8;
|
||||
dataSize = extendedView.getUint32(i + 4, true);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dataSize === 0) {
|
||||
// Estimate from file size
|
||||
dataSize = file.size - dataOffset;
|
||||
}
|
||||
|
||||
return { sampleRate, numChannels, bitsPerSample, dataOffset, dataSize };
|
||||
}
|
||||
|
||||
// Read a chunk of samples from WAV file
|
||||
async function readWavChunk(
|
||||
file: File,
|
||||
header: WavHeader,
|
||||
startSample: number,
|
||||
numSamples: number
|
||||
): Promise<{ left: Float32Array; right: Float32Array }> {
|
||||
const bytesPerSample = header.bitsPerSample / 8;
|
||||
const bytesPerFrame = bytesPerSample * header.numChannels;
|
||||
|
||||
const startByte = header.dataOffset + (startSample * bytesPerFrame);
|
||||
const endByte = Math.min(startByte + (numSamples * bytesPerFrame), file.size);
|
||||
|
||||
const chunk = await file.slice(startByte, endByte).arrayBuffer();
|
||||
const view = new DataView(chunk);
|
||||
|
||||
const actualSamples = Math.floor(chunk.byteLength / bytesPerFrame);
|
||||
const left = new Float32Array(actualSamples);
|
||||
const right = new Float32Array(actualSamples);
|
||||
|
||||
for (let i = 0; i < actualSamples; i++) {
|
||||
const offset = i * bytesPerFrame;
|
||||
|
||||
if (header.bitsPerSample === 16) {
|
||||
left[i] = view.getInt16(offset, true) / 32768;
|
||||
right[i] = header.numChannels > 1
|
||||
? view.getInt16(offset + 2, true) / 32768
|
||||
: left[i];
|
||||
} else if (header.bitsPerSample === 24) {
|
||||
const l = (view.getUint8(offset) | (view.getUint8(offset + 1) << 8) | (view.getInt8(offset + 2) << 16));
|
||||
left[i] = l / 8388608;
|
||||
if (header.numChannels > 1) {
|
||||
const r = (view.getUint8(offset + 3) | (view.getUint8(offset + 4) << 8) | (view.getInt8(offset + 5) << 16));
|
||||
right[i] = r / 8388608;
|
||||
} else {
|
||||
right[i] = left[i];
|
||||
}
|
||||
} else if (header.bitsPerSample === 32) {
|
||||
left[i] = view.getFloat32(offset, true);
|
||||
right[i] = header.numChannels > 1
|
||||
? view.getFloat32(offset + 4, true)
|
||||
: left[i];
|
||||
} else {
|
||||
// 8-bit
|
||||
left[i] = (view.getUint8(offset) - 128) / 128;
|
||||
right[i] = header.numChannels > 1
|
||||
? (view.getUint8(offset + 1) - 128) / 128
|
||||
: left[i];
|
||||
}
|
||||
}
|
||||
|
||||
return { left, right };
|
||||
}
|
||||
|
||||
export const useOfflineVideoExport = () => {
|
||||
const [state, setState] = useState<ExportState>({
|
||||
isExporting: false,
|
||||
progress: 0,
|
||||
error: null,
|
||||
stage: 'idle',
|
||||
fps: 0,
|
||||
});
|
||||
|
||||
const cancelledRef = useRef(false);
|
||||
|
||||
const generateVideoWithAudio = useCallback(async (
|
||||
audioFile: File,
|
||||
drawFrame: (ctx: CanvasRenderingContext2D, width: number, height: number, leftData: Uint8Array, rightData: Uint8Array) => void,
|
||||
options: ExportOptions
|
||||
): Promise<Blob | null> => {
|
||||
cancelledRef.current = false;
|
||||
setState({ isExporting: true, progress: 0, error: null, stage: 'preparing', fps: 0 });
|
||||
|
||||
try {
|
||||
const { fps, width, height } = options;
|
||||
const isWav = audioFile.name.toLowerCase().endsWith('.wav');
|
||||
|
||||
console.log(`Starting memory-efficient export: ${audioFile.name} (${(audioFile.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
|
||||
let sampleRate: number;
|
||||
let totalSamples: number;
|
||||
let getChunk: (startSample: number, numSamples: number) => Promise<{ left: Float32Array; right: Float32Array }>;
|
||||
|
||||
if (isWav) {
|
||||
// Memory-efficient WAV streaming
|
||||
console.log('Using streaming WAV parser (memory efficient)');
|
||||
const header = await parseWavHeader(audioFile);
|
||||
sampleRate = header.sampleRate;
|
||||
const bytesPerSample = header.bitsPerSample / 8 * header.numChannels;
|
||||
totalSamples = Math.floor(header.dataSize / bytesPerSample);
|
||||
|
||||
getChunk = (startSample, numSamples) => readWavChunk(audioFile, header, startSample, numSamples);
|
||||
|
||||
console.log(`WAV: ${header.numChannels}ch, ${sampleRate}Hz, ${header.bitsPerSample}bit, ${totalSamples} samples`);
|
||||
} else {
|
||||
// For non-WAV files, we need to decode (uses more memory)
|
||||
console.log('Non-WAV file, using AudioContext decode (higher memory)');
|
||||
const arrayBuffer = await audioFile.arrayBuffer();
|
||||
const audioContext = new AudioContext();
|
||||
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
|
||||
sampleRate = audioBuffer.sampleRate;
|
||||
totalSamples = audioBuffer.length;
|
||||
|
||||
const leftChannel = audioBuffer.getChannelData(0);
|
||||
const rightChannel = audioBuffer.numberOfChannels > 1 ? audioBuffer.getChannelData(1) : leftChannel;
|
||||
|
||||
await audioContext.close();
|
||||
|
||||
getChunk = async (startSample, numSamples) => {
|
||||
const end = Math.min(startSample + numSamples, totalSamples);
|
||||
return {
|
||||
left: leftChannel.slice(startSample, end),
|
||||
right: rightChannel.slice(startSample, end),
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
if (cancelledRef.current) {
|
||||
setState({ isExporting: false, progress: 0, error: 'Cancelled', stage: 'idle', fps: 0 });
|
||||
return null;
|
||||
}
|
||||
|
||||
const duration = totalSamples / sampleRate;
|
||||
const totalFrames = Math.ceil(duration * fps);
|
||||
const samplesPerFrame = Math.floor(sampleRate / fps);
|
||||
const fftSize = 2048;
|
||||
|
||||
console.log(`Duration: ${duration.toFixed(2)}s, ${totalFrames} frames @ ${fps}fps`);
|
||||
|
||||
setState(prev => ({ ...prev, stage: 'rendering', progress: 5 }));
|
||||
|
||||
// Create canvas
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext('2d', { alpha: false, desynchronized: true });
|
||||
|
||||
if (!ctx) throw new Error('Could not create canvas context');
|
||||
|
||||
// Setup video recording
|
||||
const stream = canvas.captureStream(0);
|
||||
const videoTrack = stream.getVideoTracks()[0];
|
||||
|
||||
const mimeType = MediaRecorder.isTypeSupported('video/webm;codecs=vp9')
|
||||
? 'video/webm;codecs=vp9'
|
||||
: 'video/webm;codecs=vp8';
|
||||
|
||||
const videoChunks: Blob[] = [];
|
||||
const recorder = new MediaRecorder(stream, {
|
||||
mimeType,
|
||||
videoBitsPerSecond: 20_000_000,
|
||||
});
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) videoChunks.push(e.data);
|
||||
};
|
||||
|
||||
// Start recording
|
||||
recorder.start(1000);
|
||||
|
||||
const startTime = performance.now();
|
||||
let framesProcessed = 0;
|
||||
|
||||
// Process frames in batches, loading audio chunks as needed
|
||||
const chunkSizeFrames = 120; // Process 2 seconds at a time (at 60fps)
|
||||
const samplesPerChunk = chunkSizeFrames * samplesPerFrame + fftSize;
|
||||
|
||||
for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += chunkSizeFrames) {
|
||||
if (cancelledRef.current) {
|
||||
recorder.stop();
|
||||
setState({ isExporting: false, progress: 0, error: 'Cancelled', stage: 'idle', fps: 0 });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Load audio chunk for this batch
|
||||
const startSample = frameIndex * samplesPerFrame;
|
||||
const { left: leftChunk, right: rightChunk } = await getChunk(startSample, samplesPerChunk);
|
||||
|
||||
// Process frames in this chunk
|
||||
const endFrame = Math.min(frameIndex + chunkSizeFrames, totalFrames);
|
||||
|
||||
for (let f = frameIndex; f < endFrame; f++) {
|
||||
const localOffset = (f - frameIndex) * samplesPerFrame;
|
||||
|
||||
// Extract waveform data for this frame
|
||||
const leftData = new Uint8Array(fftSize);
|
||||
const rightData = new Uint8Array(fftSize);
|
||||
|
||||
for (let i = 0; i < fftSize; i++) {
|
||||
const sampleIndex = localOffset + Math.floor((i / fftSize) * samplesPerFrame);
|
||||
|
||||
if (sampleIndex >= 0 && sampleIndex < leftChunk.length) {
|
||||
leftData[i] = Math.round((leftChunk[sampleIndex] * 128) + 128);
|
||||
rightData[i] = Math.round((rightChunk[sampleIndex] * 128) + 128);
|
||||
} else {
|
||||
leftData[i] = 128;
|
||||
rightData[i] = 128;
|
||||
}
|
||||
}
|
||||
|
||||
// Draw frame
|
||||
drawFrame(ctx, width, height, leftData, rightData);
|
||||
|
||||
// Capture frame
|
||||
const track = videoTrack as unknown as { requestFrame?: () => void };
|
||||
if (track.requestFrame) track.requestFrame();
|
||||
|
||||
framesProcessed++;
|
||||
}
|
||||
|
||||
// Update progress
|
||||
const elapsed = (performance.now() - startTime) / 1000;
|
||||
const currentFps = Math.round(framesProcessed / elapsed);
|
||||
const progress = 5 + Math.round((framesProcessed / totalFrames) * 85);
|
||||
setState(prev => ({ ...prev, progress, fps: currentFps }));
|
||||
|
||||
// Yield to main thread
|
||||
await new Promise(r => setTimeout(r, 0));
|
||||
}
|
||||
|
||||
// Stop recording
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
recorder.stop();
|
||||
|
||||
// Wait for recorder to finish
|
||||
await new Promise<void>(resolve => {
|
||||
const checkInterval = setInterval(() => {
|
||||
if (recorder.state === 'inactive') {
|
||||
clearInterval(checkInterval);
|
||||
resolve();
|
||||
}
|
||||
}, 100);
|
||||
});
|
||||
|
||||
const videoBlob = new Blob(videoChunks, { type: mimeType });
|
||||
console.log(`Video rendered: ${(videoBlob.size / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
setState(prev => ({ ...prev, stage: 'encoding', progress: 92 }));
|
||||
|
||||
// Mux audio with video (streaming approach)
|
||||
const finalBlob = await muxAudioVideo(videoBlob, audioFile, duration, fps);
|
||||
|
||||
setState({ isExporting: false, progress: 100, error: null, stage: 'complete', fps: 0 });
|
||||
console.log(`Export complete: ${(finalBlob.size / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
return finalBlob;
|
||||
|
||||
} catch (err) {
|
||||
console.error('Export error:', err);
|
||||
const message = err instanceof Error ? err.message : 'Export failed';
|
||||
setState({ isExporting: false, progress: 0, error: message, stage: 'idle', fps: 0 });
|
||||
return null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const cancelExport = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
}, []);
|
||||
|
||||
const downloadBlob = useCallback((blob: Blob, filename: string) => {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...state,
|
||||
generateVideoWithAudio,
|
||||
cancelExport,
|
||||
downloadBlob,
|
||||
};
|
||||
};
|
||||
|
||||
// Memory-efficient muxing using real-time playback
|
||||
async function muxAudioVideo(
|
||||
videoBlob: Blob,
|
||||
audioFile: File,
|
||||
duration: number,
|
||||
fps: number
|
||||
): Promise<Blob> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const videoUrl = URL.createObjectURL(videoBlob);
|
||||
const audioUrl = URL.createObjectURL(audioFile);
|
||||
|
||||
const video = document.createElement('video');
|
||||
const audio = document.createElement('audio');
|
||||
|
||||
video.src = videoUrl;
|
||||
video.muted = true;
|
||||
video.playbackRate = 4; // Speed up playback
|
||||
audio.src = audioUrl;
|
||||
audio.playbackRate = 4;
|
||||
|
||||
const cleanup = () => {
|
||||
URL.revokeObjectURL(videoUrl);
|
||||
URL.revokeObjectURL(audioUrl);
|
||||
};
|
||||
|
||||
Promise.all([
|
||||
new Promise<void>((res, rej) => {
|
||||
video.onloadedmetadata = () => res();
|
||||
video.onerror = () => rej(new Error('Failed to load video'));
|
||||
}),
|
||||
new Promise<void>((res, rej) => {
|
||||
audio.onloadedmetadata = () => res();
|
||||
audio.onerror = () => rej(new Error('Failed to load audio'));
|
||||
}),
|
||||
]).then(() => {
|
||||
const audioContext = new AudioContext();
|
||||
const audioSource = audioContext.createMediaElementSource(audio);
|
||||
const audioDestination = audioContext.createMediaStreamDestination();
|
||||
audioSource.connect(audioDestination);
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = video.videoWidth || 1920;
|
||||
canvas.height = video.videoHeight || 1080;
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
const canvasStream = canvas.captureStream(fps);
|
||||
|
||||
const combinedStream = new MediaStream([
|
||||
...canvasStream.getVideoTracks(),
|
||||
...audioDestination.stream.getAudioTracks(),
|
||||
]);
|
||||
|
||||
const mimeType = MediaRecorder.isTypeSupported('video/webm;codecs=vp9,opus')
|
||||
? 'video/webm;codecs=vp9,opus'
|
||||
: 'video/webm;codecs=vp8,opus';
|
||||
|
||||
const chunks: Blob[] = [];
|
||||
const recorder = new MediaRecorder(combinedStream, {
|
||||
mimeType,
|
||||
videoBitsPerSecond: 20_000_000,
|
||||
audioBitsPerSecond: 320_000,
|
||||
});
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunks.push(e.data);
|
||||
};
|
||||
|
||||
recorder.onstop = () => {
|
||||
cleanup();
|
||||
audioContext.close();
|
||||
resolve(new Blob(chunks, { type: mimeType }));
|
||||
};
|
||||
|
||||
recorder.onerror = () => {
|
||||
cleanup();
|
||||
reject(new Error('Muxing failed'));
|
||||
};
|
||||
|
||||
const drawLoop = () => {
|
||||
if (video.ended || audio.ended) {
|
||||
setTimeout(() => recorder.stop(), 100);
|
||||
return;
|
||||
}
|
||||
ctx.drawImage(video, 0, 0);
|
||||
requestAnimationFrame(drawLoop);
|
||||
};
|
||||
|
||||
recorder.start(100);
|
||||
video.currentTime = 0;
|
||||
audio.currentTime = 0;
|
||||
video.play();
|
||||
audio.play();
|
||||
drawLoop();
|
||||
}).catch(err => {
|
||||
cleanup();
|
||||
console.warn('Muxing failed, returning video only:', err);
|
||||
resolve(videoBlob);
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user