Merge pull request #22857 from marcelofg55/jsmic

Implemented audio input support for JavaScript audio driver

platform/javascript/audio_driver_javascript.cpp

@@ -44,6 +44,11 @@ extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() {
 	AudioDriverJavaScript::singleton->mix_to_js();
 }
 
+extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) {
+
+	AudioDriverJavaScript::singleton->process_capture(sample);
+}
+
 void AudioDriverJavaScript::mix_to_js() {
 
 	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
@@ -51,31 +56,39 @@ void AudioDriverJavaScript::mix_to_js() {
 	int32_t *stream_buffer = reinterpret_cast<int32_t *>(internal_buffer);
 	audio_server_process(sample_count, stream_buffer);
 	for (int i = 0; i < sample_count * channel_count; i++) {
-		internal_buffer[i] = float(stream_buffer[i] >> 16) / 32768.0;
+		internal_buffer[i] = float(stream_buffer[i] >> 16) / 32768.f;
 	}
 }
 
+void AudioDriverJavaScript::process_capture(float sample) {
+
+	int32_t sample32 = int32_t(sample * 32768.f) * (1U << 16);
+	input_buffer_write(sample32);
+}
+
 Error AudioDriverJavaScript::init() {
 
 	/* clang-format off */
 	EM_ASM({
 		_audioDriver_audioContext = new (window.AudioContext || window.webkitAudioContext);
+		_audioDriver_audioInput = null;
+		_audioDriver_inputStream = null;
 		_audioDriver_scriptNode = null;
 	});
 	/* clang-format on */
 
 	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
 	/* clang-format off */
-	int buffer_length = EM_ASM_INT({
+	buffer_length = EM_ASM_INT({
 		var CHANNEL_COUNT = $0;
 
 		var channelCount = _audioDriver_audioContext.destination.channelCount;
 		try {
 			// Try letting the browser recommend a buffer length.
-			_audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(0, 0, channelCount);
+			_audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(0, 2, channelCount);
 		} catch (e) {
 			// ...otherwise, default to 4096.
-			_audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(4096, 0, channelCount);
+			_audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(4096, 2, channelCount);
 		}
 		_audioDriver_scriptNode.connect(_audioDriver_audioContext.destination);
 
@@ -91,6 +104,7 @@ Error AudioDriverJavaScript::init() {
 			memdelete_arr(internal_buffer);
 		internal_buffer = memnew_arr(float, buffer_length *channel_count);
 	}
+
 	return internal_buffer ? OK : ERR_OUT_OF_MEMORY;
 }
 
@@ -101,11 +115,13 @@ void AudioDriverJavaScript::start() {
 		var INTERNAL_BUFFER_PTR = $0;
 
 		var audioDriverMixFunction = cwrap('audio_driver_js_mix');
+		var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
 		_audioDriver_scriptNode.onaudioprocess = function(audioProcessingEvent) {
 			audioDriverMixFunction();
-			// The output buffer contains the samples that will be modified and played.
+
+			var input = audioProcessingEvent.inputBuffer;
 			var output = audioProcessingEvent.outputBuffer;
-			var input = HEAPF32.subarray(
+			var internalBuffer = HEAPF32.subarray(
 					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT,
 					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
 
@@ -113,8 +129,16 @@ void AudioDriverJavaScript::start() {
 				var outputData = output.getChannelData(channel);
 				// Loop through samples.
 				for (var sample = 0; sample < outputData.length; sample++) {
-					// Set output equal to input.
-					outputData[sample] = input[sample * output.numberOfChannels + channel];
+					outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
+				}
+			}
+
+			if (_audioDriver_audioInput) {
+				var inputDataL = input.getChannelData(0);
+				var inputDataR = input.getChannelData(1);
+				for (var i = 0; i < inputDataL.length; i++) {
+					audioDriverProcessCapture(inputDataL[i]);
+					audioDriverProcessCapture(inputDataR[i]);
 				}
 			}
 		};
@@ -152,14 +176,74 @@ void AudioDriverJavaScript::finish() {
 	/* clang-format off */
 	EM_ASM({
 		_audioDriver_audioContext = null;
+		_audioDriver_audioInput = null;
 		_audioDriver_scriptNode = null;
 	});
 	/* clang-format on */
-	memdelete_arr(internal_buffer);
-	internal_buffer = NULL;
+
+	if (internal_buffer) {
+		memdelete_arr(internal_buffer);
+		internal_buffer = NULL;
+	}
+}
+
+Error AudioDriverJavaScript::capture_start() {
+
+	input_buffer_init(buffer_length);
+
+	/* clang-format off */
+	EM_ASM({
+		function gotMediaInput(stream) {
+			_audioDriver_inputStream = stream;
+			_audioDriver_audioInput = _audioDriver_audioContext.createMediaStreamSource(stream);
+			_audioDriver_audioInput.connect(_audioDriver_scriptNode);
+		}
+
+		function gotMediaInputError(e) {
+			console.log(e);
+		}
+
+		if (navigator.mediaDevices.getUserMedia) {
+			navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
+		} else {
+			if (!navigator.getUserMedia)
+				navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
+			navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
+		}
+	});
+	/* clang-format on */
+
+	return OK;
+}
+
+Error AudioDriverJavaScript::capture_stop() {
+
+	/* clang-format off */
+	EM_ASM({
+		if (_audioDriver_inputStream) {
+			const tracks = _audioDriver_inputStream.getTracks();
+			for (var i = 0; i < tracks.length; i++) {
+				tracks[i].stop();
+			}
+			_audioDriver_inputStream = null;
+		}
+
+		if (_audioDriver_audioInput) {
+			_audioDriver_audioInput.disconnect();
+			_audioDriver_audioInput = null;
+		}
+
+	});
+	/* clang-format on */
+
+	input_buffer.clear();
+
+	return OK;
 }
 
 AudioDriverJavaScript::AudioDriverJavaScript() {
 
+	internal_buffer = NULL;
+
 	singleton = this;
 }

platform/javascript/audio_driver_javascript.h

@@ -37,8 +37,12 @@ class AudioDriverJavaScript : public AudioDriver {
 
 	float *internal_buffer;
 
+	int buffer_length;
+
 public:
 	void mix_to_js();
+	void process_capture(float sample);
+
 	static AudioDriverJavaScript *singleton;
 
 	virtual const char *get_name() const;
@@ -51,6 +55,9 @@ public:
 	virtual void unlock();
 	virtual void finish();
 
+	virtual Error capture_start();
+	virtual Error capture_stop();
+
 	AudioDriverJavaScript();
 };
 

platform/javascript/detect.py

@@ -122,6 +122,7 @@ def configure(env):
     ## Link flags
 
     env.Append(LINKFLAGS=['-s', 'BINARYEN=1'])
+    env.Append(LINKFLAGS=['-s', 'BINARYEN_TRAP_MODE=\'clamp\''])
 
     # Allow increasing memory buffer size during runtime. This is efficient
     # when using WebAssembly (in comparison to asm.js) and works well for