generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor OpenSL audio driver

Browse Source 
Message:
I want to start this review, the basica framework is almost done.

Description:
This implementation is similar to current one, but
1. followed the design doc at
https://docs.google.com/a/google.com/document/d/1g5q2SVtkFPl2OSjvSF3eeLb_S7sCnbINxiBUES6XLCM/edit
which uses two threads, playout thread and recording thread, uses large
audio buffer, etc.
2. google code style.

What are missing in this cl,
1. a better way to control schedule/thread priority.
2. java/jni interface to better support what cannot be done in OpenSL.

Please take a review, thanks.
Review URL: https://webrtc-codereview.appspot.com/902004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3040 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
leozwang@webrtc.org 2012-11-02 16:48:03 +00:00
parent 737ed3bfa5
commit 5f9970fb0b
2 changed files with 1316 additions and 1549 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1512
webrtc/modules/audio_device/android/audio_device_opensles_android.cc
View File

File diff suppressed because it is too large Load Diff

271
webrtc/modules/audio_device/android/audio_device_opensles_android.h
View File

@ -8,14 +8,10 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_OPENSLES_ANDROID_H #ifndef SRC_MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_OPENSLES_ANDROID_H_
#define WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_OPENSLES_ANDROID_H #define SRC_MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_OPENSLES_ANDROID_H_
#include "audio_device_generic.h" #include <jni.h>
#include "critical_section_wrapper.h"
#include <jni.h> // For accessing AudioDeviceAndroid.java
#include <queue>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
@ -23,41 +19,55 @@
#include <SLES/OpenSLES_Android.h> #include <SLES/OpenSLES_Android.h>
#include <SLES/OpenSLES_AndroidConfiguration.h> #include <SLES/OpenSLES_AndroidConfiguration.h>
namespace webrtc #include <queue>
{
#include "modules/audio_device/audio_device_generic.h"
#include "system_wrappers/interface/critical_section_wrapper.h"
namespace webrtc {
class EventWrapper; class EventWrapper;
const WebRtc_UWord32 N_MAX_INTERFACES = 3; const WebRtc_UWord32 N_MAX_INTERFACES = 3;
const WebRtc_UWord32 N_MAX_OUTPUT_DEVICES = 6; const WebRtc_UWord32 N_MAX_OUTPUT_DEVICES = 6;
const WebRtc_UWord32 N_MAX_INPUT_DEVICES = 3; const WebRtc_UWord32 N_MAX_INPUT_DEVICES = 3;
const WebRtc_UWord32 N_REC_SAMPLES_PER_SEC = 16000;//44000; // Default fs const WebRtc_UWord32 N_REC_SAMPLES_PER_SEC = 16000; // Default fs
const WebRtc_UWord32 N_PLAY_SAMPLES_PER_SEC = 16000;//44000; // Default fs const WebRtc_UWord32 N_PLAY_SAMPLES_PER_SEC = 16000; // Default fs
const WebRtc_UWord32 N_REC_CHANNELS = 1; // default is mono recording const WebRtc_UWord32 N_REC_CHANNELS = 1;
const WebRtc_UWord32 N_PLAY_CHANNELS = 1; // default is mono playout const WebRtc_UWord32 N_PLAY_CHANNELS = 1;
const WebRtc_UWord32 REC_BUF_SIZE_IN_SAMPLES = 480; // Handle max 10 ms @ 48 kHz const WebRtc_UWord32 REC_BUF_SIZE_IN_SAMPLES = 480;
const WebRtc_UWord32 PLAY_BUF_SIZE_IN_SAMPLES = 480; const WebRtc_UWord32 PLAY_BUF_SIZE_IN_SAMPLES = 480;
const WebRtc_UWord32 REC_MAX_TEMP_BUF_SIZE_PER_10ms =
N_REC_CHANNELS * REC_BUF_SIZE_IN_SAMPLES * sizeof(int16_t);
const WebRtc_UWord32 PLAY_MAX_TEMP_BUF_SIZE_PER_10ms =
N_PLAY_CHANNELS * PLAY_BUF_SIZE_IN_SAMPLES * sizeof(int16_t);
// Number of the buffers in playout queue // Number of the buffers in playout queue
const WebRtc_UWord16 N_PLAY_QUEUE_BUFFERS = 2; const WebRtc_UWord16 N_PLAY_QUEUE_BUFFERS = 8;
// Number of buffers in recording queue // Number of buffers in recording queue
const WebRtc_UWord16 N_REC_QUEUE_BUFFERS = 2; // TODO(xian): Reduce the numbers of buffers to improve the latency.
// Number of 10 ms recording blocks in rec buffer const WebRtc_UWord16 N_REC_QUEUE_BUFFERS = 8;
const WebRtc_UWord16 N_REC_BUFFERS = 20; // Some values returned from getMinBufferSize
// (Nexus S playout 72ms, recording 64ms)
// (Galaxy, 167ms, 44ms)
// (Nexus 7, 72ms, 48ms)
// (Xoom 92ms, 40ms)
class ThreadWrapper; class ThreadWrapper;
class AudioDeviceAndroidOpenSLES: public AudioDeviceGeneric class AudioDeviceAndroidOpenSLES: public AudioDeviceGeneric {
{
public: public:
AudioDeviceAndroidOpenSLES(const WebRtc_Word32 id); explicit AudioDeviceAndroidOpenSLES(const WebRtc_Word32 id);
~AudioDeviceAndroidOpenSLES(); ~AudioDeviceAndroidOpenSLES();
// Retrieve the currently utilized audio layer // Retrieve the currently utilized audio layer
virtual WebRtc_Word32 virtual WebRtc_Word32
ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const; ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const; // NOLINT
// Main initializaton and termination // Main initializaton and termination
virtual WebRtc_Word32 Init(); virtual WebRtc_Word32 Init();
@ -85,10 +95,10 @@ public:
SetRecordingDevice(AudioDeviceModule::WindowsDeviceType device); SetRecordingDevice(AudioDeviceModule::WindowsDeviceType device);
// Audio transport initialization // Audio transport initialization
virtual WebRtc_Word32 PlayoutIsAvailable(bool& available); virtual WebRtc_Word32 PlayoutIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 InitPlayout(); virtual WebRtc_Word32 InitPlayout();
virtual bool PlayoutIsInitialized() const; virtual bool PlayoutIsInitialized() const;
virtual WebRtc_Word32 RecordingIsAvailable(bool& available); virtual WebRtc_Word32 RecordingIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 InitRecording(); virtual WebRtc_Word32 InitRecording();
virtual bool RecordingIsInitialized() const; virtual bool RecordingIsInitialized() const;
@ -107,69 +117,82 @@ public:
// Volume control based on the Windows Wave API (Windows only) // Volume control based on the Windows Wave API (Windows only)
virtual WebRtc_Word32 SetWaveOutVolume(WebRtc_UWord16 volumeLeft, virtual WebRtc_Word32 SetWaveOutVolume(WebRtc_UWord16 volumeLeft,
WebRtc_UWord16 volumeRight); WebRtc_UWord16 volumeRight);
virtual WebRtc_Word32 WaveOutVolume(WebRtc_UWord16& volumeLeft, virtual WebRtc_Word32 WaveOutVolume(
WebRtc_UWord16& volumeRight) const; WebRtc_UWord16& volumeLeft, // NOLINT
WebRtc_UWord16& volumeRight) const; // NOLINT
// Audio mixer initialization // Audio mixer initialization
virtual WebRtc_Word32 SpeakerIsAvailable(bool& available); virtual WebRtc_Word32 SpeakerIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 InitSpeaker(); virtual WebRtc_Word32 InitSpeaker();
virtual bool SpeakerIsInitialized() const; virtual bool SpeakerIsInitialized() const;
SLPlayItf playItf; virtual WebRtc_Word32 MicrophoneIsAvailable(
virtual WebRtc_Word32 MicrophoneIsAvailable(bool& available); bool& available);
virtual WebRtc_Word32 InitMicrophone(); virtual WebRtc_Word32 InitMicrophone();
virtual bool MicrophoneIsInitialized() const; virtual bool MicrophoneIsInitialized() const;
// Speaker volume controls // Speaker volume controls
virtual WebRtc_Word32 SpeakerVolumeIsAvailable(bool& available); virtual WebRtc_Word32 SpeakerVolumeIsAvailable(
bool& available); // NOLINT
virtual WebRtc_Word32 SetSpeakerVolume(WebRtc_UWord32 volume); virtual WebRtc_Word32 SetSpeakerVolume(WebRtc_UWord32 volume);
virtual WebRtc_Word32 SpeakerVolume(WebRtc_UWord32& volume) const; virtual WebRtc_Word32 SpeakerVolume(
virtual WebRtc_Word32 MaxSpeakerVolume(WebRtc_UWord32& maxVolume) const; WebRtc_UWord32& volume) const; // NOLINT
virtual WebRtc_Word32 MinSpeakerVolume(WebRtc_UWord32& minVolume) const; virtual WebRtc_Word32 MaxSpeakerVolume(
virtual WebRtc_Word32 SpeakerVolumeStepSize(WebRtc_UWord16& stepSize) const; WebRtc_UWord32& maxVolume) const; // NOLINT
virtual WebRtc_Word32 MinSpeakerVolume(
WebRtc_UWord32& minVolume) const; // NOLINT
virtual WebRtc_Word32 SpeakerVolumeStepSize(
WebRtc_UWord16& stepSize) const; // NOLINT
// Microphone volume controls // Microphone volume controls
virtual WebRtc_Word32 MicrophoneVolumeIsAvailable(bool& available); virtual WebRtc_Word32 MicrophoneVolumeIsAvailable(
bool& available); // NOLINT
virtual WebRtc_Word32 SetMicrophoneVolume(WebRtc_UWord32 volume); virtual WebRtc_Word32 SetMicrophoneVolume(WebRtc_UWord32 volume);
virtual WebRtc_Word32 MicrophoneVolume(WebRtc_UWord32& volume) const; virtual WebRtc_Word32 MicrophoneVolume(
virtual WebRtc_Word32 MaxMicrophoneVolume(WebRtc_UWord32& maxVolume) const; WebRtc_UWord32& volume) const; // NOLINT
virtual WebRtc_Word32 MinMicrophoneVolume(WebRtc_UWord32& minVolume) const; virtual WebRtc_Word32 MaxMicrophoneVolume(
WebRtc_UWord32& maxVolume) const; // NOLINT
virtual WebRtc_Word32 MinMicrophoneVolume(
WebRtc_UWord32& minVolume) const; // NOLINT
virtual WebRtc_Word32 virtual WebRtc_Word32
MicrophoneVolumeStepSize(WebRtc_UWord16& stepSize) const; MicrophoneVolumeStepSize(WebRtc_UWord16& stepSize) const; // NOLINT
// Speaker mute control // Speaker mute control
virtual WebRtc_Word32 SpeakerMuteIsAvailable(bool& available); virtual WebRtc_Word32 SpeakerMuteIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 SetSpeakerMute(bool enable); virtual WebRtc_Word32 SetSpeakerMute(bool enable);
virtual WebRtc_Word32 SpeakerMute(bool& enabled) const; virtual WebRtc_Word32 SpeakerMute(bool& enabled) const; // NOLINT
// Microphone mute control // Microphone mute control
virtual WebRtc_Word32 MicrophoneMuteIsAvailable(bool& available); virtual WebRtc_Word32 MicrophoneMuteIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 SetMicrophoneMute(bool enable); virtual WebRtc_Word32 SetMicrophoneMute(bool enable);
virtual WebRtc_Word32 MicrophoneMute(bool& enabled) const; virtual WebRtc_Word32 MicrophoneMute(bool& enabled) const; // NOLINT
// Microphone boost control // Microphone boost control
virtual WebRtc_Word32 MicrophoneBoostIsAvailable(bool& available); virtual WebRtc_Word32 MicrophoneBoostIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 SetMicrophoneBoost(bool enable); virtual WebRtc_Word32 SetMicrophoneBoost(bool enable);
virtual WebRtc_Word32 MicrophoneBoost(bool& enabled) const; virtual WebRtc_Word32 MicrophoneBoost(bool& enabled) const; // NOLINT
// Stereo support // Stereo support
virtual WebRtc_Word32 StereoPlayoutIsAvailable(bool& available); virtual WebRtc_Word32 StereoPlayoutIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 SetStereoPlayout(bool enable); virtual WebRtc_Word32 SetStereoPlayout(bool enable);
virtual WebRtc_Word32 StereoPlayout(bool& enabled) const; virtual WebRtc_Word32 StereoPlayout(bool& enabled) const; // NOLINT
virtual WebRtc_Word32 StereoRecordingIsAvailable(bool& available); virtual WebRtc_Word32 StereoRecordingIsAvailable(bool& available); // NOLINT
virtual WebRtc_Word32 SetStereoRecording(bool enable); virtual WebRtc_Word32 SetStereoRecording(bool enable);
virtual WebRtc_Word32 StereoRecording(bool& enabled) const; virtual WebRtc_Word32 StereoRecording(bool& enabled) const; // NOLINT
// Delay information and control // Delay information and control
virtual WebRtc_Word32 virtual WebRtc_Word32
SetPlayoutBuffer(const AudioDeviceModule::BufferType type, SetPlayoutBuffer(const AudioDeviceModule::BufferType type,
WebRtc_UWord16 sizeMS); WebRtc_UWord16 sizeMS);
virtual WebRtc_Word32 PlayoutBuffer(AudioDeviceModule::BufferType& type, virtual WebRtc_Word32 PlayoutBuffer(
AudioDeviceModule::BufferType& type, // NOLINT
WebRtc_UWord16& sizeMS) const; WebRtc_UWord16& sizeMS) const;
virtual WebRtc_Word32 PlayoutDelay(WebRtc_UWord16& delayMS) const; virtual WebRtc_Word32 PlayoutDelay(
virtual WebRtc_Word32 RecordingDelay(WebRtc_UWord16& delayMS) const; WebRtc_UWord16& delayMS) const; // NOLINT
virtual WebRtc_Word32 RecordingDelay(
WebRtc_UWord16& delayMS) const; // NOLINT
// CPU load // CPU load
virtual WebRtc_Word32 CPULoad(WebRtc_UWord16& load) const; virtual WebRtc_Word32 CPULoad(WebRtc_UWord16& load) const; // NOLINT
// Error and warning information // Error and warning information
virtual bool PlayoutWarning() const; virtual bool PlayoutWarning() const;
@ -186,27 +209,25 @@ public:
// Speaker audio routing // Speaker audio routing
virtual WebRtc_Word32 SetLoudspeakerStatus(bool enable); virtual WebRtc_Word32 SetLoudspeakerStatus(bool enable);
virtual WebRtc_Word32 GetLoudspeakerStatus(bool& enable) const; virtual WebRtc_Word32 GetLoudspeakerStatus(bool& enable) const; // NOLINT
private: private:
// Lock // Lock
void Lock() void Lock() {
{ crit_sect_.Enter();
_critSect.Enter();
}; };
void UnLock() void UnLock() {
{ crit_sect_.Leave();
_critSect.Leave();
}; };
static void PlayerSimpleBufferQueueCallback( static void PlayerSimpleBufferQueueCallback(
SLAndroidSimpleBufferQueueItf queueItf, SLAndroidSimpleBufferQueueItf queueItf,
void *pContext); void *pContext);
void PlayerSimpleBufferQueueCallbackHandler(
SLAndroidSimpleBufferQueueItf queueItf);
static void RecorderSimpleBufferQueueCallback( static void RecorderSimpleBufferQueueCallback(
SLAndroidSimpleBufferQueueItf queueItf, SLAndroidSimpleBufferQueueItf queueItf,
void *pContext); void *pContext);
void PlayerSimpleBufferQueueCallbackHandler(
SLAndroidSimpleBufferQueueItf queueItf);
void RecorderSimpleBufferQueueCallbackHandler( void RecorderSimpleBufferQueueCallbackHandler(
SLAndroidSimpleBufferQueueItf queueItf); SLAndroidSimpleBufferQueueItf queueItf);
void CheckErr(SLresult res); void CheckErr(SLresult res);
@ -218,99 +239,79 @@ private:
// Init // Init
WebRtc_Word32 InitSampleRate(); WebRtc_Word32 InitSampleRate();
// Threads
static bool RecThreadFunc(void*);
static bool PlayThreadFunc(void*);
bool RecThreadProcess();
bool PlayThreadProcess();
// Misc // Misc
AudioDeviceBuffer* _ptrAudioBuffer; AudioDeviceBuffer* voe_audio_buffer_;
CriticalSectionWrapper& _critSect; CriticalSectionWrapper& crit_sect_;
WebRtc_Word32 _id; WebRtc_Word32 id_;
// audio unit // audio unit
SLObjectItf _slEngineObject; SLObjectItf sles_engine_;
// playout device // playout device
SLObjectItf _slPlayer; SLObjectItf sles_player_;
SLEngineItf _slEngine; SLEngineItf sles_engine_itf_;
SLPlayItf _slPlayerPlay; SLPlayItf sles_player_itf_;
SLAndroidSimpleBufferQueueItf _slPlayerSimpleBufferQueue; SLAndroidSimpleBufferQueueItf sles_player_sbq_itf_;
SLObjectItf _slOutputMixObject; SLObjectItf sles_output_mixer_;
SLVolumeItf _slSpeakerVolume; SLVolumeItf sles_speaker_volume_;
// recording device // recording device
SLObjectItf _slRecorder; SLObjectItf sles_recorder_;
SLRecordItf _slRecorderRecord; SLRecordItf sles_recorder_itf_;
SLAudioIODeviceCapabilitiesItf _slAudioIODeviceCapabilities; SLAndroidSimpleBufferQueueItf sles_recorder_sbq_itf_;
SLAndroidSimpleBufferQueueItf _slRecorderSimpleBufferQueue; SLDeviceVolumeItf sles_mic_volume_;
SLDeviceVolumeItf _slMicVolume; WebRtc_UWord32 mic_dev_id_;
WebRtc_UWord32 _micDeviceId; WebRtc_UWord32 play_warning_, play_error_;
WebRtc_UWord32 _recQueueSeq; WebRtc_UWord32 rec_warning_, rec_error_;
// Events
EventWrapper& _timeEventRec;
// Threads
ThreadWrapper* _ptrThreadRec;
WebRtc_UWord32 _recThreadID;
// TODO(xians), remove the following flag
bool _recThreadIsInitialized;
// Playout buffer
WebRtc_Word8 _playQueueBuffer[N_PLAY_QUEUE_BUFFERS][2
* PLAY_BUF_SIZE_IN_SAMPLES];
WebRtc_UWord32 _playQueueSeq;
// States // States
bool _recordingDeviceIsSpecified; bool is_recording_dev_specified_;
bool _playoutDeviceIsSpecified; bool is_playout_dev_specified_;
bool _initialized; bool is_initialized_;
bool _recording; bool is_recording_;
bool _playing; bool is_playing_;
bool _recIsInitialized; bool is_rec_initialized_;
bool _playIsInitialized; bool is_play_initialized_;
bool _micIsInitialized; bool is_mic_initialized_;
bool _speakerIsInitialized; bool is_speaker_initialized_;
// Warnings and errors
WebRtc_UWord16 _playWarning;
WebRtc_UWord16 _playError;
WebRtc_UWord16 _recWarning;
WebRtc_UWord16 _recError;
// Delay // Delay
WebRtc_UWord16 _playoutDelay; WebRtc_UWord16 playout_delay_;
WebRtc_UWord16 _recordingDelay; WebRtc_UWord16 recording_delay_;
// AGC state // AGC state
bool _AGC; bool agc_enabled_;
// The sampling rate to use with Audio Device Buffer // Threads
WebRtc_UWord32 _adbSampleRate; ThreadWrapper* rec_thread_;
// Stored device properties WebRtc_UWord32 rec_thread_id_;
WebRtc_UWord32 _samplingRateIn; // Sampling frequency for Mic static bool RecThreadFunc(void* context);
WebRtc_UWord32 _samplingRateOut; // Sampling frequency for Speaker bool RecThreadFuncImpl();
WebRtc_UWord32 _maxSpeakerVolume; // The maximum speaker volume value EventWrapper& rec_timer_;
WebRtc_UWord32 _minSpeakerVolume; // The minimum speaker volume value
bool _loudSpeakerOn;
// Recording buffer used by the queues. WebRtc_UWord32 mic_sampling_rate_;
int8_t rec_buffer_[N_REC_BUFFERS][2 * REC_BUF_SIZE_IN_SAMPLES]; WebRtc_UWord32 speaker_sampling_rate_;
WebRtc_UWord32 max_speaker_vol_;
WebRtc_UWord32 min_speaker_vol_;
bool loundspeaker_on_;
// Queues accessed by both callback thread and recording thread after SLDataFormat_PCM player_pcm_;
// recording has been started. SLDataFormat_PCM record_pcm_;
std::queue<int8_t*> rec_worker_queue_;
std::queue<int8_t*> rec_available_queue_;
// Queue accssed by only callback thread after recording has been started. std::queue<WebRtc_Word8*> rec_queue_;
std::queue<int8_t*> rec_callback_queue_; std::queue<WebRtc_Word8*> rec_voe_audio_queue_;
std::queue<WebRtc_Word8*> rec_voe_ready_queue_;
WebRtc_Word8 rec_buf_[N_REC_QUEUE_BUFFERS][
N_REC_CHANNELS * sizeof(int16_t) * REC_BUF_SIZE_IN_SAMPLES];
WebRtc_Word8 rec_voe_buf_[N_REC_QUEUE_BUFFERS][
N_REC_CHANNELS * sizeof(int16_t) * REC_BUF_SIZE_IN_SAMPLES];
// Flag to protect setting the recording thread priority multiple times. std::queue<WebRtc_Word8*> play_queue_;
bool is_thread_priority_set_; WebRtc_Word8 play_buf_[N_PLAY_QUEUE_BUFFERS][
N_PLAY_CHANNELS * sizeof(int16_t) * PLAY_BUF_SIZE_IN_SAMPLES];
}; };
} // namespace webrtc } // namespace webrtc
#endif // WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_OPENSLES_ANDROID_H #endif // SRC_MODULES_AUDIO_DEVICE_ANDROID_AUDIO_DEVICE_OPENSLES_ANDROID_H_