Andoird使用AudioTrack以及OpenSLES渲染音频

以你之姓@ 2022-05-24 00:13 67阅读 0赞

最近音视频开发学习到了第四章，这一章讲的是移动平台的音视频渲染，对于AudioTrack以及OpenSLES相当于又复习了一遍。这一章结合之前的ffmpeg解码来进行mp3文件的播放。主要的难点有linux多线程的同步，生产者消费者模型等等。如果对ffmpeg解码不熟悉的可以看[https://blog.csdn.net/a568478312/article/details/80268498][https_blog.csdn.net_a568478312_article_details_80268498]，这篇文章的重点在于api的使用以及多线程的控制。

#### 使用AudioTrack播放音频 ####

AudioTrack的API其实用起来比较方便，一般需要结合解码器来使用。首先我们需要知道pcm数据的采样率，比特率，声道数等信息，用来之后初始化AudioTrack。和之前的解码一样，这里同样使用ffmpeg。

private boolean initMetaData(String path) {
            int[] metaArray = new int[]{
       0, 0, 0};
            decoder.getMusicMetaByPath(path, metaArray);
            this.sampleRateInHz = metaArray[0];
            this.bitRate = metaArray[1];
            this.channel=metaArray[2];
            if (sampleRateInHz <= 0 || bitRate <= 0) {
                return false;
            }
            totalCapacity = (new File(path)).length();
            mp3CapacityPerSec = bitRate / BITS_PER_BYTE;
            if (mp3CapacityPerSec == 0) {
                return false;
            }
            duration = (int) (totalCapacity / mp3CapacityPerSec);
            Log.d("tedu", " bitrate duration: " + duration);
            int byteCountPerSec = sampleRateInHz * CHANNEL_PER_FRAME * BITS_PER_CHANNEL / BITS_PER_BYTE;
            //比特率不变 采样率是被压缩的
            Log.d("tedu", "initMetaData: bitRate " + bitRate + "mp3CapacityPerSec " + mp3CapacityPerSec +
                    "byteCountPerSec  " + byteCountPerSec);
    
            DECODE_BUFFER_SIZE = (int) ((byteCountPerSec / 2) * 0.2);
    
            initPlayState();
            seekBaseMillsTime = 0;
            audioTrackBaseHeadPosition = 0;
            return true;
        }

接着初始化解码的核心功能，并且计算出来之后每一个解码buffer的大小，接着初始化队列以及开启线程。PacketPool维护一个链表实现的队列，用来保存解码的裸数据。

void AccompanyDecoderController::init(const char* accompanyPath,
            float packetBufferTimePercent) {
        //初始化两个全局变量
        volume = 1.0f;
        accompanyMax = 1.0f;
    
        //计算计算出伴奏和原唱的bufferSize
        int accompanyByteCountPerSec = accompanySampleRate * CHANNEL_PER_FRAME
                * BITS_PER_CHANNEL / BITS_PER_BYTE;
        accompanyPacketBufferSize = (int) ((accompanyByteCountPerSec / 2)
                * packetBufferTimePercent);
        //初始化两个decoder
        initAccompanyDecoder(accompanyPath);
        //初始化队列以及开启线程
        packetPool = PacketPool::GetInstance();
        packetPool->initDecoderAccompanyPacketQueue();
        initDecoderThread();
    }

下面是开启解码线程。

void AccompanyDecoderController::initDecoderThread() {
        isRunning = true;
        pthread_mutex_init(&mLock, NULL);
        pthread_cond_init(&mCondition, NULL);
        pthread_create(&songDecoderThread, NULL, startDecoderThread, this);
    }

解码线程中不断去文件中解码数据，并且将解码数据放入队列中，如果队列的数量超过阈值，释放锁并进行等待。

void* AccompanyDecoderController::startDecoderThread(void* ptr) {
        LOGI("enter AccompanyDecoderController::startDecoderThread");
        AccompanyDecoderController* decoderController =
                (AccompanyDecoderController *) ptr;
        int getLockCode = pthread_mutex_lock(&decoderController->mLock);
        while (decoderController->isRunning) {
            decoderController->decodeSongPacket();
            if (decoderController->packetPool->geDecoderAccompanyPacketQueueSize() > QUEUE_SIZE_MAX_THRESHOLD) {
                pthread_cond_wait(&decoderController->mCondition,
                        &decoderController->mLock);
            }
        }
        pthread_mutex_unlock(&decoderController->mLock);
    }

接着我们初始化AudioTrack,注意这里的最好是使用它提供的方法来进行计算。然后使用我们获取的采样率，声道数之类的将AudioTrack初始化。这里需要注意AudioTrack的两种播放模式，一种是流的模式一种静态模式。

*  AudioTrack.MODE\_STREAM 在播放的文件过大的时候使用，避免内存问题。
 *  AudioTrack.MODE\_STATIC 一次性写入所有数据，一般播放小文件。

private void initAudioTrack() {
            int bufferSize = AudioTrack.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat);
            audioTrack = new AudioTrack(this.audioStreamType, sampleRateInHz, channelConfig, audioFormat, bufferSize,
                    AudioTrack.MODE_STREAM);
        }

接下来我开启java线程来不断从native的队列里读取数据。

class PlayerThread implements Runnable {
        
            private short[] samples;
    
            @Override
            public void run() {
                int sample_count = 0;
                boolean isPlayTemp = isPlaying = false;
                try {
                        //?0.2S的数据
                    samples = new short[DECODE_BUFFER_SIZE];
                    short[] fakeSamples = new short[DECODE_BUFFER_SIZE];
                    int[] extraSlientSampleSize = new int[1];
                    // int sampleDurationMs = (int)((float)DECODE_BUFFER_SIZE /
                    // sampleRateInHz / CHANNEL_PER_FRAME * 1000);
                    while (!isStop) {
                        // long start = System.currentTimeMillis();
                        extraSlientSampleSize[0] = 0;
                        sample_count = decoder.readSamples(samples, extraSlientSampleSize);
    
                        // Log.i(TAG, "end Native Mp3 playing......
                        // DECODE_BUFFER_SIZE="+DECODE_BUFFER_SIZE+"
                        // sample_count="+sample_count);
    
                        if (sample_count == -2) {
                            try {
                                Thread.sleep(10);
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            // Log.i(TAG, "WARN : no play data");
                            continue;
                        }
                        if (sample_count < 0) {
                            break;
                        }
                        while (true) {
                            synchronized (NativeMp3Player.class) {
                                isPlayTemp = isPlaying;
                            }
                            if (isPlayTemp)
                                break;
                            else
                                Thread.yield();
                        }
    
                        if (null != audioTrack && audioTrack.getState() != AudioTrack.STATE_UNINITIALIZED) {
                            audioTrack.setStereoVolume(1, 1);
                            if (extraSlientSampleSize[0] > 0) {
                                int totalSize = extraSlientSampleSize[0] + sample_count;
                                short[] playSamples = new short[totalSize];
                                System.arraycopy(samples, 0, playSamples, extraSlientSampleSize[0], sample_count);
                                audioTrack.write(playSamples, 0, totalSize);
                            } else {
                                audioTrack.write(samples, 0, sample_count);
                            }
                        }
                    }
    
                    decoder.destory();
                } catch (Error e) {
                    e.printStackTrace();
                }
                samples = null;
    
                // Log.i(TAG, "end Native Mp3 player thread...");
            }
        }

读取队列里的解码数据，并且在数据数量小于阈值时，通知解码线程进行解码。

int AccompanyDecoderController::readSamples(short* samples, int size,
            int* slientSizeArr) {
        int result = -1;
        AudioPacket* accompanyPacket = NULL;
        packetPool->getDecoderAccompanyPacket(&accompanyPacket, true);
        if (NULL != accompanyPacket) {
            int samplePacketSize = accompanyPacket->size;
            if (samplePacketSize != -1 && samplePacketSize <= size) {
                //copy the raw data to samples
                memcpy(samples, accompanyPacket->buffer, samplePacketSize * 2);
                adjustSamplesVolume(samples, samplePacketSize,
                        volume / accompanyMax);
                delete accompanyPacket;
                result = samplePacketSize;
            }
        } else {
            result = -2;
        }
        if (packetPool->geDecoderAccompanyPacketQueueSize() < QUEUE_SIZE_MIN_THRESHOLD) {
            int getLockCode = pthread_mutex_lock(&mLock);
            if (result != -1) {
                pthread_cond_signal(&mCondition);
            }
            pthread_mutex_unlock (&mLock);
        }
        return result;
    }

最后再简单梳理一下流程：

1.  使用ffmepge解码mp3文件的采样率，比特率等数据。
2.  生产者线程解码pcm数据并放入队列中
3.  消费者线程从队列中获取数据并写入audioTrack中进行播放。

源码整个看下来只要熟悉使用linux下的多线程的pthread\_mutex\_t mLock，pthread\_cond\_t mCondition来进行线程间的同步，没有太大难点。

#### 使用OpenSLES来播放音频 ####

OpenSL ES是可以让你使用C或者C++来实现高性能，低延迟音频API。ES代表Embedded Systems，在嵌入式系统中单独设计。使用OpenSL ES可以省去native层到java层的数据拷贝的开销，更加高效，然而API设计的并不是特别简单，总感觉略坑。不过使用下来大部分还是模板代码，稍微熟悉一下还是没什么大问题的。要使用OpenSLES我们需要先在CMakeLists.txt中引入库。

add_library( libmedia
                 STATIC
                 opensl_es_context.cpp
                 sound_service.cpp
                  )
    
    
    target_link_libraries(
                           libmedia
                           #加入OpenSLES
                           OpenSLES
                            )

接着引入头文件

#include <SLES/OpenSLES.h>
    #include <SLES/OpenSLES_Android.h>

然后我们就可以初始化OpenSL ES的环境了。首先了解一下，这套API是使用C语言来实现面向对象的设计，通用的使用流程是。

*  初始化SLObjectItf对象
 *  Realize初始化的对象
 *  通过对象获取相关接口来使用

了解了之后我们来正式看代码。

##### 1、创建播放引擎engineObject 这是一个SLObjectItf的对象。 #####

SLresult createEngine() {
            // OpenSL ES for Android is designed to be thread-safe,
            // so this option request will be ignored, but it will
            // make the source code portable to other platforms.
            SLEngineOption engineOption[] = {
        {(SLuint32) SL_ENGINEOPTION_THREADSAFE, (SLuint32) SL_BOOLEAN_TRUE}};
    
            // Create the OpenSL ES engine object
            return slCreateEngine(&engineObject, ARRAY_LEN(engineOption), engineOption, 0,
                                  0, 0
            );
    
        }

接着我们Realize初始化的引擎对象

void OpenSLESContext::init() {
        LOGI("createEngine");
        SLresult result = createEngine();
        LOGI("createEngine result is s%", ResultToString(result));
        if (SL_RESULT_SUCCESS == result) {
            LOGI("Realize the engine object");
            // Realize the engine object
            result = RealizeObject(engineObject);
            if (SL_RESULT_SUCCESS == result) {
                LOGI("Get the engine interface");
                // Get the engine interface
                result = GetEngineInterface();
            }
        }
    }

##### 2、初始化混音器，并Realize #####

简单说明下，创建的OpenSL的对象都至少实现了一个接口，但是其他的接口是可选的或者不支持的。这时候需要自己去指定接口(mids),接口数量，以及是否绝对需要(mreq)。如果指定了绝对需要的，然而不支持，就会报错，返回SL\_RESULT\_FEATURE\_UNSUPPORTED。

SLresult CreateOutputMix() {
             const SLInterfaceID mids[1] = {SL_IID_ENVIRONMENTALREVERB};
            const SLboolean mreq[1] = {SL_BOOLEAN_TRUE};
             return (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObject, 1, mids, mreq);
        }

LOGI("Realize output mix object");
        result = RealizeObject(outputMixObject);
        if (SL_RESULT_SUCCESS != result) {
            return result;
        }

##### 3、创建播放器对象，并Realize #####

SLresult CreateBufferQueueAudioPlayer() {
            SLDataLocator_AndroidBufferQueue dataSourceLocator = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
                                                                  1
            };
    
            //PCM data source format
    //      SLDataFormat_PCM dataSourceFormat = { SL_DATAFORMAT_PCM, // format type
    //              2, // channel count
    //              accompanySampleRate, // samples per second in millihertz
    //              16, // bits per sample
    //              16, // container size
    //              SL_SPEAKER_FRONT_CENTER, // channel mask
    //              SL_BYTEORDER_LITTLEENDIAN // endianness
    //              };
    
            int samplesPerSec = opensl_get_sample_rate(accompanySampleRate);
           //使用pcm数据格式，还可以使用URI等
            SLDataFormat_PCM dataSourceFormat={
                    SL_DATAFORMAT_PCM,
                    static_cast<SLuint32>(channels),
                    static_cast<SLuint32>(samplesPerSec),
                    SL_PCMSAMPLEFORMAT_FIXED_16, // bits per sample
                    SL_PCMSAMPLEFORMAT_FIXED_16, // container size
                    SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT, // channel mask
                    SL_BYTEORDER_LITTLEENDIAN
            };
    
            // Data source is a simple buffer queue with PCM format
            SLDataSource dataSource={&dataSourceLocator,&dataSourceFormat};
    
            // Output mix locator for data sink
            SLDataLocator_OutputMix dataSinkLocator={SL_DATALOCATOR_OUTPUTMIX,
                    outputMixObject
            };
    
            // Data sink is an output mix
            SLDataSink dataSink = { &dataSinkLocator, // locator
                                    0 // format
            };
            // Interfaces that are requested
            SLInterfaceID  interfaceIds[]={SL_IID_BUFFERQUEUE};
    
            // Required interfaces. If the required interfaces
            // are not available the request will fail
            SLboolean requiredInterfaces[] = { SL_BOOLEAN_TRUE // for SL_IID_BUFFERQUEUE
            };
    
            return (*engineEngine)->CreateAudioPlayer(engineEngine,&audioPlayerObject,&dataSource,&dataSink,
                                                      ARRAY_LEN(interfaceIds), interfaceIds, requiredInterfaces
            );
        }

接着同样实现

LOGI("Realize audio player object");
        // Realize audio player object
        result = RealizeObject(audioPlayerObject);
        if (SL_RESULT_SUCCESS != result) {
            return result;
        }

##### 4、获取播放队列接口，并注册回调 #####

SLresult GetAudioPlayerBufferQueueInterface() {
            // Get the buffer queue interface
            return (*audioPlayerObject)->GetInterface(audioPlayerObject, SL_IID_BUFFERQUEUE, &audioPlayerBufferQueue);
        };

SLresult SoundService::RegisterPlayerCallback() {
        return (*audioPlayerBufferQueue)->RegisterCallback(audioPlayerBufferQueue, PlayerCallback,
                                                           this);
    }

在回调函数中入队数据，这里解码和之前的AudioTrack是一样的，我们只需要在回调函数拿队列的数据就行了。

static void PlayerCallback(SLAndroidSimpleBufferQueueItf bq,void *context){
            SoundService* service= static_cast<SoundService *>(context);
            service->producePacket();
        }

void SoundService::producePacket() {
        LOGI("SoundService::producePacket() audio player call back method... ");
        // Read data
        byte *audioBuffer = buffer + (currentFrame % bufferNums) * packetBufferSize;
        int result = -1;
        if (NULL != decoderController) {
            result = decoderController->readSamples(target, packetBufferSize / 2, NULL);
            LOGI("enter SoundService::producePacket() PLAYING_STATE_PLAYING packetBufferSize=%d, result=%d",
                 packetBufferSize, result);
        }
        if (0 < result) {
          //转成char数组后入队数据
            convertByteArrayFromShortArray(target, result, audioBuffer);
            (*audioPlayerBufferQueue)->Enqueue(audioPlayerBufferQueue, audioBuffer, result * 2);
    
        } else {
            LOGI("before DestroyContext");
            //      DestroyContext();
            LOGI("after DestroyContext");
            JNIEnv *env;
            //播放完毕后会叫java函数
            //获取对应线程的env
            if (g_jvm->AttachCurrentThread(&env, NULL) != JNI_OK) {
                LOGE("%s: AttachCurrentThread() failed", __FUNCTION__);
            }
            jclass jcls = env->GetObjectClass(obj);
            jmethodID onCompletionCallBack = env->GetMethodID(jcls, "onCompletion", "()V");
            LOGI("before env->CallVoidMethod");
            env->CallVoidMethod(obj, onCompletionCallBack);
            LOGI("after env->CallVoidMethod");
    
            if (g_jvm->DetachCurrentThread() != JNI_OK) {
                LOGE("%s: DetachCurrentThread() failed", __FUNCTION__);
            }
    
        }
        currentFrame = (currentFrame + 1) % bufferNums;
    
    }

##### 5、设置播放状态并手动调用一次回调函数 #####

Lresult SoundService::play() {
        //一开始需要主动调用回调函数，或者直接首先填充数据
        LOGI("enter SoundService::play()...");
        // Set the audio player state playing
        LOGI("Set the audio player state playing");
        if (playingState == PLAYING_STATE_PLAYING)
            return 0;
        SLresult result = SetAudioPlayerStatePlaying();
        if (SL_RESULT_SUCCESS != result) {
            return result;
        }
        LOGI("Set the slient audio player state playing");
    //    result = SetSlientAudioPlayerStatePlaying();
    //    if (SL_RESULT_SUCCESS != result) {
        
    //        return result;
    //    }
        LOGI(" Enqueue the first buffer to start");
    
    //    for (int i = 0; i < bufferNums; i++) {
        
    //        int result = -1;
    //        if (NULL != decoderController) {
        
    //            result = decoderController->readSamples(target, packetBufferSize / 2, NULL);
    //        }
    //        LOGI("read_count %d", result);
    //        // If data is read
    //        if (0 < result) {
        
    //            convertByteArrayFromShortArray(target, result, buffer + i * packetBufferSize);
    //            (*audioPlayerBufferQueue)->Enqueue(audioPlayerBufferQueue,
    //                                               buffer + i * packetBufferSize, result * 2);
    //        }
    //    }
        playingState = PLAYING_STATE_PLAYING;
        PlayerCallback(nullptr, this);
    
        LOGI("out SoundService::play()...");
    
        return SL_RESULT_SUCCESS;
    }

Android Studio 3.1.2写c++的方法没有返回值编译器不提示报错，然而运行的时候却特么会崩溃，真的是醉了，native使用ndk-stack快速调试，可以看这篇文章:[https://blog.csdn.net/a568478312/article/details/78182422][https_blog.csdn.net_a568478312_article_details_78182422]  
最后附上说明OpenSL ES文档[https://www.khronos.org/registry/OpenSL-ES/specs/OpenSL\_ES\_Specification\_1.0.1.pdf][https_www.khronos.org_registry_OpenSL-ES_specs_OpenSL_ES_Specification_1.0.1.pdf]  
源码:[https://download.csdn.net/download/a568478312/10421205][https_download.csdn.net_download_a568478312_10421205]

[https_blog.csdn.net_a568478312_article_details_80268498]: https://blog.csdn.net/a568478312/article/details/80268498
[https_blog.csdn.net_a568478312_article_details_78182422]: https://blog.csdn.net/a568478312/article/details/78182422
[https_www.khronos.org_registry_OpenSL-ES_specs_OpenSL_ES_Specification_1.0.1.pdf]: https://www.khronos.org/registry/OpenSL-ES/specs/OpenSL_ES_Specification_1.0.1.pdf
[https_download.csdn.net_download_a568478312_10421205]: https://download.csdn.net/download/a568478312/10421205