audio/internal/readerdriver: Do mixing on Ebiten side

Creating a new AudioQueue is more expensive than expected, and
this caused a little blocking the game when playing a lot of SEs.

Closes #1702
Updates #1680
This commit is contained in:
Hajime Hoshi 2021-06-25 01:39:44 +09:00
parent 4573883b03
commit 17869143c0
4 changed files with 120 additions and 623 deletions

View File

@ -25,8 +25,6 @@ import "C"
import ( import (
"fmt" "fmt"
"io"
"runtime"
"sync" "sync"
"time" "time"
"unsafe" "unsafe"
@ -36,56 +34,20 @@ func IsAvailable() bool {
return true return true
} }
type audioQueuePoolItem struct { const (
queue C.AudioQueueRef float32SizeInBytes = 4
bufs []C.AudioQueueBufferRef )
}
const audioQueuePoolMaxItemNum = 32 // 32 is an arbitrary number. func newAudioQueue(sampleRate, channelNum, bitDepthInBytes int) (C.AudioQueueRef, []C.AudioQueueBufferRef, error) {
type audioQueuePool struct {
c *context
unused []audioQueuePoolItem
used []audioQueuePoolItem
m sync.Mutex
}
func (a *audioQueuePool) Prepare(context *context) error {
a.c = context
for i := 0; i < audioQueuePoolMaxItemNum; i++ {
if _, _, err := a.Get(); err != nil {
return err
}
}
a.unused = a.used
a.used = a.used[:0]
return nil
}
func (a *audioQueuePool) Get() (C.AudioQueueRef, []C.AudioQueueBufferRef, error) {
a.m.Lock()
defer a.m.Unlock()
if len(a.unused) > 0 {
q := a.unused[0]
a.unused = a.unused[1:]
a.used = append(a.used, q)
return q.queue, q.bufs, nil
}
flags := C.kAudioFormatFlagIsPacked
if a.c.bitDepthInBytes != 1 {
flags |= C.kAudioFormatFlagIsSignedInteger
}
desc := C.AudioStreamBasicDescription{ desc := C.AudioStreamBasicDescription{
mSampleRate: C.double(a.c.sampleRate), mSampleRate: C.double(sampleRate),
mFormatID: C.kAudioFormatLinearPCM, mFormatID: C.kAudioFormatLinearPCM,
mFormatFlags: C.UInt32(flags), mFormatFlags: C.kAudioFormatFlagIsFloat,
mBytesPerPacket: C.UInt32(a.c.channelNum * a.c.bitDepthInBytes), mBytesPerPacket: C.UInt32(channelNum * float32SizeInBytes),
mFramesPerPacket: 1, mFramesPerPacket: 1,
mBytesPerFrame: C.UInt32(a.c.channelNum * a.c.bitDepthInBytes), mBytesPerFrame: C.UInt32(channelNum * float32SizeInBytes),
mChannelsPerFrame: C.UInt32(a.c.channelNum), mChannelsPerFrame: C.UInt32(channelNum),
mBitsPerChannel: C.UInt32(8 * a.c.bitDepthInBytes), mBitsPerChannel: C.UInt32(8 * float32SizeInBytes),
} }
var audioQueue C.AudioQueueRef var audioQueue C.AudioQueueRef
@ -100,65 +62,37 @@ func (a *audioQueuePool) Get() (C.AudioQueueRef, []C.AudioQueueBufferRef, error)
return nil, nil, fmt.Errorf("readerdriver: AudioQueueNewFormat with StreamFormat failed: %d", osstatus) return nil, nil, fmt.Errorf("readerdriver: AudioQueueNewFormat with StreamFormat failed: %d", osstatus)
} }
size := a.c.oneBufferSize() bufs := make([]C.AudioQueueBufferRef, 0, 4)
bufs := make([]C.AudioQueueBufferRef, 0, 2)
for len(bufs) < cap(bufs) { for len(bufs) < cap(bufs) {
var buf C.AudioQueueBufferRef var buf C.AudioQueueBufferRef
if osstatus := C.AudioQueueAllocateBuffer(audioQueue, C.UInt32(size), &buf); osstatus != C.noErr { if osstatus := C.AudioQueueAllocateBuffer(audioQueue, bufferSizeInBytes, &buf); osstatus != C.noErr {
return nil, nil, fmt.Errorf("readerdriver: AudioQueueAllocateBuffer failed: %d", osstatus) return nil, nil, fmt.Errorf("readerdriver: AudioQueueAllocateBuffer failed: %d", osstatus)
} }
buf.mAudioDataByteSize = C.UInt32(size) buf.mAudioDataByteSize = bufferSizeInBytes
bufs = append(bufs, buf) bufs = append(bufs, buf)
} }
a.used = append(a.used, audioQueuePoolItem{
queue: audioQueue,
bufs: bufs,
})
return audioQueue, bufs, nil return audioQueue, bufs, nil
} }
func (a *audioQueuePool) Put(audioQueue C.AudioQueueRef) error {
a.m.Lock()
defer a.m.Unlock()
for i, q := range a.used {
if q.queue != audioQueue {
continue
}
a.used = append(a.used[:i], a.used[i+1:]...)
if len(a.unused)+len(a.used) < audioQueuePoolMaxItemNum {
a.unused = append(a.unused, q)
break
}
// As the pool is too big, remove the AudioQueue.
for _, b := range q.bufs {
if osstatus := C.AudioQueueFreeBuffer(q.queue, b); osstatus != C.noErr {
return fmt.Errorf("readerdriver: AudioQueueFreeBuffer failed: %d", osstatus)
}
}
if osstatus := C.AudioQueueDispose(q.queue, C.true); osstatus != C.noErr {
return fmt.Errorf("readerdriver: AudioQueueDispose failed: %d", osstatus)
}
break
}
return nil
}
type context struct { type context struct {
sampleRate int sampleRate int
channelNum int channelNum int
bitDepthInBytes int bitDepthInBytes int
audioQueuePool audioQueuePool audioQueue C.AudioQueueRef
unqueuedBuffers []C.AudioQueueBufferRef
cond *sync.Cond
players *players
} }
// TOOD: Convert the error code correctly. // TOOD: Convert the error code correctly.
// See https://stackoverflow.com/questions/2196869/how-do-you-convert-an-iphone-osstatus-code-to-something-useful // See https://stackoverflow.com/questions/2196869/how-do-you-convert-an-iphone-osstatus-code-to-something-useful
var theContext *context
func NewContext(sampleRate, channelNum, bitDepthInBytes int) (Context, chan struct{}, error) { func NewContext(sampleRate, channelNum, bitDepthInBytes int) (Context, chan struct{}, error) {
ready := make(chan struct{}) ready := make(chan struct{})
close(ready) close(ready)
@ -167,564 +101,101 @@ func NewContext(sampleRate, channelNum, bitDepthInBytes int) (Context, chan stru
sampleRate: sampleRate, sampleRate: sampleRate,
channelNum: channelNum, channelNum: channelNum,
bitDepthInBytes: bitDepthInBytes, bitDepthInBytes: bitDepthInBytes,
cond: sync.NewCond(&sync.Mutex{}),
players: newPlayers(),
} }
if err := c.audioQueuePool.Prepare(c); err != nil { theContext = c
q, bs, err := newAudioQueue(sampleRate, channelNum, bitDepthInBytes)
if err != nil {
return nil, nil, err return nil, nil, err
} }
c.audioQueue = q
c.unqueuedBuffers = bs
C.ebiten_readerdriver_setNotificationHandler() C.ebiten_readerdriver_setNotificationHandler()
if osstatus := C.AudioQueueStart(c.audioQueue, nil); osstatus != C.noErr {
return nil, nil, fmt.Errorf("readerdriver: AudioQueueStart failed: %d", osstatus)
}
go c.loop()
return c, ready, nil return c, ready, nil
} }
func (c *context) wait() {
c.cond.L.Lock()
defer c.cond.L.Unlock()
for len(c.unqueuedBuffers) == 0 {
c.cond.Wait()
}
}
func (c *context) loop() {
buf32 := make([]float32, bufferSizeInBytes/4)
for {
c.wait()
c.appendBuffer(buf32)
}
}
func (c *context) appendBuffer(buf32 []float32) {
c.cond.L.Lock()
defer c.cond.L.Unlock()
buf := c.unqueuedBuffers[0]
c.unqueuedBuffers = c.unqueuedBuffers[1:]
for i := range buf32 {
buf32[i] = 0
}
c.players.read(buf32)
for i, f := range buf32 {
*(*float32)(unsafe.Pointer(uintptr(buf.mAudioData) + uintptr(i)*float32SizeInBytes)) = f
}
if osstatus := C.AudioQueueEnqueueBuffer(c.audioQueue, buf, 0, nil); osstatus != C.noErr {
// TODO: Treat the error correctly
panic(fmt.Errorf("readerdriver: AudioQueueEnqueueBuffer failed: %d", osstatus))
}
}
func (c *context) Suspend() error { func (c *context) Suspend() error {
return thePlayers.suspend() if osstatus := C.AudioQueuePause(c.audioQueue); osstatus != C.noErr {
return fmt.Errorf("readerdriver: AudioQueuePause failed: %d", osstatus)
}
return nil
} }
func (c *context) Resume() error { func (c *context) Resume() error {
return thePlayers.resume() try:
} if osstatus := C.AudioQueueStart(c.audioQueue, nil); osstatus != C.noErr {
const AVAudioSessionErrorCodeSiriIsRecording = 0x73697269 // 'siri'
type player struct { if osstatus == AVAudioSessionErrorCodeSiriIsRecording {
p *playerImpl
}
type playerImpl struct {
context *context
src io.Reader
id int
audioQueue C.AudioQueueRef
buf []byte
unqueuedBufs []C.AudioQueueBufferRef
state playerState
err error
eof bool
volume float64
m sync.Mutex
}
type players struct {
players map[C.AudioQueueRef]*playerImpl
toResume map[*playerImpl]struct{}
cond *sync.Cond
}
func (p *players) add(player *playerImpl, audioQueue C.AudioQueueRef) {
p.cond.L.Lock()
defer p.cond.L.Unlock()
if p.players == nil {
p.players = map[C.AudioQueueRef]*playerImpl{}
}
runLoop := len(p.players) == 0
p.players[audioQueue] = player
if runLoop {
// Use the only one loop for multiple players (#1662).
go p.loop()
}
}
func (p *players) get(audioQueue C.AudioQueueRef) *playerImpl {
p.cond.L.Lock()
defer p.cond.L.Unlock()
return p.players[audioQueue]
}
func (p *players) remove(audioQueue C.AudioQueueRef) {
p.cond.L.Lock()
defer p.cond.L.Unlock()
pl, ok := p.players[audioQueue]
if !ok {
return
}
delete(p.players, audioQueue)
delete(p.toResume, pl)
p.cond.Signal()
}
func (p *players) shouldWait() bool {
if len(p.players) == 0 {
return false
}
for _, pl := range p.players {
if pl.canReadSourceToBuffer() {
return false
}
}
return true
}
func (p *players) wait() bool {
p.cond.L.Lock()
defer p.cond.L.Unlock()
for p.shouldWait() {
p.cond.Wait()
}
return len(p.players) > 0
}
func (p *players) loop() {
var players []*playerImpl
for {
if !p.wait() {
return
}
p.cond.L.Lock()
players = players[:0]
for _, pl := range p.players {
players = append(players, pl)
}
p.cond.L.Unlock()
for _, pl := range players {
pl.readSourceToBuffer()
}
}
}
func (p *players) suspend() error {
p.cond.L.Lock()
defer p.cond.L.Unlock()
for _, pl := range p.players {
if !pl.IsPlaying() {
continue
}
// TODO: Is this OK to Pause instead of Close?
// Oboe (Android) closes players when suspending to avoid hogging audio resources which other apps could use.
pl.Pause()
if err := pl.Err(); err != nil {
return err
}
if p.toResume == nil {
p.toResume = map[*playerImpl]struct{}{}
}
p.toResume[pl] = struct{}{}
}
return nil
}
func (p *players) resume() error {
// playerImpl's Play can touch p. Avoid the deadlock.
p.cond.L.Lock()
players := map[*playerImpl]struct{}{}
for pl := range p.toResume {
players[pl] = struct{}{}
delete(p.toResume, pl)
}
p.cond.L.Unlock()
for pl := range players {
pl.Play()
if err := pl.Err(); err != nil {
return err
}
}
return nil
}
var thePlayers = &players{
cond: sync.NewCond(&sync.Mutex{}),
}
func (c *context) NewPlayer(src io.Reader) Player {
p := &player{
p: &playerImpl{
context: c,
src: src,
volume: 1,
},
}
runtime.SetFinalizer(p, (*player).Close)
return p
}
func (p *player) Err() error {
return p.p.Err()
}
func (p *playerImpl) Err() error {
p.m.Lock()
defer p.m.Unlock()
return p.err
}
func (p *player) Play() {
p.p.Play()
}
func (p *playerImpl) Play() {
// Call Play asynchronously since AudioQueuePrime and AudioQueuePlay might take long.
ch := make(chan struct{})
go func() {
p.m.Lock()
defer p.m.Unlock()
close(ch)
p.playImpl()
}()
// Wait until the mutex is locked in the above goroutine.
<-ch
}
func (p *playerImpl) playImpl() {
if p.err != nil {
return
}
if p.state != playerPaused {
return
}
if p.audioQueue == nil {
audioQueue, audioQueueBuffers, err := p.context.audioQueuePool.Get()
if err != nil {
p.setErrorImpl(err)
return
}
p.audioQueue = audioQueue
p.unqueuedBufs = audioQueueBuffers
C.AudioQueueSetParameter(p.audioQueue, C.kAudioQueueParam_Volume, C.AudioQueueParameterValue(p.volume))
p.m.Unlock()
thePlayers.add(p, p.audioQueue)
p.m.Lock()
}
buf := make([]byte, p.context.maxBufferSize())
for len(p.buf) < p.context.maxBufferSize() {
n, err := p.src.Read(buf)
if err != nil && err != io.EOF {
p.setErrorImpl(err)
return
}
p.buf = append(p.buf, buf[:n]...)
if err == io.EOF {
p.eof = true
break
}
}
bufs := make([]C.AudioQueueBufferRef, len(p.unqueuedBufs))
copy(bufs, p.unqueuedBufs)
var unenqueued []C.AudioQueueBufferRef
for _, buf := range bufs {
queued, err := p.appendBufferImpl(buf)
if err != nil {
p.setErrorImpl(err)
return
}
if !queued {
unenqueued = append(unenqueued, buf)
}
}
p.unqueuedBufs = unenqueued
if len(p.unqueuedBufs) == 2 && p.eof {
p.state = playerPaused
return
}
for {
if osstatus := C.AudioQueuePrime(p.audioQueue, 0, nil); osstatus != C.noErr {
// AudioQueuePrime might fail just after recovering from Siri.
if osstatus == C.AVAudioSessionErrorCodeSiriIsRecording {
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
continue goto try
} }
p.setErrorImpl(fmt.Errorf("readerdriver: AudioQueuePrime failed: %d", osstatus)) return fmt.Errorf("readerdriver: AudioQueueStart failed: %d", osstatus)
return
} }
break
}
if osstatus := C.AudioQueueStart(p.audioQueue, nil); osstatus != C.noErr {
p.setErrorImpl(fmt.Errorf("readerdriver: AudioQueueStart failed: %d", osstatus))
return
}
p.state = playerPlay
}
func (p *player) Pause() {
p.p.Pause()
}
func (p *playerImpl) Pause() {
p.m.Lock()
defer p.m.Unlock()
if p.err != nil {
return
}
if p.state != playerPlay {
return
}
if p.audioQueue == nil {
return
}
if osstatus := C.AudioQueuePause(p.audioQueue); osstatus != C.noErr && p.err == nil {
p.setErrorImpl(fmt.Errorf("readerdriver: AudioQueuePause failed: %d", osstatus))
return
}
p.state = playerPaused
}
func (p *player) Reset() {
p.p.Reset()
}
func (p *playerImpl) Reset() {
p.m.Lock()
defer p.m.Unlock()
p.resetImpl()
}
func (p *playerImpl) resetImpl() {
if p.err != nil {
return
}
if p.state == playerClosed {
return
}
if p.audioQueue == nil {
return
}
// AudioQueueReset is not efficient (#1650, #1680).
// Discard the current AudioQueue and recreate one when playing this player again.
if err := p.closeAudioQueue(); err != nil {
p.setErrorImpl(err)
return
}
p.state = playerPaused
p.buf = p.buf[:0]
p.eof = false
thePlayers.cond.Signal()
}
func (p *player) IsPlaying() bool {
return p.p.IsPlaying()
}
func (p *playerImpl) IsPlaying() bool {
p.m.Lock()
defer p.m.Unlock()
return p.state == playerPlay
}
func (p *player) Volume() float64 {
return p.p.Volume()
}
func (p *playerImpl) Volume() float64 {
p.m.Lock()
defer p.m.Unlock()
return p.volume
}
func (p *player) SetVolume(volume float64) {
p.p.SetVolume(volume)
}
func (p *playerImpl) SetVolume(volume float64) {
p.m.Lock()
defer p.m.Unlock()
p.volume = volume
if p.audioQueue == nil {
return
}
C.AudioQueueSetParameter(p.audioQueue, C.kAudioQueueParam_Volume, C.AudioQueueParameterValue(volume))
}
func (p *player) UnplayedBufferSize() int {
return p.p.UnplayedBufferSize()
}
func (p *playerImpl) UnplayedBufferSize() int {
p.m.Lock()
defer p.m.Unlock()
return len(p.buf)
}
func (p *player) Close() error {
runtime.SetFinalizer(p, nil)
return p.p.Close()
}
func (p *playerImpl) Close() error {
p.m.Lock()
defer p.m.Unlock()
return p.closeImpl()
}
func (p *playerImpl) closeImpl() error {
if err := p.closeAudioQueue(); err != nil && p.err == nil {
// setErrorImpl calls closeImpl. Do not call this.
p.err = err
}
p.state = playerClosed
return p.err
}
func (p *playerImpl) closeAudioQueue() error {
if p.audioQueue == nil {
return nil
}
// Even if reuseLater is true, AudioQueuePause is not efficient for reusing.
// AudioQueueStart takes long if the AudioQueueStop is not called.
// AudioQueueStop might invoke AudioQueueReset. Unlock the mutex here to avoid a deadlock.
q := p.audioQueue
p.m.Unlock()
osstatus := C.AudioQueueStop(q, C.true)
p.m.Lock()
if osstatus != C.noErr && p.err == nil {
return fmt.Errorf("readerdriver: AudioQueueStop failed: %d", osstatus)
}
// All the AudioQueueBuffers are already dequeued. It is safe to dispose the AudioQueue and its buffers.
if err := p.context.audioQueuePool.Put(p.audioQueue); err != nil && p.err == nil {
return err
}
p.m.Unlock()
thePlayers.remove(p.audioQueue)
p.m.Lock()
p.audioQueue = nil
return nil return nil
} }
//export ebiten_readerdriver_render //export ebiten_readerdriver_render
func ebiten_readerdriver_render(inUserData unsafe.Pointer, inAQ C.AudioQueueRef, inBuffer C.AudioQueueBufferRef) { func ebiten_readerdriver_render(inUserData unsafe.Pointer, inAQ C.AudioQueueRef, inBuffer C.AudioQueueBufferRef) {
p := thePlayers.get(inAQ) theContext.cond.L.Lock()
// The player might be already closed. defer theContext.cond.L.Unlock()
if p == nil { theContext.unqueuedBuffers = append(theContext.unqueuedBuffers, inBuffer)
return theContext.cond.Signal()
}
queued, err := p.appendBuffer(inBuffer)
if err != nil {
p.setError(err)
return
}
if queued {
return
}
p.m.Lock()
defer p.m.Unlock()
p.unqueuedBufs = append(p.unqueuedBufs, inBuffer)
if len(p.unqueuedBufs) == 2 && p.eof {
p.resetImpl()
}
}
func (p *playerImpl) appendBuffer(inBuffer C.AudioQueueBufferRef) (bool, error) {
p.m.Lock()
defer p.m.Unlock()
return p.appendBufferImpl(inBuffer)
}
func (p *playerImpl) appendBufferImpl(inBuffer C.AudioQueueBufferRef) (bool, error) {
if p.eof && len(p.buf) == 0 {
return false, nil
}
oneBufferSize := p.context.oneBufferSize()
n := oneBufferSize
if len(p.buf) < n {
n = len(p.buf)
}
buf := p.buf[:n]
for i, b := range buf {
*(*byte)(unsafe.Pointer(uintptr(inBuffer.mAudioData) + uintptr(i))) = b
}
for i := len(buf); i < oneBufferSize; i++ {
*(*byte)(unsafe.Pointer(uintptr(inBuffer.mAudioData) + uintptr(i))) = 0
}
if osstatus := C.AudioQueueEnqueueBuffer(p.audioQueue, inBuffer, 0, nil); osstatus != C.noErr {
// This can happen just after resetting.
if osstatus == C.kAudioQueueErr_EnqueueDuringReset {
return false, nil
}
return false, fmt.Errorf("readerdriver: AudioQueueEnqueueBuffer failed: %d", osstatus)
}
p.buf = p.buf[n:]
thePlayers.cond.Signal()
return true, nil
}
func (p *playerImpl) setError(err error) {
p.m.Lock()
defer p.m.Unlock()
p.setErrorImpl(err)
}
func (p *playerImpl) setErrorImpl(err error) {
p.err = err
p.closeImpl()
}
func (p *playerImpl) canReadSourceToBuffer() bool {
p.m.Lock()
defer p.m.Unlock()
if p.eof {
return false
}
return len(p.buf) < p.context.maxBufferSize()
}
func (p *playerImpl) readSourceToBuffer() {
p.m.Lock()
defer p.m.Unlock()
if p.err != nil {
return
}
if p.state == playerClosed {
return
}
maxBufferSize := p.context.maxBufferSize()
if len(p.buf) >= maxBufferSize {
return
}
buf := make([]byte, maxBufferSize)
n, err := p.src.Read(buf)
if err != nil && err != io.EOF {
p.setError(err)
return
}
p.buf = append(p.buf, buf[:n]...)
if err == io.EOF && len(p.buf) == 0 {
p.eof = true
}
} }
//export ebiten_readerdriver_setGlobalPause //export ebiten_readerdriver_setGlobalPause
func ebiten_readerdriver_setGlobalPause() { func ebiten_readerdriver_setGlobalPause() {
thePlayers.suspend() theContext.Suspend()
} }
//export ebiten_readerdriver_setGlobalResume //export ebiten_readerdriver_setGlobalResume
func ebiten_readerdriver_setGlobalResume() { func ebiten_readerdriver_setGlobalResume() {
thePlayers.resume() theContext.Resume()
} }

View File

@ -0,0 +1,24 @@
// Copyright 2021 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build darwin && ios
// +build darwin,ios
package readerdriver
// 12288 seems necessary at least on iPod touch (7th).
// With 48000[Hz] stereo, the maximum delay is (12288 / 4 / 2 [samples]) / 48000 [Hz] = 0.032 [sec].
// '4' is float32 size in bytes. '2' is a number of channels for stereo.
const bufferSizeInBytes = 12288

View File

@ -19,3 +19,5 @@ package readerdriver
// #cgo LDFLAGS: -framework AppKit // #cgo LDFLAGS: -framework AppKit
import "C" import "C"
const bufferSizeInBytes = 2048

View File

@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//go:build aix || dragonfly || freebsd || hurd || illumos || linux || netbsd || openbsd || solaris //go:build aix || darwin || dragonfly || freebsd || hurd || illumos || linux || netbsd || openbsd || solaris
// +build aix dragonfly freebsd hurd illumos linux netbsd openbsd solaris // +build aix darwin dragonfly freebsd hurd illumos linux netbsd openbsd solaris
package readerdriver package readerdriver