audio/internal/oboe: Implement self-mixing and re-enable Oboe

Updates #1549
Updates #1656
Closes #1660

audio/internal/oboe/binding_android.cpp

@@ -21,93 +21,132 @@
 #include <set>
 #include <vector>
 
+#include <android/log.h>
+
 namespace {
 
-class Player : public oboe::AudioStreamDataCallback {
+class Player;
+
+class Stream : public oboe::AudioStreamDataCallback {
 public:
-  static const char *Suspend() {
+  // GetInstance returns the instance of Stream. Only one Stream object is used
-    std::lock_guard<std::mutex> lock(PlayersMutex());
+  // in one process. It is because multiple streams can be problematic in both
-    for (Player *player : GetPlayers()) {
+  // AAudio and OpenSL (#1656, #1660).
-      if (!player->IsPlaying()) {
+  static Stream &GetInstance();
-        continue;
-      }
-      // Close should be called rather than Pause for onPause.
-      // https://github.com/google/oboe/blob/master/docs/GettingStarted.md
-      if (const char *msg = player->Close(); msg) {
-        return msg;
-      }
-      GetPlayersToResume().insert(player);
-    }
-    return nullptr;
-  }
 
-  static const char *Resume() {
+  const char *Play(int sample_rate, int channel_num, int bit_depth_in_bytes);
-    std::lock_guard<std::mutex> lock(PlayersMutex());
+  const char *Pause();
-    for (Player *player : GetPlayersToResume()) {
+  const char *Resume();
-      if (const char *msg = player->Play(); msg) {
+  const char *Close();
-        return msg;
-      }
-    }
-    GetPlayersToResume().clear();
-    return nullptr;
-  }
 
-  Player(int sample_rate, int channel_num, int bit_depth_in_bytes,
+  void AddPlayer(Player *player);
-         double volume, uintptr_t go_player)
+  void RemovePlayer(Player *player);
-      : sample_rate_{sample_rate}, channel_num_{channel_num},
+
-        bit_depth_in_bytes_{bit_depth_in_bytes}, go_player_{go_player} {
+  oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboe_stream,
+                                        void *audio_data,
+                                        int32_t num_frames) override;
+
+private:
+  Stream();
+  int sample_rate_ = 0;
+  int channel_num_ = 0;
+  int bit_depth_in_bytes_ = 0;
+
+  std::mutex mutex_;
+  std::set<Player *> players_;
+  std::shared_ptr<oboe::AudioStream> stream_;
+};
+
+class Player {
+public:
+  Player(double volume, uintptr_t go_player) : go_player_{go_player} {
     std::atomic_store(&volume_, volume);
-    {
+    Stream::GetInstance().AddPlayer(this);
-      std::lock_guard<std::mutex> lock(PlayersMutex());
-      GetPlayers().insert(this);
-    }
   }
 
+  ~Player() { Stream::GetInstance().RemovePlayer(this); }
+
   void SetVolume(double volume) { std::atomic_store(&volume_, volume); }
 
-  bool IsPlaying() { return stream_->getState() == oboe::StreamState::Started; }
+  void Play() { std::atomic_store(&playing_, true); }
+
+  void Pause() { std::atomic_store(&playing_, false); }
+
+  bool IsPlaying() { return std::atomic_load(&playing_); }
 
   void AppendBuffer(uint8_t *data, int length) {
-    // Sync this constants with internal/readerdriver/driver.go
-    const size_t bytes_per_sample = channel_num_ * bit_depth_in_bytes_;
-    const size_t one_buffer_size = sample_rate_ * channel_num_ *
-                                   bit_depth_in_bytes_ / 4 / bytes_per_sample *
-                                   bytes_per_sample;
-    const size_t max_buffer_size = one_buffer_size * 2;
-
     std::lock_guard<std::mutex> lock(mutex_);
     buf_.insert(buf_.end(), data, data + length);
   }
 
-  const char *Play() {
+  size_t GetUnplayedBufferSize() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return buf_.size();
+  }
+
+  size_t Read(std::vector<float> &buf) {
+    if (!std::atomic_load(&playing_)) {
+      return 0;
+    }
+
+    const double volume = std::atomic_load(&volume_);
+    size_t copy_num = 0;
+    {
+      // TODO: Do not use a lock in onAudioReady.
+      // https://google.github.io/oboe/reference/classoboe_1_1_audio_stream_data_callback.html#ad8a3a9f609df5fd3a5d885cbe1b2204d
+      std::lock_guard<std::mutex> lock(mutex_);
+      copy_num = std::min(buf.size(), buf_.size() / 2);
+      for (size_t i = 0; i < copy_num; i++) {
+        int16_t v = static_cast<int16_t>(buf_[2 * i]) |
+                    (static_cast<int16_t>(buf_[2 * i + 1]) << 8);
+        buf[i] = static_cast<float>(v) / (1 << 15) * volume;
+      }
+      buf_.erase(buf_.begin(), buf_.begin() + copy_num * 2);
+    }
+
+    if (copy_num) {
+      ebiten_oboe_onWrittenCallback(go_player_);
+    }
+    return copy_num;
+  }
+
+private:
+  const uintptr_t go_player_;
+
+  std::atomic<double> volume_{1.0};
+  std::atomic<bool> playing_;
+  std::vector<uint8_t> buf_;
+  std::mutex mutex_;
+};
+
+Stream &Stream::GetInstance() {
+  static Stream *stream = new Stream();
+  return *stream;
+}
+
+const char *Stream::Play(int sample_rate, int channel_num,
+                         int bit_depth_in_bytes) {
+  sample_rate_ = sample_rate;
+  channel_num_ = channel_num;
+  bit_depth_in_bytes_ = bit_depth_in_bytes;
+
+  // TODO: Enable bit_depth_in_bytes_ == 1
   if (bit_depth_in_bytes_ != 2) {
     return "bit_depth_in_bytes_ must be 2 but not";
   }
 
-    int retry_count = 0;
-  retry:
   if (!stream_) {
     oboe::AudioStreamBuilder builder;
     oboe::Result result =
         builder.setDirection(oboe::Direction::Output)
             ->setPerformanceMode(oboe::PerformanceMode::LowLatency)
             ->setSharingMode(oboe::SharingMode::Shared)
-              ->setFormat(oboe::AudioFormat::I16)
+            ->setFormat(oboe::AudioFormat::Float)
             ->setChannelCount(channel_num_)
             ->setSampleRate(sample_rate_)
             ->setDataCallback(this)
             ->openStream(stream_);
     if (result != oboe::Result::OK) {
-        // openStream might fail with oboe::Result::ErrorInternal when too many
-        // streams are opened in a short time (#1645).
-        if (result == oboe::Result::ErrorInternal && retry_count < 100) {
-          retry_count++;
-          // Sleep 10[ms]
-          timespec ts = {};
-          ts.tv_nsec = 10000000;
-          nanosleep(&ts, nullptr);
-          goto retry;
-        }
       return oboe::convertToText(result);
     }
   }
@@ -121,7 +160,7 @@ public:
   return nullptr;
 }
 
-  const char *Pause() {
+const char *Stream::Pause() {
   if (!stream_) {
     return nullptr;
   }
@@ -131,7 +170,18 @@ public:
   return nullptr;
 }
 
-  const char *Close() {
+const char *Stream::Resume() {
+  if (!stream_) {
+    return "Play is not called yet at Resume";
+  }
+  if (oboe::Result result = stream_->start(); result != oboe::Result::OK) {
+    return oboe::convertToText(result);
+  }
+  return nullptr;
+}
+
+const char *Stream::Close() {
+  // Nobody calls this so far.
   if (!stream_) {
     return nullptr;
   }
@@ -145,90 +195,62 @@ public:
   return nullptr;
 }
 
-  const char *CloseAndRemove() {
+void Stream::AddPlayer(Player *player) {
-    // Close and remove self from the players atomically.
-    // Otherwise, a removed player might be resumed at Resume unexpectedly.
-    std::lock_guard<std::mutex> lock(PlayersMutex());
-    const char *msg = Close();
-    GetPlayers().erase(this);
-    GetPlayersToResume().erase(this);
-    return msg;
-  }
-
-  int GetUnplayedBufferSize() {
   std::lock_guard<std::mutex> lock(mutex_);
-    return buf_.size();
+  players_.insert(player);
 }
 
-  oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboe_stream,
+void Stream::RemovePlayer(Player *player) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  players_.erase(player);
+}
+
+oboe::DataCallbackResult Stream::onAudioReady(oboe::AudioStream *oboe_stream,
                                               void *audio_data,
-                                        int32_t num_frames) override {
+                                              int32_t num_frames) {
-    size_t num_bytes = num_frames * channel_num_ * bit_depth_in_bytes_;
+  size_t num = num_frames * channel_num_;
-    std::vector<uint8_t> buf(num_bytes);
+  std::vector<std::vector<float>> bufs;
   {
     // TODO: Do not use a lock in onAudioReady.
     // https://google.github.io/oboe/reference/classoboe_1_1_audio_stream_data_callback.html#ad8a3a9f609df5fd3a5d885cbe1b2204d
     std::lock_guard<std::mutex> lock(mutex_);
-      size_t copy_bytes = std::min(num_bytes, buf_.size());
+    bufs.resize(players_.size());
-      std::copy(buf_.begin(), buf_.begin() + copy_bytes, buf.begin());
+    size_t i = 0;
-      buf_.erase(buf_.begin(), buf_.begin() + copy_bytes);
+    for (Player *player : players_) {
-      onWrittenCallback(go_player_);
+      bufs[i].resize(num);
-    }
+      player->Read(bufs[i]);
-
+      i++;
-    if (const double volume = std::atomic_load(&volume_); volume < 1) {
-      for (int i = 0; i < buf.size() / 2; i++) {
-        int16_t v = static_cast<int16_t>(buf[2 * i]) |
-                    (static_cast<int16_t>(buf[2 * i + 1]) << 8);
-        v = static_cast<int16_t>(static_cast<double>(v) * volume);
-        buf[2 * i] = static_cast<uint8_t>(v);
-        buf[2 * i + 1] = static_cast<uint8_t>(v >> 8);
     }
   }
 
-    std::copy(buf.begin(), buf.end(), reinterpret_cast<uint8_t *>(audio_data));
+  float *dst = reinterpret_cast<float *>(audio_data);
+  for (int i = 0; i < num; i++) {
+    dst[i] = 0;
+    for (const std::vector<float> buf : bufs) {
+      dst[i] += buf[i];
+    }
+  }
   return oboe::DataCallbackResult::Continue;
 }
 
-private:
+Stream::Stream() = default;
-  static std::set<Player *> &GetPlayers() {
-    static std::set<Player *> players;
-    return players;
-  }
-
-  static std::set<Player *> &GetPlayersToResume() {
-    static std::set<Player *> players_to_resume;
-    return players_to_resume;
-  }
-
-  static std::mutex &PlayersMutex() {
-    static std::mutex mutex;
-    return mutex;
-  }
-
-  const int sample_rate_;
-  const int channel_num_;
-  const int bit_depth_in_bytes_;
-  const uintptr_t go_player_;
-
-  std::atomic<double> volume_{1.0};
-  std::vector<uint8_t> buf_;
-  std::mutex mutex_;
-  std::shared_ptr<oboe::AudioStream> stream_;
-};
 
 } // namespace
 
 extern "C" {
 
-const char *ebiten_oboe_Suspend() { return Player::Suspend(); }
+const char *ebiten_oboe_Play(int sample_rate, int channel_num,
+                             int bit_depth_in_bytes) {
+  return Stream::GetInstance().Play(sample_rate, channel_num,
+                                    bit_depth_in_bytes);
+}
 
-const char *ebiten_oboe_Resume() { return Player::Resume(); }
+const char *ebiten_oboe_Suspend() { return Stream::GetInstance().Pause(); }
 
-PlayerID ebiten_oboe_Player_Create(int sample_rate, int channel_num,
+const char *ebiten_oboe_Resume() { return Stream::GetInstance().Resume(); }
-                                   int bit_depth_in_bytes, double volume,
+
-                                   uintptr_t go_player) {
+PlayerID ebiten_oboe_Player_Create(double volume, uintptr_t go_player) {
-  Player *p = new Player(sample_rate, channel_num, bit_depth_in_bytes, volume,
+  Player *p = new Player(volume, go_player);
-                         go_player);
   return reinterpret_cast<PlayerID>(p);
 }
 
@@ -243,12 +265,12 @@ void ebiten_oboe_Player_AppendBuffer(PlayerID audio_player, uint8_t *data,
   p->AppendBuffer(data, length);
 }
 
-const char *ebiten_oboe_Player_Play(PlayerID audio_player) {
+void ebiten_oboe_Player_Play(PlayerID audio_player) {
   Player *p = reinterpret_cast<Player *>(audio_player);
-  return p->Play();
+  p->Play();
 }
 
-const char *ebiten_oboe_Player_Pause(PlayerID audio_player) {
+void ebiten_oboe_Player_Pause(PlayerID audio_player) {
   Player *p = reinterpret_cast<Player *>(audio_player);
   return p->Pause();
 }
@@ -258,11 +280,9 @@ void ebiten_oboe_Player_SetVolume(PlayerID audio_player, double volume) {
   p->SetVolume(volume);
 }
 
-const char *ebiten_oboe_Player_Close(PlayerID audio_player) {
+void ebiten_oboe_Player_Close(PlayerID audio_player) {
   Player *p = reinterpret_cast<Player *>(audio_player);
-  const char *msg = p->CloseAndRemove();
   delete p;
-  return msg;
 }
 
 int ebiten_oboe_Player_UnplayedBufferSize(PlayerID audio_player) {

audio/internal/oboe/binding_android.go

@@ -31,6 +31,13 @@ import (
 	"unsafe"
 )
 
+func Play(sampleRate, channelNum, bitDepthInBytes int) error {
+	if msg := C.ebiten_oboe_Play(C.int(sampleRate), C.int(channelNum), C.int(bitDepthInBytes)); msg != nil {
+		return fmt.Errorf("oboe: Play failed: %s", C.GoString(msg))
+	}
+	return nil
+}
+
 func Suspend() error {
 	if msg := C.ebiten_oboe_Suspend(); msg != nil {
 		return fmt.Errorf("oboe: Suspend failed: %s", C.GoString(msg))
@@ -54,17 +61,17 @@ type Player struct {
 	m sync.Mutex
 }
 
-func NewPlayer(sampleRate, channelNum, bitDepthInBytes int, volume float64, onWritten func()) *Player {
+func NewPlayer(volume float64, onWritten func()) *Player {
 	p := &Player{
 		onWritten: onWritten,
 	}
-	p.player = C.ebiten_oboe_Player_Create(C.int(sampleRate), C.int(channelNum), C.int(bitDepthInBytes), C.double(volume), C.uintptr_t(uintptr(unsafe.Pointer(p))))
+	p.player = C.ebiten_oboe_Player_Create(C.double(volume), C.uintptr_t(uintptr(unsafe.Pointer(p))))
 	runtime.SetFinalizer(p, (*Player).Close)
 	return p
 }
 
-//export onWrittenCallback
+//export ebiten_oboe_onWrittenCallback
-func onWrittenCallback(player C.uintptr_t) {
+func ebiten_oboe_onWrittenCallback(player C.uintptr_t) {
 	p := (*Player)(unsafe.Pointer(uintptr(player)))
 	p.onWritten()
 }
@@ -92,9 +99,7 @@ func (p *Player) Play() error {
 	if p.player == 0 {
 		return fmt.Errorf("oboe: player is already closed at Play")
 	}
-	if msg := C.ebiten_oboe_Player_Play(p.player); msg != nil {
+	C.ebiten_oboe_Player_Play(p.player)
-		return fmt.Errorf("oboe: Player_Play failed: %s", C.GoString(msg))
-	}
 	return nil
 }
 
@@ -105,9 +110,7 @@ func (p *Player) Pause() error {
 	if p.player == 0 {
 		return fmt.Errorf("oboe: player is already closed at Pause")
 	}
-	if msg := C.ebiten_oboe_Player_Pause(p.player); msg != nil {
+	C.ebiten_oboe_Player_Pause(p.player)
-		return fmt.Errorf("oboe: Player_Pause failed: %s", C.GoString(msg))
-	}
 	return nil
 }
 
@@ -125,9 +128,7 @@ func (p *Player) Close() error {
 	if p.player == 0 {
 		return fmt.Errorf("oboe: player is already closed at Close")
 	}
-	if msg := C.ebiten_oboe_Player_Close(p.player); msg != nil {
+	C.ebiten_oboe_Player_Close(p.player)
-		return fmt.Errorf("oboe: Player_Close failed: %s", C.GoString(msg))
-	}
 	p.player = 0
 	return nil
 }

audio/internal/oboe/binding_android.h

@@ -25,17 +25,17 @@ extern "C" {
 
 typedef uintptr_t PlayerID;
 
+const char *ebiten_oboe_Play(int sample_rate, int channel_num,
+                             int bit_depth_in_bytes);
 const char *ebiten_oboe_Suspend();
 const char *ebiten_oboe_Resume();
-PlayerID ebiten_oboe_Player_Create(int sample_rate, int channel_num,
+PlayerID ebiten_oboe_Player_Create(double volume, uintptr_t go_player);
-                                   int bit_depth_in_bytes, double volume,
-                                   uintptr_t go_player);
 bool ebiten_oboe_Player_IsPlaying(PlayerID audio_player);
 void ebiten_oboe_Player_AppendBuffer(PlayerID audio_player, uint8_t *data,
                                      int length);
-const char *ebiten_oboe_Player_Play(PlayerID audio_player);
+void ebiten_oboe_Player_Play(PlayerID audio_player);
-const char *ebiten_oboe_Player_Pause(PlayerID audio_player);
+void ebiten_oboe_Player_Pause(PlayerID audio_player);
-const char *ebiten_oboe_Player_Close(PlayerID audio_player);
+void ebiten_oboe_Player_Close(PlayerID audio_player);
 void ebiten_oboe_Player_SetVolume(PlayerID audio_player, double volume);
 int ebiten_oboe_Player_UnplayedBufferSize(PlayerID audio_player);
 

audio/internal/readerdriver/driver_android.go

@@ -18,14 +18,12 @@ import (
 	"io"
 	"runtime"
 	"sync"
-	"time"
 
 	"github.com/hajimehoshi/ebiten/v2/audio/internal/oboe"
 )
 
 func IsAvailable() bool {
-	// TODO: Enable this after #1660 is fixed
+	return true
-	return false
 }
 
 type context struct {
@@ -43,6 +41,9 @@ func NewContext(sampleRate int, channelNum int, bitDepthInBytes int) (Context, c
 		channelNum:      channelNum,
 		bitDepthInBytes: bitDepthInBytes,
 	}
+	if err := oboe.Play(sampleRate, channelNum, bitDepthInBytes); err != nil {
+		return nil, nil, err
+	}
 	return c, ready, nil
 }
 
@@ -120,7 +121,7 @@ func (p *player) playImpl() {
 	defer p.cond.Signal()
 	var runLoop bool
 	if p.p == nil {
-		p.p = oboe.NewPlayer(p.context.sampleRate, p.context.channelNum, p.context.bitDepthInBytes, p.volume, func() {
+		p.p = oboe.NewPlayer(p.volume, func() {
 			p.cond.Signal()
 		})
 		runLoop = true
@@ -161,7 +162,10 @@ func (p *player) IsPlaying() bool {
 func (p *player) Reset() {
 	p.cond.L.Lock()
 	defer p.cond.L.Unlock()
+	p.resetImpl()
+}
 
+func (p *player) resetImpl() {
 	if p.err != nil {
 		return
 	}
@@ -298,12 +302,8 @@ func (p *player) loop() {
 
 		// Now p.resetImpl() doesn't close the stream gracefully. Then buffer size check is necessary here.
 		if p.eof && p.p.UnplayedBufferSize() == 0 {
-			// Even when the unplayed buffer size is 0,
+			p.resetImpl()
-			// the audio data in the hardware might not be played yet (#1632).
-			// Just wait for a while.
 			p.cond.L.Unlock()
-			time.Sleep(100 * time.Millisecond)
-			p.Reset()
 			return
 		}
 		p.cond.L.Unlock()