audio: Implement the reader player for browsers

Updates #1458

audio/readerplayer.go

@@ -15,11 +15,29 @@
 package audio
 
 import (
+	"fmt"
 	"io"
 	"sync"
 	"time"
 )
 
+// TODO: The term 'buffer' is confusing. Name each buffer with good terms.
+
+// oneBufferSize returns the size of one buffer in the player implementation.
+func oneBufferSize(sampleRate int) int {
+	return sampleRate * channelNum * bitDepthInBytes / 4
+}
+
+// maxBufferSize returns the maximum size of the buffer for the audio source.
+// This buffer is used when unreading on pausing the player.
+func maxBufferSize(sampleRate int) int {
+	// Actually *2 should be enough in most cases,
+	// but in some implementation (e.g, go2cpp), a player might have more UnplayedBufferSize values.
+	// As a safe margin, use *4 value.
+	// TODO: Ensure the maximum value of UnplayedBufferSize on all the platforms.
+	return oneBufferSize(sampleRate) * 4
+}
+
 // readerDriver represents a driver using io.ReadClosers.
 type readerDriver interface {
 	NewPlayer(io.Reader) readerDriverPlayer
@@ -72,25 +90,32 @@ func (f *readerPlayerFactory) newPlayerImpl(context *Context, src io.Reader) (pl
 	return p, nil
 }
 
-func (p *readerPlayer) ensurePlayer() {
+func (p *readerPlayer) ensurePlayer() error {
 	// Initialize the underlying player lazily to enable calling NewContext in an 'init' function.
 	// Accessing the underlying player functions requires the environment to be already initialized,
 	// but if Ebiten is used for a shared library, the timing when init functions are called
 	// is unexpectable.
 	// e.g. a variable for JVM on Android might not be set.
 	if p.factory.driver == nil {
-		p.factory.driver = newReaderDriverImpl(p.factory.sampleRate)
+		d, err := newReaderDriverImpl(p.context)
+		if err != nil {
+			return err
+		}
+		p.factory.driver = d
 	}
 	if p.player == nil {
 		p.player = p.factory.driver.NewPlayer(p.src)
 	}
-	// TODO: If some error happens, call p.context.setError().
+	return nil
 }
 
 func (p *readerPlayer) Play() {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return
+	}
 
 	p.player.Play()
 	p.context.addPlayer(p)
@@ -99,15 +124,23 @@ func (p *readerPlayer) Play() {
 func (p *readerPlayer) Pause() {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return
+	}
 
+	n := p.player.UnplayedBufferSize()
 	p.player.Pause()
+	p.src.Unread(int(n))
 }
 
 func (p *readerPlayer) IsPlaying() bool {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return false
+	}
 
 	return p.player.IsPlaying()
 }
@@ -115,7 +148,10 @@ func (p *readerPlayer) IsPlaying() bool {
 func (p *readerPlayer) Volume() float64 {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return 0
+	}
 
 	return p.player.Volume()
 }
@@ -123,7 +159,10 @@ func (p *readerPlayer) Volume() float64 {
 func (p *readerPlayer) SetVolume(volume float64) {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return
+	}
 
 	p.player.SetVolume(volume)
 }
@@ -131,16 +170,21 @@ func (p *readerPlayer) SetVolume(volume float64) {
 func (p *readerPlayer) Close() error {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
 
 	p.context.removePlayer(p)
-	return p.player.Close()
+	if p.player != nil {
+		return p.player.Close()
+	}
+	return nil
 }
 
 func (p *readerPlayer) Current() time.Duration {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
+	if err := p.ensurePlayer(); err != nil {
+		p.context.setError(err)
+		return 0
+	}
 
 	sample := (p.src.Current() - p.player.UnplayedBufferSize()) / bytesPerSample
 	return time.Duration(sample) * time.Second / time.Duration(p.factory.sampleRate)
@@ -153,14 +197,15 @@ func (p *readerPlayer) Rewind() error {
 func (p *readerPlayer) Seek(offset time.Duration) error {
 	p.m.Lock()
 	defer p.m.Unlock()
-	p.ensurePlayer()
 
-	if p.player.IsPlaying() {
-		defer func() {
-			p.player.Play()
-		}()
+	if p.player != nil {
+		if p.player.IsPlaying() {
+			defer func() {
+				p.player.Play()
+			}()
+		}
+		p.player.Reset()
 	}
-	p.player.Reset()
 	return p.src.Seek(offset)
 }
 
@@ -172,6 +217,8 @@ type timeStream struct {
 	r          io.Reader
 	sampleRate int
 	pos        int64
+	buf        []byte
+	unread     int
 }
 
 func newTimeStream(r io.Reader, sampleRate int) (*timeStream, error) {
@@ -190,9 +237,28 @@ func newTimeStream(r io.Reader, sampleRate int) (*timeStream, error) {
 	return s, nil
 }
 
+func (s *timeStream) Unread(n int) {
+	if s.unread+n > len(s.buf) {
+		panic(fmt.Sprintf("audio: too much unreading: %d, the buffer size: %d, unreading position: %d", n, len(s.buf), s.unread))
+	}
+	s.unread += n
+	s.pos -= int64(n)
+}
+
 func (s *timeStream) Read(buf []byte) (int, error) {
+	if s.unread > 0 {
+		n := copy(buf, s.buf[len(s.buf)-s.unread:])
+		s.unread -= n
+		s.pos += int64(n)
+		return n, nil
+	}
+
 	n, err := s.r.Read(buf)
 	s.pos += int64(n)
+	s.buf = append(s.buf, buf[:n]...)
+	if m := maxBufferSize(s.sampleRate); len(s.buf) > m {
+		s.buf = s.buf[len(s.buf)-m:]
+	}
 	return n, err
 }
 
@@ -213,6 +279,8 @@ func (s *timeStream) Seek(offset time.Duration) error {
 	}
 
 	s.pos = pos
+	s.buf = s.buf[:0]
+	s.unread = 0
 	return nil
 }
 

audio/readerplayer_default.go

@@ -25,6 +25,6 @@ func isReaderContextAvailable() bool {
 	return false
 }
 
-func newReaderDriverImpl(sampleRate int) readerDriver {
+func newReaderDriverImpl(context *Context) (readerDriver, error) {
 	panic(fmt.Sprintf("audio: newReaderDriver is not available on this environment: GOOS=%s", runtime.GOOS))
 }

audio/readerplayer_js.go

@@ -15,18 +15,234 @@
 package audio
 
 import (
+	"errors"
 	"io"
+	"reflect"
+	"runtime"
 	"syscall/js"
+	"unsafe"
 
 	"github.com/hajimehoshi/ebiten/v2/audio/internal/go2cpp"
 )
 
 func isReaderContextAvailable() bool {
-	return js.Global().Get("go2cpp").Truthy()
+	return true
 }
 
-func newReaderDriverImpl(sampleRate int) readerDriver {
-	return &go2cppDriverWrapper{go2cpp.NewContext(sampleRate, channelNum, bitDepthInBytes)}
+type readerDriverImpl struct {
+	context      *Context
+	audioContext js.Value
+	ready        bool
+	callbacks    map[string]js.Func
+}
+
+func newReaderDriverImpl(context *Context) (readerDriver, error) {
+	if js.Global().Get("go2cpp").Truthy() {
+		return &go2cppDriverWrapper{go2cpp.NewContext(context.sampleRate, channelNum, bitDepthInBytes)}, nil
+	}
+
+	class := js.Global().Get("AudioContext")
+	if !class.Truthy() {
+		class = js.Global().Get("webkitAudioContext")
+	}
+	if !class.Truthy() {
+		return nil, errors.New("audio: AudioContext or webkitAudioContext was not found")
+	}
+	options := js.Global().Get("Object").New()
+	options.Set("sampleRate", context.sampleRate)
+
+	d := &readerDriverImpl{
+		context:      context,
+		audioContext: class.New(options),
+	}
+
+	setCallback := func(event string) js.Func {
+		var f js.Func
+		f = js.FuncOf(func(this js.Value, arguments []js.Value) interface{} {
+			if !d.ready {
+				d.audioContext.Call("resume")
+				d.ready = true
+			}
+			js.Global().Get("document").Call("removeEventListener", event, f)
+			return nil
+		})
+		js.Global().Get("document").Call("addEventListener", event, f)
+		d.callbacks[event] = f
+		return f
+	}
+
+	// Browsers require user interaction to start the audio.
+	// https://developers.google.com/web/updates/2017/09/autoplay-policy-changes#webaudio
+	d.callbacks = map[string]js.Func{}
+	setCallback("touchend")
+	setCallback("keyup")
+	setCallback("mouseup")
+
+	return d, nil
+}
+
+type readerPlayerState int
+
+const (
+	readerPlayerPaused readerPlayerState = iota
+	readerPlayerPlay
+	readerPlayerClosed
+)
+
+type readerDriverPlayerImpl struct {
+	driver *readerDriverImpl
+	src    io.Reader
+	eof    bool
+	state  readerPlayerState
+	gain   js.Value
+
+	nextPos           float64
+	bufferSourceNodes []js.Value
+	appendBufferFunc  js.Func
+}
+
+func (d *readerDriverImpl) NewPlayer(src io.Reader) readerDriverPlayer {
+	p := &readerDriverPlayerImpl{
+		driver: d,
+		src:    src,
+		gain:   d.audioContext.Call("createGain"),
+	}
+	p.appendBufferFunc = js.FuncOf(p.appendBuffer)
+	p.gain.Call("connect", d.audioContext.Get("destination"))
+	runtime.SetFinalizer(p, (*readerDriverPlayerImpl).Close)
+	return p
+}
+
+func (d *readerDriverImpl) Close() error {
+	// TODO: Implement this
+	return nil
+}
+
+func (p *readerDriverPlayerImpl) Pause() {
+	if p.state != readerPlayerPlay {
+		return
+	}
+
+	// Change the state first. appendBuffer is called as an 'ended' callback.
+	for _, n := range p.bufferSourceNodes {
+		n.Set("onended", nil)
+		n.Call("stop")
+		n.Call("disconnect")
+	}
+	p.state = readerPlayerPaused
+	p.bufferSourceNodes = p.bufferSourceNodes[:0]
+	p.nextPos = 0
+}
+
+func (p *readerDriverPlayerImpl) appendBuffer(this js.Value, args []js.Value) interface{} {
+	// appendBuffer is called as the 'ended' callback of a buffer.
+	// 'this' is an AudioBufferSourceNode that already finishes its playing.
+	for i, n := range p.bufferSourceNodes {
+		if n.Equal(this) {
+			p.bufferSourceNodes = append(p.bufferSourceNodes[:i], p.bufferSourceNodes[i+1:]...)
+			break
+		}
+	}
+
+	if p.state != readerPlayerPlay {
+		return nil
+	}
+
+	if p.eof {
+		if len(p.bufferSourceNodes) == 0 {
+			p.Pause()
+		}
+		return nil
+	}
+
+	c := p.driver.audioContext.Get("currentTime").Float()
+	if p.nextPos < c {
+		// The exact current time might be too early. Add some delay on purpose to avoid buffer overlapping.
+		p.nextPos = c + 1.0/60.0
+	}
+
+	bs := make([]byte, oneBufferSize(p.driver.context.sampleRate))
+	n, err := io.ReadFull(p.src, bs)
+	if err != nil {
+		if err != io.EOF && err != io.ErrUnexpectedEOF {
+			p.driver.context.setError(err)
+			return nil
+		}
+		p.eof = true
+	}
+	if n == 0 {
+		return nil
+	}
+	bs = bs[:n]
+	l, r := toLR(bs)
+	tl, tr := float32SliceToTypedArray(l), float32SliceToTypedArray(r)
+
+	buf := p.driver.audioContext.Call("createBuffer", channelNum, len(bs)/channelNum/bitDepthInBytes, p.driver.context.sampleRate)
+	if buf.Get("copyToChannel").Truthy() {
+		buf.Call("copyToChannel", tl, 0, 0)
+		buf.Call("copyToChannel", tr, 1, 0)
+	} else {
+		// copyToChannel is not defined on Safari 11.
+		buf.Call("getChannelData", 0).Call("set", tl)
+		buf.Call("getChannelData", 1).Call("set", tr)
+	}
+
+	s := p.driver.audioContext.Call("createBufferSource")
+	s.Set("buffer", buf)
+	s.Set("onended", p.appendBufferFunc)
+	s.Call("connect", p.gain)
+	s.Call("start", p.nextPos)
+	p.nextPos += buf.Get("duration").Float()
+	p.bufferSourceNodes = append(p.bufferSourceNodes, s)
+
+	return nil
+}
+
+func (p *readerDriverPlayerImpl) Play() {
+	if p.state != readerPlayerPaused {
+		return
+	}
+	p.state = readerPlayerPlay
+	p.appendBuffer(js.Undefined(), nil)
+	p.appendBuffer(js.Undefined(), nil)
+}
+
+func (p *readerDriverPlayerImpl) IsPlaying() bool {
+	return p.state == readerPlayerPlay
+}
+
+func (p *readerDriverPlayerImpl) Reset() {
+	if p.state == readerPlayerClosed {
+		return
+	}
+
+	p.Pause()
+	p.eof = false
+}
+
+func (p *readerDriverPlayerImpl) Volume() float64 {
+	return p.gain.Get("gain").Get("value").Float()
+}
+
+func (p *readerDriverPlayerImpl) SetVolume(volume float64) {
+	p.gain.Get("gain").Set("value", volume)
+}
+
+func (p *readerDriverPlayerImpl) UnplayedBufferSize() int64 {
+	// This is not an accurate buffer size as part of the buffers might already be consumed.
+	var sec float64
+	for _, n := range p.bufferSourceNodes {
+		sec += n.Get("buffer").Get("duration").Float()
+	}
+	return int64(sec * float64(p.driver.context.sampleRate*channelNum*bitDepthInBytes))
+}
+
+func (p *readerDriverPlayerImpl) Close() error {
+	runtime.SetFinalizer(p, nil)
+	p.Reset()
+	p.state = readerPlayerClosed
+	p.appendBufferFunc.Release()
+	return nil
 }
 
 type go2cppDriverWrapper struct {
@@ -40,3 +256,28 @@ func (w *go2cppDriverWrapper) NewPlayer(r io.Reader) readerDriverPlayer {
 func (w *go2cppDriverWrapper) Close() error {
 	return w.c.Close()
 }
+
+func toLR(data []byte) ([]float32, []float32) {
+	const max = 1 << 15
+
+	l := make([]float32, len(data)/4)
+	r := make([]float32, len(data)/4)
+	for i := 0; i < len(data)/4; i++ {
+		l[i] = float32(int16(data[4*i])|int16(data[4*i+1])<<8) / max
+		r[i] = float32(int16(data[4*i+2])|int16(data[4*i+3])<<8) / max
+	}
+	return l, r
+}
+
+func float32SliceToTypedArray(s []float32) js.Value {
+	h := (*reflect.SliceHeader)(unsafe.Pointer(&s))
+	h.Len *= 4
+	h.Cap *= 4
+	bs := *(*[]byte)(unsafe.Pointer(h))
+
+	a := js.Global().Get("Uint8Array").New(len(bs))
+	js.CopyBytesToJS(a, bs)
+	runtime.KeepAlive(s)
+	buf := a.Get("buffer")
+	return js.Global().Get("Float32Array").New(buf, a.Get("byteOffset"), a.Get("byteLength").Int()/4)
+}