From 43f505b3a086b346bfc73c82dc96e659a9a147d6 Mon Sep 17 00:00:00 2001 From: Hajime Hoshi Date: Sun, 7 Jul 2024 18:55:37 +0900 Subject: [PATCH] audio: use float32 format under the hood Updates #2160 --- audio/audio.go | 15 +++-- audio/audio_test.go | 18 +++++ audio/context.go | 2 +- audio/internal/convert/float32.go | 93 ++++++++++++++++++++++++++ audio/internal/convert/float32_test.go | 84 +++++++++++++++++++++++ audio/player.go | 25 ++++--- 6 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 audio/internal/convert/float32.go create mode 100644 audio/internal/convert/float32_test.go diff --git a/audio/audio.go b/audio/audio.go index 4f749588e..b3729fb91 100644 --- a/audio/audio.go +++ b/audio/audio.go @@ -46,8 +46,9 @@ import ( ) const ( - channelCount = 2 - bitDepthInBytesInt16 = 2 + channelCount = 2 + bitDepthInBytesInt16 = 2 + bitDepthInBytesFloat32 = 4 ) // A Context represents a current state of audio. @@ -188,13 +189,13 @@ func (c *Context) addPlayingPlayer(p *playerImpl) { c.playingPlayers[p] = struct{}{} // Check the source duplication - srcs := map[io.Reader]struct{}{} + srcs := map[any]struct{}{} for p := range c.playingPlayers { - if _, ok := srcs[p.source()]; ok { + if _, ok := srcs[p.sourceIdent()]; ok { c.err = errors.New("audio: a same source is used by multiple Player") return } - srcs[p.source()] = struct{}{} + srcs[p.sourceIdent()] = struct{}{} } } @@ -323,7 +324,9 @@ type Player struct { // A Player doesn't close src even if src implements io.Closer. // Closing the source is src owner's responsibility. func (c *Context) NewPlayer(src io.Reader) (*Player, error) { - pi, err := c.playerFactory.newPlayer(c, src, bitDepthInBytesInt16) + _, seekable := src.(io.Seeker) + f32Src := convert.NewFloat32BytesReaderFromInt16BytesReader(src) + pi, err := c.playerFactory.newPlayer(c, f32Src, seekable, src, bitDepthInBytesFloat32) if err != nil { return nil, err } diff --git a/audio/audio_test.go b/audio/audio_test.go index d3bdf7078..e1d1bfd17 100644 --- a/audio/audio_test.go +++ b/audio/audio_test.go @@ -116,3 +116,21 @@ func TestPauseBeforeInit(t *testing.T) { t.Error(err) } } + +type emptySource struct{} + +func (emptySource) Read(buf []byte) (int, error) { + return len(buf), nil +} + +func TestNonSeekableSource(t *testing.T) { + setup() + defer teardown() + + p, err := context.NewPlayer(emptySource{}) + if err != nil { + t.Fatal(err) + } + + p.Play() +} diff --git a/audio/context.go b/audio/context.go index 37002b102..72c48fa90 100644 --- a/audio/context.go +++ b/audio/context.go @@ -24,7 +24,7 @@ func newContext(sampleRate int) (context, chan struct{}, error) { ctx, ready, err := oto.NewContext(&oto.NewContextOptions{ SampleRate: sampleRate, ChannelCount: channelCount, - Format: oto.FormatSignedInt16LE, + Format: oto.FormatFloat32LE, }) err = addErrorInfoForContextCreation(err) return &contextProxy{ctx}, ready, err diff --git a/audio/internal/convert/float32.go b/audio/internal/convert/float32.go new file mode 100644 index 000000000..bf75b079d --- /dev/null +++ b/audio/internal/convert/float32.go @@ -0,0 +1,93 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package convert + +import ( + "io" + "math" +) + +func NewFloat32BytesReaderFromInt16BytesReader(r io.Reader) io.Reader { + return &float32BytesReader{r: r} +} + +type float32BytesReader struct { + r io.Reader + eof bool + i16Buf []byte +} + +func (r *float32BytesReader) Read(buf []byte) (int, error) { + if r.eof && len(r.i16Buf) == 0 { + return 0, io.EOF + } + + if i16LenToFill := len(buf) / 4 * 2; len(r.i16Buf) < i16LenToFill && !r.eof { + origLen := len(r.i16Buf) + if cap(r.i16Buf) < i16LenToFill { + r.i16Buf = append(r.i16Buf, make([]byte, i16LenToFill-origLen)...) + } + + // Read int16 bytes. + n, err := r.r.Read(r.i16Buf[origLen:i16LenToFill]) + if err != nil && err != io.EOF { + return 0, err + } + if err == io.EOF { + r.eof = true + } + r.i16Buf = r.i16Buf[:origLen+n] + } + + // Convert int16 bytes to float32 bytes and fill buf. + samplesToFill := min(len(r.i16Buf)/2, len(buf)/4) + for i := 0; i < samplesToFill; i++ { + vi16l := r.i16Buf[2*i] + vi16h := r.i16Buf[2*i+1] + v := float32(int16(vi16l)|int16(vi16h)<<8) / (1 << 15) + vf32 := math.Float32bits(v) + buf[4*i] = byte(vf32) + buf[4*i+1] = byte(vf32 >> 8) + buf[4*i+2] = byte(vf32 >> 16) + buf[4*i+3] = byte(vf32 >> 24) + } + + // Copy the remaining part for the next read. + copy(r.i16Buf, r.i16Buf[samplesToFill*2:]) + r.i16Buf = r.i16Buf[:len(r.i16Buf)-samplesToFill*2] + + return samplesToFill * 4, nil +} + +func (r *float32BytesReader) Seek(offset int64, whence int) (int64, error) { + s, ok := r.r.(io.Seeker) + if !ok { + panic("float32: the source must be io.Seeker when seeking but not") + } + r.i16Buf = r.i16Buf[:0] + r.eof = false + n, err := s.Seek(offset/4*2, whence) + if err != nil { + return 0, err + } + return n / 2 * 4, nil +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/audio/internal/convert/float32_test.go b/audio/internal/convert/float32_test.go new file mode 100644 index 000000000..91c92b567 --- /dev/null +++ b/audio/internal/convert/float32_test.go @@ -0,0 +1,84 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package convert_test + +import ( + "bytes" + "crypto/rand" + "io" + "testing" + "unsafe" + + "github.com/hajimehoshi/ebiten/v2/audio/internal/convert" +) + +func randInt16s(n int) []int16 { + r := make([]int16, n) + if _, err := rand.Read(unsafe.Slice((*byte)(unsafe.Pointer(&r[0])), len(r)*2)); err != nil { + panic(err) + } + return r +} + +func TestFloat32(t *testing.T) { + cases := []struct { + In []int16 + }{ + { + In: nil, + }, + { + In: []int16{-32768, 0, 32767}, + }, + { + In: []int16{0, 0, 0, 0, 0, 0, 0, 0}, + }, + { + In: randInt16s(256), + }, + { + In: randInt16s(65536), + }, + } + for _, c := range cases { + // Note that unsafe.SliceData is available as of Go 1.20. + var in, out []byte + if len(c.In) > 0 { + outF32 := make([]float32, len(c.In)) + for i := range c.In { + outF32[i] = float32(c.In[i]) / (1 << 15) + } + in = unsafe.Slice((*byte)(unsafe.Pointer(&c.In[0])), len(c.In)*2) + out = unsafe.Slice((*byte)(unsafe.Pointer(&outF32[0])), len(outF32)*4) + } + r := convert.NewFloat32BytesReaderFromInt16BytesReader(bytes.NewReader(in)) + var got []byte + for { + var buf [97]byte + n, err := r.Read(buf[:]) + got = append(got, buf[:n]...) + if err != nil { + if err != io.EOF { + t.Fatal(err) + } + break + } + } + want := out + if !bytes.Equal(got, want) { + t.Errorf("got: %v, want: %v", got, want) + } + } +} diff --git a/audio/player.go b/audio/player.go index 1e52856f3..252c182b0 100644 --- a/audio/player.go +++ b/audio/player.go @@ -66,6 +66,8 @@ type playerImpl struct { context *Context player player src io.Reader + seekable bool + srcIdent any stream *timeStream factory *playerFactory initBufferSize int @@ -86,12 +88,14 @@ type playerImpl struct { m sync.Mutex } -func (f *playerFactory) newPlayer(context *Context, src io.Reader, bitDepthInBytes int) (*playerImpl, error) { +func (f *playerFactory) newPlayer(context *Context, src io.Reader, seekable bool, srcIdent any, bitDepthInBytes int) (*playerImpl, error) { f.m.Lock() defer f.m.Unlock() p := &playerImpl{ src: src, + seekable: seekable, + srcIdent: srcIdent, context: context, factory: f, lastSamples: -1, @@ -165,7 +169,7 @@ func (p *playerImpl) ensurePlayer() error { } if p.stream == nil { - s, err := newTimeStream(p.src, p.factory.sampleRate, p.bytesPerSample/channelCount) + s, err := newTimeStream(p.src, p.seekable, p.factory.sampleRate, p.bytesPerSample/channelCount) if err != nil { return err } @@ -324,8 +328,8 @@ func (p *playerImpl) SetBufferSize(bufferSize time.Duration) { p.player.SetBufferSize(bufferSizeInBytes) } -func (p *playerImpl) source() io.Reader { - return p.src +func (p *playerImpl) sourceIdent() any { + return p.srcIdent } func (p *playerImpl) onContextSuspended() { @@ -384,6 +388,7 @@ func (p *playerImpl) updatePosition() { type timeStream struct { r io.Reader + seekable bool sampleRate int pos int64 bytesPerSample int @@ -393,15 +398,16 @@ type timeStream struct { m sync.Mutex } -func newTimeStream(r io.Reader, sampleRate int, bitDepthInBytes int) (*timeStream, error) { +func newTimeStream(r io.Reader, seekable bool, sampleRate int, bitDepthInBytes int) (*timeStream, error) { s := &timeStream{ r: r, + seekable: seekable, sampleRate: sampleRate, bytesPerSample: bitDepthInBytes * channelCount, } - if seeker, ok := s.r.(io.Seeker); ok { + if seekable { // Get the current position of the source. - pos, err := seeker.Seek(0, io.SeekCurrent) + pos, err := s.r.(io.Seeker).Seek(0, io.SeekCurrent) if err != nil { return nil, err } @@ -423,12 +429,11 @@ func (s *timeStream) Seek(offset int64, whence int) (int64, error) { s.m.Lock() defer s.m.Unlock() - seeker, ok := s.r.(io.Seeker) - if !ok { + if !s.seekable { // TODO: Should this return an error? panic("audio: the source must be io.Seeker when seeking but not") } - pos, err := seeker.Seek(offset, whence) + pos, err := s.r.(io.Seeker).Seek(offset, whence) if err != nil { return pos, err }