diff --git a/examples/texti18n/main.go b/examples/texti18n/main.go index 0ff389ced..779490dc3 100644 --- a/examples/texti18n/main.go +++ b/examples/texti18n/main.go @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// This example is a demonstration to render languages that cannot be rendered with the `text` package. -// We plan to provide a useful API to render them more easily (#2454). Stay tuned! - package main import ( diff --git a/examples/video/license.md b/examples/video/license.md new file mode 100644 index 000000000..087619688 --- /dev/null +++ b/examples/video/license.md @@ -0,0 +1,9 @@ +# License + +## `shibuya.mpg`, `shibuya_noaudio.mpg` + +https://commons.wikimedia.org/wiki/File:Shibuya_Crossing,_Tokyo,_Japan_(video).webm + +"Shibuya Crossing, Tokyo, Japan (video).webm" by Basile Morin + +The Creative Commons Attribution-Share Alike 4.0 International license diff --git a/examples/video/main.go b/examples/video/main.go new file mode 100644 index 000000000..4c9f66661 --- /dev/null +++ b/examples/video/main.go @@ -0,0 +1,98 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "log" + "os" + + "github.com/hajimehoshi/ebiten/v2" + "github.com/hajimehoshi/ebiten/v2/audio" + "github.com/hajimehoshi/ebiten/v2/ebitenutil" +) + +//go:embed shibuya.mpg +var shibuya_mpg []byte + +type Game struct { + player *mpegPlayer + err error +} + +func (g *Game) Update() error { + if g.err != nil { + return g.err + } + return nil +} + +func (g *Game) Draw(screen *ebiten.Image) { + if g.err != nil { + return + } + if err := g.player.Draw(screen); err != nil { + g.err = err + } + ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.ActualFPS())) +} + +func (g *Game) Layout(outsideWidth, outsideHeight int) (screenWidth, screenHeight int) { + return outsideWidth, outsideHeight +} + +func main() { + // Initialize audio context. + _ = audio.NewContext(48000) + + // If you want to play your own video, the video must be an MPEG-1 video with 48000 audio sample rate. + // You can convert the video to MPEG-1 with the below command: + // + // ffmpeg -i YOUR_VIDEO -c:v mpeg1video -q:v 8 -c:a mp2 -format mpeg -ar 48000 output.mpg + // + // You can adjust quality by changing -q:v value. A lower value indicates better quality. + var in io.ReadSeeker + if len(os.Args) > 1 { + f, err := os.Open(os.Args[1]) + if err != nil { + log.Fatal(err) + } + defer func() { + _ = f.Close() + }() + in = f + } else { + in = bytes.NewReader(shibuya_mpg) + fmt.Println("Play the default video. You can specify a video file as an argument.") + } + + player, err := newMPEGPlayer(in) + if err != nil { + log.Fatal(err) + } + g := &Game{ + player: player, + } + player.Play() + + ebiten.SetWindowTitle("Video (Ebitengine Demo)") + ebiten.SetWindowResizingMode(ebiten.WindowResizingModeEnabled) + if err := ebiten.RunGame(g); err != nil { + log.Fatal(err) + } +} diff --git a/examples/video/mpegplayer.go b/examples/video/mpegplayer.go new file mode 100644 index 000000000..9a1fa948d --- /dev/null +++ b/examples/video/mpegplayer.go @@ -0,0 +1,292 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "image" + "io" + "sync" + "time" + + "github.com/gen2brain/mpeg" + "github.com/hajimehoshi/ebiten/v2" + "github.com/hajimehoshi/ebiten/v2/audio" +) + +type mpegPlayer struct { + mpg *mpeg.MPEG + + // yCbCrImage is the current frame image in YCbCr format. + // An MPEG frame is stored in this image first. + // Then, this image data is converted to RGB to frameImage. + yCbCrImage *ebiten.Image + + // yCbCrBytes is the byte slice to store YCbCr data. + // This includes Y, Cb, Cr, and alpha (always 0xff) data for each pixel. + yCbCrBytes []byte + + // yCbCrShader is the shader to convert YCbCr to RGB. + yCbCrShader *ebiten.Shader + + // frameImage is the current frame image in RGB format. + frameImage *ebiten.Image + + audioPlayer *audio.Player + + // These members are used when the video doesn't have an audio stream. + refTime time.Time + + m sync.Mutex +} + +func newMPEGPlayer(src io.Reader) (*mpegPlayer, error) { + mpg, err := mpeg.New(src) + if err != nil { + return nil, err + } + if mpg.NumVideoStreams() == 0 { + return nil, fmt.Errorf("video: no video streams") + } + if !mpg.HasHeaders() { + return nil, fmt.Errorf("video: missing headers") + } + + p := &mpegPlayer{ + mpg: mpg, + yCbCrImage: ebiten.NewImage(mpg.Width(), mpg.Height()), + yCbCrBytes: make([]byte, 4*mpg.Width()*mpg.Height()), + frameImage: ebiten.NewImage(mpg.Width(), mpg.Height()), + } + + s, err := ebiten.NewShader([]byte(`package main + +//kage:unit pixels + +func Fragment(dstPos vec4, srcPos vec2, color vec4) vec4 { + // For this calculation, see the comment in the standard library color.YCbCrToRGB function. + c := imageSrc0UnsafeAt(srcPos) + return vec4( + c.x + 1.40200 * (c.z-0.5), + c.x - 0.34414 * (c.y-0.5) - 0.71414 * (c.z-0.5), + c.x + 1.77200 * (c.y-0.5), + 1, + ) +} +`)) + if err != nil { + return nil, err + } + p.yCbCrShader = s + + // If the video doesn't have an audio stream, initialization is done. + if mpg.NumAudioStreams() == 0 { + return p, nil + } + + // If the video has an audio stream, initialize an audio player. + ctx := audio.CurrentContext() + if ctx == nil { + return nil, fmt.Errorf("video: audio.Context is not initialized") + } + if mpg.Channels() != 2 { + return nil, fmt.Errorf("video: mpeg audio stream must be 2 but was %d", mpg.Channels()) + } + if ctx.SampleRate() != mpg.Samplerate() { + return nil, fmt.Errorf("video: mpeg audio stream sample rate %d doesn't match with audio context sample rate %d", mpg.Samplerate(), ctx.SampleRate()) + } + + mpg.SetAudioFormat(mpeg.AudioS16) + + audioPlayer, err := ctx.NewPlayer(&mpegAudio{ + audio: mpg.Audio(), + m: &p.m, + }) + if err != nil { + return nil, err + } + p.audioPlayer = audioPlayer + + return p, nil +} + +// updateFrame upadtes the current video frame. +func (p *mpegPlayer) updateFrame() error { + p.m.Lock() + defer p.m.Unlock() + + var pos float64 + if p.audioPlayer != nil { + pos = p.audioPlayer.Position().Seconds() + } else { + if p.refTime != (time.Time{}) { + pos = time.Since(p.refTime).Seconds() + } + } + + video := p.mpg.Video() + if video.HasEnded() { + p.frameImage.Clear() + return nil + } + + d := 1 / p.mpg.Framerate() + var mpegFrame *mpeg.Frame + for video.Time()+d <= pos && !video.HasEnded() { + mpegFrame = video.Decode() + } + + if mpegFrame == nil { + return nil + } + + img := mpegFrame.YCbCr() + if img.SubsampleRatio != image.YCbCrSubsampleRatio420 { + return fmt.Errorf("video: subsample ratio must be 4:2:0") + } + w, h := p.mpg.Width(), p.mpg.Height() + for j := 0; j < h; j++ { + yi := j * img.YStride + ci := (j / 2) * img.CStride + // Create temporary slices to encourage BCE (boundary-checking elimination). + ys := img.Y[yi : yi+w] + cbs := img.Cb[ci : ci+w/2] + crs := img.Cr[ci : ci+w/2] + for i := 0; i < w; i++ { + idx := 4 * (j*w + i) + buf := p.yCbCrBytes[idx : idx+3] + buf[0] = ys[i] + buf[1] = cbs[i/2] + buf[2] = crs[i/2] + // p.yCbCrBytes[3] = 0xff is not needed as the shader ignores this part. + } + } + + p.yCbCrImage.WritePixels(p.yCbCrBytes) + + // Converting YCbCr to RGB on CPU is slow. Use a shader instead. + op := &ebiten.DrawRectShaderOptions{} + op.Images[0] = p.yCbCrImage + op.Blend = ebiten.BlendCopy + p.frameImage.DrawRectShader(w, h, p.yCbCrShader, op) + + return nil +} + +// Draw draws the current frame onto the given screen. +func (p *mpegPlayer) Draw(screen *ebiten.Image) error { + if err := p.updateFrame(); err != nil { + return err + } + + frame := p.frameImage + sw, sh := screen.Bounds().Dx(), screen.Bounds().Dy() + fw, fh := frame.Bounds().Dx(), frame.Bounds().Dy() + + op := ebiten.DrawImageOptions{} + wf, hf := float64(sw)/float64(fw), float64(sh)/float64(fh) + s := wf + if hf < wf { + s = hf + } + op.GeoM.Scale(s, s) + + offsetX, offsetY := float64(screen.Bounds().Min.X), float64(screen.Bounds().Min.Y) + op.GeoM.Translate(offsetX+(float64(sw)-float64(fw)*s)/2, offsetY+(float64(sh)-float64(fh)*s)/2) + op.Filter = ebiten.FilterLinear + + screen.DrawImage(frame, &op) + return nil +} + +// Play starts playing the video. +func (p *mpegPlayer) Play() { + p.m.Lock() + defer p.m.Unlock() + + if p.mpg.HasEnded() { + return + } + + if p.audioPlayer != nil { + if p.audioPlayer.IsPlaying() { + return + } + // Play refers (*mpegAudio).Read function, where the same mutex is used. + // In order to avoid dead lock, use a different goroutine to start playing. + // This issue happens especially on Windows where goroutines at Play are avoided in Oto (#1768). + // TODO: Remove this hack in the future (ebitengine/oto#235). + go p.audioPlayer.Play() + return + } + + if p.refTime != (time.Time{}) { + return + } + p.refTime = time.Now() +} + +type mpegAudio struct { + audio *mpeg.Audio + + // leftovers is the remaining audio samples of the previous Read call. + leftovers []byte + + // m is the mutex shared with the mpegPlayer. + // As *mpeg.MPEG is not concurrent safe, this mutex is necessary. + m *sync.Mutex +} + +func (a *mpegAudio) Read(buf []byte) (int, error) { + a.m.Lock() + defer a.m.Unlock() + + var readBytes int + if len(a.leftovers) > 0 { + n := copy(buf, a.leftovers) + readBytes += n + buf = buf[n:] + + copy(a.leftovers, a.leftovers[n:]) + a.leftovers = a.leftovers[:len(a.leftovers)-n] + } + + for len(buf) > 0 && !a.audio.HasEnded() { + mpegSamples := a.audio.Decode() + if mpegSamples == nil { + break + } + + bs := make([]byte, len(mpegSamples.S16)*2) + for i, s := range mpegSamples.S16 { + bs[i*2] = byte(s) + bs[i*2+1] = byte(s >> 8) + } + + n := copy(buf, bs) + readBytes += n + buf = buf[n:] + + if n < len(bs) { + a.leftovers = append(a.leftovers, bs[n:]...) + break + } + } + + if a.audio.HasEnded() { + return readBytes, io.EOF + } + return readBytes, nil +} diff --git a/examples/video/shibuya.mpg b/examples/video/shibuya.mpg new file mode 100644 index 000000000..873bec604 Binary files /dev/null and b/examples/video/shibuya.mpg differ diff --git a/examples/video/shibuya_noaudio.mpg b/examples/video/shibuya_noaudio.mpg new file mode 100644 index 000000000..0de116350 Binary files /dev/null and b/examples/video/shibuya_noaudio.mpg differ diff --git a/go.mod b/go.mod index 81ba5cdd2..a45785ef2 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/ebitengine/hideconsole v1.0.0 github.com/ebitengine/oto/v3 v3.3.0-alpha.1 github.com/ebitengine/purego v0.8.0-alpha.1 + github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703 github.com/hajimehoshi/bitmapfont/v3 v3.0.0 github.com/hajimehoshi/go-mp3 v0.3.4 diff --git a/go.sum b/go.sum index 161bb5327..f3d81e503 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/ebitengine/oto/v3 v3.3.0-alpha.1 h1:J2nBmQwPLKc4+yLObytq1jKNydI96l6Ej github.com/ebitengine/oto/v3 v3.3.0-alpha.1/go.mod h1:T2/VV0UWG97GEEf4kORMU2nCneYT/YmwSTxPutSVaUg= github.com/ebitengine/purego v0.8.0-alpha.1 h1:52AgJTNaQRi7YtOtdJl4hkxNWhAGMxuDuDjOVIp5Ojk= github.com/ebitengine/purego v0.8.0-alpha.1/go.mod h1:y8L+ZRLphbdPW2xs41fur/KaW57yTzrFsqsclHyHrTM= +github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f h1:ysqRe+lvUiL0dH5XzkH0Bz68bFMPJ4f5Si4L/HD9SGk= +github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f/go.mod h1:i/ebyRRv/IoHixuZ9bElZnXbmfoUVPGQpdsJ4sVuX38= github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703 h1:AqtMl9yw7r319Ah4W2afQm3Ql+PEsQKHds18tGvKhog= github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703/go.mod h1:2+owI/sxa73XA581LAzVuEBZ3WEEV2pXeDswCH/3i1I= github.com/go-text/typesetting-utils v0.0.0-20240317173224-1986cbe96c66 h1:GUrm65PQPlhFSKjLPGOZNPNxLCybjzjYBzjfoBGaDUY=