Set max dst images to 8 + some wording

This reverts commit 6f3f58cb32.

Reason: decoding the video already follows the framerate of the MPEG
data and skips rendering when necessary, so there is no need to adjust
FPS from Update.

examples/mrt/main.go

@@ -31,7 +31,7 @@ const (
 )
 
 var (
-	dsts = [4]*ebiten.Image{
+	dsts = [8]*ebiten.Image{
 		/*ebiten.NewImage(dstSize, dstSize),
 		ebiten.NewImage(dstSize, dstSize),
 		ebiten.NewImage(dstSize, dstSize),

examples/texti18n/main.go

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This example is a demonstration to render languages that cannot be rendered with the `text` package.
-// We plan to provide a useful API to render them more easily (#2454). Stay tuned!
-
 package main
 
 import (

examples/video/license.md

@@ -0,0 +1,9 @@
+# License
+
+## `shibuya.mpg`, `shibuya_noaudio.mpg`
+
+https://commons.wikimedia.org/wiki/File:Shibuya_Crossing,_Tokyo,_Japan_(video).webm
+
+"Shibuya Crossing, Tokyo, Japan (video).webm" by Basile Morin
+
+The Creative Commons Attribution-Share Alike 4.0 International license

examples/video/main.go

@@ -0,0 +1,98 @@
+// Copyright 2024 The Ebitengine Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/hajimehoshi/ebiten/v2"
+	"github.com/hajimehoshi/ebiten/v2/audio"
+	"github.com/hajimehoshi/ebiten/v2/ebitenutil"
+)
+
+//go:embed shibuya.mpg
+var shibuya_mpg []byte
+
+type Game struct {
+	player *mpegPlayer
+	err    error
+}
+
+func (g *Game) Update() error {
+	if g.err != nil {
+		return g.err
+	}
+	return nil
+}
+
+func (g *Game) Draw(screen *ebiten.Image) {
+	if g.err != nil {
+		return
+	}
+	if err := g.player.Draw(screen); err != nil {
+		g.err = err
+	}
+	ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.ActualFPS()))
+}
+
+func (g *Game) Layout(outsideWidth, outsideHeight int) (screenWidth, screenHeight int) {
+	return outsideWidth, outsideHeight
+}
+
+func main() {
+	// Initialize audio context.
+	_ = audio.NewContext(48000)
+
+	// If you want to play your own video, the video must be an MPEG-1 video with 48000 audio sample rate.
+	// You can convert the video to MPEG-1 with the below command:
+	//
+	//     ffmpeg -i YOUR_VIDEO -c:v mpeg1video -q:v 8 -c:a mp2 -format mpeg -ar 48000 output.mpg
+	//
+	// You can adjust quality by changing -q:v value. A lower value indicates better quality.
+	var in io.ReadSeeker
+	if len(os.Args) > 1 {
+		f, err := os.Open(os.Args[1])
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer func() {
+			_ = f.Close()
+		}()
+		in = f
+	} else {
+		in = bytes.NewReader(shibuya_mpg)
+		fmt.Println("Play the default video. You can specify a video file as an argument.")
+	}
+
+	player, err := newMPEGPlayer(in)
+	if err != nil {
+		log.Fatal(err)
+	}
+	g := &Game{
+		player: player,
+	}
+	player.Play()
+
+	ebiten.SetWindowTitle("Video (Ebitengine Demo)")
+	ebiten.SetWindowResizingMode(ebiten.WindowResizingModeEnabled)
+	if err := ebiten.RunGame(g); err != nil {
+		log.Fatal(err)
+	}
+}

examples/video/mpegplayer.go

@@ -0,0 +1,292 @@
+// Copyright 2024 The Ebitengine Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"image"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/gen2brain/mpeg"
+	"github.com/hajimehoshi/ebiten/v2"
+	"github.com/hajimehoshi/ebiten/v2/audio"
+)
+
+type mpegPlayer struct {
+	mpg *mpeg.MPEG
+
+	// yCbCrImage is the current frame image in YCbCr format.
+	// An MPEG frame is stored in this image first.
+	// Then, this image data is converted to RGB to frameImage.
+	yCbCrImage *ebiten.Image
+
+	// yCbCrBytes is the byte slice to store YCbCr data.
+	// This includes Y, Cb, Cr, and alpha (always 0xff) data for each pixel.
+	yCbCrBytes []byte
+
+	// yCbCrShader is the shader to convert YCbCr to RGB.
+	yCbCrShader *ebiten.Shader
+
+	// frameImage is the current frame image in RGB format.
+	frameImage *ebiten.Image
+
+	audioPlayer *audio.Player
+
+	// These members are used when the video doesn't have an audio stream.
+	refTime time.Time
+
+	m sync.Mutex
+}
+
+func newMPEGPlayer(src io.Reader) (*mpegPlayer, error) {
+	mpg, err := mpeg.New(src)
+	if err != nil {
+		return nil, err
+	}
+	if mpg.NumVideoStreams() == 0 {
+		return nil, fmt.Errorf("video: no video streams")
+	}
+	if !mpg.HasHeaders() {
+		return nil, fmt.Errorf("video: missing headers")
+	}
+
+	p := &mpegPlayer{
+		mpg:        mpg,
+		yCbCrImage: ebiten.NewImage(mpg.Width(), mpg.Height()),
+		yCbCrBytes: make([]byte, 4*mpg.Width()*mpg.Height()),
+		frameImage: ebiten.NewImage(mpg.Width(), mpg.Height()),
+	}
+
+	s, err := ebiten.NewShader([]byte(`package main
+
+//kage:unit pixels
+
+func Fragment(dstPos vec4, srcPos vec2, color vec4) vec4 {
+	// For this calculation, see the comment in the standard library color.YCbCrToRGB function.
+	c := imageSrc0UnsafeAt(srcPos)
+	return vec4(
+		c.x + 1.40200 * (c.z-0.5),
+		c.x - 0.34414 * (c.y-0.5) - 0.71414 * (c.z-0.5),
+		c.x + 1.77200 * (c.y-0.5),
+		1,
+	)
+}
+`))
+	if err != nil {
+		return nil, err
+	}
+	p.yCbCrShader = s
+
+	// If the video doesn't have an audio stream, initialization is done.
+	if mpg.NumAudioStreams() == 0 {
+		return p, nil
+	}
+
+	// If the video has an audio stream, initialize an audio player.
+	ctx := audio.CurrentContext()
+	if ctx == nil {
+		return nil, fmt.Errorf("video: audio.Context is not initialized")
+	}
+	if mpg.Channels() != 2 {
+		return nil, fmt.Errorf("video: mpeg audio stream must be 2 but was %d", mpg.Channels())
+	}
+	if ctx.SampleRate() != mpg.Samplerate() {
+		return nil, fmt.Errorf("video: mpeg audio stream sample rate %d doesn't match with audio context sample rate %d", mpg.Samplerate(), ctx.SampleRate())
+	}
+
+	mpg.SetAudioFormat(mpeg.AudioS16)
+
+	audioPlayer, err := ctx.NewPlayer(&mpegAudio{
+		audio: mpg.Audio(),
+		m:     &p.m,
+	})
+	if err != nil {
+		return nil, err
+	}
+	p.audioPlayer = audioPlayer
+
+	return p, nil
+}
+
+// updateFrame upadtes the current video frame.
+func (p *mpegPlayer) updateFrame() error {
+	p.m.Lock()
+	defer p.m.Unlock()
+
+	var pos float64
+	if p.audioPlayer != nil {
+		pos = p.audioPlayer.Position().Seconds()
+	} else {
+		if p.refTime != (time.Time{}) {
+			pos = time.Since(p.refTime).Seconds()
+		}
+	}
+
+	video := p.mpg.Video()
+	if video.HasEnded() {
+		p.frameImage.Clear()
+		return nil
+	}
+
+	d := 1 / p.mpg.Framerate()
+	var mpegFrame *mpeg.Frame
+	for video.Time()+d <= pos && !video.HasEnded() {
+		mpegFrame = video.Decode()
+	}
+
+	if mpegFrame == nil {
+		return nil
+	}
+
+	img := mpegFrame.YCbCr()
+	if img.SubsampleRatio != image.YCbCrSubsampleRatio420 {
+		return fmt.Errorf("video: subsample ratio must be 4:2:0")
+	}
+	w, h := p.mpg.Width(), p.mpg.Height()
+	for j := 0; j < h; j++ {
+		yi := j * img.YStride
+		ci := (j / 2) * img.CStride
+		// Create temporary slices to encourage BCE (boundary-checking elimination).
+		ys := img.Y[yi : yi+w]
+		cbs := img.Cb[ci : ci+w/2]
+		crs := img.Cr[ci : ci+w/2]
+		for i := 0; i < w; i++ {
+			idx := 4 * (j*w + i)
+			buf := p.yCbCrBytes[idx : idx+3]
+			buf[0] = ys[i]
+			buf[1] = cbs[i/2]
+			buf[2] = crs[i/2]
+			// p.yCbCrBytes[3] = 0xff is not needed as the shader ignores this part.
+		}
+	}
+
+	p.yCbCrImage.WritePixels(p.yCbCrBytes)
+
+	// Converting YCbCr to RGB on CPU is slow. Use a shader instead.
+	op := &ebiten.DrawRectShaderOptions{}
+	op.Images[0] = p.yCbCrImage
+	op.Blend = ebiten.BlendCopy
+	p.frameImage.DrawRectShader(w, h, p.yCbCrShader, op)
+
+	return nil
+}
+
+// Draw draws the current frame onto the given screen.
+func (p *mpegPlayer) Draw(screen *ebiten.Image) error {
+	if err := p.updateFrame(); err != nil {
+		return err
+	}
+
+	frame := p.frameImage
+	sw, sh := screen.Bounds().Dx(), screen.Bounds().Dy()
+	fw, fh := frame.Bounds().Dx(), frame.Bounds().Dy()
+
+	op := ebiten.DrawImageOptions{}
+	wf, hf := float64(sw)/float64(fw), float64(sh)/float64(fh)
+	s := wf
+	if hf < wf {
+		s = hf
+	}
+	op.GeoM.Scale(s, s)
+
+	offsetX, offsetY := float64(screen.Bounds().Min.X), float64(screen.Bounds().Min.Y)
+	op.GeoM.Translate(offsetX+(float64(sw)-float64(fw)*s)/2, offsetY+(float64(sh)-float64(fh)*s)/2)
+	op.Filter = ebiten.FilterLinear
+
+	screen.DrawImage(frame, &op)
+	return nil
+}
+
+// Play starts playing the video.
+func (p *mpegPlayer) Play() {
+	p.m.Lock()
+	defer p.m.Unlock()
+
+	if p.mpg.HasEnded() {
+		return
+	}
+
+	if p.audioPlayer != nil {
+		if p.audioPlayer.IsPlaying() {
+			return
+		}
+		// Play refers (*mpegAudio).Read function, where the same mutex is used.
+		// In order to avoid dead lock, use a different goroutine to start playing.
+		// This issue happens especially on Windows where goroutines at Play are avoided in Oto (#1768).
+		// TODO: Remove this hack in the future (ebitengine/oto#235).
+		go p.audioPlayer.Play()
+		return
+	}
+
+	if p.refTime != (time.Time{}) {
+		return
+	}
+	p.refTime = time.Now()
+}
+
+type mpegAudio struct {
+	audio *mpeg.Audio
+
+	// leftovers is the remaining audio samples of the previous Read call.
+	leftovers []byte
+
+	// m is the mutex shared with the mpegPlayer.
+	// As *mpeg.MPEG is not concurrent safe, this mutex is necessary.
+	m *sync.Mutex
+}
+
+func (a *mpegAudio) Read(buf []byte) (int, error) {
+	a.m.Lock()
+	defer a.m.Unlock()
+
+	var readBytes int
+	if len(a.leftovers) > 0 {
+		n := copy(buf, a.leftovers)
+		readBytes += n
+		buf = buf[n:]
+
+		copy(a.leftovers, a.leftovers[n:])
+		a.leftovers = a.leftovers[:len(a.leftovers)-n]
+	}
+
+	for len(buf) > 0 && !a.audio.HasEnded() {
+		mpegSamples := a.audio.Decode()
+		if mpegSamples == nil {
+			break
+		}
+
+		bs := make([]byte, len(mpegSamples.S16)*2)
+		for i, s := range mpegSamples.S16 {
+			bs[i*2] = byte(s)
+			bs[i*2+1] = byte(s >> 8)
+		}
+
+		n := copy(buf, bs)
+		readBytes += n
+		buf = buf[n:]
+
+		if n < len(bs) {
+			a.leftovers = append(a.leftovers, bs[n:]...)
+			break
+		}
+	}
+
+	if a.audio.HasEnded() {
+		return readBytes, io.EOF
+	}
+	return readBytes, nil
+}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/ebitengine/hideconsole v1.0.0
 	github.com/ebitengine/oto/v3 v3.3.0-alpha.1
 	github.com/ebitengine/purego v0.8.0-alpha.1
+	github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f
 	github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703
 	github.com/hajimehoshi/bitmapfont/v3 v3.0.0
 	github.com/hajimehoshi/go-mp3 v0.3.4

go.sum

@@ -6,6 +6,8 @@ github.com/ebitengine/oto/v3 v3.3.0-alpha.1 h1:J2nBmQwPLKc4+yLObytq1jKNydI96l6Ej
 github.com/ebitengine/oto/v3 v3.3.0-alpha.1/go.mod h1:T2/VV0UWG97GEEf4kORMU2nCneYT/YmwSTxPutSVaUg=
 github.com/ebitengine/purego v0.8.0-alpha.1 h1:52AgJTNaQRi7YtOtdJl4hkxNWhAGMxuDuDjOVIp5Ojk=
 github.com/ebitengine/purego v0.8.0-alpha.1/go.mod h1:y8L+ZRLphbdPW2xs41fur/KaW57yTzrFsqsclHyHrTM=
+github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f h1:ysqRe+lvUiL0dH5XzkH0Bz68bFMPJ4f5Si4L/HD9SGk=
+github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f/go.mod h1:i/ebyRRv/IoHixuZ9bElZnXbmfoUVPGQpdsJ4sVuX38=
 github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703 h1:AqtMl9yw7r319Ah4W2afQm3Ql+PEsQKHds18tGvKhog=
 github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703/go.mod h1:2+owI/sxa73XA581LAzVuEBZ3WEEV2pXeDswCH/3i1I=
 github.com/go-text/typesetting-utils v0.0.0-20240317173224-1986cbe96c66 h1:GUrm65PQPlhFSKjLPGOZNPNxLCybjzjYBzjfoBGaDUY=

internal/atlas/image.go

@@ -590,7 +590,7 @@ func drawTrianglesMRT(dsts [graphics.ShaderDstImageCount]*Image, srcs [graphics.
 		// i and a source image might share the same atlas even though i != src.
 		for _, dst := range dsts {
 			if src != nil && dst != nil && dst.backend.image == src.backend.image {
-				panic("atlas: Image.DrawTriangles: source must be different from the receiver")
+				panic("atlas: DrawTrianglesMRT: source must be different from the destination images")
 			}
 		}
 	}

internal/buffered/image.go

@@ -218,7 +218,7 @@ func DrawTrianglesMRT(dsts [graphics.ShaderDstImageCount]*Image, srcs [graphics.
 				continue
 			}
 			if dst == src {
-				panic("buffered: DrawTrianglesMRT: source images must be different from the destination")
+				panic("buffered: DrawTrianglesMRT: source images must be different from the destination images")
 			}
 		}
 		if src != nil {

internal/graphics/vertex.go

@@ -16,7 +16,11 @@ package graphics
 
 const (
 	ShaderSrcImageCount = 4
-	ShaderDstImageCount = 4
+	// The minimum guaranteed value for the number of target seems to be 8
+	// OpenGL(8): https://www.khronos.org/opengl/wiki/Framebuffer_Object#Framebuffer_Object_Structure
+	// DirectX11(8): https://learn.microsoft.com/en-us/windows/win32/direct3d11/d3d10-graphics-programming-guide-output-merger-stage#multiple-rendertargets-overview
+	// Metal(8): Page 7 of 15: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+	ShaderDstImageCount = 8
 
 	// PreservedUniformVariablesCount represents the number of preserved uniform variables.
 	// Any shaders in Ebitengine must have these uniform variables.

internal/shaderir/ir_test.go

@@ -938,12 +938,12 @@ void F0(float l0, float l1, thread float& l2) {
 				},
 				Attributes: []shaderir.Type{
 					{Main: shaderir.Vec4},
-					{Main: shaderir.Float},
 					{Main: shaderir.Vec2},
+					{Main: shaderir.Vec4},
 				},
 				Varyings: []shaderir.Type{
-					{Main: shaderir.Float},
 					{Main: shaderir.Vec2},
+					{Main: shaderir.Vec4},
 				},
 				VertexFunc: shaderir.VertexFunc{
 					Block: block(
@@ -967,10 +967,10 @@ void F0(float l0, float l1, thread float& l2) {
 			GlslVS: glslVertexPrelude + `
 uniform float U0;
 in vec4 A0;
-in float A1;
-in vec2 A2;
-out float V0;
-out vec2 V1;
+in vec2 A1;
+in vec4 A2;
+out vec2 V0;
+out vec4 V1;
 
 void main(void) {
 	gl_Position = A0;
@@ -979,8 +979,8 @@ void main(void) {
 }`,
 			GlslFS: glslFragmentPrelude + `
 uniform float U0;
-in float V0;
-in vec2 V1;`,
+in vec2 V0;
+in vec4 V1;`,
 		},
 		{
 			Name: "FragmentFunc",
@@ -991,12 +991,12 @@ in vec2 V1;`,
 				},
 				Attributes: []shaderir.Type{
 					{Main: shaderir.Vec4},
-					{Main: shaderir.Float},
 					{Main: shaderir.Vec2},
+					{Main: shaderir.Vec4},
 				},
 				Varyings: []shaderir.Type{
-					{Main: shaderir.Float},
 					{Main: shaderir.Vec2},
+					{Main: shaderir.Vec4},
 				},
 				VertexFunc: shaderir.VertexFunc{
 					Block: block(
@@ -1016,34 +1016,39 @@ in vec2 V1;`,
 						),
 					),
 				},
+				ColorsOutCount: 1,
 				FragmentFunc: shaderir.FragmentFunc{
 					Block: block(
 						[]shaderir.Type{
-							{Main: shaderir.Float},
 							{Main: shaderir.Vec2},
+							{Main: shaderir.Vec4},
 						},
-						3,
-						assignStmt(
-							localVariableExpr(3),
-							localVariableExpr(0),
-						),
+						3+1,
 						assignStmt(
 							localVariableExpr(4),
 							localVariableExpr(1),
 						),
-						returnStmt(
+						assignStmt(
+							localVariableExpr(5),
 							localVariableExpr(2),
 						),
+						assignStmt(
+							localVariableExpr(3),
+							localVariableExpr(0),
+						),
+						shaderir.Stmt{
+							Type: shaderir.Return,
+						},
 					),
 				},
 			},
 			GlslVS: glslVertexPrelude + `
 uniform float U0;
 in vec4 A0;
-in float A1;
-in vec2 A2;
-out float V0;
-out vec2 V1;
+in vec2 A1;
+in vec4 A2;
+out vec2 V0;
+out vec4 V1;
 
 void main(void) {
 	gl_Position = A0;
@@ -1052,21 +1057,22 @@ void main(void) {
 }`,
 			GlslFS: glslFragmentPrelude + `
 uniform float U0;
-in float V0;
-in vec2 V1;
+in vec2 V0;
+in vec4 V1;
 
-vec4 F0(in vec4 l0, in float l1, in vec2 l2);
+void F0(in vec4 l0, in vec2 l1, in vec4 l2, out vec4 l3);
 
-vec4 F0(in vec4 l0, in float l1, in vec2 l2) {
-	float l3 = float(0);
+void F0(in vec4 l0, in vec2 l1, in vec4 l2, out vec4 l3) {
 	vec2 l4 = vec2(0);
-	l3 = l0;
+	vec4 l5 = vec4(0);
 	l4 = l1;
-	return l2;
+	l5 = l2;
+	l3 = l0;
+	return;
 }
 
 void main(void) {
-	fragColor = F0(gl_FragCoord, V0, V1);
+	F0(gl_FragCoord, V0, V1, gl_FragData[0]);
 }`,
 		},
 	}
@@ -1088,14 +1094,14 @@ void main(void) {
 					t.Errorf("%s fragment: got: %s, want: %s", tc.Name, got, want)
 				}
 			}
-			m := msl.Compile(&tc.Program)
+			/*m := msl.Compile(&tc.Program)
 			if tc.Metal != "" {
 				got := m
 				want := tc.Metal + "\n"
 				if got != want {
 					t.Errorf("%s metal: got: %s, want: %s", tc.Name, got, want)
 				}
-			}
+			}*/
 		})
 	}
 }