Compare commits

...

14 Commits

Author SHA1 Message Date
Zyko
2faf8a551d Set max dst images to 8 + some wording 2024-04-12 19:19:17 +02:00
Zyko
75b6e1fa6d Fixed conflict 2024-04-12 18:39:47 +02:00
Zyko
f021b5ded8 Fixed IR tests + skipping metal for now 2024-04-12 18:34:29 +02:00
Hajime Hoshi
88dae9c7d4 all: update gen2brain/mpeg 2024-04-13 01:14:20 +09:00
Hajime Hoshi
4a964da0aa internal/shaderir: fix type mismatches 2024-04-12 23:42:06 +09:00
Hajime Hoshi
493db3baf3 all: update mpeg to v0.3.0 2024-04-12 21:59:07 +09:00
Hajime Hoshi
2bd9472d2c examples/video: performance optimization by encouraging BCE 2024-04-12 21:45:54 +09:00
Hajime Hoshi
c105224705 examples/video: show FPS 2024-04-12 15:16:26 +09:00
Hajime Hoshi
3c1d74b2d5 examples/texti18n: update comments 2024-04-12 12:28:15 +09:00
Hajime Hoshi
8467cafe74 examples/video: fix the title 2024-04-12 00:54:20 +09:00
Hajime Hoshi
4a87339a0a examples/video: use a shader to convert YCbCr to RGB 2024-04-12 00:47:10 +09:00
Hajime Hoshi
68cc017189 Revert "examples/video: reduce video FPS"
This reverts commit 6f3f58cb32.

Reason: decoding the video already follows the framerate of the MPEG
data and skips rendering when necessary, so there is no need to adjust
FPS from Update.
2024-04-11 22:43:48 +09:00
Hajime Hoshi
6f3f58cb32 examples/video: reduce video FPS 2024-04-11 22:06:20 +09:00
Hajime Hoshi
d3befbf89b examples: add an example to play a video
Closes #110
Updates #1768
Updates ebitengine/oto#235
2024-04-11 20:31:55 +09:00
13 changed files with 447 additions and 38 deletions

View File

@ -31,7 +31,7 @@ const (
)
var (
dsts = [4]*ebiten.Image{
dsts = [8]*ebiten.Image{
/*ebiten.NewImage(dstSize, dstSize),
ebiten.NewImage(dstSize, dstSize),
ebiten.NewImage(dstSize, dstSize),

View File

@ -12,9 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// This example is a demonstration to render languages that cannot be rendered with the `text` package.
// We plan to provide a useful API to render them more easily (#2454). Stay tuned!
package main
import (

View File

@ -0,0 +1,9 @@
# License
## `shibuya.mpg`, `shibuya_noaudio.mpg`
https://commons.wikimedia.org/wiki/File:Shibuya_Crossing,_Tokyo,_Japan_(video).webm
"Shibuya Crossing, Tokyo, Japan (video).webm" by Basile Morin
The Creative Commons Attribution-Share Alike 4.0 International license

98
examples/video/main.go Normal file
View File

@ -0,0 +1,98 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
_ "embed"
"fmt"
"io"
"log"
"os"
"github.com/hajimehoshi/ebiten/v2"
"github.com/hajimehoshi/ebiten/v2/audio"
"github.com/hajimehoshi/ebiten/v2/ebitenutil"
)
//go:embed shibuya.mpg
var shibuya_mpg []byte
type Game struct {
player *mpegPlayer
err error
}
func (g *Game) Update() error {
if g.err != nil {
return g.err
}
return nil
}
func (g *Game) Draw(screen *ebiten.Image) {
if g.err != nil {
return
}
if err := g.player.Draw(screen); err != nil {
g.err = err
}
ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.ActualFPS()))
}
func (g *Game) Layout(outsideWidth, outsideHeight int) (screenWidth, screenHeight int) {
return outsideWidth, outsideHeight
}
func main() {
// Initialize audio context.
_ = audio.NewContext(48000)
// If you want to play your own video, the video must be an MPEG-1 video with 48000 audio sample rate.
// You can convert the video to MPEG-1 with the below command:
//
// ffmpeg -i YOUR_VIDEO -c:v mpeg1video -q:v 8 -c:a mp2 -format mpeg -ar 48000 output.mpg
//
// You can adjust quality by changing -q:v value. A lower value indicates better quality.
var in io.ReadSeeker
if len(os.Args) > 1 {
f, err := os.Open(os.Args[1])
if err != nil {
log.Fatal(err)
}
defer func() {
_ = f.Close()
}()
in = f
} else {
in = bytes.NewReader(shibuya_mpg)
fmt.Println("Play the default video. You can specify a video file as an argument.")
}
player, err := newMPEGPlayer(in)
if err != nil {
log.Fatal(err)
}
g := &Game{
player: player,
}
player.Play()
ebiten.SetWindowTitle("Video (Ebitengine Demo)")
ebiten.SetWindowResizingMode(ebiten.WindowResizingModeEnabled)
if err := ebiten.RunGame(g); err != nil {
log.Fatal(err)
}
}

View File

@ -0,0 +1,292 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"image"
"io"
"sync"
"time"
"github.com/gen2brain/mpeg"
"github.com/hajimehoshi/ebiten/v2"
"github.com/hajimehoshi/ebiten/v2/audio"
)
type mpegPlayer struct {
mpg *mpeg.MPEG
// yCbCrImage is the current frame image in YCbCr format.
// An MPEG frame is stored in this image first.
// Then, this image data is converted to RGB to frameImage.
yCbCrImage *ebiten.Image
// yCbCrBytes is the byte slice to store YCbCr data.
// This includes Y, Cb, Cr, and alpha (always 0xff) data for each pixel.
yCbCrBytes []byte
// yCbCrShader is the shader to convert YCbCr to RGB.
yCbCrShader *ebiten.Shader
// frameImage is the current frame image in RGB format.
frameImage *ebiten.Image
audioPlayer *audio.Player
// These members are used when the video doesn't have an audio stream.
refTime time.Time
m sync.Mutex
}
func newMPEGPlayer(src io.Reader) (*mpegPlayer, error) {
mpg, err := mpeg.New(src)
if err != nil {
return nil, err
}
if mpg.NumVideoStreams() == 0 {
return nil, fmt.Errorf("video: no video streams")
}
if !mpg.HasHeaders() {
return nil, fmt.Errorf("video: missing headers")
}
p := &mpegPlayer{
mpg: mpg,
yCbCrImage: ebiten.NewImage(mpg.Width(), mpg.Height()),
yCbCrBytes: make([]byte, 4*mpg.Width()*mpg.Height()),
frameImage: ebiten.NewImage(mpg.Width(), mpg.Height()),
}
s, err := ebiten.NewShader([]byte(`package main
//kage:unit pixels
func Fragment(dstPos vec4, srcPos vec2, color vec4) vec4 {
// For this calculation, see the comment in the standard library color.YCbCrToRGB function.
c := imageSrc0UnsafeAt(srcPos)
return vec4(
c.x + 1.40200 * (c.z-0.5),
c.x - 0.34414 * (c.y-0.5) - 0.71414 * (c.z-0.5),
c.x + 1.77200 * (c.y-0.5),
1,
)
}
`))
if err != nil {
return nil, err
}
p.yCbCrShader = s
// If the video doesn't have an audio stream, initialization is done.
if mpg.NumAudioStreams() == 0 {
return p, nil
}
// If the video has an audio stream, initialize an audio player.
ctx := audio.CurrentContext()
if ctx == nil {
return nil, fmt.Errorf("video: audio.Context is not initialized")
}
if mpg.Channels() != 2 {
return nil, fmt.Errorf("video: mpeg audio stream must be 2 but was %d", mpg.Channels())
}
if ctx.SampleRate() != mpg.Samplerate() {
return nil, fmt.Errorf("video: mpeg audio stream sample rate %d doesn't match with audio context sample rate %d", mpg.Samplerate(), ctx.SampleRate())
}
mpg.SetAudioFormat(mpeg.AudioS16)
audioPlayer, err := ctx.NewPlayer(&mpegAudio{
audio: mpg.Audio(),
m: &p.m,
})
if err != nil {
return nil, err
}
p.audioPlayer = audioPlayer
return p, nil
}
// updateFrame upadtes the current video frame.
func (p *mpegPlayer) updateFrame() error {
p.m.Lock()
defer p.m.Unlock()
var pos float64
if p.audioPlayer != nil {
pos = p.audioPlayer.Position().Seconds()
} else {
if p.refTime != (time.Time{}) {
pos = time.Since(p.refTime).Seconds()
}
}
video := p.mpg.Video()
if video.HasEnded() {
p.frameImage.Clear()
return nil
}
d := 1 / p.mpg.Framerate()
var mpegFrame *mpeg.Frame
for video.Time()+d <= pos && !video.HasEnded() {
mpegFrame = video.Decode()
}
if mpegFrame == nil {
return nil
}
img := mpegFrame.YCbCr()
if img.SubsampleRatio != image.YCbCrSubsampleRatio420 {
return fmt.Errorf("video: subsample ratio must be 4:2:0")
}
w, h := p.mpg.Width(), p.mpg.Height()
for j := 0; j < h; j++ {
yi := j * img.YStride
ci := (j / 2) * img.CStride
// Create temporary slices to encourage BCE (boundary-checking elimination).
ys := img.Y[yi : yi+w]
cbs := img.Cb[ci : ci+w/2]
crs := img.Cr[ci : ci+w/2]
for i := 0; i < w; i++ {
idx := 4 * (j*w + i)
buf := p.yCbCrBytes[idx : idx+3]
buf[0] = ys[i]
buf[1] = cbs[i/2]
buf[2] = crs[i/2]
// p.yCbCrBytes[3] = 0xff is not needed as the shader ignores this part.
}
}
p.yCbCrImage.WritePixels(p.yCbCrBytes)
// Converting YCbCr to RGB on CPU is slow. Use a shader instead.
op := &ebiten.DrawRectShaderOptions{}
op.Images[0] = p.yCbCrImage
op.Blend = ebiten.BlendCopy
p.frameImage.DrawRectShader(w, h, p.yCbCrShader, op)
return nil
}
// Draw draws the current frame onto the given screen.
func (p *mpegPlayer) Draw(screen *ebiten.Image) error {
if err := p.updateFrame(); err != nil {
return err
}
frame := p.frameImage
sw, sh := screen.Bounds().Dx(), screen.Bounds().Dy()
fw, fh := frame.Bounds().Dx(), frame.Bounds().Dy()
op := ebiten.DrawImageOptions{}
wf, hf := float64(sw)/float64(fw), float64(sh)/float64(fh)
s := wf
if hf < wf {
s = hf
}
op.GeoM.Scale(s, s)
offsetX, offsetY := float64(screen.Bounds().Min.X), float64(screen.Bounds().Min.Y)
op.GeoM.Translate(offsetX+(float64(sw)-float64(fw)*s)/2, offsetY+(float64(sh)-float64(fh)*s)/2)
op.Filter = ebiten.FilterLinear
screen.DrawImage(frame, &op)
return nil
}
// Play starts playing the video.
func (p *mpegPlayer) Play() {
p.m.Lock()
defer p.m.Unlock()
if p.mpg.HasEnded() {
return
}
if p.audioPlayer != nil {
if p.audioPlayer.IsPlaying() {
return
}
// Play refers (*mpegAudio).Read function, where the same mutex is used.
// In order to avoid dead lock, use a different goroutine to start playing.
// This issue happens especially on Windows where goroutines at Play are avoided in Oto (#1768).
// TODO: Remove this hack in the future (ebitengine/oto#235).
go p.audioPlayer.Play()
return
}
if p.refTime != (time.Time{}) {
return
}
p.refTime = time.Now()
}
type mpegAudio struct {
audio *mpeg.Audio
// leftovers is the remaining audio samples of the previous Read call.
leftovers []byte
// m is the mutex shared with the mpegPlayer.
// As *mpeg.MPEG is not concurrent safe, this mutex is necessary.
m *sync.Mutex
}
func (a *mpegAudio) Read(buf []byte) (int, error) {
a.m.Lock()
defer a.m.Unlock()
var readBytes int
if len(a.leftovers) > 0 {
n := copy(buf, a.leftovers)
readBytes += n
buf = buf[n:]
copy(a.leftovers, a.leftovers[n:])
a.leftovers = a.leftovers[:len(a.leftovers)-n]
}
for len(buf) > 0 && !a.audio.HasEnded() {
mpegSamples := a.audio.Decode()
if mpegSamples == nil {
break
}
bs := make([]byte, len(mpegSamples.S16)*2)
for i, s := range mpegSamples.S16 {
bs[i*2] = byte(s)
bs[i*2+1] = byte(s >> 8)
}
n := copy(buf, bs)
readBytes += n
buf = buf[n:]
if n < len(bs) {
a.leftovers = append(a.leftovers, bs[n:]...)
break
}
}
if a.audio.HasEnded() {
return readBytes, io.EOF
}
return readBytes, nil
}

BIN
examples/video/shibuya.mpg Normal file

Binary file not shown.

Binary file not shown.

1
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/ebitengine/hideconsole v1.0.0
github.com/ebitengine/oto/v3 v3.3.0-alpha.1
github.com/ebitengine/purego v0.8.0-alpha.1
github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f
github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703
github.com/hajimehoshi/bitmapfont/v3 v3.0.0
github.com/hajimehoshi/go-mp3 v0.3.4

2
go.sum
View File

@ -6,6 +6,8 @@ github.com/ebitengine/oto/v3 v3.3.0-alpha.1 h1:J2nBmQwPLKc4+yLObytq1jKNydI96l6Ej
github.com/ebitengine/oto/v3 v3.3.0-alpha.1/go.mod h1:T2/VV0UWG97GEEf4kORMU2nCneYT/YmwSTxPutSVaUg=
github.com/ebitengine/purego v0.8.0-alpha.1 h1:52AgJTNaQRi7YtOtdJl4hkxNWhAGMxuDuDjOVIp5Ojk=
github.com/ebitengine/purego v0.8.0-alpha.1/go.mod h1:y8L+ZRLphbdPW2xs41fur/KaW57yTzrFsqsclHyHrTM=
github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f h1:ysqRe+lvUiL0dH5XzkH0Bz68bFMPJ4f5Si4L/HD9SGk=
github.com/gen2brain/mpeg v0.3.2-0.20240412154320-a2ac4fc8a46f/go.mod h1:i/ebyRRv/IoHixuZ9bElZnXbmfoUVPGQpdsJ4sVuX38=
github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703 h1:AqtMl9yw7r319Ah4W2afQm3Ql+PEsQKHds18tGvKhog=
github.com/go-text/typesetting v0.1.1-0.20240402181327-ced1d6822703/go.mod h1:2+owI/sxa73XA581LAzVuEBZ3WEEV2pXeDswCH/3i1I=
github.com/go-text/typesetting-utils v0.0.0-20240317173224-1986cbe96c66 h1:GUrm65PQPlhFSKjLPGOZNPNxLCybjzjYBzjfoBGaDUY=

View File

@ -590,7 +590,7 @@ func drawTrianglesMRT(dsts [graphics.ShaderDstImageCount]*Image, srcs [graphics.
// i and a source image might share the same atlas even though i != src.
for _, dst := range dsts {
if src != nil && dst != nil && dst.backend.image == src.backend.image {
panic("atlas: Image.DrawTriangles: source must be different from the receiver")
panic("atlas: DrawTrianglesMRT: source must be different from the destination images")
}
}
}

View File

@ -218,7 +218,7 @@ func DrawTrianglesMRT(dsts [graphics.ShaderDstImageCount]*Image, srcs [graphics.
continue
}
if dst == src {
panic("buffered: DrawTrianglesMRT: source images must be different from the destination")
panic("buffered: DrawTrianglesMRT: source images must be different from the destination images")
}
}
if src != nil {

View File

@ -16,7 +16,11 @@ package graphics
const (
ShaderSrcImageCount = 4
ShaderDstImageCount = 4
// The minimum guaranteed value for the number of target seems to be 8
// OpenGL(8): https://www.khronos.org/opengl/wiki/Framebuffer_Object#Framebuffer_Object_Structure
// DirectX11(8): https://learn.microsoft.com/en-us/windows/win32/direct3d11/d3d10-graphics-programming-guide-output-merger-stage#multiple-rendertargets-overview
// Metal(8): Page 7 of 15: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
ShaderDstImageCount = 8
// PreservedUniformVariablesCount represents the number of preserved uniform variables.
// Any shaders in Ebitengine must have these uniform variables.

View File

@ -938,12 +938,12 @@ void F0(float l0, float l1, thread float& l2) {
},
Attributes: []shaderir.Type{
{Main: shaderir.Vec4},
{Main: shaderir.Float},
{Main: shaderir.Vec2},
{Main: shaderir.Vec4},
},
Varyings: []shaderir.Type{
{Main: shaderir.Float},
{Main: shaderir.Vec2},
{Main: shaderir.Vec4},
},
VertexFunc: shaderir.VertexFunc{
Block: block(
@ -967,10 +967,10 @@ void F0(float l0, float l1, thread float& l2) {
GlslVS: glslVertexPrelude + `
uniform float U0;
in vec4 A0;
in float A1;
in vec2 A2;
out float V0;
out vec2 V1;
in vec2 A1;
in vec4 A2;
out vec2 V0;
out vec4 V1;
void main(void) {
gl_Position = A0;
@ -979,8 +979,8 @@ void main(void) {
}`,
GlslFS: glslFragmentPrelude + `
uniform float U0;
in float V0;
in vec2 V1;`,
in vec2 V0;
in vec4 V1;`,
},
{
Name: "FragmentFunc",
@ -991,12 +991,12 @@ in vec2 V1;`,
},
Attributes: []shaderir.Type{
{Main: shaderir.Vec4},
{Main: shaderir.Float},
{Main: shaderir.Vec2},
{Main: shaderir.Vec4},
},
Varyings: []shaderir.Type{
{Main: shaderir.Float},
{Main: shaderir.Vec2},
{Main: shaderir.Vec4},
},
VertexFunc: shaderir.VertexFunc{
Block: block(
@ -1016,34 +1016,39 @@ in vec2 V1;`,
),
),
},
ColorsOutCount: 1,
FragmentFunc: shaderir.FragmentFunc{
Block: block(
[]shaderir.Type{
{Main: shaderir.Float},
{Main: shaderir.Vec2},
{Main: shaderir.Vec4},
},
3,
assignStmt(
localVariableExpr(3),
localVariableExpr(0),
),
3+1,
assignStmt(
localVariableExpr(4),
localVariableExpr(1),
),
returnStmt(
assignStmt(
localVariableExpr(5),
localVariableExpr(2),
),
assignStmt(
localVariableExpr(3),
localVariableExpr(0),
),
shaderir.Stmt{
Type: shaderir.Return,
},
),
},
},
GlslVS: glslVertexPrelude + `
uniform float U0;
in vec4 A0;
in float A1;
in vec2 A2;
out float V0;
out vec2 V1;
in vec2 A1;
in vec4 A2;
out vec2 V0;
out vec4 V1;
void main(void) {
gl_Position = A0;
@ -1052,21 +1057,22 @@ void main(void) {
}`,
GlslFS: glslFragmentPrelude + `
uniform float U0;
in float V0;
in vec2 V1;
in vec2 V0;
in vec4 V1;
vec4 F0(in vec4 l0, in float l1, in vec2 l2);
void F0(in vec4 l0, in vec2 l1, in vec4 l2, out vec4 l3);
vec4 F0(in vec4 l0, in float l1, in vec2 l2) {
float l3 = float(0);
void F0(in vec4 l0, in vec2 l1, in vec4 l2, out vec4 l3) {
vec2 l4 = vec2(0);
l3 = l0;
vec4 l5 = vec4(0);
l4 = l1;
return l2;
l5 = l2;
l3 = l0;
return;
}
void main(void) {
fragColor = F0(gl_FragCoord, V0, V1);
F0(gl_FragCoord, V0, V1, gl_FragData[0]);
}`,
},
}
@ -1088,14 +1094,14 @@ void main(void) {
t.Errorf("%s fragment: got: %s, want: %s", tc.Name, got, want)
}
}
m := msl.Compile(&tc.Program)
/*m := msl.Compile(&tc.Program)
if tc.Metal != "" {
got := m
want := tc.Metal + "\n"
if got != want {
t.Errorf("%s metal: got: %s, want: %s", tc.Name, got, want)
}
}
}*/
})
}
}