internal/ui: bug fix: deadlock at Layout with Ebiten's image functions called

Closes #2079

internal/processtest/testdata/issue2079.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The Ebiten Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"errors"
+	"image/color"
+	"time"
+
+	"github.com/hajimehoshi/ebiten/v2"
+)
+
+var regularTermination = errors.New("regular termination")
+
+type Game struct {
+	count int
+}
+
+func (g *Game) Update() error {
+	g.count++
+	if g.count >= 2 {
+		return regularTermination
+	}
+	return nil
+}
+
+func (g *Game) Draw(screen *ebiten.Image) {
+}
+
+func (g *Game) Layout(width, height int) (int, int) {
+	// Ebiten's image function should be able to be called even in Layout.
+	done := make(chan struct{})
+	timeout := time.After(time.Second)
+	go func() {
+		select {
+		case <-done:
+		case <-timeout:
+			panic("timeout")
+		}
+	}()
+	defer close(done)
+
+	i := ebiten.NewImage(width, height)
+	i.Fill(color.White)
+	i.Dispose()
+	return width, height
+}
+
+func main() {
+	if err := ebiten.RunGame(&Game{}); err != nil && err != regularTermination {
+		panic(err)
+	}
+}

internal/ui/context.go

@@ -79,7 +79,7 @@ func (c *context) forceUpdateFrame(graphicsDriver graphicsdriver.Graphics, outsi
 	return nil
 }
 
-func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, updateCount int, outsideWidth, outsideHeight float64, deviceScaleFactor float64) error {
+func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, updateCount int, outsideWidth, outsideHeight float64, deviceScaleFactor float64) (err error) {
 	if err := theGlobalState.error(); err != nil {
 		return err
 	}
@@ -90,16 +90,29 @@ func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, update
 		return nil
 	}
 
-	// ForceUpdate can be invoked even if the context is not initialized yet (#1591).
-	if w, h := c.layoutGame(outsideWidth, outsideHeight, deviceScaleFactor); w == 0 || h == 0 {
-		return nil
-	}
-
 	debug.Logf("----\n")
 
 	if err := buffered.BeginFrame(graphicsDriver); err != nil {
 		return err
 	}
+	defer func() {
+		// All the vertices data are consumed at the end of the frame, and the data backend can be
+		// available after that. Until then, lock the vertices backend.
+		err1 := graphics.LockAndResetVertices(func() error {
+			if err := buffered.EndFrame(graphicsDriver); err != nil {
+				return err
+			}
+			return nil
+		})
+		if err == nil {
+			err = err1
+		}
+	}()
+
+	// ForceUpdate can be invoked even if the context is not initialized yet (#1591).
+	if w, h := c.layoutGame(outsideWidth, outsideHeight, deviceScaleFactor); w == 0 || h == 0 {
+		return nil
+	}
 
 	// Ensure that Update is called once before Draw so that Update can be used for initialization.
 	if !c.updateCalled && updateCount == 0 {
@@ -128,12 +141,7 @@ func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, update
 
 	// All the vertices data are consumed at the end of the frame, and the data backend can be
 	// available after that. Until then, lock the vertices backend.
-	return graphics.LockAndResetVertices(func() error {
+	return nil
-		if err := buffered.EndFrame(graphicsDriver); err != nil {
-			return err
-		}
-		return nil
-	})
 }
 
 func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics) {