internal/ui: add sleep for an environment where vsync doesn't work

Updates #2952

internal/ui/context.go

@@ -54,7 +54,7 @@ type context struct {
 	offscreenHeight float64
 
 	isOffscreenModified bool
-	lastDrawTime        time.Time
+	lastSwapBufferTime  time.Time
 
 	skipCount int
 
@@ -70,7 +70,14 @@ func newContext(game Game) *context {
 
 func (c *context) updateFrame(graphicsDriver graphicsdriver.Graphics, outsideWidth, outsideHeight float64, deviceScaleFactor float64, ui *UserInterface) error {
 	// TODO: If updateCount is 0 and vsync is disabled, swapping buffers can be skipped.
-	return c.updateFrameImpl(graphicsDriver, clock.UpdateFrame(), outsideWidth, outsideHeight, deviceScaleFactor, ui, false)
+	needsSwapBuffers, err := c.updateFrameImpl(graphicsDriver, clock.UpdateFrame(), outsideWidth, outsideHeight, deviceScaleFactor, ui, false)
+	if err != nil {
+		return err
+	}
+	if err := c.swapBuffersOrWait(needsSwapBuffers, graphicsDriver, ui.FPSMode() == FPSModeVsyncOn); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (c *context) forceUpdateFrame(graphicsDriver graphicsdriver.Graphics, outsideWidth, outsideHeight float64, deviceScaleFactor float64, ui *UserInterface) error {
@@ -81,33 +88,32 @@ func (c *context) forceUpdateFrame(graphicsDriver graphicsdriver.Graphics, outsi
 		n = 2
 	}
 	for i := 0; i < n; i++ {
-		if err := c.updateFrameImpl(graphicsDriver, 1, outsideWidth, outsideHeight, deviceScaleFactor, ui, true); err != nil {
+		needsSwapBuffers, err := c.updateFrameImpl(graphicsDriver, 1, outsideWidth, outsideHeight, deviceScaleFactor, ui, true)
+		if err != nil {
+			return err
+		}
+		if err := c.swapBuffersOrWait(needsSwapBuffers, graphicsDriver, ui.FPSMode() == FPSModeVsyncOn); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, updateCount int, outsideWidth, outsideHeight float64, deviceScaleFactor float64, ui *UserInterface, forceDraw bool) (err error) {
+func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, updateCount int, outsideWidth, outsideHeight float64, deviceScaleFactor float64, ui *UserInterface, forceDraw bool) (needsSwapBuffers bool, err error) {
 	// The given outside size can be 0 e.g. just after restoring from the fullscreen mode on Windows (#1589)
 	// Just ignore such cases. Otherwise, creating a zero-sized framebuffer causes a panic.
 	if outsideWidth == 0 || outsideHeight == 0 {
-		return nil
+		return false, nil
 	}
 
 	debug.FrameLogf("----\n")
 
 	if err := atlas.BeginFrame(graphicsDriver); err != nil {
-		return err
+		return false, err
 	}
-
 	defer func() {
 		if err1 := atlas.EndFrame(); err1 != nil && err == nil {
-			err = err1
+			needsSwapBuffers = false
-			return
-		}
-
-		if err1 := atlas.SwapBuffers(graphicsDriver); err1 != nil && err == nil {
 			err = err1
 			return
 		}
@@ -115,17 +121,17 @@ func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, update
 
 	// Flush deferred functions, like reading pixels from GPU.
 	if err := c.processFuncsInFrame(ui); err != nil {
-		return err
+		return false, err
 	}
 
 	// ForceUpdate can be invoked even if the context is not initialized yet (#1591).
 	if w, h := c.layoutGame(outsideWidth, outsideHeight, deviceScaleFactor); w == 0 || h == 0 {
-		return nil
+		return false, nil
 	}
 
 	// Update the input state after the layout is updated as a cursor position is affected by the layout.
 	if err := ui.updateInputState(); err != nil {
-		return err
+		return false, err
 	}
 
 	// Ensure that Update is called once before Draw so that Update can be used for initialization.
@@ -143,15 +149,15 @@ func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, update
 		})
 
 		if err := hook.RunBeforeUpdateHooks(); err != nil {
-			return err
+			return false, err
 		}
 		if err := c.game.Update(); err != nil {
-			return err
+			return false, err
 		}
 
 		// Catch the error that happened at (*Image).At.
 		if err := ui.error(); err != nil {
-			return err
+			return false, err
 		}
 
 		ui.tick.Add(1)
@@ -160,13 +166,40 @@ func (c *context) updateFrameImpl(graphicsDriver graphicsdriver.Graphics, update
 	// Update window icons during a frame, since an icon might be *ebiten.Image and
 	// getting pixels from it needs to be in a frame (#1468).
 	if err := ui.updateIconIfNeeded(); err != nil {
-		return err
+		return false, err
 	}
 
 	// Draw the game.
-	if err := c.drawGame(graphicsDriver, ui, forceDraw); err != nil {
+	return c.drawGame(graphicsDriver, ui, forceDraw)
+}
+
+func (c *context) swapBuffersOrWait(needsSwapBuffers bool, graphicsDriver graphicsdriver.Graphics, vsyncEnabled bool) error {
+	now := time.Now()
+	defer func() {
+		c.lastSwapBufferTime = now
+	}()
+
+	if needsSwapBuffers {
+		if err := atlas.SwapBuffers(graphicsDriver); err != nil {
 			return err
 		}
+	}
+
+	var waitTime time.Duration
+	if !needsSwapBuffers {
+		// When swapping buffers is skipped and Draw is called too early, sleep for a while to suppress CPU usages (#2890).
+		waitTime = time.Second / 60
+	} else if vsyncEnabled {
+		// In some environments, e.g. Linux on Parallels, SwapBuffers doesn't wait for the vsync (#2952).
+		// In the case when the display has high refresh rates like 240 [Hz], the wait time should be small.
+		waitTime = time.Millisecond
+	}
+	if waitTime > 0 {
+		if delta := waitTime - now.Sub(c.lastSwapBufferTime); delta > 0 {
+			println(waitTime.String(), now.Sub(c.lastSwapBufferTime).String())
+			time.Sleep(delta)
+		}
+	}
 
 	return nil
 }
@@ -179,7 +212,7 @@ func (c *context) newOffscreenImage(w, h int) *Image {
 	return img
 }
 
-func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics, ui *UserInterface, forceDraw bool) error {
+func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics, ui *UserInterface, forceDraw bool) (needSwapBuffers bool, err error) {
 	if (c.offscreen.imageType == atlas.ImageTypeVolatile) != ui.IsScreenClearedEveryFrame() {
 		w, h := c.offscreen.width, c.offscreen.height
 		c.offscreen.Deallocate()
@@ -197,7 +230,7 @@ func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics, ui *UserInter
 	}
 
 	if err := c.game.DrawOffscreen(); err != nil {
-		return err
+		return false, err
 	}
 
 	const maxSkipCount = 4
@@ -210,12 +243,10 @@ func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics, ui *UserInter
 		c.skipCount = 0
 	}
 
-	now := time.Now()
+	if c.skipCount >= maxSkipCount {
-	defer func() {
+		return false, nil
-		c.lastDrawTime = now
+	}
-	}()
 
-	if c.skipCount < maxSkipCount {
 	if graphicsDriver.NeedsClearingScreen() {
 		// This clear is needed for fullscreen mode or some mobile platforms (#622).
 		c.screen.clear()
@@ -226,12 +257,7 @@ func (c *context) drawGame(graphicsDriver graphicsdriver.Graphics, ui *UserInter
 	// The final screen is never used as the rendering source.
 	// Flush its buffer here just in case.
 	c.screen.flushBufferIfNeeded()
-	} else if delta := time.Second/60 - now.Sub(c.lastDrawTime); delta > 0 {
+	return true, nil
-		// When swapping buffers is skipped and Draw is called too early, sleep for a while to suppress CPU usages (#2890).
-		time.Sleep(delta)
-	}
-
-	return nil
 }
 
 func (c *context) layoutGame(outsideWidth, outsideHeight float64, deviceScaleFactor float64) (int, int) {
@@ -240,8 +266,13 @@ func (c *context) layoutGame(outsideWidth, outsideHeight float64, deviceScaleFac
 		panic("ui: Layout must return positive numbers")
 	}
 
-	c.screenWidth = outsideWidth * deviceScaleFactor
+	screenWidth := outsideWidth * deviceScaleFactor
-	c.screenHeight = outsideHeight * deviceScaleFactor
+	screenHeight := outsideHeight * deviceScaleFactor
+	if c.screenWidth != screenWidth || c.screenHeight != screenHeight {
+		c.skipCount = 0
+	}
+	c.screenWidth = screenWidth
+	c.screenHeight = screenHeight
 	c.offscreenWidth = owf
 	c.offscreenHeight = ohf
 

internal/ui/ui_android.go

@@ -18,15 +18,19 @@ package ui
 #include <jni.h>
 #include <stdlib.h>
 
-// Basically same as:
+// The following JNI code works as this pseudo Java code:
 //
 //     WindowService windowService = context.getSystemService(Context.WINDOW_SERVICE);
 //     Display display = windowManager.getDefaultDisplay();
 //     DisplayMetrics displayMetrics = new DisplayMetrics();
 //     display.getRealMetrics(displayMetrics);
-//     this.deviceScale = displayMetrics.density;
+//     return displayMetrics.widthPixels, displayMetrics.heightPixels, displayMetrics.density;
 //
-static float deviceScale(uintptr_t java_vm, uintptr_t jni_env, uintptr_t ctx) {
+static void displayInfo(int* width, int* height, float* scale, uintptr_t java_vm, uintptr_t jni_env, uintptr_t ctx) {
+  *width = 0;
+  *height = 0;
+  *scale = 1;
+
   JavaVM* vm = (JavaVM*)java_vm;
   JNIEnv* env = (JNIEnv*)jni_env;
   jobject context = (jobject)ctx;
@@ -64,7 +68,15 @@ static float deviceScale(uintptr_t java_vm, uintptr_t jni_env, uintptr_t ctx) {
       env, display,
       (*env)->GetMethodID(env, android_view_Display, "getRealMetrics", "(Landroid/util/DisplayMetrics;)V"),
       displayMetrics);
-  const float density =
+  *width =
+      (*env)->GetIntField(
+          env, displayMetrics,
+          (*env)->GetFieldID(env, android_util_DisplayMetrics, "widthPixels", "I"));
+  *height =
+      (*env)->GetIntField(
+          env, displayMetrics,
+          (*env)->GetFieldID(env, android_util_DisplayMetrics, "heightPixels", "I"));
+  *scale =
       (*env)->GetFloatField(
           env, displayMetrics,
           (*env)->GetFieldID(env, android_util_DisplayMetrics, "density", "F"));
@@ -78,15 +90,12 @@ static float deviceScale(uintptr_t java_vm, uintptr_t jni_env, uintptr_t ctx) {
   (*env)->DeleteLocalRef(env, windowManager);
   (*env)->DeleteLocalRef(env, display);
   (*env)->DeleteLocalRef(env, displayMetrics);
-
-  return density;
 }
 */
 import "C"
 
 import (
 	"errors"
-	"fmt"
 
 	"github.com/ebitengine/gomobile/app"
 
@@ -119,18 +128,27 @@ func (*graphicsDriverCreatorImpl) newPlayStation5() (graphicsdriver.Graphics, er
 	return nil, errors.New("ui: PlayStation 5 is not supported in this environment")
 }
 
-func (u *UserInterface) deviceScaleFactor() float64 {
-	var s float64
-	if err := app.RunOnJVM(func(vm, env, ctx uintptr) error {
-		// TODO: This might be crash when this is called from init(). How can we detect this?
-		s = float64(C.deviceScale(C.uintptr_t(vm), C.uintptr_t(env), C.uintptr_t(ctx)))
-		return nil
-	}); err != nil {
-		panic(fmt.Sprintf("devicescale: error %v", err))
-	}
-	return s
-}
-
 func dipToNativePixels(x float64, scale float64) float64 {
 	return x * scale
 }
+
+func dipFromNativePixels(x float64, scale float64) float64 {
+	return x / scale
+}
+
+func (u *UserInterface) displayInfo() (int, int, float64, bool) {
+	var cWidth, cHeight C.int
+	var cScale C.float
+	if err := app.RunOnJVM(func(vm, env, ctx uintptr) error {
+		C.displayInfo(&cWidth, &cHeight, &cScale, C.uintptr_t(vm), C.uintptr_t(env), C.uintptr_t(ctx))
+		return nil
+	}); err != nil {
+		// JVM is not ready yet.
+		// TODO: Fix gomobile to detect the error type for this case.
+		return 0, 0, 1, false
+	}
+	scale := float64(cScale)
+	width := int(dipFromNativePixels(float64(cWidth), scale))
+	height := int(dipFromNativePixels(float64(cHeight), scale))
+	return width, height, scale, true
+}

internal/ui/ui_ios.go

@@ -19,31 +19,43 @@ package ui
 //
 // #import <UIKit/UIKit.h>
 //
-// static double devicePixelRatioOnMainThread(UIView* view) {
+// static void displayInfoOnMainThread(float* width, float* height, float* scale, UIView* view) {
+//   *width = 0;
+//   *height = 0;
+//   *scale = 1;
 //   UIWindow* window = view.window;
 //   if (!window) {
-//     return 1;
+//     return;
 //   }
 //   UIWindowScene* scene = window.windowScene;
 //   if (!scene) {
-//     return 1;
+//     return;
 //   }
-//   return scene.screen.nativeScale;
+//   CGRect bounds = scene.screen.bounds;
+//   *width = bounds.size.width;
+//   *height = bounds.size.height;
+//   *scale = scene.screen.nativeScale;
 // }
 //
-// static double devicePixelRatio(uintptr_t viewPtr) {
+// static void displayInfo(float* width, float* height, float* scale, uintptr_t viewPtr) {
+//   *width = 0;
+//   *height = 0;
+//   *scale = 1;
 //   if (!viewPtr) {
-//     return 1;
+//     return;
 //   }
 //   UIView* view = (__bridge UIView*)(void*)viewPtr;
 //   if ([NSThread isMainThread]) {
-//     return devicePixelRatioOnMainThread(view);
+//     displayInfoOnMainThread(width, height, scale, view);
+//     return;
 //   }
-//   __block double scale;
+//   __block float w, h, s;
 //   dispatch_sync(dispatch_get_main_queue(), ^{
-//     scale = devicePixelRatioOnMainThread(view);
+//     displayInfoOnMainThread(&w, &h, &s, view);
 //   });
-//   return scale;
+//   *width = w;
+//   *height = h;
+//   *scale = s;
 // }
 import "C"
 
@@ -113,10 +125,24 @@ func (u *UserInterface) IsGL() (bool, error) {
 	return u.GraphicsLibrary() == GraphicsLibraryOpenGL, nil
 }
 
-func (u *UserInterface) deviceScaleFactor() float64 {
-	return float64(C.devicePixelRatio(C.uintptr_t(u.uiView.Load())))
-}
-
 func dipToNativePixels(x float64, scale float64) float64 {
 	return x
 }
+
+func dipFromNativePixels(x float64, scale float64) float64 {
+	return x
+}
+
+func (u *UserInterface) displayInfo() (int, int, float64, bool) {
+	view := u.uiView.Load()
+	if view == 0 {
+		return 0, 0, 1, false
+	}
+
+	var cWidth, cHeight, cScale C.float
+	C.displayInfo(&cWidth, &cHeight, &cScale, C.uintptr_t(view))
+	scale := float64(cScale)
+	width := int(dipFromNativePixels(float64(cWidth), scale))
+	height := int(dipFromNativePixels(float64(cHeight), scale))
+	return width, height, scale, true
+}

internal/ui/ui_mobile.go

@@ -268,8 +268,10 @@ func (u *UserInterface) Window() Window {
 }
 
 type Monitor struct {
+	width             int
+	height            int
 	deviceScaleFactor float64
-	deviceScaleFactorOnce sync.Once
+	inited            atomic.Bool
 
 	m sync.Mutex
 }
@@ -280,22 +282,35 @@ func (m *Monitor) Name() string {
 	return ""
 }
 
-func (m *Monitor) DeviceScaleFactor() float64 {
+func (m *Monitor) ensureInit() {
+	if m.inited.Load() {
+		return
+	}
+
 	m.m.Lock()
 	defer m.m.Unlock()
+	// Re-check the state since the state might be changed while locking.
+	if m.inited.Load() {
+		return
+	}
+	width, height, scale, ok := theUI.displayInfo()
+	if !ok {
+		return
+	}
+	m.width = width
+	m.height = height
+	m.deviceScaleFactor = scale
+	m.inited.Store(true)
+}
 
-	// The device scale factor can be obtained after the main function starts, especially on Android.
+func (m *Monitor) DeviceScaleFactor() float64 {
-	// Initialize this lazily.
+	m.ensureInit()
-	m.deviceScaleFactorOnce.Do(func() {
-		// Assume that the device scale factor never changes on mobiles.
-		m.deviceScaleFactor = theUI.deviceScaleFactor()
-	})
 	return m.deviceScaleFactor
 }
 
 func (m *Monitor) Size() (int, int) {
-	// TODO: Return a valid value.
+	m.ensureInit()
-	return 0, 0
+	return m.width, m.height
 }
 
 func (u *UserInterface) AppendMonitors(mons []*Monitor) []*Monitor {

monitor.go

@@ -32,8 +32,7 @@ func (m *MonitorType) Name() string {
 // DeviceScaleFactor returns a meaningful value on high-DPI display environment,
 // otherwise DeviceScaleFactor returns 1.
 //
-// DeviceScaleFactor might panic on init function on some devices like Android.
+// On mobiles, DeviceScaleFactor returns 1 before the game starts e.g. in init functions.
-// Then, it is not recommended to call DeviceScaleFactor from init functions.
 func (m *MonitorType) DeviceScaleFactor() float64 {
 	return (*ui.Monitor)(m).DeviceScaleFactor()
 }
@@ -42,7 +41,7 @@ func (m *MonitorType) DeviceScaleFactor() float64 {
 // This is the same as the screen size in fullscreen mode.
 // The returned value can be given to SetSize function if the perfectly fit fullscreen is needed.
 //
-// On mobiles, Size returns (0, 0) so far.
+// On mobiles, Size returns (0, 0) before the game starts e.g. in init functions.
 //
 // Size's use cases are limited. If you are making a fullscreen application, you can use RunGame and
 // the Game interface's Layout function instead. If you are making a not-fullscreen application but the application's