diff --git a/gameforui.go b/gameforui.go
index 7394bc7d2..f0867262b 100644
--- a/gameforui.go
+++ b/gameforui.go
@@ -60,7 +60,5 @@ func (c *gameForUI) Update() error {
 }
 
 func (c *gameForUI) Draw() {
-	// TODO: This is a dirty hack to fix #2362. Move setVerticesCache to ui.Image if possible.
-	c.offscreen.resolveSetVerticesCacheIfNeeded()
 	c.game.Draw(c.offscreen)
 }
diff --git a/image.go b/image.go
index 58a334791..671938278 100644
--- a/image.go
+++ b/image.go
@@ -32,11 +32,13 @@ import (
 type Image struct {
 	// addr holds self to check copying.
 	// See strings.Builder for similar examples.
-	addr             *Image
-	image            *ui.Image
-	original         *Image
-	setVerticesCache map[[2]int][4]byte
-	bounds           image.Rectangle
+	addr     *Image
+	image    *ui.Image
+	original *Image
+	bounds   image.Rectangle
+
+	// Do not add a 'cache' member that are resolved lazily.
+	// This tends to forget resolving the cache easily (#2362).
 }
 
 var emptyImage *Image
@@ -53,80 +55,6 @@ func (i *Image) copyCheck() {
 	}
 }
 
-func (i *Image) resolveSetVerticesCacheIfNeeded() {
-	if i.isSubImage() {
-		i = i.original
-	}
-
-	if len(i.setVerticesCache) == 0 {
-		return
-	}
-
-	l := len(i.setVerticesCache)
-	vs := graphics.Vertices(l * 4)
-	is := make([]uint16, l*6)
-	sx, sy := emptyImage.adjustPositionF32(1, 1)
-	var idx int
-	for p, c := range i.setVerticesCache {
-		dx := float32(p[0])
-		dy := float32(p[1])
-
-		var crf, cgf, cbf, caf float32
-		if c[3] != 0 {
-			crf = float32(c[0]) / float32(c[3])
-			cgf = float32(c[1]) / float32(c[3])
-			cbf = float32(c[2]) / float32(c[3])
-			caf = float32(c[3]) / 0xff
-		}
-
-		vs[graphics.VertexFloatCount*4*idx] = dx
-		vs[graphics.VertexFloatCount*4*idx+1] = dy
-		vs[graphics.VertexFloatCount*4*idx+2] = sx
-		vs[graphics.VertexFloatCount*4*idx+3] = sy
-		vs[graphics.VertexFloatCount*4*idx+4] = crf
-		vs[graphics.VertexFloatCount*4*idx+5] = cgf
-		vs[graphics.VertexFloatCount*4*idx+6] = cbf
-		vs[graphics.VertexFloatCount*4*idx+7] = caf
-		vs[graphics.VertexFloatCount*4*idx+8] = dx + 1
-		vs[graphics.VertexFloatCount*4*idx+9] = dy
-		vs[graphics.VertexFloatCount*4*idx+10] = sx + 1
-		vs[graphics.VertexFloatCount*4*idx+11] = sy
-		vs[graphics.VertexFloatCount*4*idx+12] = crf
-		vs[graphics.VertexFloatCount*4*idx+13] = cgf
-		vs[graphics.VertexFloatCount*4*idx+14] = cbf
-		vs[graphics.VertexFloatCount*4*idx+15] = caf
-		vs[graphics.VertexFloatCount*4*idx+16] = dx
-		vs[graphics.VertexFloatCount*4*idx+17] = dy + 1
-		vs[graphics.VertexFloatCount*4*idx+18] = sx
-		vs[graphics.VertexFloatCount*4*idx+19] = sy + 1
-		vs[graphics.VertexFloatCount*4*idx+20] = crf
-		vs[graphics.VertexFloatCount*4*idx+21] = cgf
-		vs[graphics.VertexFloatCount*4*idx+22] = cbf
-		vs[graphics.VertexFloatCount*4*idx+23] = caf
-		vs[graphics.VertexFloatCount*4*idx+24] = dx + 1
-		vs[graphics.VertexFloatCount*4*idx+25] = dy + 1
-		vs[graphics.VertexFloatCount*4*idx+26] = sx + 1
-		vs[graphics.VertexFloatCount*4*idx+27] = sy + 1
-		vs[graphics.VertexFloatCount*4*idx+28] = crf
-		vs[graphics.VertexFloatCount*4*idx+29] = cgf
-		vs[graphics.VertexFloatCount*4*idx+30] = cbf
-		vs[graphics.VertexFloatCount*4*idx+31] = caf
-
-		is[6*idx] = uint16(4 * idx)
-		is[6*idx+1] = uint16(4*idx + 1)
-		is[6*idx+2] = uint16(4*idx + 2)
-		is[6*idx+3] = uint16(4*idx + 1)
-		is[6*idx+4] = uint16(4*idx + 2)
-		is[6*idx+5] = uint16(4*idx + 3)
-
-		idx++
-	}
-	i.setVerticesCache = nil
-
-	srcs := [graphics.ShaderImageCount]*ui.Image{emptyImage.image}
-	i.image.DrawTriangles(srcs, vs, is, affine.ColorMIdentity{}, graphicsdriver.CompositeModeCopy, graphicsdriver.FilterNearest, graphicsdriver.AddressUnsafe, i.adjustedRegion(), graphicsdriver.Region{}, [graphics.ShaderImageCount - 1][2]float32{}, nil, nil, false, true)
-}
-
 // Size returns the size of the image.
 func (i *Image) Size() (width, height int) {
 	s := i.Bounds().Size()
@@ -156,7 +84,6 @@ func (i *Image) Fill(clr color.Color) {
 	if i.isDisposed() {
 		return
 	}
-	i.setVerticesCache = nil
 
 	var crf, cgf, cbf, caf float32
 	cr, cg, cb, ca := clr.RGBA()
@@ -283,9 +210,6 @@ func (i *Image) DrawImage(img *Image, options *DrawImageOptions) {
 		return
 	}
 
-	img.resolveSetVerticesCacheIfNeeded()
-	i.resolveSetVerticesCacheIfNeeded()
-
 	// Calculate vertices before locking because the user can do anything in
 	// options.ImageParts interface without deadlock (e.g. Call Image functions).
 	if options == nil {
@@ -439,9 +363,6 @@ func (i *Image) DrawTriangles(vertices []Vertex, indices []uint16, img *Image, o
 	}
 	// TODO: Check the maximum value of indices and len(vertices)?
 
-	img.resolveSetVerticesCacheIfNeeded()
-	i.resolveSetVerticesCacheIfNeeded()
-
 	if options == nil {
 		options = &DrawTrianglesOptions{}
 	}
@@ -547,8 +468,6 @@ func (i *Image) DrawTrianglesShader(vertices []Vertex, indices []uint16, shader
 	}
 	// TODO: Check the maximum value of indices and len(vertices)?
 
-	i.resolveSetVerticesCacheIfNeeded()
-
 	if options == nil {
 		options = &DrawTrianglesShaderOptions{}
 	}
@@ -593,7 +512,6 @@ func (i *Image) DrawTrianglesShader(vertices []Vertex, indices []uint16, shader
 				panic("ebiten: all the source images must be the same size with the rectangle")
 			}
 		}
-		img.resolveSetVerticesCacheIfNeeded()
 		imgs[i] = img.image
 	}
 
@@ -669,8 +587,6 @@ func (i *Image) DrawRectShader(width, height int, shader *Shader, options *DrawR
 		return
 	}
 
-	i.resolveSetVerticesCacheIfNeeded()
-
 	if options == nil {
 		options = &DrawRectShaderOptions{}
 	}
@@ -688,7 +604,6 @@ func (i *Image) DrawRectShader(width, height int, shader *Shader, options *DrawR
 		if w, h := img.Size(); width != w || height != h {
 			panic("ebiten: all the source images must be the same size with the rectangle")
 		}
-		img.resolveSetVerticesCacheIfNeeded()
 		imgs[i] = img.image
 	}
 
@@ -808,8 +723,6 @@ func (i *Image) ReadPixels(pixels []byte) {
 		return
 	}
 
-	i.resolveSetVerticesCacheIfNeeded()
-
 	x, y := i.adjustPosition(b.Min.X, b.Min.Y)
 	i.image.ReadPixels(pixels, x, y, b.Dx(), b.Dy())
 }
@@ -854,10 +767,8 @@ func (i *Image) at(x, y int) (r, g, b, a byte) {
 	if !image.Pt(x, y).In(i.Bounds()) {
 		return 0, 0, 0, 0
 	}
+
 	x, y = i.adjustPosition(x, y)
-	if c, ok := i.setVerticesCache[[2]int{x, y}]; ok {
-		return c[0], c[1], c[2], c[3]
-	}
 	var pix [4]byte
 	i.image.ReadPixels(pix[:], x, y, 1, 1)
 	return pix[0], pix[1], pix[2], pix[3]
@@ -884,16 +795,9 @@ func (i *Image) Set(x, y int, clr color.Color) {
 		i = i.original
 	}
 
-	if i.setVerticesCache == nil {
-		i.setVerticesCache = map[[2]int][4]byte{}
-	}
 	dx, dy := i.adjustPosition(x, y)
 	cr, cg, cb, ca := clr.RGBA()
-	i.setVerticesCache[[2]int{dx, dy}] = [4]byte{byte(cr / 0x101), byte(cg / 0x101), byte(cb / 0x101), byte(ca / 0x101)}
-	// One square requires 6 indices (= 2 triangles).
-	if len(i.setVerticesCache) >= graphics.IndicesCount/6 {
-		i.resolveSetVerticesCacheIfNeeded()
-	}
+	i.image.WritePixels([]byte{byte(cr / 0x101), byte(cg / 0x101), byte(cb / 0x101), byte(ca / 0x101)}, dx, dy, 1, 1)
 }
 
 // Dispose disposes the image data.
@@ -916,7 +820,6 @@ func (i *Image) Dispose() {
 	}
 	i.image.MarkDisposed()
 	i.image = nil
-	i.setVerticesCache = nil
 }
 
 // WritePixels replaces the pixels of the image.
@@ -936,8 +839,6 @@ func (i *Image) WritePixels(pixels []byte) {
 		return
 	}
 
-	i.resolveSetVerticesCacheIfNeeded()
-
 	r := i.Bounds()
 	x, y := i.adjustPosition(r.Min.X, r.Min.Y)
 	// Do not need to copy pixels here.
diff --git a/internal/ui/image.go b/internal/ui/image.go
index 819e2c927..7695295a0 100644
--- a/internal/ui/image.go
+++ b/internal/ui/image.go
@@ -35,6 +35,8 @@ type Image struct {
 	width    int
 	height   int
 	volatile bool
+
+	dotsCache map[[2]int][4]byte
 }
 
 func NewImage(width, height int, imageType atlas.ImageType) *Image {
@@ -52,14 +54,18 @@ func (i *Image) MarkDisposed() {
 	}
 	i.mipmap.MarkDisposed()
 	i.mipmap = nil
+	i.dotsCache = nil
 }
 
 func (i *Image) DrawTriangles(srcs [graphics.ShaderImageCount]*Image, vertices []float32, indices []uint16, colorm affine.ColorM, mode graphicsdriver.CompositeMode, filter graphicsdriver.Filter, address graphicsdriver.Address, dstRegion, srcRegion graphicsdriver.Region, subimageOffsets [graphics.ShaderImageCount - 1][2]float32, shader *Shader, uniforms [][]float32, evenOdd bool, canSkipMipmap bool) {
+	i.resolveDotsCacheIfNeeded()
+
 	var srcMipmaps [graphics.ShaderImageCount]*mipmap.Mipmap
 	for i, src := range srcs {
 		if src == nil {
 			continue
 		}
+		src.resolveDotsCacheIfNeeded()
 		srcMipmaps[i] = src.mipmap
 	}
 
@@ -72,6 +78,23 @@ func (i *Image) DrawTriangles(srcs [graphics.ShaderImageCount]*Image, vertices [
 }
 
 func (i *Image) WritePixels(pix []byte, x, y, width, height int) {
+	if width == 1 && height == 1 {
+		if i.dotsCache == nil {
+			i.dotsCache = map[[2]int][4]byte{}
+		}
+
+		var clr [4]byte
+		copy(clr[:], pix)
+		i.dotsCache[[2]int{x, y}] = clr
+
+		// One square requires 6 indices (= 2 triangles).
+		if len(i.dotsCache) >= graphics.IndicesCount/6 {
+			i.resolveDotsCacheIfNeeded()
+		}
+		return
+	}
+
+	i.resolveDotsCacheIfNeeded()
 	i.mipmap.WritePixels(pix, x, y, width, height)
 }
 
@@ -81,6 +104,17 @@ func (i *Image) ReadPixels(pixels []byte, x, y, width, height int) {
 		return
 	}
 
+	if width == 1 && height == 1 {
+		if c, ok := i.dotsCache[[2]int{x, y}]; ok {
+			copy(pixels, c[:])
+			return
+		}
+		// Do not call resolveDotsCacheIfNeeded here. This would slow (image/draw).Draw.
+		// See ebiten.TestImageDrawOver.
+	} else {
+		i.resolveDotsCacheIfNeeded()
+	}
+
 	if err := theUI.readPixels(i.mipmap, pixels, x, y, width, height); err != nil {
 		if panicOnErrorOnReadingPixels {
 			panic(err)
@@ -93,6 +127,82 @@ func (i *Image) DumpScreenshot(name string, blackbg bool) (string, error) {
 	return theUI.dumpScreenshot(i.mipmap, name, blackbg)
 }
 
+func (i *Image) resolveDotsCacheIfNeeded() {
+	if len(i.dotsCache) == 0 {
+		return
+	}
+
+	l := len(i.dotsCache)
+	vs := graphics.Vertices(l * 4)
+	is := make([]uint16, l*6)
+	sx, sy := float32(1), float32(1)
+	var idx int
+	for p, c := range i.dotsCache {
+		dx := float32(p[0])
+		dy := float32(p[1])
+
+		var crf, cgf, cbf, caf float32
+		if c[3] != 0 {
+			crf = float32(c[0]) / float32(c[3])
+			cgf = float32(c[1]) / float32(c[3])
+			cbf = float32(c[2]) / float32(c[3])
+			caf = float32(c[3]) / 0xff
+		}
+
+		vs[graphics.VertexFloatCount*4*idx] = dx
+		vs[graphics.VertexFloatCount*4*idx+1] = dy
+		vs[graphics.VertexFloatCount*4*idx+2] = sx
+		vs[graphics.VertexFloatCount*4*idx+3] = sy
+		vs[graphics.VertexFloatCount*4*idx+4] = crf
+		vs[graphics.VertexFloatCount*4*idx+5] = cgf
+		vs[graphics.VertexFloatCount*4*idx+6] = cbf
+		vs[graphics.VertexFloatCount*4*idx+7] = caf
+		vs[graphics.VertexFloatCount*4*idx+8] = dx + 1
+		vs[graphics.VertexFloatCount*4*idx+9] = dy
+		vs[graphics.VertexFloatCount*4*idx+10] = sx + 1
+		vs[graphics.VertexFloatCount*4*idx+11] = sy
+		vs[graphics.VertexFloatCount*4*idx+12] = crf
+		vs[graphics.VertexFloatCount*4*idx+13] = cgf
+		vs[graphics.VertexFloatCount*4*idx+14] = cbf
+		vs[graphics.VertexFloatCount*4*idx+15] = caf
+		vs[graphics.VertexFloatCount*4*idx+16] = dx
+		vs[graphics.VertexFloatCount*4*idx+17] = dy + 1
+		vs[graphics.VertexFloatCount*4*idx+18] = sx
+		vs[graphics.VertexFloatCount*4*idx+19] = sy + 1
+		vs[graphics.VertexFloatCount*4*idx+20] = crf
+		vs[graphics.VertexFloatCount*4*idx+21] = cgf
+		vs[graphics.VertexFloatCount*4*idx+22] = cbf
+		vs[graphics.VertexFloatCount*4*idx+23] = caf
+		vs[graphics.VertexFloatCount*4*idx+24] = dx + 1
+		vs[graphics.VertexFloatCount*4*idx+25] = dy + 1
+		vs[graphics.VertexFloatCount*4*idx+26] = sx + 1
+		vs[graphics.VertexFloatCount*4*idx+27] = sy + 1
+		vs[graphics.VertexFloatCount*4*idx+28] = crf
+		vs[graphics.VertexFloatCount*4*idx+29] = cgf
+		vs[graphics.VertexFloatCount*4*idx+30] = cbf
+		vs[graphics.VertexFloatCount*4*idx+31] = caf
+
+		is[6*idx] = uint16(4 * idx)
+		is[6*idx+1] = uint16(4*idx + 1)
+		is[6*idx+2] = uint16(4*idx + 2)
+		is[6*idx+3] = uint16(4*idx + 1)
+		is[6*idx+4] = uint16(4*idx + 2)
+		is[6*idx+5] = uint16(4*idx + 3)
+
+		idx++
+	}
+	i.dotsCache = nil
+
+	srcs := [graphics.ShaderImageCount]*mipmap.Mipmap{emptyImage.mipmap}
+	dr := graphicsdriver.Region{
+		X:      0,
+		Y:      0,
+		Width:  float32(i.width),
+		Height: float32(i.height),
+	}
+	i.mipmap.DrawTriangles(srcs, vs, is, affine.ColorMIdentity{}, graphicsdriver.CompositeModeCopy, graphicsdriver.FilterNearest, graphicsdriver.AddressUnsafe, dr, graphicsdriver.Region{}, [graphics.ShaderImageCount - 1][2]float32{}, nil, nil, false, true)
+}
+
 func DumpImages(dir string) (string, error) {
 	return theUI.dumpImages(dir)
 }