diff --git a/internal/affine/colorm.go b/internal/affine/colorm.go
index 06072d675..eed0f0234 100644
--- a/internal/affine/colorm.go
+++ b/internal/affine/colorm.go
@@ -17,6 +17,7 @@ package affine
 import (
 	"image/color"
 	"math"
+	"sync"
 )
 
 // ColorMDim is a dimension of a ColorM.
@@ -441,15 +442,16 @@ func (c *ColorM) Concat(other *ColorM) *ColorM {
 // Scale scales the matrix by (r, g, b, a).
 func (c *ColorM) Scale(r, g, b, a float32) *ColorM {
 	if !c.isInited() {
-		return &ColorM{
-			body: []float32{
-				r, 0, 0, 0,
-				0, g, 0, 0,
-				0, 0, b, 0,
-				0, 0, 0, a,
-			},
-		}
+		return getCachedScalingColorM(r, g, b, a)
 	}
+
+	if c.ScaleOnly() {
+		if c.body == nil {
+			return getCachedScalingColorM(r, g, b, a)
+		}
+		return getCachedScalingColorM(r * c.body[0], g * c.body[5], b * c.body[10], a * c.body[15])
+	}
+
 	eb := make([]float32, len(colorMIdentityBody))
 	if c.body != nil {
 		copy(eb, c.body)
@@ -549,3 +551,62 @@ func (c *ColorM) ChangeHSV(hueTheta float64, saturationScale float32, valueScale
 	c = c.Concat(yCbCrToRgb)
 	return c
 }
+
+type cachedScalingColorMKey struct {
+	r, g, b, a float32
+}
+
+type cachedScalingColorMValue struct {
+	c     *ColorM
+	atime uint64
+}
+
+var (
+	cachedScalingColorM  = map[cachedScalingColorMKey]*cachedScalingColorMValue{}
+	cachedScalingColorMM sync.Mutex
+	cacheMonotonicClock  uint64
+)
+
+func getCachedScalingColorM(r, g, b, a float32) *ColorM {
+	key := cachedScalingColorMKey{r, g, b, a}
+
+	cachedScalingColorMM.Lock()
+	defer cachedScalingColorMM.Unlock()
+
+	cacheMonotonicClock++
+	now := cacheMonotonicClock
+
+	if v, ok := cachedScalingColorM[key]; ok {
+		v.atime = now
+		return v.c
+	}
+
+	const maxCacheSize = 512 // An arbitrary number
+
+	for len(cachedScalingColorM) >= maxCacheSize {
+		var oldest uint64 = math.MaxUint64
+		var oldestKey cachedScalingColorMKey
+		for k, v := range cachedScalingColorM {
+			if v.atime < oldest {
+				oldestKey = k
+				oldest = v.atime
+			}
+		}
+		delete(cachedScalingColorM, oldestKey)
+	}
+
+	v := &cachedScalingColorMValue{
+		c: &ColorM{
+			body: []float32{
+				r, 0, 0, 0,
+				0, g, 0, 0,
+				0, 0, b, 0,
+				0, 0, 0, a,
+			},
+		},
+		atime: now,
+	}
+	cachedScalingColorM[key] = v
+
+	return v.c
+}
diff --git a/internal/affine/colorm_test.go b/internal/affine/colorm_test.go
index 684594927..26c67f8ca 100644
--- a/internal/affine/colorm_test.go
+++ b/internal/affine/colorm_test.go
@@ -22,6 +22,33 @@ import (
 	. "github.com/hajimehoshi/ebiten/v2/internal/affine"
 )
 
+func TestColorMScale(t *testing.T) {
+	cases := []struct {
+		In  *ColorM
+		Out *ColorM
+	}{
+		{
+			nil,
+			(*ColorM)(nil).Scale(0.25, 0.5, 0.75, 1),
+		},
+		{
+			(*ColorM)(nil).Scale(0.5, 0.5, 0.5, 0.8),
+			(*ColorM)(nil).Scale(0.125, 0.25, 0.375, 0.8),
+		},
+		{
+			(*ColorM)(nil).Translate(0, 0, 0, 0),
+			(*ColorM)(nil).Scale(0.25, 0.5, 0.75, 1),
+		},
+	}
+	for _, c := range cases {
+		got := c.In.Scale(0.25, 0.5, 0.75, 1)
+		want := c.Out
+		if got != want {
+			t.Errorf("%v.Scale(): got: %v, want: %v", c.In, got, want)
+		}
+	}
+}
+
 func TestColorMScaleOnly(t *testing.T) {
 	cases := []struct {
 		In  *ColorM
diff --git a/text/text.go b/text/text.go
index aaa66e966..df8de9ddd 100644
--- a/text/text.go
+++ b/text/text.go
@@ -27,7 +27,6 @@ import (
 	"golang.org/x/image/math/fixed"
 
 	"github.com/hajimehoshi/ebiten/v2"
-	"github.com/hajimehoshi/ebiten/v2/internal/colormcache"
 	"github.com/hajimehoshi/ebiten/v2/internal/hooks"
 )
 
@@ -167,13 +166,19 @@ func Draw(dst *ebiten.Image, text string, face font.Face, x, y int, clr color.Co
 	textM.Lock()
 	defer textM.Unlock()
 
+	cr, cg, cb, ca := clr.RGBA()
+	if ca == 0 {
+		return
+	}
+
+	var colorm ebiten.ColorM
+	colorm.Scale(float64(cr)/float64(ca), float64(cg)/float64(ca), float64(cb)/float64(ca), float64(ca)/0xffff)
+
 	fx, fy := fixed.I(x), fixed.I(y)
 	prevR := rune(-1)
 
 	faceHeight := face.Metrics().Height
 
-	colorm := colormcache.ColorToColorM(clr)
-
 	for _, r := range text {
 		if prevR >= 0 {
 			fx += face.Kern(prevR, r)