diff --git a/internal/atlas/image.go b/internal/atlas/image.go
index 6f2f6cb96..430aed5f6 100644
--- a/internal/atlas/image.go
+++ b/internal/atlas/image.go
@@ -230,7 +230,6 @@ func (i *Image) paddingSize() int {
 
 func (i *Image) ensureIsolatedFromSource(backends []*backend) {
 	i.resetUsedAsSourceCount()
-	imagesUsedAsDestination[i] = struct{}{}
 
 	if i.backend == nil {
 		// `theSourceBackendsForOneFrame` already includes `backends`.
@@ -246,6 +245,8 @@ func (i *Image) ensureIsolatedFromSource(backends []*backend) {
 		return
 	}
 
+	imagesUsedAsDestination[i] = struct{}{}
+
 	// Check if i has the same backend as the given backends.
 	var needsIsolation bool
 	for _, b := range backends {