all: add a new package shaderprecomp

The current implementation is only for macOS so far.

Updates #2861
This commit is contained in:
Hajime Hoshi 2024-04-20 23:48:33 +09:00
parent d7df5ebcbd
commit c46f62e184
17 changed files with 599 additions and 12 deletions

3
.gitignore vendored
View File

@ -7,3 +7,6 @@
.vscode
go.work
go.work.sum
*.metallib
!dummy.metallib

View File

@ -0,0 +1,33 @@
// Copyright 2020 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ignore
//kage:unit pixels
package main
var Time float
var Cursor vec2
func Fragment(dstPos vec4, srcPos vec2, color vec4) vec4 {
pos := (dstPos.xy - imageDstOrigin()) / imageDstSize()
pos += Cursor / imageDstSize() / 4
clr := 0.0
clr += sin(pos.x*cos(Time/15)*80) + cos(pos.y*cos(Time/15)*10)
clr += sin(pos.y*sin(Time/10)*40) + cos(pos.x*sin(Time/25)*40)
clr += sin(pos.x*sin(Time/5)*10) + sin(pos.y*sin(Time/35)*80)
clr *= sin(Time/10) * 0.5
return vec4(clr, clr*0.5, sin(clr+Time/3)*0.75, 1)
}

View File

@ -0,0 +1,73 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
_ "embed"
"log"
"github.com/hajimehoshi/ebiten/v2"
"github.com/hajimehoshi/ebiten/v2/ebitenutil"
)
//go:embed defaultshader.go
var defaultShaderSourceBytes []byte
type Game struct {
defaultShader *ebiten.Shader
counter int
}
func (g *Game) Update() error {
g.counter++
if g.defaultShader == nil {
s, err := ebiten.NewShader(defaultShaderSourceBytes)
if err != nil {
return err
}
g.defaultShader = s
}
return nil
}
func (g *Game) Draw(screen *ebiten.Image) {
cx, cy := ebiten.CursorPosition()
w, h := screen.Bounds().Dx(), screen.Bounds().Dy()
op := &ebiten.DrawRectShaderOptions{}
op.Uniforms = map[string]interface{}{
"Time": float32(g.counter) / float32(ebiten.TPS()),
"Cursor": []float32{float32(cx), float32(cy)},
}
screen.DrawRectShader(w, h, g.defaultShader, op)
msg := `This is a test for shader precompilation.
Precompilation works only on macOS so far.
Note that this example still works even without shader precompilation.`
ebitenutil.DebugPrint(screen, msg)
}
func (g *Game) Layout(outsideWidth, outsideHeight int) (screenWidth, screenHeight int) {
return outsideWidth, outsideHeight
}
func main() {
if err := registerPrecompiledShaders(); err != nil {
log.Fatal(err)
}
if err := ebiten.RunGame(&Game{}); err != nil {
log.Fatal(err)
}
}

View File

@ -0,0 +1 @@
This is a dummy .metallib file to trick Go's embed package.

View File

@ -0,0 +1,103 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ignore
// This is a program to generate precompiled Metal libraries.
//
// See https://developer.apple.com/documentation/metal/shader_libraries/building_a_shader_library_by_precompiling_source_files.
package main
import (
"os"
"os/exec"
"path/filepath"
"golang.org/x/sync/errgroup"
"github.com/hajimehoshi/ebiten/v2/shaderprecomp"
)
func main() {
if err := run(); err != nil {
panic(err)
}
}
func run() error {
tmpdir, err := os.MkdirTemp("", "")
if err != nil {
return err
}
defer os.RemoveAll(tmpdir)
srcs := shaderprecomp.AppendBuildinShaderSources(nil)
defaultSrcBytes, err := os.ReadFile(filepath.Join("..", "defaultshader.go"))
if err != nil {
return err
}
defaultSrc, err := shaderprecomp.NewShaderSource(defaultSrcBytes)
if err != nil {
return err
}
srcs = append(srcs, defaultSrc)
var wg errgroup.Group
for _, src := range srcs {
source := src
wg.Go(func() error {
return compile(source, tmpdir)
})
}
if err := wg.Wait(); err != nil {
return err
}
return nil
}
func compile(source *shaderprecomp.ShaderSource, tmpdir string) error {
id := source.ID().String()
metalFilePath := filepath.Join(tmpdir, id+".metal")
f, err := os.Create(metalFilePath)
if err != nil {
return err
}
defer f.Close()
if err := shaderprecomp.CompileToMSL(f, source); err != nil {
return err
}
if err := f.Sync(); err != nil {
return err
}
irFilePath := filepath.Join(tmpdir, id+".ir")
cmd := exec.Command("xcrun", "-sdk", "macosx", "metal", "-o", irFilePath, "-c", metalFilePath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
metallibFilePath := id + ".metallib"
cmd = exec.Command("xcrun", "-sdk", "macosx", "metallib", "-o", metallibFilePath, irFilePath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,17 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:generate go run gen.go
package metallib

View File

@ -0,0 +1,52 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"embed"
"errors"
"fmt"
"io/fs"
"os"
"github.com/hajimehoshi/ebiten/v2/shaderprecomp"
)
//go:embed metallib/*.metallib
var metallibs embed.FS
func registerPrecompiledShaders() error {
srcs := shaderprecomp.AppendBuildinShaderSources(nil)
defaultShaderSource, err := shaderprecomp.NewShaderSource(defaultShaderSourceBytes)
if err != nil {
return err
}
srcs = append(srcs, defaultShaderSource)
for _, src := range srcs {
name := src.ID().String() + ".metallib"
lib, err := metallibs.ReadFile("metallib/" + name)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
fmt.Fprintf(os.Stderr, "precompiled Metal library %s was not found. Run 'go generate' for 'metallib' directory to generate them\n", name)
continue
}
return err
}
shaderprecomp.RegisterMetalLibrary(src, lib)
}
return nil
}

View File

@ -0,0 +1,27 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !darwin
package main
import (
"fmt"
"os"
)
func registerPrecompiledShaders() error {
fmt.Fprintf(os.Stderr, "precompiled shaders are not available in this environment.\n")
return nil
}

View File

@ -197,3 +197,13 @@ func Fragment(dstPos vec4, srcPos vec2, color vec4) vec4 {
return vec4(0)
}
`)
func AppendShaderSources(sources [][]byte) [][]byte {
for filter := Filter(0); filter < FilterCount; filter++ {
for address := Address(0); address < AddressCount; address++ {
sources = append(sources, ShaderSource(filter, address, false), ShaderSource(filter, address, true))
}
}
sources = append(sources, ScreenShaderSource, ClearShaderSource)
return sources
}

View File

@ -161,8 +161,8 @@ func __vertex(dstPos vec2, srcPos vec2, color vec4) (vec4, vec2, vec4) {
return shaderSuffix, nil
}
func CompileShader(src []byte) (*shaderir.Program, error) {
unit, err := shader.ParseCompilerDirectives(src)
func completeShaderSource(fragmentSrc []byte) ([]byte, error) {
unit, err := shader.ParseCompilerDirectives(fragmentSrc)
if err != nil {
return nil, err
}
@ -172,14 +172,23 @@ func CompileShader(src []byte) (*shaderir.Program, error) {
}
var buf bytes.Buffer
buf.Write(src)
buf.Write(fragmentSrc)
buf.WriteString(suffix)
return buf.Bytes(), nil
}
func CompileShader(fragmentSrc []byte) (*shaderir.Program, error) {
src, err := completeShaderSource(fragmentSrc)
if err != nil {
return nil, err
}
const (
vert = "__vertex"
frag = "Fragment"
)
ir, err := shader.Compile(buf.Bytes(), vert, frag, ShaderImageCount)
ir, err := shader.Compile(src, vert, frag, ShaderImageCount)
if err != nil {
return nil, err
}
@ -193,3 +202,11 @@ func CompileShader(src []byte) (*shaderir.Program, error) {
return ir, nil
}
func CalcSourceHash(fragmentSrc []byte) (shaderir.SourceHash, error) {
src, err := completeShaderSource(fragmentSrc)
if err != nil {
return shaderir.SourceHash{}, err
}
return shaderir.CalcSourceHash(src), nil
}

View File

@ -0,0 +1,39 @@
// Copyright 2024 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mtl
import (
"unsafe"
"github.com/ebitengine/purego"
)
var libSystem uintptr
var (
dispatchDataCreate func(buffer unsafe.Pointer, size uint, queue uintptr, destructor uintptr) uintptr
dispatchRelease func(obj uintptr)
)
func init() {
lib, err := purego.Dlopen("/usr/lib/libSystem.B.dylib", purego.RTLD_LAZY|purego.RTLD_GLOBAL)
if err != nil {
panic(err)
}
libSystem = lib
purego.RegisterLibFunc(&dispatchDataCreate, libSystem, "dispatch_data_create")
purego.RegisterLibFunc(&dispatchRelease, libSystem, "dispatch_release")
}

View File

@ -493,6 +493,7 @@ var (
sel_supportsFeatureSet = objc.RegisterName("supportsFeatureSet:")
sel_newCommandQueue = objc.RegisterName("newCommandQueue")
sel_newLibraryWithSource_options_error = objc.RegisterName("newLibraryWithSource:options:error:")
sel_newLibraryWithData_error = objc.RegisterName("newLibraryWithData:error:")
sel_release = objc.RegisterName("release")
sel_retain = objc.RegisterName("retain")
sel_new = objc.RegisterName("new")
@ -652,6 +653,27 @@ func (d Device) MakeLibrary(source string, opt CompileOptions) (Library, error)
return Library{l}, nil
}
// MakeLibraryWithData creates a Metal library instance from a data instance that contains the functions in a precompiled Metal library.
//
// Reference: https://developer.apple.com/documentation/metal/mtldevice/1433391-makelibrary
func (d Device) MakeLibraryWithData(buffer []byte) (Library, error) {
defer runtime.KeepAlive(buffer)
data := dispatchDataCreate(unsafe.Pointer(&buffer[0]), uint(len(buffer)), 0, 0)
defer dispatchRelease(data)
var err cocoa.NSError
l := d.device.Send(
sel_newLibraryWithData_error,
data,
unsafe.Pointer(&err),
)
if l == 0 {
return Library{}, errors.New(cocoa.NSString{ID: err.Send(sel_localizedDescription)}.String())
}
return Library{l}, nil
}
// MakeRenderPipelineState creates a render pipeline state object.
//
// Reference: https://developer.apple.com/documentation/metal/mtldevice/1433369-makerenderpipelinestate.

View File

@ -16,6 +16,7 @@ package metal
import (
"fmt"
"sync"
"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver"
"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal/mtl"
@ -23,6 +24,38 @@ import (
"github.com/hajimehoshi/ebiten/v2/internal/shaderir/msl"
)
type precompiledLibraries struct {
binaries map[shaderir.SourceHash][]byte
m sync.Mutex
}
func (c *precompiledLibraries) put(hash shaderir.SourceHash, bin []byte) {
c.m.Lock()
defer c.m.Unlock()
if c.binaries == nil {
c.binaries = map[shaderir.SourceHash][]byte{}
}
if _, ok := c.binaries[hash]; ok {
panic(fmt.Sprintf("metal: the precompiled library for the hash %s is already registered", hash.String()))
}
c.binaries[hash] = bin
}
func (c *precompiledLibraries) get(hash shaderir.SourceHash) ([]byte, bool) {
c.m.Lock()
defer c.m.Unlock()
bin, ok := c.binaries[hash]
return bin, ok
}
var thePrecompiledLibraries precompiledLibraries
func RegisterPrecompiledLibrary(hash shaderir.SourceHash, bin []byte) {
thePrecompiledLibraries.put(hash, bin)
}
type shaderRpsKey struct {
blend graphicsdriver.Blend
stencilMode stencilMode
@ -37,6 +70,8 @@ type Shader struct {
fs mtl.Function
vs mtl.Function
rpss map[shaderRpsKey]mtl.RenderPipelineState
libraryPrecompiled bool
}
func newShader(device mtl.Device, id graphicsdriver.ShaderID, program *shaderir.Program) (*Shader, error) {
@ -61,24 +96,42 @@ func (s *Shader) Dispose() {
}
s.vs.Release()
s.fs.Release()
s.lib.Release()
// Do not release s.lib if this is precompiled. This is a shared precompiled library.
if !s.libraryPrecompiled {
s.lib.Release()
}
}
func (s *Shader) init(device mtl.Device) error {
src := msl.Compile(s.ir)
lib, err := device.MakeLibrary(src, mtl.CompileOptions{})
if err != nil {
return fmt.Errorf("metal: device.MakeLibrary failed: %w, source: %s", err, src)
var src string
if libBin, ok := thePrecompiledLibraries.get(s.ir.SourceHash); ok {
lib, err := device.MakeLibraryWithData(libBin)
if err != nil {
return err
}
s.lib = lib
} else {
src = msl.Compile(s.ir)
lib, err := device.MakeLibrary(src, mtl.CompileOptions{})
if err != nil {
return fmt.Errorf("metal: device.MakeLibrary failed: %w, source: %s", err, src)
}
s.lib = lib
}
s.lib = lib
vs, err := s.lib.MakeFunction(msl.VertexName)
if err != nil {
return fmt.Errorf("metal: lib.MakeFunction for vertex failed: %w, source: %s", err, src)
if src != "" {
return fmt.Errorf("metal: lib.MakeFunction for vertex failed: %w, source: %s", err, src)
}
return fmt.Errorf("metal: lib.MakeFunction for vertex failed: %w", err)
}
fs, err := s.lib.MakeFunction(msl.FragmentName)
if err != nil {
return fmt.Errorf("metal: lib.MakeFunction for fragment failed: %w, source: %s", err, src)
if src != "" {
return fmt.Errorf("metal: lib.MakeFunction for fragment failed: %w, source: %s", err, src)
}
return fmt.Errorf("metal: lib.MakeFunction for fragment failed: %w", err)
}
s.fs = fs
s.vs = vs

View File

@ -202,6 +202,7 @@ func Compile(src []byte, vertexEntry, fragmentEntry string, textureCount int) (*
fragmentEntry: fragmentEntry,
unit: unit,
}
s.ir.SourceHash = shaderir.CalcSourceHash(src)
s.global.ir = &shaderir.Block{}
s.parse(f)

View File

@ -16,8 +16,10 @@
package shaderir
import (
"encoding/hex"
"go/constant"
"go/token"
"hash/fnv"
"sort"
"strings"
)
@ -29,6 +31,21 @@ const (
Pixels
)
type SourceHash [16]byte
func CalcSourceHash(source []byte) SourceHash {
h := fnv.New128a()
_, _ = h.Write(source)
var hash SourceHash
h.Sum(hash[:0])
return hash
}
func (s SourceHash) String() string {
return hex.EncodeToString(s[:])
}
type Program struct {
UniformNames []string
Uniforms []Type
@ -40,6 +57,8 @@ type Program struct {
FragmentFunc FragmentFunc
Unit Unit
SourceHash SourceHash
uniformFactors []uint32
}

View File

@ -0,0 +1,70 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shaderprecomp
import (
"github.com/hajimehoshi/ebiten/v2/internal/builtinshader"
"github.com/hajimehoshi/ebiten/v2/internal/graphics"
"github.com/hajimehoshi/ebiten/v2/internal/shaderir"
)
// AppendBuildinShaderSources appends all the built-in shader sources to the given slice.
//
// Do not modify the content of the shader source.
//
// AppendBuildinShaderSources is concurrent-safe.
func AppendBuildinShaderSources(sources []*ShaderSource) []*ShaderSource {
for _, s := range builtinshader.AppendShaderSources(nil) {
src, err := NewShaderSource(s)
if err != nil {
panic(err)
}
sources = append(sources, src)
}
return sources
}
// ShaderSource is an object encapsulating a shader source code.
type ShaderSource struct {
source []byte
id ShaderSourceID
}
// NewShaderSource creates a new ShaderSource object from the given source code.
func NewShaderSource(source []byte) (*ShaderSource, error) {
hash, err := graphics.CalcSourceHash(source)
if err != nil {
return nil, err
}
return &ShaderSource{
source: source,
id: ShaderSourceID(hash),
}, nil
}
// ID returns a unique identifier for the shader source.
// The ShaderSourceID value must be the same for the same shader source and the same Ebitengine version.
// There is no guarantee that the ShaderSourceID value is the same between different Ebitengine versions.
func (s *ShaderSource) ID() ShaderSourceID {
return s.id
}
// ShaderSourceID is a uniuqe identifier for a shader source.
type ShaderSourceID [16]byte
// String returns a string representation of the shader source ID.
func (s ShaderSourceID) String() string {
return shaderir.SourceHash(s).String()
}

View File

@ -0,0 +1,47 @@
// Copyright 2024 The Ebitengine Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shaderprecomp
import (
"io"
"github.com/hajimehoshi/ebiten/v2/internal/graphics"
"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal"
"github.com/hajimehoshi/ebiten/v2/internal/shaderir"
"github.com/hajimehoshi/ebiten/v2/internal/shaderir/msl"
)
// CompileToMSL compiles the shader source to Metal Shader Language, and writes the result to w.
//
// CompileToMSL is concurrent-safe.
func CompileToMSL(w io.Writer, source *ShaderSource) error {
ir, err := graphics.CompileShader(source.source)
if err != nil {
return err
}
if _, err = w.Write([]byte(msl.Compile(ir))); err != nil {
return err
}
return nil
}
// RegisterMetalLibrary registers a precompiled Metal library for a shader source.
// library must be the content of a .metallib file.
// For more details, see https://developer.apple.com/documentation/metal/shader_libraries/building_a_shader_library_by_precompiling_source_files.
//
// RegisterMetalLibrary is concurrent-safe.
func RegisterMetalLibrary(source *ShaderSource, library []byte) {
metal.RegisterPrecompiledLibrary(shaderir.SourceHash(source.ID()), library)
}