From 10d96601250f34a9e7d0ba5691b04edb230c0d3b Mon Sep 17 00:00:00 2001 From: Hajime Hoshi Date: Sun, 5 May 2024 16:52:10 +0900 Subject: [PATCH] shaderprecomp: implement for Windows Closes #2861 --- .gitignore | 2 + examples/shaderprecomp/fxc/dummy.fxc | 1 + examples/shaderprecomp/fxc/gen.go | 130 ++++++++++++++++++ examples/shaderprecomp/fxc/generate.go | 17 +++ examples/shaderprecomp/main.go | 1 + examples/shaderprecomp/register_others.go | 2 +- examples/shaderprecomp/register_windows.go | 65 +++++++++ .../graphicsdriver/directx/d3d_windows.go | 17 ++- .../graphicsdriver/directx/shader_windows.go | 73 +++++++++- shaderprecomp/shaderprecomp_windows.go | 65 +++++++++ 10 files changed, 366 insertions(+), 7 deletions(-) create mode 100644 examples/shaderprecomp/fxc/dummy.fxc create mode 100644 examples/shaderprecomp/fxc/gen.go create mode 100644 examples/shaderprecomp/fxc/generate.go create mode 100644 examples/shaderprecomp/register_windows.go create mode 100644 shaderprecomp/shaderprecomp_windows.go diff --git a/.gitignore b/.gitignore index 719c6998d..981311b14 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,7 @@ go.work go.work.sum +*.fxc +!dummy.fxc *.metallib !dummy.metallib diff --git a/examples/shaderprecomp/fxc/dummy.fxc b/examples/shaderprecomp/fxc/dummy.fxc new file mode 100644 index 000000000..2a3f3dd8e --- /dev/null +++ b/examples/shaderprecomp/fxc/dummy.fxc @@ -0,0 +1 @@ +This is a dummy .fxc file to trick Go's embed package. diff --git a/examples/shaderprecomp/fxc/gen.go b/examples/shaderprecomp/fxc/gen.go new file mode 100644 index 000000000..8d8215069 --- /dev/null +++ b/examples/shaderprecomp/fxc/gen.go @@ -0,0 +1,130 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build ignore + +// This is a program to generate precompiled HLSL blobs (FXC files). +// +// See https://learn.microsoft.com/en-us/windows/win32/direct3dtools/fxc. +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/hajimehoshi/ebiten/v2/shaderprecomp" +) + +func main() { + if err := run(); err != nil { + panic(err) + } +} + +func run() error { + if _, err := exec.LookPath("fxc.exe"); err != nil { + if errors.Is(err, exec.ErrNotFound) { + fmt.Fprintln(os.Stderr, "fxc.exe not found. Please install Windows SDK.") + fmt.Fprintln(os.Stderr, "See https://learn.microsoft.com/en-us/windows/win32/direct3dtools/fxc for more details.") + fmt.Fprintln(os.Stderr, "On PowerShell, you can add a path to the PATH environment variable temporarily like:") + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, ` & (Get-Process -Id $PID).Path { $env:PATH="C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64;"+$env:PATH; go generate .\examples\shaderprecomp\fxc\ }`) + fmt.Fprintln(os.Stderr) + os.Exit(1) + } + return err + } + + tmpdir, err := os.MkdirTemp("", "") + if err != nil { + return err + } + defer os.RemoveAll(tmpdir) + + srcs := shaderprecomp.AppendBuildinShaderSources(nil) + + defaultSrcBytes, err := os.ReadFile(filepath.Join("..", "defaultshader.go")) + if err != nil { + return err + } + defaultSrc, err := shaderprecomp.NewShaderSource(defaultSrcBytes) + if err != nil { + return err + } + srcs = append(srcs, defaultSrc) + + for _, src := range srcs { + // Compiling sources in parallel causes a mixed error message on the console. + if err := compile(src, tmpdir); err != nil { + return err + } + } + return nil +} + +func generateHSLSFiles(source *shaderprecomp.ShaderSource, tmpdir string) (vs, ps string, err error) { + id := source.ID().String() + + vsHLSLFilePath := filepath.Join(tmpdir, id+"_vs.hlsl") + vsf, err := os.Create(vsHLSLFilePath) + if err != nil { + return "", "", err + } + defer vsf.Close() + + psHLSLFilePath := filepath.Join(tmpdir, id+"_ps.hlsl") + psf, err := os.Create(psHLSLFilePath) + if err != nil { + return "", "", err + } + defer psf.Close() + + if err := shaderprecomp.CompileToHLSL(vsf, psf, source); err != nil { + return "", "", err + } + + return vsHLSLFilePath, psHLSLFilePath, nil +} + +func compile(source *shaderprecomp.ShaderSource, tmpdir string) error { + // Generate HLSL files. Make sure this process doesn't have any handlers of the files. + // Without closing the files, fxc.exe cannot access the files. + vsHLSLFilePath, psHLSLFilePath, err := generateHSLSFiles(source, tmpdir) + if err != nil { + return err + } + + id := source.ID().String() + + vsFXCFilePath := id + "_vs.fxc" + cmd := exec.Command("fxc.exe", "/nologo", "/O3", "/T", shaderprecomp.HLSLVertexShaderProfile, "/E", shaderprecomp.HLSLVertexShaderEntryPoint, "/Fo", vsFXCFilePath, vsHLSLFilePath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + psFXCFilePath := id + "_ps.fxc" + cmd = exec.Command("fxc.exe", "/nologo", "/O3", "/T", shaderprecomp.HLSLPixelShaderProfile, "/E", shaderprecomp.HLSLPixelShaderEntryPoint, "/Fo", psFXCFilePath, psHLSLFilePath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + return nil +} diff --git a/examples/shaderprecomp/fxc/generate.go b/examples/shaderprecomp/fxc/generate.go new file mode 100644 index 000000000..772999805 --- /dev/null +++ b/examples/shaderprecomp/fxc/generate.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate go run gen.go + +package fxc diff --git a/examples/shaderprecomp/main.go b/examples/shaderprecomp/main.go index 971d8aa04..60d85edce 100644 --- a/examples/shaderprecomp/main.go +++ b/examples/shaderprecomp/main.go @@ -67,6 +67,7 @@ func main() { if err := registerPrecompiledShaders(); err != nil { log.Fatal(err) } + ebiten.SetWindowTitle("Ebitengine Example (Shader Precompilation)") if err := ebiten.RunGame(&Game{}); err != nil { log.Fatal(err) } diff --git a/examples/shaderprecomp/register_others.go b/examples/shaderprecomp/register_others.go index 2f9c757ef..b0e37ea6f 100644 --- a/examples/shaderprecomp/register_others.go +++ b/examples/shaderprecomp/register_others.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !darwin +//go:build !darwin && !windows package main diff --git a/examples/shaderprecomp/register_windows.go b/examples/shaderprecomp/register_windows.go new file mode 100644 index 000000000..34ac7437a --- /dev/null +++ b/examples/shaderprecomp/register_windows.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "embed" + "errors" + "fmt" + "io/fs" + "os" + + "github.com/hajimehoshi/ebiten/v2/shaderprecomp" +) + +// https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/ + +//go:embed fxc/*.fxc +var fxcs embed.FS + +func registerPrecompiledShaders() error { + srcs := shaderprecomp.AppendBuildinShaderSources(nil) + defaultShaderSource, err := shaderprecomp.NewShaderSource(defaultShaderSourceBytes) + if err != nil { + return err + } + srcs = append(srcs, defaultShaderSource) + + for _, src := range srcs { + vsname := src.ID().String() + "_vs.fxc" + vs, err := fxcs.ReadFile("fxc/" + vsname) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + fmt.Fprintf(os.Stderr, "precompiled HLSL library %s was not found. Run 'go generate' for 'fxc' directory to generate them.\n", vsname) + continue + } + return err + } + + psname := src.ID().String() + "_ps.fxc" + ps, err := fxcs.ReadFile("fxc/" + psname) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + fmt.Fprintf(os.Stderr, "precompiled HLSL library %s was not found. Run 'go generate' for 'fxc' directory to generate them.\n", psname) + continue + } + return err + } + + shaderprecomp.RegisterFXCs(src, vs, ps) + } + + return nil +} diff --git a/internal/graphicsdriver/directx/d3d_windows.go b/internal/graphicsdriver/directx/d3d_windows.go index be90829a1..845bbda1c 100644 --- a/internal/graphicsdriver/directx/d3d_windows.go +++ b/internal/graphicsdriver/directx/d3d_windows.go @@ -71,7 +71,8 @@ const ( ) var ( - procD3DCompile *windows.LazyProc + procD3DCompile *windows.LazyProc + procD3DCreateBlob *windows.LazyProc ) func init() { @@ -93,6 +94,7 @@ func init() { } procD3DCompile = d3dcompiler.NewProc("D3DCompile") + procD3DCreateBlob = d3dcompiler.NewProc("D3DCreateBlob") } func isD3DCompilerDLLAvailable() bool { @@ -135,6 +137,19 @@ func _D3DCompile(srcData []byte, sourceName string, pDefines []_D3D_SHADER_MACRO return code, nil } +func _D3DCreateBlob(size uint) (*_ID3DBlob, error) { + if !isD3DCompilerDLLAvailable() { + return nil, fmt.Errorf("directx: d3dcompiler_*.dll is missing in this environment") + } + + var blob *_ID3DBlob + r, _, _ := procD3DCreateBlob.Call(uintptr(size), uintptr(unsafe.Pointer(&blob))) + if uint32(r) != uint32(windows.S_OK) { + return nil, fmt.Errorf("directx: D3DCreateBlob failed: %w", handleError(windows.Handle(uint32(r)))) + } + return blob, nil +} + type _D3D_SHADER_MACRO struct { Name *byte Definition *byte diff --git a/internal/graphicsdriver/directx/shader_windows.go b/internal/graphicsdriver/directx/shader_windows.go index a8003b7a2..172ec1638 100644 --- a/internal/graphicsdriver/directx/shader_windows.go +++ b/internal/graphicsdriver/directx/shader_windows.go @@ -16,6 +16,8 @@ package directx import ( "fmt" + "sync" + "unsafe" "golang.org/x/sync/errgroup" @@ -24,12 +26,57 @@ import ( "github.com/hajimehoshi/ebiten/v2/internal/shaderir/hlsl" ) +const ( + VertexShaderProfile = "vs_4_0" + PixelShaderProfile = "ps_4_0" + + VertexShaderEntryPoint = "VSMain" + PixelShaderEntryPoint = "PSMain" +) + +type fxcPair struct { + vertex []byte + pixel []byte +} + +type precompiledFXCs struct { + binaries map[shaderir.SourceHash]fxcPair + m sync.Mutex +} + +func (c *precompiledFXCs) put(hash shaderir.SourceHash, vertex, pixel []byte) { + c.m.Lock() + defer c.m.Unlock() + + if c.binaries == nil { + c.binaries = map[shaderir.SourceHash]fxcPair{} + } + if _, ok := c.binaries[hash]; ok { + panic(fmt.Sprintf("directx: the precompiled library for the hash %s is already registered", hash.String())) + } + c.binaries[hash] = fxcPair{ + vertex: vertex, + pixel: pixel, + } +} + +func (c *precompiledFXCs) get(hash shaderir.SourceHash) ([]byte, []byte) { + c.m.Lock() + defer c.m.Unlock() + + f := c.binaries[hash] + return f.vertex, f.pixel +} + +var thePrecompiledFXCs precompiledFXCs + +func RegisterPrecompiledFXCs(hash shaderir.SourceHash, vertex, pixel []byte) { + thePrecompiledFXCs.put(hash, vertex, pixel) +} + var vertexShaderCache = map[string]*_ID3DBlob{} func compileShader(program *shaderir.Program) (vsh, psh *_ID3DBlob, ferr error) { - vs, ps := hlsl.Compile(program) - var flag uint32 = uint32(_D3DCOMPILE_OPTIMIZATION_LEVEL3) - defer func() { if ferr == nil { return @@ -42,6 +89,22 @@ func compileShader(program *shaderir.Program) (vsh, psh *_ID3DBlob, ferr error) } }() + if vshBin, pshBin := thePrecompiledFXCs.get(program.SourceHash); vshBin != nil && pshBin != nil { + var err error + if vsh, err = _D3DCreateBlob(uint(len(vshBin))); err != nil { + return nil, nil, err + } + if psh, err = _D3DCreateBlob(uint(len(pshBin))); err != nil { + return nil, nil, err + } + copy(unsafe.Slice((*byte)(vsh.GetBufferPointer()), vsh.GetBufferSize()), vshBin) + copy(unsafe.Slice((*byte)(psh.GetBufferPointer()), psh.GetBufferSize()), pshBin) + return vsh, psh, nil + } + + vs, ps := hlsl.Compile(program) + var flag uint32 = uint32(_D3DCOMPILE_OPTIMIZATION_LEVEL3) + var wg errgroup.Group // Vertex shaders are likely the same. If so, reuse the same _ID3DBlob. @@ -58,7 +121,7 @@ func compileShader(program *shaderir.Program) (vsh, psh *_ID3DBlob, ferr error) } }() wg.Go(func() error { - v, err := _D3DCompile([]byte(vs), "shader", nil, nil, "VSMain", "vs_4_0", flag, 0) + v, err := _D3DCompile([]byte(vs), "shader", nil, nil, VertexShaderEntryPoint, VertexShaderProfile, flag, 0) if err != nil { return fmt.Errorf("directx: D3DCompile for VSMain failed, original source: %s, %w", vs, err) } @@ -67,7 +130,7 @@ func compileShader(program *shaderir.Program) (vsh, psh *_ID3DBlob, ferr error) }) } wg.Go(func() error { - p, err := _D3DCompile([]byte(ps), "shader", nil, nil, "PSMain", "ps_4_0", flag, 0) + p, err := _D3DCompile([]byte(ps), "shader", nil, nil, PixelShaderEntryPoint, PixelShaderProfile, flag, 0) if err != nil { return fmt.Errorf("directx: D3DCompile for PSMain failed, original source: %s, %w", ps, err) } diff --git a/shaderprecomp/shaderprecomp_windows.go b/shaderprecomp/shaderprecomp_windows.go new file mode 100644 index 000000000..f1b96bdbb --- /dev/null +++ b/shaderprecomp/shaderprecomp_windows.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shaderprecomp + +import ( + "io" + + "github.com/hajimehoshi/ebiten/v2/internal/graphics" + "github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/directx" + "github.com/hajimehoshi/ebiten/v2/internal/shaderir" + "github.com/hajimehoshi/ebiten/v2/internal/shaderir/hlsl" +) + +const ( + // HLSLVertexShaderProfile is the target profile for vertex shaders. + HLSLVertexShaderProfile = directx.VertexShaderProfile + + // HLSLPixelShaderProfile is the target profile for pixel shaders. + HLSLPixelShaderProfile = directx.PixelShaderProfile + + // HLSLVertexShaderEntryPoint is the entry point name for vertex shaders. + HLSLVertexShaderEntryPoint = directx.VertexShaderEntryPoint + + // HLSLPixelShaderEntryPoint is the entry point name for pixel shaders. + HLSLPixelShaderEntryPoint = directx.PixelShaderEntryPoint +) + +// CompileToHLSL compiles the shader source to High-Level Shader Language to writers. +// +// CompileToHLSL is concurrent-safe. +func CompileToHLSL(vertexWriter, pixelWriter io.Writer, source *ShaderSource) error { + ir, err := graphics.CompileShader(source.source) + if err != nil { + return err + } + vs, ps := hlsl.Compile(ir) + if _, err = vertexWriter.Write([]byte(vs)); err != nil { + return err + } + if _, err = pixelWriter.Write([]byte(ps)); err != nil { + return err + } + return nil +} + +// RegisterFXCs registers a precompiled HLSL (FXC) for a shader source. +// vertexFXC and pixelFXC must be the content of .fxc files generated by `fxc` command. +// For more details, see https://learn.microsoft.com/en-us/windows/win32/direct3dtools/dx-graphics-tools-fxc-using. +// +// RegisterFXCs is concurrent-safe. +func RegisterFXCs(source *ShaderSource, vertexFXC, pixelFXC []byte) { + directx.RegisterPrecompiledFXCs(shaderir.SourceHash(source.ID()), vertexFXC, pixelFXC) +}