From 99bbe7138c38a45b5b64e083b7427f37d4667c1f Mon Sep 17 00:00:00 2001 From: Hajime Hoshi Date: Sun, 10 Nov 2024 19:08:32 +0900 Subject: [PATCH] internal: add shaderlister This adds a new compiler directive `//ebitengine:shader` indicating a shader source. A new tool internal/shaderlister can iterates all the shader strings with the directive. The tool might be exposed in the future. Updates #3157 --- internal/shaderlister/main.go | 299 ++++++++++++++++++ internal/shaderlister/shaderlister_test.go | 65 ++++ internal/shaderlister/shaderlistertest/def.go | 68 ++++ .../shaderlister/shaderlistertest2/def.go | 22 ++ 4 files changed, 454 insertions(+) create mode 100644 internal/shaderlister/main.go create mode 100644 internal/shaderlister/shaderlister_test.go create mode 100644 internal/shaderlister/shaderlistertest/def.go create mode 100644 internal/shaderlister/shaderlistertest2/def.go diff --git a/internal/shaderlister/main.go b/internal/shaderlister/main.go new file mode 100644 index 000000000..191024d86 --- /dev/null +++ b/internal/shaderlister/main.go @@ -0,0 +1,299 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "go/ast" + "go/constant" + "go/token" + "go/types" + "log/slog" + "os" + "os/exec" + "regexp" + "slices" + "strings" + + "golang.org/x/tools/go/ast/inspector" + "golang.org/x/tools/go/packages" +) + +func main() { + if err := xmain(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type Shader struct { + Package string + File string + Source string +} + +func xmain() error { + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "shaderlister [package]") + os.Exit(2) + } + flag.Parse() + if len(flag.Args()) < 1 { + flag.Usage() + } + + pkg := flag.Arg(0) + deps, err := listDeps(pkg) + if err != nil { + return err + } + deps = slices.DeleteFunc(deps, func(name string) bool { + // A standard library should not have a directive for shaders. Skip them. + if isStandardImportPath(name) { + return true + } + // A semi-standard library should not have a directive for shaders. Skip them. + if strings.HasPrefix(name, "golang.org/x/") { + return true + } + return false + }) + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo, + }, append([]string{pkg}, deps...)...) + if err != nil { + return err + } + + var shaders []Shader + + for _, pkg := range pkgs { + shaders = appendShaderSources(shaders, pkg) + } + + w := bufio.NewWriter(os.Stdout) + enc := json.NewEncoder(w) + enc.SetIndent("", "\t") + if err := enc.Encode(shaders); err != nil { + return err + } + if err := w.Flush(); err != nil { + return err + } + + return nil +} + +func listDeps(pkg string) ([]string, error) { + cmd := exec.Command("go", "list", "-f", `{{join .Deps ","}}`, pkg) + out, err := cmd.Output() + if err != nil { + if e, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("go list failed: %w\n%s", e, e.Stderr) + } + return nil, err + } + return strings.Split(strings.TrimSpace(string(out)), ","), nil +} + +// isStandardImportPath reports whether $GOROOT/src/path should be considered part of the standard distribution. +// +// This is based on the implementation in the standard library (cmd/go/internal/search/search.go). +func isStandardImportPath(path string) bool { + head, _, _ := strings.Cut(path, "/") + return !strings.Contains(head, ".") +} + +const directive = "ebitengine:shader" + +var reDirective = regexp.MustCompile(`(?m)^\s*//` + regexp.QuoteMeta(directive)) + +func hasShaderDirectiveInComment(commentGroup *ast.CommentGroup) bool { + for _, line := range commentGroup.List { + if reDirective.MatchString(line.Text) { + return true + } + } + return false +} + +func appendShaderSources(shaders []Shader, pkg *packages.Package) []Shader { + topLevelDecls := map[ast.Decl]struct{}{} + for _, file := range pkg.Syntax { + for _, decl := range file.Decls { + topLevelDecls[decl] = struct{}{} + } + } + isTopLevelDecl := func(decl ast.Decl) bool { + _, ok := topLevelDecls[decl] + return ok + } + + var genDeclStack []*ast.GenDecl + + in := inspector.New(pkg.Syntax) + in.Nodes([]ast.Node{ + (*ast.GenDecl)(nil), + (*ast.ValueSpec)(nil), + }, func(n ast.Node, push bool) bool { + switch n := n.(type) { + case *ast.GenDecl: + genDecl := n + + // It is possible to check whether decl.Tok is token.CONST or not, + // but move on without checking it for better warning messages. + + if push { + // If the GenDecl is with parentheses (e.g. `const ( ... )`), check the GenDecl's comment. + // The directive doesn't work, so if the directive is found, warn it. + if genDecl.Lparen != token.NoPos { + if genDecl.Doc != nil && hasShaderDirectiveInComment(genDecl.Doc) { + pos := pkg.Fset.Position(genDecl.Doc.Pos()) + slog.Warn(fmt.Sprintf("misplaced %s directive", directive), + "package", pkg.PkgPath, + "file", pos.Filename, + "line", pos.Line, + "column", pos.Column) + } + } else { + if genDecl.Doc == nil { + return false + } + if !hasShaderDirectiveInComment(genDecl.Doc) { + return false + } + } + + // It is possible to check whether genCecl is top-level or not, + // but move on without checking it for better warning messages. + + genDeclStack = append(genDeclStack, genDecl) + } else { + genDeclStack = genDeclStack[:len(genDeclStack)-1] + } + return true + + case *ast.ValueSpec: + spec := n + + genDecl := genDeclStack[len(genDeclStack)-1] + + // If the ValueSpec is in parentheses (e.g. `const ( ... )`), check the ValueSpec's comment. + if genDecl.Lparen != token.NoPos { + if spec.Doc == nil { + return false + } + if !hasShaderDirectiveInComment(spec.Doc) { + return false + } + } + + var docPos token.Pos + if spec.Doc != nil { + docPos = spec.Doc.Pos() + } else { + docPos = genDecl.Doc.Pos() + } + + if !isTopLevelDecl(genDecl) { + pos := pkg.Fset.Position(docPos) + slog.Warn(fmt.Sprintf("misplaced %s directive", directive), + "package", pkg.PkgPath, + "file", pos.Filename, + "line", pos.Line, + "column", pos.Column) + return false + } + + // Avoid multiple names like `const a, b = "foo", "bar"` to avoid confusions. + if len(spec.Names) != 1 { + pos := pkg.Fset.Position(docPos) + slog.Warn(fmt.Sprintf("%s cannot apply to multiple declarations", directive), + "package", pkg.PkgPath, + "file", pos.Filename, + "line", pos.Line, + "column", pos.Column) + return false + } + + // Check if the ValueSpec is a const declaration. + name := spec.Names[0] + def := pkg.TypesInfo.Defs[name] + c, ok := def.(*types.Const) + if !ok { + pos := pkg.Fset.Position(docPos) + slog.Warn(fmt.Sprintf("%s cannot apply to %s", directive, objectTypeString(def)), + "package", pkg.PkgPath, + "file", pos.Filename, + "line", pos.Line, + "column", pos.Column) + return false + } + + // Check the constant type. + val := c.Val() + if val.Kind() != constant.String { + pos := pkg.Fset.Position(docPos) + slog.Warn(fmt.Sprintf("%s cannot apply to const type of %s", directive, val.Kind()), + "package", pkg.PkgPath, + "file", pos.Filename, + "line", pos.Line, + "column", pos.Column) + return false + } + + shaders = append(shaders, Shader{ + Package: pkg.PkgPath, + File: pkg.Fset.Position(spec.Pos()).Filename, + Source: constant.StringVal(val), + }) + return false + + default: + return false + } + }) + + return shaders +} + +func objectTypeString(obj types.Object) string { + switch obj := obj.(type) { + case *types.PkgName: + return "package" + case *types.Const: + return "const" + case *types.TypeName: + return "type" + case *types.Var: + if obj.IsField() { + return "field" + } + return "var" + case *types.Func: + return "func" + case *types.Label: + return "label" + case *types.Builtin: + return "builtin" + case *types.Nil: + return "nil" + default: + return fmt.Sprintf("objectTypeString(%T)", obj) + } +} diff --git a/internal/shaderlister/shaderlister_test.go b/internal/shaderlister/shaderlister_test.go new file mode 100644 index 000000000..a2a189c00 --- /dev/null +++ b/internal/shaderlister/shaderlister_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main_test + +import ( + "cmp" + "encoding/json" + "fmt" + "os/exec" + "slices" + "testing" +) + +func TestRun(t *testing.T) { + cmd := exec.Command("go", "run", "github.com/hajimehoshi/ebiten/v2/internal/shaderlister", "github.com/hajimehoshi/ebiten/v2/internal/shaderlister/shaderlistertest") + out, err := cmd.Output() + if err != nil { + if err, ok := err.(*exec.ExitError); ok { + t.Fatalf("Error: %v\n%s", err, err.Stderr) + } + t.Fatal(err) + } + + type shader struct { + Package string + File string + Source string + } + var shaders []shader + if err := json.Unmarshal(out, &shaders); err != nil { + t.Fatal(err) + } + + slices.SortFunc(shaders, func(s1, s2 shader) int { + return cmp.Compare(s1.Source, s2.Source) + }) + + if got, want := len(shaders), 6; got != want { + t.Fatalf("len(shaders): got: %d, want: %d", got, want) + } + + for i, s := range shaders { + if s.Package == "" { + t.Errorf("s.Package is empty: %v", s) + } + if s.File == "" { + t.Errorf("s.File is empty: %v", s) + } + if got, want := s.Source, fmt.Sprintf("shader %d", i+1); got != want { + t.Errorf("s.Source: got: %q, want: %q", got, want) + } + } +} diff --git a/internal/shaderlister/shaderlistertest/def.go b/internal/shaderlister/shaderlistertest/def.go new file mode 100644 index 000000000..baa78b148 --- /dev/null +++ b/internal/shaderlister/shaderlistertest/def.go @@ -0,0 +1,68 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shaderlistertest + +import "github.com/hajimehoshi/ebiten/v2/internal/shaderlister/shaderlistertest2" + +//ebitengine:shader +const _ = "shader 1" + +const ( + //ebitengine:shader + _ = "shader 2" + + //ebitengine:shader + a = "shader 3" + + //ebitengine:invalid + b = "not shader" + + //ebitengine:shader + c = "shader" + " 4" +) + +//ebitengine:invalid +const _ = "not shader" + +//ebitengine:shader +const d = shaderlistertest2.S + " 5" + +const _ = "not shader" + +//ebitengine:shader +const ( + _ = "ignored" // The directive is misplaced. +) + +//ebitengine:shader +var _ = "ignored" // The directive doesn't work for var. + +func f() { + //ebitengine:shader + const _ = "ignored" // The directive doesn't work for non-top-level const. + + const ( + //ebitengine:shader + _ = "ignored" // The directive doesn't work for non-top-level const. + ) +} + +//ebitengine:shader +const _, _ = "ignored", "ignored again" // multiple consts are ignored to avoid confusion. + +const ( + //ebitengine:shader + _, _ = "ignored", "ignored again" // multiple consts are ignored to avoid confusion. +) diff --git a/internal/shaderlister/shaderlistertest2/def.go b/internal/shaderlister/shaderlistertest2/def.go new file mode 100644 index 000000000..007b772a6 --- /dev/null +++ b/internal/shaderlister/shaderlistertest2/def.go @@ -0,0 +1,22 @@ +// Copyright 2024 The Ebitengine Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shaderlistertest2 + +const S = "shader" + +//ebitengine:shader +const _ = "shader 6" + +const _ = "not shader"