gen.go

Documentation: cuelang.org/go/pkg

     1  // Copyright 2018 The CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //go:build ignore
    16  
    17  // gen.go generates the pkg.go files inside the packages under the pkg directory.
    18  //
    19  // It takes the list of packages from the packages.txt.
    20  //
    21  // Be sure to also update an entry in pkg/pkg.go, if so desired.
    22  package main
    23  
    24  // TODO generate ../register.go too.
    25  
    26  import (
    27  	"bytes"
    28  	_ "embed"
    29  	"flag"
    30  	"fmt"
    31  	"go/constant"
    32  	"go/format"
    33  	"go/token"
    34  	"go/types"
    35  	"log"
    36  	"math/big"
    37  	"os"
    38  	"path"
    39  	"path/filepath"
    40  	"sort"
    41  	"strings"
    42  	"text/template"
    43  
    44  	"golang.org/x/tools/go/packages"
    45  
    46  	"cuelang.org/go/cue"
    47  	"cuelang.org/go/cue/build"
    48  	"cuelang.org/go/cue/errors"
    49  	cueformat "cuelang.org/go/cue/format"
    50  	"cuelang.org/go/internal"
    51  	"cuelang.org/go/internal/core/runtime"
    52  )
    53  
    54  const genFile = "pkg.go"
    55  
    56  //go:embed packages.txt
    57  var packagesStr string
    58  
    59  type headerParams struct {
    60  	GoPkg  string
    61  	CUEPkg string
    62  
    63  	PackageDoc  string
    64  	PackageDefs string
    65  }
    66  
    67  var header = template.Must(template.New("").Parse(
    68  	`// Code generated by cuelang.org/go/pkg/gen. DO NOT EDIT.
    69  
    70  {{if .PackageDoc}}
    71  {{.PackageDoc -}}
    72  //     {{.PackageDefs}}
    73  {{end -}}
    74  package {{.GoPkg}}
    75  
    76  {{if .CUEPkg -}}
    77  import (
    78  	"cuelang.org/go/internal/core/adt"
    79  	"cuelang.org/go/internal/pkg"
    80  )
    81  
    82  func init() {
    83  	pkg.Register({{printf "%q" .CUEPkg}}, p)
    84  }
    85  
    86  var _ = adt.TopKind // in case the adt package isn't used
    87  {{end}}
    88  `))
    89  
    90  const pkgParent = "cuelang.org/go/pkg"
    91  
    92  func main() {
    93  	flag.Parse()
    94  	log.SetFlags(log.Lshortfile)
    95  	log.SetOutput(os.Stdout)
    96  
    97  	var packagesList []string
    98  	for _, pkg := range strings.Fields(packagesStr) {
    99  		if pkg == "path" {
   100  			// TODO remove this special case. Currently the path
   101  			// pkg.go file cannot be generated automatically but that
   102  			// will be possible when we can attach arbitrary signatures
   103  			// to builtin functions.
   104  			continue
   105  		}
   106  		packagesList = append(packagesList, path.Join(pkgParent, pkg))
   107  	}
   108  
   109  	cfg := &packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes}
   110  	pkgs, err := packages.Load(cfg, packagesList...)
   111  	if err != nil {
   112  		fmt.Fprintf(os.Stderr, "load: %v\n", err)
   113  		os.Exit(1)
   114  	}
   115  	if packages.PrintErrors(pkgs) > 0 {
   116  		os.Exit(1)
   117  	}
   118  	for _, pkg := range pkgs {
   119  		if err := generate(pkg); err != nil {
   120  			log.Fatalf("%s: %v", pkg, err)
   121  		}
   122  	}
   123  }
   124  
   125  type generator struct {
   126  	dir        string
   127  	w          *bytes.Buffer
   128  	cuePkgPath string
   129  	first      bool
   130  }
   131  
   132  func generate(pkg *packages.Package) error {
   133  	// go/packages supports multiple build systems, including some which don't keep
   134  	// a Go package entirely within a single directory.
   135  	// However, we know for certain that CUE uses modules, so it is the case here.
   136  	// We can figure out the directory from the first Go file.
   137  	pkgDir := filepath.Dir(pkg.GoFiles[0])
   138  	cuePkg := strings.TrimPrefix(pkg.PkgPath, pkgParent+"/")
   139  	g := generator{
   140  		dir:        pkgDir,
   141  		cuePkgPath: cuePkg,
   142  		w:          &bytes.Buffer{},
   143  	}
   144  
   145  	params := headerParams{
   146  		GoPkg:  pkg.Name,
   147  		CUEPkg: cuePkg,
   148  	}
   149  	// As a special case, the "tool" package cannot be imported from CUE.
   150  	skipRegister := params.CUEPkg == "tool"
   151  	if skipRegister {
   152  		params.CUEPkg = ""
   153  	}
   154  
   155  	if doc, err := os.ReadFile(filepath.Join(pkgDir, "doc.txt")); err == nil {
   156  		defs, err := os.ReadFile(filepath.Join(pkgDir, pkg.Name+".cue"))
   157  		if err != nil {
   158  			return err
   159  		}
   160  		i := bytes.Index(defs, []byte("package "+pkg.Name))
   161  		defs = defs[i+len("package "+pkg.Name)+1:]
   162  		defs = bytes.TrimRight(defs, "\n")
   163  		defs = bytes.ReplaceAll(defs, []byte("\n"), []byte("\n//\t"))
   164  		params.PackageDoc = string(doc)
   165  		params.PackageDefs = string(defs)
   166  	}
   167  
   168  	if err := header.Execute(g.w, params); err != nil {
   169  		return err
   170  	}
   171  
   172  	if !skipRegister {
   173  		fmt.Fprintf(g.w, "var p = &pkg.Package{\nNative: []*pkg.Builtin{")
   174  		g.first = true
   175  		if err := g.processGo(pkg); err != nil {
   176  			return err
   177  		}
   178  		fmt.Fprintf(g.w, "},\n")
   179  		if err := g.processCUE(); err != nil {
   180  			return err
   181  		}
   182  		fmt.Fprintf(g.w, "}\n")
   183  	}
   184  
   185  	b, err := format.Source(g.w.Bytes())
   186  	if err != nil {
   187  		fmt.Printf("go/format error on %s: %v\n", pkg.PkgPath, err)
   188  		b = g.w.Bytes() // write the unformatted source
   189  	}
   190  
   191  	filename := filepath.Join(pkgDir, genFile)
   192  
   193  	if err := os.WriteFile(filename, b, 0666); err != nil {
   194  		return err
   195  	}
   196  	return nil
   197  }
   198  
   199  func (g *generator) sep() {
   200  	if g.first {
   201  		g.first = false
   202  		return
   203  	}
   204  	fmt.Fprint(g.w, ", ")
   205  }
   206  
   207  // processCUE mixes in CUE definitions defined in the package directory.
   208  func (g *generator) processCUE() error {
   209  	// Note: we avoid using the cue/load and the cuecontext packages
   210  	// because they depend on the standard library which is what this
   211  	// command is generating - cyclic dependencies are undesirable in general.
   212  	ctx := newContext()
   213  	val, err := loadCUEPackage(ctx, g.dir, g.cuePkgPath)
   214  	if err != nil {
   215  		if errors.Is(err, errNoCUEFiles) {
   216  			return nil
   217  		}
   218  		errors.Print(os.Stderr, err, nil)
   219  		return fmt.Errorf("error processing %s: %v", g.cuePkgPath, err)
   220  	}
   221  
   222  	v := val.Syntax(cue.Raw())
   223  	// fmt.Printf("%T\n", v)
   224  	// fmt.Println(astinternal.DebugStr(v))
   225  	n := internal.ToExpr(v)
   226  	b, err := cueformat.Node(n)
   227  	if err != nil {
   228  		return err
   229  	}
   230  	b = bytes.ReplaceAll(b, []byte("\n\n"), []byte("\n"))
   231  	// Try to use a Go string with backquotes, for readability.
   232  	// If not possible due to cueSrc itself having backquotes,
   233  	// use a single-line double quoted string, removing tabs for brevity.
   234  	// We don't use strconv.CanBackquote as it is for quoting as a single line.
   235  	if cueSrc := string(b); !strings.Contains(cueSrc, "`") {
   236  		fmt.Fprintf(g.w, "CUE: `%s`,\n", cueSrc)
   237  	} else {
   238  		cueSrc = strings.ReplaceAll(cueSrc, "\t", "")
   239  		fmt.Fprintf(g.w, "CUE: %q,\n", cueSrc)
   240  	}
   241  	return nil
   242  }
   243  
   244  func (g *generator) processGo(pkg *packages.Package) error {
   245  	// We sort the objects by their original source code position.
   246  	// Otherwise go/types defaults to sorting by name strings.
   247  	// We could remove this code if we were fine with sorting by name.
   248  	scope := pkg.Types.Scope()
   249  	type objWithPos struct {
   250  		obj types.Object
   251  		pos token.Position
   252  	}
   253  	var objs []objWithPos
   254  	for _, name := range scope.Names() {
   255  		obj := scope.Lookup(name)
   256  		objs = append(objs, objWithPos{obj, pkg.Fset.Position(obj.Pos())})
   257  	}
   258  	sort.Slice(objs, func(i, j int) bool {
   259  		obj1, obj2 := objs[i], objs[j]
   260  		if obj1.pos.Filename == obj2.pos.Filename {
   261  			return obj1.pos.Line < obj2.pos.Line
   262  		}
   263  		return obj1.pos.Filename < obj2.pos.Filename
   264  	})
   265  
   266  	for _, obj := range objs {
   267  		obj := obj.obj // no longer need the token.Position
   268  		if !obj.Exported() {
   269  			continue
   270  		}
   271  		// TODO: support type declarations.
   272  		switch obj := obj.(type) {
   273  		case *types.Const:
   274  			var value string
   275  			switch v := obj.Val(); v.Kind() {
   276  			case constant.Bool, constant.Int, constant.String:
   277  				// TODO: convert octal numbers
   278  				value = v.ExactString()
   279  			case constant.Float:
   280  				var rat big.Rat
   281  				rat.SetString(v.ExactString())
   282  				var float big.Float
   283  				float.SetRat(&rat)
   284  				value = float.Text('g', -1)
   285  			default:
   286  				fmt.Printf("Dropped entry %s.%s (%T: %v)\n", g.cuePkgPath, obj.Name(), v.Kind(), v.ExactString())
   287  				continue
   288  			}
   289  			g.sep()
   290  			fmt.Fprintf(g.w, "{\nName: %q,\n Const: %q,\n}", obj.Name(), value)
   291  		case *types.Func:
   292  			g.genFunc(obj)
   293  		}
   294  	}
   295  	return nil
   296  }
   297  
   298  var errorType = types.Universe.Lookup("error").Type()
   299  
   300  func (g *generator) genFunc(fn *types.Func) {
   301  	sign := fn.Type().(*types.Signature)
   302  	if sign.Recv() != nil {
   303  		return
   304  	}
   305  	params := sign.Params()
   306  	results := sign.Results()
   307  	if results == nil || (results.Len() != 1 && results.At(1).Type() != errorType) {
   308  		fmt.Printf("Dropped func %s.%s: must have one return value or a value and an error %v\n", g.cuePkgPath, fn.Name(), sign)
   309  		return
   310  	}
   311  
   312  	g.sep()
   313  	fmt.Fprintf(g.w, "{\n")
   314  	defer fmt.Fprintf(g.w, "}")
   315  
   316  	fmt.Fprintf(g.w, "Name: %q,\n", fn.Name())
   317  
   318  	args := []string{}
   319  	vals := []string{}
   320  	kind := []string{}
   321  	for i := 0; i < params.Len(); i++ {
   322  		param := params.At(i)
   323  		typ := strings.Title(g.goKind(param.Type()))
   324  		argKind := g.goToCUE(param.Type())
   325  		vals = append(vals, fmt.Sprintf("c.%s(%d)", typ, len(args)))
   326  		args = append(args, param.Name())
   327  		kind = append(kind, argKind)
   328  	}
   329  
   330  	fmt.Fprintf(g.w, "Params: []pkg.Param{\n")
   331  	for _, k := range kind {
   332  		fmt.Fprintf(g.w, "{Kind: %s},\n", k)
   333  	}
   334  	fmt.Fprintf(g.w, "\n},\n")
   335  
   336  	fmt.Fprintf(g.w, "Result: %s,\n", g.goToCUE(results.At(0).Type()))
   337  
   338  	argList := strings.Join(args, ", ")
   339  	valList := strings.Join(vals, ", ")
   340  	init := ""
   341  	if len(args) > 0 {
   342  		init = fmt.Sprintf("%s := %s", argList, valList)
   343  	}
   344  
   345  	fmt.Fprintf(g.w, "Func: func(c *pkg.CallCtxt) {")
   346  	defer fmt.Fprintln(g.w, "},")
   347  	fmt.Fprintln(g.w)
   348  	if init != "" {
   349  		fmt.Fprintln(g.w, init)
   350  	}
   351  	fmt.Fprintln(g.w, "if c.Do() {")
   352  	defer fmt.Fprintln(g.w, "}")
   353  	if results.Len() == 1 {
   354  		fmt.Fprintf(g.w, "c.Ret = %s(%s)", fn.Name(), argList)
   355  	} else {
   356  		fmt.Fprintf(g.w, "c.Ret, c.Err = %s(%s)", fn.Name(), argList)
   357  	}
   358  }
   359  
   360  // TODO(mvdan): goKind and goToCUE still use a lot of strings; simplify.
   361  
   362  func (g *generator) goKind(typ types.Type) string {
   363  	if ptr, ok := typ.(*types.Pointer); ok {
   364  		typ = ptr.Elem()
   365  	}
   366  	switch str := types.TypeString(typ, nil); str {
   367  	case "math/big.Int":
   368  		return "bigInt"
   369  	case "math/big.Float":
   370  		return "bigFloat"
   371  	case "math/big.Rat":
   372  		return "bigRat"
   373  	case "cuelang.org/go/internal/core/adt.Bottom":
   374  		return "error"
   375  	case "github.com/cockroachdb/apd/v3.Decimal":
   376  		return "decimal"
   377  	case "cuelang.org/go/internal/pkg.List":
   378  		return "cueList"
   379  	case "cuelang.org/go/internal/pkg.Struct":
   380  		return "struct"
   381  	case "[]*github.com/cockroachdb/apd/v3.Decimal":
   382  		return "decimalList"
   383  	case "cuelang.org/go/cue.Value":
   384  		return "value"
   385  	case "cuelang.org/go/cue.List":
   386  		return "list"
   387  	case "[]string":
   388  		return "stringList"
   389  	case "[]byte":
   390  		return "bytes"
   391  	case "[]cuelang.org/go/cue.Value":
   392  		return "list"
   393  	case "io.Reader":
   394  		return "reader"
   395  	case "time.Time":
   396  		return "string"
   397  	default:
   398  		return str
   399  	}
   400  }
   401  
   402  func (g *generator) goToCUE(typ types.Type) (cueKind string) {
   403  	// TODO: detect list and structs types for return values.
   404  	switch k := g.goKind(typ); k {
   405  	case "error":
   406  		cueKind += "adt.BottomKind"
   407  	case "bool":
   408  		cueKind += "adt.BoolKind"
   409  	case "bytes", "reader":
   410  		cueKind += "adt.BytesKind|adt.StringKind"
   411  	case "string":
   412  		cueKind += "adt.StringKind"
   413  	case "int", "int8", "int16", "int32", "rune", "int64",
   414  		"uint", "byte", "uint8", "uint16", "uint32", "uint64",
   415  		"bigInt":
   416  		cueKind += "adt.IntKind"
   417  	case "float64", "bigRat", "bigFloat", "decimal":
   418  		cueKind += "adt.NumKind"
   419  	case "list", "decimalList", "stringList", "cueList":
   420  		cueKind += "adt.ListKind"
   421  	case "struct":
   422  		cueKind += "adt.StructKind"
   423  	case "value":
   424  		// Must use callCtxt.value method for these types and resolve manually.
   425  		cueKind += "adt.TopKind" // TODO: can be more precise
   426  	default:
   427  		switch {
   428  		case strings.HasPrefix(k, "[]"):
   429  			cueKind += "adt.ListKind"
   430  		case strings.HasPrefix(k, "map["):
   431  			cueKind += "adt.StructKind"
   432  		default:
   433  			// log.Println("Unknown type:", k)
   434  			// Must use callCtxt.value method for these types and resolve manually.
   435  			cueKind += "adt.TopKind" // TODO: can be more precise
   436  		}
   437  	}
   438  	return cueKind
   439  }
   440  
   441  var errNoCUEFiles = errors.New("no CUE files in directory")
   442  
   443  // loadCUEPackage loads a CUE package as a value. We avoid using cue/load because
   444  // that depends on the standard library and as this generator is generating the standard
   445  // library, we don't want that cyclic dependency.
   446  // It only has to deal with the fairly limited subset of CUE packages that are
   447  // present inside pkg/....
   448  func loadCUEPackage(ctx *cue.Context, dir string, pkgPath string) (cue.Value, error) {
   449  	inst := &build.Instance{
   450  		PkgName:     path.Base(pkgPath),
   451  		Dir:         dir,
   452  		DisplayPath: pkgPath,
   453  		ImportPath:  pkgPath,
   454  	}
   455  	cuefiles, err := filepath.Glob(filepath.Join(dir, "*.cue"))
   456  	if err != nil {
   457  		return cue.Value{}, err
   458  	}
   459  	if len(cuefiles) == 0 {
   460  		return cue.Value{}, errNoCUEFiles
   461  	}
   462  	for _, file := range cuefiles {
   463  		if err := inst.AddFile(file, nil); err != nil {
   464  			return cue.Value{}, err
   465  		}
   466  	}
   467  	if err := inst.Complete(); err != nil {
   468  		return cue.Value{}, err
   469  	}
   470  	vals, err := ctx.BuildInstances([]*build.Instance{inst})
   471  	if err != nil {
   472  		return cue.Value{}, err
   473  	}
   474  	return vals[0], nil
   475  }
   476  
   477  // Avoid using cuecontext.New because that package imports
   478  // the entire stdlib which we are generating.
   479  func newContext() *cue.Context {
   480  	r := runtime.New()
   481  	return (*cue.Context)(r)
   482  }
   483
View as plain text