1 // Copyright 2019 CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package protobuf defines functionality for parsing protocol buffer 16 // definitions and instances. 17 // 18 // Proto definition mapping follows the guidelines of mapping Proto to JSON as 19 // discussed in https://developers.google.com/protocol-buffers/docs/proto3, and 20 // carries some of the mapping further when possible with CUE. 21 // 22 // # Package Paths 23 // 24 // If a .proto file contains a go_package directive, it will be used as the 25 // destination package fo the generated .cue files. A common use case is to 26 // generate the CUE in the same directory as the .proto definition. If a 27 // destination package is not within the current CUE module, it will be written 28 // relative to the pkg directory. 29 // 30 // If a .proto file does not specify go_package, it will convert a proto package 31 // "google.parent.sub" to the import path "googleapis.com/google/parent/sub". 32 // It is safe to mix package with and without a go_package within the same 33 // project. 34 // 35 // # Type Mappings 36 // 37 // The following type mappings of definitions apply: 38 // 39 // Proto type CUE type/def Comments 40 // message struct Message fields become CUE fields, whereby 41 // names are mapped to lowerCamelCase. 42 // enum e1 | e2 | ... Where ex are strings. A separate mapping is 43 // generated to obtain the numeric values. 44 // map<K, V> { <>: V } All keys are converted to strings. 45 // repeated V [...V] null is accepted as the empty list []. 46 // bool bool 47 // string string 48 // bytes bytes A base64-encoded string when converted to JSON. 49 // int32, fixed32 int32 An integer with bounds as defined by int32. 50 // uint32 uint32 An integer with bounds as defined by uint32. 51 // int64, fixed64 int64 An integer with bounds as defined by int64. 52 // uint64 uint64 An integer with bounds as defined by uint64. 53 // float float32 A number with bounds as defined by float32. 54 // double float64 A number with bounds as defined by float64. 55 // Struct struct See struct.proto. 56 // Value _ See struct.proto. 57 // ListValue [...] See struct.proto. 58 // NullValue null See struct.proto. 59 // BoolValue bool See struct.proto. 60 // StringValue string See struct.proto. 61 // NumberValue number See struct.proto. 62 // StringValue string See struct.proto. 63 // Empty close({}) 64 // Timestamp time.Time See struct.proto. 65 // Duration time.Duration See struct.proto. 66 // 67 // Protobuf definitions can be annotated with CUE constraints that are included 68 // in the generated CUE: 69 // 70 // (cue.val) string CUE expression defining a constraint for this 71 // field. The string may refer to other fields 72 // in a message definition using their JSON name. 73 // 74 // (cue.opt) FieldOptions 75 // required bool Defines the field is required. Use with 76 // caution. 77 package protobuf 78 79 // TODO mappings: 80 // 81 // Wrapper types various types 2, "2", "foo", true, "true", null, 0, … Wrappers use the same representation in JSON as the wrapped primitive type, except that null is allowed and preserved during data conversion and transfer. 82 // FieldMask string "f.fooBar,h" See field_mask.proto. 83 // Any {"@type":"url", See struct.proto. 84 // f1: value, 85 // ...} 86 87 import ( 88 "os" 89 "path/filepath" 90 "slices" 91 "strings" 92 93 "cuelang.org/go/cue/ast" 94 "cuelang.org/go/cue/build" 95 "cuelang.org/go/cue/errors" 96 "cuelang.org/go/cue/format" 97 "cuelang.org/go/cue/parser" 98 "cuelang.org/go/cue/token" 99 "cuelang.org/go/internal" 100 101 // Generated protobuf CUE may use builtins. Ensure that these can always be 102 // found, even if the user does not use cue/load or another package that 103 // triggers its loading. 104 // 105 // TODO: consider whether just linking in the necessary packages suffices. 106 // It probably does, but this may reorder some of the imports, which may, 107 // in turn, change the numbering, which can be confusing while debugging. 108 _ "cuelang.org/go/pkg" 109 ) 110 111 // Config specifies the environment into which to parse a proto definition file. 112 type Config struct { 113 // Root specifies the root of the CUE project, which typically coincides 114 // with, for example, a version control repository root or the Go module. 115 // Any imports of proto files within the directory tree of this of this root 116 // are considered to be "project files" and are generated at the 117 // corresponding location with this hierarchy. Any other imports are 118 // considered to be external. Files for such imports are rooted under the 119 // $Root/pkg/, using the Go package path specified in the .proto file. 120 Root string 121 122 // Module is the Go package import path of the module root. It is the value 123 // as after "module" in a cue.mod/modules.cue file, if a module file is 124 // present. 125 Module string // TODO: determine automatically if unspecified. 126 127 // Paths defines the include directory in which to search for imports. 128 Paths []string 129 130 // PkgName specifies the package name for a generated CUE file. A value 131 // will be derived from the Go package name if undefined. 132 PkgName string 133 134 // EnumMode defines whether enums should be set as integer values, instead 135 // of strings. 136 // 137 // json value is a string, corresponding to the standard JSON mapping 138 // of Protobuf. The value is associated with a #enumValue 139 // to allow the json+pb interpretation to interpret integers 140 // as well. 141 // 142 // int value is an integer associated with an #enumValue definition 143 // The json+pb interpreter uses the definition names in the 144 // disjunction of the enum to interpret strings. 145 // 146 EnumMode string 147 } 148 149 // An Extractor converts a collection of proto files, typically belonging to one 150 // repo or module, to CUE. It thereby observes the CUE package layout. 151 // 152 // CUE observes the same package layout as Go and requires .proto files to have 153 // the go_package directive. Generated CUE files are put in the same directory 154 // as their corresponding .proto files if the .proto files are located in the 155 // specified Root (or current working directory if none is specified). 156 // All other imported files are assigned to the CUE pkg dir ($Root/pkg) 157 // according to their Go package import path. 158 type Extractor struct { 159 root string 160 cwd string 161 module string 162 paths []string 163 pkgName string 164 enumMode string 165 166 fileCache map[string]result 167 imports map[string]*build.Instance 168 169 errs errors.Error 170 done bool 171 } 172 173 type result struct { 174 p *protoConverter 175 err error 176 } 177 178 // NewExtractor creates an Extractor. If the configuration contained any errors 179 // it will be observable by the Err method fo the Extractor. It is safe, 180 // however, to only check errors after building the output. 181 func NewExtractor(c *Config) *Extractor { 182 cwd, _ := os.Getwd() 183 b := &Extractor{ 184 root: c.Root, 185 cwd: cwd, 186 paths: c.Paths, 187 pkgName: c.PkgName, 188 module: c.Module, 189 enumMode: c.EnumMode, 190 fileCache: map[string]result{}, 191 imports: map[string]*build.Instance{}, 192 } 193 194 if b.root == "" { 195 b.root = b.cwd 196 } 197 198 return b 199 } 200 201 // Err returns the errors accumulated during testing. The returned error may be 202 // of type cuelang.org/go/cue/errors.List. 203 func (b *Extractor) Err() error { 204 return b.errs 205 } 206 207 func (b *Extractor) addErr(err error) { 208 b.errs = errors.Append(b.errs, errors.Promote(err, "unknown error")) 209 } 210 211 // AddFile adds a proto definition file to be converted into CUE by the builder. 212 // Relatives paths are always taken relative to the Root with which the b is 213 // configured. 214 // 215 // AddFile assumes that the proto file compiles with protoc and may not report 216 // an error if it does not. Imports are resolved using the paths defined in 217 // Config. 218 func (b *Extractor) AddFile(filename string, src interface{}) error { 219 if b.done { 220 err := errors.Newf(token.NoPos, 221 "protobuf: cannot call AddFile: Instances was already called") 222 b.errs = errors.Append(b.errs, err) 223 return err 224 } 225 if b.root != b.cwd && !filepath.IsAbs(filename) { 226 filename = filepath.Join(b.root, filename) 227 } 228 _, err := b.parse(filename, src) 229 return err 230 } 231 232 // TODO: some way of (recursively) adding multiple proto files with filter. 233 234 // Files returns a File for each proto file that was added or imported, 235 // recursively. 236 func (b *Extractor) Files() (files []*ast.File, err error) { 237 defer func() { err = b.Err() }() 238 b.done = true 239 240 instances, err := b.Instances() 241 if err != nil { 242 return nil, err 243 } 244 245 for _, p := range instances { 246 files = append(files, p.Files...) 247 } 248 return files, nil 249 } 250 251 // Instances creates a build.Instances for every package for which a proto file 252 // was added to the builder. This includes transitive dependencies. It does not 253 // write the generated files to disk. 254 // 255 // The returned instances can be passed to cue.Build to generated the 256 // corresponding CUE instances. 257 // 258 // All import paths are located within the specified Root, where external 259 // packages are located under $Root/pkg. Instances for builtin (like time) 260 // packages may be omitted, and if not will have no associated files. 261 func (b *Extractor) Instances() (instances []*build.Instance, err error) { 262 defer func() { err = b.Err() }() 263 b.done = true 264 265 for _, r := range b.fileCache { 266 if r.err != nil { 267 b.addErr(r.err) 268 continue 269 } 270 inst := b.getInst(r.p) 271 if inst == nil { 272 continue 273 } 274 275 // Set canonical CUE path for generated file. 276 f := r.p.file 277 base := filepath.Base(f.Filename) 278 base = base[:len(base)-len(".proto")] + "_proto_gen.cue" 279 f.Filename = filepath.Join(inst.Dir, base) 280 buf, err := format.Node(f) 281 if err != nil { 282 b.addErr(err) 283 // return nil, err 284 continue 285 } 286 f, err = parser.ParseFile(f.Filename, buf, parser.ParseComments) 287 if err != nil { 288 b.addErr(err) 289 continue 290 } 291 292 inst.Files = append(inst.Files, f) 293 294 for pkg := range r.p.imported { 295 inst.ImportPaths = append(inst.ImportPaths, pkg) 296 } 297 } 298 299 for _, p := range b.imports { 300 instances = append(instances, p) 301 slices.Sort(p.ImportPaths) 302 p.ImportPaths = slices.Compact(p.ImportPaths) 303 for _, i := range p.ImportPaths { 304 if imp := b.imports[i]; imp != nil { 305 p.Imports = append(p.Imports, imp) 306 } 307 } 308 309 slices.SortFunc(p.Files, func(a, b *ast.File) int { 310 return strings.Compare(a.Filename, b.Filename) 311 }) 312 } 313 slices.SortFunc(instances, func(a, b *build.Instance) int { 314 return strings.Compare(a.ImportPath, b.ImportPath) 315 }) 316 317 if err != nil { 318 return instances, err 319 } 320 return instances, nil 321 } 322 323 func (b *Extractor) getInst(p *protoConverter) *build.Instance { 324 if b.errs != nil { 325 return nil 326 } 327 importPath := p.qualifiedImportPath() 328 if importPath == "" { 329 err := errors.Newf(token.NoPos, 330 "no package clause for proto package %q in file %s", p.id, p.file.Filename) 331 b.errs = errors.Append(b.errs, err) 332 // TODO: find an alternative. Is proto package good enough? 333 return nil 334 } 335 336 dir := b.root 337 path := p.importPath() 338 file := p.file.Filename 339 if !filepath.IsAbs(file) { 340 file = filepath.Join(b.root, p.file.Filename) 341 } 342 // Determine whether the generated file should be included in place, or 343 // within cue.mod. 344 inPlace := strings.HasPrefix(file, b.root) 345 if !strings.HasPrefix(path, b.module) { 346 // b.module is either "", in which case we assume the setting for 347 // inPlace, or not, in which case the module in the protobuf must 348 // correspond with that of the proto package. 349 inPlace = false 350 } 351 if !inPlace { 352 dir = filepath.Join(internal.GenPath(dir), path) 353 } else { 354 dir = filepath.Dir(p.file.Filename) 355 } 356 357 // TODO: verify module name from go_package option against that of actual 358 // CUE module. Maybe keep this old code for some strict mode? 359 // want := filepath.Dir(p.file.Filename) 360 // dir = filepath.Join(dir, path[len(b.module)+1:]) 361 // if !filepath.IsAbs(want) { 362 // want = filepath.Join(b.root, want) 363 // } 364 // if dir != want { 365 // err := errors.Newf(token.NoPos, 366 // "file %s mapped to inconsistent path %s; module name %q may be inconsistent with root dir %s", 367 // want, dir, b.module, b.root, 368 // ) 369 // b.errs = errors.Append(b.errs, err) 370 // } 371 372 inst := b.imports[importPath] 373 if inst == nil { 374 inst = &build.Instance{ 375 Root: b.root, 376 Dir: dir, 377 ImportPath: importPath, 378 PkgName: p.shortPkgName, 379 DisplayPath: p.protoPkg, 380 } 381 b.imports[importPath] = inst 382 } 383 return inst 384 } 385 386 // Extract parses a single proto file and returns its contents translated to a CUE 387 // file. If src is not nil, it will use this as the contents of the file. It may 388 // be a string, []byte or io.Reader. Otherwise Extract will open the given file 389 // name at the fully qualified path. 390 // 391 // Extract assumes the proto file compiles with protoc and may not report an error 392 // if it does not. Imports are resolved using the paths defined in Config. 393 func Extract(filename string, src interface{}, c *Config) (f *ast.File, err error) { 394 if c == nil { 395 c = &Config{} 396 } 397 b := NewExtractor(c) 398 399 p, err := b.parse(filename, src) 400 if err != nil { 401 return nil, err 402 } 403 p.file.Filename = filename[:len(filename)-len(".proto")] + "_gen.cue" 404 return p.file, b.Err() 405 } 406 407 // TODO 408 // func GenDefinition 409 410 // func MarshalText(cue.Value) (string, error) { 411 // return "", nil 412 // } 413 414 // func MarshalBytes(cue.Value) ([]byte, error) { 415 // return nil, nil 416 // } 417 418 // func UnmarshalText(descriptor cue.Value, b string) (ast.Expr, error) { 419 // return nil, nil 420 // } 421 422 // func UnmarshalBytes(descriptor cue.Value, b []byte) (ast.Expr, error) { 423 // return nil, nil 424 // } 425