1 // Copyright 2018 The CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package load 16 17 import ( 18 "context" 19 "io" 20 "os" 21 "path/filepath" 22 "strings" 23 24 "cuelang.org/go/cue/ast" 25 "cuelang.org/go/cue/build" 26 "cuelang.org/go/cue/errors" 27 "cuelang.org/go/cue/token" 28 "cuelang.org/go/internal" 29 "cuelang.org/go/internal/cueexperiment" 30 "cuelang.org/go/mod/modconfig" 31 "cuelang.org/go/mod/modfile" 32 "cuelang.org/go/mod/module" 33 ) 34 35 const ( 36 cueSuffix = ".cue" 37 modDir = "cue.mod" 38 moduleFile = "module.cue" 39 ) 40 41 // FromArgsUsage is a partial usage message that applications calling 42 // FromArgs may wish to include in their -help output. 43 // 44 // Some of the aspects of this documentation, like flags and handling '--' need 45 // to be implemented by the tools. 46 const FromArgsUsage = ` 47 <args> is a list of arguments denoting a set of instances of the form: 48 49 <package>* <file_args>* 50 51 1. A list of source files 52 53 CUE files are parsed, loaded and unified into a single instance. All files 54 must have the same package name. 55 56 Data files, like YAML or JSON, are handled in one of two ways: 57 58 a. Explicitly mapped into a single CUE namespace, using the --path, --files 59 and --list flags. In this case these are unified into a single instance 60 along with any other CUE files. 61 62 b. Treated as a stream of data elements that each is optionally unified with 63 a single instance, which either consists of the other CUE files specified 64 on the command line or a single package. 65 66 By default, the format of files is derived from the file extension. 67 This behavior may be modified with file arguments of the form <qualifiers>: 68 For instance, 69 70 cue eval foo.cue json: bar.data 71 72 indicates that the bar.data file should be interpreted as a JSON file. 73 A qualifier applies to all files following it until the next qualifier. 74 75 The following qualifiers are available: 76 77 encodings 78 cue CUE definitions and data 79 json JSON data, one value only 80 jsonl newline-separated JSON values 81 yaml a YAML file, may contain a stream 82 proto Protobuf definitions 83 84 interpretations 85 jsonschema data encoding describes JSON Schema 86 openapi data encoding describes Open API 87 88 formats 89 data output as -- or only accept -- data 90 graph data allowing references or anchors 91 schema output as schema; defaults JSON files to JSON Schema 92 def full definitions, including documentation 93 94 2. A list of relative directories to denote a package instance. 95 96 Each directory matching the pattern is loaded as a separate instance. 97 The instance contains all files in this directory and ancestor directories, 98 up to the module root, with the same package name. The package name must 99 be either uniquely determined by the files in the given directory, or 100 explicitly defined using a package name qualifier. For instance, ./...:foo 101 selects all packages named foo in the any subdirectory of the current 102 working directory. 103 104 3. An import path referring to a directory within the current module 105 106 All CUE files in that directory, and all the ancestor directories up to the 107 module root (if applicable), with a package name corresponding to the base 108 name of the directory or the optional explicit package name are loaded into 109 a single instance. 110 111 Examples, assume a module name of acme.org/root: 112 mod.test/foo package in cue.mod 113 ./foo package corresponding to foo directory 114 .:bar package in current directory with package name bar 115 ` 116 117 // GenPath reports the directory in which to store generated 118 // files. 119 func GenPath(root string) string { 120 return internal.GenPath(root) 121 } 122 123 // A Config configures load behavior. 124 type Config struct { 125 // TODO: allow passing a cuecontext to be able to lookup and verify builtin 126 // packages at loading time. 127 128 // Context specifies the context for the load operation. 129 Context *build.Context 130 131 // A Module is a collection of packages and instances that are within the 132 // directory hierarchy rooted at the module root. The module root can be 133 // marked with a cue.mod file. 134 ModuleRoot string 135 136 // Module specifies the module prefix. If not empty, this value must match 137 // the module field of an existing cue.mod file. 138 Module string 139 140 // modFile holds the contents of the module file, or nil 141 // if no module file was present. If non-nil, then 142 // after calling Config.complete, modFile.Module will be 143 // equal to Module. 144 modFile *modfile.File 145 146 // Package defines the name of the package to be loaded. If this is not set, 147 // the package must be uniquely defined from its context. Special values: 148 // _ load files without a package 149 // * load all packages. Files without packages are loaded 150 // in the _ package. 151 Package string 152 153 // Dir is the base directory for import path resolution. 154 // For example, it is used to determine the main module, 155 // and rooted import paths starting with "./" are relative to it. 156 // If Dir is empty, the current directory is used. 157 Dir string 158 159 // Tags defines boolean tags or key-value pairs to select files to build 160 // or be injected as values in fields. 161 // 162 // Each string is of the form 163 // 164 // key [ "=" value ] 165 // 166 // where key is a valid CUE identifier and value valid CUE scalar. 167 // 168 // The Tags values are used to both select which files get included in a 169 // build and to inject values into the AST. 170 // 171 // 172 // File selection 173 // 174 // Files with an attribute of the form @if(expr) before a package clause 175 // are conditionally included if expr resolves to true, where expr refers to 176 // boolean values in Tags. 177 // 178 // It is an error for a file to have more than one @if attribute or to 179 // have a @if attribute without or after a package clause. 180 // 181 // 182 // Value injection 183 // 184 // The Tags values are also used to inject values into fields with a 185 // @tag attribute. 186 // 187 // For any field of the form 188 // 189 // field: x @tag(key) 190 // 191 // and Tags value for which the name matches key, the field will be 192 // modified to 193 // 194 // field: x & "value" 195 // 196 // By default, the injected value is treated as a string. Alternatively, a 197 // "type" option of the @tag attribute allows a value to be interpreted as 198 // an int, number, or bool. For instance, for a field 199 // 200 // field: x @tag(key,type=int) 201 // 202 // an entry "key=2" modifies the field to 203 // 204 // field: x & 2 205 // 206 // Valid values for type are "int", "number", "bool", and "string". 207 // 208 // A @tag attribute can also define shorthand values, which can be injected 209 // into the fields without having to specify the key. For instance, for 210 // 211 // environment: string @tag(env,short=prod|staging) 212 // 213 // the Tags entry "prod" sets the environment field to the value "prod". 214 // This is equivalent to a Tags entry of "env=prod". 215 // 216 // The use of @tag does not preclude using any of the usual CUE constraints 217 // to limit the possible values of a field. For instance 218 // 219 // environment: "prod" | "staging" @tag(env,short=prod|staging) 220 // 221 // ensures the user may only specify "prod" or "staging". 222 Tags []string 223 224 // TagVars defines a set of key value pair the values of which may be 225 // referenced by tags. 226 // 227 // Use DefaultTagVars to get a pre-loaded map with supported values. 228 TagVars map[string]TagVar 229 230 // Include all files, regardless of tags. 231 AllCUEFiles bool 232 233 // Deprecated: use Tags 234 BuildTags []string 235 releaseTags []string 236 237 // If Tests is set, the loader includes not just the packages 238 // matching a particular pattern but also any related test packages. 239 Tests bool 240 241 // If Tools is set, the loader includes tool files associated with 242 // a package. 243 Tools bool 244 245 // filesMode indicates that files are specified 246 // explicitly on the command line. 247 filesMode bool 248 249 // If DataFiles is set, the loader includes entries for directories that 250 // have no CUE files, but have recognized data files that could be converted 251 // to CUE. 252 DataFiles bool 253 254 // StdRoot specifies an alternative directory for standard libraries. 255 // This is mostly used for bootstrapping. 256 StdRoot string 257 258 // ParseFile is called to read and parse each file when preparing a 259 // package's syntax tree. It must be safe to call ParseFile simultaneously 260 // from multiple goroutines. If ParseFile is nil, the loader will uses 261 // parser.ParseFile. 262 // 263 // ParseFile should parse the source from src and use filename only for 264 // recording position information. 265 // 266 // An application may supply a custom implementation of ParseFile to change 267 // the effective file contents or the behavior of the parser, or to modify 268 // the syntax tree. 269 ParseFile func(name string, src interface{}) (*ast.File, error) 270 271 // Overlay provides a mapping of absolute file paths to file contents. If 272 // the file with the given path already exists, the parser will use the 273 // alternative file contents provided by the map. 274 Overlay map[string]Source 275 276 // Stdin defines an alternative for os.Stdin for the file "-". When used, 277 // the corresponding build.File will be associated with the full buffer. 278 Stdin io.Reader 279 280 // Registry is used to fetch CUE module dependencies. 281 // 282 // When nil, if the modules experiment is enabled 283 // (CUE_EXPERIMENT=modules), [modconfig.NewRegistry] 284 // will be used to create a registry instance using the 285 // usual cmd/cue conventions for environment variables 286 // (but see the Env field below). 287 // 288 // THIS IS EXPERIMENTAL. API MIGHT CHANGE. 289 Registry modconfig.Registry 290 291 // Env provides environment variables for use in the configuration. 292 // Currently this is only used in the construction of the Registry 293 // value (see above). If this is nil, the current process's environment 294 // will be used. 295 Env []string 296 297 fileSystem fileSystem 298 } 299 300 func (c *Config) stdin() io.Reader { 301 if c.Stdin == nil { 302 return os.Stdin 303 } 304 return c.Stdin 305 } 306 307 type importPath string 308 309 type fsPath string 310 311 func addImportQualifier(pkg importPath, name string) (importPath, errors.Error) { 312 if name != "" { 313 s := string(pkg) 314 if i := strings.LastIndexByte(s, '/'); i >= 0 { 315 s = s[i+1:] 316 } 317 if i := strings.LastIndexByte(s, ':'); i >= 0 { 318 // should never happen, but just in case. 319 s = s[i+1:] 320 if s != name { 321 return "", errors.Newf(token.NoPos, 322 "non-matching package names (%s != %s)", s, name) 323 } 324 } else if s != name { 325 pkg += importPath(":" + name) 326 } 327 } 328 329 return pkg, nil 330 } 331 332 // Complete updates the configuration information. After calling complete, 333 // the following invariants hold: 334 // - c.Dir is an absolute path. 335 // - c.ModuleRoot is an absolute path 336 // - c.Module is set to the module import prefix if there is a cue.mod file 337 // with the module property. 338 // - c.loader != nil 339 // - c.cache != "" 340 // 341 // It does not initialize c.Context, because that requires the 342 // loader in order to use for build.Loader. 343 func (c Config) complete() (cfg *Config, err error) { 344 // Each major CUE release should add a tag here. 345 // Old tags should not be removed. That is, the cue1.x tag is present 346 // in all releases >= CUE 1.x. Code that requires CUE 1.x or later should 347 // say "+build cue1.x", and code that should only be built before CUE 1.x 348 // (perhaps it is the stub to use in that case) should say "+build !cue1.x". 349 c.releaseTags = []string{"cue0.1"} 350 351 if c.Dir == "" { 352 c.Dir, err = os.Getwd() 353 if err != nil { 354 return nil, err 355 } 356 } else if c.Dir, err = filepath.Abs(c.Dir); err != nil { 357 return nil, err 358 } 359 360 // TODO: we could populate this already with absolute file paths, 361 // but relative paths cannot be added. Consider what is reasonable. 362 if err := c.fileSystem.init(c.Dir, c.Overlay); err != nil { 363 return nil, err 364 } 365 366 // TODO: determine root on a package basis. Maybe we even need a 367 // pkgname.cue.mod 368 // Look to see if there is a cue.mod. 369 if c.ModuleRoot == "" { 370 // Only consider the current directory by default 371 c.ModuleRoot = c.Dir 372 if root := c.findRoot(c.Dir); root != "" { 373 c.ModuleRoot = root 374 } 375 } else if !filepath.IsAbs(c.ModuleRoot) { 376 c.ModuleRoot = filepath.Join(c.Dir, c.ModuleRoot) 377 } 378 // Note: if cueexperiment.Flags.Modules _isn't_ set but c.Registry 379 // is, we consider that a good enough hint that modules support 380 // should be enabled and hence don't return an error in that case. 381 if cueexperiment.Flags.Modules && c.Registry == nil { 382 registry, err := modconfig.NewRegistry(&modconfig.Config{ 383 Env: c.Env, 384 }) 385 if err != nil { 386 // If there's an error in the registry configuration, 387 // don't error immediately, but only when we actually 388 // need to resolve modules. 389 registry = errorRegistry{err} 390 } 391 c.Registry = registry 392 } 393 if err := c.loadModule(); err != nil { 394 return nil, err 395 } 396 return &c, nil 397 } 398 399 // loadModule loads the module file, resolves and downloads module 400 // dependencies. It sets c.Module if it's empty or checks it for 401 // consistency with the module file otherwise. 402 func (c *Config) loadModule() error { 403 // TODO: also make this work if run from outside the module? 404 mod := filepath.Join(c.ModuleRoot, modDir) 405 info, cerr := c.fileSystem.stat(mod) 406 if cerr != nil { 407 return nil 408 } 409 // TODO remove support for legacy non-directory module.cue file 410 // by returning an error if info.IsDir is false. 411 if info.IsDir() { 412 mod = filepath.Join(mod, moduleFile) 413 } 414 f, cerr := c.fileSystem.openFile(mod) 415 if cerr != nil { 416 return nil 417 } 418 defer f.Close() 419 data, err := io.ReadAll(f) 420 if err != nil { 421 return err 422 } 423 parseModFile := modfile.ParseNonStrict 424 if c.Registry == nil { 425 parseModFile = modfile.ParseLegacy 426 } 427 mf, err := parseModFile(data, mod) 428 if err != nil { 429 return err 430 } 431 c.modFile = mf 432 if mf.Module == "" { 433 // Backward compatibility: allow empty module.cue file. 434 // TODO maybe check that the rest of the fields are empty too? 435 return nil 436 } 437 if c.Module != "" && c.Module != mf.Module { 438 return errors.Newf(token.NoPos, "inconsistent modules: got %q, want %q", mf.Module, c.Module) 439 } 440 c.Module = mf.Module 441 return nil 442 } 443 444 func (c Config) isRoot(dir string) bool { 445 fs := &c.fileSystem 446 // Note: cue.mod used to be a file. We still allow both to match. 447 _, err := fs.stat(filepath.Join(dir, modDir)) 448 return err == nil 449 } 450 451 // findRoot returns the module root that's ancestor 452 // of the given absolute directory path, or "" if none was found. 453 func (c Config) findRoot(absDir string) string { 454 abs := absDir 455 for { 456 if c.isRoot(abs) { 457 return abs 458 } 459 d := filepath.Dir(abs) 460 if filepath.Base(filepath.Dir(abs)) == modDir { 461 // The package was located within a "cue.mod" dir and there was 462 // not cue.mod found until now. So there is no root. 463 return "" 464 } 465 if len(d) >= len(abs) { 466 return "" // reached top of file system, no cue.mod 467 } 468 abs = d 469 } 470 } 471 472 func (c *Config) newErrInstance(err error) *build.Instance { 473 i := c.Context.NewInstance("", nil) 474 i.Root = c.ModuleRoot 475 i.Module = c.Module 476 i.Err = errors.Promote(err, "instance") 477 return i 478 } 479 480 // errorRegistry implements [modconfig.Registry] by returning err from all methods. 481 type errorRegistry struct { 482 err error 483 } 484 485 func (r errorRegistry) Requirements(ctx context.Context, m module.Version) ([]module.Version, error) { 486 return nil, r.err 487 } 488 489 func (r errorRegistry) Fetch(ctx context.Context, m module.Version) (module.SourceLoc, error) { 490 return module.SourceLoc{}, r.err 491 } 492 493 func (r errorRegistry) ModuleVersions(ctx context.Context, mpath string) ([]string, error) { 494 return nil, r.err 495 } 496