ref.go

Documentation: cuelang.org/go/encoding/jsonschema

     1  // Copyright 2020 CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package jsonschema
    16  
    17  import (
    18  	"net/url"
    19  	"path"
    20  	"strconv"
    21  	"strings"
    22  
    23  	"cuelang.org/go/cue"
    24  	"cuelang.org/go/cue/ast"
    25  	"cuelang.org/go/cue/errors"
    26  	"cuelang.org/go/cue/token"
    27  	"cuelang.org/go/internal"
    28  )
    29  
    30  func (d *decoder) parseRef(p token.Pos, str string) []string {
    31  	u, err := url.Parse(str)
    32  	if err != nil {
    33  		d.addErr(errors.Newf(p, "invalid JSON reference: %s", err))
    34  		return nil
    35  	}
    36  
    37  	if u.Host != "" || u.Path != "" {
    38  		d.addErr(errors.Newf(p, "external references (%s) not supported", str))
    39  		// TODO: handle
    40  		//    host:
    41  		//      If the host corresponds to a package known to cue,
    42  		//      load it from there. It would prefer schema converted to
    43  		//      CUE, although we could consider loading raw JSON schema
    44  		//      if present.
    45  		//      If not present, advise the user to run cue get.
    46  		//    path:
    47  		//      Look up on file system or relatively to authority location.
    48  		return nil
    49  	}
    50  
    51  	if !path.IsAbs(u.Fragment) {
    52  		d.addErr(errors.Newf(p, "anchors (%s) not supported", u.Fragment))
    53  		// TODO: support anchors
    54  		return nil
    55  	}
    56  
    57  	// NOTE: Go bug?: url.URL has no raw representation of the fragment. This
    58  	// means that %2F gets translated to `/` before it can be split. This, in
    59  	// turn, means that field names cannot have a `/` as name.
    60  
    61  	return splitFragment(u)
    62  }
    63  
    64  // resolveURI parses a URI from n and resolves it in the current context.
    65  // To resolve it in the current context, it looks for the closest URI from
    66  // an $id in the parent scopes and the uses the URI resolution to get the
    67  // new URI.
    68  //
    69  // This method is used to resolve any URI, including those from $id and $ref.
    70  func (s *state) resolveURI(n cue.Value) *url.URL {
    71  	str, ok := s.strValue(n)
    72  	if !ok {
    73  		return nil
    74  	}
    75  
    76  	u, err := url.Parse(str)
    77  	if err != nil {
    78  		s.addErr(errors.Newf(n.Pos(), "invalid JSON reference: %s", err))
    79  		return nil
    80  	}
    81  
    82  	for {
    83  		if s.id != nil {
    84  			u = s.id.ResolveReference(u)
    85  			break
    86  		}
    87  		if s.up == nil {
    88  			break
    89  		}
    90  		s = s.up
    91  	}
    92  
    93  	return u
    94  }
    95  
    96  const topSchema = "_schema"
    97  
    98  // makeCUERef converts a URI into a CUE reference for the current location.
    99  // The returned identifier (or first expression in a selection chain), is
   100  // hardwired to point to the resolved value. This will allow astutil.Sanitize
   101  // to automatically unshadow any shadowed variables.
   102  func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
   103  	a := splitFragment(u)
   104  
   105  	switch fn := s.cfg.Map; {
   106  	case fn != nil:
   107  		// TODO: This block is only used in case s.cfg.Map is set, which is
   108  		// currently only used for OpenAPI. Handling should be brought more in
   109  		// line with JSON schema.
   110  		a, err := fn(n.Pos(), a)
   111  		if err != nil {
   112  			s.addErr(errors.Newf(n.Pos(), "invalid reference %q: %v", u, err))
   113  			return nil
   114  		}
   115  		if len(a) == 0 {
   116  			// TODO: should we allow inserting at root level?
   117  			s.addErr(errors.Newf(n.Pos(),
   118  				"invalid empty reference returned by map for %q", u))
   119  			return nil
   120  		}
   121  		sel, ok := a[0].(ast.Expr)
   122  		if !ok {
   123  			sel = &ast.BadExpr{}
   124  		}
   125  		for _, l := range a[1:] {
   126  			switch x := l.(type) {
   127  			case *ast.Ident:
   128  				sel = &ast.SelectorExpr{X: sel, Sel: x}
   129  
   130  			case *ast.BasicLit:
   131  				sel = &ast.IndexExpr{X: sel, Index: x}
   132  			}
   133  		}
   134  		return sel
   135  	}
   136  
   137  	var ident *ast.Ident
   138  
   139  	for ; ; s = s.up {
   140  		if s.up == nil {
   141  			switch {
   142  			case u.Host == "" && u.Path == "",
   143  				s.id != nil && s.id.Host == u.Host && s.id.Path == u.Path:
   144  				if len(a) == 0 {
   145  					// refers to the top of the file. We will allow this by
   146  					// creating a helper schema as such:
   147  					//   _schema: {...}
   148  					//   _schema
   149  					// This is created at the finalization stage if
   150  					// hasSelfReference is set.
   151  					s.hasSelfReference = true
   152  
   153  					ident = ast.NewIdent(topSchema)
   154  					ident.Node = s.obj
   155  					return ident
   156  				}
   157  
   158  				ident, a = s.getNextIdent(n, a)
   159  
   160  			case u.Host != "":
   161  				// Reference not found within scope. Create an import reference.
   162  
   163  				// TODO: allow the configuration to specify a map from
   164  				// URI domain+paths to CUE packages.
   165  
   166  				// TODO: currently only $ids that are in scope can be
   167  				// referenced. We could consider doing an extra pass to record
   168  				// all '$id's in a file to be able to link to them even if they
   169  				// are not in scope.
   170  				p := u.Path
   171  
   172  				base := path.Base(p)
   173  				if !ast.IsValidIdent(base) {
   174  					base = strings.TrimSuffix(base, ".json")
   175  					if !ast.IsValidIdent(base) {
   176  						// Find something more clever to do there. For now just
   177  						// pick "schema" as the package name.
   178  						base = "schema"
   179  					}
   180  					p += ":" + base
   181  				}
   182  
   183  				ident = ast.NewIdent(base)
   184  				ident.Node = &ast.ImportSpec{Path: ast.NewString(u.Host + p)}
   185  
   186  			default:
   187  				// Just a path, not sure what that means.
   188  				s.errf(n, "unknown domain for reference %q", u)
   189  				return nil
   190  			}
   191  			break
   192  		}
   193  
   194  		if s.id == nil {
   195  			continue
   196  		}
   197  
   198  		if s.id.Host == u.Host && s.id.Path == u.Path {
   199  			if len(a) == 0 {
   200  				if len(s.idRef) == 0 {
   201  					// This is a reference to either root or a schema for which
   202  					// we do not yet support references. See Issue #386.
   203  					if s.up.up != nil {
   204  						s.errf(n, "cannot refer to internal schema %q", u)
   205  						return nil
   206  					}
   207  
   208  					// This is referring to the root scope. There is a dummy
   209  					// state above the root state that we need to update.
   210  					s = s.up
   211  
   212  					// refers to the top of the file. We will allow this by
   213  					// creating a helper schema as such:
   214  					//   _schema: {...}
   215  					//   _schema
   216  					// This is created at the finalization stage if
   217  					// hasSelfReference is set.
   218  					s.hasSelfReference = true
   219  					ident = ast.NewIdent(topSchema)
   220  					ident.Node = s.obj
   221  					return ident
   222  				}
   223  
   224  				x := s.idRef[0]
   225  				if !x.isDef && !ast.IsValidIdent(x.name) {
   226  					s.errf(n, "referring to field %q not supported", x.name)
   227  					return nil
   228  				}
   229  				e := ast.NewIdent(x.name)
   230  				if len(s.idRef) == 1 {
   231  					return e
   232  				}
   233  				return newSel(e, s.idRef[1])
   234  			}
   235  			ident, a = s.getNextIdent(n, a)
   236  			ident.Node = s.obj
   237  			break
   238  		}
   239  	}
   240  
   241  	return s.newSel(ident, n, a)
   242  }
   243  
   244  // getNextSelector translates a JSON Reference path into a CUE path by consuming
   245  // the first path elements and returning the corresponding CUE label.
   246  func (s *state) getNextSelector(v cue.Value, a []string) (l label, tail []string) {
   247  	switch elem := a[0]; elem {
   248  	case "$defs", "definitions":
   249  		if len(a) == 1 {
   250  			s.errf(v, "cannot refer to %s section: must refer to one of its elements", a[0])
   251  			return label{}, nil
   252  		}
   253  
   254  		if name := "#" + a[1]; ast.IsValidIdent(name) {
   255  			return label{name, true}, a[2:]
   256  		}
   257  
   258  		return label{"#", true}, a[1:]
   259  
   260  	case "properties":
   261  		if len(a) == 1 {
   262  			s.errf(v, "cannot refer to %s section: must refer to one of its elements", a[0])
   263  			return label{}, nil
   264  		}
   265  
   266  		return label{a[1], false}, a[2:]
   267  
   268  	default:
   269  		return label{elem, false}, a[1:]
   270  
   271  	case "additionalProperties",
   272  		"patternProperties",
   273  		"items",
   274  		"additionalItems":
   275  		// TODO: as a temporary workaround, include the schema verbatim.
   276  		// TODO: provide definitions for these in CUE.
   277  		s.errf(v, "referring to field %q not yet supported", elem)
   278  
   279  		// Other known fields cannot be supported.
   280  		return label{}, nil
   281  	}
   282  }
   283  
   284  // newSel converts a JSON Reference path and initial CUE identifier to
   285  // a CUE selection path.
   286  func (s *state) newSel(e ast.Expr, v cue.Value, a []string) ast.Expr {
   287  	for len(a) > 0 {
   288  		var label label
   289  		label, a = s.getNextSelector(v, a)
   290  		e = newSel(e, label)
   291  	}
   292  	return e
   293  }
   294  
   295  // newSel converts label to a CUE index and creates an expression to index
   296  // into e.
   297  func newSel(e ast.Expr, label label) ast.Expr {
   298  	if label.isDef {
   299  		return ast.NewSel(e, label.name)
   300  
   301  	}
   302  	if ast.IsValidIdent(label.name) && !internal.IsDefOrHidden(label.name) {
   303  		return ast.NewSel(e, label.name)
   304  	}
   305  	return &ast.IndexExpr{X: e, Index: ast.NewString(label.name)}
   306  }
   307  
   308  func (s *state) setField(lab label, f *ast.Field) {
   309  	x := s.getRef(lab)
   310  	x.field = f
   311  	s.setRef(lab, x)
   312  	x = s.getRef(lab)
   313  }
   314  
   315  func (s *state) getRef(lab label) refs {
   316  	if s.fieldRefs == nil {
   317  		s.fieldRefs = make(map[label]refs)
   318  	}
   319  	x, ok := s.fieldRefs[lab]
   320  	if !ok {
   321  		if lab.isDef ||
   322  			(ast.IsValidIdent(lab.name) && !internal.IsDefOrHidden(lab.name)) {
   323  			x.ident = lab.name
   324  		} else {
   325  			x.ident = "_X" + strconv.Itoa(s.decoder.numID)
   326  			s.decoder.numID++
   327  		}
   328  		s.fieldRefs[lab] = x
   329  	}
   330  	return x
   331  }
   332  
   333  func (s *state) setRef(lab label, r refs) {
   334  	s.fieldRefs[lab] = r
   335  }
   336  
   337  // getNextIdent gets the first CUE reference from a JSON Reference path and
   338  // converts it to a CUE identifier.
   339  func (s *state) getNextIdent(v cue.Value, a []string) (resolved *ast.Ident, tail []string) {
   340  	lab, a := s.getNextSelector(v, a)
   341  
   342  	x := s.getRef(lab)
   343  	ident := ast.NewIdent(x.ident)
   344  	x.refs = append(x.refs, ident)
   345  	s.setRef(lab, x)
   346  
   347  	return ident, a
   348  }
   349  
   350  // linkReferences resolves identifiers to relevant nodes. This allows
   351  // astutil.Sanitize to unshadow nodes if necessary.
   352  func (s *state) linkReferences() {
   353  	for _, r := range s.fieldRefs {
   354  		if r.field == nil {
   355  			// TODO: improve error message.
   356  			s.errf(cue.Value{}, "reference to non-existing value %q", r.ident)
   357  			continue
   358  		}
   359  
   360  		// link resembles the link value. See astutil.Resolve.
   361  		var link ast.Node
   362  
   363  		ident, ok := r.field.Label.(*ast.Ident)
   364  		if ok && ident.Name == r.ident {
   365  			link = r.field.Value
   366  		} else if len(r.refs) > 0 {
   367  			r.field.Label = &ast.Alias{
   368  				Ident: ast.NewIdent(r.ident),
   369  				Expr:  r.field.Label.(ast.Expr),
   370  			}
   371  			link = r.field
   372  		}
   373  
   374  		for _, i := range r.refs {
   375  			i.Node = link
   376  		}
   377  	}
   378  }
   379  
   380  // splitFragment splits the fragment part of a URI into path components. The
   381  // result may be an empty slice.
   382  //
   383  // TODO: this requires RawFragment introduced in go1.15 to function properly.
   384  // As for now, CUE still uses go1.12.
   385  func splitFragment(u *url.URL) []string {
   386  	if u.Fragment == "" {
   387  		return nil
   388  	}
   389  	s := strings.TrimRight(u.Fragment[1:], "/")
   390  	if s == "" {
   391  		return nil
   392  	}
   393  	return strings.Split(s, "/")
   394  }
   395  
   396  func (d *decoder) mapRef(p token.Pos, str string, ref []string) []ast.Label {
   397  	fn := d.cfg.Map
   398  	if fn == nil {
   399  		fn = jsonSchemaRef
   400  	}
   401  	a, err := fn(p, ref)
   402  	if err != nil {
   403  		if str == "" {
   404  			str = "#/" + strings.Join(ref, "/")
   405  		}
   406  		d.addErr(errors.Newf(p, "invalid reference %q: %v", str, err))
   407  		return nil
   408  	}
   409  	if len(a) == 0 {
   410  		// TODO: should we allow inserting at root level?
   411  		if str == "" {
   412  			str = "#/" + strings.Join(ref, "/")
   413  		}
   414  		d.addErr(errors.Newf(p,
   415  			"invalid empty reference returned by map for %q", str))
   416  		return nil
   417  	}
   418  	return a
   419  }
   420  
   421  func jsonSchemaRef(p token.Pos, a []string) ([]ast.Label, error) {
   422  	// TODO: technically, references could reference a
   423  	// non-definition. We disallow this case for the standard
   424  	// JSON Schema interpretation. We could detect cases that
   425  	// are not definitions and then resolve those as literal
   426  	// values.
   427  	if len(a) != 2 || (a[0] != "definitions" && a[0] != "$defs") {
   428  		return nil, errors.Newf(p,
   429  			// Don't mention the ability to use $defs, as this definition seems
   430  			// to already have been withdrawn from the JSON Schema spec.
   431  			"$ref must be of the form #/definitions/...")
   432  	}
   433  	name := a[1]
   434  	if ast.IsValidIdent(name) &&
   435  		name != rootDefs[1:] &&
   436  		!internal.IsDefOrHidden(name) {
   437  		return []ast.Label{ast.NewIdent("#" + name)}, nil
   438  	}
   439  	return []ast.Label{ast.NewIdent(rootDefs), ast.NewString(name)}, nil
   440  }
   441
View as plain text