1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package yaml 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "io" 24 "reflect" 25 "strconv" 26 27 "sigs.k8s.io/yaml/goyaml.v2" 28 ) 29 30 // Marshal marshals obj into JSON using stdlib json.Marshal, and then converts JSON to YAML using JSONToYAML (see that method for more reference) 31 func Marshal(obj interface{}) ([]byte, error) { 32 jsonBytes, err := json.Marshal(obj) 33 if err != nil { 34 return nil, fmt.Errorf("error marshaling into JSON: %w", err) 35 } 36 37 return JSONToYAML(jsonBytes) 38 } 39 40 // JSONOpt is a decoding option for decoding from JSON format. 41 type JSONOpt func(*json.Decoder) *json.Decoder 42 43 // Unmarshal first converts the given YAML to JSON, and then unmarshals the JSON into obj. Options for the 44 // standard library json.Decoder can be optionally specified, e.g. to decode untyped numbers into json.Number instead of float64, or to disallow unknown fields (but for that purpose, see also UnmarshalStrict). obj must be a non-nil pointer. 45 // 46 // Important notes about the Unmarshal logic: 47 // 48 // - Decoding is case-insensitive, unlike the rest of Kubernetes API machinery, as this is using the stdlib json library. This might be confusing to users. 49 // - This decodes any number (although it is an integer) into a float64 if the type of obj is unknown, e.g. *map[string]interface{}, *interface{}, or *[]interface{}. This means integers above +/- 2^53 will lose precision when round-tripping. Make a JSONOpt that calls d.UseNumber() to avoid this. 50 // - Duplicate fields, including in-case-sensitive matches, are ignored in an undefined order. Note that the YAML specification forbids duplicate fields, so this logic is more permissive than it needs to. See UnmarshalStrict for an alternative. 51 // - Unknown fields, i.e. serialized data that do not map to a field in obj, are ignored. Use d.DisallowUnknownFields() or UnmarshalStrict to override. 52 // - As per the YAML 1.1 specification, which yaml.v2 used underneath implements, literal 'yes' and 'no' strings without quotation marks will be converted to true/false implicitly. 53 // - YAML non-string keys, e.g. ints, bools and floats, are converted to strings implicitly during the YAML to JSON conversion process. 54 // - There are no compatibility guarantees for returned error values. 55 func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { 56 return unmarshal(yamlBytes, obj, yaml.Unmarshal, opts...) 57 } 58 59 // UnmarshalStrict is similar to Unmarshal (please read its documentation for reference), with the following exceptions: 60 // 61 // - Duplicate fields in an object yield an error. This is according to the YAML specification. 62 // - If obj, or any of its recursive children, is a struct, presence of fields in the serialized data unknown to the struct will yield an error. 63 func UnmarshalStrict(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { 64 return unmarshal(yamlBytes, obj, yaml.UnmarshalStrict, append(opts, DisallowUnknownFields)...) 65 } 66 67 // unmarshal unmarshals the given YAML byte stream into the given interface, 68 // optionally performing the unmarshalling strictly 69 func unmarshal(yamlBytes []byte, obj interface{}, unmarshalFn func([]byte, interface{}) error, opts ...JSONOpt) error { 70 jsonTarget := reflect.ValueOf(obj) 71 72 jsonBytes, err := yamlToJSONTarget(yamlBytes, &jsonTarget, unmarshalFn) 73 if err != nil { 74 return fmt.Errorf("error converting YAML to JSON: %w", err) 75 } 76 77 err = jsonUnmarshal(bytes.NewReader(jsonBytes), obj, opts...) 78 if err != nil { 79 return fmt.Errorf("error unmarshaling JSON: %w", err) 80 } 81 82 return nil 83 } 84 85 // jsonUnmarshal unmarshals the JSON byte stream from the given reader into the 86 // object, optionally applying decoder options prior to decoding. We are not 87 // using json.Unmarshal directly as we want the chance to pass in non-default 88 // options. 89 func jsonUnmarshal(reader io.Reader, obj interface{}, opts ...JSONOpt) error { 90 d := json.NewDecoder(reader) 91 for _, opt := range opts { 92 d = opt(d) 93 } 94 if err := d.Decode(&obj); err != nil { 95 return fmt.Errorf("while decoding JSON: %v", err) 96 } 97 return nil 98 } 99 100 // JSONToYAML converts JSON to YAML. Notable implementation details: 101 // 102 // - Duplicate fields, are case-sensitively ignored in an undefined order. 103 // - The sequence indentation style is compact, which means that the "- " marker for a YAML sequence will be on the same indentation level as the sequence field name. 104 // - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. 105 func JSONToYAML(j []byte) ([]byte, error) { 106 // Convert the JSON to an object. 107 var jsonObj interface{} 108 109 // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the 110 // Go JSON library doesn't try to pick the right number type (int, float, 111 // etc.) when unmarshalling to interface{}, it just picks float64 112 // universally. go-yaml does go through the effort of picking the right 113 // number type, so we can preserve number type throughout this process. 114 err := yaml.Unmarshal(j, &jsonObj) 115 if err != nil { 116 return nil, err 117 } 118 119 // Marshal this object into YAML. 120 yamlBytes, err := yaml.Marshal(jsonObj) 121 if err != nil { 122 return nil, err 123 } 124 125 return yamlBytes, nil 126 } 127 128 // YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, 129 // passing JSON through this method should be a no-op. 130 // 131 // Some things YAML can do that are not supported by JSON: 132 // - In YAML you can have binary and null keys in your maps. These are invalid 133 // in JSON, and therefore int, bool and float keys are converted to strings implicitly. 134 // - Binary data in YAML with the !!binary tag is not supported. If you want to 135 // use binary data with this library, encode the data as base64 as usual but do 136 // not use the !!binary tag in your YAML. This will ensure the original base64 137 // encoded data makes it all the way through to the JSON. 138 // - And more... read the YAML specification for more details. 139 // 140 // Notable about the implementation: 141 // 142 // - Duplicate fields are case-sensitively ignored in an undefined order. Note that the YAML specification forbids duplicate fields, so this logic is more permissive than it needs to. See YAMLToJSONStrict for an alternative. 143 // - As per the YAML 1.1 specification, which yaml.v2 used underneath implements, literal 'yes' and 'no' strings without quotation marks will be converted to true/false implicitly. 144 // - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. 145 // - There are no compatibility guarantees for returned error values. 146 func YAMLToJSON(y []byte) ([]byte, error) { 147 return yamlToJSONTarget(y, nil, yaml.Unmarshal) 148 } 149 150 // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, 151 // returning an error on any duplicate field names. 152 func YAMLToJSONStrict(y []byte) ([]byte, error) { 153 return yamlToJSONTarget(y, nil, yaml.UnmarshalStrict) 154 } 155 156 func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, unmarshalFn func([]byte, interface{}) error) ([]byte, error) { 157 // Convert the YAML to an object. 158 var yamlObj interface{} 159 err := unmarshalFn(yamlBytes, &yamlObj) 160 if err != nil { 161 return nil, err 162 } 163 164 // YAML objects are not completely compatible with JSON objects (e.g. you 165 // can have non-string keys in YAML). So, convert the YAML-compatible object 166 // to a JSON-compatible object, failing with an error if irrecoverable 167 // incompatibilties happen along the way. 168 jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) 169 if err != nil { 170 return nil, err 171 } 172 173 // Convert this object to JSON and return the data. 174 jsonBytes, err := json.Marshal(jsonObj) 175 if err != nil { 176 return nil, err 177 } 178 return jsonBytes, nil 179 } 180 181 func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { 182 var err error 183 184 // Resolve jsonTarget to a concrete value (i.e. not a pointer or an 185 // interface). We pass decodingNull as false because we're not actually 186 // decoding into the value, we're just checking if the ultimate target is a 187 // string. 188 if jsonTarget != nil { 189 jsonUnmarshaler, textUnmarshaler, pointerValue := indirect(*jsonTarget, false) 190 // We have a JSON or Text Umarshaler at this level, so we can't be trying 191 // to decode into a string. 192 if jsonUnmarshaler != nil || textUnmarshaler != nil { 193 jsonTarget = nil 194 } else { 195 jsonTarget = &pointerValue 196 } 197 } 198 199 // If yamlObj is a number or a boolean, check if jsonTarget is a string - 200 // if so, coerce. Else return normal. 201 // If yamlObj is a map or array, find the field that each key is 202 // unmarshaling to, and when you recurse pass the reflect.Value for that 203 // field back into this function. 204 switch typedYAMLObj := yamlObj.(type) { 205 case map[interface{}]interface{}: 206 // JSON does not support arbitrary keys in a map, so we must convert 207 // these keys to strings. 208 // 209 // From my reading of go-yaml v2 (specifically the resolve function), 210 // keys can only have the types string, int, int64, float64, binary 211 // (unsupported), or null (unsupported). 212 strMap := make(map[string]interface{}) 213 for k, v := range typedYAMLObj { 214 // Resolve the key to a string first. 215 var keyString string 216 switch typedKey := k.(type) { 217 case string: 218 keyString = typedKey 219 case int: 220 keyString = strconv.Itoa(typedKey) 221 case int64: 222 // go-yaml will only return an int64 as a key if the system 223 // architecture is 32-bit and the key's value is between 32-bit 224 // and 64-bit. Otherwise the key type will simply be int. 225 keyString = strconv.FormatInt(typedKey, 10) 226 case float64: 227 // Stolen from go-yaml to use the same conversion to string as 228 // the go-yaml library uses to convert float to string when 229 // Marshaling. 230 s := strconv.FormatFloat(typedKey, 'g', -1, 32) 231 switch s { 232 case "+Inf": 233 s = ".inf" 234 case "-Inf": 235 s = "-.inf" 236 case "NaN": 237 s = ".nan" 238 } 239 keyString = s 240 case bool: 241 if typedKey { 242 keyString = "true" 243 } else { 244 keyString = "false" 245 } 246 default: 247 return nil, fmt.Errorf("unsupported map key of type: %s, key: %+#v, value: %+#v", 248 reflect.TypeOf(k), k, v) 249 } 250 251 // jsonTarget should be a struct or a map. If it's a struct, find 252 // the field it's going to map to and pass its reflect.Value. If 253 // it's a map, find the element type of the map and pass the 254 // reflect.Value created from that type. If it's neither, just pass 255 // nil - JSON conversion will error for us if it's a real issue. 256 if jsonTarget != nil { 257 t := *jsonTarget 258 if t.Kind() == reflect.Struct { 259 keyBytes := []byte(keyString) 260 // Find the field that the JSON library would use. 261 var f *field 262 fields := cachedTypeFields(t.Type()) 263 for i := range fields { 264 ff := &fields[i] 265 if bytes.Equal(ff.nameBytes, keyBytes) { 266 f = ff 267 break 268 } 269 // Do case-insensitive comparison. 270 if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { 271 f = ff 272 } 273 } 274 if f != nil { 275 // Find the reflect.Value of the most preferential 276 // struct field. 277 jtf := t.Field(f.index[0]) 278 strMap[keyString], err = convertToJSONableObject(v, &jtf) 279 if err != nil { 280 return nil, err 281 } 282 continue 283 } 284 } else if t.Kind() == reflect.Map { 285 // Create a zero value of the map's element type to use as 286 // the JSON target. 287 jtv := reflect.Zero(t.Type().Elem()) 288 strMap[keyString], err = convertToJSONableObject(v, &jtv) 289 if err != nil { 290 return nil, err 291 } 292 continue 293 } 294 } 295 strMap[keyString], err = convertToJSONableObject(v, nil) 296 if err != nil { 297 return nil, err 298 } 299 } 300 return strMap, nil 301 case []interface{}: 302 // We need to recurse into arrays in case there are any 303 // map[interface{}]interface{}'s inside and to convert any 304 // numbers to strings. 305 306 // If jsonTarget is a slice (which it really should be), find the 307 // thing it's going to map to. If it's not a slice, just pass nil 308 // - JSON conversion will error for us if it's a real issue. 309 var jsonSliceElemValue *reflect.Value 310 if jsonTarget != nil { 311 t := *jsonTarget 312 if t.Kind() == reflect.Slice { 313 // By default slices point to nil, but we need a reflect.Value 314 // pointing to a value of the slice type, so we create one here. 315 ev := reflect.Indirect(reflect.New(t.Type().Elem())) 316 jsonSliceElemValue = &ev 317 } 318 } 319 320 // Make and use a new array. 321 arr := make([]interface{}, len(typedYAMLObj)) 322 for i, v := range typedYAMLObj { 323 arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) 324 if err != nil { 325 return nil, err 326 } 327 } 328 return arr, nil 329 default: 330 // If the target type is a string and the YAML type is a number, 331 // convert the YAML type to a string. 332 if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { 333 // Based on my reading of go-yaml, it may return int, int64, 334 // float64, or uint64. 335 var s string 336 switch typedVal := typedYAMLObj.(type) { 337 case int: 338 s = strconv.FormatInt(int64(typedVal), 10) 339 case int64: 340 s = strconv.FormatInt(typedVal, 10) 341 case float64: 342 s = strconv.FormatFloat(typedVal, 'g', -1, 32) 343 case uint64: 344 s = strconv.FormatUint(typedVal, 10) 345 case bool: 346 if typedVal { 347 s = "true" 348 } else { 349 s = "false" 350 } 351 } 352 if len(s) > 0 { 353 yamlObj = interface{}(s) 354 } 355 } 356 return yamlObj, nil 357 } 358 } 359 360 // JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice, 361 // without going through a byte representation. A nil or empty map[string]interface{} input is 362 // converted to an empty map, i.e. yaml.MapSlice(nil). 363 // 364 // interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice. 365 // 366 // int64 and float64 are down casted following the logic of github.com/go-yaml/yaml: 367 // - float64s are down-casted as far as possible without data-loss to int, int64, uint64. 368 // - int64s are down-casted to int if possible without data-loss. 369 // 370 // Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. 371 // 372 // string, bool and any other types are unchanged. 373 func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { 374 if len(j) == 0 { 375 return nil 376 } 377 ret := make(yaml.MapSlice, 0, len(j)) 378 for k, v := range j { 379 ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) 380 } 381 return ret 382 } 383 384 func jsonToYAMLValue(j interface{}) interface{} { 385 switch j := j.(type) { 386 case map[string]interface{}: 387 if j == nil { 388 return interface{}(nil) 389 } 390 return JSONObjectToYAMLObject(j) 391 case []interface{}: 392 if j == nil { 393 return interface{}(nil) 394 } 395 ret := make([]interface{}, len(j)) 396 for i := range j { 397 ret[i] = jsonToYAMLValue(j[i]) 398 } 399 return ret 400 case float64: 401 // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151 402 if i64 := int64(j); j == float64(i64) { 403 if i := int(i64); i64 == int64(i) { 404 return i 405 } 406 return i64 407 } 408 if ui64 := uint64(j); j == float64(ui64) { 409 return ui64 410 } 411 return j 412 case int64: 413 if i := int(j); j == int64(i) { 414 return i 415 } 416 return j 417 } 418 return j 419 } 420