reader.go

Documentation: github.com/google/gnostic-models/compiler

     1  // Copyright 2017 Google LLC. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package compiler
    16  
    17  import (
    18  	"fmt"
    19  	"io/ioutil"
    20  	"log"
    21  	"net/http"
    22  	"net/url"
    23  	"path/filepath"
    24  	"strings"
    25  	"sync"
    26  
    27  	yaml "gopkg.in/yaml.v3"
    28  )
    29  
    30  var verboseReader = false
    31  
    32  var fileCache map[string][]byte
    33  var infoCache map[string]*yaml.Node
    34  
    35  var fileCacheEnable = true
    36  var infoCacheEnable = true
    37  
    38  // These locks are used to synchronize accesses to the fileCache and infoCache
    39  // maps (above). They are global state and can throw thread-related errors
    40  // when modified from separate goroutines. The general strategy is to protect
    41  // all public functions in this file with mutex Lock() calls. As a result, to
    42  // avoid deadlock, these public functions should not call other public
    43  // functions, so some public functions have private equivalents.
    44  // In the future, we might consider replacing the maps with sync.Map and
    45  // eliminating these mutexes.
    46  var fileCacheMutex sync.Mutex
    47  var infoCacheMutex sync.Mutex
    48  
    49  func initializeFileCache() {
    50  	if fileCache == nil {
    51  		fileCache = make(map[string][]byte, 0)
    52  	}
    53  }
    54  
    55  func initializeInfoCache() {
    56  	if infoCache == nil {
    57  		infoCache = make(map[string]*yaml.Node, 0)
    58  	}
    59  }
    60  
    61  // EnableFileCache turns on file caching.
    62  func EnableFileCache() {
    63  	fileCacheMutex.Lock()
    64  	defer fileCacheMutex.Unlock()
    65  	fileCacheEnable = true
    66  }
    67  
    68  // EnableInfoCache turns on parsed info caching.
    69  func EnableInfoCache() {
    70  	infoCacheMutex.Lock()
    71  	defer infoCacheMutex.Unlock()
    72  	infoCacheEnable = true
    73  }
    74  
    75  // DisableFileCache turns off file caching.
    76  func DisableFileCache() {
    77  	fileCacheMutex.Lock()
    78  	defer fileCacheMutex.Unlock()
    79  	fileCacheEnable = false
    80  }
    81  
    82  // DisableInfoCache turns off parsed info caching.
    83  func DisableInfoCache() {
    84  	infoCacheMutex.Lock()
    85  	defer infoCacheMutex.Unlock()
    86  	infoCacheEnable = false
    87  }
    88  
    89  // RemoveFromFileCache removes an entry from the file cache.
    90  func RemoveFromFileCache(fileurl string) {
    91  	fileCacheMutex.Lock()
    92  	defer fileCacheMutex.Unlock()
    93  	if !fileCacheEnable {
    94  		return
    95  	}
    96  	initializeFileCache()
    97  	delete(fileCache, fileurl)
    98  }
    99  
   100  // RemoveFromInfoCache removes an entry from the info cache.
   101  func RemoveFromInfoCache(filename string) {
   102  	infoCacheMutex.Lock()
   103  	defer infoCacheMutex.Unlock()
   104  	if !infoCacheEnable {
   105  		return
   106  	}
   107  	initializeInfoCache()
   108  	delete(infoCache, filename)
   109  }
   110  
   111  // GetInfoCache returns the info cache map.
   112  func GetInfoCache() map[string]*yaml.Node {
   113  	infoCacheMutex.Lock()
   114  	defer infoCacheMutex.Unlock()
   115  	if infoCache == nil {
   116  		initializeInfoCache()
   117  	}
   118  	return infoCache
   119  }
   120  
   121  // ClearFileCache clears the file cache.
   122  func ClearFileCache() {
   123  	fileCacheMutex.Lock()
   124  	defer fileCacheMutex.Unlock()
   125  	fileCache = make(map[string][]byte, 0)
   126  }
   127  
   128  // ClearInfoCache clears the info cache.
   129  func ClearInfoCache() {
   130  	infoCacheMutex.Lock()
   131  	defer infoCacheMutex.Unlock()
   132  	infoCache = make(map[string]*yaml.Node)
   133  }
   134  
   135  // ClearCaches clears all caches.
   136  func ClearCaches() {
   137  	ClearFileCache()
   138  	ClearInfoCache()
   139  }
   140  
   141  // FetchFile gets a specified file from the local filesystem or a remote location.
   142  func FetchFile(fileurl string) ([]byte, error) {
   143  	fileCacheMutex.Lock()
   144  	defer fileCacheMutex.Unlock()
   145  	return fetchFile(fileurl)
   146  }
   147  
   148  func fetchFile(fileurl string) ([]byte, error) {
   149  	var bytes []byte
   150  	initializeFileCache()
   151  	if fileCacheEnable {
   152  		bytes, ok := fileCache[fileurl]
   153  		if ok {
   154  			if verboseReader {
   155  				log.Printf("Cache hit %s", fileurl)
   156  			}
   157  			return bytes, nil
   158  		}
   159  		if verboseReader {
   160  			log.Printf("Fetching %s", fileurl)
   161  		}
   162  	}
   163  	response, err := http.Get(fileurl)
   164  	if err != nil {
   165  		return nil, err
   166  	}
   167  	defer response.Body.Close()
   168  	if response.StatusCode != 200 {
   169  		return nil, fmt.Errorf("Error downloading %s: %s", fileurl, response.Status)
   170  	}
   171  	bytes, err = ioutil.ReadAll(response.Body)
   172  	if fileCacheEnable && err == nil {
   173  		fileCache[fileurl] = bytes
   174  	}
   175  	return bytes, err
   176  }
   177  
   178  // ReadBytesForFile reads the bytes of a file.
   179  func ReadBytesForFile(filename string) ([]byte, error) {
   180  	fileCacheMutex.Lock()
   181  	defer fileCacheMutex.Unlock()
   182  	return readBytesForFile(filename)
   183  }
   184  
   185  func readBytesForFile(filename string) ([]byte, error) {
   186  	// is the filename a url?
   187  	fileurl, _ := url.Parse(filename)
   188  	if fileurl.Scheme != "" {
   189  		// yes, fetch it
   190  		bytes, err := fetchFile(filename)
   191  		if err != nil {
   192  			return nil, err
   193  		}
   194  		return bytes, nil
   195  	}
   196  	// no, it's a local filename
   197  	bytes, err := ioutil.ReadFile(filename)
   198  	if err != nil {
   199  		return nil, err
   200  	}
   201  	return bytes, nil
   202  }
   203  
   204  // ReadInfoFromBytes unmarshals a file as a *yaml.Node.
   205  func ReadInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
   206  	infoCacheMutex.Lock()
   207  	defer infoCacheMutex.Unlock()
   208  	return readInfoFromBytes(filename, bytes)
   209  }
   210  
   211  func readInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
   212  	initializeInfoCache()
   213  	if infoCacheEnable {
   214  		cachedInfo, ok := infoCache[filename]
   215  		if ok {
   216  			if verboseReader {
   217  				log.Printf("Cache hit info for file %s", filename)
   218  			}
   219  			return cachedInfo, nil
   220  		}
   221  		if verboseReader {
   222  			log.Printf("Reading info for file %s", filename)
   223  		}
   224  	}
   225  	var info yaml.Node
   226  	err := yaml.Unmarshal(bytes, &info)
   227  	if err != nil {
   228  		return nil, err
   229  	}
   230  	if infoCacheEnable && len(filename) > 0 {
   231  		infoCache[filename] = &info
   232  	}
   233  	return &info, nil
   234  }
   235  
   236  // ReadInfoForRef reads a file and return the fragment needed to resolve a $ref.
   237  func ReadInfoForRef(basefile string, ref string) (*yaml.Node, error) {
   238  	fileCacheMutex.Lock()
   239  	defer fileCacheMutex.Unlock()
   240  	infoCacheMutex.Lock()
   241  	defer infoCacheMutex.Unlock()
   242  	initializeInfoCache()
   243  	if infoCacheEnable {
   244  		info, ok := infoCache[ref]
   245  		if ok {
   246  			if verboseReader {
   247  				log.Printf("Cache hit for ref %s#%s", basefile, ref)
   248  			}
   249  			return info, nil
   250  		}
   251  		if verboseReader {
   252  			log.Printf("Reading info for ref %s#%s", basefile, ref)
   253  		}
   254  	}
   255  	basedir, _ := filepath.Split(basefile)
   256  	parts := strings.Split(ref, "#")
   257  	var filename string
   258  	if parts[0] != "" {
   259  		filename = parts[0]
   260  		if _, err := url.ParseRequestURI(parts[0]); err != nil {
   261  			// It is not an URL, so the file is local
   262  			filename = basedir + parts[0]
   263  		}
   264  	} else {
   265  		filename = basefile
   266  	}
   267  	bytes, err := readBytesForFile(filename)
   268  	if err != nil {
   269  		return nil, err
   270  	}
   271  	info, err := readInfoFromBytes(filename, bytes)
   272  	if info != nil && info.Kind == yaml.DocumentNode {
   273  		info = info.Content[0]
   274  	}
   275  	if err != nil {
   276  		log.Printf("File error: %v\n", err)
   277  	} else {
   278  		if info == nil {
   279  			return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
   280  		}
   281  		if len(parts) > 1 {
   282  			path := strings.Split(parts[1], "/")
   283  			for i, key := range path {
   284  				if i > 0 {
   285  					m := info
   286  					if true {
   287  						found := false
   288  						for i := 0; i < len(m.Content); i += 2 {
   289  							if m.Content[i].Value == key {
   290  								info = m.Content[i+1]
   291  								found = true
   292  							}
   293  						}
   294  						if !found {
   295  							infoCache[ref] = nil
   296  							return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
   297  						}
   298  					}
   299  				}
   300  			}
   301  		}
   302  	}
   303  	if infoCacheEnable {
   304  		infoCache[ref] = info
   305  	}
   306  	return info, nil
   307  }
   308
View as plain text