compiler/reader.go

// Copyright 2017 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"sync"

	yaml "gopkg.in/yaml.v3"
)

var verboseReader = false

var fileCache map[string][]byte
var infoCache map[string]*yaml.Node

var fileCacheEnable = true
var infoCacheEnable = true

// These locks are used to synchronize accesses to the fileCache and infoCache
// maps (above). They are global state and can throw thread-related errors
// when modified from separate goroutines. The general strategy is to protect
// all public functions in this file with mutex Lock() calls. As a result, to
// avoid deadlock, these public functions should not call other public
// functions, so some public functions have private equivalents.
// In the future, we might consider replacing the maps with sync.Map and
// eliminating these mutexes.
var fileCacheMutex sync.Mutex
var infoCacheMutex sync.Mutex

func initializeFileCache() {
	if fileCache == nil {
		fileCache = make(map[string][]byte, 0)
	}
}

func initializeInfoCache() {
	if infoCache == nil {
		infoCache = make(map[string]*yaml.Node, 0)
	}
}

// EnableFileCache turns on file caching.
func EnableFileCache() {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	fileCacheEnable = true
}

// EnableInfoCache turns on parsed info caching.
func EnableInfoCache() {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	infoCacheEnable = true
}

// DisableFileCache turns off file caching.
func DisableFileCache() {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	fileCacheEnable = false
}

// DisableInfoCache turns off parsed info caching.
func DisableInfoCache() {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	infoCacheEnable = false
}

// RemoveFromFileCache removes an entry from the file cache.
func RemoveFromFileCache(fileurl string) {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	if !fileCacheEnable {
		return
	}
	initializeFileCache()
	delete(fileCache, fileurl)
}

// RemoveFromInfoCache removes an entry from the info cache.
func RemoveFromInfoCache(filename string) {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	if !infoCacheEnable {
		return
	}
	initializeInfoCache()
	delete(infoCache, filename)
}

// GetInfoCache returns the info cache map.
func GetInfoCache() map[string]*yaml.Node {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	if infoCache == nil {
		initializeInfoCache()
	}
	return infoCache
}

// ClearFileCache clears the file cache.
func ClearFileCache() {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	fileCache = make(map[string][]byte, 0)
}

// ClearInfoCache clears the info cache.
func ClearInfoCache() {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	infoCache = make(map[string]*yaml.Node)
}

// ClearCaches clears all caches.
func ClearCaches() {
	ClearFileCache()
	ClearInfoCache()
}

// FetchFile gets a specified file from the local filesystem or a remote location.
func FetchFile(fileurl string) ([]byte, error) {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	return fetchFile(fileurl)
}

func fetchFile(fileurl string) ([]byte, error) {
	var bytes []byte
	initializeFileCache()
	if fileCacheEnable {
		bytes, ok := fileCache[fileurl]
		if ok {
			if verboseReader {
				log.Printf("Cache hit %s", fileurl)
			}
			return bytes, nil
		}
		if verboseReader {
			log.Printf("Fetching %s", fileurl)
		}
	}
	response, err := http.Get(fileurl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	if response.StatusCode != 200 {
		return nil, fmt.Errorf("Error downloading %s: %s", fileurl, response.Status)
	}
	bytes, err = ioutil.ReadAll(response.Body)
	if fileCacheEnable && err == nil {
		fileCache[fileurl] = bytes
	}
	return bytes, err
}

// ReadBytesForFile reads the bytes of a file.
func ReadBytesForFile(filename string) ([]byte, error) {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	return readBytesForFile(filename)
}

func readBytesForFile(filename string) ([]byte, error) {
	// is the filename a url?
	fileurl, _ := url.Parse(filename)
	if fileurl.Scheme != "" {
		// yes, fetch it
		bytes, err := fetchFile(filename)
		if err != nil {
			return nil, err
		}
		return bytes, nil
	}
	// no, it's a local filename
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return bytes, nil
}

// ReadInfoFromBytes unmarshals a file as a *yaml.Node.
func ReadInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	return readInfoFromBytes(filename, bytes)
}

func readInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
	initializeInfoCache()
	if infoCacheEnable {
		cachedInfo, ok := infoCache[filename]
		if ok {
			if verboseReader {
				log.Printf("Cache hit info for file %s", filename)
			}
			return cachedInfo, nil
		}
		if verboseReader {
			log.Printf("Reading info for file %s", filename)
		}
	}
	var info yaml.Node
	err := yaml.Unmarshal(bytes, &info)
	if err != nil {
		return nil, err
	}
	if infoCacheEnable && len(filename) > 0 {
		infoCache[filename] = &info
	}
	return &info, nil
}

// ReadInfoForRef reads a file and return the fragment needed to resolve a $ref.
func ReadInfoForRef(basefile string, ref string) (*yaml.Node, error) {
	fileCacheMutex.Lock()
	defer fileCacheMutex.Unlock()
	infoCacheMutex.Lock()
	defer infoCacheMutex.Unlock()
	initializeInfoCache()
	if infoCacheEnable {
		info, ok := infoCache[ref]
		if ok {
			if verboseReader {
				log.Printf("Cache hit for ref %s#%s", basefile, ref)
			}
			return info, nil
		}
		if verboseReader {
			log.Printf("Reading info for ref %s#%s", basefile, ref)
		}
	}
	basedir, _ := filepath.Split(basefile)
	parts := strings.Split(ref, "#")
	var filename string
	if parts[0] != "" {
		filename = parts[0]
		if _, err := url.ParseRequestURI(parts[0]); err != nil {
			// It is not an URL, so the file is local
			filename = basedir + parts[0]
		}
	} else {
		filename = basefile
	}
	bytes, err := readBytesForFile(filename)
	if err != nil {
		return nil, err
	}
	info, err := readInfoFromBytes(filename, bytes)
	if info != nil && info.Kind == yaml.DocumentNode {
		info = info.Content[0]
	}
	if err != nil {
		log.Printf("File error: %v\n", err)
	} else {
		if info == nil {
			return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
		}
		if len(parts) > 1 {
			path := strings.Split(parts[1], "/")
			for i, key := range path {
				if i > 0 {
					m := info
					if true {
						found := false
						for i := 0; i < len(m.Content); i += 2 {
							if m.Content[i].Value == key {
								info = m.Content[i+1]
								found = true
							}
						}
						if !found {
							infoCache[ref] = nil
							return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
						}
					}
				}
			}
		}
	}
	if infoCacheEnable {
		infoCache[ref] = info
	}
	return info, nil
}