<|start_filename|>filedata_test.go<|end_filename|>
package conflate

import (
	"errors"
	"github.com/stretchr/testify/assert"
	pkgurl "net/url"
	"os"
	"testing"
)

func testFiledataNew(t *testing.T, data []byte, path string) (filedata, error) {
	url, err := pkgurl.Parse(path)
	assert.Nil(t, err)
	return newFiledata(data, *url)
}

func testFiledataNewAssert(t *testing.T, data []byte, path string) filedata {
	fd, err := testFiledataNew(t, data, path)
	assert.Nil(t, err)
	return fd
}

func TestFiledata_WrapErrorNil(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "myurl")
	assert.Nil(t, err)
	err = fd.wrapError(nil)
	assert.Nil(t, err)
}

func TestFiledata_WrapError(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "myurl")
	assert.Nil(t, err)
	err = errors.New("My Error")
	err = fd.wrapError(err)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "My Error")
	assert.Contains(t, err.Error(), "Error processing myurl")
}

func TestFiledata_WrapErrorBlank(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "")
	assert.Nil(t, err)
	err1 := errors.New("My Error")
	err2 := fd.wrapError(err1)
	assert.NotNil(t, err2)
	assert.Equal(t, err1, err2)
}

func TestFiledata_JSONAsAny(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "file")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_JSONAsUnknown(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "file.unknown")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_JSONAsJSON(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "file.json")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_JSONAsJSN(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "file.jsn")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_JSONAsTOML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalJSON, "file.toml")
	assert.NotNil(t, err)
	assert.Nil(t, fd.obj)
}

func TestFiledata_YAMLAsAny(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalYAML, "file")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_YAMLAsUnknown(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalYAML, "file.unknown")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_YAMLAsYAML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalYAML, "file.yaml")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_YAMLAsYML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalYAML, "file.yml")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_YAMLAsTOML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalYAML, "file.toml")
	assert.NotNil(t, err)
	assert.Nil(t, fd.obj)
}

func TestFiledata_TOMLAsAny(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalTOML, "file")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_TOMLAsUnknown(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalTOML, "file.unknown")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_TOMLAsTOML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalTOML, "file.toml")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_TOMLAsTML(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalTOML, "file.tml")
	assert.Nil(t, err)
	assert.Equal(t, fd.obj, testMarshalData)
}

func TestFiledata_TOMLAsJSON(t *testing.T) {
	fd, err := testFiledataNew(t, testMarshalTOML, "file.json")
	assert.NotNil(t, err)
	assert.Nil(t, fd.obj)
}

func TestFiledata_NoIncludes(t *testing.T) {
	fd, err := testLoader.wrapFiledata([]byte(`{"x": 1}`))
	assert.Nil(t, err)
	assert.Nil(t, fd.obj[Includes])
	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
}

func TestFiledata_BlankIncludes(t *testing.T) {
	fd, err := testLoader.wrapFiledata([]byte(`{"includes":[], "x": 1}`))
	assert.Nil(t, err)
	assert.Nil(t, fd.obj[Includes])
	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
}

func TestFiledata_NullIncludes(t *testing.T) {
	_, err := testLoader.wrapFiledata([]byte(`{"includes":null}`))
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "not valid against the schema")
}

func TestFiledata_Includes(t *testing.T) {
	fd, err := testLoader.wrapFiledata([]byte(`{"includes":["test1", "test2"], "x": 1}`))
	assert.Nil(t, err)
	assert.Equal(t, fd.includes, []string{"test1", "test2"})
	assert.Nil(t, fd.obj[Includes])
	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
}

func TestFiledata_ExtractError(t *testing.T) {
	old := getSchema
	getSchema = func() map[string]interface{} { return map[string]interface{}{} }
	defer func() { getSchema = old }()
	_, err := testLoader.wrapFiledata([]byte(`{"includes": "not array"}`))
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "Could not extract includes")
}

func TestFiledata_IncludesError(t *testing.T) {
	_, err := testLoader.wrapFiledata([]byte(`{"includes": "not array"}`))
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "not valid against the schema")
}

func TestFiledata_Expand(t *testing.T) {
	w := os.Getenv("W")
	x := os.Getenv("X")
	y := os.Getenv("Y")
	z := os.Getenv("Z")
	os.Setenv("W", "$W")
	os.Setenv("X", `"x"`)
	os.Setenv("Y", `y`)
	os.Setenv("Z", `$Y`)
	defer func() {
		os.Setenv("W", w)
		os.Setenv("X", x)
		os.Setenv("Y", y)
		os.Setenv("Z", z)
	}()
	b := recursiveExpand([]byte(`{"W":"$W","X":$X,"Y":"$Y","Z":"$Z"}`))
	assert.Equal(t, string(b), string(`{"W":"$W","X":"x","Y":"y","Z":"y"}`))
}

func TestFiledatas_Unmarshal(t *testing.T) {
	fds := filedatas{
		testFiledataNewAssert(t, testMarshalJSON, "file.json"),
		testFiledataNewAssert(t, testMarshalYAML, "file.yaml"),
		testFiledataNewAssert(t, testMarshalTOML, "file.toml"),
	}
	assert.Equal(t, fds.objs(), []interface{}{testMarshalData, testMarshalData, testMarshalData})
}

func TestFiledatas_DifferentIncludes(t *testing.T) {
	old := Includes
	Includes = "using"
	defer func() { Includes = old }()
	fd, err := testLoader.wrapFiledata([]byte(`{"using":["test1", "test2"], "x": 1}`))
	assert.Nil(t, err)
	assert.Equal(t, fd.includes, []string{"test1", "test2"})
	assert.Nil(t, fd.obj[Includes])
	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
}

func TestFiledatas_NoIncludes(t *testing.T) {
	old := Includes
	Includes = "using"
	defer func() { Includes = old }()
	fd, err := testLoader.wrapFiledata([]byte(`{"includes":["test1", "test2"]}`))
	assert.Nil(t, err)
	assert.Empty(t, fd.includes)
	assert.Equal(t, fd.obj, map[string]interface{}{"includes": []interface{}{"test1", "test2"}})
}

func TestFiledatas_IgnoreIncludes(t *testing.T) {
	old := Includes
	Includes = ""
	defer func() { Includes = old }()
	fd, err := testLoader.wrapFiledata([]byte(`{"":["test1", "test2"]}`))
	assert.Nil(t, err)
	assert.Empty(t, fd.includes)
	assert.Equal(t, fd.obj, map[string]interface{}{"": []interface{}{"test1", "test2"}})
}


<|start_filename|>marshal.go<|end_filename|>
package conflate

import (
	"bytes"
	"encoding/json"
	"github.com/BurntSushi/toml"
	"github.com/ghodss/yaml"
)

func jsonMarshalAll(data ...interface{}) ([][]byte, error) {
	var outs [][]byte
	for i, datum := range data {
		out, err := jsonMarshal(datum)
		if err != nil {
			return nil, wrapError(err, "Could not marshal data %v", i)
		}
		outs = append(outs, out)
	}
	return outs, nil
}

func jsonMarshalUnmarshal(in interface{}, out interface{}) error {
	data, err := jsonMarshal(in)
	if err != nil {
		return err
	}
	return JSONUnmarshal(data, out)
}

// JSONUnmarshal unmarshals the data as JSON
func JSONUnmarshal(data []byte, out interface{}) error {
	err := json.Unmarshal(data, out)
	if err != nil {
		return wrapError(err, "The data could not be unmarshalled as json")
	}
	return nil
}

// YAMLUnmarshal unmarshals the data as YAML
func YAMLUnmarshal(data []byte, out interface{}) error {
	err := yaml.Unmarshal(data, out)
	if err != nil {
		return wrapError(err, "The data could not be unmarshalled as yaml")
	}
	return nil
}

// TOMLUnmarshal unmarshals the data as TOML
func TOMLUnmarshal(data []byte, out interface{}) error {
	err := toml.Unmarshal(data, out)
	if err != nil {
		return wrapError(err, "The data could not be unmarshalled as toml")
	}
	return nil
}

func jsonMarshal(data interface{}) ([]byte, error) {
	buffer := bytes.Buffer{}
	encoder := json.NewEncoder(&buffer)
	encoder.SetEscapeHTML(false)
	encoder.SetIndent("", "  ")
	err := encoder.Encode(data)
	if err != nil {
		return nil, wrapError(err, "The data could not be marshalled to json")
	}
	return buffer.Bytes(), nil
}

func yamlMarshal(in interface{}) ([]byte, error) {
	data, err := yaml.Marshal(in)
	if err != nil {
		return nil, wrapError(err, "The data could not be marshalled to yaml")
	}
	return data, nil
}

func tomlMarshal(in interface{}) (out []byte, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = wrapError(makeError("%v", r), "The data could not be marshalled to toml")
		}
	}()
	buf := bytes.Buffer{}
	enc := toml.NewEncoder(&buf)
	err = enc.Encode(in)
	if err != nil {
		return nil, wrapError(err, "The data could not be marshalled to toml")
	}
	out = buf.Bytes()
	return out, nil
}


<|start_filename|>context.go<|end_filename|>
package conflate

import (
	"fmt"
	"path"
)

func rootContext() context {
	return context("#")
}

func (c context) String() string {
	return string(c)
}

func (c context) add(s ...string) context {
	return context(path.Join(c.String(), path.Join(s...)))
}

func (c context) addInt(i int) context {
	return context(fmt.Sprintf("%v[%v]", c.String(), i))
}


<|start_filename|>error.go<|end_filename|>
package conflate

import (
	"fmt"
)

type context string

func makeError(msg string, args ...interface{}) error {
	return fmt.Errorf(msg, args...)
}

func makeContextError(ctx context, msg string, args ...interface{}) error {
	return makeError("%v (%v)", makeError(msg, args...), ctx)
}

func wrapError(err error, msg string, args ...interface{}) error {
	if err == nil {
		return nil
	}
	return makeError("%v : %v", makeError(msg, args...), err)
}

func detailError(err error, msg string, args ...interface{}) error {
	return makeError("%v. %v", err, makeError(msg, args...))
}


<|start_filename|>schema.go<|end_filename|>
package conflate

import (
	"github.com/xeipuuv/gojsonreference"
	"github.com/xeipuuv/gojsonschema"
	"net/url"
	"reflect"
	"strings"
)

// Schema contains a JSON v4 schema
type Schema struct {
	s interface{}
}

// NewSchemaFile loads a JSON v4 schema from the given path
func NewSchemaFile(path string) (*Schema, error) {
	url, err := toURL(nil, path)
	if err != nil {
		return nil, wrapError(err, "Failed to obtain url to schema file")
	}
	return NewSchemaURL(url)
}

// NewSchemaURL loads a JSON v4 schema from the given URL
func NewSchemaURL(url url.URL) (*Schema, error) {
	data, err := loadURL(url)
	if err != nil {
		return nil, wrapError(err, "Failed to load schema url %v", url)
	}
	return NewSchemaData(data)
}

// NewSchemaData loads a JSON v4 schema from the given data
func NewSchemaData(data []byte) (*Schema, error) {
	var s interface{}
	err := JSONUnmarshal(data, &s)
	if err != nil {
		return nil, wrapError(err, "Schema is not valid json")
	}
	return NewSchemaGo(s)
}

// NewSchemaGo creates a Schema instance from a schema represented as a golang object
func NewSchemaGo(s interface{}) (*Schema, error) {
	err := validateSchema(s)
	if err != nil {
		return nil, wrapError(err, "The schema is not valid against the meta-schema http://json-schema.org/draft-04/schema")
	}
	return &Schema{s: s}, nil
}

// Validate checks the given golang data against the schema
func (s *Schema) Validate(data interface{}) error {
	if s == nil {
		return makeError("Schema is not set")
	}
	return validate(data, s.s)
}

// ApplyDefaults adds default values defined in the schema to the data pointed to by pData
func (s *Schema) ApplyDefaults(pData interface{}) error {
	if s == nil {
		return makeError("Schema is not set")
	}
	return applyDefaults(pData, s.s)
}

var metaSchema interface{}

func validateSchema(s interface{}) error {
	if metaSchema == nil {
		err := JSONUnmarshal(metaSchemaData, &metaSchema)
		if err != nil {
			return wrapError(err, "Could not load json meta-schema")
		}
	}
	return validate(s, metaSchema)
}

func validate(data interface{}, schema interface{}) error {
	dataLoader := gojsonschema.NewGoLoader(data)
	schemaLoader := gojsonschema.NewGoLoader(schema)
	formatErrs.clear()
	result, err := gojsonschema.Validate(schemaLoader, dataLoader)
	if err != nil {
		return wrapError(err, "An error occurred during validation")
	}
	err = processResult(result)
	return wrapError(err, "Schema validation failed")
}

func processResult(result *gojsonschema.Result) error {
	if !result.Valid() {
		err := makeError("The document is not valid against the schema")
		for _, rerr := range result.Errors() {
			ctx := convertJSONContext(rerr.Context().String())
			ctxErr := makeContextError(ctx, rerr.Description())

			ferr := formatErrs.get(rerr.Details()["format"], rerr.Value())
			if ferr != nil {
				ctxErr = detailError(ctxErr, ferr.Error())
			}
			err = wrapError(ctxErr, err.Error())
		}
		return err
	}
	return nil
}

func convertJSONContext(jsonCtx string) context {
	parts := strings.Split(jsonCtx, ".")
	return rootContext().add(parts[1:]...)
}

func applyDefaults(pData interface{}, schema interface{}) error {
	err := applyDefaultsRecursive(rootContext(), schema, pData, schema)
	return wrapError(err, "The defaults could not be applied")
}

func applyDefaultsRecursive(ctx context, rootSchema interface{}, pData interface{}, schema interface{}) error {
	if pData == nil {
		return makeContextError(ctx, "Destination value must not be nil")
	}
	pDataVal := reflect.ValueOf(pData)
	if pDataVal.Kind() != reflect.Ptr {
		return makeContextError(ctx, "Destination value must be a pointer")
	}
	dataVal := pDataVal.Elem()
	data := dataVal.Interface()

	schemaNode, ok := schema.(map[string]interface{})
	if !ok {
		return makeContextError(ctx, "Schema section is not a map")
	}

	val, ok := schemaNode["$ref"]
	if ok {
		ref, ok := val.(string)
		if !ok {
			return makeContextError(ctx, makeError("Reference is not a string '%v'", ref).Error())
		}
		jref, err := gojsonreference.NewJsonReference(ref)
		if err != nil {
			return makeContextError(ctx, wrapError(err, "Invalid reference '%v'", ref).Error())
		}
		subSchema, _, err := jref.GetPointer().Get(rootSchema)
		if subSchema == nil || err != nil {
			return makeContextError(ctx, wrapError(err, "Cannot find reference '%v'", ref).Error())
		}
		return applyDefaultsRecursive(ctx.add(ref), rootSchema, pData, subSchema)
	}

	schemaType, ok := schemaNode["type"]
	if !ok {
		if hasKey(schemaNode, "anyOf", "allOf", "oneOf", "not") {
			// the schema is valid, so it is not an error, but we do not support these types of schema yet
			return nil
		}
		return makeContextError(ctx, "Schema section does not have a valid 'type' attribute")
	}

	if value, ok := schemaNode["default"]; ok && data == nil {
		defaultVal := reflect.ValueOf(value)
		dataVal.Set(defaultVal)
		data = dataVal.Interface()
	}

	var err error
	switch schemaType {
	case "object":
		err = applyObjectDefaults(ctx, rootSchema, data, schemaNode)
	case "array":
		err = applyArrayDefaults(ctx, rootSchema, data, schemaNode)
	}
	return err
}

func hasKey(m map[string]interface{}, keys ...string) bool {
	for _, key := range keys {
		if _, ok := m[key]; ok {
			return true
		}
	}
	return false
}

func applyObjectDefaults(ctx context, rootSchema interface{}, data interface{}, schemaNode map[string]interface{}) error {
	if data == nil {
		return nil
	}
	dataProps, ok := data.(map[string]interface{})
	if !ok {
		return makeContextError(ctx, "Node should be an 'object'")
	}
	if dataProps == nil {
		return nil
	}
	var schemaProps map[string]interface{}
	if props, ok := schemaNode["properties"]; ok {
		schemaProps = props.(map[string]interface{})
		for name, schemaProp := range schemaProps {
			dataProp := dataProps[name]
			err := applyDefaultsRecursive(ctx.add(name), rootSchema, &dataProp, schemaProp)
			if err != nil {
				return wrapError(err, "Failed to apply defaults to object property")
			}
			if dataProp != nil {
				dataProps[name] = dataProp
			}
		}
	}
	if addProps, ok := schemaNode["additionalProperties"]; ok {
		if addProps, ok = addProps.(map[string]interface{}); ok {
			for name, dataProp := range dataProps {
				if schemaProps == nil || schemaProps[name] == nil {
					err := applyDefaultsRecursive(ctx.add(name), rootSchema, &dataProp, addProps)
					if err != nil {
						return wrapError(err, "Failed to apply defaults to additional object property")
					}
					if dataProp != nil {
						dataProps[name] = dataProp
					}
				}
			}
		}
	}
	return nil
}

func applyArrayDefaults(ctx context, rootSchema interface{}, data interface{}, schemaNode map[string]interface{}) error {
	if data == nil {
		return nil
	}
	dataItems, ok := data.([]interface{})
	if !ok {
		return makeContextError(ctx, "Node should be an 'array'")
	}
	if items, ok := schemaNode["items"]; ok {
		schemaItem := items.(map[string]interface{})
		for i, dataItem := range dataItems {
			err := applyDefaultsRecursive(ctx.addInt(i), rootSchema, &dataItem, schemaItem)
			if err != nil {
				return wrapError(err, "Failed to apply defaults to array item")
			}
			if dataItem != nil {
				dataItems[i] = dataItem
			}
		}
	}
	return nil
}

var metaSchemaData = []byte(`
{
    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "Core schema meta-schema",
    "definitions": {
        "schemaArray": {
            "type": "array",
            "minItems": 1,
            "items": { "$ref": "#" }
        },
        "positiveInteger": {
            "type": "integer",
            "minimum": 0
        },
        "positiveIntegerDefault0": {
            "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ]
        },
        "simpleTypes": {
            "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ]
        },
        "stringArray": {
            "type": "array",
            "items": { "type": "string" },
            "minItems": 1,
            "uniqueItems": true
        }
    },
    "type": "object",
    "properties": {
        "id": {
            "type": "string",
            "format": "uri"
        },
        "$schema": {
            "type": "string",
            "format": "uri"
        },
        "title": {
            "type": "string"
        },
        "description": {
            "type": "string"
        },
        "default": {},
        "multipleOf": {
            "type": "number",
            "minimum": 0,
            "exclusiveMinimum": true
        },
        "maximum": {
            "type": "number"
        },
        "exclusiveMaximum": {
            "type": "boolean",
            "default": false
        },
        "minimum": {
            "type": "number"
        },
        "exclusiveMinimum": {
            "type": "boolean",
            "default": false
        },
        "maxLength": { "$ref": "#/definitions/positiveInteger" },
        "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" },
        "pattern": {
            "type": "string",
            "format": "regex"
        },
        "additionalItems": {
            "anyOf": [
                { "type": "boolean" },
                { "$ref": "#" }
            ],
            "default": {}
        },
        "items": {
            "anyOf": [
                { "$ref": "#" },
                { "$ref": "#/definitions/schemaArray" }
            ],
            "default": {}
        },
        "maxItems": { "$ref": "#/definitions/positiveInteger" },
        "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" },
        "uniqueItems": {
            "type": "boolean",
            "default": false
        },
        "maxProperties": { "$ref": "#/definitions/positiveInteger" },
        "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" },
        "required": { "$ref": "#/definitions/stringArray" },
        "additionalProperties": {
            "anyOf": [
                { "type": "boolean" },
                { "$ref": "#" }
            ],
            "default": {}
        },
        "definitions": {
            "type": "object",
            "additionalProperties": { "$ref": "#" },
            "default": {}
        },
        "properties": {
            "type": "object",
            "additionalProperties": { "$ref": "#" },
            "default": {}
        },
        "patternProperties": {
            "type": "object",
            "additionalProperties": { "$ref": "#" },
            "default": {}
        },
        "dependencies": {
            "type": "object",
            "additionalProperties": {
                "anyOf": [
                    { "$ref": "#" },
                    { "$ref": "#/definitions/stringArray" }
                ]
            }
        },
        "enum": {
            "type": "array",
            "minItems": 1,
            "uniqueItems": true
        },
        "type": {
            "anyOf": [
                { "$ref": "#/definitions/simpleTypes" },
                {
                    "type": "array",
                    "items": { "$ref": "#/definitions/simpleTypes" },
                    "minItems": 1,
                    "uniqueItems": true
                }
            ]
        },
        "allOf": { "$ref": "#/definitions/schemaArray" },
        "anyOf": { "$ref": "#/definitions/schemaArray" },
        "oneOf": { "$ref": "#/definitions/schemaArray" },
        "not": { "$ref": "#" }
    },
    "dependencies": {
        "exclusiveMaximum": [ "maximum" ],
        "exclusiveMinimum": [ "minimum" ]
    },
    "default": {}
}`)


<|start_filename|>doc.go<|end_filename|>
//Package conflate is a library that helps to merge and validate data from multiple formats (JSON/YAML/TOML), and multiple locations (filesystem paths and urls).
package conflate


<|start_filename|>loader.go<|end_filename|>
package conflate

import (
	"io/ioutil"
	"net"
	"net/http"
	pkgurl "net/url"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"
)

var (
	goos        = runtime.GOOS
	emptyURL    = pkgurl.URL{}
	getwd       = os.Getwd
	driveLetter = regexp.MustCompile(`^[A-Za-z]:.*$`)
)

type loader struct {
	newFiledata func([]byte, pkgurl.URL) (filedata, error)
}

func (l *loader) loadURLsRecursive(parentUrls []pkgurl.URL, urls ...pkgurl.URL) (filedatas, error) {
	var allData filedatas
	for _, url := range urls {
		data, err := l.loadURLRecursive(parentUrls, url)
		if err != nil {
			return nil, err
		}
		allData = append(allData, data...)
	}
	return allData, nil
}

func (l *loader) loadURLRecursive(parentUrls []pkgurl.URL, url pkgurl.URL) (filedatas, error) {
	data, err := loadURL(url)
	if err != nil {
		return nil, err
	}
	fdata, err := l.newFiledata(data, url)
	if err != nil {
		return nil, err
	}
	return l.loadDatumRecursive(parentUrls, &url, fdata)
}

func (l *loader) loadDataRecursive(parentUrls []pkgurl.URL, data ...filedata) (filedatas, error) {
	var allData filedatas
	for _, datum := range data {
		childData, err := l.loadDatumRecursive(parentUrls, nil, datum)
		if err != nil {
			return nil, err
		}
		allData = append(allData, childData...)
	}
	return allData, nil
}

func (l *loader) loadDatumRecursive(parentUrls []pkgurl.URL, url *pkgurl.URL, data filedata) (filedatas, error) {
	if data.isEmpty() {
		return nil, nil
	}
	if containsURL(url, parentUrls) {
		return nil, makeError("The url recursively includes itself (%v)", url)
	}
	childUrls, err := toURLs(url, data.includes...)
	if err != nil {
		return nil, err
	}
	var newParentUrls []pkgurl.URL
	newParentUrls = append(newParentUrls, parentUrls...)
	if url != nil {
		newParentUrls = append(newParentUrls, *url)
	}
	childData, err := l.loadURLsRecursive(newParentUrls, childUrls...)
	if err != nil {
		return nil, err
	}
	var allData filedatas
	allData = append(allData, childData...)
	allData = append(allData, data)
	return allData, nil
}

func (l *loader) wrapFiledata(bytes []byte) (filedata, error) {
	return l.newFiledata(bytes, emptyURL)
}

func (l *loader) wrapFiledatas(bytes ...[]byte) (filedatas, error) {
	var fds []filedata
	for _, b := range bytes {
		fd, err := l.wrapFiledata(b)
		if err != nil {
			return nil, err
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

func loadURL(url pkgurl.URL) ([]byte, error) {
	if url.Scheme == "file" {
		// attempt to load locally handling case where we are loading from fifo etc
		b, err := ioutil.ReadFile(getPath(url.Path))
		if err == nil {
			return b, nil
		}
	}
	client := http.Client{Transport: newTransport()}
	resp, err := client.Get(url.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		return nil, makeError("Failed to load url : %v : %v", resp.StatusCode, url.String())
	}
	return data, err
}

func newTransport() *http.Transport {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	transport.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	return transport
}

func toURLs(rootURL *pkgurl.URL, paths ...string) ([]pkgurl.URL, error) {
	var urls []pkgurl.URL
	for _, path := range paths {
		url, err := toURL(rootURL, path)
		if err != nil {
			return nil, err
		}
		urls = append(urls, url)
	}
	return urls, nil
}

func toURL(rootURL *pkgurl.URL, path string) (pkgurl.URL, error) {
	if path == "" {
		return emptyURL, makeError("The file path is blank")
	}
	var err error
	if rootURL == nil {
		rootURL, err = workingDir()
		if err != nil {
			return emptyURL, err
		}
	}
	url, err := pkgurl.Parse(setPath(path))
	if err != nil {
		return emptyURL, wrapError(err, "Could not parse path")
	}
	if !url.IsAbs() {
		url = rootURL.ResolveReference(url)
		url.RawQuery = rootURL.RawQuery
	}
	return *url, nil
}

func containsURL(searchURL *pkgurl.URL, urls []pkgurl.URL) bool {
	if searchURL == nil {
		return false
	}
	for _, url := range urls {
		if url == *searchURL {
			return true
		}
	}
	return false
}

func workingDir() (*pkgurl.URL, error) {
	rootPath, err := getwd()
	if err != nil {
		return nil, err
	}
	rootURL, err := pkgurl.Parse("file://" + setPath(rootPath) + "/")
	if err != nil {
		return nil, err
	}
	return rootURL, nil
}

func setPath(path string) string {
	if goos == "windows" {
		// https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/
		path = strings.Replace(path, `\`, `/`, -1)
		path = strings.TrimLeft(path, `/`)
		if driveLetter.MatchString(path) {
			path = `/` + path
		}
	}
	return path
}

func getPath(path string) string {
	if goos == "windows" {
		// https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/
		path = strings.TrimLeft(path, `/`)
		if !driveLetter.MatchString(path) {
			path = `//` + path
		}
		path = strings.Replace(path, `/`, `\`, -1)
	}
	return path
}


<|start_filename|>filedata.go<|end_filename|>
package conflate

import (
	pkgurl "net/url"
	"os"
	"path/filepath"
	"strings"
)

type filedata struct {
	url      pkgurl.URL
	data     []byte
	obj      map[string]interface{}
	includes []string
}

var emptyFiledata = filedata{}

type filedatas []filedata

// UnmarshallerFunc defines the type of function used for unmarshalling data
type UnmarshallerFunc func([]byte, interface{}) error

// UnmarshallerFuncs defines the type for a slice of UnmarshallerFunc
type UnmarshallerFuncs []UnmarshallerFunc

// UnmarshallerMap defines the type of a map of string to UnmarshallerFuncs
type UnmarshallerMap map[string]UnmarshallerFuncs

// Unmarshallers is a list of unmarshalling functions to be used for given file extensions. The unmarshaller slice for the blank file extension is used when no match is found.
var Unmarshallers = UnmarshallerMap{
	".json": {JSONUnmarshal},
	".jsn":  {JSONUnmarshal},
	".yaml": {YAMLUnmarshal},
	".yml":  {YAMLUnmarshal},
	".toml": {TOMLUnmarshal},
	".tml":  {TOMLUnmarshal},
	"":      {JSONUnmarshal, YAMLUnmarshal, TOMLUnmarshal},
}

func newFiledata(data []byte, url pkgurl.URL) (filedata, error) {
	fd := filedata{data: data, url: url}
	err := fd.unmarshal()
	if err != nil {
		return emptyFiledata, err
	}
	err = fd.validate()
	if err != nil {
		return emptyFiledata, err
	}
	err = fd.extractIncludes()
	if err != nil {
		return emptyFiledata, err
	}
	return fd, nil
}

func newExpandedFiledata(data []byte, url pkgurl.URL) (filedata, error) {
	return newFiledata(recursiveExpand(data), url)
}

func (fd *filedata) wrapError(err error) error {
	if fd.url == emptyURL {
		return err
	}
	return wrapError(err, "Error processing %v", fd.url.String())
}

func (fd *filedata) validate() error {
	return fd.wrapError(validate(fd.obj, getSchema()))
}

func (fd *filedata) unmarshal() error {
	ext := strings.ToLower(filepath.Ext(fd.url.Path))
	unmarshallers, ok := Unmarshallers[ext]
	if !ok {
		unmarshallers = Unmarshallers[""]
	}
	err := makeError("Could not unmarshal data")
	for _, unmarshal := range unmarshallers {
		uerr := unmarshal(fd.data, &fd.obj)
		if uerr == nil {
			return nil
		}
		err = wrapError(uerr, err.Error())
	}
	return err
}

func (fd *filedata) extractIncludes() error {
	if Includes == "" {
		return nil
	}
	err := jsonMarshalUnmarshal(fd.obj[Includes], &fd.includes)
	if err != nil {
		return wrapError(err, "Could not extract includes")
	}
	delete(fd.obj, Includes)
	return nil
}

func (fds filedatas) objs() []interface{} {
	var objs []interface{}
	for _, fd := range fds {
		objs = append(objs, fd.obj)
	}
	return objs
}

func (fd *filedata) isEmpty() bool {
	return fd == nil || fd.obj == nil
}

func recursiveExpand(b []byte) []byte {
	const maxExpansions = 10
	var c int
	for i := 0; i < maxExpansions; i++ {
		b, c = expand(b)
		if c == 0 {
			return b
		}
	}
	return b
}

func expand(b []byte) ([]byte, int) {
	var c int
	return []byte(os.Expand(string(b),
		func(name string) string {
			val, ok := os.LookupEnv(name)
			if ok {
				c++
				return val
			}
			return "$" + name
		})), c
}

var getSchema = getDefaultSchema

func getDefaultSchema() map[string]interface{} {
	return map[string]interface{}{
		"anyOf": []interface{}{
			map[string]interface{}{
				"type": "object",
				"properties": map[string]interface{}{
					Includes: map[string]interface{}{
						"type": "array",
						"items": map[string]interface{}{
							"type": "string",
						},
					},
				},
			},
			map[string]interface{}{
				"type": "null",
			},
		},
	}
}


<|start_filename|>error_test.go<|end_filename|>
package conflate

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestMakeError_NoArgs(t *testing.T) {
	err := makeError("err msg")
	assert.Equal(t, err.Error(), "err msg")
}

func TestMakeError_WithArgs(t *testing.T) {
	err := makeError("err %v", "msg")
	assert.Equal(t, err.Error(), "err msg")
}

func TestMakeContextError_NoArgs(t *testing.T) {
	err := makeContextError("ctx", "err msg")
	assert.Equal(t, err.Error(), "err msg (ctx)")
}

func TestMakeContextError_WithArgs(t *testing.T) {
	err := makeContextError("ctx", "err %v", "msg")
	assert.Equal(t, err.Error(), "err msg (ctx)")
}

func TestWrapError_Nil(t *testing.T) {
	err := wrapError(nil, "an error")
	assert.Nil(t, err)
}

func TestWrapError_NoArgs(t *testing.T) {
	err := wrapError(makeError("err2"), "err1")
	assert.Equal(t, err.Error(), "err1 : err2")
}

func TestWrapError_WithArgs(t *testing.T) {
	err := wrapError(makeError("err2"), "err%v", 1)
	assert.Equal(t, err.Error(), "err1 : err2")
}


<|start_filename|>context_test.go<|end_filename|>
package conflate

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestRootContext_String(t *testing.T) {
	ctx := rootContext()
	assert.Equal(t, "#", ctx.String())
}

func TestMakeContext_Add(t *testing.T) {
	ctx := rootContext()
	ctx2 := ctx.add("parent").add("child")
	assert.Equal(t, "#", ctx.String())
	assert.Equal(t, "#/parent/child", ctx2.String())
}

func TestMakeContext_AddInt(t *testing.T) {
	ctx := rootContext()
	ctx2 := ctx.add("parent").addInt(3)
	assert.Equal(t, "#", ctx.String())
	assert.Equal(t, "#/parent[3]", ctx2.String())
}


<|start_filename|>marshal_test.go<|end_filename|>
package conflate

import (
	"errors"
	"github.com/stretchr/testify/assert"
	"testing"
)

// --------

func TestJSONMarshalAll(t *testing.T) {
	data, err := jsonMarshalAll("a", "b", "c")
	assert.Nil(t, err)
	assert.Equal(t, 3, len(data))
	assert.Equal(t, []byte("\"a\"\n"), data[0])
	assert.Equal(t, []byte("\"b\"\n"), data[1])
	assert.Equal(t, []byte("\"c\"\n"), data[2])
}

func TestJSONMarshalAll_Error(t *testing.T) {
	mockMarshal := func(obj interface{}) ([]byte, error) {
		return nil, errors.New("my error")
	}
	data, err := jsonMarshalAll(mockMarshal, "a")
	assert.NotNil(t, err)
	assert.Nil(t, data)
	assert.Contains(t, err.Error(), "The data could not be marshalled")
}

// --------

func TestJSONMarshalUnmarshal(t *testing.T) {
	var out interface{}
	err := jsonMarshalUnmarshal(testMarshalData, &out)
	assert.Nil(t, err)
	assert.Equal(t, testMarshalData, out)
}

func TestJSONMarshalUnmarshal_MarshalError(t *testing.T) {
	var out interface{}
	err := jsonMarshalUnmarshal(testMarshalDataInvalid, out)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "could not be marshalled to json")
}

func TestJSONMarshalUnmarshal_UnmarshalError(t *testing.T) {
	err := jsonMarshalUnmarshal(testMarshalData, testMarshalDataInvalid)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "could not be unmarshalled as json")
}

// --------

func TestJSONUnmarshal(t *testing.T) {
	var out interface{}
	err := JSONUnmarshal(testMarshalJSON, &out)
	assert.Nil(t, err)
	assert.Equal(t, testMarshalData, out)
}

func TestJSONUnmarshal_Error(t *testing.T) {
	var out interface{}
	err := JSONUnmarshal(testMarshalInvalid, &out)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "could not be unmarshalled as json")
}

func TestYAMLUnmarshal(t *testing.T) {
	var out interface{}
	err := YAMLUnmarshal(testMarshalYAML, &out)
	assert.Nil(t, err)
	assert.Equal(t, testMarshalData, out)
}

func TestYAMLUnmarshal_Error(t *testing.T) {
	var out interface{}
	err := YAMLUnmarshal(testMarshalInvalid, &out)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "could not be unmarshalled as yaml")
}

func TestTOMLUnmarshal(t *testing.T) {
	var out interface{}
	err := TOMLUnmarshal(testMarshalTOML, &out)
	assert.Nil(t, err)
	assert.Equal(t, testMarshalData, out)
}

func TestTOMLUnmarshal_Error(t *testing.T) {
	var out interface{}
	err := TOMLUnmarshal(testMarshalInvalid, &out)
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "could not be unmarshalled as toml")
}

// --------

func TestJSONMarshal(t *testing.T) {
	out, err := jsonMarshal(testMarshalData)
	assert.Nil(t, err)
	assert.Equal(t, string(testMarshalJSON), string(out))
}

func TestJSONMarshal_Error(t *testing.T) {
	out, err := jsonMarshal(testMarshalDataInvalid)
	assert.NotNil(t, err)
	assert.Nil(t, out)

	assert.Contains(t, err.Error(), "marshalled to json")
}

func TestYAMLMarshal(t *testing.T) {
	out, err := yamlMarshal(testMarshalData)
	assert.Nil(t, err)
	assert.Equal(t, string(testMarshalYAML), string(out))
}

func TestYAMLMarshal_Error(t *testing.T) {
	out, err := yamlMarshal(testMarshalDataInvalid)
	assert.NotNil(t, err)
	assert.Nil(t, out)
	assert.Contains(t, err.Error(), "marshalled to yaml")
}

func TestTOMLMarshal(t *testing.T) {
	out, err := tomlMarshal(testMarshalData)
	assert.Nil(t, err)
	assert.Equal(t, string(testMarshalTOML), string(out))
}

func TestTOMLMarshal_PanicError(t *testing.T) {
	out, err := tomlMarshal(testMarshalDataInvalid)
	assert.NotNil(t, err)
	assert.Nil(t, out)
	assert.Contains(t, err.Error(), "marshalled to toml")
}

func TestTOMLMarshal_Error(t *testing.T) {
	in := []interface{}{123, "123"}
	out, err := tomlMarshal(in)
	assert.NotNil(t, err)
	assert.Nil(t, out)
	assert.Contains(t, err.Error(), "marshalled to toml")
}

// --------

var (
	testValue              = `value!£$%^&*()_+-={}[]:@~;'#<>?,./|`
	testMarshalData        = map[string]interface{}{"key": testValue}
	testMarshalDataInvalid = func() {}
	testMarshalJSON        = []byte(`{
  "key": "` + testValue + `"
}
`)
	testMarshalYAML = []byte(`key: ` + testValue + `
`)
	testMarshalTOML = []byte(`key = "` + testValue + `"
`)
	testMarshalInvalid = []byte(`{invalid`)
)
