chore(kontemplate): Prepare kontemplate for depot-merge

This merge will not yet include moving over to buildGo.nix, as support
for testing and such is not present in that library yet.

ops/kontemplate/context/context.go

@@ -0,0 +1,266 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+package context
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/tazjin/kontemplate/util"
+)
+
+type ResourceSet struct {
+	// Name of the resource set. This can be used in include/exclude statements during kontemplate runs.
+	Name string `json:"name"`
+
+	// Path to the folder containing the files for this resource set. This defaults to the value of the 'name' field
+	// if unset.
+	Path string `json:"path"`
+
+	// Values to include when interpolating resources from this resource set.
+	Values map[string]interface{} `json:"values"`
+
+	// Args to pass on to kubectl for this resource set.
+	Args []string `json:"args"`
+
+	// Nested resource sets to include
+	Include []ResourceSet `json:"include"`
+
+	// Parent resource set for flattened resource sets. Should not be manually specified.
+	Parent string
+}
+
+type Context struct {
+	// The name of the kubectl context
+	Name string `json:"context"`
+
+	// Global variables that should be accessible by all resource sets
+	Global map[string]interface{} `json:"global"`
+
+	// File names of YAML or JSON files including extra variables that should be globally accessible
+	VariableImportFiles []string `json:"import"`
+
+	// The resource sets to include in this context
+	ResourceSets []ResourceSet `json:"include"`
+
+	// Variables imported from additional files
+	ImportedVars map[string]interface{}
+
+	// Explicitly set variables (via `--var`) that should override all others
+	ExplicitVars map[string]interface{}
+
+	// This field represents the absolute path to the context base directory and should not be manually specified.
+	BaseDir string
+}
+
+func contextLoadingError(filename string, cause error) error {
+	return fmt.Errorf("Context loading failed on file %s due to: \n%v", filename, cause)
+}
+
+// Attempt to load and deserialise a Context from the specified file.
+func LoadContext(filename string, explicitVars *[]string) (*Context, error) {
+	var ctx Context
+	err := util.LoadData(filename, &ctx)
+
+	if err != nil {
+		return nil, contextLoadingError(filename, err)
+	}
+
+	ctx.BaseDir = path.Dir(filename)
+
+	// Prepare the resource sets by resolving parents etc.
+	ctx.ResourceSets = flattenPrepareResourceSetPaths(&ctx.BaseDir, &ctx.ResourceSets)
+
+	// Add variables explicitly specified on the command line
+	ctx.ExplicitVars, err = loadExplicitVars(explicitVars)
+	if err != nil {
+		return nil, fmt.Errorf("Error setting explicit variables: %v\n", err)
+	}
+
+	// Add variables loaded from import files
+	ctx.ImportedVars, err = ctx.loadImportedVariables()
+	if err != nil {
+		return nil, contextLoadingError(filename, err)
+	}
+
+	// Merge variables defined at different levels. The
+	// `mergeContextValues` function is documented with the merge
+	// hierarchy.
+	ctx.ResourceSets = ctx.mergeContextValues()
+
+	if err != nil {
+		return nil, contextLoadingError(filename, err)
+	}
+
+	return &ctx, nil
+}
+
+// Kontemplate supports specifying additional variable files with the
+// `import` keyword. This function loads those variable files and
+// merges them together with the context's other global variables.
+func (ctx *Context) loadImportedVariables() (map[string]interface{}, error) {
+	allImportedVars := make(map[string]interface{})
+
+	for _, file := range ctx.VariableImportFiles {
+		// Ensure that the filename is not merged with the baseDir if
+		// it is set to an absolute path.
+		var filePath string
+		if path.IsAbs(file) {
+			filePath = file
+		} else {
+			filePath = path.Join(ctx.BaseDir, file)
+		}
+
+		var importedVars map[string]interface{}
+		err := util.LoadData(filePath, &importedVars)
+
+		if err != nil {
+			return nil, err
+		}
+
+		allImportedVars = *util.Merge(&allImportedVars, &importedVars)
+	}
+
+	return allImportedVars, nil
+}
+
+// Correctly prepares the file paths for resource sets by inferring implicit paths and flattening resource set
+// collections, i.e. resource sets that themselves have an additional 'include' field set.
+// Those will be regarded as a short-hand for including multiple resource sets from a subfolder.
+// See https://github.com/tazjin/kontemplate/issues/9 for more information.
+func flattenPrepareResourceSetPaths(baseDir *string, rs *[]ResourceSet) []ResourceSet {
+	flattened := make([]ResourceSet, 0)
+
+	for _, r := range *rs {
+		// If a path is not explicitly specified it should default to the resource set name.
+		// This is also the classic behaviour prior to kontemplate 1.2
+		if r.Path == "" {
+			r.Path = r.Name
+		}
+
+		// Paths are made absolute by resolving them relative to the context base,
+		// unless absolute paths were specified.
+		if !path.IsAbs(r.Path) {
+			r.Path = path.Join(*baseDir, r.Path)
+		}
+
+		if len(r.Include) == 0 {
+			flattened = append(flattened, r)
+		} else {
+			for _, subResourceSet := range r.Include {
+				if subResourceSet.Path == "" {
+					subResourceSet.Path = subResourceSet.Name
+				}
+
+				subResourceSet.Parent = r.Name
+				subResourceSet.Name = path.Join(r.Name, subResourceSet.Name)
+				subResourceSet.Path = path.Join(r.Path, subResourceSet.Path)
+				subResourceSet.Values = *util.Merge(&r.Values, &subResourceSet.Values)
+				flattened = append(flattened, subResourceSet)
+			}
+		}
+	}
+
+	return flattened
+}
+
+// Merges the context and resource set variables according in the
+// desired precedence order.
+//
+// For now the reasoning behind the merge order is from least specific
+// in relation to the cluster configuration, which means that the
+// precedence is (in ascending order):
+//
+// 1. Default values in resource sets.
+// 2. Values imported from files (via `import:`)
+// 3. Global values in a cluster configuration
+// 4. Values set in a resource set's `include`-section
+// 5. Explicit values set on the CLI (`--var`)
+//
+// For a discussion on the reasoning behind this order, please consult
+// https://github.com/tazjin/kontemplate/issues/142
+func (ctx *Context) mergeContextValues() []ResourceSet {
+	updated := make([]ResourceSet, len(ctx.ResourceSets))
+
+	// Merging has to happen separately for every individual
+	// resource set to make use of the default values:
+	for i, rs := range ctx.ResourceSets {
+		// Begin by loading default values from the resource
+		// sets configuration.
+		//
+		// Resource sets are used across different cluster
+		// contexts and the default values in them have the
+		// lowest precedence.
+		defaultValues := loadDefaultValues(&rs, ctx)
+
+		// Continue by merging default values with values
+		// imported from external files. Those values are also
+		// used across cluster contexts, but have higher
+		// precedence than defaults.
+		merged := util.Merge(defaultValues, &ctx.ImportedVars)
+
+		// Merge global values defined in the cluster context:
+		merged = util.Merge(merged, &ctx.Global)
+
+		// Merge values configured in the resource set's
+		// `include` section:
+		merged = util.Merge(merged, &rs.Values)
+
+		// Merge values defined explicitly on the CLI:
+		merged = util.Merge(merged, &ctx.ExplicitVars)
+
+		// Continue with the newly merged resource set:
+		rs.Values = *merged
+		updated[i] = rs
+	}
+
+	return updated
+}
+
+// Loads default values for a resource set collection from
+// path/to/set/default.{json|yaml}.
+func loadDefaultValues(rs *ResourceSet, c *Context) *map[string]interface{} {
+	var defaultVars map[string]interface{}
+
+	for _, filename := range util.DefaultFilenames {
+		err := util.LoadData(path.Join(rs.Path, filename), &defaultVars)
+		if err == nil {
+			return &defaultVars
+		}
+	}
+
+	// The actual error is not inspected here. The reasoning for
+	// this is that in case of serious problems (e.g. permission
+	// issues with the folder / folder not existing) failure will
+	// occur a bit later anyways.
+	//
+	// Otherwise we'd have to differentiate between
+	// file-not-found-errors (no default values specified) and
+	// other errors here.
+	return &rs.Values
+}
+
+// Prepares the variables specified explicitly via `--var` when
+// executing kontemplate for adding to the context.
+func loadExplicitVars(vars *[]string) (map[string]interface{}, error) {
+	explicitVars := make(map[string]interface{}, len(*vars))
+
+	for _, v := range *vars {
+		varParts := strings.SplitN(v, "=", 2)
+		if len(varParts) != 2 {
+			return nil, fmt.Errorf(`invalid explicit variable provided (%s), name and value should be separated with "="`, v)
+		}
+
+		explicitVars[varParts[0]] = varParts[1]
+	}
+
+	return explicitVars, nil
+}

ops/kontemplate/context/context_test.go

@@ -0,0 +1,353 @@
+// Copyright (C) 2016-2019  Vincent Ambo <mail@tazj.in>
+//
+// This file is part of Kontemplate.
+//
+// Kontemplate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+package context
+
+import (
+	"reflect"
+	"testing"
+)
+
+var noExplicitVars []string = make([]string, 0)
+
+func TestLoadFlatContextFromFile(t *testing.T) {
+	ctx, err := LoadContext("testdata/flat-test.yaml", &noExplicitVars)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		Global: map[string]interface{}{
+			"globalVar": "lizards",
+		},
+		ResourceSets: []ResourceSet{
+			{
+				Name: "some-api",
+				Path: "testdata/some-api",
+				Values: map[string]interface{}{
+					"apiPort":          float64(4567), // yep!
+					"importantFeature": true,
+					"version":          "1.0-0e6884d",
+					"globalVar":        "lizards",
+				},
+				Include: nil,
+				Parent:  "",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded context and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestLoadContextWithArgs(t *testing.T) {
+	ctx, err := LoadContext("testdata/flat-with-args-test.yaml", &noExplicitVars)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		ResourceSets: []ResourceSet{
+			{
+				Name:   "some-api",
+				Path:   "testdata/some-api",
+				Values: make(map[string]interface{}, 0),
+				Args: []string{
+					"--as=some-user",
+					"--as-group=hello:world",
+					"--as-banana",
+					"true",
+				},
+				Include: nil,
+				Parent:  "",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded context and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestLoadContextWithResourceSetCollections(t *testing.T) {
+	ctx, err := LoadContext("testdata/collections-test.yaml", &noExplicitVars)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		Global: map[string]interface{}{
+			"globalVar": "lizards",
+		},
+		ResourceSets: []ResourceSet{
+			{
+				Name: "some-api",
+				Path: "testdata/some-api",
+				Values: map[string]interface{}{
+					"apiPort":          float64(4567), // yep!
+					"importantFeature": true,
+					"version":          "1.0-0e6884d",
+					"globalVar":        "lizards",
+				},
+				Include: nil,
+				Parent:  "",
+			},
+			{
+				Name: "collection/nested",
+				Path: "testdata/collection/nested",
+				Values: map[string]interface{}{
+					"lizards":   "good",
+					"globalVar": "lizards",
+				},
+				Include: nil,
+				Parent:  "collection",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded context and expected context did not match")
+		t.Fail()
+	}
+
+}
+
+func TestSubresourceVariableInheritance(t *testing.T) {
+	ctx, err := LoadContext("testdata/parent-variables.yaml", &noExplicitVars)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		ResourceSets: []ResourceSet{
+			{
+				Name: "parent/child",
+				Path: "testdata/parent/child",
+				Values: map[string]interface{}{
+					"foo": "bar",
+					"bar": "baz",
+				},
+				Include: nil,
+				Parent:  "parent",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestSubresourceVariableInheritanceOverride(t *testing.T) {
+	ctx, err := LoadContext("testdata/parent-variable-override.yaml", &noExplicitVars)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		ResourceSets: []ResourceSet{
+			{
+				Name: "parent/child",
+				Path: "testdata/parent/child",
+				Values: map[string]interface{}{
+					"foo": "newvalue",
+				},
+				Include: nil,
+				Parent:  "parent",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestDefaultValuesLoading(t *testing.T) {
+	ctx, err := LoadContext("testdata/default-loading.yaml", &noExplicitVars)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	rs := ctx.ResourceSets[0]
+	if rs.Values["defaultValues"] != "loaded" {
+		t.Errorf("Default values not loaded from YAML file")
+		t.Fail()
+	}
+
+	if rs.Values["override"] != "notAtAll" {
+		t.Error("Default values should not override other values")
+		t.Fail()
+	}
+}
+
+func TestImportValuesLoading(t *testing.T) {
+	ctx, err := LoadContext("testdata/import-vars-simple.yaml", &noExplicitVars)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := map[string]interface{}{
+		"override": "true",
+		"music": map[string]interface{}{
+			"artist": "Pallida",
+			"track":  "Tractor Beam",
+		},
+	}
+
+	if !reflect.DeepEqual(ctx.ImportedVars, expected) {
+		t.Error("Expected imported values after loading imports did not match!")
+		t.Fail()
+	}
+}
+
+func TestExplicitPathLoading(t *testing.T) {
+	ctx, err := LoadContext("testdata/explicit-path.yaml", &noExplicitVars)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		ResourceSets: []ResourceSet{
+			{
+				Name: "some-api-europe",
+				Path: "testdata/some-api",
+				Values: map[string]interface{}{
+					"location": "europe",
+				},
+				Include: nil,
+				Parent:  "",
+			},
+			{
+				Name: "some-api-asia",
+				Path: "testdata/some-api",
+				Values: map[string]interface{}{
+					"location": "asia",
+				},
+				Include: nil,
+				Parent:  "",
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded context and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestExplicitSubresourcePathLoading(t *testing.T) {
+	ctx, err := LoadContext("testdata/explicit-subresource-path.yaml", &noExplicitVars)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	expected := Context{
+		Name: "k8s.prod.mydomain.com",
+		ResourceSets: []ResourceSet{
+			{
+				Name:   "parent/child",
+				Path:   "testdata/parent-path/child-path",
+				Parent: "parent",
+				Values: make(map[string]interface{}, 0),
+			},
+		},
+		BaseDir:      "testdata",
+		ImportedVars: make(map[string]interface{}, 0),
+		ExplicitVars: make(map[string]interface{}, 0),
+	}
+
+	if !reflect.DeepEqual(*ctx, expected) {
+		t.Error("Loaded context and expected context did not match")
+		t.Fail()
+	}
+}
+
+func TestSetVariablesFromArguments(t *testing.T) {
+	vars := []string{"version=some-service-version"}
+	ctx, _ := LoadContext("testdata/default-loading.yaml", &vars)
+
+	if version := ctx.ExplicitVars["version"]; version != "some-service-version" {
+		t.Errorf(`Expected variable "version" to have value "some-service-version" but was "%s"`, version)
+	}
+}
+
+func TestSetInvalidVariablesFromArguments(t *testing.T) {
+	vars := []string{"version: some-service-version"}
+	_, err := LoadContext("testdata/default-loading.yaml", &vars)
+
+	if err == nil {
+		t.Error("Expected invalid variable to return an error")
+	}
+}
+
+// This test ensures that variables are merged in the correct order.
+// Please consult the test data in `testdata/merging`.
+func TestValueMergePrecedence(t *testing.T) {
+	cliVars:= []string{"cliVar=cliVar"}
+	ctx, _ := LoadContext("testdata/merging/context.yaml", &cliVars)
+
+	expected := map[string]interface{}{
+		"defaultVar": "defaultVar",
+		"importVar": "importVar",
+		"globalVar": "globalVar",
+		"includeVar": "includeVar",
+		"cliVar": "cliVar",
+	}
+
+	result := ctx.ResourceSets[0].Values
+
+	if !reflect.DeepEqual(expected, result) {
+		t.Errorf("Merged values did not match expected result: \n%v", result)
+		t.Fail()
+	}
+}

ops/kontemplate/context/testdata/collections-test.yaml

@@ -0,0 +1,15 @@
+---
+context: k8s.prod.mydomain.com
+global:
+  globalVar: lizards
+include:
+  - name: some-api
+    values:
+      version: 1.0-0e6884d
+      importantFeature: true
+      apiPort: 4567
+  - name: collection
+    include:
+      - name: nested
+        values:
+          lizards: good

ops/kontemplate/context/testdata/default-loading.yaml

@@ -0,0 +1,6 @@
+---
+context: default-loading
+include:
+  - name: default
+    values:
+      override: notAtAll

ops/kontemplate/context/testdata/default/default.yaml

@@ -0,0 +1,2 @@
+defaultValues: loaded
+override: noop

ops/kontemplate/context/testdata/explicit-path.yaml

@@ -0,0 +1,11 @@
+---
+context: k8s.prod.mydomain.com
+include:
+  - name: some-api-europe
+    path: some-api
+    values:
+      location: europe
+  - name: some-api-asia
+    path: some-api
+    values:
+      location: asia

ops/kontemplate/context/testdata/explicit-subresource-path.yaml

@@ -0,0 +1,8 @@
+---
+context: k8s.prod.mydomain.com
+include:
+  - name: parent
+    path: parent-path
+    include:
+      - name: child
+        path: child-path

ops/kontemplate/context/testdata/flat-test.yaml

@@ -0,0 +1,10 @@
+---
+context: k8s.prod.mydomain.com
+global:
+  globalVar: lizards
+include:
+  - name: some-api
+    values:
+      version: 1.0-0e6884d
+      importantFeature: true
+      apiPort: 4567

ops/kontemplate/context/testdata/flat-with-args-test.yaml

@@ -0,0 +1,9 @@
+---
+context: k8s.prod.mydomain.com
+include:
+  - name: some-api
+    args:
+      - --as=some-user
+      - --as-group=hello:world
+      - --as-banana
+      - "true"

ops/kontemplate/context/testdata/import-vars-simple.yaml

@@ -0,0 +1,5 @@
+---
+context: k8s.prod.mydomain.com
+import:
+  - test-vars.yaml
+include: []

ops/kontemplate/context/testdata/merging/context.yaml

@@ -0,0 +1,15 @@
+# This context file is intended to test the merge hierarchy of
+# variables defined at different levels.
+---
+context: merging.in.kontemplate.works
+global:
+  globalVar: globalVar
+  includeVar: should be overridden (global)
+  cliVar: should be overridden (global)
+import:
+  - import-vars.yaml
+include:
+  - name: resource
+    values:
+      includeVar: includeVar
+      cliVar: should be overridden (include)

ops/kontemplate/context/testdata/merging/import-vars.yaml

@@ -0,0 +1,4 @@
+importVar: importVar
+globalVar: should be overridden (import)
+includeVar: should be overridden (import)
+cliVar: should be overridden (import)

ops/kontemplate/context/testdata/merging/resource/default.yaml

@@ -0,0 +1,5 @@
+defaultVar: defaultVar
+importVar: should be overridden (default)
+globalVar: should be overridden (default)
+includeVar: should be overridden (default)
+cliVar: should be overridden (default)

ops/kontemplate/context/testdata/merging/resource/output.yaml

@@ -0,0 +1,5 @@
+defaultVar: {{ .defaultVar }}
+importVar: {{ .importVar }}
+globalVar: {{ .globalVar }}
+includeVar: {{ .includeVar }}
+cliVar: {{ .cliVar }}

ops/kontemplate/context/testdata/parent-variable-override.yaml

@@ -0,0 +1,10 @@
+---
+context: k8s.prod.mydomain.com
+include:
+  - name: parent
+    values:
+      foo: bar
+    include:
+      - name: child
+        values:
+          foo: newvalue

ops/kontemplate/context/testdata/parent-variables.yaml

@@ -0,0 +1,10 @@
+---
+context: k8s.prod.mydomain.com
+include:
+  - name: parent
+    values:
+      foo: bar
+    include:
+      - name: child
+        values:
+          bar: baz

ops/kontemplate/context/testdata/test-vars-override.yaml

@@ -0,0 +1,3 @@
+---
+override: 3
+place: Oslo

ops/kontemplate/context/testdata/test-vars.yaml

@@ -0,0 +1,5 @@
+---
+override: 'true'
+music:
+  artist: Pallida
+  track: Tractor Beam