Skip to content

Commit e47fa61

Browse files
pieternpietern
authored
Skeleton for configuration loading and mutation (#92)
Load a tree of configuration files anchored at `bundle.yml` into the `config.Root` struct. All mutations (from setting defaults to merging files) are observable through the `mutator.Mutator` interface.
1 parent 6a8c9f2 commit e47fa61

38 files changed

+1027
-0
lines changed

‎bundle/bundle.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package bundle
2+
3+
import (
4+
"path/filepath"
5+
6+
"github.com/databricks/bricks/bundle/config"
7+
)
8+
9+
type Bundle struct {
10+
Config config.Root
11+
}
12+
13+
func Load(path string) (*Bundle, error) {
14+
bundle := &Bundle{
15+
Config: config.Root{
16+
Path: path,
17+
},
18+
}
19+
err := bundle.Config.Load(filepath.Join(path, config.FileName))
20+
if err != nil {
21+
return nil, err
22+
}
23+
return bundle, nil
24+
}

‎bundle/bundle_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package bundle
2+
3+
import (
4+
"os"
5+
"testing"
6+
7+
"github.com/stretchr/testify/assert"
8+
"github.com/stretchr/testify/require"
9+
)
10+
11+
func TestLoadNotExists(t *testing.T) {
12+
b, err := Load("/doesntexist")
13+
assert.True(t, os.IsNotExist(err))
14+
assert.Nil(t, b)
15+
}
16+
17+
func TestLoadExists(t *testing.T) {
18+
b, err := Load("./config/tests/basic")
19+
require.Nil(t, err)
20+
assert.Equal(t, "basic", b.Config.Bundle.Name)
21+
}

‎bundle/config/bundle.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package config
2+
3+
type Bundle struct {
4+
Name string `json:"name,omitempty"`
5+
6+
// TODO
7+
// Default cluster to run commands on (Python, Scala).
8+
// DefaultCluster string `json:"default_cluster,omitempty"`
9+
10+
// TODO
11+
// Default warehouse to run SQL on.
12+
// DefaultWarehouse string `json:"default_warehouse,omitempty"`
13+
}

‎bundle/config/environment.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package config
2+
3+
// Environment defines overrides for a single environment.
4+
// This structure is recursively merged into the root configuration.
5+
type Environment struct {
6+
Bundle *Bundle `json:"bundle,omitempty"`
7+
8+
Workspace *Workspace `json:"workspace,omitempty"`
9+
10+
Resources *Resources `json:"resources,omitempty"`
11+
}

‎bundle/config/mutator/default_environment.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package mutator
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/databricks/bricks/bundle/config"
7+
)
8+
9+
type defineDefaultEnvironment struct {
10+
name string
11+
}
12+
13+
// DefineDefaultEnvironment adds an environment named "default"
14+
// to the configuration if none have been defined.
15+
func DefineDefaultEnvironment() Mutator {
16+
return &defineDefaultEnvironment{
17+
name: "default",
18+
}
19+
}
20+
21+
func (m *defineDefaultEnvironment) Name() string {
22+
return fmt.Sprintf("DefineDefaultEnvironment(%s)", m.name)
23+
}
24+
25+
func (m *defineDefaultEnvironment) Apply(root *config.Root) ([]Mutator, error) {
26+
// Nothing to do if the configuration has at least 1 environment.
27+
if root.Environments != nil || len(root.Environments) > 0 {
28+
return nil, nil
29+
}
30+
31+
// Define default environment.
32+
root.Environments = make(map[string]*config.Environment)
33+
root.Environments[m.name] = &config.Environment{}
34+
return nil, nil
35+
}

‎bundle/config/mutator/default_environment_test.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package mutator_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/databricks/bricks/bundle/config"
7+
"github.com/databricks/bricks/bundle/config/mutator"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestDefaultEnvironment(t *testing.T) {
13+
root := &config.Root{}
14+
_, err := mutator.DefineDefaultEnvironment().Apply(root)
15+
require.NoError(t, err)
16+
env, ok := root.Environments["default"]
17+
assert.True(t, ok)
18+
assert.Equal(t, &config.Environment{}, env)
19+
}
20+
21+
func TestDefaultEnvironmentAlreadySpecified(t *testing.T) {
22+
root := &config.Root{
23+
Environments: map[string]*config.Environment{
24+
"development": {},
25+
},
26+
}
27+
_, err := mutator.DefineDefaultEnvironment().Apply(root)
28+
require.NoError(t, err)
29+
_, ok := root.Environments["default"]
30+
assert.False(t, ok)
31+
}

‎bundle/config/mutator/default_include.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package mutator
2+
3+
import (
4+
"github.com/databricks/bricks/bundle/config"
5+
"golang.org/x/exp/slices"
6+
)
7+
8+
type defineDefaultInclude struct {
9+
include []string
10+
}
11+
12+
// DefineDefaultInclude sets the list of includes to a default if it hasn't been set.
13+
func DefineDefaultInclude() Mutator {
14+
return &defineDefaultInclude{
15+
// When we support globstar we can collapse below into a single line.
16+
include: []string{
17+
// Load YAML files in the same directory.
18+
"*.yml",
19+
// Load YAML files in subdirectories.
20+
"*/*.yml",
21+
},
22+
}
23+
}
24+
25+
func (m *defineDefaultInclude) Name() string {
26+
return "DefineDefaultInclude"
27+
}
28+
29+
func (m *defineDefaultInclude) Apply(root *config.Root) ([]Mutator, error) {
30+
if len(root.Include) == 0 {
31+
root.Include = slices.Clone(m.include)
32+
}
33+
return nil, nil
34+
}

‎bundle/config/mutator/default_include_test.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package mutator_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/databricks/bricks/bundle/config"
7+
"github.com/databricks/bricks/bundle/config/mutator"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestDefaultInclude(t *testing.T) {
13+
root := &config.Root{}
14+
_, err := mutator.DefineDefaultInclude().Apply(root)
15+
require.NoError(t, err)
16+
assert.Equal(t, []string{"*.yml", "*/*.yml"}, root.Include)
17+
}

‎bundle/config/mutator/mutator.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package mutator
2+
3+
import "github.com/databricks/bricks/bundle/config"
4+
5+
// Mutator is the interface types that mutate the bundle configuration.
6+
// This makes every mutation observable and debuggable.
7+
type Mutator interface {
8+
// Name returns the mutators name.
9+
Name() string
10+
11+
// Apply mutates the specified configuration object.
12+
// It optionally returns a list of mutators to invoke immediately after this mutator.
13+
// This is used when processing all configuration files in the tree; each file gets
14+
// its own mutator instance.
15+
Apply(*config.Root) ([]Mutator, error)
16+
}
17+
18+
func DefaultMutators() []Mutator {
19+
return []Mutator{
20+
DefineDefaultInclude(),
21+
ProcessRootIncludes(),
22+
DefineDefaultEnvironment(),
23+
}
24+
}
25+
26+
func DefaultMutatorsForEnvironment(env string) []Mutator {
27+
return append(DefaultMutators(), SelectEnvironment(env))
28+
}
29+
30+
func Apply(root *config.Root, ms []Mutator) error {
31+
if len(ms) == 0 {
32+
return nil
33+
}
34+
for _, m := range ms {
35+
ms_, err := m.Apply(root)
36+
if err != nil {
37+
return err
38+
}
39+
// Apply recursively.
40+
err = Apply(root, ms_)
41+
if err != nil {
42+
return err
43+
}
44+
}
45+
return nil
46+
}

‎bundle/config/mutator/process_include.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package mutator
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/databricks/bricks/bundle/config"
7+
)
8+
9+
type processInclude struct {
10+
fullPath string
11+
relPath string
12+
}
13+
14+
// ProcessInclude loads the configuration at [fullPath] and merges it into the configuration.
15+
func ProcessInclude(fullPath, relPath string) Mutator {
16+
return &processInclude{
17+
fullPath: fullPath,
18+
relPath: relPath,
19+
}
20+
}
21+
22+
func (m *processInclude) Name() string {
23+
return fmt.Sprintf("ProcessInclude(%s)", m.relPath)
24+
}
25+
26+
func (m *processInclude) Apply(root *config.Root) ([]Mutator, error) {
27+
this, err := config.Load(m.fullPath)
28+
if err != nil {
29+
return nil, err
30+
}
31+
return nil, root.Merge(this)
32+
}

0 commit comments

Comments
 (0)