Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ jobs:
plugin-secrets-version: 3.15.0
plugin-diff-version: 3.5.0
extra-helmfile-flags:
v1mode:
- helm-version: v3.9.4
kustomize-version: v4.5.7
# We assume that the helm-secrets plugin is supposed to
Expand All @@ -71,23 +72,34 @@ jobs:
plugin-secrets-version: 4.0.0
plugin-diff-version: 3.6.0
extra-helmfile-flags:
v1mode:
- helm-version: v3.10.3
kustomize-version: v4.4.1
plugin-secrets-version: 3.15.0
plugin-diff-version: 3.5.0
extra-helmfile-flags:
v1mode:
- helm-version: v3.10.3
kustomize-version: v4.5.7
plugin-secrets-version: 4.0.0
plugin-diff-version: 3.6.0
extra-helmfile-flags:
v1mode:
# Helmfile v1
- helm-version: v3.10.3
kustomize-version: v4.5.7
plugin-secrets-version: 4.0.0
plugin-diff-version: 3.6.0
extra-helmfile-flags:
v1mode: "true"
Comment on lines +88 to +94

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yxxhero We now run the integration test suite for the v1 mode once per build, with the representative versions of helmfile dependencies, so that we won't break any v1 functionality until we finally stop maintaining v0.

# In case you need to test some optional helmfile features,
# enable it via extra-helmfile-flags below.
- helm-version: v3.10.3
kustomize-version: v4.5.7
plugin-secrets-version: 4.0.0
plugin-diff-version: 3.6.0
extra-helmfile-flags: "--enable-live-output"
v1mode:
steps:
- uses: actions/checkout@v2
- name: Cache libraries
Expand Down Expand Up @@ -131,6 +143,7 @@ jobs:
HELMFILE_HELM3: 1
TERM: xterm
EXTRA_HELMFILE_FLAGS: ${{ matrix.extra-helmfile-flags }}
HELMFILE_V1MODE: ${{ matrix.v1mode }}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, HELMFILE_V1MODE is defined in pkg/runtime/runtime.go

run: make integration
e2e_tests:
needs: tests
Expand Down
30 changes: 20 additions & 10 deletions pkg/app/app_template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/helmfile/helmfile/pkg/exectest"
ffs "github.com/helmfile/helmfile/pkg/filesystem"
"github.com/helmfile/helmfile/pkg/helmexec"
"github.com/helmfile/helmfile/pkg/runtime"
"github.com/helmfile/helmfile/pkg/testhelper"
"github.com/helmfile/helmfile/pkg/yaml"
)

func TestTemplate(t *testing.T) {
Expand Down Expand Up @@ -309,20 +309,21 @@ releases:
}

func TestTemplate_StrictParsing(t *testing.T) {
v := yaml.GoccyGoYaml
yaml.GoccyGoYaml = true
t.Cleanup(func() {
yaml.GoccyGoYaml = v
})

type testcase struct {
ns string
error string
goccyGoYaml bool
ns string
error string
}

check := func(t *testing.T, tc testcase) {
t.Helper()

v := runtime.GoccyGoYaml
runtime.GoccyGoYaml = tc.goccyGoYaml
t.Cleanup(func() {
runtime.GoccyGoYaml = v
})
Comment on lines +321 to +325

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an idiom I used widely across many test sources to ensure we modify the GoccyGoYaml runtime variable only while running the (sub-)test.


var helm = &exectest.Helm{
FailOnUnexpectedList: true,
FailOnUnexpectedDiff: true,
Expand Down Expand Up @@ -381,8 +382,9 @@ releases:
})
}

t.Run("fail due to known field", func(t *testing.T) {
t.Run("fail due to unknown field with goccy/go-yaml", func(t *testing.T) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might also see this idiom, the same test is run with goccy/goyaml and with yaml.v2, so that we are sure that we don't break anything while migrating to goccy/goyaml or helmfile v1.

check(t, testcase{
goccyGoYaml: true,
error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1: [4:3] unknown field "foobar"
2 | releases:
3 | - name: app1
Expand All @@ -391,6 +393,14 @@ releases:
5 | chart: incubator/raw`,
})
})

t.Run("fail due to unknown field with gopkg.in/yaml.v2", func(t *testing.T) {
check(t, testcase{
goccyGoYaml: false,
error: `in ./helmfile.yaml: failed to read helmfile.yaml: reading document at index 1: yaml: unmarshal errors:
line 4: field foobar not found in type state.ReleaseSpec`,
})
})
}

func TestTemplate_CyclicInheritance(t *testing.T) {
Expand Down
27 changes: 23 additions & 4 deletions pkg/app/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"path/filepath"
"reflect"
"regexp"
"runtime"
goruntime "runtime"
"strings"
"sync"
"testing"
Expand All @@ -25,6 +25,7 @@ import (
ffs "github.com/helmfile/helmfile/pkg/filesystem"
"github.com/helmfile/helmfile/pkg/helmexec"
"github.com/helmfile/helmfile/pkg/remote"
"github.com/helmfile/helmfile/pkg/runtime"
"github.com/helmfile/helmfile/pkg/state"
"github.com/helmfile/helmfile/pkg/testhelper"
"github.com/helmfile/helmfile/pkg/testutil"
Expand Down Expand Up @@ -4338,7 +4339,15 @@ myrelease4 testNamespace true true id:myrelease1 mychart1
assert.Equal(t, expected, out)
}

func TestSetValuesTemplate(t *testing.T) {
func testSetValuesTemplate(t *testing.T, goccyGoYaml bool) {
t.Helper()

v := runtime.GoccyGoYaml
runtime.GoccyGoYaml = goccyGoYaml
t.Cleanup(func() {
runtime.GoccyGoYaml = v
})

files := map[string]string{
"/path/to/helmfile.yaml": `
releases:
Expand All @@ -4357,7 +4366,7 @@ releases:
`,
}
expectedValues := []interface{}{
map[interface{}]interface{}{"val1": "zipkin"},
map[string]interface{}{"val1": "zipkin"},
map[string]interface{}{"val2": "val2"}}
expectedSetValues := []state.SetValue{
{Name: "name-zipkin", Value: "val-zipkin"},
Expand Down Expand Up @@ -4402,7 +4411,17 @@ releases:
}
}

func TestSetValuesTemplate(t *testing.T) {
t.Run("with goccy/go-yaml", func(t *testing.T) {
testSetValuesTemplate(t, true)
})

t.Run("with gopkg.in/yaml.v2", func(t *testing.T) {
testSetValuesTemplate(t, false)
})
}

func location() string {
_, fn, line, _ := runtime.Caller(1)
_, fn, line, _ := goruntime.Caller(1)
return fmt.Sprintf("%s:%d", filepath.Base(fn), line)
}
2 changes: 2 additions & 0 deletions pkg/envvar/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ const (
Helm3 = "HELMFILE_HELM3"
UpgradeNoticeDisabled = "HELMFILE_UPGRADE_NOTICE_DISABLED"
V1Mode = "HELMFILE_V1MODE"
GoccyGoYaml = "HELMFILE_GOCCY_GOYAML"
CacheHome = "HELMFILE_CACHE_HOME"

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't strictly necessary to change our yaml library. However I added this because this helped tidy up our e2e/snapshot test to be runnable with/without v1 mode.
The envvar is used in pkg/remote/remote.go and set in snapshot_test.go

)
4 changes: 4 additions & 0 deletions pkg/remote/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ func init() {
}

func CacheDir() string {
if h := os.Getenv(envvar.CacheHome); h != "" {
return h
}

dir, err := os.UserCacheDir()
if err != nil {
// fall back to relative path with hidden directory
Expand Down
22 changes: 21 additions & 1 deletion pkg/runtime/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,23 @@ import (
var (
V1Mode bool

// GoccyGoYaml is set to true in order to let Helmfile use
// goccy/go-yaml instead of gopkg.in/yaml.v2.
// It's false by default in Helmfile v0.x and true by default for Helmfile v1.x.
GoccyGoYaml bool

// We set this via ldflags at build-time so that we can use the
// value specified at the build time as the runtime default.
v1Mode string
)

func Info() string {
return fmt.Sprintf("V1 mode = %v", V1Mode)
yamlLib := "gopkg.in/yaml.v2"
if GoccyGoYaml {
yamlLib = "goccy/go-yaml"
}

return fmt.Sprintf("V1 mode = %v\nYAML library = %v", V1Mode, yamlLib)
}

func init() {
Expand All @@ -34,4 +44,14 @@ func init() {
default:
V1Mode, _ = strconv.ParseBool(v1Mode)
}

// You can switch the YAML library at runtime via an envvar:
switch os.Getenv(envvar.GoccyGoYaml) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Although goccy/go-yaml is the default for v1, you can opt in/out in both v0 and v1, by setting this envvar.

case "true":
GoccyGoYaml = true
case "false":
GoccyGoYaml = false
default:
GoccyGoYaml = V1Mode
}
}
16 changes: 14 additions & 2 deletions pkg/state/release.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package state
import (
"fmt"

"github.com/helmfile/helmfile/pkg/maputil"
"github.com/helmfile/helmfile/pkg/tmpl"
"github.com/helmfile/helmfile/pkg/yaml"
)
Expand Down Expand Up @@ -107,13 +108,18 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, string(serialized), err)
}

var deserialized map[interface{}]interface{}
var deserialized map[string]interface{}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yaml.v2 deserializes nested yaml dict to map[interface{}]interface{} while yaml.v3 and goccy/go-yaml serializes it to map[string]interface{}.
My original design was to ensure any nested dict is deserialized to the latter, by calling maputil.CastKeysToStrings


if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil {
return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
}

result.ValuesTemplate[i] = deserialized
m, err := maputil.CastKeysToStrings(deserialized)
if err != nil {
return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err)
}

result.ValuesTemplate[i] = m
}
}

Expand All @@ -130,6 +136,12 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R
return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
}
result.Values[i] = s.String()
case map[interface{}]interface{}:

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

m, err := maputil.CastKeysToStrings(ts)
if err != nil {
return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err)
}
result.Values[i] = m
}
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/state/state_exec_tmpl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func TestHelmState_executeTemplates(t *testing.T) {
Verify: nil,
Name: "app",
Namespace: "dev",
Values: []interface{}{map[interface{}]interface{}{"key": "app-val0"}},
Values: []interface{}{map[string]interface{}{"key": "app-val0"}},

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

},
},
}
Expand Down
9 changes: 8 additions & 1 deletion pkg/tmpl/context_funcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/helmfile/helmfile/pkg/envvar"
"github.com/helmfile/helmfile/pkg/helmexec"
"github.com/helmfile/helmfile/pkg/maputil"
"github.com/helmfile/helmfile/pkg/yaml"
)

Expand Down Expand Up @@ -286,11 +287,17 @@ func ToYaml(v interface{}) (string, error) {
}

func FromYaml(str string) (Values, error) {
m := Values{}
m := map[string]interface{}{}

if err := yaml.Unmarshal([]byte(str), &m); err != nil {
return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
}

m, err := maputil.CastKeysToStrings(m)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/helmfile/helmfile/pull/609/files#r1058925709

This is especially important here because this function is supposed to be used like fromYaml | toJson within helmfile templates. Go's json pkg which powers toJson is unable to serialize map[interface{}]interface{} deserialized by yaml.v2. This CastKeysToStrings call normalizes it so that both yaml.v2 and goccy/go-yaml implementation of FromYaml is compatible with ToJson.

if err != nil {
return nil, fmt.Errorf("%s, offending yaml: %s", err, str)
}

return m, nil
}

Expand Down
Loading