Skip to content

Commit d4b3b13

Browse files
jondotjondot
committed
implement rolling updates workflows
1 parent 9b6b33a commit d4b3b13

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+5830
-74
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ preflight
44
.vscode/
55
node_modules/
66
coverage.out
7+
todo.txt

‎README.md‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,13 @@ $ brew tap spectralops/tap && brew install preflight
4949
$ curl -L https://XXX | preflight run sha256=1ce...2244a6e86
5050
⌛️ Preflight starting
5151
❌ Preflight failed:
52-
Digest does not match.
53-
Expected: <...>
54-
Actual: <...>
52+
Digest does not match.
53+
54+
Expected:
55+
<...>
56+
57+
Actual:
58+
<...>
5559
5660
Information:
5761
It is recommended to inspect the modified file contents.
@@ -117,6 +121,35 @@ steps:
117121
```
118122
119123
----
124+
125+
## :bulb: Dealing with changing runnables & auto updates
126+
127+
When updating an old binary or script to a new updated version, there will be at least two (2) valid digests "live" and just replacing the single digest used will fail for the older runnable which may still be running somewhere.
128+
129+
```
130+
$ preflight <hash list|https://url/to/hash-list>
131+
```
132+
133+
To support updates and rolling/auto updates of scripts and binaries we basically need to validate against `<old hash>` + `<new hash>` at all times, until everyone upgrades to the new script. Preflight validates against a `list of hashes` or better, give it a _live_ URL of `valid hashes` and it will validate against it.
134+
135+
136+
```
137+
curl .. | ./ci/preflight run sha256=d6aa3207c4908d123bd8af62ec0538e3f2b9f257c3de62fad4e29cd3b59b41d9,sha256=<new hash>,...
138+
```
139+
140+
Or to a live URL:
141+
```
142+
curl .. | ./ci/preflight run https://dl.example.com/hashes.txt
143+
```
144+
145+
146+
Use this when:
147+
148+
* Use multiple digests verbatim, when your runnables change often, but not too often
149+
* Use a URL when your runnables change often. Remember to follow the chain of trust. This will now mean that:
150+
* Your hash list URL is now a source of trust
151+
* Visually: we're swapping the chain of trust like so `curl <foreign trust> | ./ci/preflight <own trust>`
152+
120153
## :running: Running scripts and binaries
121154

122155
**Piping:**

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ require (
1212
github.com/pkg/errors v0.9.1 // indirect
1313
github.com/sergi/go-diff v1.2.0 // indirect
1414
github.com/stretchr/testify v1.6.1 // indirect
15+
github.com/thoas/go-funk v0.8.0
1516
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 // indirect
1617
)

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
3333
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
3434
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
3535
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
36+
github.com/thoas/go-funk v0.8.0 h1:JP9tKSvnpFVclYgDM0Is7FD9M4fhPvqA0s0BsXmzSRQ=
37+
github.com/thoas/go-funk v0.8.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
3638
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
3739
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
3840
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 h1:V066+OYJ66oTjnhm4Yrn7SXIwSCiDQJxpBxmvqb1N1c=

‎main.go‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ import (
1212

1313
var CLI struct {
1414
Run struct {
15-
Hash string `arg name:"hash" help:"Hash to verify. Format: sha256=<hash>"`
15+
Hash string `arg name:"hash|url" help:"Hash to verify. You can provide a list seperated by a comma (,) and no space. Format: sha256=<hash>[,sha256=<hash2>,...], Or a URL to a flat file to fetch with a list of hashes, one per line. Format: https://example.com/file.txt"`
1616
Cmd []string `arg optional name:"cmd" help:"Command to execute"`
1717
} `cmd help:"Verify and run a command"`
1818

1919
Check struct {
20-
Hash string `arg name:"hash" help:"Hash to verify. Format: sha256=<hash>"`
20+
Hash string `arg name:"hash|url" help:"Hash to verify. You can provide a list seperated by a comma (,) and no space. Format: sha256=<hash>[,sha256=<hash2>,...], Or a URL to a flat file to fetch with a list of hashes, one per line. Format: https://example.com/file.txt"`
2121
Cmd []string `arg optional name:"cmd" help:"Command to execute"`
2222
} `cmd help:"Verify a command"`
2323

@@ -54,7 +54,7 @@ func main() {
5454
preflight := pkg.NewPreflight(lookup)
5555

5656
switch ctx.Command() {
57-
case "run <hash>":
57+
case "run <hash|url>":
5858
// piping
5959
var fin io.Reader = os.Stdin
6060
s, err := ioutil.ReadAll(fin)
@@ -68,13 +68,13 @@ func main() {
6868
os.Exit(1)
6969
}
7070

71-
case "run <hash> <cmd>":
71+
case "run <hash|url> <cmd>":
7272
err := preflight.Exec(CLI.Run.Cmd, CLI.Run.Hash)
7373
if err != nil {
7474
os.Exit(1)
7575
}
7676

77-
case "check <hash>":
77+
case "check <hash|url>":
7878
// piping
7979
var fin io.Reader = os.Stdin
8080
s, err := ioutil.ReadAll(fin)
@@ -84,21 +84,30 @@ func main() {
8484
}
8585

8686
content := string(s)
87-
res := preflight.Check(content, CLI.Check.Hash)
87+
res, err := preflight.Check(content, CLI.Check.Hash)
88+
if err != nil {
89+
fmt.Printf("Error: %v\n", err)
90+
os.Exit(1)
91+
}
8892
if !res.Ok {
8993
preflight.Porcelain.CheckFailed(res)
9094
os.Exit(1)
9195
}
9296
fmt.Print(content) // give back so piping can continue
9397

94-
case "check <hash> <cmd>":
98+
case "check <hash|url> <cmd>":
9599
s, err := ioutil.ReadFile(CLI.Check.Cmd[0])
96100
if err != nil {
97101
fmt.Printf("cannot open %v: %v", CLI.Check.Cmd[0], err)
98102
os.Exit(1)
99103
}
100104

101-
res := preflight.Check(string(s), CLI.Check.Hash)
105+
res, err := preflight.Check(string(s), CLI.Check.Hash)
106+
if err != nil {
107+
fmt.Printf("Error: %v\n", err)
108+
os.Exit(1)
109+
}
110+
102111
if !res.Ok {
103112
preflight.Porcelain.CheckFailed(res)
104113
os.Exit(1)
@@ -115,8 +124,13 @@ func main() {
115124
CLI.Create.Digest = "sha256"
116125
}
117126

118-
res := preflight.Check(string(s), fmt.Sprintf("%v=?", CLI.Create.Digest))
119-
if res.Lookup.Vulnerable {
127+
res, err := preflight.Check(string(s), fmt.Sprintf("%v=?", CLI.Create.Digest))
128+
if err != nil {
129+
fmt.Printf("Error: %v\n", err)
130+
os.Exit(1)
131+
}
132+
133+
if res.HasLookupVulns() {
120134
preflight.Porcelain.CheckFailed(res)
121135
os.Exit(1)
122136
}
@@ -133,8 +147,9 @@ func main() {
133147
CLI.Create.Digest = "sha256"
134148
}
135149

136-
res := preflight.Check(string(s), fmt.Sprintf("%v=?", CLI.Create.Digest))
137-
if res.Lookup.Vulnerable {
150+
res, err := preflight.Check(string(s), fmt.Sprintf("%v=?", CLI.Create.Digest))
151+
152+
if res.HasLookupVulns() {
138153
preflight.Porcelain.CheckFailed(res)
139154
os.Exit(1)
140155
}

‎pkg/core.go‎

Lines changed: 117 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ package pkg
44
import (
55
//nolint
66
"crypto/md5"
7+
"errors"
8+
"net/http"
9+
710
//nolint
811
"crypto/sha1"
912
"crypto/sha256"
@@ -12,6 +15,8 @@ import (
1215
"os"
1316
"os/exec"
1417
"strings"
18+
19+
"github.com/thoas/go-funk"
1520
)
1621

1722
type Digest struct {
@@ -20,7 +25,7 @@ type Digest struct {
2025
MD5 string
2126
}
2227

23-
func (d *Digest) Verify(s Signature) (ok bool, content string) {
28+
func (d *Digest) Verify(s Signature) (ok bool, expectedHash string) {
2429
switch s.digest {
2530
case "sha1":
2631
return d.SHA1 == s.content, d.SHA1
@@ -30,27 +35,47 @@ func (d *Digest) Verify(s Signature) (ok bool, content string) {
3035
return d.SHA256 == s.content, d.SHA256
3136
}
3237
}
38+
func (d *Digest) String() string {
39+
return fmt.Sprintf("sha256=%v\nOR: sha1=%v\nOR: md5=%v", d.SHA256, d.SHA1, d.MD5)
40+
}
3341

3442
type Signature struct {
3543
content string
3644
digest string
3745
}
3846

47+
func (s *Signature) String() string {
48+
return fmt.Sprintf("%v=%v", s.digest, s.content)
49+
}
50+
3951
type CheckResult struct {
40-
Lookup LookupResult
41-
ExpectedDigest string
42-
ActualDigest string
43-
Ok bool
52+
LookupResult *LookupResult
53+
ValidDigest *Signature
54+
ExpectedDigests []Signature
55+
ActualDigest Digest
56+
Ok bool
4457
}
4558

4659
func (c *CheckResult) Error() error {
4760
if c.Ok {
4861
return nil
4962
}
50-
if c.Lookup.Vulnerable {
63+
64+
if c.HasLookupVulns() {
5165
return fmt.Errorf("vulnerable digest: %v", c.ActualDigest)
66+
} else if c.HasValidationVulns() {
67+
return fmt.Errorf("digest mismatch: actual: %v, expected: %v", c.ActualDigest, c.ExpectedDigests)
68+
} else {
69+
return errors.New("unknown result error")
5270
}
53-
return fmt.Errorf("digest mismatch: actual: %v, expected: %v", c.ActualDigest, c.ExpectedDigest)
71+
}
72+
73+
func (c *CheckResult) HasValidationVulns() bool {
74+
return c.ValidDigest == nil && len(c.ExpectedDigests) > 0
75+
}
76+
77+
func (c *CheckResult) HasLookupVulns() bool {
78+
return c.LookupResult != nil && c.LookupResult.Vulnerable
5479
}
5580

5681
type Lookup interface {
@@ -77,21 +102,29 @@ func GetLookup() (Lookup, error) {
77102
return &NoLookup{}, nil
78103
}
79104

80-
func digestTuple(s, sig string) (Digest, Signature) {
81-
parts := strings.Split(sig, "=")
82-
signature := Signature{content: sig, digest: "sha256"}
83-
if len(parts) > 1 {
84-
signature = Signature{content: parts[1], digest: strings.ToLower(parts[0])}
85-
}
86-
87-
digest := Digest{
105+
func createDigest(s string) Digest {
106+
return Digest{
88107
//nolint
89108
SHA1: fmt.Sprintf("%x", sha1.Sum([]byte(s))),
90109
//nolint
91110
MD5: fmt.Sprintf("%x", md5.Sum([]byte(s))),
92111
SHA256: fmt.Sprintf("%x", sha256.Sum256([]byte(s))),
93112
}
113+
}
94114

115+
func createSignature(sig string) Signature {
116+
parts := strings.Split(sig, "=")
117+
signature := Signature{content: sig, digest: "sha256"}
118+
if len(parts) > 1 {
119+
signature = Signature{content: parts[1], digest: strings.ToLower(parts[0])}
120+
}
121+
122+
return signature
123+
}
124+
125+
func digestTuple(s, sig string) (Digest, Signature) {
126+
signature := createSignature(sig)
127+
digest := createDigest(s)
95128
return digest, signature
96129
}
97130

@@ -102,22 +135,54 @@ func NewPreflight(lookup Lookup) *Preflight {
102135
}
103136
}
104137

105-
func (a *Preflight) Check(script, sig string) CheckResult {
106-
digest, signature := digestTuple(script, sig)
107-
lookup := a.Lookup.Hash(digest)
108-
verifyOk, actual := digest.Verify(signature)
109-
return CheckResult{
110-
Lookup: lookup,
111-
ActualDigest: actual,
112-
ExpectedDigest: signature.content,
113-
Ok: verifyOk && !lookup.Vulnerable,
138+
func (a *Preflight) Check(script, siglist string) (*CheckResult, error) {
139+
sigs, err := parsehashList(siglist)
140+
if err != nil {
141+
return nil, err
114142
}
143+
digest := createDigest(script)
144+
145+
res := funk.Find(sigs, func(s Signature) bool {
146+
ok, _ := digest.Verify(s)
147+
return ok
148+
})
149+
if res == nil {
150+
return &CheckResult{
151+
ExpectedDigests: sigs,
152+
ActualDigest: digest,
153+
ValidDigest: nil,
154+
Ok: false,
155+
}, nil
156+
}
157+
158+
validDigest := res.(Signature)
159+
// parseHashlist(sig) -> []Signature
160+
// check should return err, wired by parseHashlist + hash lookup
161+
// XXX untangle the digest tuple:
162+
// createDigest
163+
// parseSignature, accept sig []string
164+
// "verify" a signature
165+
// "validate" a hash
166+
// 0. get digest object
167+
// 1. parse all digs, then verify them. if one passes, we're OK
168+
// 2. next, the one that passes we want to lookup
169+
lookup := a.Lookup.Hash(digest)
170+
return &CheckResult{
171+
ExpectedDigests: sigs,
172+
ActualDigest: digest,
173+
ValidDigest: &validDigest,
174+
LookupResult: &lookup,
175+
Ok: !lookup.Vulnerable,
176+
}, nil
115177
}
116178

117179
// XXX: windows/powershell needs a different function
118180
func (a *Preflight) ExecPiped(script, sig string) error {
119181
a.Porcelain.Start(a)
120-
check := a.Check(script, sig)
182+
check, err := a.Check(script, sig)
183+
if err != nil {
184+
return err
185+
}
121186
if !check.Ok {
122187
a.Porcelain.CheckFailed(check)
123188
return check.Error()
@@ -137,7 +202,11 @@ func (a *Preflight) Exec(args []string, sig string) error {
137202
if err != nil {
138203
return fmt.Errorf("cannot open %v: %v", args[0], err)
139204
}
140-
check := a.Check(string(s), sig)
205+
check, err := a.Check(string(s), sig)
206+
if err != nil {
207+
return err
208+
}
209+
141210
if !check.Ok {
142211
a.Porcelain.CheckFailed(check)
143212
return check.Error()
@@ -152,3 +221,25 @@ func (a *Preflight) Exec(args []string, sig string) error {
152221
command.Stderr = os.Stderr
153222
return command.Run()
154223
}
224+
225+
func parsehashList(hashArg string) ([]Signature, error) {
226+
var sigs []string
227+
if strings.Contains(hashArg, ",") {
228+
sigs = strings.Split(hashArg, ",")
229+
} else if strings.HasPrefix(hashArg, "http") {
230+
resp, err := http.Get(hashArg) //nolint
231+
if err != nil {
232+
return nil, fmt.Errorf("cannot parse hash URL: %v", err)
233+
}
234+
defer resp.Body.Close()
235+
res, err := ioutil.ReadAll(resp.Body)
236+
if err != nil {
237+
return nil, fmt.Errorf("cannot read hash URL content: %v", err)
238+
}
239+
sigs = strings.Split(string(res), "\n")
240+
} else {
241+
sigs = []string{hashArg}
242+
}
243+
244+
return funk.Map(sigs, createSignature).([]Signature), nil
245+
}

0 commit comments

Comments
 (0)