Skip to content

Commit cb935bf

Browse files
fuweidqiutongs
authored andcommitted
pkg/failpoint: init failpoint package
Failpoint is used to control the fail during API call when testing, especially the API is complicated like CRI-RunPodSandbox. It can help us to test the unexpected behavior without mock. The control design is based on freebsd fail(9), but simpler. REF: https://www.freebsd.org/cgi/man.cgi?query=fail&sektion=9&apropos=0&manpath=FreeBSD%2B10.0-RELEASE Signed-off-by: Wei Fu <[email protected]> (cherry picked from commit ffd59ba) Signed-off-by: Qiutong Song <[email protected]>
1 parent 2cbb524 commit cb935bf

2 files changed

Lines changed: 427 additions & 0 deletions

File tree

pkg/failpoint/fail.go

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package failpoint
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
"strconv"
23+
"strings"
24+
"sync"
25+
"time"
26+
)
27+
28+
// Type is the type of failpoint to specifies which action to take.
29+
type Type int
30+
31+
const (
32+
// TypeInvalid is invalid type
33+
TypeInvalid Type = iota
34+
// TypeOff takes no action
35+
TypeOff
36+
// TypeError triggers failpoint error with specified argument
37+
TypeError
38+
// TypePanic triggers panic with specified argument
39+
TypePanic
40+
// TypeDelay sleeps with the specified number of milliseconds
41+
TypeDelay
42+
)
43+
44+
// String returns the name of type.
45+
func (t Type) String() string {
46+
switch t {
47+
case TypeOff:
48+
return "off"
49+
case TypeError:
50+
return "error"
51+
case TypePanic:
52+
return "panic"
53+
case TypeDelay:
54+
return "delay"
55+
default:
56+
return "invalid"
57+
}
58+
}
59+
60+
// Failpoint is used to add code points where error or panic may be injected by
61+
// user. The user controlled variable will be parsed for how the error injected
62+
// code should fire. There is the way to set the rule for failpoint.
63+
//
64+
// <count>*<type>[(arg)][-><more terms>]
65+
//
66+
// The <type> argument specifies which action to take; it can be one of:
67+
//
68+
// off: Takes no action (does not trigger failpoint and no argument)
69+
// error: Triggers failpoint error with specified argument(string)
70+
// panic: Triggers panic with specified argument(string)
71+
// delay: Sleep the specified number of milliseconds
72+
//
73+
// The <count>* modifiers prior to <type> control when <type> is executed. For
74+
// example, "5*error(oops)" means "return error oops 5 times total". The
75+
// operator -> can be used to express cascading terms. If you specify
76+
// <term1>-><term2>, it means that if <term1> does not execute, <term2> will
77+
// be evaluated. If you want the error injected code should fire in second
78+
// call, you can specify "1*off->1*error(oops)".
79+
//
80+
// Based on fail(9) freebsd: https://www.freebsd.org/cgi/man.cgi?query=fail&sektion=9&apropos=0&manpath=FreeBSD%2B10.0-RELEASE
81+
type Failpoint struct {
82+
sync.Mutex
83+
84+
fnName string
85+
entries []*failpointEntry
86+
}
87+
88+
// NewFailpoint returns failpoint control.
89+
func NewFailpoint(fnName string, terms string) (*Failpoint, error) {
90+
entries, err := parseTerms([]byte(terms))
91+
if err != nil {
92+
return nil, err
93+
}
94+
95+
return &Failpoint{
96+
fnName: fnName,
97+
entries: entries,
98+
}, nil
99+
}
100+
101+
// Evaluate evaluates a failpoint.
102+
func (fp *Failpoint) Evaluate() error {
103+
var target *failpointEntry
104+
105+
func() {
106+
fp.Lock()
107+
defer fp.Unlock()
108+
109+
for _, entry := range fp.entries {
110+
if entry.count == 0 {
111+
continue
112+
}
113+
114+
entry.count--
115+
target = entry
116+
break
117+
}
118+
}()
119+
120+
if target == nil {
121+
return nil
122+
}
123+
return target.evaluate()
124+
}
125+
126+
// Failpoint returns the current state of control in string format.
127+
func (fp *Failpoint) Marshal() string {
128+
fp.Lock()
129+
defer fp.Unlock()
130+
131+
res := make([]string, 0, len(fp.entries))
132+
for _, entry := range fp.entries {
133+
res = append(res, entry.marshal())
134+
}
135+
return strings.Join(res, "->")
136+
}
137+
138+
type failpointEntry struct {
139+
typ Type
140+
arg interface{}
141+
count int64
142+
}
143+
144+
func newFailpointEntry() *failpointEntry {
145+
return &failpointEntry{
146+
typ: TypeInvalid,
147+
count: 0,
148+
}
149+
}
150+
151+
func (fpe *failpointEntry) marshal() string {
152+
base := fmt.Sprintf("%d*%s", fpe.count, fpe.typ)
153+
switch fpe.typ {
154+
case TypeOff:
155+
return base
156+
case TypeError, TypePanic:
157+
return fmt.Sprintf("%s(%s)", base, fpe.arg.(string))
158+
case TypeDelay:
159+
return fmt.Sprintf("%s(%d)", base, fpe.arg.(time.Duration)/time.Millisecond)
160+
default:
161+
return base
162+
}
163+
}
164+
165+
func (fpe *failpointEntry) evaluate() error {
166+
switch fpe.typ {
167+
case TypeOff:
168+
return nil
169+
case TypeError:
170+
return fmt.Errorf("%v", fpe.arg)
171+
case TypePanic:
172+
panic(fpe.arg)
173+
case TypeDelay:
174+
time.Sleep(fpe.arg.(time.Duration))
175+
return nil
176+
default:
177+
panic("invalid failpoint type")
178+
}
179+
}
180+
181+
func parseTerms(term []byte) ([]*failpointEntry, error) {
182+
var entry *failpointEntry
183+
var err error
184+
185+
// count*type[(arg)]
186+
term, entry, err = parseTerm(term)
187+
if err != nil {
188+
return nil, err
189+
}
190+
191+
res := []*failpointEntry{entry}
192+
193+
// cascading terms
194+
for len(term) > 0 {
195+
if !bytes.HasPrefix(term, []byte("->")) {
196+
return nil, fmt.Errorf("invalid cascading terms: %s", string(term))
197+
}
198+
199+
term = term[2:]
200+
term, entry, err = parseTerm(term)
201+
if err != nil {
202+
return nil, fmt.Errorf("failed to parse cascading term: %w", err)
203+
}
204+
205+
res = append(res, entry)
206+
}
207+
return res, nil
208+
}
209+
210+
func parseTerm(term []byte) ([]byte, *failpointEntry, error) {
211+
var err error
212+
var entry = newFailpointEntry()
213+
214+
// count*
215+
term, err = parseInt64(term, '*', &entry.count)
216+
if err != nil {
217+
return nil, nil, err
218+
}
219+
220+
// type[(arg)]
221+
term, err = parseType(term, entry)
222+
return term, entry, err
223+
}
224+
225+
func parseType(term []byte, entry *failpointEntry) ([]byte, error) {
226+
var nameToTyp = map[string]Type{
227+
"off": TypeOff,
228+
"error(": TypeError,
229+
"panic(": TypePanic,
230+
"delay(": TypeDelay,
231+
}
232+
233+
var found bool
234+
for name, typ := range nameToTyp {
235+
if bytes.HasPrefix(term, []byte(name)) {
236+
found = true
237+
term = term[len(name):]
238+
entry.typ = typ
239+
break
240+
}
241+
}
242+
243+
if !found {
244+
return nil, fmt.Errorf("invalid type format: %s", string(term))
245+
}
246+
247+
switch entry.typ {
248+
case TypePanic, TypeError:
249+
endIdx := bytes.IndexByte(term, ')')
250+
if endIdx <= 0 {
251+
return nil, fmt.Errorf("invalid argument for %s type", entry.typ)
252+
}
253+
entry.arg = string(term[:endIdx])
254+
return term[endIdx+1:], nil
255+
case TypeOff:
256+
// do nothing
257+
return term, nil
258+
case TypeDelay:
259+
var msVal int64
260+
var err error
261+
262+
term, err = parseInt64(term, ')', &msVal)
263+
if err != nil {
264+
return nil, err
265+
}
266+
entry.arg = time.Millisecond * time.Duration(msVal)
267+
return term, nil
268+
default:
269+
panic("unreachable")
270+
}
271+
}
272+
273+
func parseInt64(term []byte, terminate byte, val *int64) ([]byte, error) {
274+
i := 0
275+
276+
for ; i < len(term); i++ {
277+
if b := term[i]; b < '0' || b > '9' {
278+
break
279+
}
280+
}
281+
282+
if i == 0 || i == len(term) || term[i] != terminate {
283+
return nil, fmt.Errorf("failed to parse int64 because of invalid terminate byte: %s", string(term))
284+
}
285+
286+
v, err := strconv.ParseInt(string(term[:i]), 10, 64)
287+
if err != nil {
288+
return nil, fmt.Errorf("failed to parse int64 from %s: %v", string(term[:i]), err)
289+
}
290+
291+
*val = v
292+
return term[i+1:], nil
293+
}

0 commit comments

Comments
 (0)