Skip to content

Commit 6a3f31b

Browse files
committed
add init-period flag for faster initial sync retries
1 parent 7ffab78 commit 6a3f31b

2 files changed

Lines changed: 151 additions & 2 deletions

File tree

‎main.go‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,12 @@ func main() {
186186
flPeriod := pflag.Duration("period",
187187
envDuration(10*time.Second, "GITSYNC_PERIOD", "GIT_SYNC_PERIOD"),
188188
"how long to wait between syncs, must be >= 10ms; --wait overrides this")
189+
flInitPeriod := pflag.Duration("init-period",
190+
envDuration(0, "GITSYNC_INIT_PERIOD"),
191+
"how long to wait between sync attempts until the first success, must be >= 10ms if set; if unset, --period is used")
192+
flInitTimeout := pflag.Duration("init-timeout",
193+
envDuration(0, "GITSYNC_INIT_TIMEOUT"),
194+
"the max time allowed for the initial sync to succeed; if unset, there is no timeout (retries forever until --max-failures)")
189195
flSyncTimeout := pflag.Duration("sync-timeout",
190196
envDuration(120*time.Second, "GITSYNC_SYNC_TIMEOUT", "GIT_SYNC_SYNC_TIMEOUT"),
191197
"the total time allowed for one complete sync, must be >= 10ms; --timeout overrides this")
@@ -469,6 +475,12 @@ func main() {
469475
if *flPeriod < 10*time.Millisecond {
470476
fatalConfigErrorf(log, true, "invalid flag: --period must be at least 10ms")
471477
}
478+
if *flInitPeriod != 0 && *flInitPeriod < 10*time.Millisecond {
479+
fatalConfigErrorf(log, true, "invalid flag: --init-period must be at least 10ms")
480+
}
481+
if *flInitTimeout != 0 && *flInitTimeout < 10*time.Millisecond {
482+
fatalConfigErrorf(log, true, "invalid flag: --init-timeout must be at least 10ms")
483+
}
472484

473485
if *flDeprecatedChmod != 0 {
474486
fatalConfigErrorf(log, true, "deprecated flag: --change-permissions is no longer supported")
@@ -912,20 +924,35 @@ func main() {
912924

913925
failCount := 0
914926
syncCount := uint64(0)
927+
initialSyncDone := false
928+
var initStart time.Time
929+
if *flInitPeriod != 0 || *flInitTimeout != 0 {
930+
initStart = time.Now()
931+
}
915932
for {
916933
start := time.Now()
917934
ctx, cancel := context.WithTimeout(context.Background(), *flSyncTimeout)
918935

919936
if changed, hash, err := git.SyncRepo(ctx, refreshCreds); err != nil {
920937
failCount++
921938
updateSyncMetrics(metricKeyError, start)
939+
if isInitTimedOut(*flInitTimeout, initialSyncDone, initStart) {
940+
log.Error(err, "initial sync timed out", "initTimeout", flInitTimeout.String(), "failCount", failCount)
941+
os.Exit(1)
942+
}
922943
if *flMaxFailures >= 0 && failCount >= *flMaxFailures {
923944
// Exit after too many retries, maybe the error is not recoverable.
924945
log.Error(err, "too many failures, aborting", "failCount", failCount)
925946
os.Exit(1)
926947
}
927948
log.Error(err, "error syncing repo, will retry", "failCount", failCount)
928949
} else {
950+
if !initialSyncDone {
951+
initialSyncDone = true
952+
if *flInitPeriod != 0 {
953+
log.V(0).Info("initial sync complete, switching to normal period", "initPeriod", flInitPeriod.String(), "period", flPeriod.String())
954+
}
955+
}
929956
// this might have been called before, but also might not have
930957
setRepoReady()
931958
// We treat the first loop as a sync, including sending hooks.
@@ -989,12 +1016,14 @@ func main() {
9891016
log.DeleteErrorFile()
9901017
}
9911018

992-
log.V(3).Info("next sync", "waitTime", flPeriod.String(), "syncCount", syncCount)
1019+
// Use init-period for retries before the first successful sync.
1020+
waitTime := chooseWaitTime(*flPeriod, *flInitPeriod, initialSyncDone)
1021+
log.V(3).Info("next sync", "waitTime", waitTime.String(), "syncCount", syncCount)
9931022
cancel()
9941023

9951024
// Sleep until the next sync. If syncSig is set then the sleep may
9961025
// be interrupted by that signal.
997-
t := time.NewTimer(*flPeriod)
1026+
t := time.NewTimer(waitTime)
9981027
select {
9991028
case <-t.C:
10001029
case <-sigChan:
@@ -1144,6 +1173,24 @@ func setRepoReady() {
11441173
repoReady = true
11451174
}
11461175

1176+
// chooseWaitTime returns the appropriate wait duration based on whether the
1177+
// initial sync has completed. If initPeriod is non-zero and the initial sync
1178+
// is not yet done, it returns initPeriod. Otherwise it returns period.
1179+
func chooseWaitTime(period, initPeriod time.Duration, initialSyncDone bool) time.Duration {
1180+
if !initialSyncDone && initPeriod != 0 {
1181+
return initPeriod
1182+
}
1183+
return period
1184+
}
1185+
1186+
// isInitTimedOut returns true if the initial sync has exceeded the init-timeout.
1187+
func isInitTimedOut(initTimeout time.Duration, initialSyncDone bool, initStart time.Time) bool {
1188+
if initTimeout == 0 || initialSyncDone {
1189+
return false
1190+
}
1191+
return time.Since(initStart) >= initTimeout
1192+
}
1193+
11471194
// Do no work, but don't do something that triggers go's runtime into thinking
11481195
// it is deadlocked.
11491196
func sleepForever() {
@@ -2589,6 +2636,18 @@ OPTIONS
25892636
dynamic credentials from an external secrets system).
25902637
See also $GITSYNC_PASSWORD.
25912638
2639+
--init-period <duration>, $GITSYNC_INIT_PERIOD
2640+
How long to wait between sync attempts until the first successful
2641+
sync. Once the initial sync succeeds, --period is used instead.
2642+
This must be at least 10ms if set. If not specified, --period is
2643+
used for all sync attempts.
2644+
2645+
--init-timeout <duration>, $GITSYNC_INIT_TIMEOUT
2646+
The maximum amount of time to keep retrying the initial sync
2647+
before failing. If not specified, git-sync retries forever (or
2648+
until --max-failures is reached). This must be at least 10ms if
2649+
set.
2650+
25922651
--period <duration>, $GITSYNC_PERIOD
25932652
How long to wait between sync attempts. This must be at least
25942653
10ms. This flag obsoletes --wait, but if --wait is specified, it

‎main_test.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,96 @@ func TestTouch(t *testing.T) {
425425
}
426426
}
427427

428+
func TestChooseWaitTime(t *testing.T) {
429+
period := 10 * time.Second
430+
initPeriod := 500 * time.Millisecond
431+
432+
cases := []struct {
433+
name string
434+
period time.Duration
435+
initPeriod time.Duration
436+
initialSyncDone bool
437+
expected time.Duration
438+
}{{
439+
name: "no init-period, not done",
440+
period: period,
441+
initPeriod: 0,
442+
initialSyncDone: false,
443+
expected: period,
444+
}, {
445+
name: "no init-period, done",
446+
period: period,
447+
initPeriod: 0,
448+
initialSyncDone: true,
449+
expected: period,
450+
}, {
451+
name: "init-period set, not done",
452+
period: period,
453+
initPeriod: initPeriod,
454+
initialSyncDone: false,
455+
expected: initPeriod,
456+
}, {
457+
name: "init-period set, done",
458+
period: period,
459+
initPeriod: initPeriod,
460+
initialSyncDone: true,
461+
expected: period,
462+
}}
463+
464+
for _, tc := range cases {
465+
t.Run(tc.name, func(t *testing.T) {
466+
got := chooseWaitTime(tc.period, tc.initPeriod, tc.initialSyncDone)
467+
if got != tc.expected {
468+
t.Errorf("expected %v, got %v", tc.expected, got)
469+
}
470+
})
471+
}
472+
}
473+
474+
func TestIsInitTimedOut(t *testing.T) {
475+
cases := []struct {
476+
name string
477+
initTimeout time.Duration
478+
initialSyncDone bool
479+
elapsed time.Duration
480+
expected bool
481+
}{{
482+
name: "no timeout set",
483+
initTimeout: 0,
484+
initialSyncDone: false,
485+
elapsed: time.Hour,
486+
expected: false,
487+
}, {
488+
name: "timeout set, already done",
489+
initTimeout: time.Second,
490+
initialSyncDone: true,
491+
elapsed: time.Hour,
492+
expected: false,
493+
}, {
494+
name: "timeout set, not done, not expired",
495+
initTimeout: 10 * time.Second,
496+
initialSyncDone: false,
497+
elapsed: time.Second,
498+
expected: false,
499+
}, {
500+
name: "timeout set, not done, expired",
501+
initTimeout: time.Second,
502+
initialSyncDone: false,
503+
elapsed: 2 * time.Second,
504+
expected: true,
505+
}}
506+
507+
for _, tc := range cases {
508+
t.Run(tc.name, func(t *testing.T) {
509+
initStart := time.Now().Add(-tc.elapsed)
510+
got := isInitTimedOut(tc.initTimeout, tc.initialSyncDone, initStart)
511+
if got != tc.expected {
512+
t.Errorf("expected %v, got %v", tc.expected, got)
513+
}
514+
})
515+
}
516+
}
517+
428518
func TestHasGitLockFile(t *testing.T) {
429519
testCases := map[string]struct {
430520
inputFilePath []string

0 commit comments

Comments
 (0)