Skip to content

Commit 581e671

Browse files
Renato CostaDarrylWong
Renato Costa
authored and
DarrylWong
committed
roachtest: randomly run with runtime assertions by default
This changes the semantics of `t.Cockroach()` to use a binary with runtime assertions and metamorphic constants enabled by default. Performance tests (indicated by the benchmark TestSpec) will continue using the standard binary, without runtime assertions, as usual. This commit also opts-out other tests that cannot easily be run with runtime assertions or metamorphic constants enabled, most often due to timeouts and metamorphic constant incompatibility. Resolves #86678. Informs #94986. Epic: none Release note: None
1 parent b22b3f1 commit 581e671

31 files changed

+311
-84
lines changed

‎build/teamcity/cockroach/nightlies/roachtest_compile_component.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,13 @@ case "$component" in
8383
;;
8484
roachtest)
8585
# Roachtest binary.
86-
bazel_args=(//pkg/cmd/roachtest)
86+
# N.B. We always compile the roachtest binary with crdb_test so the same serialization
87+
# on the wire is established with cockroach binaries built under crdb_test.
88+
# E.g. KVNemesisSeq is used only under crdb_test builds. If the cockroach binary is
89+
# built with crdb_test, it will expect this field to be sent by the roachtest runner.
90+
# Note that the opposite is not true, and a cockroach binary built without crdb_test
91+
# is still compatible with a roachtest binary built with it.
92+
bazel_args=(//pkg/cmd/roachtest --crdb_test)
8793
artifacts=("pkg/cmd/roachtest/roachtest_/roachtest:bin/roachtest.$os-$arch")
8894
;;
8995
*)

‎build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
77
source "$dir/teamcity-support.sh" # For $root
88
source "$dir/teamcity-bazel-support.sh" # For run_bazel
99

10-
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY" \
10+
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED" \
1111
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

‎build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
77
source "$dir/teamcity-support.sh" # For $root
88
source "$dir/teamcity-bazel-support.sh" # For run_bazel
99

10-
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY" \
10+
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED" \
1111
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

‎build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
77
source "$dir/teamcity-support.sh" # For $root
88
source "$dir/teamcity-bazel-support.sh" # For run_bazel
99

10-
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY" \
10+
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED" \
1111
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

‎pkg/cmd/roachtest/cluster.go‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"fmt"
1818
"io"
1919
"io/fs"
20+
"math/rand"
2021
"net"
2122
"net/url"
2223
"os"
@@ -689,8 +690,13 @@ type clusterImpl struct {
689690
// BAZEL_COVER_DIR will be set to this value when starting a node.
690691
goCoverDir string
691692

692-
os string // OS of the cluster
693-
arch vm.CPUArch // CPU architecture of the cluster
693+
os string // OS of the cluster
694+
arch vm.CPUArch // CPU architecture of the cluster
695+
randomSeed struct {
696+
mu syncutil.Mutex
697+
seed *int64
698+
}
699+
694700
// destroyState contains state related to the cluster's destruction.
695701
destroyState destroyState
696702
}
@@ -2003,10 +2009,16 @@ func (c *clusterImpl) StartE(
20032009

20042010
startOpts.RoachprodOpts.EncryptedStores = c.encAtRest
20052011

2006-
if !envExists(settings.Env, "COCKROACH_CRASH_ON_SPAN_USE_AFTER_FINISH") {
2007-
// Panic on span use-after-Finish, so we catch such bugs.
2008-
settings.Env = append(settings.Env, "COCKROACH_CRASH_ON_SPAN_USE_AFTER_FINISH=true")
2012+
setUnlessExists := func(name string, value interface{}) {
2013+
if !envExists(settings.Env, name) {
2014+
settings.Env = append(settings.Env, fmt.Sprintf("%s=%s", name, fmt.Sprint(value)))
2015+
}
20092016
}
2017+
// Panic on span use-after-Finish, so we catch such bugs.
2018+
setUnlessExists("COCKROACH_CRASH_ON_SPAN_USE_AFTER_FINISH", true)
2019+
// Set the same seed on every node, to be used by builds with
2020+
// runtime assertions enabled.
2021+
setUnlessExists("COCKROACH_RANDOM_SEED", c.cockroachRandomSeed())
20102022

20112023
// Needed for backward-compat on crdb_internal.ranges{_no_leases}.
20122024
// Remove in v23.2.
@@ -2071,6 +2083,38 @@ func (c *clusterImpl) RefetchCertsFromNode(ctx context.Context, node int) error
20712083
})
20722084
}
20732085

2086+
// SetRandomSeed sets the random seed to be used by the cluster. If
2087+
// not called, clusters generate a random seed from the global
2088+
// generator in the `rand` package. This function must be called
2089+
// before any nodes in the cluster start.
2090+
func (c *clusterImpl) SetRandomSeed(seed int64) {
2091+
c.randomSeed.seed = &seed
2092+
}
2093+
2094+
// cockroachRandomSeed returns the `COCKROACH_RANDOM_SEED` to be used
2095+
// by this cluster. The seed may have been previously set by a
2096+
// previous call to `StartE`, or by the user via `SetRandomSeed`. If
2097+
// not set, this function will generate a seed and return it.
2098+
func (c *clusterImpl) cockroachRandomSeed() int64 {
2099+
c.randomSeed.mu.Lock()
2100+
defer c.randomSeed.mu.Unlock()
2101+
2102+
// If the user provided a seed via environment variable, always use
2103+
// that, even if the test attempts to set a different seed.
2104+
if seedStr := os.Getenv(test.EnvAssertionsEnabledSeed); seedStr != "" {
2105+
seedOverride, err := strconv.ParseInt(seedStr, 0, 64)
2106+
if err != nil {
2107+
panic(fmt.Sprintf("error parsing %s: %s", test.EnvAssertionsEnabledSeed, err))
2108+
}
2109+
c.randomSeed.seed = &seedOverride
2110+
} else if c.randomSeed.seed == nil {
2111+
seed := rand.Int63()
2112+
c.randomSeed.seed = &seed
2113+
}
2114+
2115+
return *c.randomSeed.seed
2116+
}
2117+
20742118
// Start is like StartE() except that it will fatal the test on error.
20752119
func (c *clusterImpl) Start(
20762120
ctx context.Context,

‎pkg/cmd/roachtest/cluster/cluster_interface.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ type Cluster interface {
4949
ctx context.Context, content, dest string, mode os.FileMode, opts ...option.Option,
5050
) error
5151

52+
// SetRandomSeed allows tests to set their own random seed to be
53+
// used by builds with runtime assertions enabled.
54+
SetRandomSeed(seed int64)
55+
5256
// Starting and stopping CockroachDB.
5357

5458
StartE(ctx context.Context, l *logger.Logger, startOpts option.StartOpts, settings install.ClusterSettings, opts ...option.Option) error

‎pkg/cmd/roachtest/cluster_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,11 @@ func (t testWrapper) Cockroach() string {
7272
return "./dummy-path/to/cockroach"
7373
}
7474

75-
func (t testWrapper) CockroachShort() string {
75+
func (t testWrapper) StandardCockroach() string {
76+
return "./dummy-path/to/cockroach"
77+
}
78+
79+
func (t testWrapper) RuntimeAssertionsCockroach() string {
7680
return "./dummy-path/to/cockroach-short"
7781
}
7882

‎pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,10 @@ func UploadVersion(
135135
newVersion *Version,
136136
) (string, error) {
137137
dstBinary := BinaryPathForVersion(t, newVersion)
138-
srcBinary := t.Cockroach()
138+
// Run with standard binary as older versions retrieved through roachprod stage
139+
// are not currently available with crdb_test enabled.
140+
// TODO(DarrylWong): Compile older versions with crdb_test flag.
141+
srcBinary := t.StandardCockroach()
139142

140143
overrideBinary, isOverriden := t.VersionsBinaryOverride()[newVersion.String()]
141144
if isOverriden {

‎pkg/cmd/roachtest/run.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ package main
1313
import (
1414
"context"
1515
"fmt"
16+
"math/rand"
1617
"net/http"
1718
"os"
1819
"os/signal"
@@ -29,6 +30,7 @@ import (
2930
"github.com/cockroachdb/cockroach/pkg/util/log"
3031
"github.com/cockroachdb/cockroach/pkg/util/log/logconfig"
3132
"github.com/cockroachdb/cockroach/pkg/util/log/logpb"
33+
"github.com/cockroachdb/cockroach/pkg/util/randutil"
3234
"github.com/cockroachdb/cockroach/pkg/util/stop"
3335
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
3436
"github.com/cockroachdb/errors"
@@ -41,6 +43,7 @@ var (
4143

4244
parallelism int
4345
cpuQuota int
46+
globalSeed int64
4447

4548
// Path to a local dir where the test logs and artifacts collected from
4649
// cluster will be placed.
@@ -153,6 +156,9 @@ func addRunBenchCommonFlags(cmd *cobra.Command) {
153156
cmd.Flags().BoolVar(
154157
&forceCloudCompat, "force-cloud-compat", false, "Includes tests that are not marked as compatible with the cloud used")
155158
addSuiteAndOwnerFlags(cmd)
159+
cmd.Flags().Int64Var(
160+
&globalSeed, "global-seed", randutil.NewPseudoSeed(),
161+
"The global random seed used for all tests.")
156162
}
157163

158164
func addRunFlags(runCmd *cobra.Command) {
@@ -179,6 +185,8 @@ func addBenchFlags(benchCmd *cobra.Command) {
179185
// runTests is the main function for the run and bench commands.
180186
// Assumes initRunFlagsBinariesAndLibraries was called.
181187
func runTests(register func(registry.Registry), filter *registry.TestFilter) error {
188+
//lint:ignore SA1019 deprecated
189+
rand.Seed(globalSeed)
182190
r := makeTestRegistry(cloud)
183191

184192
// actual registering of tests
@@ -257,6 +265,7 @@ func runTests(register func(registry.Registry), filter *registry.TestFilter) err
257265
literalArtifactsDir: literalArtifactsDir,
258266
runnerLogPath: runnerLogPath,
259267
}
268+
l.Printf("global random seed: %d", globalSeed)
260269
go func() {
261270
if err := http.ListenAndServe(
262271
fmt.Sprintf(":%d", promPort),

‎pkg/cmd/roachtest/test/test_interface.go‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,25 @@ import (
2020
// cluster.
2121
const DefaultCockroachPath = "./cockroach-default"
2222

23+
// EnvAssertionsEnabledSeed is the name of the environment variable
24+
// that, when set, causes roachtest to use a binary with runtime
25+
// assertions enabled (if available), using the random seed contained
26+
// in that environment variable.
27+
var EnvAssertionsEnabledSeed = "ROACHTEST_ASSERTIONS_ENABLED_SEED"
28+
2329
// Test is the interface through which roachtests interact with the
2430
// test harness.
2531
type Test interface {
26-
Cockroach() string // path to main cockroach binary
27-
// CockroachShort returns the path to cockroach-short binary compiled with
28-
// --crdb_test build tag, or an empty string if no such binary was given.
29-
CockroachShort() string
32+
// StandardCockroach returns path to main cockroach binary, compiled
33+
// without runtime assertions.
34+
StandardCockroach() string
35+
// RuntimeAssertionsCockroach returns the path to cockroach-short
36+
// binary compiled with --crdb_test build tag, or an empty string if
37+
// no such binary was given.
38+
RuntimeAssertionsCockroach() string
39+
// Cockroach returns either StandardCockroach or RuntimeAssertionsCockroach,
40+
// picked randomly.
41+
Cockroach() string
3042
Name() string
3143
BuildVersion() *version.Version
3244
IsBuildVersion(string) bool // "vXX.YY"

0 commit comments

Comments
 (0)