Skip to content

Commit a7fdd01

Browse files
shs96cshs96c
committed
[bazel] Mark remaining tests as being skipped for remote builds
1 parent 96494b4 commit a7fdd01

9 files changed

Lines changed: 132 additions & 10 deletions

‎.bazelrc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,3 +136,6 @@ build:remote-ci --bes_upload_mode=wait_for_upload_complete
136136
# make reasonable usage of everything, everywhere, all at once.
137137
build:remote --local_cpu_resources='HOST_CPUS*10'
138138
build:remote --local_ram_resources='HOST_RAM*4.0'
139+
140+
# A small hint that we're running our tests remotely
141+
test:remote --test_env=REMOTE_BUILD=1

‎java/private/selenium_test.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(),
7777
stripped_args.pop("data", None)
7878
stripped_args.pop("jvm_flags", None)
7979
stripped_args.pop("tags", None)
80+
inherited_env = stripped_args.pop("env_inherit", []) + ["REMOTE_BUILD"]
8081

8182
all_tests = []
8283

@@ -94,6 +95,7 @@ def selenium_test(name, test_class, size = "medium", browsers = BROWSERS.keys(),
9495
# Only allow linting on the default test
9596
tags = BROWSERS[browser]["tags"] + tags + ([] if test == name else ["no-lint"]),
9697
data = BROWSERS[browser]["data"] + data,
98+
env_inherit = inherited_env,
9799
**stripped_args
98100
)
99101
if browser == default_browser:

‎java/test/org/openqa/selenium/ClickTest.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,6 @@
1717

1818
package org.openqa.selenium;
1919

20-
import static org.assertj.core.api.Assertions.assertThat;
21-
import static org.openqa.selenium.WaitingConditions.newWindowIsOpened;
22-
import static org.openqa.selenium.WaitingConditions.pageSourceToContain;
23-
import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
24-
import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
25-
import static org.openqa.selenium.testing.drivers.Browser.IE;
26-
import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
27-
28-
import java.util.Set;
2920
import org.junit.jupiter.api.BeforeEach;
3021
import org.junit.jupiter.api.Test;
3122
import org.openqa.selenium.testing.Ignore;
@@ -34,6 +25,16 @@
3425
import org.openqa.selenium.testing.NotYetImplemented;
3526
import org.openqa.selenium.testing.SwitchToTopAfterTest;
3627

28+
import java.util.Set;
29+
30+
import static org.assertj.core.api.Assertions.assertThat;
31+
import static org.openqa.selenium.WaitingConditions.newWindowIsOpened;
32+
import static org.openqa.selenium.WaitingConditions.pageSourceToContain;
33+
import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
34+
import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
35+
import static org.openqa.selenium.testing.drivers.Browser.IE;
36+
import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
37+
3738
class ClickTest extends JupiterTestBase {
3839

3940
@BeforeEach

‎java/test/org/openqa/selenium/CookieImplementationTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import static org.assertj.core.api.Assertions.assertThat;
2121
import static org.junit.jupiter.api.Assumptions.assumeTrue;
2222
import static org.openqa.selenium.testing.drivers.Browser.ALL;
23+
import static org.openqa.selenium.testing.drivers.Browser.CHROME;
24+
import static org.openqa.selenium.testing.drivers.Browser.FIREFOX;
2325
import static org.openqa.selenium.testing.drivers.Browser.IE;
2426
import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
2527

@@ -32,6 +34,7 @@
3234
import org.openqa.selenium.environment.DomainHelper;
3335
import org.openqa.selenium.testing.Ignore;
3436
import org.openqa.selenium.testing.JupiterTestBase;
37+
import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds;
3538
import org.openqa.selenium.testing.NotYetImplemented;
3639
import org.openqa.selenium.testing.SwitchToTopAfterTest;
3740

@@ -81,6 +84,7 @@ public void testShouldGetCookieByName() {
8184

8285
@Test
8386
@NotYetImplemented(SAFARI)
87+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
8488
public void testShouldBeAbleToAddCookie() {
8589
String key = generateUniqueKey();
8690
String value = "foo";
@@ -200,6 +204,8 @@ public void testAddCookiesWithDifferentPathsThatAreRelatedToOurs() {
200204
@SwitchToTopAfterTest
201205
@Test
202206
@Ignore(SAFARI)
207+
@NotWorkingInRemoteBazelBuilds(CHROME)
208+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
203209
public void testGetCookiesInAFrame() {
204210
driver.get(domainHelper.getUrlForFirstValidHostname("/common/animals"));
205211
Cookie cookie1 = new Cookie.Builder("fish", "cod").path("/common/animals").build();
@@ -239,6 +245,7 @@ public void testShouldNotGetCookieOnDifferentDomain() {
239245

240246
@Test
241247
@NotYetImplemented(SAFARI)
248+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
242249
public void testShouldBeAbleToAddToADomainWhichIsRelatedToTheCurrentDomain() {
243250
String cookieName = "name";
244251
assertCookieIsNotPresentWithName(cookieName);
@@ -263,6 +270,7 @@ public void testsShouldNotGetCookiesRelatedToCurrentDomainWithoutLeadingPeriod()
263270
}
264271

265272
@Test
273+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
266274
void testShouldBeAbleToIncludeLeadingPeriodInDomainName() {
267275
String cookieName = "name";
268276
assertCookieIsNotPresentWithName(cookieName);
@@ -277,6 +285,7 @@ void testShouldBeAbleToIncludeLeadingPeriodInDomainName() {
277285

278286
@Test
279287
@NotYetImplemented(SAFARI)
288+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
280289
public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception {
281290
URI url = new URI(driver.getCurrentUrl());
282291
String host = url.getHost() + ":" + url.getPort();
@@ -291,6 +300,8 @@ public void testShouldBeAbleToSetDomainToTheCurrentDomain() throws Exception {
291300

292301
@Test
293302
@NotYetImplemented(SAFARI)
303+
@NotWorkingInRemoteBazelBuilds(CHROME)
304+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
294305
public void testShouldWalkThePathToDeleteACookie() {
295306
Cookie cookie1 = new Cookie.Builder("fish", "cod").build();
296307
driver.manage().addCookie(cookie1);
@@ -320,6 +331,7 @@ public void testShouldWalkThePathToDeleteACookie() {
320331

321332
@Test
322333
@NotYetImplemented(SAFARI)
334+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
323335
public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws Exception {
324336
URI uri = new URI(driver.getCurrentUrl());
325337
String host = String.format("%s:%d", uri.getHost(), uri.getPort());
@@ -335,6 +347,8 @@ public void testShouldIgnoreThePortNumberOfTheHostWhenSettingTheCookie() throws
335347

336348
@Test
337349
@NotYetImplemented(SAFARI)
350+
@NotWorkingInRemoteBazelBuilds(CHROME)
351+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
338352
public void testCookieEqualityAfterSetAndGet() {
339353
driver.get(domainHelper.getUrlForFirstValidHostname("animals"));
340354

@@ -379,6 +393,8 @@ public void testRetainsCookieExpiry() {
379393
@Test
380394
@Ignore(IE)
381395
@Ignore(SAFARI)
396+
@NotWorkingInRemoteBazelBuilds(CHROME)
397+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
382398
public void canHandleSecureCookie() {
383399
driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals"));
384400

@@ -395,6 +411,8 @@ public void canHandleSecureCookie() {
395411
@Test
396412
@Ignore(IE)
397413
@Ignore(SAFARI)
414+
@NotWorkingInRemoteBazelBuilds(CHROME)
415+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
398416
public void testRetainsCookieSecure() {
399417
driver.get(domainHelper.getSecureUrlForFirstValidHostname("animals"));
400418

@@ -411,6 +429,8 @@ public void testRetainsCookieSecure() {
411429

412430
@Test
413431
@Ignore(SAFARI)
432+
@NotWorkingInRemoteBazelBuilds(CHROME)
433+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
414434
public void canHandleHttpOnlyCookie() {
415435
Cookie addedCookie =
416436
new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build();
@@ -424,6 +444,8 @@ public void canHandleHttpOnlyCookie() {
424444

425445
@Test
426446
@Ignore(SAFARI)
447+
@NotWorkingInRemoteBazelBuilds(CHROME)
448+
@NotWorkingInRemoteBazelBuilds(FIREFOX)
427449
public void testRetainsHttpOnlyFlag() {
428450
Cookie addedCookie =
429451
new Cookie.Builder("fish", "cod").path("/common/animals").isHttpOnly(true).build();

‎java/test/org/openqa/selenium/JavascriptEnabledDriverTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,15 @@
2323
import static org.openqa.selenium.WaitingConditions.elementValueToEqual;
2424
import static org.openqa.selenium.WaitingConditions.windowToBeSwitchedToWithName;
2525
import static org.openqa.selenium.support.ui.ExpectedConditions.titleIs;
26+
import static org.openqa.selenium.testing.drivers.Browser.CHROME;
2627
import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
2728

2829
import org.junit.jupiter.api.Test;
2930
import org.openqa.selenium.interactions.Locatable;
3031
import org.openqa.selenium.support.ui.ExpectedConditions;
3132
import org.openqa.selenium.testing.JupiterTestBase;
3233
import org.openqa.selenium.testing.NoDriverAfterTest;
34+
import org.openqa.selenium.testing.NotWorkingInRemoteBazelBuilds;
3335
import org.openqa.selenium.testing.NotYetImplemented;
3436

3537
class JavascriptEnabledDriverTest extends JupiterTestBase {
@@ -61,6 +63,7 @@ public void testDocumentShouldReflectLatestDom() {
6163
}
6264

6365
@Test
66+
@NotWorkingInRemoteBazelBuilds(value = CHROME)
6467
void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() {
6568
driver.get(pages.formPage);
6669

@@ -71,6 +74,7 @@ void testShouldWaitForLoadsToCompleteAfterJavascriptCausesANewPageToLoad() {
7174
}
7275

7376
@Test
77+
@NotWorkingInRemoteBazelBuilds(value = CHROME)
7478
void testShouldBeAbleToFindElementAfterJavascriptCausesANewPageToLoad() {
7579
driver.get(pages.formPage);
7680

‎java/test/org/openqa/selenium/testing/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ java_library(
2222
"NeedsFreshDriver.java",
2323
"NoDriverAfterTest.java",
2424
"NoDriverBeforeTest.java",
25+
"NotWorkingInRemoteBazelBuilds.java",
26+
"NotWorkingInRemoteBazelBuildsList.java",
2527
"NotYetImplemented.java",
2628
"NotYetImplementedList.java",
2729
"SwitchToTopAfterTest.java",

‎java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuilds.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package org.openqa.selenium.testing;
2+
3+
import org.openqa.selenium.testing.drivers.Browser;
4+
5+
import java.lang.annotation.ElementType;
6+
import java.lang.annotation.Repeatable;
7+
import java.lang.annotation.Retention;
8+
import java.lang.annotation.RetentionPolicy;
9+
import java.lang.annotation.Target;
10+
11+
@Retention(RetentionPolicy.RUNTIME)
12+
@Target(ElementType.METHOD)
13+
@Repeatable(NotWorkingInRemoteBazelBuildsList.class)
14+
public @interface NotWorkingInRemoteBazelBuilds {
15+
16+
Browser value() default Browser.ALL;
17+
18+
}

‎java/test/org/openqa/selenium/testing/NotWorkingInRemoteBazelBuildsList.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.openqa.selenium.testing;
2+
3+
import java.lang.annotation.ElementType;
4+
import java.lang.annotation.Retention;
5+
import java.lang.annotation.RetentionPolicy;
6+
import java.lang.annotation.Target;
7+
8+
@Retention(RetentionPolicy.RUNTIME)
9+
@Target({ElementType.METHOD})
10+
public @interface NotWorkingInRemoteBazelBuildsList {
11+
NotWorkingInRemoteBazelBuilds[] value();
12+
}

‎java/test/org/openqa/selenium/testing/SeleniumExtension.java‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public class SeleniumExtension
6767
private final CaptureLoggingRule captureLoggingRule;
6868

6969
private boolean failedWithNotYetImplemented = false;
70+
private boolean failedWithRemoteBuild = false;
7071

7172
public SeleniumExtension() {
7273
this(Duration.ofSeconds(10), Duration.ofSeconds(5));
@@ -115,6 +116,9 @@ public void beforeEach(ExtensionContext context) throws Exception {
115116

116117
// NotYetImplementedRule
117118
failedWithNotYetImplemented = false;
119+
120+
// Remote builds
121+
failedWithRemoteBuild = false;
118122
}
119123

120124
@Override
@@ -144,6 +148,20 @@ public void afterEach(ExtensionContext context) throws Exception {
144148
testMethod.map(Method::getName).orElse(""),
145149
current));
146150
}
151+
152+
NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context);
153+
boolean isNotExpectedToWork = notWorkingInRemoteBuilds.check();
154+
155+
if (isNotExpectedToWork && !failedWithRemoteBuild) {
156+
Optional<Class<?>> testClass = context.getTestClass();
157+
Optional<Method> testMethod = context.getTestMethod();
158+
throw new Exception(
159+
String.format(
160+
"%s.%s is not yet expected to work on remote builds using %s, but it already works!",
161+
testClass.map(Class::getName).orElse(""),
162+
testMethod.map(Method::getName).orElse(""),
163+
Browser.detect()));
164+
}
147165
}
148166

149167
@Override
@@ -199,10 +217,19 @@ public void handleTestExecutionException(ExtensionContext context, Throwable thr
199217
// NotYetImplementedRule
200218
NotYetImplementedRule notYetImplementedRule = new NotYetImplementedRule(context);
201219
if (notYetImplementedRule.check()) {
202-
// Expected
203220
failedWithNotYetImplemented = true;
221+
}
222+
223+
NotWorkingInRemoteBazelBuildsRule notWorkingInRemoteBuilds = new NotWorkingInRemoteBazelBuildsRule(context);
224+
if (notWorkingInRemoteBuilds.check()) {
225+
failedWithRemoteBuild = true;
226+
}
227+
228+
if (failedWithNotYetImplemented || failedWithRemoteBuild) {
229+
// Expected failures.
204230
return;
205231
}
232+
206233
throw throwable;
207234
}
208235

@@ -309,6 +336,37 @@ public boolean check() throws Exception {
309336
findRepeatableAnnotations(element, NotYetImplemented.class);
310337
return notImplemented(notYetImplementedList) || notImplemented(notYetImplemented.stream());
311338
}
339+
340+
}
341+
342+
private static class NotWorkingInRemoteBazelBuildsRule {
343+
ExtensionContext context;
344+
private final Browser current = Objects.requireNonNull(Browser.detect());
345+
346+
public NotWorkingInRemoteBazelBuildsRule(ExtensionContext context) {
347+
this.context = context;
348+
}
349+
350+
private boolean notWorkingYet(Optional<NotWorkingInRemoteBazelBuilds> list) {
351+
return list.isPresent() && notWorkingYet(Stream.of(list.get()));
352+
}
353+
354+
private boolean notWorkingYet(Stream<NotWorkingInRemoteBazelBuilds> value) {
355+
return value.anyMatch(notWorking -> current.matches(notWorking.value()));
356+
}
357+
358+
public boolean check() {
359+
if (!Objects.equals("1", System.getenv("REMOTE_BUILD"))) {
360+
return false;
361+
}
362+
363+
Optional<AnnotatedElement> element = context.getElement();
364+
Optional<NotWorkingInRemoteBazelBuilds> notWorkingList =
365+
findAnnotation(element, NotWorkingInRemoteBazelBuilds.class);
366+
List<NotWorkingInRemoteBazelBuilds> notWorking =
367+
findRepeatableAnnotations(element, NotWorkingInRemoteBazelBuilds.class);
368+
return notWorkingYet(notWorkingList) || notWorkingYet(notWorking.stream());
369+
}
312370
}
313371

314372
private static class SwitchToTopRule {

0 commit comments

Comments
 (0)