Skip to content

Commit a05aa45

Browse files
echauchotechauchot
authored
Merge pull request #15381 from egalpin/BEAM-10990-elasticsearch-response-filtering
Merge pull request #15381: [BEAM-10990] Elasticsearch response filtering and [BEAM-5172] Tries to reduce ES UTest flakiness
2 parents 8578c9a + f7691e6 commit a05aa45

9 files changed

Lines changed: 647 additions & 111 deletions

File tree

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ configurations.all {
3939
}
4040
}
4141

42+
test {
43+
maxParallelForks = 1
44+
}
45+
4246
dependencies {
4347
testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "testRuntime")
4448
testCompile "org.elasticsearch.test:framework:$elastic_search_version"

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.junit.AfterClass;
3434
import org.junit.Before;
3535
import org.junit.BeforeClass;
36-
import org.junit.Ignore;
3736
import org.junit.Rule;
3837
import org.junit.Test;
3938
import org.junit.rules.ExpectedException;
@@ -84,7 +83,6 @@ public void setup() throws IOException {
8483

8584
@Rule public TestPipeline pipeline = TestPipeline.create();
8685

87-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
8886
@Test
8987
public void testSizes() throws Exception {
9088
// need to create the index using the helper method (not create it at first insertion)
@@ -134,6 +132,18 @@ public void testWriteWithErrors() throws Exception {
134132
elasticsearchIOTestCommon.testWriteWithErrors();
135133
}
136134

135+
@Test
136+
public void testWriteWithErrorsReturned() throws Exception {
137+
elasticsearchIOTestCommon.setPipeline(pipeline);
138+
elasticsearchIOTestCommon.testWriteWithErrorsReturned();
139+
}
140+
141+
@Test
142+
public void testWriteWithErrorsReturnedAllowedErrors() throws Exception {
143+
elasticsearchIOTestCommon.setPipeline(pipeline);
144+
elasticsearchIOTestCommon.testWriteWithErrorsReturnedAllowedErrors();
145+
}
146+
137147
@Test
138148
public void testWriteWithMaxBatchSize() throws Exception {
139149
elasticsearchIOTestCommon.testWriteWithMaxBatchSize();
@@ -144,7 +154,6 @@ public void testWriteWithMaxBatchSizeBytes() throws Exception {
144154
elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
145155
}
146156

147-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
148157
@Test
149158
public void testSplit() throws Exception {
150159
// need to create the index using the helper method (not create it at first insertion)
@@ -254,4 +263,15 @@ public void testWriteWithIsDeleteFn() throws Exception {
254263
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithPartialUpdates();
255264
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithoutPartialUpdate();
256265
}
266+
267+
@Test
268+
public void testDocumentCoder() throws Exception {
269+
elasticsearchIOTestCommon.testDocumentCoder();
270+
}
271+
272+
@Test
273+
public void testPDone() throws Exception {
274+
elasticsearchIOTestCommon.setPipeline(pipeline);
275+
elasticsearchIOTestCommon.testPipelineDone();
276+
}
257277
}

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ configurations.all {
3939
}
4040
}
4141

42+
test {
43+
maxParallelForks = 1
44+
}
45+
4246
dependencies {
4347
testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "testRuntime")
4448
testCompile "org.elasticsearch.test:framework:$elastic_search_version"

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.junit.AfterClass;
3434
import org.junit.Before;
3535
import org.junit.BeforeClass;
36-
import org.junit.Ignore;
3736
import org.junit.Rule;
3837
import org.junit.Test;
3938
import org.junit.rules.ExpectedException;
@@ -83,7 +82,6 @@ public void setup() throws IOException {
8382

8483
@Rule public TestPipeline pipeline = TestPipeline.create();
8584

86-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
8785
@Test
8886
public void testSizes() throws Exception {
8987
// need to create the index using the helper method (not create it at first insertion)
@@ -133,6 +131,18 @@ public void testWriteWithErrors() throws Exception {
133131
elasticsearchIOTestCommon.testWriteWithErrors();
134132
}
135133

134+
@Test
135+
public void testWriteWithErrorsReturned() throws Exception {
136+
elasticsearchIOTestCommon.setPipeline(pipeline);
137+
elasticsearchIOTestCommon.testWriteWithErrorsReturned();
138+
}
139+
140+
@Test
141+
public void testWriteWithErrorsReturnedAllowedErrors() throws Exception {
142+
elasticsearchIOTestCommon.setPipeline(pipeline);
143+
elasticsearchIOTestCommon.testWriteWithErrorsReturnedAllowedErrors();
144+
}
145+
136146
@Test
137147
public void testWriteWithMaxBatchSize() throws Exception {
138148
elasticsearchIOTestCommon.testWriteWithMaxBatchSize();
@@ -143,7 +153,6 @@ public void testWriteWithMaxBatchSizeBytes() throws Exception {
143153
elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
144154
}
145155

146-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
147156
@Test
148157
public void testSplit() throws Exception {
149158
// need to create the index using the helper method (not create it at first insertion)
@@ -247,4 +256,15 @@ public void testWriteWithIsDeleteFn() throws Exception {
247256
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithPartialUpdates();
248257
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithoutPartialUpdate();
249258
}
259+
260+
@Test
261+
public void testDocumentCoder() throws Exception {
262+
elasticsearchIOTestCommon.testDocumentCoder();
263+
}
264+
265+
@Test
266+
public void testPDone() throws Exception {
267+
elasticsearchIOTestCommon.setPipeline(pipeline);
268+
elasticsearchIOTestCommon.testPipelineDone();
269+
}
250270
}

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests :: 7.x"
2828
ext.summary = "Tests of ElasticsearchIO on Elasticsearch 7.x"
2929

3030
def log4j_version = "2.14.1"
31-
def elastic_search_version = "7.9.2"
31+
def elastic_search_version = "7.13.4"
3232

3333
configurations.all {
3434
resolutionStrategy {
@@ -39,6 +39,10 @@ configurations.all {
3939
}
4040
}
4141

42+
test {
43+
maxParallelForks = 1
44+
}
45+
4246
dependencies {
4347
testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "testRuntime")
4448
testCompile "org.elasticsearch.test:framework:$elastic_search_version"

‎sdks/java/io/elasticsearch-tests/elasticsearch-tests-7/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.junit.AfterClass;
3434
import org.junit.Before;
3535
import org.junit.BeforeClass;
36-
import org.junit.Ignore;
3736
import org.junit.Rule;
3837
import org.junit.Test;
3938
import org.junit.rules.ExpectedException;
@@ -84,7 +83,6 @@ public void setup() throws IOException {
8483

8584
@Rule public TestPipeline pipeline = TestPipeline.create();
8685

87-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
8886
@Test
8987
public void testSizes() throws Exception {
9088
// need to create the index using the helper method (not create it at first insertion)
@@ -134,6 +132,18 @@ public void testWriteWithErrors() throws Exception {
134132
elasticsearchIOTestCommon.testWriteWithErrors();
135133
}
136134

135+
@Test
136+
public void testWriteWithErrorsReturned() throws Exception {
137+
elasticsearchIOTestCommon.setPipeline(pipeline);
138+
elasticsearchIOTestCommon.testWriteWithErrorsReturned();
139+
}
140+
141+
@Test
142+
public void testWriteWithErrorsReturnedAllowedErrors() throws Exception {
143+
elasticsearchIOTestCommon.setPipeline(pipeline);
144+
elasticsearchIOTestCommon.testWriteWithErrorsReturnedAllowedErrors();
145+
}
146+
137147
@Test
138148
public void testWriteWithMaxBatchSize() throws Exception {
139149
elasticsearchIOTestCommon.testWriteWithMaxBatchSize();
@@ -144,7 +154,6 @@ public void testWriteWithMaxBatchSizeBytes() throws Exception {
144154
elasticsearchIOTestCommon.testWriteWithMaxBatchSizeBytes();
145155
}
146156

147-
@Ignore("https://issues.apache.org/jira/browse/BEAM-5172")
148157
@Test
149158
public void testSplit() throws Exception {
150159
// need to create the index using the helper method (not create it at first insertion)
@@ -248,4 +257,15 @@ public void testWriteWithIsDeleteFn() throws Exception {
248257
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithPartialUpdates();
249258
elasticsearchIOTestCommon.testWriteWithIsDeletedFnWithoutPartialUpdate();
250259
}
260+
261+
@Test
262+
public void testDocumentCoder() throws Exception {
263+
elasticsearchIOTestCommon.testDocumentCoder();
264+
}
265+
266+
@Test
267+
public void testPDone() throws Exception {
268+
elasticsearchIOTestCommon.setPipeline(pipeline);
269+
elasticsearchIOTestCommon.testPipelineDone();
270+
}
251271
}

0 commit comments

Comments
 (0)