Skip to content

Commit eb20d81

Browse files
kevinjqliuCopilot
andcommitted
Open-API: Avoid mixing shaded and unshaded cloud SDKs
Reuse cloud bundle dependency declarations as loose runtime JARs, following the Kafka Connect distribution, rather than staging shaded bundles alongside their unshaded dependencies. Cover unique SDK class definitions and ADLS FileIO initialization to prevent the Azure linkage regression. Generated-by: GitHub Copilot Co-authored-by: Copilot App <[email protected]>
1 parent 50f0b05 commit eb20d81

3 files changed

Lines changed: 73 additions & 11 deletions

File tree

‎build.gradle‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1142,10 +1142,6 @@ project(':iceberg-open-api') {
11421142
testImplementation libs.junit.suite.engine
11431143
testImplementation libs.assertj.core
11441144

1145-
testImplementation project(':iceberg-aws-bundle')
1146-
testImplementation project(':iceberg-gcp-bundle')
1147-
testImplementation project(':iceberg-azure-bundle')
1148-
11491145
testFixturesImplementation project(':iceberg-api')
11501146
testFixturesImplementation project(':iceberg-core')
11511147
testFixturesImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
@@ -1186,10 +1182,19 @@ project(':iceberg-open-api') {
11861182
testFixturesImplementation libs.sqlite.jdbc
11871183

11881184
testFixturesCompileOnly libs.apiguardian
1185+
}
1186+
1187+
afterEvaluate {
1188+
def awsBundle = evaluationDependsOn(':iceberg-aws-bundle')
1189+
def gcpBundle = evaluationDependsOn(':iceberg-gcp-bundle')
1190+
def azureBundle = evaluationDependsOn(':iceberg-azure-bundle')
11891191

1190-
testFixturesRuntimeOnly project(':iceberg-aws-bundle')
1191-
testFixturesRuntimeOnly project(':iceberg-azure-bundle')
1192-
testFixturesRuntimeOnly project(':iceberg-gcp-bundle')
1192+
dependencies {
1193+
// Use loose SDK JARs, as in the Kafka Connect distribution, without the shaded bundles.
1194+
awsBundle.configurations.runtimeClasspath.allDependencies.each { add('testFixturesRuntimeOnly', it.copy()) }
1195+
gcpBundle.configurations.runtimeClasspath.allDependencies.each { add('testFixturesRuntimeOnly', it.copy()) }
1196+
azureBundle.configurations.runtimeClasspath.allDependencies.each { add('testFixturesRuntimeOnly', it.copy()) }
1197+
}
11931198
}
11941199

11951200
test {

‎docker/iceberg-rest-fixture/README.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,16 @@ docker image rm -f apache/iceberg-rest-fixture && docker build -t apache/iceberg
9999

100100
The staging task writes the test-fixtures JAR and its runtime dependencies to
101101
`open-api/build/iceberg-rest-server/libs/`, with the existing OpenAPI `LICENSE`
102-
and `NOTICE` alongside `libs/`. It builds only the required runtime dependencies,
103-
including the cloud bundles, rather than every project's `shadowJar`.
102+
and `NOTICE` alongside `libs/`. Like the Kafka Connect distribution, it uses the
103+
cloud bundles' dependency declarations to stage individual SDK JARs, not the
104+
shaded cloud bundle JARs. This avoids mixing shaded and unshaded SDK classes.
105+
Only the required runtime dependencies are built, including bundled Guava.
104106
Staging is explicit: the normal OpenAPI `build` does not create this directory.
105107

106108
The image starts `RESTCatalogServer` with `libs/*` on the Java classpath; it does
107109
not use an OpenAPI shaded runtime JAR. The published test-fixtures JAR remains
108-
available to test consumers. Changing this packaging does not prune runtime
109-
dependencies or complete the LICENSE/NOTICE audit; those are separate follow-ups.
110+
available to test consumers. Backend dependency pruning and the LICENSE/NOTICE
111+
audit remain separate follow-ups.
110112

111113
## Browse
112114

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iceberg.rest;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import java.io.IOException;
24+
import java.util.Collections;
25+
import java.util.Map;
26+
import org.apache.iceberg.CatalogUtil;
27+
import org.apache.iceberg.io.FileIO;
28+
import org.junit.jupiter.api.Test;
29+
import org.junit.jupiter.params.ParameterizedTest;
30+
import org.junit.jupiter.params.provider.ValueSource;
31+
32+
class TestRESTFixtureRuntime {
33+
@ParameterizedTest
34+
@ValueSource(
35+
strings = {
36+
"software/amazon/awssdk/services/s3/S3Client.class",
37+
"com/azure/core/http/netty/NettyAsyncHttpClientBuilder.class",
38+
"com/google/cloud/storage/Storage.class"
39+
})
40+
void cloudSdkClassesHaveSingleDefinition(String resource) throws IOException {
41+
assertThat(Collections.list(getClass().getClassLoader().getResources(resource))).hasSize(1);
42+
}
43+
44+
@Test
45+
void azureFileIOCanInitialize() {
46+
String location = "abfss://[email protected]/test.parquet";
47+
try (FileIO fileIO =
48+
CatalogUtil.loadFileIO(
49+
"org.apache.iceberg.azure.adlsv2.ADLSFileIO",
50+
Map.of("adls.sas-token.account.dfs.core.windows.net", "test-token"),
51+
null)) {
52+
assertThat(fileIO.newInputFile(location).location()).isEqualTo(location);
53+
}
54+
}
55+
}

0 commit comments

Comments
 (0)