Skip to content

Commit 0350472

Browse files
OIiveirraOIiveirra
committed
[test](regression) Support Azure in migrated ORC fixture loads
### What problem does this PR solve? Issue Number: N/A Related PR: #69593 Problem Summary: The migrated ARRAY and MAP fixtures always used an AWS client even when the regression provider was Azure. Select the existing Azure Blob SDK for Azure upload/cleanup and pass the selected provider and endpoint to S3 TVF. Resolve the framework Azure endpoint placeholder while preserving explicit endpoints. Keep S3 uploads, original fixtures and all existing assertions. ### Release note None ### Check List (For Author) - Test: Eight scripts compiled with the existing SDK dependencies; 12 loopback SDK upload/cleanup scenarios passed, including mixed-case Azure, default/explicit endpoints and injected SQL failure. SQL was intercepted in this probe; no Doris runtime regression or live cloud validation was performed. git diff --check passed. - Behavior changed: No (test data-loading adaptation only) - Does this need documentation: No
1 parent 63b2f2a commit 0350472

2 files changed

Lines changed: 58 additions & 8 deletions

File tree

‎regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
import com.azure.storage.blob.BlobClientBuilder
19+
import com.azure.storage.common.StorageSharedKeyCredential
20+
1821
suite("test_array_with_single_row_max_than_uint16", "query") {
1922
// define a sql table
2023
def testTable = "arr_dd"
@@ -38,19 +41,41 @@ suite("test_array_with_single_row_max_than_uint16", "query") {
3841
// prepare data
3942
String bucket = getS3BucketName()
4043
String key = "regression/local_tvf_migration/${UUID.randomUUID()}/arr_max.orc"
41-
def s3Client = getS3Client()
42-
s3Client.putObject(bucket, key, new File(context.config.dataPath, "query_p0/arr_max.orc"))
44+
String provider = getS3Provider()
45+
String endpoint = getS3Endpoint()
46+
File fixture = new File(context.config.dataPath, "query_p0/arr_max.orc")
47+
def deleteFixture
48+
if (provider.equalsIgnoreCase("AZURE")) {
49+
if (endpoint == "azure-endpoint") {
50+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
51+
} else if (!endpoint.contains("://")) {
52+
endpoint = "https://${endpoint}"
53+
}
54+
def blobClient = new BlobClientBuilder()
55+
.endpoint(endpoint)
56+
.containerName(bucket)
57+
.blobName(key)
58+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
59+
.buildClient()
60+
blobClient.uploadFromFile(fixture.absolutePath)
61+
deleteFixture = { blobClient.delete() }
62+
} else {
63+
def s3Client = getS3Client()
64+
s3Client.putObject(bucket, key, fixture)
65+
deleteFixture = { s3Client.deleteObject(bucket, key) }
66+
}
4367
try {
4468
sql """INSERT INTO ${testTable} SELECT k1, k2, c_decimal FROM S3(
4569
"uri" = "s3://${bucket}/${key}",
4670
"format" = "orc",
47-
"s3.endpoint" = "${getS3Endpoint()}",
71+
"s3.endpoint" = "${endpoint}",
72+
"provider" = "${provider}",
4873
"s3.region" = "${getS3Region()}",
4974
"s3.access_key" = "${getS3AK()}",
5075
"s3.secret_key" = "${getS3SK()}"
5176
)"""
5277
} finally {
53-
s3Client.deleteObject(bucket, key)
78+
deleteFixture()
5479
}
5580
sql "sync"
5681
qt_sql "SELECT COUNT(*) FROM ${testTable}"

‎regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
import com.azure.storage.blob.BlobClientBuilder
19+
import com.azure.storage.common.StorageSharedKeyCredential
20+
1821
suite("map_uniq_with_local_tvf", "p0") {
1922
def table_name = "map_uniq"
2023
sql "DROP TABLE IF EXISTS ${table_name};"
@@ -36,19 +39,41 @@ suite("map_uniq_with_local_tvf", "p0") {
3639

3740
String bucket = getS3BucketName()
3841
String key = "regression/local_tvf_migration/${UUID.randomUUID()}/mm.orc"
39-
def s3Client = getS3Client()
40-
s3Client.putObject(bucket, key, new File(context.config.dataPath, "types/complex_types/mm.orc"))
42+
String provider = getS3Provider()
43+
String endpoint = getS3Endpoint()
44+
File fixture = new File(context.config.dataPath, "types/complex_types/mm.orc")
45+
def deleteFixture
46+
if (provider.equalsIgnoreCase("AZURE")) {
47+
if (endpoint == "azure-endpoint") {
48+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
49+
} else if (!endpoint.contains("://")) {
50+
endpoint = "https://${endpoint}"
51+
}
52+
def blobClient = new BlobClientBuilder()
53+
.endpoint(endpoint)
54+
.containerName(bucket)
55+
.blobName(key)
56+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
57+
.buildClient()
58+
blobClient.uploadFromFile(fixture.absolutePath)
59+
deleteFixture = { blobClient.delete() }
60+
} else {
61+
def s3Client = getS3Client()
62+
s3Client.putObject(bucket, key, fixture)
63+
deleteFixture = { s3Client.deleteObject(bucket, key) }
64+
}
4165
try {
4266
sql """INSERT INTO ${table_name} SELECT id, m FROM S3(
4367
"uri" = "s3://${bucket}/${key}",
4468
"format" = "orc",
45-
"s3.endpoint" = "${getS3Endpoint()}",
69+
"s3.endpoint" = "${endpoint}",
70+
"provider" = "${provider}",
4671
"s3.region" = "${getS3Region()}",
4772
"s3.access_key" = "${getS3AK()}",
4873
"s3.secret_key" = "${getS3SK()}"
4974
)"""
5075
} finally {
51-
s3Client.deleteObject(bucket, key)
76+
deleteFixture()
5277
}
5378
sql "sync"
5479
qt_sql "SELECT COUNT(*) FROM ${table_name}"

0 commit comments

Comments
 (0)