Skip to content

Commit ef4eca4

Browse files
OIiveirraOIiveirra
committed
[test](regression) Support Azure in UUID fixture uploads
### What problem does this PR solve? Issue Number: N/A Related PR: #69591 Problem Summary: Removing the disabled LOCAL fallback makes three UUID fixture suites use an AWS upload even for Azure. Use the already-declared Azure Blob SDK for Azure uploads and pass the matching provider/endpoint to TVF. Preserve explicit endpoints, the existing S3 upload path, original fixtures and all schema/value/profile assertions. This change is limited to these three existing suites. ### Release note None ### Check List (For Author) - Test: Six scripts compiled with existing SDK dependencies. Twelve loopback SDK scenarios verified the three exact UUID setup/upload blocks, Azure default/explicit endpoints, S3 uploads, byte-identical fixtures, matching TVF parameters and fixture availability for later queries. No SQL execution, Doris runtime regression or live cloud validation was performed. git diff --check passed. - Behavior changed: No (test data-loading adaptation only) - Does this need documentation: No
1 parent 30ce3e7 commit ef4eca4

3 files changed

Lines changed: 69 additions & 6 deletions

File tree

‎regression-test/suites/datatype_p0/uuid/test_uuid_csv_schema.groovy‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,40 @@
1616
// under the License.
1717

1818
// External checklist: T04, R02, R09, Q01-Q03, W06, V01-V02.
19+
import com.azure.storage.blob.BlobClientBuilder
20+
import com.azure.storage.common.StorageSharedKeyCredential
21+
1922
suite("test_uuid_csv_schema", "p0,external") {
2023
sql "SET enable_sql_cache = false"
2124
sql "SET enable_query_cache = false"
25+
String provider = getS3Provider()
26+
String endpoint = getS3Endpoint()
27+
if (provider.equalsIgnoreCase("AZURE")) {
28+
if (endpoint == "azure-endpoint") {
29+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
30+
} else if (!endpoint.contains("://")) {
31+
endpoint = "https://${endpoint}"
32+
}
33+
}
2234
String token = UUID.randomUUID().toString()
2335
def sources = [:]
2436
for (String kind : ['values', 'invalid']) {
2537
File fixture = new File(context.dataPath, "test_data/uuid_external_${kind}.csv")
2638
String bucket = context.config.otherConfigs.get('s3BucketName')
2739
String key = "uuid_csv_schema/${token}/${kind}.csv"
28-
getS3Client().putObject(bucket, key, fixture)
40+
if (provider.equalsIgnoreCase("AZURE")) {
41+
new BlobClientBuilder()
42+
.endpoint(endpoint)
43+
.containerName(bucket)
44+
.blobName(key)
45+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
46+
.buildClient().uploadFromFile(fixture.absolutePath)
47+
} else {
48+
getS3Client().putObject(bucket, key, fixture)
49+
}
2950
sources[kind] = """S3('uri'='s3://${bucket}/${key}','format'='csv',
3051
'column_separator'=',','csv_schema'='id:int;u:uuid',
31-
's3.endpoint'='${getS3Endpoint()}','s3.region'='${getS3Region()}',
52+
'provider'='${provider}','s3.endpoint'='${endpoint}','s3.region'='${getS3Region()}',
3253
's3.access_key'='${getS3AK()}','s3.secret_key'='${getS3SK()}')"""
3354
}
3455
for (boolean scannerV2 : [false, true]) {

‎regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,42 @@
1717

1818

1919
// Checklist: G08 G11 H01 H03 H08.
20+
import com.azure.storage.blob.BlobClientBuilder
21+
import com.azure.storage.common.StorageSharedKeyCredential
22+
2023
suite("test_uuid_parquet_native", "p0,external") {
2124

2225
// Native UUID files are generated by generate_uuid_parquet.py (Arrow 21+). Unlike OUTFILE
2326
// roundtrips, their footer explicitly has UUID / FIXED_LEN_BYTE_ARRAY(16).
2427
sql "SET enable_sql_cache = false"
2528
sql "SET enable_query_cache = false"
29+
String provider = getS3Provider()
30+
String endpoint = getS3Endpoint()
31+
if (provider.equalsIgnoreCase("AZURE")) {
32+
if (endpoint == "azure-endpoint") {
33+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
34+
} else if (!endpoint.contains("://")) {
35+
endpoint = "https://${endpoint}"
36+
}
37+
}
2638
String token = UUID.randomUUID().toString()
2739
for (String encoding : ['plain','dictionary']) {
2840
File fixture = new File(context.dataPath, "test_data/uuid_${encoding}.parquet")
2941
String source
3042
String bucket = context.config.otherConfigs.get('s3BucketName')
3143
String key = "uuid_parquet_native/${token}/${encoding}.parquet"
32-
getS3Client().putObject(bucket,key,fixture)
44+
if (provider.equalsIgnoreCase("AZURE")) {
45+
new BlobClientBuilder()
46+
.endpoint(endpoint)
47+
.containerName(bucket)
48+
.blobName(key)
49+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
50+
.buildClient().uploadFromFile(fixture.absolutePath)
51+
} else {
52+
getS3Client().putObject(bucket,key,fixture)
53+
}
3354
source = """s3('uri'='s3://${bucket}/${key}','format'='parquet',
34-
's3.endpoint'='${getS3Endpoint()}','s3.region'='${getS3Region()}',
55+
'provider'='${provider}','s3.endpoint'='${endpoint}','s3.region'='${getS3Region()}',
3556
's3.access_key'='${getS3AK()}','s3.secret_key'='${getS3SK()}')"""
3657
for (boolean scannerV2 : [false, true]) {
3758
sql "SET enable_file_scanner_v2 = ${scannerV2}"

‎regression-test/suites/datatype_p0/uuid/test_uuid_parquet_predicate.groovy‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,40 @@
1616
// under the License.
1717

1818
// Prove safe fallback for UUID bounds and actual pruning on the companion integer column.
19+
import com.azure.storage.blob.BlobClientBuilder
20+
import com.azure.storage.common.StorageSharedKeyCredential
21+
1922
suite("test_uuid_parquet_predicate", "p0,external") {
2023
sql "SET enable_sql_cache = false"
2124
sql "SET enable_query_cache = false"
2225
sql "SET enable_condition_cache = false"
2326
sql "SET enable_profile = true"
27+
String provider = getS3Provider()
28+
String endpoint = getS3Endpoint()
29+
if (provider.equalsIgnoreCase("AZURE")) {
30+
if (endpoint == "azure-endpoint") {
31+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
32+
} else if (!endpoint.contains("://")) {
33+
endpoint = "https://${endpoint}"
34+
}
35+
}
2436
String token = UUID.randomUUID().toString()
2537
String source
2638
File fixture = new File(context.dataPath, "test_data/uuid_predicate.parquet")
2739
String bucket = context.config.otherConfigs.get('s3BucketName')
2840
String key = "uuid_parquet_predicate/${token}/predicate.parquet"
29-
getS3Client().putObject(bucket, key, fixture)
41+
if (provider.equalsIgnoreCase("AZURE")) {
42+
new BlobClientBuilder()
43+
.endpoint(endpoint)
44+
.containerName(bucket)
45+
.blobName(key)
46+
.credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
47+
.buildClient().uploadFromFile(fixture.absolutePath)
48+
} else {
49+
getS3Client().putObject(bucket, key, fixture)
50+
}
3051
source = """S3('uri'='s3://${bucket}/${key}','format'='parquet',
31-
's3.endpoint'='${getS3Endpoint()}','s3.region'='${getS3Region()}',
52+
'provider'='${provider}','s3.endpoint'='${endpoint}','s3.region'='${getS3Region()}',
3253
's3.access_key'='${getS3AK()}','s3.secret_key'='${getS3SK()}')"""
3354
for (boolean scannerV2 : [false, true]) {
3455
sql "SET enable_file_scanner_v2 = ${scannerV2}"

0 commit comments

Comments
 (0)