|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +import com.sun.net.httpserver.HttpExchange |
| 19 | +import com.sun.net.httpserver.HttpHandler |
| 20 | +import com.sun.net.httpserver.HttpServer |
| 21 | +import groovy.json.JsonOutput |
| 22 | +import groovy.json.JsonSlurper |
| 23 | + |
| 24 | +import java.net.InetSocketAddress |
| 25 | +import java.util.concurrent.ConcurrentHashMap |
| 26 | +import java.util.concurrent.Executors |
| 27 | + |
| 28 | +// t$labels shows the labels a REST catalog returns with each table load (apache/iceberg#15750). The docker REST |
| 29 | +// fixture returns none (a fixture option is apache/iceberg#18047), so this suite puts a small proxy in front of |
| 30 | +// it: the proxy forwards every request unchanged and adds this suite's labels to the load-table responses, which |
| 31 | +// is all a labelling catalog does differently. Everything after the HTTP response is the real path: the iceberg |
| 32 | +// REST client parses the labels, FE plans the labels split and BE reads it through the JNI system-table scanner. |
| 33 | +suite("test_iceberg_rest_labels", "p0,external") { |
| 34 | + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableIcebergTest"))) { |
| 35 | + return |
| 36 | + } |
| 37 | + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") |
| 38 | + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") |
| 39 | + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") |
| 40 | + String restUri = "http://${externalEnvIp}:${restPort}" |
| 41 | + |
| 42 | + // Labels the proxy adds, keyed by "<namespace>.<table>". A table without an entry keeps the fixture's |
| 43 | + // response as is, without labels. |
| 44 | + Map<String, Map> labelsByTable = new ConcurrentHashMap<>() |
| 45 | + // Lengths and hop-by-hop headers belong to each connection. Accept-Encoding is dropped so that the fixture |
| 46 | + // answers in plain JSON the proxy can rewrite. |
| 47 | + Set<String> droppedRequestHeaders = ["host", "connection", "keep-alive", "content-length", "transfer-encoding", |
| 48 | + "expect", "accept-encoding"] as Set |
| 49 | + def loadTablePath = ~'^/v1/namespaces/([^/]+)/tables/([^/]+)$' |
| 50 | + def proxyExecutor = Executors.newFixedThreadPool(4) |
| 51 | + HttpServer proxy = HttpServer.create(new InetSocketAddress("0.0.0.0", 0), 0) |
| 52 | + proxy.setExecutor(proxyExecutor) |
| 53 | + proxy.createContext("/", { HttpExchange exchange -> |
| 54 | + try { |
| 55 | + String rawPath = exchange.requestURI.rawPath |
| 56 | + String rawQuery = exchange.requestURI.rawQuery |
| 57 | + def upstream = (HttpURLConnection) new URL(restUri + rawPath + (rawQuery == null ? "" : "?" + rawQuery)) |
| 58 | + .openConnection() |
| 59 | + upstream.setRequestMethod(exchange.requestMethod) |
| 60 | + upstream.setConnectTimeout(10000) |
| 61 | + upstream.setReadTimeout(60000) |
| 62 | + exchange.requestHeaders.each { String name, List<String> values -> |
| 63 | + if (!droppedRequestHeaders.contains(name.toLowerCase())) { |
| 64 | + values.each { upstream.addRequestProperty(name, it) } |
| 65 | + } |
| 66 | + } |
| 67 | + byte[] requestBody = exchange.requestBody.bytes |
| 68 | + if (requestBody.length > 0) { |
| 69 | + upstream.setDoOutput(true) |
| 70 | + upstream.outputStream.withCloseable { it.write(requestBody) } |
| 71 | + } |
| 72 | + int status = upstream.responseCode |
| 73 | + InputStream upstreamBody = status >= 400 ? upstream.errorStream : upstream.inputStream |
| 74 | + byte[] body = upstreamBody == null ? new byte[0] : upstreamBody.withCloseable { it.bytes } |
| 75 | + def loadTable = loadTablePath.matcher(rawPath) |
| 76 | + if ("GET" == exchange.requestMethod && status == 200 && loadTable.matches()) { |
| 77 | + Map labels = labelsByTable.get(loadTable.group(1) + "." + loadTable.group(2)) |
| 78 | + if (labels != null) { |
| 79 | + def response = new JsonSlurper().parseText(new String(body, "UTF-8")) |
| 80 | + response.labels = labels |
| 81 | + body = JsonOutput.toJson(response).getBytes("UTF-8") |
| 82 | + } |
| 83 | + } |
| 84 | + // Pass back only the content type. The fixture's validators describe the body before the rewrite: a |
| 85 | + // forwarded ETag would let the iceberg client revalidate its cached load and keep stale labels. |
| 86 | + String contentType = upstream.getHeaderField("Content-Type") |
| 87 | + if (contentType != null) { |
| 88 | + exchange.responseHeaders.set("Content-Type", contentType) |
| 89 | + } |
| 90 | + exchange.sendResponseHeaders(status, body.length == 0 ? -1 : body.length) |
| 91 | + if (body.length > 0) { |
| 92 | + exchange.responseBody.withCloseable { it.write(body) } |
| 93 | + } |
| 94 | + } finally { |
| 95 | + exchange.close() |
| 96 | + } |
| 97 | + } as HttpHandler) |
| 98 | + proxy.start() |
| 99 | + try { |
| 100 | + // FE loads the tables through the proxy, so it must be reachable from FE; like the HTTP TVF suite, use |
| 101 | + // httpTvfHost when the regression runner is not on the docker host. |
| 102 | + String proxyHost = context.config.otherConfigs.get("httpTvfHost") ?: externalEnvIp |
| 103 | + String proxyUri = "http://${proxyHost}:${proxy.address.port}" |
| 104 | + |
| 105 | + sql "drop catalog if exists test_iceberg_rest_labels" |
| 106 | + sql """create catalog test_iceberg_rest_labels properties ( |
| 107 | + 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'uri' = '${proxyUri}', |
| 108 | + 's3.access_key' = 'admin', 's3.secret_key' = 'password', |
| 109 | + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.region' = 'us-east-1' |
| 110 | + )""" |
| 111 | + // With a zero table cache TTL every statement loads the table, and so its labels, again. |
| 112 | + sql "drop catalog if exists test_iceberg_rest_labels_nocache" |
| 113 | + sql """create catalog test_iceberg_rest_labels_nocache properties ( |
| 114 | + 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'uri' = '${proxyUri}', |
| 115 | + 's3.access_key' = 'admin', 's3.secret_key' = 'password', |
| 116 | + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.region' = 'us-east-1', |
| 117 | + 'meta.cache.iceberg.table.ttl-second' = '0' |
| 118 | + )""" |
| 119 | + // Vended credentials turn the cross-query table cache off altogether, so this catalog reads the table that |
| 120 | + // each statement loads instead of a cached one. |
| 121 | + sql "drop catalog if exists test_iceberg_rest_labels_vended" |
| 122 | + sql """create catalog test_iceberg_rest_labels_vended properties ( |
| 123 | + 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'uri' = '${proxyUri}', |
| 124 | + 's3.access_key' = 'admin', 's3.secret_key' = 'password', |
| 125 | + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.region' = 'us-east-1', |
| 126 | + 'iceberg.rest.vended-credentials-enabled' = 'true' |
| 127 | + )""" |
| 128 | + // The same tables read straight from the fixture, which returns no labels. The tables are created through |
| 129 | + // it, so no proxied catalog caches a load before its labels are set. |
| 130 | + sql "drop catalog if exists test_iceberg_rest_labels_direct" |
| 131 | + sql """create catalog test_iceberg_rest_labels_direct properties ( |
| 132 | + 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'uri' = '${restUri}', |
| 133 | + 's3.access_key' = 'admin', 's3.secret_key' = 'password', |
| 134 | + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.region' = 'us-east-1' |
| 135 | + )""" |
| 136 | + |
| 137 | + sql "create database if not exists test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db" |
| 138 | + sql "drop table if exists test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled" |
| 139 | + sql """create table test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled ( |
| 140 | + id int, email string, phone string, address struct<city: string, zip: string> |
| 141 | + )""" |
| 142 | + sql """insert into test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled values |
| 143 | + (1, '[email protected]', '555-0101', named_struct('city', 'Paris', 'zip', '75001')), |
| 144 | + (2, '[email protected]', '555-0102', named_struct('city', 'Lyon', 'zip', '69001'))""" |
| 145 | + sql "drop table if exists test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled_empty" |
| 146 | + sql "create table test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled_empty (id int, note string)" |
| 147 | + sql "drop table if exists test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.unlabeled" |
| 148 | + sql "create table test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.unlabeled (id int)" |
| 149 | + sql "insert into test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.unlabeled values (1), (2)" |
| 150 | + |
| 151 | + // Field labels are keyed by field id, which the catalog assigns: read the ids from the fixture. |
| 152 | + def loaded = new JsonSlurper().parseText( |
| 153 | + new URL("${restUri}/v1/namespaces/test_iceberg_rest_labels_db/tables/labeled").getText( |
| 154 | + connectTimeout: 10000, readTimeout: 10000)) |
| 155 | + def schema = loaded.metadata.schemas.find { it["schema-id"] == loaded.metadata["current-schema-id"] } |
| 156 | + def emailId = schema.fields.find { it.name == "email" }.id |
| 157 | + def phoneId = schema.fields.find { it.name == "phone" }.id |
| 158 | + def cityId = schema.fields.find { it.name == "address" }.type.fields.find { it.name == "city" }.id |
| 159 | + labelsByTable.put("test_iceberg_rest_labels_db.labeled", [ |
| 160 | + "object-labels": ["owner": "team-a", "cost.center": "cc-42", "steward": "数据平台组"], |
| 161 | + "fields": [ |
| 162 | + ["field-id": emailId, "labels": ["classification": "pii"]], |
| 163 | + ["field-id": phoneId, "labels": ["classification": "pii", "masking": "partial"]], |
| 164 | + ["field-id": cityId, "labels": ["classification": "location"]]]]) |
| 165 | + labelsByTable.put("test_iceberg_rest_labels_db.labeled_empty", ["object-labels": ["owner": "team-b"]]) |
| 166 | + |
| 167 | + sql "switch test_iceberg_rest_labels" |
| 168 | + sql "use test_iceberg_rest_labels_db" |
| 169 | + order_qt_desc "desc labeled\$labels" |
| 170 | + // One row per label. A field label names its field from the current schema, a nested one by its full path. |
| 171 | + order_qt_all "select * from labeled\$labels" |
| 172 | + // BE reads the split's rows by position in the order of the columns it asks for, so another order or a |
| 173 | + // subset of the columns must still put every value in its own column. KEY is a reserved word in Doris, |
| 174 | + // so the key column is quoted. |
| 175 | + order_qt_projection "select value, `key`, field_name, scope, field_id from labeled\$labels" |
| 176 | + order_qt_one_column "select `key` from labeled\$labels" |
| 177 | + order_qt_filter_field "select field_name, `key`, value from labeled\$labels where scope = 'field' and value = 'pii'" |
| 178 | + order_qt_filter_object "select `key`, value from labeled\$labels where field_id is null" |
| 179 | + // COUNT counts labels rather than taking the table's record count: two records, seven labels. |
| 180 | + order_qt_count "select count(*) from labeled\$labels" |
| 181 | + order_qt_count_table "select count(*) from labeled" |
| 182 | + // Labels come with the table load, not with a snapshot: a table without any commit has them too. |
| 183 | + order_qt_no_snapshot "select * from labeled_empty\$labels" |
| 184 | + // A table the catalog returns no labels for, and the labelled table read without the proxy. |
| 185 | + order_qt_unlabeled "select count(*) from unlabeled\$labels" |
| 186 | + order_qt_direct "select count(*) from test_iceberg_rest_labels_direct.test_iceberg_rest_labels_db.labeled\$labels" |
| 187 | + // Without the table cache the labels come straight from the statement's own table load. |
| 188 | + order_qt_vended "select * from test_iceberg_rest_labels_vended.test_iceberg_rest_labels_db.labeled\$labels" |
| 189 | + |
| 190 | + // field_name follows the field id through schema evolution: a renamed column keeps its labels under its new |
| 191 | + // name, and a dropped one keeps its field_id with a NULL field_name. |
| 192 | + sql "alter table labeled rename column email email_address" |
| 193 | + sql "alter table labeled drop column phone" |
| 194 | + order_qt_schema_evolution "select * from labeled\$labels" |
| 195 | + // Labels are not versioned: a time-travel pin on t$labels is ignored, so the rows are those of the current |
| 196 | + // load and field names still come from the current schema, not from the schema of the pinned snapshot. |
| 197 | + def snapshotId = sql("select snapshot_id from labeled\$snapshots")[0][0] |
| 198 | + order_qt_time_travel "select * from labeled\$labels for version as of ${snapshotId}" |
| 199 | + |
| 200 | + // Labels belong to the table load, so they follow the table cache like the rest of the table: the cached |
| 201 | + // catalog keeps the labels it loaded until REFRESH TABLE, the uncached one sees new labels at once. |
| 202 | + labelsByTable.put("test_iceberg_rest_labels_db.labeled_empty", ["object-labels": ["owner": "team-c", "tier": "gold"]]) |
| 203 | + order_qt_cached "select `key`, value from labeled_empty\$labels" |
| 204 | + order_qt_uncached "select `key`, value from test_iceberg_rest_labels_nocache.test_iceberg_rest_labels_db.labeled_empty\$labels" |
| 205 | + sql "refresh table labeled_empty" |
| 206 | + order_qt_refreshed "select `key`, value from labeled_empty\$labels" |
| 207 | + |
| 208 | + // Only a REST catalog returns labels with the table load, so no other catalog offers t$labels. |
| 209 | + sql "drop catalog if exists test_iceberg_rest_labels_hadoop" |
| 210 | + sql """create catalog test_iceberg_rest_labels_hadoop properties ( |
| 211 | + 'type' = 'iceberg', 'iceberg.catalog.type' = 'hadoop', |
| 212 | + 'warehouse' = 's3a://warehouse/test_iceberg_rest_labels_hadoop', |
| 213 | + 's3.access_key' = 'admin', 's3.secret_key' = 'password', |
| 214 | + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.region' = 'us-east-1' |
| 215 | + )""" |
| 216 | + sql "create database if not exists test_iceberg_rest_labels_hadoop.test_iceberg_rest_labels_db" |
| 217 | + sql "drop table if exists test_iceberg_rest_labels_hadoop.test_iceberg_rest_labels_db.hadoop_table" |
| 218 | + sql "create table test_iceberg_rest_labels_hadoop.test_iceberg_rest_labels_db.hadoop_table (id int)" |
| 219 | + test { |
| 220 | + sql "select * from test_iceberg_rest_labels_hadoop.test_iceberg_rest_labels_db.hadoop_table\$labels" |
| 221 | + exception "Unknown sys table 'hadoop_table\$labels'" |
| 222 | + } |
| 223 | + |
| 224 | + // The proxy stops with the suite, so its catalogs go too; the tables stay readable through the direct one. |
| 225 | + sql "switch internal" |
| 226 | + sql "drop catalog test_iceberg_rest_labels" |
| 227 | + sql "drop catalog test_iceberg_rest_labels_nocache" |
| 228 | + sql "drop catalog test_iceberg_rest_labels_vended" |
| 229 | + } finally { |
| 230 | + proxy.stop(0) |
| 231 | + proxyExecutor.shutdownNow() |
| 232 | + } |
| 233 | +} |
0 commit comments