|
| 1 | +import org.codehaus.groovy.runtime.IOGroovyMethods |
| 2 | + |
| 3 | +suite ("test_compaction_schema_change") { |
| 4 | + def tableName = "lineorder" |
| 5 | + def columns = """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, |
| 6 | + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, |
| 7 | + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""" |
| 8 | + |
| 9 | + sql """ DROP TABLE IF EXISTS ${tableName} """ |
| 10 | + |
| 11 | + sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( |
| 12 | + `lo_orderkey` bigint(20) NOT NULL COMMENT "", |
| 13 | + `lo_linenumber` bigint(20) NOT NULL COMMENT "", |
| 14 | + `lo_custkey` int(11) NOT NULL COMMENT "", |
| 15 | + `lo_partkey` int(11) NOT NULL COMMENT "", |
| 16 | + `lo_suppkey` int(11) NOT NULL COMMENT "", |
| 17 | + `lo_orderdate` int(11) NOT NULL COMMENT "", |
| 18 | + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", |
| 19 | + `lo_shippriority` int(11) NOT NULL COMMENT "", |
| 20 | + `lo_quantity` bigint(20) NOT NULL COMMENT "", |
| 21 | + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", |
| 22 | + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", |
| 23 | + `lo_discount` bigint(20) NOT NULL COMMENT "", |
| 24 | + `lo_revenue` bigint(20) NOT NULL COMMENT "", |
| 25 | + `lo_supplycost` bigint(20) NOT NULL COMMENT "", |
| 26 | + `lo_tax` bigint(20) NOT NULL COMMENT "", |
| 27 | + `lo_commitdate` bigint(20) NOT NULL COMMENT "", |
| 28 | + `lo_shipmode` varchar(11) NOT NULL COMMENT "" |
| 29 | + ) |
| 30 | + PARTITION BY RANGE(`lo_orderdate`) |
| 31 | + (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), |
| 32 | + PARTITION p1993 VALUES [("19930101"), ("19940101")), |
| 33 | + PARTITION p1994 VALUES [("19940101"), ("19950101")), |
| 34 | + PARTITION p1995 VALUES [("19950101"), ("19960101")), |
| 35 | + PARTITION p1996 VALUES [("19960101"), ("19970101")), |
| 36 | + PARTITION p1997 VALUES [("19970101"), ("19980101")), |
| 37 | + PARTITION p1998 VALUES [("19980101"), ("19990101"))) |
| 38 | + DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 1 |
| 39 | + PROPERTIES ( |
| 40 | + "replication_num" = "1" |
| 41 | + );""" |
| 42 | + |
| 43 | + Thread thread = new Thread(() -> { |
| 44 | + streamLoad { |
| 45 | + // a default db 'regression_test' is specified in |
| 46 | + // ${DORIS_HOME}/conf/regression-conf.groovy |
| 47 | + table tableName |
| 48 | + |
| 49 | + // default label is UUID: |
| 50 | + // set 'label' UUID.randomUUID().toString() |
| 51 | + |
| 52 | + // default column_separator is specify in doris fe config, usually is '\t'. |
| 53 | + // this line change to ',' |
| 54 | + set 'column_separator', '|' |
| 55 | + set 'compress_type', 'GZ' |
| 56 | + set 'columns', columns |
| 57 | + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. |
| 58 | + // also, you can stream load a http stream, e.g. http://xxx/some.csv |
| 59 | + file """${getS3Url() + '/regression/ssb/sf1/' + tableName}.tbl.gz""" |
| 60 | + |
| 61 | + time 10000 // limit inflight 10s |
| 62 | + |
| 63 | + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows |
| 64 | + |
| 65 | + // if declared a check callback, the default check condition will ignore. |
| 66 | + // So you must check all condition |
| 67 | + check { result, exception, startTime, endTime -> |
| 68 | + if (exception != null) { |
| 69 | + throw exception |
| 70 | + } |
| 71 | + log.info("Stream load result: ${result}".toString()) |
| 72 | + def json = parseJson(result) |
| 73 | + assertEquals("success", json.Status.toLowerCase()) |
| 74 | + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) |
| 75 | + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) |
| 76 | + } |
| 77 | + } |
| 78 | + }) |
| 79 | + |
| 80 | + Thread.sleep(60000) |
| 81 | + |
| 82 | + sql """ alter table ${tableName} add column test_column int default '1'; """ |
| 83 | + sql """ insert into ${tableName} values (-1, 1, 1, 1, 1, 1, '1', 1, 1, 1, 1, 1, 1, 1, 1, 1, '1', 100); """ |
| 84 | + qt_compaction """ select test_column from ${tableName} where lo_orderkey = -1; """ |
| 85 | + |
| 86 | + thread.join() |
| 87 | + |
| 88 | + // wait to auto compaction |
| 89 | + Thread.sleep(60000) |
| 90 | + |
| 91 | + qt_compaction """ select test_column from ${tableName} where lo_orderkey = -1; """ |
| 92 | +} |
0 commit comments