Skip to content

datetime/datetime64 column timezone changed unexpectedly #39

@yokofly

Description

@yokofly

tp_time timezone changed incorrectly

CREATE STREAM normal
(
  `stock` int64,
  `_tp_time` datetime64(3, 'Asia/Shanghai') DEFAULT now64(3, 'Asia/Shanghai') CODEC(DoubleDelta, LZ4)
)

CREATE STREAM new_normal
(
  `stock` int64,
  `_tp_time` datetime64(3, 'Asia/Shanghai') DEFAULT now64(3, 'Asia/Shanghai') CODEC(DoubleDelta, LZ4)
)

insert into normal(stock) select number as stock FROM system.numbers LIMIT 10;
➜ ✗ ./sling run --src-conn LOCAL --src-stream normal --tgt-conn LOCAL --tgt-object new_normal --mode full-refresh -d
2025-06-23 02:46:12 DBG Using Timeplus version, skipping update check
2025-06-23 02:46:12 DBG conn credentials of LOCAL from sling env yaml overwritten by env var LOCAL
2025-06-23 02:46:12 DBG conn credentials of HISTORICAL_TIMEPLUS from sling env yaml overwritten by env var HISTORICAL_TIMEPLUS
2025-06-23 02:46:12 DBG Force SLING_PROCESS_BW to false for timeplus database
2025-06-23 02:46:12 DBG Sling version: 1.2.20-stable-timeplus (linux amd64)
2025-06-23 02:46:12 DBG type is db-db
2025-06-23 02:46:12 DBG using: {"columns":null,"mode":"full-refresh","transforms":null}
2025-06-23 02:46:12 DBG using source options: {"empty_as_null":false,"null_if":"NULL","datetime_format":"AUTO","max_decimals":11}
2025-06-23 02:46:12 DBG using target options: {"batch_limit":50000,"datetime_format":"auto","file_max_rows":0,"max_decimals":11,"use_bulk":true,"add_new_columns":true,"adjust_column_type":false,"column_casing":"source"}
2025-06-23 02:46:12 DBG opened "proton" connection (conn-proton-NXo)
2025-06-23 02:46:12 DBG opened "proton" connection (conn-proton-Yt2)
2025-06-23 02:46:12 INF connecting to source database (proton)
2025-06-23 02:46:12 INF connecting to target database (proton)
2025-06-23 02:46:12 INF Creating intermediate configuration
2025-06-23 02:46:12 INF Exporting data from source Proton database
2025-06-23 02:46:12 DBG proton to proton first stage: writedbtofile using source options: {"empty_as_null":false,"null_if":"NULL","datetime_format":"AUTO","max_decimals":11}
2025-06-23 02:46:12 DBG proton to proton first stage: writedbtofile using target options: {"header":true,"concurrency":7,"delimiter":"~","format":"csv","max_decimals":11,"use_bulk":true}
2025-06-23 02:46:12 DBG opened "proton" connection (conn-proton-Diu)
2025-06-23 02:46:12 INF connecting to source database (proton)
2025-06-23 02:46:12 INF reading from source database
2025-06-23 02:46:12 DBG select * from table(`default`.`normal`)
2025-06-23 02:46:12 INF writing to target file system (file)
2025-06-23 02:46:12 DBG opened "file" connection (conn-file-q3a)
2025-06-23 02:46:12 DBG writing to file:///app/timeplus_database_20250623_024612_normal_to_default.`new_normal`.csv [fileRowLimit=0 fileBytesLimit=0 compression= concurrency=7 useBufferedStream=false fileFormat=csv singleFile=true]
2025-06-23 02:46:12 DBG wrote 345 B: 10 rows [626 r/s]
2025-06-23 02:46:12 INF wrote 10 rows [625 r/s] to /app/timeplus_database_20250623_024612_normal_to_default.`new_normal`.csv
2025-06-23 02:46:12 DBG closed "proton" connection (conn-proton-NXo)
2025-06-23 02:46:12 DBG closed "proton" connection (conn-proton-Yt2)
2025-06-23 02:46:12 DBG closed "proton" connection (conn-proton-Diu)
2025-06-23 02:46:12 INF Checking exported data
2025-06-23 02:46:12 INF Preparing to import data to target Proton database
2025-06-23 02:46:12 INF Importing data to target Proton database
2025-06-23 02:46:12 DBG proton to proton second stage: filetodb using source options: {"header":true,"format":"csv","delimiter":"~","max_decimals":11}
2025-06-23 02:46:12 DBG proton to proton second stage: filetodb using target options: {"batch_limit":50000,"datetime_format":"auto","file_max_rows":0,"max_decimals":11,"use_bulk":true,"add_new_columns":true,"adjust_column_type":false,"column_casing":"source"}
2025-06-23 02:46:12 DBG opened "proton" connection (conn-proton-44G)
2025-06-23 02:46:12 INF connecting to target database (proton)
2025-06-23 02:46:12 INF reading from source file system (file)
2025-06-23 02:46:12 DBG opened "file" connection (conn-file-vOQ)
2025-06-23 02:46:12 DBG reading single datastream from file:///app/timeplus_database_20250623_024612_normal_to_default.`new_normal`.csv [format=csv]
2025-06-23 02:46:12 DBG merging csv readers of 1 files [concurrency=3] from file:///app/timeplus_database_20250623_024612_normal_to_default.`new_normal`.csv
2025-06-23 02:46:12 DBG processing reader from file:///app/timeplus_database_20250623_024612_normal_to_default.`new_normal`.csv
2025-06-23 02:46:12 INF writing to target database [mode: full-refresh]
2025-06-23 02:46:12 INF streaming data (direct insert)
2025-06-23 02:46:12 DBG use `default`
2025-06-23 02:46:12 DBG 10 ROWS COPIED
2025-06-23 02:46:17 DBG select count(*) as cnt from table(`default`.`new_normal`)
2025-06-23 02:46:17 INF inserted 10 rows into default.`new_normal` in 5 secs [2 r/s]
2025-06-23 02:46:17 DBG closed "proton" connection (conn-proton-44G)
2025-06-23 02:46:17 INF Transferred 10 rows between Proton databases in 5 secs [2 r/s]
2025-06-23 02:46:17 INF execution succeeded
➜ ✗ ../timeplusd client -q "select * from table(new_normal) FORMAT CSV"                                            
0,"2025-06-24 01:45:38.143"
1,"2025-06-24 01:45:38.143"
2,"2025-06-24 01:45:38.143"
3,"2025-06-24 01:45:38.143"
4,"2025-06-24 01:45:38.143"
5,"2025-06-24 01:45:38.143"
6,"2025-06-24 01:45:38.143"
7,"2025-06-24 01:45:38.143"
8,"2025-06-24 01:45:38.143"
9,"2025-06-24 01:45:38.143"
➜ ✗ ../timeplusd client -q "select * from table(normal) FORMAT CSV" 
0,"2025-06-23 17:45:38.143"
1,"2025-06-23 17:45:38.143"
2,"2025-06-23 17:45:38.143"
3,"2025-06-23 17:45:38.143"
4,"2025-06-23 17:45:38.143"
5,"2025-06-23 17:45:38.143"
6,"2025-06-23 17:45:38.143"
7,"2025-06-23 17:45:38.143"
8,"2025-06-23 17:45:38.143"
9,"2025-06-23 17:45:38.143"
➜ ✗ 

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions