-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Closed
Description
Search before asking
- I had searched in the issues and found no similar issues.
What happened
After updating to the current dev branch cluster, s3File failed to run the task
SeaTunnel Version
dev
SeaTunnel Config
{
"env": {
"job.name": "Excel输入测试",
"job.mode": "batch"
},
"preHandler": [
],
"source": [
{
"plugin_name": "S3File",
"resourceType": "file",
"path": "/file-node/upload/excel/669634f980e02dd229a3eb04.xlsx",
"fs.s3a.endpoint": "******",
"fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
"access_key": "P7HpCVa0Eth8X3dM",
"secret_key": "xxM9cJL5QKqq9M7jvwkQ5LJgsQSVvIay",
"bucket": "s3a://ottomi",
"file_format_type": "excel",
"skip_header_row_number": 1,
"date_format": "yyyy-MM-dd",
"datetime_format": "yyyy-MM-dd HH:mm:ss",
"time_format": "HH:mm:ss",
"schema": {
"fields": {
}
},
"resources": [
"testOut_0.xlsx"
],
"result_table_name": "ot_768f13511d68480b8ee8b4112f6efa6b",
"ottomi_node_type": "ExcelSource"
}
],
"transform": [
],
"sink": [
{
"file_format_type": "excel",
"sheet_name": "ottomi",
"custom_filename": true,
"file_name_expression": "testaaa",
"is_enable_transaction": false,
"batch_size": 1000000000,
"max_rows_in_memory ": 2048,
"plugin_name": "S3File",
"bucket": "s3a://ottomi",
"fs.s3a.endpoint": "******",
"fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
"access_key": "P7HpCVa0Eth8X3dM",
"secret_key": "xxM9cJL5QKqq9M7jvwkQ5LJgsQSVvIay",
"tmp_path": "/tmp/ottomi",
"path": "/file-node/download/1789908261255188482/excel/",
"result_table_name": "ot_1e0968446fb242608701085865694ca2",
"ottomi_node_type": "ExcelSink",
"source_table_name": [
"ot_768f13511d68480b8ee8b4112f6efa6b"
]
}
]
}
Running Command
bin/seatunnel.sh -c demo.jsonError Exception
Caused by: java.io.IOException: org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider instantiation exception: java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;Ljava/lang/Object;)V
at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:678) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:566) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:256) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3303) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:476) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:227) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.initialize(HadoopFileSystemProxy.java:222) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.getFileSystem(HadoopFileSystemProxy.java:171) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.listStatus(HadoopFileSystemProxy.java:154) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.source.reader.AbstractReadStrategy.getFileNamesByPath(AbstractReadStrategy.java:94) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig.parseFilePaths(BaseFileSourceConfig.java:73) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig.<init>(BaseFileSourceConfig.java:64) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.S3FileSourceConfig.<init>(S3FileSourceConfig.java:44) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.MultipleTableS3FileSourceConfig.getBaseSourceConfig(MultipleTableS3FileSourceConfig.java:32) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig.parseFromFileSourceConfig(BaseMultipleTableFileSourceConfig.java:54) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig.<init>(BaseMultipleTableFileSourceConfig.java:39) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.MultipleTableS3FileSourceConfig.<init>(MultipleTableS3FileSourceConfig.java:27) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.S3FileSource.<init>(S3FileSource.java:28) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.S3FileSourceFactory.lambda$createSource$0(S3FileSourceFactory.java:48) ~[?:?]
at org.apache.seatunnel.api.table.factory.FactoryUtil.createAndPrepareSource(FactoryUtil.java:112) ~[seatunnel-starter.jar:2.3.6-SNAPSHOT]
at org.apache.seatunnel.api.table.factory.FactoryUtil.createAndPrepareSource(FactoryUtil.java:73) ~[seatunnel-starter.jar:2.3.6-SNAPSHOT]
... 14 more
Caused by: java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;Ljava/lang/Object;)V
at org.apache.hadoop.fs.s3a.S3AUtils.lookupPassword(S3AUtils.java:742) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.<init>(SimpleAWSCredentialsProvider.java:59) ~[hadoop-aws-3.1.4.jar:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_212]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_212]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_212]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_212]
at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:635) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:566) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:256) ~[hadoop-aws-3.1.4.jar:?]
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3303) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:476) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:227) ~[seatunnel-hadoop3-3.1.4-uber.jar:2.3.6-SNAPSHOT]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.initialize(HadoopFileSystemProxy.java:222) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.getFileSystem(HadoopFileSystemProxy.java:171) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.listStatus(HadoopFileSystemProxy.java:154) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.source.reader.AbstractReadStrategy.getFileNamesByPath(AbstractReadStrategy.java:94) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig.parseFilePaths(BaseFileSourceConfig.java:73) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSourceConfig.<init>(BaseFileSourceConfig.java:64) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.S3FileSourceConfig.<init>(S3FileSourceConfig.java:44) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.MultipleTableS3FileSourceConfig.getBaseSourceConfig(MultipleTableS3FileSourceConfig.java:32) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig.parseFromFileSourceConfig(BaseMultipleTableFileSourceConfig.java:54) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.config.BaseMultipleTableFileSourceConfig.<init>(BaseMultipleTableFileSourceConfig.java:39) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.config.MultipleTableS3FileSourceConfig.<init>(MultipleTableS3FileSourceConfig.java:27) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.S3FileSource.<init>(S3FileSource.java:28) ~[?:?]
at org.apache.seatunnel.connectors.seatunnel.file.s3.source.S3FileSourceFactory.lambda$createSource$0(S3FileSourceFactory.java:48) ~[?:?]
at org.apache.seatunnel.api.table.factory.FactoryUtil.createAndPrepareSource(FactoryUtil.java:112) ~[seatunnel-starter.jar:2.3.6-SNAPSHOT]
at org.apache.seatunnel.api.table.factory.FactoryUtil.createAndPrepareSource(FactoryUtil.java:73) ~[seatunnel-starter.jar:2.3.6-SNAPSHOT]
... 14 more
Zeta or Flink or Spark Version
2.3.6-dev
Java or Scala Version
1.8
Screenshots
Are you willing to submit PR?
- Yes I am willing to submit a PR!
Code of Conduct
- I agree to follow this project's Code of Conduct

