Lazy init for StorageObjectStorageCluster

ianton-ru · ianton-ru · commit cf3e7748dd2f · 2025-08-05T16:54:47.000+02:00
diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp
@@ -445,7 +445,8 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
         context_copy,
         /* comment */"",
         getFormatSettings(context_copy),
-        LoadingStrictnessLevel::CREATE);
+        LoadingStrictnessLevel::CREATE,
+        /* lazy_init */true);
 }
 
 DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator(
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp
@@ -87,26 +87,52 @@ StorageObjectStorageCluster::StorageObjectStorageCluster(
     ContextPtr context_,
     const String & comment_,
     std::optional<FormatSettings> format_settings_,
-    LoadingStrictnessLevel mode_)
+    LoadingStrictnessLevel mode_,
+    bool lazy_init)
     : IStorageCluster(
         cluster_name_, table_id_, getLogger(fmt::format("{}({})", configuration_->getEngineName(), table_id_.table_name)))
     , configuration{configuration_}
     , object_storage(object_storage_)
     , cluster_name_in_settings(false)
 {
     configuration->initPartitionStrategy(partition_by, columns_in_table_or_function_definition, context_);
-    /// We allow exceptions to be thrown on update(),
-    /// because Cluster engine can only be used as table function,
-    /// so no lazy initialization is allowed.
-    configuration->update(
-        object_storage,
-        context_,
-        /* if_not_updated_before */false,
-        /* check_consistent_with_previous_metadata */true);
+
+    const bool need_resolve_columns_or_format = columns_in_table_or_function_definition.empty() || (configuration->getFormat() == "auto");
+    const bool do_lazy_init = lazy_init && !need_resolve_columns_or_format;
+
+    auto log_ = getLogger("StorageObjectStorageCluster");
+
+    try
+    {
+        if (!do_lazy_init)
+        {
+            /// We allow exceptions to be thrown on update(),
+            /// because Cluster engine can only be used as table function,
+            /// so no lazy initialization is allowed.
+            configuration->update(
+                object_storage,
+                context_,
+                /* if_not_updated_before */false,
+                /* check_consistent_with_previous_metadata */true);
+        }
+    }
+    catch (...)
+    {
+        // If we don't have format or schema yet, we can't ignore failed configuration update,
+        // because relevant configuration is crucial for format and schema inference
+        if (mode_ <= LoadingStrictnessLevel::CREATE || need_resolve_columns_or_format)
+        {
+            throw;
+        }
+        tryLogCurrentException(log_);
+    }
 
     ColumnsDescription columns{columns_in_table_or_function_definition};
     std::string sample_path;
-    resolveSchemaAndFormat(columns, object_storage, configuration, {}, sample_path, context_);
+    if (need_resolve_columns_or_format)
+        resolveSchemaAndFormat(columns, object_storage, configuration, {}, sample_path, context_);
+    else
+        validateSupportedColumns(columns, *configuration);
     configuration->check(context_);
 
     if (sample_path.empty() && context_->getSettingsRef()[Setting::use_hive_partitioning] && !configuration->isDataLakeConfiguration() && !configuration->getPartitionStrategy())
@@ -162,7 +188,9 @@ StorageObjectStorageCluster::StorageObjectStorageCluster(
         format_settings_,
         mode_,
         /* distributed_processing */false,
-        partition_by);
+        partition_by,
+        /* is_table_function */false,
+        /* lazy_init */lazy_init);
 
     auto virtuals_ = getVirtualsPtr();
     if (virtuals_)
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h
@@ -23,7 +23,8 @@ class StorageObjectStorageCluster : public IStorageCluster
         ContextPtr context_,
         const String & comment_,
         std::optional<FormatSettings> format_settings_,
-        LoadingStrictnessLevel mode_);
+        LoadingStrictnessLevel mode_,
+        bool lazy_init = false);
 
     std::string getName() const override;
 
diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.cpp b/src/TableFunctions/TableFunctionObjectStorageCluster.cpp
@@ -47,7 +47,8 @@ StoragePtr TableFunctionObjectStorageCluster<Definition, Configuration, is_data_
             /* mode */ LoadingStrictnessLevel::CREATE,
             /* distributed_processing */ true,
             /* partition_by_ */Base::partition_by,
-            /* is_table_function */true);
+            /* is_table_function */true,
+            /* lazy_init */ true);
     }
     else
     {
@@ -62,7 +63,8 @@ StoragePtr TableFunctionObjectStorageCluster<Definition, Configuration, is_data_
             context,
             /* comment */ String{},
             /* format_settings */ std::nullopt, /// No format_settings
-            /* mode */ LoadingStrictnessLevel::CREATE);
+            /* mode */ LoadingStrictnessLevel::CREATE,
+            /* lazy_init */ true);
     }
 
     storage->startup();

Original file line number	Diff line number	Diff line change
`@@ -445,7 +445,8 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con`
`445`	`445`	`context_copy,`
`446`	`446`	`/* comment */"",`
`447`	`447`	`getFormatSettings(context_copy),`
`448`		`- LoadingStrictnessLevel::CREATE);`
	`448`	`+ LoadingStrictnessLevel::CREATE,`
	`449`	`+ /* lazy_init */true);`
`449`	`450`	`}`
`450`	`451`
`451`	`452`	`DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator(`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,8 @@ StoragePtr TableFunctionObjectStorageCluster<Definition, Configuration, is_data_`
`47`	`47`	`/* mode */ LoadingStrictnessLevel::CREATE,`
`48`	`48`	`/* distributed_processing */ true,`
`49`	`49`	`/* partition_by_ */Base::partition_by,`
`50`		`- /* is_table_function */true);`
	`50`	`+ /* is_table_function */true,`
	`51`	`+ /* lazy_init */ true);`
`51`	`52`	`}`
`52`	`53`	`else`
`53`	`54`	`{`
`@@ -62,7 +63,8 @@ StoragePtr TableFunctionObjectStorageCluster<Definition, Configuration, is_data_`
`62`	`63`	`context,`
`63`	`64`	`/* comment */ String{},`
`64`	`65`	`/* format_settings */ std::nullopt, /// No format_settings`
`65`		`- /* mode */ LoadingStrictnessLevel::CREATE);`
	`66`	`+ /* mode */ LoadingStrictnessLevel::CREATE,`
	`67`	`+ /* lazy_init */ true);`
`66`	`68`	`}`
`67`	`69`
`68`	`70`	`storage->startup();`