Safer IObjectStorage::tryGetObjectMetadata() (ignore only object existence error)

azat · azat · commit 94da831bc275 · 2025-09-25T18:24:08.000+02:00
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
@@ -325,6 +325,18 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path) c
     return result;
 }
 
+std::optional<ObjectMetadata> AzureObjectStorage::tryGetObjectMetadata(const std::string & path) const
+try
+{
+    return getObjectMetadata(path);
+}
+catch (const Azure::Storage::StorageException & e)
+{
+    if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound)
+        return {};
+    throw;
+}
+
 void AzureObjectStorage::copyObject( /// NOLINT
     const StoredObject & object_from,
     const StoredObject & object_to,
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
@@ -74,6 +74,8 @@ class AzureObjectStorage : public IObjectStorage
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
+    std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const override;
+
     ObjectStorageConnectionInfoPtr getConnectionInfo() const override;
 
     void copyObject( /// NOLINT
diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
@@ -202,6 +202,11 @@ ObjectMetadata CachedObjectStorage::getObjectMetadata(const std::string & path)
     return object_storage->getObjectMetadata(path);
 }
 
+std::optional<ObjectMetadata> CachedObjectStorage::tryGetObjectMetadata(const std::string & path) const
+{
+    return object_storage->tryGetObjectMetadata(path);
+}
+
 void CachedObjectStorage::shutdown()
 {
     object_storage->shutdown();
diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
@@ -67,6 +67,8 @@ class CachedObjectStorage final : public IObjectStorage
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
+    std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const override;
+
     ObjectStorageConnectionInfoPtr getConnectionInfo() const override { return object_storage->getConnectionInfo(); }
 
     void shutdown() override;
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
@@ -151,12 +151,26 @@ void HDFSObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 }
 
 ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string & path) const
+{
+    auto metadata = tryGetObjectMetadata(path);
+    if (!metadata.has_value())
+        throw Exception(ErrorCodes::HDFS_ERROR,
+                        "{} does not exist. Error: {}", path, hdfsGetLastError());
+    return metadata.value();
+}
+
+std::optional<ObjectMetadata> HDFSObjectStorage::tryGetObjectMetadata(const std::string & path) const
 {
     initializeHDFSFS();
     auto * file_info = wrapErr<hdfsFileInfo *>(hdfsGetPathInfo, hdfs_fs.get(), path.data());
     if (!file_info)
+    {
+        if (errno == ENOENT)
+            return {};
+
         throw Exception(ErrorCodes::HDFS_ERROR,
                         "Cannot get file info for: {}. Error: {}", path, hdfsGetLastError());
+    }
 
     ObjectMetadata metadata;
     metadata.size_bytes = static_cast<size_t>(file_info->mSize);
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
@@ -84,6 +84,8 @@ class HDFSObjectStorage : public IObjectStorage, public HDFSErrorWrapper
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
+    std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const override;
+
     void copyObject( /// NOLINT
         const StoredObject & object_from,
         const StoredObject & object_to,
diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp
@@ -44,18 +44,6 @@ ObjectStorageIteratorPtr IObjectStorage::iterate(const std::string & path_prefix
     return std::make_shared<ObjectStorageIteratorFromList>(std::move(files));
 }
 
-std::optional<ObjectMetadata> IObjectStorage::tryGetObjectMetadata(const std::string & path) const
-{
-    try
-    {
-        return getObjectMetadata(path);
-    }
-    catch (...)
-    {
-        return {};
-    }
-}
-
 ThreadPool & IObjectStorage::getThreadPoolWriter()
 {
     auto context = Context::getGlobalContextInstance();
diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h
@@ -197,14 +197,13 @@ class IObjectStorage
     /// List objects recursively by certain prefix. Use it instead of listObjects, if you want to list objects lazily.
     virtual ObjectStorageIteratorPtr iterate(const std::string & path_prefix, size_t max_keys) const;
 
-    /// Get object metadata if supported. It should be possible to receive
-    /// at least size of object
-    virtual std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const;
-
     /// Get object metadata if supported. It should be possible to receive
     /// at least size of object
     virtual ObjectMetadata getObjectMetadata(const std::string & path) const = 0;
 
+    /// Same as getObjectMetadata(), but ignores if object does not exist.
+    virtual std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const = 0;
+
     virtual ObjectStorageConnectionInfoPtr getConnectionInfo() const { return nullptr; }
 
     /// Read single object
diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
@@ -148,6 +148,29 @@ ObjectMetadata LocalObjectStorage::getObjectMetadata(const std::string & path) c
     return object_metadata;
 }
 
+std::optional<ObjectMetadata> LocalObjectStorage::tryGetObjectMetadata(const std::string & path) const
+{
+    ObjectMetadata object_metadata;
+    LOG_TEST(log, "Getting metadata for path: {}", path);
+
+    std::error_code error;
+    auto time = fs::last_write_time(path, error);
+    if (error)
+    {
+        if (error == std::errc::no_such_file_or_directory)
+            return {};
+        throw fs::filesystem_error("Got unexpected error while getting last write time", path, error);
+    }
+
+    /// no_such_file_or_directory is ignored only for last_write_time for consistency
+    object_metadata.size_bytes = fs::file_size(path);
+
+    object_metadata.etag = std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(time.time_since_epoch()).count());
+    object_metadata.last_modified = Poco::Timestamp::fromEpochTime(
+        std::chrono::duration_cast<std::chrono::seconds>(time.time_since_epoch()).count());
+    return object_metadata;
+}
+
 void LocalObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t/* max_keys */) const
 {
     if (!fs::exists(path) || !fs::is_directory(path))
diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h
@@ -61,6 +61,8 @@ class LocalObjectStorage : public IObjectStorage
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
+    std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const override;
+
     void listObjects(const std::string & path, RelativePathsWithMetadata & children, size_t max_keys) const override;
 
     bool existsOrHasAnyChild(const std::string & path) const override;
diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp
@@ -283,6 +283,11 @@ ObjectMetadata WebObjectStorage::getObjectMetadata(const std::string & /* path *
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Metadata is not supported for {}", getName());
 }
 
+std::optional<ObjectMetadata> WebObjectStorage::tryGetObjectMetadata(const std::string & /* path */) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Metadata is not supported for {}", getName());
+}
+
 ObjectStorageConnectionInfoPtr WebObjectStorage::getConnectionInfo() const
 {
     return getWebObjectStorageConnectionInfo(url);
diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h
@@ -51,6 +51,7 @@ class WebObjectStorage : public IObjectStorage, WithContext
     void removeObjectsIfExist(const StoredObjects & objects) override;
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
+    std::optional<ObjectMetadata> tryGetObjectMetadata(const std::string & path) const override;
 
     ObjectStorageConnectionInfoPtr getConnectionInfo() const override;
 

Original file line number	Diff line number	Diff line change
`@@ -202,6 +202,11 @@ ObjectMetadata CachedObjectStorage::getObjectMetadata(const std::string & path)`
`202`	`202`	`return object_storage->getObjectMetadata(path);`
`203`	`203`	`}`
`204`	`204`
	`205`	`+std::optional<ObjectMetadata> CachedObjectStorage::tryGetObjectMetadata(const std::string & path) const`
	`206`	`+{`
	`207`	`+ return object_storage->tryGetObjectMetadata(path);`
	`208`	`+}`
	`209`	`+`
`205`	`210`	`void CachedObjectStorage::shutdown()`
`206`	`211`	`{`
`207`	`212`	`object_storage->shutdown();`