diff --git a/velox/common/file/FileSystems.cpp b/velox/common/file/FileSystems.cpp index aa738b6a43ba4..f4e84e587b16c 100644 --- a/velox/common/file/FileSystems.cpp +++ b/velox/common/file/FileSystems.cpp @@ -115,7 +115,8 @@ class LocalFileSystem : public FileSystem { std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options) override { + const FileOptions& options, + io::IoStatistics* ioStats) override { return std::make_unique( extractPath(path), executor_.get(), options.bufferIo); } diff --git a/velox/common/file/FileSystems.h b/velox/common/file/FileSystems.h index 1829215b625bc..45a54d1f60fd0 100644 --- a/velox/common/file/FileSystems.h +++ b/velox/common/file/FileSystems.h @@ -16,6 +16,7 @@ #pragma once #include "velox/common/base/Exceptions.h" +#include "velox/common/io/IoStatistics.h" #include "velox/common/memory/MemoryPool.h" #include @@ -103,7 +104,8 @@ class FileSystem { /// Returns a ReadFile handle for a given file path virtual std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options = {}) = 0; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) = 0; /// Returns a WriteFile handle for a given file path virtual std::unique_ptr openFileForWrite( diff --git a/velox/common/file/tests/FaultyFileSystem.h b/velox/common/file/tests/FaultyFileSystem.h index f85314c75e8f3..b0e39189b00f6 100644 --- a/velox/common/file/tests/FaultyFileSystem.h +++ b/velox/common/file/tests/FaultyFileSystem.h @@ -54,7 +54,8 @@ class FaultyFileSystem : public FileSystem { std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options = {}) override; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) override; std::unique_ptr openFileForWrite( std::string_view path, diff --git a/velox/common/io/IoStatistics.cpp b/velox/common/io/IoStatistics.cpp index 7dfddc6dc4831..2593ba44ce1af 100644 --- a/velox/common/io/IoStatistics.cpp +++ b/velox/common/io/IoStatistics.cpp @@ -108,6 +108,12 @@ IoStatistics::operationStats() const { return operationStats_; } +std::unordered_map IoStatistics::storageStats() + const { + std::lock_guard lock{storageStatsMutex_}; + return storageStats_; +} + void IoStatistics::merge(const IoStatistics& other) { rawBytesRead_ += other.rawBytesRead_; rawBytesWritten_ += other.rawBytesWritten_; diff --git a/velox/common/io/IoStatistics.h b/velox/common/io/IoStatistics.h index 2111a8877b475..045ef19b75fb0 100644 --- a/velox/common/io/IoStatistics.h +++ b/velox/common/io/IoStatistics.h @@ -23,6 +23,7 @@ #include #include +#include "velox/common/base/RuntimeMetrics.h" namespace facebook::velox::io { @@ -140,6 +141,7 @@ class IoStatistics { const uint64_t partialThrottleCount = 0); std::unordered_map operationStats() const; + std::unordered_map storageStats() const; void merge(const IoStatistics& other); @@ -172,7 +174,9 @@ class IoStatistics { IoCounter queryThreadIoLatency_; std::unordered_map operationStats_; + std::unordered_map storageStats_; mutable std::mutex operationStatsMutex_; + mutable std::mutex storageStatsMutex_; }; } // namespace facebook::velox::io diff --git a/velox/connectors/hive/HiveDataSource.cpp b/velox/connectors/hive/HiveDataSource.cpp index 0edfb3a0ea7cb..18993b3707858 100644 --- a/velox/connectors/hive/HiveDataSource.cpp +++ b/velox/connectors/hive/HiveDataSource.cpp @@ -513,6 +513,9 @@ std::unordered_map HiveDataSource::runtimeStats() { if (numBucketConversion_ > 0) { res.insert({"numBucketConversion", RuntimeCounter(numBucketConversion_)}); } + for (const auto& storageStats : ioStats_->storageStats()) { + res.insert({storageStats.first, storageStats.second}); + } return res; } diff --git a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h index c0d3d60ccdee5..096b26018b272 100644 --- a/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h @@ -40,7 +40,8 @@ class AbfsFileSystem : public FileSystem { std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options = {}) override; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) override; std::unique_ptr openFileForWrite( std::string_view path, diff --git a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h index 34daff8d6c64f..03fa8842f8779 100644 --- a/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/gcs/GcsFileSystem.h @@ -41,7 +41,8 @@ class GcsFileSystem : public FileSystem { /// [[https://cloud.google.com/storage/docs/samples/storage-stream-file-download]]. std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options = {}) override; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) override; /// Initialize a WriteFile /// First the method google::cloud::storage::Client::GetObjectMetadata diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h index b541ec629baf1..6c7ad7badeb23 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h @@ -55,7 +55,8 @@ class HdfsFileSystem : public FileSystem { std::unique_ptr openFileForRead( std::string_view path, - const FileOptions& options = {}) override; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) override; std::unique_ptr openFileForWrite( std::string_view path, diff --git a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h index c1e73198d48f1..66645047074b6 100644 --- a/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h +++ b/velox/connectors/hive/storage_adapters/s3fs/S3FileSystem.h @@ -37,7 +37,8 @@ class S3FileSystem : public FileSystem { std::unique_ptr openFileForRead( std::string_view s3Path, - const FileOptions& options = {}) override; + const FileOptions& options = {}, + io::IoStatistics* ioStats = {}) override; std::unique_ptr openFileForWrite( std::string_view s3Path,