diff --git a/.asf.yaml b/.asf.yaml index 55f588ddb5d6..7d40bfd59e32 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -36,8 +36,6 @@ github: main: required_pull_request_reviews: required_approving_review_count: 1 - gh-pages: - whatever: Just a placeholder to make it take effects custom_subjects: new_discussion: "{title}" edit_discussion: "Re: {title}" diff --git a/.github/workflows/ci_bindings_ruby.yml b/.github/workflows/ci_bindings_ruby.yml index 5dcaf608de0f..13238e65c834 100644 --- a/.github/workflows/ci_bindings_ruby.yml +++ b/.github/workflows/ci_bindings_ruby.yml @@ -42,17 +42,27 @@ permissions: jobs: test: runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby-version: ["3.1", "3.2", "3.3"] + env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps BUNDLE_GEMFILE: ${{ github.workspace }}/bindings/ruby/Gemfile + steps: - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.1' + ruby-version: ${{ matrix.ruby-version }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Setup Rust toolchain uses: ./.github/actions/setup - - name: Bundle with rake - working-directory: "bindings/ruby" + + - name: Run tests and lint + working-directory: bindings/ruby run: | bundle exec rake diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2ca5beeabbf9..2fb8660c0fcb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -631,6 +631,8 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: website/build publish_branch: gh-pages + # This allows us to make our publish branch with only the latest commit. + force_orphan: true - name: Clear build run: rm -rf ./website/build diff --git a/bindings/ruby/.gitignore b/bindings/ruby/.gitignore index f96f9a8ab876..6661c2f1f89a 100644 --- a/bindings/ruby/.gitignore +++ b/bindings/ruby/.gitignore @@ -4,10 +4,9 @@ /coverage/ /doc/ /pkg/ -/spec/reports/ /tmp/ +/target/ *.bundle *.so -.rspec_status Gemfile.lock Cargo.lock diff --git a/bindings/ruby/.rspec b/bindings/ruby/.rspec deleted file mode 100644 index 34c5164d9b56..000000000000 --- a/bindings/ruby/.rspec +++ /dev/null @@ -1,3 +0,0 @@ ---format documentation ---color ---require spec_helper diff --git a/bindings/ruby/.standard.yml b/bindings/ruby/.standard.yml index 3450e5599330..c199ca278772 100644 --- a/bindings/ruby/.standard.yml +++ b/bindings/ruby/.standard.yml @@ -17,4 +17,4 @@ # For available configuration options, see: # https://github.com/testdouble/standard -ruby_version: 2.6 +ruby_version: 3.1 diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml index d9fe81be04b5..50c6842af66e 100644 --- a/bindings/ruby/Cargo.toml +++ b/bindings/ruby/Cargo.toml @@ -33,7 +33,7 @@ doc = false name = "opendal_ruby" [dependencies] -magnus = { version = "0.5", features = ["bytes-crate"] } +magnus = { version = "0.7.1", features = ["bytes"] } # this crate won't be published, we always use the local version opendal = { version = ">=0", path = "../../core", features = [ # These are default features before v0.46. TODO: change to optional features @@ -53,7 +53,9 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-webhdfs", "services-azfile", ] } -rb-sys = { version = "0.9.77", default-features = false } +rb-sys = { version = "0.9.102", default-features = false } +# should be in sync with opendal +bytes = "1.6" [build-dependencies] rb-sys-env = "0.1.2" diff --git a/bindings/ruby/Gemfile b/bindings/ruby/Gemfile index 0d12b266c4b6..5f8523302722 100644 --- a/bindings/ruby/Gemfile +++ b/bindings/ruby/Gemfile @@ -19,11 +19,15 @@ source "https://rubygems.org" -# Specify your gem's dependencies in opendal.gemspec +# Includes runtime dependencies from opendal.gemspec gemspec -gem "rake", "~> 13.0" -gem "rake-compiler", "~> 1.2.0" -gem "rspec", "~> 3.12.0" -gem "standard", "~> 1.3" -gem "rb_sys", "~> 0.9.39" +group :development, :test do + gem "rake", ">= 13.0" + gem "rb_sys", "~> 0.9.102" # for Makefile generation in extconf.rb + gem "rake-compiler", "~> 1.2.0" + gem "minitest", "~> 5.25.0" + gem "minitest-reporters", "~> 1.7.1" + gem "activesupport", "~> 7.2.1" + gem "standard", "~> 1.3" +end diff --git a/bindings/ruby/README.md b/bindings/ruby/README.md index f45b455c312a..334fa8936abc 100644 --- a/bindings/ruby/README.md +++ b/bindings/ruby/README.md @@ -17,13 +17,13 @@ bundle Build bindings: ```shell -rake compile +bundle exec rake compile ``` Run tests: ```shell -rake spec +bundle exec rake spec ``` ## License and Trademarks diff --git a/bindings/ruby/Rakefile b/bindings/ruby/Rakefile index 16d56ec23286..802064805b2f 100644 --- a/bindings/ruby/Rakefile +++ b/bindings/ruby/Rakefile @@ -17,18 +17,26 @@ # frozen_string_literal: true +require "bundler/gem_tasks" require "rake/testtask" require "rake/extensiontask" -require "rspec/core/rake_task" -require "bundler/gem_tasks" require "standard/rake" -RSpec::Core::RakeTask.new(:spec) +GEMSPEC = Gem::Specification.load("opendal.gemspec") +CRATE_PACKAGE_NAME = "opendal-ruby" -Rake::ExtensionTask.new do |ext| +Rake::ExtensionTask.new(CRATE_PACKAGE_NAME, GEMSPEC) do |ext| ext.name = "opendal_ruby" ext.ext_dir = "." ext.lib_dir = "lib/opendal_ruby" end -task default: %i[clobber compile spec standard] +Rake::Task[:test].prerequisites << :compile + +Rake::TestTask.new do |t| + t.libs << "lib" + t.libs << "test" + t.pattern = "test/**/*_test.rb" +end + +task default: %i[clobber compile test standard] diff --git a/bindings/ruby/extconf.rb b/bindings/ruby/extconf.rb index 98ec2ea83d32..7b6e02c41198 100644 --- a/bindings/ruby/extconf.rb +++ b/bindings/ruby/extconf.rb @@ -16,6 +16,9 @@ # under the License. require "mkmf" +# We use rb_sys for makefile generation only. +# We can use `RB_SYS_CARGO_PROFILE` to choose Cargo profile +# Read more https://github.com/oxidize-rb/rb-sys/blob/main/gem/README.md require "rb_sys/mkmf" create_rust_makefile("opendal_ruby/opendal_ruby") diff --git a/bindings/ruby/opendal.gemspec b/bindings/ruby/opendal.gemspec index 59c25e684b76..df716d91f6c7 100644 --- a/bindings/ruby/opendal.gemspec +++ b/bindings/ruby/opendal.gemspec @@ -27,7 +27,6 @@ Gem::Specification.new do |spec| spec.summary = "OpenDAL Ruby Binding" spec.homepage = "https://opendal.apache.org/" - spec.required_ruby_version = ">= 2.6.0" spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'" @@ -46,16 +45,19 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - # Uncomment to register a new dependency of your gem - # spec.add_dependency "example-gem", "~> 1.0" spec.extensions = ["./extconf.rb"] - # needed until rubygems supports Rust support is out of beta - spec.add_dependency "rb_sys", "~> 0.9.39" - - # only needed when developing or packaging your gem - spec.add_development_dependency "rake-compiler", "~> 1.2.0" - - # For more information and examples about making a new gem, check out our - # guide at: https://bundler.io/guides/creating_gem.html + # Rubygems is a default gem that is a part of Ruby core. + # Rubygems 3.3.11 supports building gem with Cargo. + # Read more https://github.com/rubygems/rubygems/blob/master/CHANGELOG.md#3311--2022-04-07 + # + # Ruby 3.1.3 includes Rubygems 3.3.26 + # Read more https://stdgems.org/3.1.3/ + # + # use a Ruby version which: + # - supports new Rubygems with the ability of compilation of Rust gem + # - not end of life + spec.required_ruby_version = ">= 3.1.3" + + # intentionally skipping rb_sys gem because newer Rubygems will be present end diff --git a/bindings/ruby/src/capability.rs b/bindings/ruby/src/capability.rs new file mode 100644 index 000000000000..01a9676b2a97 --- /dev/null +++ b/bindings/ruby/src/capability.rs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use magnus::class; +use magnus::method; +use magnus::prelude::*; +use magnus::Error; +use magnus::RModule; + +use crate::*; + +// This name follows `attr_accessor` in Ruby +macro_rules! define_accessors { + ($struct:ty, { $( $field:ident : $type:ty ),+ $(,)? }) => { + impl $struct { + $( + pub fn $field(&self) -> $type { + self.0.$field + } + )+ + } + }; +} +macro_rules! bind_methods_to_ruby { + ($ruby_class:ident, { $( $field:ident ),+ $(,)? }) => { + $( + $ruby_class.define_method(stringify!($field), method!(Capability::$field, 0))?; + )+ + }; +} + +/// Capability describes OpenDAL supported operations by current Operator. +#[magnus::wrap(class = "OpenDAL::Capability", free_immediately, size)] +pub struct Capability(ocore::Capability); + +impl Capability { + pub fn new(capability: ocore::Capability) -> Self { + Self(capability) + } +} + +define_accessors!(Capability, { + stat: bool, + stat_with_if_match: bool, + stat_with_if_none_match: bool, + stat_with_override_cache_control: bool, + stat_with_override_content_disposition: bool, + stat_with_override_content_type: bool, + stat_with_version: bool, + read: bool, + read_with_if_match: bool, + read_with_if_none_match: bool, + read_with_override_cache_control: bool, + read_with_override_content_disposition: bool, + read_with_override_content_type: bool, + read_with_version: bool, + write: bool, + write_can_multi: bool, + write_can_empty: bool, + write_can_append: bool, + write_with_content_type: bool, + write_with_content_disposition: bool, + write_with_cache_control: bool, + write_with_if_none_match: bool, + write_with_user_metadata: bool, + write_multi_max_size: Option, + write_multi_min_size: Option, + write_multi_align_size: Option, + write_total_max_size: Option, + create_dir: bool, + delete: bool, + delete_with_version: bool, + copy: bool, + rename: bool, + list: bool, + list_with_limit: bool, + list_with_start_after: bool, + list_with_recursive: bool, + list_with_version: bool, + presign: bool, + presign_read: bool, + presign_stat: bool, + presign_write: bool, + batch: bool, + batch_delete: bool, + batch_max_operations: Option, + blocking: bool +}); + +// includes class into the Ruby module +pub fn include(gem_module: &RModule) -> Result<(), Error> { + let class = gem_module.define_class("Capability", class::object())?; + bind_methods_to_ruby!(class, { + stat, + stat_with_if_match, + stat_with_if_none_match, + stat_with_override_cache_control, + stat_with_override_content_disposition, + stat_with_override_content_type, + stat_with_version, + read, + read_with_if_match, + read_with_if_none_match, + read_with_override_cache_control, + read_with_override_content_disposition, + read_with_override_content_type, + read_with_version, + write, + write_can_multi, + write_can_empty, + write_can_append, + write_with_content_type, + write_with_content_disposition, + write_with_cache_control, + write_with_if_none_match, + write_with_user_metadata, + write_multi_max_size, + write_multi_min_size, + write_multi_align_size, + write_total_max_size, + create_dir, + delete, + delete_with_version, + copy, + rename, + list, + list_with_limit, + list_with_start_after, + list_with_recursive, + list_with_version, + presign, + presign_read, + presign_stat, + presign_write, + batch, + batch_delete, + batch_max_operations, + blocking + }); + + Ok(()) +} diff --git a/bindings/ruby/src/lib.rs b/bindings/ruby/src/lib.rs index c47e9af88a5a..5de42706927a 100644 --- a/bindings/ruby/src/lib.rs +++ b/bindings/ruby/src/lib.rs @@ -15,124 +15,30 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; -use std::str::FromStr; - -use magnus::class; -use magnus::define_module; -use magnus::error::Result; use magnus::exception; use magnus::function; -use magnus::method; -use magnus::prelude::*; use magnus::Error; -use magnus::RString; -use opendal as od; - -#[magnus::wrap(class = "OpenDAL::Operator", free_immediately, size)] -#[derive(Clone, Debug)] -pub struct Operator(od::BlockingOperator); - -impl Operator { - pub fn new(scheme: String, options: Option>) -> Result { - let scheme = od::Scheme::from_str(&scheme) - .map_err(|err| { - od::Error::new(od::ErrorKind::Unexpected, "unsupported scheme").set_source(err) - }) - .map_err(format_magnus_error)?; - let options = options.unwrap_or_default(); - - let op = od::Operator::via_iter(scheme, options) - .map_err(format_magnus_error)? - .blocking(); - Ok(Operator(op)) - } - - /// Read the whole path into string. - pub fn read(&self, path: String) -> Result { - let bytes = self.0.read(&path).map_err(format_magnus_error)?; - Ok(RString::from_slice(&bytes.to_vec())) - } - - /// Write string into given path. - pub fn write(&self, path: String, bs: RString) -> Result<()> { - self.0 - .write(&path, bs.to_bytes()) - .map_err(format_magnus_error) - } - - /// Get current path's metadata **without cache** directly. - pub fn stat(&self, path: String) -> Result { - self.0 - .stat(&path) - .map_err(format_magnus_error) - .map(Metadata) - } -} +use magnus::Ruby; -#[magnus::wrap(class = "OpenDAL::Metadata", free_immediately, size)] -pub struct Metadata(od::Metadata); +// We will use `ocore::` to represents opendal rust core functionalities. +// This convention aligns with the Python binding. +pub use ::opendal as ocore; -impl Metadata { - /// Content-Disposition of this object - pub fn content_disposition(&self) -> Option<&str> { - self.0.content_disposition() - } - - /// Content length of this entry. - pub fn content_length(&self) -> u64 { - self.0.content_length() - } - - /// Content MD5 of this entry. - pub fn content_md5(&self) -> Option<&str> { - self.0.content_md5() - } - - /// Content Type of this entry. - pub fn content_type(&self) -> Option<&str> { - self.0.content_type() - } - - /// ETag of this entry. - pub fn etag(&self) -> Option<&str> { - self.0.etag() - } - - /// Returns `True` if this is a file. - pub fn is_file(&self) -> bool { - self.0.is_file() - } - - /// Returns `True` if this is a directory. - pub fn is_dir(&self) -> bool { - self.0.is_dir() - } -} +mod capability; +mod metadata; +mod operator; -fn format_magnus_error(err: od::Error) -> Error { +pub fn format_magnus_error(err: ocore::Error) -> Error { Error::new(exception::runtime_error(), err.to_string()) } +/// Apache OpenDAL™ Ruby binding #[magnus::init] -fn init() -> Result<()> { - let namespace = define_module("OpenDAL")?; - let operator_class = namespace.define_class("Operator", class::object())?; - operator_class.define_singleton_method("new", function!(Operator::new, 2))?; - operator_class.define_method("read", method!(Operator::read, 1))?; - operator_class.define_method("write", method!(Operator::write, 2))?; - operator_class.define_method("stat", method!(Operator::stat, 1))?; +fn init(ruby: &Ruby) -> Result<(), Error> { + let gem_module = ruby.define_module("OpenDAL")?; + let _ = operator::include(&gem_module); + let _ = metadata::include(&gem_module); + let _ = capability::include(&gem_module); - let metadata_class = namespace.define_class("Metadata", class::object())?; - metadata_class.define_method( - "content_disposition", - method!(Metadata::content_disposition, 0), - )?; - metadata_class.define_method("content_length", method!(Metadata::content_length, 0))?; - metadata_class.define_method("content_md5", method!(Metadata::content_md5, 0))?; - metadata_class.define_method("content_type", method!(Metadata::content_type, 0))?; - metadata_class.define_method("etag", method!(Metadata::etag, 0))?; - metadata_class.define_method("is_file", method!(Metadata::is_file, 0))?; - metadata_class.define_method("is_dir", method!(Metadata::is_dir, 0))?; Ok(()) } diff --git a/bindings/ruby/src/metadata.rs b/bindings/ruby/src/metadata.rs new file mode 100644 index 000000000000..c9e495772811 --- /dev/null +++ b/bindings/ruby/src/metadata.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use magnus::class; +use magnus::method; +use magnus::prelude::*; +use magnus::Error; +use magnus::RModule; + +use crate::*; + +#[magnus::wrap(class = "OpenDAL::Metadata", free_immediately, size)] +pub struct Metadata(ocore::Metadata); + +impl Metadata { + pub fn new(metadata: ocore::Metadata) -> Self { + Self(metadata) + } +} + +impl Metadata { + /// Content-Disposition of this object + pub fn content_disposition(&self) -> Option<&str> { + self.0.content_disposition() + } + + /// Content length of this entry. + pub fn content_length(&self) -> u64 { + self.0.content_length() + } + + /// Content MD5 of this entry. + pub fn content_md5(&self) -> Option<&str> { + self.0.content_md5() + } + + /// Content Type of this entry. + pub fn content_type(&self) -> Option<&str> { + self.0.content_type() + } + + /// ETag of this entry. + pub fn etag(&self) -> Option<&str> { + self.0.etag() + } + + /// Returns `True` if this is a file. + pub fn is_file(&self) -> bool { + self.0.is_file() + } + + /// Returns `True` if this is a directory. + pub fn is_dir(&self) -> bool { + self.0.is_dir() + } +} + +pub fn include(gem_module: &RModule) -> Result<(), Error> { + let class = gem_module.define_class("Metadata", class::object())?; + class.define_method( + "content_disposition", + method!(Metadata::content_disposition, 0), + )?; + class.define_method("content_length", method!(Metadata::content_length, 0))?; + class.define_method("content_md5", method!(Metadata::content_md5, 0))?; + class.define_method("content_type", method!(Metadata::content_type, 0))?; + class.define_method("etag", method!(Metadata::etag, 0))?; + class.define_method("file?", method!(Metadata::is_file, 0))?; + class.define_method("dir?", method!(Metadata::is_dir, 0))?; + + Ok(()) +} diff --git a/bindings/ruby/src/operator.rs b/bindings/ruby/src/operator.rs new file mode 100644 index 000000000000..997b55a607b6 --- /dev/null +++ b/bindings/ruby/src/operator.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::str::FromStr; + +use magnus::class; +use magnus::method; +use magnus::prelude::*; +use magnus::Error; +use magnus::RModule; +use magnus::RString; + +use crate::capability::Capability; +use crate::metadata::Metadata; +use crate::*; + +#[magnus::wrap(class = "OpenDAL::Operator", free_immediately, size)] +#[derive(Clone, Debug)] +struct Operator(ocore::BlockingOperator); + +impl Operator { + fn new(scheme: String, options: Option>) -> Result { + let scheme = ocore::Scheme::from_str(&scheme) + .map_err(|err| { + ocore::Error::new(ocore::ErrorKind::Unexpected, "unsupported scheme") + .set_source(err) + }) + .map_err(format_magnus_error)?; + let options = options.unwrap_or_default(); + + let op = ocore::Operator::via_iter(scheme, options) + .map_err(format_magnus_error)? + .blocking(); + Ok(Operator(op)) + } + + /// Reads the whole path into string. + fn read(&self, path: String) -> Result { + let buffer = self.0.read(&path).map_err(format_magnus_error)?; + Ok(buffer.to_bytes()) + } + + /// Writes string into given path. + fn write(&self, path: String, bs: RString) -> Result<(), Error> { + self.0 + .write(&path, bs.to_bytes()) + .map_err(format_magnus_error) + } + + /// Gets current path's metadata **without cache** directly. + fn stat(&self, path: String) -> Result { + self.0 + .stat(&path) + .map_err(format_magnus_error) + .map(Metadata::new) + } + + /// Gets capabilities of the current operator + fn capability(&self) -> Result { + let capability = self.0.info().full_capability(); + Ok(Capability::new(capability)) + } + + /// Creates directory recursively similar as `mkdir -p` + /// The ending path must be `/`. Otherwise, OpenDAL throws `NotADirectory` error. + fn create_dir(&self, path: String) -> Result<(), Error> { + self.0.create_dir(&path).map_err(format_magnus_error) + } + + /// Deletes given path + fn delete(&self, path: String) -> Result<(), Error> { + self.0.delete(&path).map_err(format_magnus_error) + } + + /// Returns if this path exists + fn exists(&self, path: String) -> Result { + self.0.exists(&path).map_err(format_magnus_error) + } + + /// Renames a file from `from` to `to` + fn rename(&self, from: String, to: String) -> Result<(), Error> { + self.0.rename(&from, &to).map_err(format_magnus_error) + } + + /// Removes the path and all nested directories and files recursively + fn remove_all(&self, path: String) -> Result<(), Error> { + self.0.remove_all(&path).map_err(format_magnus_error) + } + + /// Copies a file from `from` to `to`. + fn copy(&self, from: String, to: String) -> Result<(), Error> { + self.0.copy(&from, &to).map_err(format_magnus_error) + } +} + +pub fn include(gem_module: &RModule) -> Result<(), Error> { + let class = gem_module.define_class("Operator", class::object())?; + class.define_singleton_method("new", function!(Operator::new, 2))?; + class.define_method("read", method!(Operator::read, 1))?; + class.define_method("write", method!(Operator::write, 2))?; + class.define_method("stat", method!(Operator::stat, 1))?; + class.define_method("capability", method!(Operator::capability, 0))?; + class.define_method("create_dir", method!(Operator::create_dir, 1))?; + class.define_method("delete", method!(Operator::delete, 1))?; + class.define_method("exist?", method!(Operator::exists, 1))?; + class.define_method("rename", method!(Operator::rename, 2))?; + class.define_method("remove_all", method!(Operator::remove_all, 1))?; + class.define_method("copy", method!(Operator::copy, 2))?; + + Ok(()) +} diff --git a/bindings/ruby/test/blocking_op_test.rb b/bindings/ruby/test/blocking_op_test.rb new file mode 100644 index 000000000000..b6cee4d716c4 --- /dev/null +++ b/bindings/ruby/test/blocking_op_test.rb @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" + +class OpenDalTest < ActiveSupport::TestCase + setup do + @root = Dir.mktmpdir + File.write("#{@root}/sample", "Sample data for testing") + @op = OpenDAL::Operator.new("fs", {"root" => @root}) + end + + teardown do + FileUtils.remove_entry(@root) if File.exist?(@root) + end + + test "write writes to a file" do + @op.write("/file", "OpenDAL Ruby is ready.") + + assert_equal "OpenDAL Ruby is ready.", File.read("/#{@root}/file") + end + + test "write writes binary" do + # writes 32-bit signed integers + @op.write("/file", [67305985, -50462977].pack("l*")) + + assert_equal [67305985, -50462977], File.binread("/#{@root}/file").unpack("l*") + end + + test "read reads file data" do + data = @op.read("/sample") + + assert_equal "Sample data for testing", data + end + + test "stat returns file metadata" do + stat = @op.stat("sample") + + assert stat.is_a?(OpenDAL::Metadata) + assert_equal 23, stat.content_length + assert stat.file? + end + + test "create_dir creates directory" do + @op.create_dir("/new/directory/") + assert File.directory?("#{@root}/new/directory/") + end + + test "exists returns existence" do + assert @op.exist?("sample") + end + + test "delete removes file" do + @op.delete("/sample") + + assert_not File.exist?("#{@root}/sample") + end + + test "rename renames file" do + @op.rename("/sample", "/new_name") + + assert_not File.exist?("#{@root}/sample") + assert File.exist?("#{@root}/new_name") + end + + test "remove_all removes files" do + @op.create_dir("/nested/directory/") + @op.write("/nested/directory/text", "content") + + @op.remove_all("/") + + assert_not File.exist?(@root) + end + + test "copy copies file" do + @op.copy("/sample", "/new_name") + + assert File.exist?("#{@root}/sample") + assert File.exist?("#{@root}/new_name") + end +end diff --git a/bindings/ruby/spec/blocking_op_spec.rb b/bindings/ruby/test/capability_test.rb similarity index 69% rename from bindings/ruby/spec/blocking_op_spec.rb rename to bindings/ruby/test/capability_test.rb index 8d555ddf4534..f1bf43fe4d70 100644 --- a/bindings/ruby/spec/blocking_op_spec.rb +++ b/bindings/ruby/test/capability_test.rb @@ -17,20 +17,26 @@ # frozen_string_literal: true -RSpec.describe OpenDAL do - before :each do +require "test_helper" + +class CapabilityTest < ActiveSupport::TestCase + setup do @op = OpenDAL::Operator.new("memory", nil) end - it "should perform basic ops" do - path = "/path/to/file" - content = "OpenDAL Ruby is ready." - @op.write(path, content) + test "has read capability" do + capability = @op.capability + + assert_not capability.nil? + assert capability.read + end - stat = @op.stat(path) - expect(stat.is_file).to eq(true) - expect(stat.content_length).to eq(content.length) + test "doesn't respond to undefined capability" do + capability = @op.capability - expect(@op.read(path)).to eq(content) + assert_not capability.nil? + assert_raises(NoMethodError) do + capability.not_exist + end end end diff --git a/bindings/ruby/spec/spec_helper.rb b/bindings/ruby/test/test_helper.rb similarity index 65% rename from bindings/ruby/spec/spec_helper.rb rename to bindings/ruby/test/test_helper.rb index 16a653f4ca15..10ae46e8b5fb 100644 --- a/bindings/ruby/spec/spec_helper.rb +++ b/bindings/ruby/test/test_helper.rb @@ -17,16 +17,20 @@ # frozen_string_literal: true -require "opendal" +require "active_support" +require "minitest/autorun" +require "minitest/reporters" -RSpec.configure do |config| - # Enable flags like --only-failures and --next-failure - config.example_status_persistence_file_path = ".rspec_status" +Minitest::Reporters.use!([Minitest::Reporters::DefaultReporter.new(color: true)]) - # Disable RSpec exposing methods globally on `Module` and `main` - config.disable_monkey_patching! +require "opendal" - config.expect_with :rspec do |c| - c.syntax = :expect - end +# Uses `ActiveSupport::TestCase` for additional features including: +# - additional assertions +# - file fixtures +# - parallel worker +# +# Read more https://edgeapi.rubyonrails.org/classes/ActiveSupport/TestCase.html +class ActiveSupport::TestCase + parallelize(workers: :number_of_processors) end diff --git a/core/Cargo.lock b/core/Cargo.lock index 96b630981cbf..deda3e10257b 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.79", + "syn 2.0.81", ] [[package]] @@ -5949,7 +5949,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.81", "version_check", "yansi", ] diff --git a/core/src/docs/rfcs/3911_deleter_api.md b/core/src/docs/rfcs/3911_deleter_api.md index 1fd70708bfdd..dfdc7a73cb33 100644 --- a/core/src/docs/rfcs/3911_deleter_api.md +++ b/core/src/docs/rfcs/3911_deleter_api.md @@ -42,22 +42,15 @@ impl Operator { ``` - `delete` is the existing API, which deletes a single file or an empty dir. -- `delete_with` is an extension of the existing `delete` API, which supports additional options, such as `recursive`. +- `delete_with` is an extension of the existing `delete` API, which supports additional options, such as `version`. - `deleter` is a new API that returns a `Deleter` instance. -- `deleter_with` is an extension of the existing `deleter` API, which supports additional options, such as `recursive`. +- `deleter_with` is an extension of the existing `deleter` API, which supports additional options, such as `concurrent`. The following new options will be available for `delete_with` and `deleter_with`: -- `recursive`: Enable recursive deletion. - `concurrent`: How many delete tasks can be performed concurrently? - `buffer`: How many files can be buffered for send in a single batch? -Users can delete a file recursively in this way: - -```rust -let _ = op.delete_with("path/to/file").recursive(true).await?; -``` - Users can delete multiple files in this way: @@ -136,7 +129,7 @@ And the `delete` API will be changed to return a `oio::Delete` instead: ```diff trait Accessor { - async fn delete(&self, path: &str, args: OpDelete) -> Result; -+ async fn delete(&self, path: &str, args: OpDelete) -> Result<(RpDelete, Self::Deleter)>; ++ async fn delete(&self, args: OpDelete) -> Result<(RpDelete, Self::Deleter)>; } ``` diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs index 8deb767e6c1f..a68538ec6dec 100644 --- a/core/src/layers/blocking.rs +++ b/core/src/layers/blocking.rs @@ -173,7 +173,7 @@ impl LayeredAccess for BlockingAccessor { &self.inner } - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { let mut meta = self.inner.info().as_ref().clone(); meta.full_capability_mut().blocking = true; meta.into() diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index 17c0c6414a01..ad26e5b7bcc4 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -110,7 +110,7 @@ impl Layer for CompleteLayer { fn layer(&self, inner: A) -> Self::LayeredAccess { CompleteAccessor { - meta: inner.info(), + info: inner.info(), inner: Arc::new(inner), } } @@ -118,7 +118,7 @@ impl Layer for CompleteLayer { /// Provide complete wrapper for backend. pub struct CompleteAccessor { - meta: Arc, + info: Arc, inner: Arc, } @@ -130,7 +130,7 @@ impl Debug for CompleteAccessor { impl CompleteAccessor { fn new_unsupported_error(&self, op: impl Into<&'static str>) -> Error { - let scheme = self.meta.scheme(); + let scheme = self.info.scheme(); let op = op.into(); Error::new( ErrorKind::Unsupported, @@ -140,7 +140,7 @@ impl CompleteAccessor { } async fn complete_create_dir(&self, path: &str, args: OpCreateDir) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if capability.create_dir { return self.inner().create_dir(path, args).await; } @@ -154,7 +154,7 @@ impl CompleteAccessor { } fn complete_blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if capability.create_dir && capability.blocking { return self.inner().blocking_create_dir(path, args); } @@ -168,7 +168,7 @@ impl CompleteAccessor { } async fn complete_stat(&self, path: &str, args: OpStat) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.stat { return Err(self.new_unsupported_error(Operation::Stat)); } @@ -218,7 +218,7 @@ impl CompleteAccessor { } fn complete_blocking_stat(&self, path: &str, args: OpStat) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.stat { return Err(self.new_unsupported_error(Operation::Stat)); } @@ -271,7 +271,7 @@ impl CompleteAccessor { path: &str, args: OpList, ) -> Result<(RpList, CompleteLister)> { - let cap = self.meta.full_capability(); + let cap = self.info.full_capability(); if !cap.list { return Err(self.new_unsupported_error(Operation::List)); } @@ -319,7 +319,7 @@ impl CompleteAccessor { path: &str, args: OpList, ) -> Result<(RpList, CompleteLister)> { - let cap = self.meta.full_capability(); + let cap = self.info.full_capability(); if !cap.list { return Err(self.new_unsupported_error(Operation::BlockingList)); } @@ -377,8 +377,8 @@ impl LayeredAccess for CompleteAccessor { } // Todo: May move the logic to the implement of Layer::layer of CompleteAccessor - fn metadata(&self) -> Arc { - let mut meta = (*self.meta).clone(); + fn info(&self) -> Arc { + let mut meta = (*self.info).clone(); let cap = meta.full_capability_mut(); if cap.list && cap.write_can_empty { cap.create_dir = true; @@ -391,7 +391,7 @@ impl LayeredAccess for CompleteAccessor { } async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.read { return Err(self.new_unsupported_error(Operation::Read)); } @@ -404,7 +404,7 @@ impl LayeredAccess for CompleteAccessor { } async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.write { return Err(self.new_unsupported_error(Operation::Write)); } @@ -413,7 +413,7 @@ impl LayeredAccess for CompleteAccessor { ErrorKind::Unsupported, format!( "service {} doesn't support operation write with append", - self.info().scheme() + self.info.scheme() ), )); } @@ -424,7 +424,7 @@ impl LayeredAccess for CompleteAccessor { } async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.copy { return Err(self.new_unsupported_error(Operation::Copy)); } @@ -433,7 +433,7 @@ impl LayeredAccess for CompleteAccessor { } async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.rename { return Err(self.new_unsupported_error(Operation::Rename)); } @@ -446,7 +446,7 @@ impl LayeredAccess for CompleteAccessor { } async fn delete(&self, path: &str, args: OpDelete) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.delete { return Err(self.new_unsupported_error(Operation::Delete)); } @@ -455,7 +455,7 @@ impl LayeredAccess for CompleteAccessor { } async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.list { return Err(self.new_unsupported_error(Operation::List)); } @@ -464,7 +464,7 @@ impl LayeredAccess for CompleteAccessor { } async fn batch(&self, args: OpBatch) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.batch { return Err(self.new_unsupported_error(Operation::Batch)); } @@ -473,7 +473,7 @@ impl LayeredAccess for CompleteAccessor { } async fn presign(&self, path: &str, args: OpPresign) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.presign { return Err(self.new_unsupported_error(Operation::Presign)); } @@ -486,7 +486,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.read || !capability.blocking { return Err(self.new_unsupported_error(Operation::Read)); } @@ -498,7 +498,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.write || !capability.blocking { return Err(self.new_unsupported_error(Operation::BlockingWrite)); } @@ -508,7 +508,7 @@ impl LayeredAccess for CompleteAccessor { ErrorKind::Unsupported, format!( "service {} doesn't support operation write with append", - self.info().scheme() + self.info.scheme() ), )); } @@ -519,7 +519,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.copy || !capability.blocking { return Err(self.new_unsupported_error(Operation::BlockingCopy)); } @@ -528,7 +528,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.rename || !capability.blocking { return Err(self.new_unsupported_error(Operation::BlockingRename)); } @@ -541,7 +541,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_delete(&self, path: &str, args: OpDelete) -> Result { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.delete || !capability.blocking { return Err(self.new_unsupported_error(Operation::BlockingDelete)); } @@ -550,7 +550,7 @@ impl LayeredAccess for CompleteAccessor { } fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> { - let capability = self.meta.full_capability(); + let capability = self.info.full_capability(); if !capability.list || !capability.blocking { return Err(self.new_unsupported_error(Operation::BlockingList)); } diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs index 95ec210a262b..a160a63772ec 100644 --- a/core/src/layers/error_context.rs +++ b/core/src/layers/error_context.rs @@ -45,14 +45,14 @@ impl Layer for ErrorContextLayer { type LayeredAccess = ErrorContextAccessor; fn layer(&self, inner: A) -> Self::LayeredAccess { - let meta = inner.info(); - ErrorContextAccessor { meta, inner } + let info = inner.info(); + ErrorContextAccessor { info, inner } } } /// Provide error context wrapper for backend. pub struct ErrorContextAccessor { - meta: Arc, + info: Arc, inner: A, } @@ -75,14 +75,14 @@ impl LayeredAccess for ErrorContextAccessor { &self.inner } - fn metadata(&self) -> Arc { - self.meta.clone() + fn info(&self) -> Arc { + self.info.clone() } async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result { self.inner.create_dir(path, args).await.map_err(|err| { err.with_operation(Operation::CreateDir) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -95,13 +95,13 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, r)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), r) + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), r) .with_range(range), ) }) .map_err(|err| { err.with_operation(Operation::Read) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) .with_context("range", range.to_string()) }) @@ -114,12 +114,12 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, w)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), w), + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), w), ) }) .map_err(|err| { err.with_operation(Operation::Write) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -127,7 +127,7 @@ impl LayeredAccess for ErrorContextAccessor { async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result { self.inner.copy(from, to, args).await.map_err(|err| { err.with_operation(Operation::Copy) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("from", from) .with_context("to", to) }) @@ -136,7 +136,7 @@ impl LayeredAccess for ErrorContextAccessor { async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result { self.inner.rename(from, to, args).await.map_err(|err| { err.with_operation(Operation::Rename) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("from", from) .with_context("to", to) }) @@ -145,7 +145,7 @@ impl LayeredAccess for ErrorContextAccessor { async fn stat(&self, path: &str, args: OpStat) -> Result { self.inner.stat(path, args).await.map_err(|err| { err.with_operation(Operation::Stat) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -153,7 +153,7 @@ impl LayeredAccess for ErrorContextAccessor { async fn delete(&self, path: &str, args: OpDelete) -> Result { self.inner.delete(path, args).await.map_err(|err| { err.with_operation(Operation::Delete) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -165,12 +165,12 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, p)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), p), + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), p), ) }) .map_err(|err| { err.with_operation(Operation::List) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -186,7 +186,7 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(path, res)| { let res = res.map_err(|err| { err.with_operation(Operation::Delete) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", &path) }); (path, res) @@ -197,14 +197,14 @@ impl LayeredAccess for ErrorContextAccessor { }) .map_err(|err| { err.with_operation(Operation::Batch) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) }) } async fn presign(&self, path: &str, args: OpPresign) -> Result { self.inner.presign(path, args).await.map_err(|err| { err.with_operation(Operation::Presign) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -212,7 +212,7 @@ impl LayeredAccess for ErrorContextAccessor { fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result { self.inner.blocking_create_dir(path, args).map_err(|err| { err.with_operation(Operation::BlockingCreateDir) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -224,13 +224,13 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, os)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), os) + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os) .with_range(range), ) }) .map_err(|err| { err.with_operation(Operation::BlockingRead) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) .with_context("range", range.to_string()) }) @@ -242,12 +242,12 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, os)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), os), + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os), ) }) .map_err(|err| { err.with_operation(Operation::BlockingWrite) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -255,7 +255,7 @@ impl LayeredAccess for ErrorContextAccessor { fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result { self.inner.blocking_copy(from, to, args).map_err(|err| { err.with_operation(Operation::BlockingCopy) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("from", from) .with_context("to", to) }) @@ -264,7 +264,7 @@ impl LayeredAccess for ErrorContextAccessor { fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result { self.inner.blocking_rename(from, to, args).map_err(|err| { err.with_operation(Operation::BlockingRename) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("from", from) .with_context("to", to) }) @@ -273,7 +273,7 @@ impl LayeredAccess for ErrorContextAccessor { fn blocking_stat(&self, path: &str, args: OpStat) -> Result { self.inner.blocking_stat(path, args).map_err(|err| { err.with_operation(Operation::BlockingStat) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -281,7 +281,7 @@ impl LayeredAccess for ErrorContextAccessor { fn blocking_delete(&self, path: &str, args: OpDelete) -> Result { self.inner.blocking_delete(path, args).map_err(|err| { err.with_operation(Operation::BlockingDelete) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } @@ -292,12 +292,12 @@ impl LayeredAccess for ErrorContextAccessor { .map(|(rp, os)| { ( rp, - ErrorContextWrapper::new(self.meta.scheme(), path.to_string(), os), + ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os), ) }) .map_err(|err| { err.with_operation(Operation::BlockingList) - .with_context("service", self.meta.scheme()) + .with_context("service", self.info.scheme()) .with_context("path", path) }) } diff --git a/core/src/layers/fastrace.rs b/core/src/layers/fastrace.rs index ad49145c6cf3..51153d616cc7 100644 --- a/core/src/layers/fastrace.rs +++ b/core/src/layers/fastrace.rs @@ -132,7 +132,7 @@ impl LayeredAccess for FastraceAccessor { } #[trace] - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { self.inner.info() } diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs index 5e02472fab6a..4a54adb81658 100644 --- a/core/src/layers/immutable_index.rs +++ b/core/src/layers/immutable_index.rs @@ -149,7 +149,7 @@ impl LayeredAccess for ImmutableIndexAccessor { } /// Add list capabilities for underlying storage services. - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { let mut meta = (*self.inner.info()).clone(); let cap = meta.full_capability_mut(); diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs index 075ffdd33838..ec929e50d63b 100644 --- a/core/src/layers/logging.rs +++ b/core/src/layers/logging.rs @@ -266,7 +266,7 @@ impl LayeredAccess for LoggingAccessor { &self.inner } - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { self.logger .log(&self.info, Operation::Info, &[], "started", None); diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs index 55a4d12f47d0..58cb7e4dfd4a 100644 --- a/core/src/layers/oteltrace.rs +++ b/core/src/layers/oteltrace.rs @@ -77,9 +77,9 @@ impl LayeredAccess for OtelTraceAccessor { &self.inner } - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { let tracer = global::tracer("opendal"); - tracer.in_span("metadata", |_cx| self.inner.info()) + tracer.in_span("info", |_cx| self.inner.info()) } async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result { diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs index ba824215134b..c59f8d5fc204 100644 --- a/core/src/layers/tracing.rs +++ b/core/src/layers/tracing.rs @@ -165,7 +165,7 @@ impl LayeredAccess for TracingAccessor { } #[tracing::instrument(level = "debug")] - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { self.inner.info() } diff --git a/core/src/raw/adapters/kv/api.rs b/core/src/raw/adapters/kv/api.rs index 8f9b8e965837..2790a53d9238 100644 --- a/core/src/raw/adapters/kv/api.rs +++ b/core/src/raw/adapters/kv/api.rs @@ -96,8 +96,8 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static { /// TODO: use default associate type `= ()` after stablized type Scanner: Scan; - /// Return the metadata of this key value accessor. - fn metadata(&self) -> Metadata; + /// Return the info of this key value accessor. + fn info(&self) -> Info; /// Get a key from service. /// @@ -196,14 +196,14 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static { } } -/// Metadata for this key value accessor. -pub struct Metadata { +/// Info for this key value accessor. +pub struct Info { scheme: Scheme, name: String, capabilities: Capability, } -impl Metadata { +impl Info { /// Create a new KeyValueAccessorInfo. pub fn new(scheme: Scheme, name: &str, capabilities: Capability) -> Self { Self { @@ -228,14 +228,3 @@ impl Metadata { self.capabilities } } - -impl From for AccessorInfo { - fn from(m: Metadata) -> AccessorInfo { - let mut am = AccessorInfo::default(); - am.set_name(m.name()); - am.set_scheme(m.scheme()); - am.set_native_capability(m.capabilities()); - - am - } -} diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs index 6b625c78b6eb..a2f378a46bca 100644 --- a/core/src/raw/adapters/kv/backend.rs +++ b/core/src/raw/adapters/kv/backend.rs @@ -72,10 +72,13 @@ impl Access for Backend { type BlockingLister = HierarchyLister; fn info(&self) -> Arc { - let mut am: AccessorInfo = self.kv.metadata().into(); + let kv_info = self.kv.info(); + let mut am: AccessorInfo = AccessorInfo::default(); am.set_root(&self.root); + am.set_scheme(kv_info.scheme()); + am.set_name(kv_info.name()); - let mut cap = am.native_capability(); + let mut cap = kv_info.capabilities(); if cap.read { cap.stat = true; } diff --git a/core/src/raw/adapters/kv/mod.rs b/core/src/raw/adapters/kv/mod.rs index c03c8d71b808..719925160421 100644 --- a/core/src/raw/adapters/kv/mod.rs +++ b/core/src/raw/adapters/kv/mod.rs @@ -21,7 +21,7 @@ mod api; pub use api::Adapter; -pub use api::Metadata; +pub use api::Info; pub use api::Scan; #[cfg(any( feature = "services-cloudflare-kv", diff --git a/core/src/raw/adapters/typed_kv/api.rs b/core/src/raw/adapters/typed_kv/api.rs index 0fecb9d01299..f1e4a95fc47a 100644 --- a/core/src/raw/adapters/typed_kv/api.rs +++ b/core/src/raw/adapters/typed_kv/api.rs @@ -45,7 +45,7 @@ use crate::Scheme; /// Ideally, we should use `typed_kv::Adapter` instead of `kv::Adapter` for /// in-memory rust libs like moka and dashmap. pub trait Adapter: Send + Sync + Debug + Unpin + 'static { - /// Get the scheme and name of current adapter. + /// Return the info of this key value accessor. fn info(&self) -> Info; /// Get a value from adapter. diff --git a/core/src/raw/layer.rs b/core/src/raw/layer.rs index f9872c6cf03e..6e6d2b802b45 100644 --- a/core/src/raw/layer.rs +++ b/core/src/raw/layer.rs @@ -128,7 +128,6 @@ pub trait Layer { /// LayeredAccess is layered accessor that forward all not implemented /// method to inner. #[allow(missing_docs)] - pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static { type Inner: Access; type Reader: oio::Read; @@ -140,7 +139,7 @@ pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static { fn inner(&self) -> &Self::Inner; - fn metadata(&self) -> Arc { + fn info(&self) -> Arc { self.inner().info() } @@ -241,86 +240,86 @@ pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static { impl Access for L { type Reader = L::Reader; - type BlockingReader = L::BlockingReader; type Writer = L::Writer; - type BlockingWriter = L::BlockingWriter; type Lister = L::Lister; + type BlockingReader = L::BlockingReader; + type BlockingWriter = L::BlockingWriter; type BlockingLister = L::BlockingLister; fn info(&self) -> Arc { - (self as &L).metadata() + LayeredAccess::info(self) } async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result { - (self as &L).create_dir(path, args).await + LayeredAccess::create_dir(self, path, args).await } async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { - (self as &L).read(path, args).await + LayeredAccess::read(self, path, args).await } async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { - (self as &L).write(path, args).await + LayeredAccess::write(self, path, args).await } async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result { - (self as &L).copy(from, to, args).await + LayeredAccess::copy(self, from, to, args).await } async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result { - (self as &L).rename(from, to, args).await + LayeredAccess::rename(self, from, to, args).await } async fn stat(&self, path: &str, args: OpStat) -> Result { - (self as &L).stat(path, args).await + LayeredAccess::stat(self, path, args).await } async fn delete(&self, path: &str, args: OpDelete) -> Result { - (self as &L).delete(path, args).await + LayeredAccess::delete(self, path, args).await } async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> { - (self as &L).list(path, args).await + LayeredAccess::list(self, path, args).await } async fn batch(&self, args: OpBatch) -> Result { - (self as &L).batch(args).await + LayeredAccess::batch(self, args).await } async fn presign(&self, path: &str, args: OpPresign) -> Result { - (self as &L).presign(path, args).await + LayeredAccess::presign(self, path, args).await } fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result { - (self as &L).blocking_create_dir(path, args) + LayeredAccess::blocking_create_dir(self, path, args) } fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> { - (self as &L).blocking_read(path, args) + LayeredAccess::blocking_read(self, path, args) } fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> { - (self as &L).blocking_write(path, args) + LayeredAccess::blocking_write(self, path, args) } fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result { - (self as &L).blocking_copy(from, to, args) + LayeredAccess::blocking_copy(self, from, to, args) } fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result { - (self as &L).blocking_rename(from, to, args) + LayeredAccess::blocking_rename(self, from, to, args) } fn blocking_stat(&self, path: &str, args: OpStat) -> Result { - (self as &L).blocking_stat(path, args) + LayeredAccess::blocking_stat(self, path, args) } fn blocking_delete(&self, path: &str, args: OpDelete) -> Result { - (self as &L).blocking_delete(path, args) + LayeredAccess::blocking_delete(self, path, args) } fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> { - (self as &L).blocking_list(path, args) + LayeredAccess::blocking_list(self, path, args) } } diff --git a/core/src/services/atomicserver/backend.rs b/core/src/services/atomicserver/backend.rs index 2a8318daa7ad..546e6d25989f 100644 --- a/core/src/services/atomicserver/backend.rs +++ b/core/src/services/atomicserver/backend.rs @@ -353,8 +353,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Atomicserver, "atomicserver", Capability { diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index 2083f124cf53..1ef193a68252 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -87,8 +87,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Cacache, &self.datadir, Capability { diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index ce68192f58e4..c2be1ddd1663 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -183,8 +183,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = kv::Scanner; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::CloudflareKv, &self.namespace_id, Capability { diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs index b1c992b1a8c6..643617431e47 100644 --- a/core/src/services/d1/backend.rs +++ b/core/src/services/d1/backend.rs @@ -260,8 +260,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::D1, &self.table, Capability { diff --git a/core/src/services/etcd/backend.rs b/core/src/services/etcd/backend.rs index d37fb35d2a80..4a6e94f13bd9 100644 --- a/core/src/services/etcd/backend.rs +++ b/core/src/services/etcd/backend.rs @@ -274,8 +274,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = kv::ScanStdIter>>; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Etcd, &self.endpoints.join(","), Capability { diff --git a/core/src/services/foundationdb/backend.rs b/core/src/services/foundationdb/backend.rs index d28b70152bcf..b9c70946b5f4 100644 --- a/core/src/services/foundationdb/backend.rs +++ b/core/src/services/foundationdb/backend.rs @@ -112,8 +112,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Foundationdb, "foundationdb", Capability { diff --git a/core/src/services/gridfs/backend.rs b/core/src/services/gridfs/backend.rs index 6d7898d1dd99..f2bb2341534c 100644 --- a/core/src/services/gridfs/backend.rs +++ b/core/src/services/gridfs/backend.rs @@ -214,8 +214,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Gridfs, &format!("{}/{}", self.database, self.bucket), Capability { diff --git a/core/src/services/libsql/backend.rs b/core/src/services/libsql/backend.rs index c0870e374678..233580c78e9f 100644 --- a/core/src/services/libsql/backend.rs +++ b/core/src/services/libsql/backend.rs @@ -307,8 +307,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Libsql, &self.table, Capability { diff --git a/core/src/services/memcached/backend.rs b/core/src/services/memcached/backend.rs index d0cc42c9211f..2b772f713e5e 100644 --- a/core/src/services/memcached/backend.rs +++ b/core/src/services/memcached/backend.rs @@ -199,8 +199,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Memcached, "memcached", Capability { diff --git a/core/src/services/mongodb/backend.rs b/core/src/services/mongodb/backend.rs index 786c34dbe80c..ed5f26a411e9 100644 --- a/core/src/services/mongodb/backend.rs +++ b/core/src/services/mongodb/backend.rs @@ -228,8 +228,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Mongodb, &format!("{}/{}", self.database, self.collection), Capability { diff --git a/core/src/services/mysql/backend.rs b/core/src/services/mysql/backend.rs index 4569431b7e9f..ada294d26691 100644 --- a/core/src/services/mysql/backend.rs +++ b/core/src/services/mysql/backend.rs @@ -190,8 +190,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Mysql, &self.table, Capability { diff --git a/core/src/services/nebula_graph/backend.rs b/core/src/services/nebula_graph/backend.rs index 9c34018bfdb2..4d70e3210a73 100644 --- a/core/src/services/nebula_graph/backend.rs +++ b/core/src/services/nebula_graph/backend.rs @@ -272,8 +272,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = kv::ScanStdIter>>; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::NebulaGraph, &self.session_config.space.clone().unwrap(), Capability { diff --git a/core/src/services/persy/backend.rs b/core/src/services/persy/backend.rs index e5317b11221d..c9234f1878cb 100644 --- a/core/src/services/persy/backend.rs +++ b/core/src/services/persy/backend.rs @@ -154,8 +154,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Persy, &self.datafile, Capability { diff --git a/core/src/services/postgresql/backend.rs b/core/src/services/postgresql/backend.rs index b7cbacc997ad..72993c3af6a1 100644 --- a/core/src/services/postgresql/backend.rs +++ b/core/src/services/postgresql/backend.rs @@ -189,8 +189,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Postgresql, &self.table, Capability { diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index 281a5ac96a43..b0722f824cec 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -113,8 +113,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Redb, &self.datadir, Capability { diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs index e04f5af29daf..19afe34c7655 100644 --- a/core/src/services/redis/backend.rs +++ b/core/src/services/redis/backend.rs @@ -329,8 +329,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Redis, self.addr.as_str(), Capability { diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs index ecbf9bdfcf09..8dc3c6d2a2ba 100644 --- a/core/src/services/rocksdb/backend.rs +++ b/core/src/services/rocksdb/backend.rs @@ -110,8 +110,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = kv::Scanner; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Rocksdb, &self.db.path().to_string_lossy(), Capability { diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs index c4cb4bdf854b..64c4c367581e 100644 --- a/core/src/services/sled/backend.rs +++ b/core/src/services/sled/backend.rs @@ -139,8 +139,8 @@ impl Debug for Adapter { impl kv::Adapter for Adapter { type Scanner = kv::Scanner; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Sled, &self.datadir, Capability { diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index 06158048341a..6f9c1aa15530 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -224,8 +224,8 @@ impl kv::Scan for SqliteScanner { impl kv::Adapter for Adapter { type Scanner = SqliteScanner; - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Sqlite, &self.table, Capability { diff --git a/core/src/services/surrealdb/backend.rs b/core/src/services/surrealdb/backend.rs index 47b91e36057f..d7de77252683 100644 --- a/core/src/services/surrealdb/backend.rs +++ b/core/src/services/surrealdb/backend.rs @@ -285,8 +285,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Surrealdb, &self.table, Capability { diff --git a/core/src/services/tikv/backend.rs b/core/src/services/tikv/backend.rs index 275dcf9bbdc4..5d37b526d5db 100644 --- a/core/src/services/tikv/backend.rs +++ b/core/src/services/tikv/backend.rs @@ -187,8 +187,8 @@ impl Adapter { impl kv::Adapter for Adapter { type Scanner = (); - fn metadata(&self) -> kv::Metadata { - kv::Metadata::new( + fn info(&self) -> kv::Info { + kv::Info::new( Scheme::Tikv, "TiKV", Capability {