From 36ea40abfde721212d67a61d4503df57e376dda6 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 15:47:07 +0000 Subject: [PATCH 01/12] Added user metadata support for cos service --- core/src/services/cos/backend.rs | 8 ++++- core/src/services/cos/core.rs | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 86d7e342ca36..1a3360ac1ffe 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -257,6 +257,7 @@ impl Access for CosBackend { stat_has_content_md5: true, stat_has_last_modified: true, stat_has_content_disposition: true, + stat_has_user_metadata: true, read: true, @@ -284,6 +285,7 @@ impl Access for CosBackend { } else { Some(usize::MAX) }, + write_with_user_metadata: true, delete: true, copy: true, @@ -311,7 +313,11 @@ impl Access for CosBackend { let status = resp.status(); match status { - StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new), + StatusCode::OK =>{ + let headers = resp.headers(); + let mut meta = self.core.parse_metadata(path, resp.headers())?; + Ok(RpStat::new(meta)) + }, _ => Err(parse_error(resp)), } } diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index 4d28b268895b..f4dd9e36bd37 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -184,6 +184,20 @@ impl CosCore { req = req.header("x-cos-forbid-overwrite", "true") } + // Set user metadata headers. + if let Some(user_metadata) = args.user_metadata() { + // before insert user defined metadata header, add prefix to the header name + if !self.check_user_metadata_key(key) { + return Err(Error::new( + ErrorKind::Unsupported, + "the format of the user metadata key is invalid, please refer the document", + )); + } + for (key, value) in user_metadata { + req = req.header(format!("x-cos-meta-{key}"), value) + } + } + let req = req.body(body).map_err(new_request_build_error)?; Ok(req) @@ -345,6 +359,20 @@ impl CosCore { req = req.header(CACHE_CONTROL, cache_control) } + // Set user metadata headers. + if let Some(user_metadata) = args.user_metadata() { + // before insert user defined metadata header, add prefix to the header name + if !self.check_user_metadata_key(key) { + return Err(Error::new( + ErrorKind::Unsupported, + "the format of the user metadata key is invalid, please refer the document", + )); + } + for (key, value) in user_metadata { + req = req.header(format!("x-cos-meta-{key}"), value) + } + } + let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?; self.sign(&mut req).await?; @@ -434,6 +462,32 @@ impl CosCore { self.sign(&mut req).await?; self.send(req).await } + + // According to https://www.tencentcloud.com/document/product/436/7746 + // there are some limits in user defined metadata key + fn check_user_metadata_key(&self, key: &str) -> bool { + key.chars().all(|c| c != '_') + } + + /// parse_metadata will parse http headers(including standards http headers + /// and user defined metadata header) into Metadata. + /// + /// # Arguments + /// + /// * `user_metadata_prefix` is the prefix of user defined metadata key + /// + /// # Notes + /// + /// before return the user defined metadata, we'll strip the user_metadata_prefix from the key + pub fn parse_metadata(&self, path: &str, headers: &HeaderMap) -> Result { + let mut m = parse_into_metadata(path, headers)?; + let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); + if !user_meta.is_empty() { + m.with_user_metadata(user_meta); + } + + Ok(m) + } } /// Result of CreateMultipartUpload From ed0dc6f73c91d22bf869b09c0e62ea4bf17524b9 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 16:02:32 +0000 Subject: [PATCH 02/12] fix --- core/src/services/cos/core.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index f4dd9e36bd37..d80635a7f6b5 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -186,14 +186,14 @@ impl CosCore { // Set user metadata headers. if let Some(user_metadata) = args.user_metadata() { - // before insert user defined metadata header, add prefix to the header name - if !self.check_user_metadata_key(key) { - return Err(Error::new( - ErrorKind::Unsupported, - "the format of the user metadata key is invalid, please refer the document", - )); - } for (key, value) in user_metadata { + // before insert user defined metadata header, add prefix to the header name + if !self.check_user_metadata_key(key) { + return Err(Error::new( + ErrorKind::Unsupported, + "the format of the user metadata key is invalid, please refer the document", + )); + } req = req.header(format!("x-cos-meta-{key}"), value) } } @@ -361,14 +361,14 @@ impl CosCore { // Set user metadata headers. if let Some(user_metadata) = args.user_metadata() { - // before insert user defined metadata header, add prefix to the header name - if !self.check_user_metadata_key(key) { - return Err(Error::new( - ErrorKind::Unsupported, - "the format of the user metadata key is invalid, please refer the document", - )); - } for (key, value) in user_metadata { + // before insert user defined metadata header, add prefix to the header name + if !self.check_user_metadata_key(key) { + return Err(Error::new( + ErrorKind::Unsupported, + "the format of the user metadata key is invalid, please refer the document", + )); + } req = req.header(format!("x-cos-meta-{key}"), value) } } From cff50c9d7e44404490d1cf93b54e55b8b294a650 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 16:03:48 +0000 Subject: [PATCH 03/12] fmt --- core/src/services/cos/backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 1a3360ac1ffe..0a9eb3f3dd64 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -313,11 +313,11 @@ impl Access for CosBackend { let status = resp.status(); match status { - StatusCode::OK =>{ + StatusCode::OK => { let headers = resp.headers(); let mut meta = self.core.parse_metadata(path, resp.headers())?; Ok(RpStat::new(meta)) - }, + } _ => Err(parse_error(resp)), } } From 255e7cef84f0a93803ff6c9f10b382f466bb9997 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 16:08:35 +0000 Subject: [PATCH 04/12] Fix --- core/src/services/cos/core.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index d80635a7f6b5..eccef01ef859 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -28,6 +28,7 @@ use http::header::IF_MATCH; use http::header::IF_NONE_MATCH; use http::Request; use http::Response; +use http::HeaderMap; use reqsign::TencentCosCredential; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; From 4206211d885ee019d2ceb9726383b3494b8a578c Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 16:17:50 +0000 Subject: [PATCH 05/12] fmt --- core/src/services/cos/backend.rs | 2 +- core/src/services/cos/core.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 0a9eb3f3dd64..921f56e7c4a6 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -315,7 +315,7 @@ impl Access for CosBackend { match status { StatusCode::OK => { let headers = resp.headers(); - let mut meta = self.core.parse_metadata(path, resp.headers())?; + let meta = self.core.parse_metadata(path, resp.headers())?; Ok(RpStat::new(meta)) } _ => Err(parse_error(resp)), diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index eccef01ef859..bdbdf58ccae8 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -26,9 +26,9 @@ use http::header::CONTENT_LENGTH; use http::header::CONTENT_TYPE; use http::header::IF_MATCH; use http::header::IF_NONE_MATCH; +use http::HeaderMap; use http::Request; use http::Response; -use http::HeaderMap; use reqsign::TencentCosCredential; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; From 38d0f97e9748eb974ef0aa55e6e06953c6c3bf60 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Sun, 5 Jan 2025 16:26:34 +0000 Subject: [PATCH 06/12] clippy --- core/src/services/cos/backend.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 921f56e7c4a6..48dfa7024242 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -314,7 +314,6 @@ impl Access for CosBackend { match status { StatusCode::OK => { - let headers = resp.headers(); let meta = self.core.parse_metadata(path, resp.headers())?; Ok(RpStat::new(meta)) } From ed49fce9c300a7d2d0d4f3138ce94531c87ea9bd Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 15:42:04 +0530 Subject: [PATCH 07/12] removed key check --- core/src/services/cos/backend.rs | 6 +++++- core/src/services/cos/core.rs | 33 -------------------------------- 2 files changed, 5 insertions(+), 34 deletions(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 48dfa7024242..655c64fb91ec 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -314,7 +314,11 @@ impl Access for CosBackend { match status { StatusCode::OK => { - let meta = self.core.parse_metadata(path, resp.headers())?; + let meta = parse_into_metadata(path, headers)?; + let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); + if !user_meta.is_empty() { + m.with_user_metadata(user_meta); + } Ok(RpStat::new(meta)) } _ => Err(parse_error(resp)), diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index bdbdf58ccae8..ef7ff14b7c48 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -188,13 +188,6 @@ impl CosCore { // Set user metadata headers. if let Some(user_metadata) = args.user_metadata() { for (key, value) in user_metadata { - // before insert user defined metadata header, add prefix to the header name - if !self.check_user_metadata_key(key) { - return Err(Error::new( - ErrorKind::Unsupported, - "the format of the user metadata key is invalid, please refer the document", - )); - } req = req.header(format!("x-cos-meta-{key}"), value) } } @@ -463,32 +456,6 @@ impl CosCore { self.sign(&mut req).await?; self.send(req).await } - - // According to https://www.tencentcloud.com/document/product/436/7746 - // there are some limits in user defined metadata key - fn check_user_metadata_key(&self, key: &str) -> bool { - key.chars().all(|c| c != '_') - } - - /// parse_metadata will parse http headers(including standards http headers - /// and user defined metadata header) into Metadata. - /// - /// # Arguments - /// - /// * `user_metadata_prefix` is the prefix of user defined metadata key - /// - /// # Notes - /// - /// before return the user defined metadata, we'll strip the user_metadata_prefix from the key - pub fn parse_metadata(&self, path: &str, headers: &HeaderMap) -> Result { - let mut m = parse_into_metadata(path, headers)?; - let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); - if !user_meta.is_empty() { - m.with_user_metadata(user_meta); - } - - Ok(m) - } } /// Result of CreateMultipartUpload From 2842b18cd280c7d74f84528f209126d245728bbe Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 15:45:29 +0530 Subject: [PATCH 08/12] removed key check --- core/src/services/cos/core.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index ef7ff14b7c48..9132f7e9fe24 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -356,13 +356,6 @@ impl CosCore { // Set user metadata headers. if let Some(user_metadata) = args.user_metadata() { for (key, value) in user_metadata { - // before insert user defined metadata header, add prefix to the header name - if !self.check_user_metadata_key(key) { - return Err(Error::new( - ErrorKind::Unsupported, - "the format of the user metadata key is invalid, please refer the document", - )); - } req = req.header(format!("x-cos-meta-{key}"), value) } } From 6a8429aaf32b307772dbc7e983be8641f5700546 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 15:46:50 +0530 Subject: [PATCH 09/12] fix --- core/src/services/cos/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 655c64fb91ec..afe412391745 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -317,7 +317,7 @@ impl Access for CosBackend { let meta = parse_into_metadata(path, headers)?; let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); if !user_meta.is_empty() { - m.with_user_metadata(user_meta); + meta.with_user_metadata(user_meta); } Ok(RpStat::new(meta)) } From db22c308cac4b4e12abbfbbdab009b7cadc4c6d4 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 15:50:55 +0530 Subject: [PATCH 10/12] fix --- core/src/services/cos/backend.rs | 1 + core/src/services/cos/core.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index afe412391745..f1fc43403833 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -314,6 +314,7 @@ impl Access for CosBackend { match status { StatusCode::OK => { + let header = resp.headers(); let meta = parse_into_metadata(path, headers)?; let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); if !user_meta.is_empty() { diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index 9132f7e9fe24..9d3adcc69488 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -26,7 +26,6 @@ use http::header::CONTENT_LENGTH; use http::header::CONTENT_TYPE; use http::header::IF_MATCH; use http::header::IF_NONE_MATCH; -use http::HeaderMap; use http::Request; use http::Response; use reqsign::TencentCosCredential; From 44dd9b811e3fa18138b03c53835deeea1aa65aeb Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 15:53:40 +0530 Subject: [PATCH 11/12] fix --- core/src/services/cos/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index f1fc43403833..6c6e696a9838 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -314,7 +314,7 @@ impl Access for CosBackend { match status { StatusCode::OK => { - let header = resp.headers(); + let headers = resp.headers(); let meta = parse_into_metadata(path, headers)?; let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); if !user_meta.is_empty() { From 38edeac9acbb53322889a2c587f0558eba57f353 Mon Sep 17 00:00:00 2001 From: geetanshjuneja Date: Tue, 7 Jan 2025 10:27:20 +0000 Subject: [PATCH 12/12] fix --- core/src/services/cos/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 6c6e696a9838..28336e55f23c 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -315,7 +315,7 @@ impl Access for CosBackend { match status { StatusCode::OK => { let headers = resp.headers(); - let meta = parse_into_metadata(path, headers)?; + let mut meta = parse_into_metadata(path, headers)?; let user_meta = parse_prefixed_headers(headers, "x-cos-meta-"); if !user_meta.is_empty() { meta.with_user_metadata(user_meta);