From bfac422bf7969c16de5f0c7ae912a9e1e9d60795 Mon Sep 17 00:00:00 2001 From: oeqqwq Date: Thu, 26 Sep 2024 11:37:03 +0800 Subject: [PATCH 1/3] repo-sync-2024-09-26T11:36:57+0800 --- .circleci/config.yml | 10 -- .circleci/release-config.yml | 126 ------------------ .github/workflows/deploy.yml | 28 ---- dataproxy_sdk/cc/file_help.h | 4 +- .../python/dataproxy/dp_file_adapter.py | 3 + 5 files changed, 5 insertions(+), 166 deletions(-) delete mode 100644 .circleci/release-config.yml delete mode 100644 .github/workflows/deploy.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e48a97..50f6dc0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,13 +137,3 @@ workflows: .bazelrc sdk-build-and-run true WORKSPACE sdk-build-and-run true - lint - sdk-publish-workflow: - when: - equal: [ "release", << pipeline.parameters.GHA_Event >> ] - jobs: - - continuation/continue: - configuration_path: .circleci/release-config.yml - # This is mandatory to trigger a pipeline when pushing a tag - filters: - tags: - only: /^v.*/ diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml deleted file mode 100644 index 6ae1d8b..0000000 --- a/.circleci/release-config.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2024 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use the latest 2.1 version of CircleCI pipeline process engine. -# See: https://circleci.com/docs/2.0/configuration-reference -version: 2.1 - -parameters: - GHA_Action: - type: string - default: "" - GHA_Actor: - type: string - default: "" - GHA_Event: - type: string - default: "" - -executors: - linux_x64_executor: # declares a reusable executor - docker: - - image: secretflow/release-ci:latest - resource_class: 2xlarge+ - shell: /bin/bash --login -eo pipefail - linux_aarch64_executor: - docker: - - image: secretflow/release-ci-aarch64:latest - resource_class: arm.2xlarge - shell: /bin/bash --login -eo pipefail - macos_arm64_executor: - macos: - xcode: 15.1.0 - resource_class: macos.m1.large.gen1 - -commands: - pip_publish: - parameters: - python_ver: - type: string - steps: - - run: - name: "build package and publish" - command: | - conda create -n build python=<< parameters.python_ver >> -y - conda activate build - - python3 -m pip install twine - - cd dataproxy_sdk/python - - python setup.py bdist_wheel - - python3 -m twine upload -r pypi -u __token__ -p ${PYPI_TWINE_TOKEN} dist/*.whl - -# Define a job to be invoked later in a workflow. -# See: https://circleci.com/docs/2.0/configuration-reference/#jobs -jobs: - linux_publish: - parameters: - python_ver: - type: string - executor: - type: string - executor: <> - steps: - - checkout - - pip_publish: - python_ver: << parameters.python_ver >> - - macOS_publish: - parameters: - python_ver: - type: string - executor: - type: string - executor: <> - steps: - - checkout - - run: - name: "Install homebrew dependencies" - command: | - brew install bazelisk cmake ninja libomp wget go - - run: - name: "Install Miniconda" - command: | - wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.1.2-0-MacOSX-arm64.sh -O ~/miniconda.sh - bash ~/miniconda.sh -b -p $HOME/miniconda - source $HOME/miniconda/bin/activate - conda init zsh bash - - pip_publish: - python_ver: << parameters.python_ver >> - -# Invoke jobs via workflows -# See: https://circleci.com/docs/2.0/configuration-reference/#workflows -workflows: - publish: - jobs: - - linux_publish: - matrix: - parameters: - python_ver: ["3.9", "3.10", "3.11"] - executor: ["linux_x64_executor", "linux_aarch64_executor"] - # This is mandatory to trigger a pipeline when pushing a tag - filters: - tags: - only: /^v.*/ - - macOS_publish: - matrix: - parameters: - python_ver: ["3.9", "3.10", "3.11"] - executor: ["macos_arm64_executor"] - # This is mandatory to trigger a pipeline when pushing a tag - filters: - tags: - only: /^v.*/ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index aba04e2..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2024 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -name: "Publish SDK Python Package" -on: - release: - types: [published] -permissions: read-all -jobs: - trigger-circleci: - runs-on: ubuntu-latest - steps: - - name: secretflow-serving-deploy - id: secretflow-serving-deploy - uses: CircleCI-Public/trigger-circleci-pipeline-action@v1.2.0 - env: - CCI_TOKEN: ${{ secrets.CCI_TOKEN }} diff --git a/dataproxy_sdk/cc/file_help.h b/dataproxy_sdk/cc/file_help.h index 91e9c0e..0d72d96 100644 --- a/dataproxy_sdk/cc/file_help.h +++ b/dataproxy_sdk/cc/file_help.h @@ -28,9 +28,9 @@ class FileHelpWrite { public: struct Options { // only orc use by sf - arrow::Compression::type compression = arrow::Compression::ZSTD; + arrow::Compression::type compression = arrow::Compression::UNCOMPRESSED; // only orc use by sf - int64_t compression_block_size = 256 * 1024; + int64_t compression_block_size = 64 * 1024; // only orc use by sf int64_t stripe_size = 64 * 1024 * 1024; diff --git a/dataproxy_sdk/python/dataproxy/dp_file_adapter.py b/dataproxy_sdk/python/dataproxy/dp_file_adapter.py index 5ba6f4d..4785267 100644 --- a/dataproxy_sdk/python/dataproxy/dp_file_adapter.py +++ b/dataproxy_sdk/python/dataproxy/dp_file_adapter.py @@ -22,9 +22,11 @@ class DataProxyFileAdapter: def __init__(self, config: proto.DataProxyConfig): self.data_proxy_file = libdataproxy.DataProxyFile(config.SerializeToString()) + def close(self): self.data_proxy_file.close() + def download_file( self, info: proto.DownloadInfo, file_path: str, file_format: proto.FileFormat ): @@ -37,6 +39,7 @@ def download_file( f"dataproxy sdk: download_file[{file_path}], type[{file_format}], size[{size}]" ) + def upload_file( self, info: proto.UploadInfo, file_path: str, file_format: proto.FileFormat ): From 344af00158820f7c187b33180603504175aa2ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=89=BE=E5=84=92?= Date: Thu, 26 Sep 2024 11:43:14 +0800 Subject: [PATCH 2/3] fix sync error --- .circleci/config.yml | 10 +++ .circleci/release-config.yml | 126 +++++++++++++++++++++++++++++++++++ .github/workflows/deploy.yml | 28 ++++++++ 3 files changed, 164 insertions(+) create mode 100644 .circleci/release-config.yml create mode 100644 .github/workflows/deploy.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 50f6dc0..7e48a97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,3 +137,13 @@ workflows: .bazelrc sdk-build-and-run true WORKSPACE sdk-build-and-run true - lint + sdk-publish-workflow: + when: + equal: [ "release", << pipeline.parameters.GHA_Event >> ] + jobs: + - continuation/continue: + configuration_path: .circleci/release-config.yml + # This is mandatory to trigger a pipeline when pushing a tag + filters: + tags: + only: /^v.*/ diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml new file mode 100644 index 0000000..6ae1d8b --- /dev/null +++ b/.circleci/release-config.yml @@ -0,0 +1,126 @@ +# Copyright 2024 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 + +parameters: + GHA_Action: + type: string + default: "" + GHA_Actor: + type: string + default: "" + GHA_Event: + type: string + default: "" + +executors: + linux_x64_executor: # declares a reusable executor + docker: + - image: secretflow/release-ci:latest + resource_class: 2xlarge+ + shell: /bin/bash --login -eo pipefail + linux_aarch64_executor: + docker: + - image: secretflow/release-ci-aarch64:latest + resource_class: arm.2xlarge + shell: /bin/bash --login -eo pipefail + macos_arm64_executor: + macos: + xcode: 15.1.0 + resource_class: macos.m1.large.gen1 + +commands: + pip_publish: + parameters: + python_ver: + type: string + steps: + - run: + name: "build package and publish" + command: | + conda create -n build python=<< parameters.python_ver >> -y + conda activate build + + python3 -m pip install twine + + cd dataproxy_sdk/python + + python setup.py bdist_wheel + + python3 -m twine upload -r pypi -u __token__ -p ${PYPI_TWINE_TOKEN} dist/*.whl + +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/2.0/configuration-reference/#jobs +jobs: + linux_publish: + parameters: + python_ver: + type: string + executor: + type: string + executor: <> + steps: + - checkout + - pip_publish: + python_ver: << parameters.python_ver >> + + macOS_publish: + parameters: + python_ver: + type: string + executor: + type: string + executor: <> + steps: + - checkout + - run: + name: "Install homebrew dependencies" + command: | + brew install bazelisk cmake ninja libomp wget go + - run: + name: "Install Miniconda" + command: | + wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.1.2-0-MacOSX-arm64.sh -O ~/miniconda.sh + bash ~/miniconda.sh -b -p $HOME/miniconda + source $HOME/miniconda/bin/activate + conda init zsh bash + - pip_publish: + python_ver: << parameters.python_ver >> + +# Invoke jobs via workflows +# See: https://circleci.com/docs/2.0/configuration-reference/#workflows +workflows: + publish: + jobs: + - linux_publish: + matrix: + parameters: + python_ver: ["3.9", "3.10", "3.11"] + executor: ["linux_x64_executor", "linux_aarch64_executor"] + # This is mandatory to trigger a pipeline when pushing a tag + filters: + tags: + only: /^v.*/ + - macOS_publish: + matrix: + parameters: + python_ver: ["3.9", "3.10", "3.11"] + executor: ["macos_arm64_executor"] + # This is mandatory to trigger a pipeline when pushing a tag + filters: + tags: + only: /^v.*/ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..aba04e2 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,28 @@ +# Copyright 2024 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: "Publish SDK Python Package" +on: + release: + types: [published] +permissions: read-all +jobs: + trigger-circleci: + runs-on: ubuntu-latest + steps: + - name: secretflow-serving-deploy + id: secretflow-serving-deploy + uses: CircleCI-Public/trigger-circleci-pipeline-action@v1.2.0 + env: + CCI_TOKEN: ${{ secrets.CCI_TOKEN }} From ca075c604bc078499c764a6704ab292a47367b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=89=BE=E5=84=92?= Date: Thu, 26 Sep 2024 11:45:22 +0800 Subject: [PATCH 3/3] format --- dataproxy_sdk/python/dataproxy/dp_file_adapter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dataproxy_sdk/python/dataproxy/dp_file_adapter.py b/dataproxy_sdk/python/dataproxy/dp_file_adapter.py index 4785267..5ba6f4d 100644 --- a/dataproxy_sdk/python/dataproxy/dp_file_adapter.py +++ b/dataproxy_sdk/python/dataproxy/dp_file_adapter.py @@ -22,11 +22,9 @@ class DataProxyFileAdapter: def __init__(self, config: proto.DataProxyConfig): self.data_proxy_file = libdataproxy.DataProxyFile(config.SerializeToString()) - def close(self): self.data_proxy_file.close() - def download_file( self, info: proto.DownloadInfo, file_path: str, file_format: proto.FileFormat ): @@ -39,7 +37,6 @@ def download_file( f"dataproxy sdk: download_file[{file_path}], type[{file_format}], size[{size}]" ) - def upload_file( self, info: proto.UploadInfo, file_path: str, file_format: proto.FileFormat ):