add wheel dependency check (#3064)

XieYunshen · web-flow · commit 0b7510740c28 · 2025-04-14T18:46:36.000+08:00
diff --git a/.github/actions/so_dependency_analyzer/action.yml b/.github/actions/so_dependency_analyzer/action.yml
@@ -0,0 +1,51 @@
+name: PaddlePaddle Packages Dependency So Analysis
+description: "PaddlePaddle Packages Dependency So Analysis Tool"
+
+inputs:
+  ce_task_name:
+    description: "Ce Task Name"
+    required: true
+  is_cuda:
+    description: "Is CUDA"
+    required: false
+    default: "True"
+  python_version:
+    description: "Python Version"
+    required: false
+    default: "3.10"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Download and Analyze Wheel
+      shell: bash
+      run: |
+        set -x
+        python -m pip install requests
+        mkdir -p SoAnalysisDir
+        python publish_scripts/so_dependency_analyzer/wheel_download.py \
+          --ce_task_name=${{ inputs.ce_task_name }} \
+          --is_cuda=${{ inputs.is_cuda }} \
+          --python_version=${{ inputs.python_version }} \
+          --output_dir=SoAnalysisDir
+
+        so_dependency_file=$(readlink -f publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py)
+        cd SoAnalysisDir
+        unzip *.whl -d .
+        python $so_dependency_file extract 
+        result_file_path=$(readlink -f so_dependencies_static.json)
+        description_file_path=$(readlink -f description.txt)
+        echo "result_file_path=${result_file_path}" >> $GITHUB_ENV
+        echo "description_file_path=${description_file_path}" >> $GITHUB_ENV
+        echo "ce_task_name=${{ inputs.ce_task_name }}" >> $GITHUB_ENV
+        echo "is_cuda=${{ inputs.is_cuda }}" >> $GITHUB_ENV
+        echo "python_version=${{ inputs.python_version }}" >> $GITHUB_ENV
+
+outputs:
+  result_file_path:
+    description: 'The path of the result file'
diff --git a/.github/workflows/so_denpency_analyzer.yml b/.github/workflows/so_denpency_analyzer.yml
@@ -0,0 +1,103 @@
+name: PaddlePaddle Packages Dependency So Analysis
+run-name: PaddlePaddle Packages Dependency So Analysis
+
+on:
+  workflow_dispatch:
+    inputs:
+      baselinesetting:
+        description: 'so denpendency baseline setting'
+        required: false
+        default: 'OFF'
+      branch_name:
+        required: false
+        type: string
+        default: 'Develop'
+  workflow_call:
+    inputs:
+      baselinesetting:
+        required: false
+        type: string
+        default: 'OFF'
+      branch_name:
+        required: false
+        type: string
+        default: 'Develop'
+  schedule:
+    - cron: "0 20 * * *"
+
+jobs:
+  so-analysis:
+    environment: CodeSync
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ce_task_name:
+          - "TagBuild-Training-Linux-Gpu-Cuda11.8-Cudnn8.6-Mkl-Avx-Gcc8.2-SelfBuiltPypiUse"
+          - "TagBuild-Training-Linux-Cpu-Mkl-Avx-Gcc82-SelfBuiltPypiUse"
+          - "TagBuild-Training-Linux-Cpu-ARM-SelfBuiltPypiUse"
+          - "TagBuild-Training-Linux-Gpu-Cuda12.6-Cudnn9.5-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse"
+        python_version: ["3.8", "3.10", "3.13"]
+        is_cuda: ["True", "False"]
+        exclude:
+          - ce_task_name: "TagBuild-Training-Linux-Cpu-Mkl-Avx-Gcc82-SelfBuiltPypiUse"
+            is_cuda: "True"
+          - ce_task_name: "TagBuild-Training-Linux-Cpu-ARM-SelfBuiltPypiUse"
+            is_cuda: "True"
+          - ce_task_name: "TagBuild-Training-Linux-Gpu-Cuda12.6-Cudnn9.5-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse"
+            is_cuda: "False"
+          - ce_task_name: "TagBuild-Training-Linux-Gpu-Cuda11.8-Cudnn8.6-Mkl-Avx-Gcc8.2-SelfBuiltPypiUse"
+            is_cuda: "False"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          sparse-checkout: |
+            .github
+            publish_scripts
+            tools
+          fetch-depth: 1
+
+      - name: Wheel Download and Analysis
+        id: analyze
+        uses: ./.github/actions/so_dependency_analyzer
+        with:
+          ce_task_name: ${{ github.event.inputs.branch_name || inputs.branch_name || 'Develop'}}-${{ matrix.ce_task_name }}
+          is_cuda: ${{ matrix.is_cuda }}
+          python_version: ${{ matrix.python_version }}
+        
+      - name: Upload so analysis result
+        env:
+          AK: ${{ secrets.BOS_AK }}
+          SK: ${{ secrets.BOS_SK }}
+          BASELINE_SETTING: ${{ github.event.inputs.baselinesetting || inputs.baselinesetting || 'OFF' }}
+        run: |
+          set -x
+          tree -L 3
+          echo "The ce_task_name input passed to the analyze step is: ${ce_task_name}"
+          echo "The python_version input passed to the analyze step is: ${python_version}"
+          echo "The result file is at: ${result_file_path}"
+          cat ${result_file_path}
+          echo "The description of the result file is:${description_file_path}"
+          cp ${description_file_path} ./
+          cp ${result_file_path} ./
+          commit_id=$(grep "^commit_id:" description.txt | cut -d':' -f2)
+          echo ${commit_id}
+          python -m pip install bce-python-sdk==0.9.29
+          if [[ ${BASELINE_SETTING} == "ON" ]];then
+            python tools/bos_tools.py description.txt paddle-qa/so_analysis/${ce_task_name}/baseline/${python_version//./}
+            python tools/bos_tools.py so_dependencies_static.json paddle-qa/so_analysis/${ce_task_name}/baseline/${python_version//./}
+          else
+            python tools/bos_tools.py description.txt paddle-qa/so_analysis/${ce_task_name}/${commit_id}/${python_version//./}
+            python tools/bos_tools.py so_dependencies_static.json paddle-qa/so_analysis/${ce_task_name}/${commit_id}/${python_version//./}
+            wget https://paddle-qa.bj.bcebos.com/so_analysis/${ce_task_name}/baseline/${python_version//./}/so_dependencies_static.json -O baseline_so_dependencies_static.json
+            exit_code=0
+            python publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py compare baseline_so_dependencies_static.json so_dependencies_static.json || exit_code=$?
+            if [ ${exit_code} -eq 0 ];then
+              echo "No change detected in so dependencies."
+              exit 0
+            else
+              echo "Change detected in so dependencies."
+              exit 1
+            fi
+          fi
diff --git a/publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py b/publish_scripts/so_dependency_analyzer/so_dependency_analyzer.py
@@ -0,0 +1,104 @@
+import os
+import subprocess
+import json
+import sys
+
+def find_so_files(root_dir):
+    """查找所有 .so 文件"""
+    so_files = []
+    for dirpath, _, filenames in os.walk(root_dir):
+        for filename in filenames:
+            if filename.endswith(".so") or ".so." in filename:
+                full_path = os.path.join(dirpath, filename)
+                so_files.append(full_path)
+    return so_files
+
+def get_so_dependencies_readelf(so_file):
+    """使用 readelf 获取 .so 的依赖"""
+    try:
+        output = subprocess.check_output(['readelf', '-d', so_file], stderr=subprocess.STDOUT, text=True)
+        deps = []
+        for line in output.splitlines():
+            if 'Shared library' in line:
+                start = line.find('[')
+                end = line.find(']')
+                if start != -1 and end != -1:
+                    lib_name = line[start+1:end]
+                    deps.append(lib_name)
+        return deps
+    except subprocess.CalledProcessError as e:
+        return [f"Error: {e.output.strip()}"]
+    
+def compare_so_dependency_jsons(json1_path, json2_path):
+    """比较两个 .so 静态依赖 JSON 文件的差异"""
+    with open(json1_path, 'r') as f1, open(json2_path, 'r') as f2:
+        deps1 = json.load(f1)
+        deps2 = json.load(f2)
+
+    so_files1 = set(deps1.keys())
+    so_files2 = set(deps2.keys())
+
+    only_in_1 = so_files1 - so_files2
+    only_in_2 = so_files2 - so_files1
+    common_files = so_files1 & so_files2
+
+    diff_results = []
+
+    if only_in_1:
+        diff_results.append(f"🔺 只在 {json1_path} 中存在的 .so 文件:\n" + "\n".join(sorted(only_in_1)))
+    if only_in_2:
+        diff_results.append(f"🔻 只在 {json2_path} 中存在的 .so 文件:\n" + "\n".join(sorted(only_in_2)))
+
+    for so_file in sorted(common_files):
+        deps_1 = set(deps1[so_file])
+        deps_2 = set(deps2[so_file])
+        if deps_1 != deps_2:
+            diff_results.append(
+                f"🔁 文件依赖不同: {so_file}\n"
+                f"  ➤ {json1_path}: {sorted(deps_1)}\n"
+                f"  ➤ {json2_path}: {sorted(deps_2)}"
+            )
+
+    if not diff_results:
+        print("✅ 两个 JSON 文件中的所有 .so 文件及其依赖完全一致。")
+    else:
+        print("⚠️ 发现差异：\n")
+        print("\n\n".join(diff_results))
+        sys.exit(1)
+
+def main(target_dir, output_json):
+    # target_dir = "paddle"  # TODO: 修改为你自己的路径
+    # output_json = "so_dependencies_static.json"
+
+    so_files = find_so_files(target_dir)
+    all_deps = {}
+
+    for so_file in so_files:
+        deps = get_so_dependencies_readelf(so_file)
+        all_deps[so_file] = deps
+
+    with open(output_json, "w") as f:
+        json.dump(all_deps, f, indent=4)
+
+    print(f"✅ 静态依赖信息已保存到 {output_json}")
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Extract or compare .so static dependencies")
+    subparsers = parser.add_subparsers(dest="command")
+
+    extract_parser = subparsers.add_parser("extract")
+    extract_parser.add_argument("--target_dir", type=str, default="paddle", help="Target directory to search for .so files")
+    extract_parser.add_argument("--output_json", type=str, default="so_dependencies_static.json", help="Output JSON file name")
+
+    compare_parser = subparsers.add_parser("compare")
+    compare_parser.add_argument("json1", type=str, help="First JSON file to compare")
+    compare_parser.add_argument("json2", type=str, help="Second JSON file to compare")
+
+    args = parser.parse_args()
+
+    if args.command == "extract":
+        main(args.target_dir, args.output_json)
+    elif args.command == "compare":
+        compare_so_dependency_jsons(args.json1, args.json2)
diff --git a/publish_scripts/so_dependency_analyzer/wheel_download.py b/publish_scripts/so_dependency_analyzer/wheel_download.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import requests
+import os
+
+def build_wheel_url(base_url, python_version="3.10", is_cuda=True):
+    """
+    根据给定的基础URL、Python版本和是否使用CUDA生成PaddlePaddle的wheel文件下载链接。
+
+    Args:
+        base_url (str): 基础URL，用于生成下载链接。
+        python_version (str, optional): Python版本，默认为"3.10"。
+        is_cuda (bool, optional): 是否使用CUDA，默认为True。
+
+    Returns:
+        str: 生成的PaddlePaddle wheel文件下载链接。
+
+    Raises:
+        Exception: 如果下载描述文件失败，抛出异常。
+        ValueError: 如果无法从描述文件中提取commit_id或wheel_version，抛出异常。
+
+    """
+    desc_url = f"{base_url}/latest/description.txt"
+    response = requests.get(desc_url)
+    if response.status_code != 200:
+        raise Exception(f"❌ 下载失败: {desc_url}")
+    
+    commit_id = None
+    wheel_version = None
+    for line in response.text.splitlines():
+        if line.startswith("commit_id:"):
+            commit_id = line.split(":", 1)[1].strip()
+        elif line.startswith("wheel_version:"):
+            wheel_version = line.split(":", 1)[1].strip()
+    
+    if not commit_id or not wheel_version:
+        raise ValueError("❗ 无法从 description.txt 中提取 commit_id 或 wheel_version")
+
+    py_tag = f"cp{python_version.replace('.', '')}"
+    if not is_cuda:
+        package_name = "paddlepaddle"
+    else:
+        package_name = "paddlepaddle_gpu"
+    if "ARM" in base_url:
+        wheel_url = (
+        f"{base_url}/{commit_id}/"
+        f"{package_name}-{wheel_version}-{py_tag}-{py_tag}-linux_aarch64.whl"
+        )
+    else:
+        wheel_url = (
+            f"{base_url}/{commit_id}/"
+            f"{package_name}-{wheel_version}-{py_tag}-{py_tag}-linux_x86_64.whl"
+        )
+    return wheel_url
+
+
+def download_file(url, save_path):
+    """
+    从给定的URL下载文件并保存到指定的路径。
+
+    Args:
+        url (str): 要下载文件的URL。
+        save_path (str): 文件保存的路径。
+
+    Returns:
+        None
+
+    Raises:
+        Exception: 如果下载失败，将引发异常。
+
+    """
+    # 确保文件夹存在
+    dir_name = os.path.dirname(save_path)
+    if not os.path.exists(dir_name):
+        os.makedirs(dir_name)
+        print(f"创建目录: {dir_name}")
+    print(f"📥 正在下载: {url}")
+    response = requests.get(url, stream=True)
+    if response.status_code != 200:
+        raise Exception(f"❌ 下载失败: {url}")
+    
+    with open(save_path, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                f.write(chunk)
+    print(f"✅ 保存成功: {save_path}")
+
+if __name__ == "__main__":
+    import argparse
+    def str2bool(v):
+        if isinstance(v, bool):
+            return v
+        if v.lower() in ('yes', 'true', 't', '1'):
+            return True
+        elif v.lower() in ('no', 'false', 'f', '0'):
+            return False
+        else:
+            raise argparse.ArgumentTypeError('Boolean value expected.')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--python_version", type=str,default='3.10', help="Python version")
+    parser.add_argument("--ce_task_name", type=str, help="CE task name")
+    parser.add_argument("--is_cuda", type=str2bool, default=True, help="Whether to use CUDA")
+    parser.add_argument("--output_dir", type=str, default=".", help="wheel package output directory")
+    args = parser.parse_args()
+    # parser.add_argument("base_url", type=str, help="Base URL of the PaddlePaddle wheel")
+    # 示例调用
+    base_url =f"https://paddle-qa.bj.bcebos.com/paddle-pipeline/{args.ce_task_name}"
+
+    wheel_url = build_wheel_url(base_url, args.python_version, args.is_cuda)
+    print("✅ 下载链接为：")
+    print(wheel_url)
+    description_url = f"{base_url}/latest/description.txt"
+    description_save_path = os.path.join(args.output_dir, "description.txt")
+    wheel_filename = os.path.basename(wheel_url)
+    wheel_save_path = os.path.join(args.output_dir, wheel_filename)
+    download_file(wheel_url, wheel_save_path)
+    download_file(description_url, description_save_path)
