From 237fe57bc249ed553c6749897ccdb5b6a78f0d7c Mon Sep 17 00:00:00 2001 From: Scott Kidder Date: Sun, 3 Nov 2024 05:55:36 +0000 Subject: [PATCH] Verify and auto-update native dependencies built in CI. --- .github/workflows/deps.yaml | 145 +++++++++++++++++++++++++++-- deps/verify_deps.py | 176 ++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 8 deletions(-) create mode 100755 deps/verify_deps.py diff --git a/.github/workflows/deps.yaml b/.github/workflows/deps.yaml index 206f1720..ff43c2da 100644 --- a/.github/workflows/deps.yaml +++ b/.github/workflows/deps.yaml @@ -26,12 +26,14 @@ jobs: - name: Install build tools run: | sudo apt-get update - sudo apt-get install nasm + sudo apt-get install nasm python3 - name: Build deps run: | + echo "Starting dependency build..." export MAKEFLAGS="-j$(nproc)" ./deps/build-deps-linux.sh + echo "Dependency build completed" - run: | git status @@ -48,14 +50,22 @@ jobs: go build go test -v - - name: Compress deps - run: tar -czf deps.tar.gz deps/linux + - name: Generate build info + run: | + ./deps/verify_deps.py generate \ + --deps-dir deps/linux \ + --platform linux \ + --commit ${{ github.sha }} + + - name: Create deps archive + run: | + tar -czf deps-linux.tar.gz deps/linux/ - name: Upload deps artifact uses: actions/upload-artifact@v4 with: name: deps-linux.tar.gz - path: deps.tar.gz + path: deps-linux.tar.gz macos: name: macOS @@ -73,8 +83,10 @@ jobs: - name: Build deps run: | + echo "Starting dependency build..." export MAKEFLAGS="-j$(nproc)" ./deps/build-deps-osx.sh + echo "Dependency build completed" - run: | git status @@ -90,12 +102,129 @@ jobs: run: | go build go test -v - - - name: Compress deps - run: tar -czf deps.tar.gz deps/osx + + - name: Generate build info + run: | + ./deps/verify_deps.py generate \ + --deps-dir deps/osx \ + --platform macos \ + --commit ${{ github.sha }} + + - name: Create deps archive + run: | + tar -czf deps-macos.tar.gz deps/osx/ - name: Upload deps artifact uses: actions/upload-artifact@v4 with: name: deps-macos.tar.gz - path: deps.tar.gz + path: deps-macos.tar.gz + + verify: + name: Verify Build Artifacts + needs: [linux, macos] + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Download Linux artifact + uses: actions/download-artifact@v4 + with: + name: deps-linux.tar.gz + path: . + + - name: Download macOS artifact + uses: actions/download-artifact@v4 + with: + name: deps-macos.tar.gz + path: . + + - name: Extract artifacts + run: | + tar xzf deps-linux.tar.gz + tar xzf deps-macos.tar.gz + + - name: Verify artifacts match checked-in deps + run: | + python3 ./deps/verify_deps.py verify \ + --deps-dir deps/linux \ + --build-info deps/linux/build-info.json + + python3 ./deps/verify_deps.py verify \ + --deps-dir deps/osx \ + --build-info deps/osx/build-info.json + + update-deps: + name: Update Checked-in Dependencies + needs: [linux, macos] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + timeout-minutes: 15 + + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Download Linux artifact + uses: actions/download-artifact@v4 + with: + name: deps-linux.tar.gz + path: . + + - name: Download macOS artifact + uses: actions/download-artifact@v4 + with: + name: deps-macos.tar.gz + path: . + + - name: Extract and update deps + run: | + # Remove existing deps directories to avoid stale files + rm -rf deps/linux/* deps/osx/* + + tar xzf deps-linux.tar.gz + tar xzf deps-macos.tar.gz + + - name: Commit updated deps + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git config --local commit.gpgsign false + + # Stage each type of file explicitly + shopt -s nullglob # Handle case where globs don't match + + # Binary files + for f in deps/*/lib/*.{so,so.*,dylib,dylib.*,a}; do + if [ -f "$f" ]; then + git add -f "$f" + fi + done + + # Header files + for f in deps/*/include/**/*.h; do + if [ -f "$f" ]; then + git add -f "$f" + fi + done + + # Build info + for f in deps/*/build-info.json; do + if [ -f "$f" ]; then + git add -f "$f" + fi + done + + # Only commit if there are changes + if ! git diff --cached --quiet; then + git commit -m "Update native dependencies from ${{ github.sha }} [skip ci] + + Dependencies built by workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + git push + else + echo "No changes to checked-in dependencies" + fi diff --git a/deps/verify_deps.py b/deps/verify_deps.py new file mode 100755 index 00000000..bc8a27a3 --- /dev/null +++ b/deps/verify_deps.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path +from typing import Dict, List, NamedTuple, Tuple + + +class BuildInfo(NamedTuple): + commit_sha: str + platform: str + files: Dict[str, str] # relative path -> sha256 + + +def calculate_checksum(file_path: Path) -> str: + """Calculate SHA-256 checksum of a file.""" + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + # Read in 1MB chunks to handle large files efficiently + for byte_block in iter(lambda: f.read(4096 * 256), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def scan_deps(deps_dir: Path) -> Dict[str, str]: + """Scan directory for dependency files and calculate their checksums.""" + checksums = {} + for file_path in deps_dir.rglob("*"): + if not file_path.is_file(): + continue + + # Only process shared libraries, static libraries, and headers + if not ( + file_path.suffix in [".so", ".dylib", ".a", ".h"] + or ( + file_path.suffix.startswith(".so.") + or file_path.suffix.startswith(".dylib.") + ) + ): + continue + + # Get path relative to deps_dir + rel_path = str(file_path.relative_to(deps_dir)) + try: + checksums[rel_path] = calculate_checksum(file_path) + except (IOError, OSError) as e: + print(f"Error processing {rel_path}: {e}", file=sys.stderr) + continue + + return checksums + + +def generate_build_info(deps_dir: Path, platform: str, commit_sha: str) -> BuildInfo: + """Generate build info for the given deps directory.""" + checksums = scan_deps(deps_dir) + return BuildInfo(commit_sha=commit_sha, platform=platform, files=checksums) + + +def verify_deps(deps_dir: Path, build_info: BuildInfo) -> Tuple[bool, List[str]]: + """Verify deps directory against build info.""" + mismatches = [] + valid = True + + # Get current state of deps directory + current_checksums = scan_deps(deps_dir) + + print(f"Found {len(current_checksums)} files to verify") + + # Check for missing or mismatched files + for rel_path, expected_checksum in build_info.files.items(): + if rel_path not in current_checksums: + mismatches.append(f"{rel_path}: file not found in deps directory") + valid = False + continue + + actual_checksum = current_checksums[rel_path] + if actual_checksum != expected_checksum: + mismatches.append( + f"{rel_path}: checksum mismatch\n" + f" expected: {expected_checksum}\n" + f" got: {actual_checksum}" + ) + valid = False + else: + print(f"Verified: {rel_path}") + + # Check for extra files + for rel_path in current_checksums: + if rel_path not in build_info.files: + mismatches.append(f"{rel_path}: extra file in deps directory") + valid = False + + return valid, mismatches + + +def main(): + parser = argparse.ArgumentParser(description="Verify Lilliput dependencies") + + # Create subparsers first + subparsers = parser.add_subparsers(dest="command", required=True) + + # Generate command + generate_parser = subparsers.add_parser( + "generate", help="Generate build info for dependencies" + ) + generate_parser.add_argument( + "--deps-dir", required=True, type=Path, help="Directory containing dependencies" + ) + generate_parser.add_argument( + "--platform", + required=True, + choices=["linux", "macos"], + help="Platform identifier", + ) + generate_parser.add_argument( + "--commit", required=True, help="Commit SHA that produced the build" + ) + generate_parser.add_argument( + "--output", type=Path, help="Output file (default: /build-info.json)" + ) + + # Verify command + verify_parser = subparsers.add_parser( + "verify", help="Verify deps against build info" + ) + verify_parser.add_argument( + "--deps-dir", required=True, type=Path, help="Directory containing dependencies" + ) + verify_parser.add_argument( + "--build-info", required=True, type=Path, help="Path to build info JSON file" + ) + + args = parser.parse_args() + + if not os.path.exists(args.deps_dir): + print(f"Error: deps directory not found: {args.deps_dir}", file=sys.stderr) + sys.exit(1) + + if args.command == "generate": + build_info = generate_build_info(args.deps_dir, args.platform, args.commit) + + output_file = args.output or args.deps_dir / "build-info.json" + + try: + with open(output_file, "w") as f: + json.dump(build_info._asdict(), f, indent=4) + print(f"Build info generated successfully: {output_file}") + except (IOError, OSError) as e: + print(f"Error writing build info: {e}", file=sys.stderr) + sys.exit(1) + + elif args.command == "verify": + try: + with open(args.build_info) as f: + build_info_dict = json.load(f) + build_info = BuildInfo(**build_info_dict) + except (IOError, OSError, json.JSONDecodeError) as e: + print(f"Error reading build info: {e}", file=sys.stderr) + sys.exit(1) + + print(f"Verifying deps against build from commit {build_info.commit_sha}") + valid, mismatches = verify_deps(args.deps_dir, build_info) + + if not valid: + print("\nVerification failed:") + for mismatch in mismatches: + print(f" {mismatch}") + sys.exit(1) + + print("\nAll dependencies verified successfully") + + +if __name__ == "__main__": + main()