Calculate checksums of native deps and auto-update in repo.

.github/workflows/deps.yaml

@@ -23,15 +23,25 @@ jobs:
       - name: Check out repo
         uses: actions/checkout@v4
 
+      - name: Cache build tools
+        uses: actions/cache@v3
+        with:
+          path: |
+            /var/cache/apt
+            /var/lib/apt
+          key: ${{ runner.os }}-build-tools-${{ hashFiles('deps/build-deps-linux.sh') }}
+
       - name: Install build tools
         run: |
           sudo apt-get update
-          sudo apt-get install nasm
+          sudo apt-get install nasm python3
 
       - name: Build deps
         run: |
+          echo "Starting dependency build..."
           export MAKEFLAGS="-j$(nproc)"
           ./deps/build-deps-linux.sh
+          echo "Dependency build completed"
 
       - run: |
           git status
@@ -48,14 +58,22 @@ jobs:
           go build
           go test -v
 
-      - name: Compress deps
-        run: tar -czf deps.tar.gz deps/linux
+      - name: Generate build info
+        run: |
+          ./deps/verify_deps.py generate \
+            --deps-dir deps/linux \
+            --platform linux \
+            --commit ${{ github.sha }}
+
+      - name: Create deps archive
+        run: |
+          tar -czf deps-linux.tar.gz deps/linux/
 
       - name: Upload deps artifact
         uses: actions/upload-artifact@v4
         with:
           name: deps-linux.tar.gz
-          path: deps.tar.gz
+          path: deps-linux.tar.gz
 
   macos:
     name: macOS
@@ -64,6 +82,16 @@ jobs:
       - name: Check out repo
         uses: actions/checkout@v4
 
+      - name: Cache Homebrew packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/Homebrew
+            ~/Library/Logs/Homebrew
+          key: ${{ runner.os }}-brew-${{ hashFiles('deps/build-deps-osx.sh') }}
+          restore-keys: |
+            ${{ runner.os }}-brew-
+
       - name: Install build tools
         run: |
           brew install autoconf
@@ -73,8 +101,10 @@ jobs:
 
       - name: Build deps
         run: |
+          echo "Starting dependency build..."
           export MAKEFLAGS="-j$(nproc)"
           ./deps/build-deps-osx.sh
+          echo "Dependency build completed"
 
       - run: |
           git status
@@ -90,12 +120,129 @@ jobs:
         run: |
           go build
           go test -v
-
-      - name: Compress deps
-        run: tar -czf deps.tar.gz deps/osx
+
+      - name: Generate build info
+        run: |
+          ./deps/verify_deps.py generate \
+            --deps-dir deps/osx \
+            --platform macos \
+            --commit ${{ github.sha }}
+
+      - name: Create deps archive
+        run: |
+          tar -czf deps-macos.tar.gz deps/osx/
 
       - name: Upload deps artifact
         uses: actions/upload-artifact@v4
         with:
           name: deps-macos.tar.gz
-          path: deps.tar.gz
+          path: deps-macos.tar.gz
+
+  verify:
+    name: Verify Build Artifacts
+    needs: [linux, macos]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Download Linux artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-linux.tar.gz
+          path: .
+
+      - name: Download macOS artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-macos.tar.gz
+          path: .
+
+      - name: Extract artifacts
+        run: |
+          tar xzf deps-linux.tar.gz
+          tar xzf deps-macos.tar.gz
+
+      - name: Verify artifacts match checked-in deps
+        run: |
+          python3 ./deps/verify_deps.py verify \
+            --deps-dir deps/linux \
+            --build-info deps/linux/build-info.json
+
+          python3 ./deps/verify_deps.py verify \
+            --deps-dir deps/osx \
+            --build-info deps/osx/build-info.json
+
+  update-deps:
+    name: Update Checked-in Dependencies
+    needs: [linux, macos]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Download Linux artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-linux.tar.gz
+          path: .
+
+      - name: Download macOS artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-macos.tar.gz
+          path: .
+
+      - name: Extract and update deps
+        run: |
+          # Remove existing deps directories to avoid stale files
+          rm -rf deps/linux/* deps/osx/*
+
+          tar xzf deps-linux.tar.gz
+          tar xzf deps-macos.tar.gz
+
+      - name: Commit updated deps
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git config --local commit.gpgsign false
+
+          # Stage each type of file explicitly
+          shopt -s nullglob # Handle case where globs don't match
+
+          # Binary files
+          for f in deps/*/lib/*.{so,so.*,dylib,dylib.*,a}; do
+            if [ -f "$f" ]; then
+              git add -f "$f"
+            fi
+          done
+
+          # Header files
+          for f in deps/*/include/**/*.h; do
+            if [ -f "$f" ]; then
+              git add -f "$f"
+            fi
+          done
+
+          # Build info
+          for f in deps/*/build-info.json; do
+            if [ -f "$f" ]; then
+              git add -f "$f"
+            fi
+          done
+
+          # Only commit if there are changes
+          if ! git diff --cached --quiet; then
+            git commit -m "Update native dependencies from ${{ github.sha }} [skip ci]
+
+            Dependencies built by workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+            git push
+          else
+            echo "No changes to checked-in dependencies"
+          fi

deps/verify_deps.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Dict, List, NamedTuple, Tuple
+
+
+class BuildInfo(NamedTuple):
+    commit_sha: str
+    platform: str
+    files: Dict[str, str]  # relative path -> sha256
+
+
+def calculate_checksum(file_path: Path) -> str:
+    """Calculate SHA-256 checksum of a file."""
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        # Read in 1MB chunks to handle large files efficiently
+        for byte_block in iter(lambda: f.read(4096*256), b""):
+            sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+def scan_deps(deps_dir: Path) -> Dict[str, str]:
+    """Scan directory for dependency files and calculate their checksums."""
+    checksums = {}
+    for file_path in deps_dir.rglob("*"):
+        if not file_path.is_file():
+            continue
+
+        # Only process shared and static libraries and headers
+        if not file_path.suffix in ['.so', '.dylib', '.a', '.h']:
+            continue
+
+        # Get path relative to deps_dir
+        rel_path = str(file_path.relative_to(deps_dir))
+        try:
+            checksums[rel_path] = calculate_checksum(file_path)
+        except (IOError, OSError) as e:
+            print(f"Error processing {rel_path}: {e}", file=sys.stderr)
+            continue
+
+    return checksums
+
+
+def generate_build_info(deps_dir: Path, platform: str, commit_sha: str) -> BuildInfo:
+    """Generate build info for the given deps directory."""
+    checksums = scan_deps(deps_dir)
+    return BuildInfo(
+        commit_sha=commit_sha,
+        platform=platform,
+        files=checksums
+    )
+
+
+def verify_deps(deps_dir: Path, build_info: BuildInfo) -> Tuple[bool, List[str]]:
+    """Verify deps directory against build info."""
+    mismatches = []
+    valid = True
+
+    # Get current state of deps directory
+    current_checksums = scan_deps(deps_dir)
+
+    print(f"Found {len(current_checksums)} files to verify")
+
+    # Check for missing or mismatched files
+    for rel_path, expected_checksum in build_info.files.items():
+        if rel_path not in current_checksums:
+            mismatches.append(f"{rel_path}: file not found in deps directory")
+            valid = False
+            continue
+
+        actual_checksum = current_checksums[rel_path]
+        if actual_checksum != expected_checksum:
+            mismatches.append(
+                f"{rel_path}: checksum mismatch\n"
+                f"  expected: {expected_checksum}\n"
+                f"  got:      {actual_checksum}"
+            )
+            valid = False
+        else:
+            print(f"Verified: {rel_path}")
+
+    # Check for extra files
+    for rel_path in current_checksums:
+        if rel_path not in build_info.files:
+            mismatches.append(f"{rel_path}: extra file in deps directory")
+            valid = False
+
+    return valid, mismatches
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Verify Lilliput dependencies")
+
+    # Create subparsers first
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Generate command
+    generate_parser = subparsers.add_parser("generate",
+                                          help="Generate build info for dependencies")
+    generate_parser.add_argument("--deps-dir", required=True, type=Path,
+                               help="Directory containing dependencies")
+    generate_parser.add_argument("--platform", required=True,
+                               choices=["linux", "macos"],
+                               help="Platform identifier")
+    generate_parser.add_argument("--commit", required=True,
+                               help="Commit SHA that produced the build")
+    generate_parser.add_argument("--output", type=Path,
+                               help="Output file (default: <deps-dir>/build-info.json)")
+
+    # Verify command
+    verify_parser = subparsers.add_parser("verify",
+                                        help="Verify deps against build info")
+    verify_parser.add_argument("--deps-dir", required=True, type=Path,
+                             help="Directory containing dependencies")
+    verify_parser.add_argument("--build-info", required=True, type=Path,
+                             help="Path to build info JSON file")
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.deps_dir):
+        print(f"Error: deps directory not found: {args.deps_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    if args.command == "generate":
+        build_info = generate_build_info(
+            args.deps_dir,
+            args.platform,
+            args.commit
+        )
+
+        output_file = args.output or args.deps_dir / "build-info.json"
+
+        try:
+            with open(output_file, "w") as f:
+                json.dump(build_info._asdict(), f, indent=4)
+            print(f"Build info generated successfully: {output_file}")
+        except (IOError, OSError) as e:
+            print(f"Error writing build info: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    elif args.command == "verify":
+        try:
+            with open(args.build_info) as f:
+                build_info_dict = json.load(f)
+            build_info = BuildInfo(**build_info_dict)
+        except (IOError, OSError, json.JSONDecodeError) as e:
+            print(f"Error reading build info: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        print(f"Verifying deps against build from commit {build_info.commit_sha}")
+        valid, mismatches = verify_deps(args.deps_dir, build_info)
+
+        if not valid:
+            print("\nVerification failed:")
+            for mismatch in mismatches:
+                print(f"  {mismatch}")
+            sys.exit(1)
+
+        print("\nAll dependencies verified successfully")
+
+
+if __name__ == "__main__":
+    main()