patch(v1.7): Live Migration and MSync support

.github/workflows/hydrun.yaml

@@ -0,0 +1,97 @@
+name: hydrun CI
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: "0 0 * * 0"
+
+jobs:
+  build-linux:
+    runs-on: ${{ matrix.target.runner }}
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        target:
+          # Binaries
+          - id: rust.x86_64
+            src: .
+            os: alpine:edge
+            flags: ""
+            cmd: ./Hydrunfile rust x86_64
+            dst: out/*
+            runner: depot-ubuntu-22.04-32
+          - id: rust.aarch64
+            src: .
+            os: alpine:edge
+            flags: ""
+            cmd: ./Hydrunfile rust aarch64
+            dst: out/*
+            runner: depot-ubuntu-22.04-arm-32
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Restore ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /tmp/ccache
+          key: cache-ccache-${{ matrix.target.id }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set up hydrun
+        run: |
+          curl -L -o /tmp/hydrun "https://github.com/pojntfx/hydrun/releases/latest/download/hydrun.linux-$(uname -m)"
+          sudo install /tmp/hydrun /usr/local/bin
+      - name: Build with hydrun
+        working-directory: ${{ matrix.target.src }}
+        run: hydrun -o ${{ matrix.target.os }} ${{ matrix.target.flags }} "${{ matrix.target.cmd }}"
+      - name: Fix permissions for output
+        run: sudo chown -R $USER .
+      - name: Save ccache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /tmp/ccache
+          key: cache-ccache-${{ matrix.target.id }}
+      - name: Upload output
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.target.id }}
+          path: ${{ matrix.target.dst }}
+
+  publish-linux:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs: build-linux
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download output
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/out
+      - name: Extract branch name
+        id: extract_branch
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Publish pre-release to GitHub releases
+        if: ${{ github.ref == 'refs/heads/firecracker-v1.7-live-migration-and-msync' || github.ref == 'refs/heads/firecracker-v1.7-live-migration-pvm-and-msync' || github.ref == 'refs/heads/main-live-migration-and-msync' }}
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: release-${{ steps.extract_branch.outputs.branch }}
+          prerelease: true
+          files: |
+            /tmp/out/*/*
+      - name: Publish release to GitHub releases
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: false
+          files: |
+            /tmp/out/*/*

Hydrunfile

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+# Rust
+if [ "$1" = "rust" ]; then
+  # Install native dependencies
+  apk add rust cargo clang-dev cmake linux-headers make git
+
+  # Configure Git
+  git config --global --add safe.directory '*'
+
+  # Build
+  cp "resources/seccomp/$2-unknown-linux-musl.json" "resources/seccomp/$2-alpine-linux-musl.json"
+  export RUSTFLAGS='-C target-feature=+crt-static'
+  cargo build --package firecracker --package jailer --package seccompiler --package rebase-snap --package cpu-template-helper --target "$2-alpine-linux-musl" --all-features --release
+
+  # Stage binaries
+  mkdir -p out
+
+  dir="./build/cargo_target/$2-alpine-linux-musl/release"
+  for file in $(ls "$dir"); do
+    if [[ -x "$dir/$file" && ! -d "$dir/$file" ]]; then
+      cp "$dir/$file" "./out/${file}.linux-$2"
+    fi
+  done
+
+  exit 0
+fi

docs/cpu_templates/cpu-template-helper.md

@@ -180,7 +180,7 @@ CPU features to a heterogeneous fleet consisting of multiple CPU models.
 ### MSRs excluded from guest CPU configuration dump
 
 | Register name                           | Index                   |
-| --------------------------------------- | ----------------------- |
+|-----------------------------------------|-------------------------|
 | MSR_IA32_TSC                            | 0x00000010              |
 | MSR_ARCH_PERFMON_PERFCTRn               | 0x000000c1 - 0x000000d2 |
 | MSR_ARCH_PERFMON_EVENTSELn              | 0x00000186 - 0x00000197 |
@@ -230,6 +230,7 @@ CPU features to a heterogeneous fleet consisting of multiple CPU models.
 | HV_X64_MSR_SYNDBG_RECV_BUFFER           | 0x400000f4              |
 | HV_X64_MSR_SYNDBG_PENDING_BUFFER        | 0x400000f5              |
 | HV_X64_MSR_SYNDBG_OPTIONS               | 0x400000ff              |
+| HV_X64_MSR_TSC_INVARIANT_CONTROL        | 0x40000118              |
 
 ### ARM registers excluded from guest CPU configuration dump
 

resources/seccomp/aarch64-unknown-linux-musl.json

@@ -35,6 +35,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -35,6 +35,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

src/firecracker/src/api_server/mod.rs

@@ -158,6 +158,14 @@ impl ApiServer {
                     &METRICS.latencies_us.diff_create_snapshot,
                     "create diff snapshot",
                 )),
+                SnapshotType::Msync => Some((
+                    &METRICS.latencies_us.diff_create_snapshot,
+                    "memory synchronization snapshot",
+                )),
+                SnapshotType::MsyncAndState => Some((
+                    &METRICS.latencies_us.diff_create_snapshot,
+                    "memory synchronization and state snapshot",
+                )),
             },
             VmmAction::LoadSnapshot(_) => {
                 Some((&METRICS.latencies_us.load_snapshot, "load snapshot"))

src/firecracker/src/api_server/request/snapshot.rs

@@ -99,6 +99,7 @@ fn parse_put_snapshot_load(body: &Body) -> Result<ParsedRequest, Error> {
         mem_backend,
         enable_diff_snapshots: snapshot_config.enable_diff_snapshots,
         resume_vm: snapshot_config.resume_vm,
+        shared: snapshot_config.shared,
     };
 
     // Construct the `ParsedRequest` object.
@@ -175,6 +176,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: false,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -202,6 +204,7 @@ mod tests {
             },
             enable_diff_snapshots: true,
             resume_vm: false,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -229,6 +232,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: true,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -253,6 +257,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: true,
+            shared: false,
         };
         let parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert_eq!(

src/firecracker/swagger/firecracker.yaml

@@ -1199,6 +1199,8 @@ definitions:
         enum:
           - Full
           - Diff
+          - Msync
+          - MsyncAndState
         description:
           Type of snapshot to create. It is optional and by default, a full
           snapshot is created.
@@ -1234,6 +1236,11 @@ definitions:
         type: boolean
         description:
           When set to true, the vm is also resumed if the snapshot load is successful.
+      shared:
+        type: boolean
+        description: When set to true and the guest memory backend is a file,
+          changes to the memory are asynchronously written back to the
+          backend as the VM is running.
 
   TokenBucket:
     type: object

src/vmm/src/arch/x86_64/msr.rs

@@ -50,7 +50,7 @@ const APIC_BASE_MSR: u32 = 0x800;
 /// Number of APIC MSR indexes
 const APIC_MSR_INDEXES: u32 = 0x400;
 
-/// Custom MSRs fall in the range 0x4b564d00-0x4b564dff
+/// Custom KVM MSRs fall in the range 0x4b564d00-0x4b564dff
 const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b56_4d00;
 const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b56_4d01;
 const MSR_KVM_ASYNC_PF_EN: u32 = 0x4b56_4d02;
@@ -236,6 +236,7 @@ static SERIALIZABLE_MSR_RANGES: &[MsrRange] = &[
     MSR_RANGE!(MSR_K7_HWCR),
     MSR_RANGE!(MSR_KVM_POLL_CONTROL),
     MSR_RANGE!(MSR_KVM_ASYNC_PF_INT),
+    MSR_RANGE!(MSR_IA32_TSX_CTRL),
 ];
 
 /// Specifies whether a particular MSR should be included in vcpu serialization.
@@ -387,6 +388,8 @@ static UNDUMPABLE_MSR_RANGES: &[MsrRange] = &[
     MSR_RANGE!(HV_X64_MSR_SYNDBG_CONTROL, 5),
     // HV_X64_MSR_SYNDBG_OPTIONS
     MSR_RANGE!(HV_X64_MSR_SYNDBG_OPTIONS),
+    // HV_X64_MSR_TSC_INVARIANT_CONTROL
+    MSR_RANGE!(HV_X64_MSR_TSC_INVARIANT_CONTROL),
 ];
 
 /// Specifies whether a particular MSR should be dumped.

src/vmm/src/logger/metrics.rs

@@ -616,6 +616,10 @@ pub struct PerformanceMetrics {
     pub vmm_full_create_snapshot: SharedStoreMetric,
     /// Measures the snapshot diff create time, at the VMM level, in microseconds.
     pub vmm_diff_create_snapshot: SharedStoreMetric,
+    /// Measures the snapshot memory synchronization time, at the VMM level, in microseconds.
+    pub vmm_msync_create_snapshot: SharedStoreMetric,
+    /// Measures the snapshot memory synchronization and state time, at the VMM level, in microseconds.
+    pub vmm_msync_and_state_create_snapshot: SharedStoreMetric,
     /// Measures the snapshot load time, at the VMM level, in microseconds.
     pub vmm_load_snapshot: SharedStoreMetric,
     /// Measures the microVM pausing duration, at the VMM level, in microseconds.
@@ -634,6 +638,8 @@ impl PerformanceMetrics {
             resume_vm: SharedStoreMetric::new(),
             vmm_full_create_snapshot: SharedStoreMetric::new(),
             vmm_diff_create_snapshot: SharedStoreMetric::new(),
+            vmm_msync_create_snapshot: SharedStoreMetric::new(),
+            vmm_msync_and_state_create_snapshot: SharedStoreMetric::new(),
             vmm_load_snapshot: SharedStoreMetric::new(),
             vmm_pause_vm: SharedStoreMetric::new(),
             vmm_resume_vm: SharedStoreMetric::new(),