Modify tests to cover ext2/3/4, XFS filesystems

.github/workflows/ci.yml

@@ -14,6 +14,8 @@ jobs:
     runs-on:
       - metal
       - ${{ matrix.arch }}
+    env:
+      FS: ext2 ext3 ext4 xfs
 
     strategy:
       matrix:
@@ -105,12 +107,23 @@ jobs:
           vagrant ssh ${{env.INSTANCE_NAME}} -c 'sudo make uninstall'
         working-directory: ${{env.BOX_DIR}}
 
+      # FIXME: enable tests on XFS for all distros.
+      # See https://github.com/elastio/elastio-snap/issues/139 about Debian 8
+      # See https://github.com/elastio/elastio-snap/issues/138 about Ubuntu 22.04
+      - name: Ignore XFS tests
+        if: "${{ matrix.distro == 'debian8' || matrix.distro == 'ubuntu2204' }}"
+        run:
+          echo FS="ext2 ext3 ext4" >> $GITHUB_ENV
+
       - name: Run tests (loop device)
-        run: vagrant ssh ${{env.INSTANCE_NAME}} -c "cd tests && sudo ./elio-test.sh"
+        run: |
+          for fs in ${FS[*]}; do
+            vagrant ssh ${{env.INSTANCE_NAME}} -c "cd tests && sudo ./elio-test.sh -f $fs"
+          done
         working-directory: ${{env.BOX_DIR}}
-        # For now tests are taking 10-20 seconds. But they can hang.
-        # 5 minutes seems to be reasonable timeout.
-        timeout-minutes: 5
+        # For now tests are taking 10-20 seconds per 1 fs on amd64 and about 2 minutes on arm64. But they can hang.
+        # 10 minutes seems to be reasonable timeout.
+        timeout-minutes: 10
 
       - name: Show dmesg
         if: always()
@@ -137,9 +150,12 @@ jobs:
         working-directory: ${{env.BOX_DIR}}
 
       - name: Run tests (qcow2 disk)
-        run: vagrant ssh ${{env.INSTANCE_NAME}} -c "cd tests && sudo ./elio-test.sh /dev/vdb1"
+        run: |
+          for fs in ${FS[*]}; do
+            vagrant ssh ${{env.INSTANCE_NAME}} -c "cd tests && sudo ./elio-test.sh -d /dev/vdb1 -f $fs"
+          done
         working-directory: ${{env.BOX_DIR}}
-        timeout-minutes: 5
+        timeout-minutes: 10
 
       - name: Show dmesg
         if: always()

tests/devicetestcase.py

@@ -33,7 +33,8 @@ def setUpClass(cls):
             util.dd("/dev/zero", cls.backing_store, 256, bs="1M")
             cls.device = util.loop_create(cls.backing_store)
 
-        util.mkfs(cls.device)
+        cls.fs = os.getenv('TEST_FS', 'ext4')
+        util.mkfs(cls.device, cls.fs)
         os.makedirs(cls.mount, exist_ok=True)
         util.mount(cls.device, cls.mount)
 

tests/elio-test.sh

@@ -11,18 +11,55 @@ if [ "$EUID" -ne 0 ]; then
     exit 1
 fi
 
+me=$(basename $0)
+filesystems=(ext2 ext3 ext4 xfs)
+
+usage()
+{
+    echo "Usage: $me -d /dev/vda2 -f ext4 | -h"
+    echo
+    echo "  -f | --filesystem : File system to test: $(echo ${filesystems[*]} | sed "s/ /, /g")."
+    echo "                      ext4 is used by default, if this parameter is not specified."
+    echo "  -d | --device     : Block device to use instead of the loopback device."
+    echo "                      NOTE: This device will be formatted during the test and all data there will be lost!"
+    echo "                      A loopback device is used by default, if this parameter is not specified."
+    echo "  -h | --help       : Show this usage help."
+}
+
+while [ "$1" != "" ]; do
+    case $1 in
+        -d | --device)      shift && export TEST_DEVICE=$1 ;;
+        -f | --filesystem)  shift && export TEST_FS=$1 ;;
+        -h | --help)        usage && exit ;;
+        *)                  echo "Wrong arguments!"
+                            usage && exit 15 ;;
+    esac
+        shift
+done
+
+if [ -n "$TEST_DEVICE" ] && ! lsblk $TEST_DEVICE >/dev/null 2>&1; then
+    echo "The script's argumet $TEST_DEVICE seems to be not a block device."
+    exit 1
+fi
+
+if [ -n "$TEST_FS" ] && ! echo ${filesystems[*]} | grep -w -q $TEST_FS; then
+    echo "The script's argumet $TEST_FS seems to be not a supported file system, one of $(echo ${filesystems[*]} | sed "s/ /, /g")."
+    exit 1
+fi
+
 packman="apt-get"
 which yum >/dev/null && packman=yum
 which pip3 >/dev/null || $packman install -y python3-pip
 
-if ! pip3 list --format=legacy 2>/dev/null | grep -q cffi ; then
+if ! pip3 list 2>/dev/null | grep -q cffi ; then
     echo "Python module CFFI is not installed. Installing it..."
     pip3 install cffi
 fi
 
 echo
 echo "elastio-snap: $(git rev-parse --short HEAD)"
 echo "kernel: $(uname -r)"
+echo "filesystem: $([ -n "$TEST_FS" ] && echo $TEST_FS || echo ext4 )"
 echo "gcc: $(gcc --version | awk 'NR==1 {print $3}')"
 echo "bash: ${BASH_VERSION}"
 echo "python: $(python3 --version)"
@@ -31,12 +68,6 @@ echo
 dmesg -c &> /dev/null
 >| dmesg.log
 
-[ ! -z "$1" ] && export TEST_DEVICE="$1"
-if ! lsblk $TEST_DEVICE >/dev/null 2>&1; then
-    echo "The script's argumet $TEST_DEVICE seems to be not a block device."
-    exit 1
-fi
-
 python3 -m unittest -v
 ret=$?
 dmesg > dmesg.log

tests/test_snapshot.py

@@ -8,6 +8,7 @@
 
 import errno
 import os
+import platform
 import unittest
 
 import elastio_snap
@@ -41,13 +42,16 @@ def test_modify_origin(self):
             f.write("jumps over the lazy dog")
 
         os.sync()
-
-        util.mount(self.snap_device, self.snap_mount, opts="ro")
+        # TODO: norecovery option, probably, should not be here after the fix of the elastio/elastio-snap#63
+        opts = "nouuid,norecovery,ro" if (self.fs == "xfs") else "ro"
+        util.mount(self.snap_device, self.snap_mount, opts)
         self.addCleanup(util.unmount, self.snap_mount)
 
         md5_snap = util.md5sum(snapfile)
         self.assertEqual(md5_orig, md5_snap)
 
+    @unittest.skipIf(os.getenv('TEST_FS') == "ext2" and int(platform.release().split(".", 1)[0]) < 4, "Broken on ext2, 3-rd kernels")
+    @unittest.skipIf(os.getenv('TEST_FS') == "xfs", "Broken on XFS, due to ignored os.sync and due to #63.")
     def test_track_writes(self):
         testfile = "{}/testfile".format(self.mount)
 

tests/util.py

@@ -28,7 +28,7 @@ def dd(ifile, ofile, count, **kwargs):
     for k, v in kwargs.items():
         cmd.append("{}={}".format(k, v))
 
-    subprocess.check_call(cmd, timeout=20)
+    subprocess.check_call(cmd, timeout=30)
 
 
 def md5sum(path):
@@ -55,6 +55,10 @@ def loop_destroy(loop):
     subprocess.check_call(cmd, timeout=10)
 
 
-def mkfs(device):
-    cmd = ["mkfs.ext4", "-F", device]
-    subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
+def mkfs(device, fs="ext4"):
+    if (fs == "xfs"):
+        cmd = ["mkfs.xfs", device, "-f"]
+    else:
+        cmd = ["mkfs." + fs, "-F", device]
+
+    subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=40)