Merge pull request tcp-acceleration-service#3 from stolet/merge_vtas_…

…dpdk21 Merge vtas dpdk21
liam0215 · Apr 19, 2023 · bda0e79 · bda0e79
2 parents d496497 + 16eb312
commit bda0e79
Show file tree

Hide file tree

Showing 107 changed files with 3,398 additions and 1,315 deletions.
diff --git a/Makefile b/Makefile
@@ -5,14 +5,16 @@
 
 CPPFLAGS += -Iinclude/
 CPPFLAGS += $(EXTRA_CPPFLAGS)
-CFLAGS += -std=gnu99 -O3 -g -Wall -Werror -march=native -fno-omit-frame-pointer -Wno-address-of-packed-member
+CFLAGS += -std=gnu99 -O3 -g -Wall -march=native
+CFLAGS += -Wno-address-of-packed-member
 CFLAGS += $(EXTRA_CFLAGS)
 CFLAGS_SHARED += $(CFLAGS) -fPIC
 LDFLAGS += -pthread -g
 LDFLAGS += $(EXTRA_LDFLAGS)
 LDLIBS += -lm -lpthread -lrt -ldl
 LDLIBS += $(EXTRA_LDLIBS)
 
+PKG_CONFIG ?= pkg-config
 PREFIX ?= /usr/local
 SBINDIR ?= $(PREFIX)/sbin
 LIBDIR ?= $(PREFIX)/lib
@@ -22,38 +24,8 @@ INCDIR ?= $(PREFIX)/include
 ##############################################################################
 # DPDK configuration
 
-# Prefix for dpdk
-RTE_SDK ?= /usr/
-# mpdts to compile
-DPDK_PMDS ?= ixgbe i40e tap virtio
-
-DPDK_CPPFLAGS += -I$(RTE_SDK)/include -I$(RTE_SDK)/include/dpdk \
-  -I$(RTE_SDK)/include/x86_64-linux-gnu/dpdk/
-DPDK_LDFLAGS+= -L$(RTE_SDK)/lib/
-DPDK_LDLIBS+= \
-  -Wl,--whole-archive \
-   $(addprefix -lrte_pmd_,$(DPDK_PMDS)) \
-  -lrte_eal \
-  -lrte_mempool \
-  -lrte_mempool_ring \
-  -lrte_hash \
-  -lrte_ring \
-  -lrte_kvargs \
-  -lrte_ethdev \
-  -lrte_mbuf \
-  -lnuma \
-  -lrte_bus_pci \
-  -lrte_pci \
-  -lrte_cmdline \
-  -lrte_timer \
-  -lrte_net \
-  -lrte_kni \
-  -lrte_bus_vdev \
-  -lrte_gso \
-  -Wl,--no-whole-archive \
-  -ldl \
-  $(EXTRA_LIBS_DPDK)
-
+DPDK_CPPFLAGS ?= $(shell $(PKG_CONFIG) --cflags libdpdk)
+DPDK_LDLIBS ?= $(shell $(PKG_CONFIG) --static --libs libdpdk)
 
 ##############################################################################
 

diff --git a/README.md b/README.md
@@ -1,6 +1,5 @@
 # Virtualized TCP Acceleration Service
 
-[![Build Status](https://travis-ci.org/tcp-acceleration-service/tas.svg?branch=master)](https://travis-ci.org/tcp-acceleration-service/tas)
 [![Documentation Status](https://readthedocs.org/projects/tas/badge/?version=latest)](https://tas.readthedocs.io/en/latest/?badge=latest)
 
 
@@ -10,16 +9,36 @@ virtualized environments.
 ## Building
 Requirements:
   * vTAS is built on top of Intel DPDK for direct access to the NIC. We have
-    tested this version with dpdk versions (17.11.9, 18.11.5, 19.11).
+    tested this version with dpdk version 21.
 
-Assuming that dpdk is installed in `~/dpdk-inst` vTAS can be built as follows:
+Assuming that dpdk is installed through the system package manager, the
+following suffices to build TAS:
 ```
-make RTE_SDK=~/dpdk-inst
+make
 ```
 
 This will build the vTAS service (binary `tas/tas`), client libraries (in
 `lib/`), and a few debugging tools (in `tools/`).
 
+
+### Using Separate DPDK Build
+
+The Makefile uses `pkg-config` to locate the dpdk libraries and headers. If you
+want to build TAS against a non-system dpdk version, the
+`PKG_CONFIG_PATH` environmental variable can point pkg-config to the right dpdk
+location. For example, with dpdk installed in `/local/antoinek/dpdk` I set
+
+```
+PKG_CONFIG_PATH=/local/antoinek/dpdk/usr/local/lib/x86_64-linux-gnu/pkgconfig/
+```
+
+Instead of manually exporting the environment variable every time, it can also
+be set by creating a `Makefile.local` file including the following line:
+
+```
+PKG_CONFIG= PKG_CONFIG_PATH=/local/antoinek/dpdk/usr/local/lib/x86_64-linux-gnu/pkgconfig/ pkg-config
+```
+
 ## Running
 
 Before running vTAS the following steps are necessary:

diff --git a/experiments/components/client.py b/experiments/components/client.py
@@ -17,26 +17,32 @@ def __init__(self, defaults, machine_config, client_config, vm_config, wmanager)
                 machine_config.is_remote)
 
     def run_bare(self, w_sudo, ld_preload):
-        self.run_benchmark_rpc(w_sudo, ld_preload)
+        self.run_benchmark_rpc(w_sudo, ld_preload, clean=False)
 
     def run_virt(self, w_sudo, ld_preload):
         ssh_com = utils.get_ssh_command(self.machine_config, self.vm_config)
         self.pane.send_keys(ssh_com)
         time.sleep(3)
         self.pane.send_keys("tas")
-        self.run_benchmark_rpc(w_sudo, ld_preload)
+        time.sleep(3)
+        self.run_benchmark_rpc(w_sudo, ld_preload, clean=False)
 
-    def run_benchmark_rpc(self, w_sudo, ld_preload):
+    def run_benchmark_rpc(self, w_sudo, ld_preload, clean):
         self.pane.send_keys('cd ' + self.client_config.comp_dir)
+
+        if clean:
+            self.pane.send_keys(self.client_config.clean_cmd)
+            time.sleep(1)
+
         self.pane.send_keys(self.client_config.comp_cmd)
+        time.sleep(1)
         self.pane.send_keys("cd " + self.client_config.tas_dir)
         time.sleep(3)
 
-        cmd = ''
-        stack = self.machine_config.stack
+        cmd = 'stdbuf -oL '
 
         if w_sudo:
-            cmd = 'sudo '
+            cmd += 'sudo -E '
 
         if ld_preload:
             cmd += 'LD_PRELOAD=' + self.client_config.lib_so + ' '
@@ -46,7 +52,6 @@ def run_benchmark_rpc(self, w_sudo, ld_preload):
                 ' | tee ' + \
                 self.client_config.out
 
-        print(cmd) 
         self.pane.send_keys(cmd)
 
     def save_log_virt(self, exp_path):
@@ -65,7 +70,18 @@ def save_log_virt(self, exp_path):
         self.save_logs_pane.send_keys(suppress_history=False, cmd='tas')
         time.sleep(1)
 
+        # Remove log from remote machine
+        ssh_com = utils.get_ssh_command(self.machine_config, self.vm_config)
+        ssh_com += " 'rm {}'".format(self.client_config.out)
+        self.save_logs_pane.send_keys(ssh_com)
+        time.sleep(3)
+        self.save_logs_pane.send_keys(suppress_history=False, cmd='tas')
+        time.sleep(1)
+
     def save_log_bare(self, exp_path):
+        # kill process to force flush to file
+        # self.save_logs_pane.send_keys("sudo pkill testclient")
+
         # self.exp_path is set in the run.py file
         split_path = exp_path.split("/")
         n = len(split_path)

diff --git a/experiments/components/proxy.py b/experiments/components/proxy.py
@@ -22,9 +22,11 @@ def run(self):
                 pane=self.pane,
                 comp_dir=self.proxy_config.comp_dir,
                 comp_cmd=self.proxy_config.comp_cmd,
+                clean_cmd=self.proxy_config.clean_cmd,
                 exec_file=self.proxy_config.exec_file,
                 out=self.proxy_config.out,
-                args='')
+                args='',
+                clean=False)
 
 class ProxyGuest(Proxy):
 
@@ -44,7 +46,9 @@ def run(self):
                 pane=self.pane,
                 comp_dir=self.proxy_config.comp_dir,
                 comp_cmd=self.proxy_config.comp_cmd,
+                clean_cmd=self.proxy_config.clean_cmd,
                 exec_file=self.proxy_config.exec_file,
                 out=self.proxy_config.out,
                 args='',
-                bg=False)
+                bg=False,
+                clean=False)
diff --git a/experiments/components/server.py b/experiments/components/server.py
@@ -15,26 +15,32 @@ def __init__(self, defaults, machine_config,
             machine_config.is_remote)
 
     def run_bare(self, w_sudo, ld_preload):
-        self.run_benchmark_rpc(w_sudo, ld_preload)
+        self.run_benchmark_rpc(w_sudo, ld_preload, clean=False)
 
     def run_virt(self, w_sudo, ld_preload):
         ssh_com = utils.get_ssh_command(self.machine_config, self.vm_config)
         self.pane.send_keys(ssh_com)
         time.sleep(3)
         self.pane.send_keys("tas")
-        self.run_benchmark_rpc(w_sudo, ld_preload)
+        self.run_benchmark_rpc(w_sudo, ld_preload, clean=False)
 
-    def run_benchmark_rpc(self, w_sudo, ld_preload):
+    def run_benchmark_rpc(self, w_sudo, ld_preload, clean):
         self.pane.send_keys('cd ' + self.server_config.comp_dir)
+
+        if clean:
+            self.pane.send_keys(self.server_config.clean_cmd)
+            time.sleep(1)
+
         self.pane.send_keys(self.server_config.comp_cmd)
+        time.sleep(1)
         self.pane.send_keys("cd " + self.server_config.tas_dir)
         time.sleep(3)
 
         cmd = ''
         stack = self.machine_config.stack
 
         if w_sudo:
-            cmd = 'sudo '
+            cmd = 'sudo -E '
 
         if ld_preload:
             cmd += 'LD_PRELOAD=' + self.server_config.lib_so + ' '
@@ -45,5 +51,4 @@ def run_benchmark_rpc(self, w_sudo, ld_preload):
                 # ' | tee ' + \
                 # self.server_config.out
 
-        print(cmd) 
         self.pane.send_keys(cmd)
diff --git a/experiments/components/tas.py b/experiments/components/tas.py
@@ -19,27 +19,35 @@ def run_bare(self):
         utils.compile_and_run(pane=self.pane,
                 comp_dir=self.tas_config.comp_dir,
                 comp_cmd=self.tas_config.comp_cmd,
+                clean_cmd=self.tas_config.clean_cmd,
                 exec_file=self.tas_config.exec_file,
                 out=self.tas_config.out,
                 args=tas_args,
-                save_log=True)
+                save_log=True,
+                clean=False)
 
     def run_virt(self):
         ssh_com = utils.get_ssh_command(self.machine_config, self.vm_config)
         self.pane.send_keys(ssh_com)
         time.sleep(3)
         self.pane.send_keys("tas")
+        time.sleep(3)
 
         tas_args = self.tas_config.args
         utils.compile_and_run(pane=self.pane,
                 comp_dir=self.tas_config.comp_dir,
                 comp_cmd=self.tas_config.comp_cmd,
+                clean_cmd=self.tas_config.clean_cmd,
                 exec_file=self.tas_config.exec_file,
                 out=self.tas_config.out,
                 args=tas_args,
-                save_log=True)
+                save_log=True,
+                clean=False)
 
     def save_log_bare(self, exp_path):
+        # # kill process to force flush to file
+        # self.pane.send_keys("sudo pkill tas")
+
         split_path = exp_path.split("/")
         n = len(split_path)
 

diff --git a/experiments/components/vm.py b/experiments/components/vm.py
@@ -12,25 +12,30 @@ def __init__(self, defaults, machine_config, vm_config, wmanager):
 
     def start(self):
         self.pane.send_keys('cd ' + self.vm_config.manager_dir)
-        start_vm_cmd = "sudo bash start-vm.sh {} {}".format(
-                self.machine_config.stack, self.vm_config.id)
+        start_vm_cmd = "sudo bash start-vm.sh {} {} {}".format(
+                self.machine_config.stack, self.vm_config.id,
+                self.machine_config.interface)
         self.pane.send_keys(start_vm_cmd)
 
         print("Started VM")
-        time.sleep(35)
+        time.sleep(25)
         self.login_vm()
 
     def enable_hugepages(self):
         cmd = "sudo mount -t hugetlbfs nodev /dev/hugepages"
         self.pane.send_keys(cmd)
-        cmd = "echo 1024 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages"
+        time.sleep(1)
+        cmd = "echo 8192 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages"
         self.pane.send_keys(cmd)
-        time.sleep(3)
+        time.sleep(5)
 
     def enable_noiommu(self, vendor_id):
         self.pane.send_keys("sudo su -")
+        time.sleep(1)
         self.pane.send_keys("sudo echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode")
+        time.sleep(1)
         self.pane.send_keys("sudo echo {} > /sys/bus/pci/drivers/vfio-pci/new_id".format(vendor_id))
+        time.sleep(1)
         self.pane.send_keys("exit")
         time.sleep(1)
 
@@ -45,7 +50,7 @@ def init_interface(self, ip, interface):
     def dpdk_bind(self, ip, interface, pci_id):
         cmd = 'cd ' + self.vm_config.manager_dir_virt
         self.pane.send_keys(cmd)
-        cmd = 'bash dpdk_bind.sh {} {} {}'.format(ip, interface, pci_id)
+        cmd = 'bash dpdk-bind.sh {} {} {}'.format(ip, interface, pci_id)
         self.pane.send_keys(cmd)
         time.sleep(3)
 
@@ -54,4 +59,16 @@ def login_vm(self):
         time.sleep(3)
         self.pane.send_keys(suppress_history=False, cmd='tas')
         self.pane.enter()
-        time.sleep(5)
+        time.sleep(5)
+
+    def shutdown(self):
+        self.pane.send_keys(suppress_history=False, cmd='whoami')
+
+        captured_pane = self.pane.capture_pane()
+        user = captured_pane[len(captured_pane) - 2]
+
+        # This means we are in the vm, so we don't 
+        # accidentally shutdown machine
+        if user == 'tas':
+            self.pane.send_keys(suppress_history=False, cmd='sudo shutdown -h now')
+            time.sleep(2)