Skip to content

Commit

Permalink
update to BF SDE 9.7.2. added CI tests for local execution
Browse files Browse the repository at this point in the history
  • Loading branch information
ralfkundel committed Jul 23, 2022
1 parent 38f83e2 commit 63d0e20
Show file tree
Hide file tree
Showing 52 changed files with 8,986 additions and 211 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
GitHub.zip
bash_vars/*
log/*
builds/*
GitHub/*
data/*
Expand All @@ -15,6 +18,7 @@ management_ui/static/includes/bootstrap.min.css_v*
targets/bmv2/data/commands1_middlebox1.txt
test.*
pastaenv/*
pastaenv_core_docker/*
targets/BF2556X-1T.zip
targets/BF2556X-1T/
build/
Expand Down
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,10 @@ We highly recommonend to use the "status check" on the bottom of the configurati


# Publications
* "How to measure the speed of light with programmable data plane hardware?"@ 2nd P4Europe Workshop: [Demo Paper](https://ieeexplore.ieee.org/abstract/document/8901871)
* "P4STA: High Performance Packet Timestamping with Programmable Packet Processors"@ IEEE/IFIP NOMS: [Paper](https://ieeexplore.ieee.org/abstract/document/9110290)
* "Microbursts in Software and Hardware-based Traffic Load Generation" @IEEE/IFIP NOMS: [Demo Paper](https://ieeexplore.ieee.org/abstract/document/9110305)

* 2022: "Network Testing Utilizing Programmable Network Hardware?"@ IEEE Communications Magazine: [Article](https://ieeexplore.ieee.org/abstract/document/9722801)
* 2020: "P4STA: High Performance Packet Timestamping with Programmable Packet Processors"@ IEEE/IFIP NOMS: [Paper](https://ieeexplore.ieee.org/abstract/document/9110290)
* 2020: "Microbursts in Software and Hardware-based Traffic Load Generation" @IEEE/IFIP NOMS: [Demo Paper](https://ieeexplore.ieee.org/abstract/document/9110305)
* 2019: "How to measure the speed of light with programmable data plane hardware?"@ 2nd P4Europe Workshop: [Demo Paper](https://ieeexplore.ieee.org/abstract/document/8901871)

# Architecture

Expand Down
8 changes: 2 additions & 6 deletions cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,13 @@
import time
import traceback

from core import P4STA_utils

dir_path = os.path.dirname(os.path.realpath(__file__))
project_path = dir_path[0:dir_path.find("/cli")]
sys.path.append(project_path)
try:
from analytics import analytics
except Exception as e:
# prevent PEP3 warning because sys append must be before
raise e

from core import P4STA_utils
from analytics import analytics

def main():
global selected_run_id
Expand Down
2 changes: 1 addition & 1 deletion extHost/dpdkExtHost/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
src/build/*
/script/install_dpdk_sudo.sh
scripts/install_dpdk_sudo.sh
18 changes: 7 additions & 11 deletions extHost/pythonExtHost/pythonHostDriver.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,17 @@ def start_external(self, file_id, multi=1, tsmax=(2 ** 32 - 1)):
answer = P4STA_utils.execute_ssh(
self.cfg["ext_host_user"],
self.cfg["ext_host_ssh"],
"python3 -c 'import pkgutil; print(1 if pkgutil.find_loader"
"(\"setproctitle\") else 0)'")
"python3 -c 'import pkgutil; print(1 if pkgutil.find_loader(\"setproctitle\") else 0)'")
if answer[0] == "0":
errors = errors + (
"Python Module 'setproctitle' not found at external host -> "
"'pip3 install setproctitle'",)
"Python Module 'setproctitle' not found at external host -> 'pip3 install setproctitle'",)
return errors

answer = P4STA_utils.execute_ssh(
self.cfg["ext_host_user"],
self.cfg["ext_host_ssh"],
"mkdir -p /home/" + self.cfg["ext_host_user"] +
"/p4sta/externalHost/python; "
"sudo killall external_host_python_receiver")
"/p4sta/externalHost/python; sudo killall external_host_python_receiver")

input = ["scp", ext_py_dir + "/pythonRawSocketExtHost.py",
self.cfg["ext_host_user"] + "@" + self.cfg[
Expand All @@ -73,8 +70,8 @@ def start_external(self, file_id, multi=1, tsmax=(2 ** 32 - 1)):
res = P4STA_utils.execute_ssh(self.cfg["ext_host_user"],
self.cfg["ext_host_ssh"], args)

print("now start python extHost")
call = "sudo ./pythonRawSocketExtHost.py --name " + file_id + \
print("start python extHost")
call = "sudo -E ./pythonRawSocketExtHost.py --name " + file_id + \
" --interface " + self.cfg["ext_host_if"] + " --multi " + str(
multi) + " --tsmax " + str(tsmax)
args = "cd /home/" + self.cfg["ext_host_user"] + \
Expand All @@ -89,8 +86,7 @@ def start_external(self, file_id, multi=1, tsmax=(2 ** 32 - 1)):
input = ["ssh",
self.cfg["ext_host_user"] + "@" + self.cfg["ext_host_ssh"],
"cd /home/" + self.cfg["ext_host_user"] +
"/p4sta/externalHost/python; cat pythonRawSocketExtHost.log; "
"exit"]
"/p4sta/externalHost/python; cat pythonRawSocketExtHost.log; exit"]
res = subprocess.run(input, stdout=subprocess.PIPE, timeout=3).stdout
result = res.decode("utf-8")

Expand Down Expand Up @@ -160,7 +156,7 @@ def get_server_install_script(self, user_name, ip):
"echo 'visudo entry " \
"already exists';\n else\n sleep 0.1\n " \
"echo $current_user' ALL=(ALL:ALL) " \
"NOPASSWD:'$1 | sudo EDITOR='tee -a' visudo;\n fi\n}\n"
"NOPASSWD:SETENV:'$1 | sudo EDITOR='tee -a' visudo;\n fi\n}\n"

with open(dir_path + "/scripts/install_python_sudo.sh", "w") as f:
f.write(add_sudo_rights_str)
Expand Down
2 changes: 1 addition & 1 deletion gui.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
source pastaenv/bin/activate
python3 manage.py runserver 0.0.0.0:9997
python3 manage.py runserver 0.0.0.0:9998
deactivate
18 changes: 11 additions & 7 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
###########################################

create_env () {
# create virtual environment and activate
python3_dir=$(which python3)
virtualenv -p $python3_dir pastaenv
source pastaenv/bin/activate
# create virtual environment and activate
python3_dir=$(which python3)
virtualenv -p $python3_dir $P4STA_VIRTUALENV
source $P4STA_VIRTUALENV/bin/activate
}

ask_and_create_key () {
Expand Down Expand Up @@ -53,7 +53,6 @@ add_sudo_rights() {
fi
}


pub_key=$(cat ~/.ssh/id_rsa.pub)
pub_key_length=`expr "$pub_key" : '.*'`

Expand All @@ -65,20 +64,25 @@ fi
printf "Setting executeable bits for management server scripts...\n"
chmod +x gui.sh cli.sh run.sh

if [ -z "$P4STA_VIRTUALENV" ]
then
P4STA_VIRTUALENV="pastaenv"
fi

if [ -z "$BATCH_MODE" ]
then
while true; do
sleep 1
read -p "Do you wish to install the dependencies on this machine? Y/N: " yn
case $yn in
[Yy]* ) add_sudo_rights $(which pkill); sudo apt update; sudo apt -y install python3-pip virtualenv net-tools shellinabox; rm -rf p4staenv/; create_env; pip3 install -r requirements.txt ; deactivate; echo "finished pip3 on webserver"; break;;
[Yy]* ) add_sudo_rights $(which pkill); sudo apt update; sudo apt -y install python3-pip virtualenv net-tools shellinabox; sudo rm -rf $P4STA_VIRTUALENV/; create_env; pip3 install --upgrade pip; python3 -m pip install -r requirements.txt ; deactivate; echo "finished pip3 on p4sta core"; break;;
[Nn]* ) break;;
* ) echo "Please answer yes or no.";;
esac
done
else
add_sudo_rights $(which pkill);
sudo apt update; sudo apt -y install python3-pip virtualenv net-tools; rm -rf pastaenv/; create_env; pip3 install -r requirements.txt ; deactivate; echo "finished pip3 on webserver";
sudo apt update; sudo apt -y install python3-pip virtualenv net-tools; sudo rm -rf $P4STA_VIRTUALENV/; create_env; pip3 install -r requirements.txt ; deactivate; echo "finished pip3 on p4sta core";
fi

mkdir -p results
Expand Down
5 changes: 3 additions & 2 deletions management_ui/views_dir/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ def updateCfg(request):
request.POST["num_grp_" + str(loadgen_grp["group"])])
servers = []
i = 1
_grp = str(loadgen_grp["group"])
for j in range(1, num_servers + 1):
s = {"id": j}
_grp = str(loadgen_grp["group"])
_key = "s" + _grp + "_" + str(i) + "_real_port"
while _key not in request.POST:
i += 1
Expand Down Expand Up @@ -158,8 +158,9 @@ def updateCfg(request):
"config parameters:" + str(e))
servers.append(s)
i += 1

print(str(request.POST["add_to_grp_" + _grp]))
if str(request.POST["add_to_grp_" + _grp]) == "1":
print("addtogrp i was here")
s = {}
s["id"] = num_servers + 1
s["real_port"] = ""
Expand Down
30 changes: 15 additions & 15 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
cycler==0.10.0
Django==2.2.24
grpcio==1.35.0
grpcio-tools==1.35.0
cycler==0.11.0
Django==2.2.28
grpcio==1.47.0
grpcio-tools==1.47.0
kiwisolver==1.3.1
matplotlib==3.3.4
numpy==1.18.0
plumbum==1.6.8
protobuf==3.17.2
numpy==1.19.5
plumbum==1.7.2
protobuf==3.19.4
pyparsing==2.4.7
python-dateutil==2.8.1
pytz==2019.3
python-dateutil==2.8.2
pytz==2022.1
rpyc==5.0.1
six==1.13.0
sqlparse==0.3.0
tabulate==0.8.6
thrift==0.13.0
google-api-core==1.25.1
google-api-python-client==1.12.8
six==1.16.0
tabulate==0.8.10
thrift==0.16.0
google-api-core==1.32.0
google-api-python-client==1.12.11
pycodestyle==2.8.0
8 changes: 7 additions & 1 deletion run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@ echo "##########################################################################
echo "Welcome to P4STA! Please select the mode you want to start the environment."
echo "To stop P4STA you just need to enter ctrl + c"
echo "###########################################################################"
source pastaenv/bin/activate

if [ -z "$P4STA_VIRTUALENV" ]
then
P4STA_VIRTUALENV="pastaenv"
echo "[LOG][run.sh] P4STA_VIRTUALENV not specified. taking default path 'pastaenv'"
fi
source $P4STA_VIRTUALENV/bin/activate
select yn in "P4STA Core + HTML-GUI (recommended)" "P4STA Core(VERBOSE) + HTML-GUI" "P4STA Core + CLI" "Just P4STA Core (VERBOSE)"; do
case $yn in
"P4STA Core + HTML-GUI (recommended)" ) (python3 core/core.py>/dev/null&); sleep 2; python3 manage.py runserver 0.0.0.0:9997; break;;
Expand Down
120 changes: 120 additions & 0 deletions run_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
AVAILABLE_TESTS="install|test_code_quality|test_ptf_bmv2|test_ptf_tofino|test_core|test_ui|test_ui_tofino|dpdk_install|test_core_dpdk|cleanup|all"
if [[ ! "$1" =~ $AVAILABLE_TESTS ]]
then
echo "Usage: $0 [ ${AVAILABLE_TESTS//|/ | } ]"
exit 1
fi

source tests/ci_tests.sh



if [[ "$1" == "install" ]]
then
set -e
job_prepare
job_install
printf "${GREEN} [SUCCESS] Succeeded installing p4sta ci test environment \n${NC}"
exit 0
fi

if [[ "$1" == "test_code_quality" ]]
then
set -e
job_code_quality
printf "${GREEN} [SUCCESS] Succeeded test_code_quality \n${NC}"
exit 0
fi


if [[ "$1" == "test_ptf_bmv2" ]]
then
set -e
job_ptf_bmv2
printf "${GREEN} [SUCCESS] Succeeded test_ptf_bmv2 \n${NC}"
exit 0
fi

if [[ "$1" == "test_ptf_tofino" ]]
then
set -e
job_ptf_tofino
printf "${GREEN} [SUCCESS] Succeeded test_ptf_tofino \n${NC}"
exit 0
fi

if [[ "$1" == "test_core" ]]
then
set -e
job_test_core
printf "${GREEN} [SUCCESS] Succeeded test_core \n${NC}"
exit 0
fi

if [[ "$1" == "test_ui" ]]
then
set -e
job_test_django_bmv2
printf "${GREEN} [SUCCESS] Succeeded test_ui \n${NC}"
exit 0
fi

if [[ "$1" == "test_ui_tofino" ]]
then
set -e
job_test_django_tofino
printf "${GREEN} [SUCCESS] Succeeded test_ui_tofino \n${NC}"
exit 0
fi

if [[ "$1" == "dpdk_install" ]]
then
set -e
job_dpdk_install
printf "${GREEN} [SUCCESS] Succeeded dpdk_install \n${NC}"
exit 0
fi

if [[ "$1" == "test_core_dpdk" ]]
then
set -e
job_test_core_dpdk
printf "${GREEN} [SUCCESS] Succeeded test_core_dpdk \n${NC}"
exit 0
fi

if [[ "$1" == "cleanup" ]]
then
set -e
job_cleanup
printf "${GREEN} [SUCCESS] Succeeded job_cleanup \n${NC}"
exit 0
fi

if [[ "$1" == "all" ]]
then
set -e
job_prepare
job_install
printf "${GREEN} [SUCCESS] Succeeded installing p4sta ci test environment \n${NC}"
job_code_quality
printf "${GREEN} [SUCCESS] Succeeded test_code_quality \n${NC}"
job_ptf_bmv2
printf "${GREEN} [SUCCESS] Succeeded test_ptf_bmv2 \n${NC}"
job_ptf_tofino
printf "${GREEN} [SUCCESS] Succeeded test_ptf_tofino \n${NC}"
job_test_core
printf "${GREEN} [SUCCESS] Succeeded test_core \n${NC}"
job_test_django_bmv2
printf "${GREEN} [SUCCESS] Succeeded test_ui \n${NC}"
job_test_django_tofino
printf "${GREEN} [SUCCESS] Succeeded test_ui_tofino \n${NC}"
job_dpdk_install
printf "${GREEN} [SUCCESS] Succeeded dpdk_install \n${NC}"
job_test_core_dpdk
printf "${GREEN} [SUCCESS] Succeeded test_core_dpdk \n${NC}"
job_cleanup
printf "${GREEN} [SUCCESS] Succeeded in all Test Jobs \n${NC}"
exit 0
fi

Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
],
"multicast": "1",
"program": "tofino_stamper_v1_0_1",
"sde": "/opt/bf-sde-9.3.0",
"sde": "/opt/bf-sde-9.7.2",
"selected_loadgen": "iperf3",
"selected_target": "Stordis_BF6064XT",
"selected_extHost": "PythonExtHost",
Expand Down
3 changes: 3 additions & 0 deletions stamper_targets/Ufispace_S9180-32X/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
data/mau.html
data/bf_input.txt

Loading

0 comments on commit 63d0e20

Please sign in to comment.