Merge pull request #8 from mrragava/oe-fuzzing

oe fuzzer target programs and onefuzz workflow

.github/workflows/github-issues.json

@@ -0,0 +1,23 @@
+{
+   "config": {
+      "auth": {
+         "user": "OE_SEC_ADMIN",
+         "personal_access_token": "OE_SEC_PAT"
+      },
+      "organization": "openenclave",
+      "repository": "openenclave-security",
+      "title": "{{ report.executable }} - {{report.crash_site}}",
+      "body": "## Files\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})\n\n## Repro\n\n `{{ repro_cmd }}`\n\n## Call Stack\n\n```{% for item in report.call_stack %}{{ item }}\n{% endfor %}```\n\n## ASAN Log\n\n```{{ report.asan_log }}```",
+      "unique_search": {
+         "field_match": ["title"],
+         "string": "{{ report.executable }}"
+      },
+      "assignees": [],
+      "labels": ["bug", "{{ report.crash_type }}"],
+      "on_duplicate": {
+         "comment": "Duplicate found.\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})",
+         "labels": ["{{ report.crash_type }}"],
+         "reopen": true
+      }
+   }
+}

.github/workflows/onefuzz-workflow.yml

@@ -0,0 +1,48 @@
+# Copyright (c) Open Enclave SDK contributors.
+# Licensed under the MIT License.
+
+name: Onefuzz Pipeline
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    container:
+      image: oeciteam/oetools-full-18.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7 
+      - name: Install pip
+        run: |
+          sudo -H python3 -m pip install --upgrade pip
+      - name: Install CMake
+        run: |
+          sudo -H python3 -m pip install cmake
+      - name: build
+        run: |
+          ./build.sh -c -i -d
+      - name: submit onefuzz job
+        env:
+          ONEFUZZ_ENDPOINT: ${{ secrets.onefuzz_endpoint }}
+          ONEFUZZ_CLIENT_ID: ${{ secrets.onefuzz_client_id }}
+          ONEFUZZ_CLIENT_SECRET: ${{ secrets.onefuzz_client_secret }}
+          ONEFUZZ_PAT: ${{ secrets.onefuzz_pat }}
+        run: |
+          set -ex
+          sudo -H python3 -m pip install onefuzz==2.23.0
+          sed -i s/OE_SEC_ADMIN/${OE_SEC_ADMIN}/ .github/workflows/github-issues.json
+          sed -i s/OE_SEC_PAT/${ONEFUZZ_PAT}/ .github/workflows/github-issues.json
+          onefuzz config --endpoint $ONEFUZZ_ENDPOINT --client_id $ONEFUZZ_CLIENT_ID --client_secret $ONEFUZZ_CLIENT_SECRET
+          ./src/dynamic/scripts/onefuzz/run.sh

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "src/static/codeql/codeql"]
 	path = src/static/codeql/codeql
 	url = https://github.com/github/codeql.git
-	branch = rc/1.26
+	branch = main
 	ignore = all
 [submodule "sut/openenclave"]
 	path = sut/openenclave
@@ -14,3 +14,7 @@
 	branch = master
 	ignore = all
 
+[submodule "3rdparty/linux-sgx"]
+	path = 3rdparty/linux-sgx
+	url = https://github.com/intel/linux-sgx.git
+	ignore = all

CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.0.0)
+project(oefuzz VERSION 1.0.0)
+set(CMAKE_C_COMPILER_ID "Clang")
+set(CMAKE_CXX_COMPILER_ID "Clang")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+include(fuzzer_settings)
+
+set(OPEN_ENCLAVE_SDK_SRC ${PROJECT_SOURCE_DIR}/sut/openenclave)
+add_subdirectory(src/dynamic/fuzzing)

README.md

@@ -2,10 +2,10 @@
 This repository will host security automation projects for Open Enclave. These scripts and toolset are used in continous intergration, developer testing and security analysis of Open Enclave. 
 
 1. Static Analysis
-    - [OpenEnclave-CodeQL](src/static/codeql/README.md)
+    - [OpenEnclave-CodeQL](src/static/codeql)
 
 2. Dynamic Analysis 
-    - OpenEnclave-Fuzzing - TBD
+    - [OpenEnclave-Fuzzing](src/dynamic/fuzzing)
 
 # Prerequisites
 ## Build dependencies

build.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+# Copyright (c) Open Enclave SDK contributors.
+# Licensed under the MIT License.
+
+set -e
+ulimit -n 4096
+
+export OE_SEC_ROOT=${PWD}
+export OE_SEC_BUILD=${OE_SEC_ROOT}/build
+export OE_SEC_TOOLS=${OE_SEC_ROOT}/build/tools
+export OE_SRC_ROOT=${OE_SEC_ROOT}/sut/openenclave
+export OE_3RDPARTY=${OE_SEC_ROOT}/3rdparty
+
+export OE_UNINSTRUMENTED_BUILD=${OE_SEC_BUILD}/oe_uninstrumented_build
+export OE_UNINSTRUMENTED_INSTALL_PREFIX=${OE_SEC_BUILD}/oe_uninstrumented_install_prefix
+export OE_SEC_CORPUS_BUILD=${OE_SEC_BUILD}/corpus_build
+export OE_SEC_ENCLAVE_CORPUS=${OE_SEC_ROOT}/src/dynamic/fuzzing/create/enclave_corpus
+
+export OE_INSTRUMENTED_BUILD=${OE_SEC_BUILD}/oe_instrumented_build
+export OE_INSTRUMENTED_INSTALL_PREFIX=${OE_SEC_BUILD}/oe_instrumented_install_prefix
+export OE_SEC_FUZZING_BUILD=${OE_SEC_BUILD}/fuzzing_build
+
+export OE_LLVM_URL="https://github.com/openenclave/openenclave-security/releases/download/v1.0/oe-llvm-1.0.zip"
+export CLANG=${OE_SEC_TOOLS}/oe-llvm-1.0/bin/clang
+export CLANG_CPP=${OE_SEC_TOOLS}/oe-llvm-1.0/bin/clang++
+
+export INTEL_SGX_SDK="https://download.01.org/intel-sgx/sgx-linux/2.13/distro/ubuntu18.04-server/sgx_linux_x64_sdk_2.13.100.4.bin"
+export INTEL_SGX_SDK_PACKAGE="sgx_linux_x64_sdk_2.13.100.4.bin"
+
+CLEAN=0
+INSTALL_DEPENDS=0
+BUILD_INTEL_SGX_PSW=0
+for i in "$@"; do
+    case $i in
+    -c | --clean)
+        CLEAN=1
+        ;;
+    esac
+    case $i in
+    -d | --depends)
+        INSTALL_DEPENDS=1
+        ;;
+    esac
+    case $i in
+    -i | --intelsdk)
+        BUILD_INTEL_SGX_PSW=1
+        ;;
+    esac
+done
+
+# Intel PSW build dependencies
+if [[ $INSTALL_DEPENDS -eq 1 ]]; then
+    sudo apt-get update
+    sudo apt-get install -y build-essential ocaml ocamlbuild automake autoconf libtool wget \
+        libssl-dev perl libssl-dev libcurl4-openssl-dev protobuf-compiler libprotobuf-dev debhelper reprepro unzip
+fi
+
+[[ ${CLEAN} -eq 1 ]] && rm -rf "${OE_SEC_BUILD}"
+[[ ! -d "${OE_SEC_BUILD}" ]] && mkdir -p "${OE_SEC_BUILD}"
+
+if [[ ! -d "${OE_SEC_TOOLS}" ]]; then
+    mkdir -p "${OE_SEC_TOOLS}"
+    pushd "${OE_SEC_TOOLS}"
+    wget "${OE_LLVM_URL}"
+    unzip oe-llvm-1.0.zip
+    popd
+fi
+
+MAKE_THREADS=$(nproc)
+
+# Building debug version of Intel PSW.
+if [[ ${BUILD_INTEL_SGX_PSW} -eq 1 ]]; then
+    if [[ ! -d "/opt/intel/sgxsdk" ]]; then
+        pushd "${OE_SEC_TOOLS}"
+        wget "${INTEL_SGX_SDK}"
+        chmod +x ./"${INTEL_SGX_SDK_PACKAGE}"
+        sudo ./"${INTEL_SGX_SDK_PACKAGE}" <<EOF
+no
+/opt/intel
+EOF
+        popd
+    fi
+
+    pushd "${OE_3RDPARTY}"
+    make -C linux-sgx clean all
+    make -C linux-sgx preparation
+    make -C linux-sgx psw DEBUG=1 -j ${MAKE_THREADS}
+    popd
+fi
+
+# Building OE in release mode to build an enclave which will used as corpus data in create api fuzzing.
+if [[ ! -d "${OE_UNINSTRUMENTED_BUILD}" ]]; then
+    mkdir -p "${OE_UNINSTRUMENTED_BUILD}"
+    mkdir -p "${OE_UNINSTRUMENTED_INSTALL_PREFIX}"
+    pushd "${OE_UNINSTRUMENTED_BUILD}"
+    cmake "${OE_SRC_ROOT}" -GNinja \
+        -DBUILD_TESTS=OFF \
+        -DCMAKE_C_COMPILER="${CLANG}" \
+        -DCMAKE_CXX_COMPILER="${CLANG_CPP}" \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DUSE_DEBUG_MALLOC=OFF \
+        -DCMAKE_INSTALL_PREFIX="${OE_UNINSTRUMENTED_INSTALL_PREFIX}"
+    ninja install -j ${MAKE_THREADS}
+    popd
+fi
+
+# Building OE in debug mode and instrumented with OE-LLVM sanitizers.
+if [[ ! -d "${OE_INSTRUMENTED_BUILD}" ]]; then
+    mkdir -p "${OE_INSTRUMENTED_BUILD}"
+    mkdir -p "${OE_INSTRUMENTED_INSTALL_PREFIX}"
+    pushd "${OE_INSTRUMENTED_BUILD}"
+    cmake "${OE_SRC_ROOT}" -GNinja \
+        -DENABLE_FUZZING=ON \
+        -DBUILD_OEGENERATE_TOOL=OFF \
+        -DBUILD_TESTS=OFF \
+        -DCMAKE_C_COMPILER="${CLANG}" \
+        -DCMAKE_CXX_COMPILER="${CLANG_CPP}" \
+        -DCMAKE_BUILD_TYPE=Debug \
+        -DCMAKE_C_FLAGS_DEBUG="-O0 -g" \
+        -DCMAKE_CXX_FLAGS_DEBUG="-O0 -g" \
+        -DUSE_DEBUG_MALLOC=OFF \
+        -DCMAKE_INSTALL_PREFIX="${OE_INSTRUMENTED_INSTALL_PREFIX}"
+    ninja install -j ${MAKE_THREADS}
+    popd
+fi
+
+# Building an enclave with un-instrumented OE-SDK.
+rm -rf "${OE_SEC_CORPUS_BUILD}"
+mkdir -p "${OE_SEC_CORPUS_BUILD}"
+pushd "${OE_SEC_CORPUS_BUILD}"
+cmake "${OE_SEC_ENCLAVE_CORPUS}" -GNinja \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_C_COMPILER="${CLANG}" \
+    -DCMAKE_CXX_COMPILER="${CLANG_CPP}" \
+    -DCMAKE_PREFIX_PATH="${OE_UNINSTRUMENTED_INSTALL_PREFIX}"
+ninja -j ${MAKE_THREADS}
+popd
+
+# Building fuzzer targets with instrumented OE-SDK.
+rm -rf "${OE_SEC_FUZZING_BUILD}"
+mkdir -p "${OE_SEC_FUZZING_BUILD}"
+pushd "${OE_SEC_FUZZING_BUILD}"
+cmake "${OE_SEC_ROOT}" -GNinja \
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DCMAKE_C_FLAGS_DEBUG="-O0 -g" \
+    -DCMAKE_CXX_FLAGS_DEBUG="-O0 -g" \
+    -DCMAKE_C_COMPILER="${CLANG}" \
+    -DCMAKE_CXX_COMPILER="${CLANG_CPP}" \
+    -DCMAKE_PREFIX_PATH="${OE_INSTRUMENTED_INSTALL_PREFIX}"
+ninja -j ${MAKE_THREADS}
+popd
+
+# Prepare artificats needed for onefuzz.
+pushd "${OE_SEC_FUZZING_BUILD}"/output/bin
+cp ${OE_SEC_ROOT}/src/dynamic/scripts/onefuzz/setup.sh ./
+cp "${OE_3RDPARTY}"/linux-sgx/build/linux/libsgx_enclave_common.so ./
+mkdir create_corpus
+cp "${OE_SEC_CORPUS_BUILD}"/output/bin/create_enclave.signed ./create_corpus/
+popd

cmake/fuzzer_settings.cmake

@@ -0,0 +1,32 @@
+# Copyright (c) Open Enclave SDK contributors.
+# Licensed under the MIT License.
+
+macro (enclave_enable_fuzzing NAME)
+  target_compile_options(${NAME}
+    PRIVATE
+    -fsanitize=enclavefuzzer,enclaveaddress
+    -fsanitize-address-instrument-interceptors
+    -fsanitize-coverage=edge,indirect-calls,no-prune
+    -fsanitize-coverage=trace-cmp,trace-div,trace-gep,trace-pc,trace-pc-guard)
+
+  target_link_options(${NAME}
+    PRIVATE
+    -fsanitize=enclavefuzzer,enclaveaddress
+    -fsanitize-address-instrument-interceptors
+    -fsanitize-coverage=edge,indirect-calls,no-prune
+    -fsanitize-coverage=trace-cmp,trace-div,trace-gep,trace-pc,trace-pc-guard)
+endmacro (enclave_enable_fuzzing)
+
+macro (host_enable_fuzzing NAME)
+  target_compile_options(${NAME}
+    PRIVATE
+    -fsanitize=fuzzer,address
+    -fsanitize-coverage=edge,indirect-calls,no-prune
+    -fsanitize-coverage=trace-cmp,trace-div,trace-gep,trace-pc,trace-pc-guard)
+
+  target_link_options(${NAME}
+    PRIVATE
+    -fsanitize=fuzzer,address
+    -fsanitize-coverage=edge,indirect-calls,no-prune
+    -fsanitize-coverage=trace-cmp,trace-div,trace-gep,trace-pc,trace-pc-guard)
+endmacro (host_enable_fuzzing)

src/dynamic/fuzzing/CMakeLists.txt

@@ -0,0 +1,19 @@
+# Copyright (c) Open Enclave SDK contributors.
+# Licensed under the MIT License.
+
+cmake_minimum_required(VERSION 3.0.0)
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
+
+find_package(OpenEnclave CONFIG REQUIRED)
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/output/bin)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/output/bin)
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/output/lib)
+
+add_subdirectory(fuzzsupport)
+add_subdirectory(fileapihook)
+add_subdirectory(create)
+add_subdirectory(hostapis)
+add_subdirectory(enclaveapis)
+add_subdirectory(sample)

src/dynamic/fuzzing/README.md

@@ -0,0 +1,11 @@
+# Fuzzing Open Enclave API
+
+This project contains fuzzer support library and libfuzzer based target programs to fuzz Open Enclave APIs exposed in [host](https://github.com/openenclave/openenclave/blob/master/include/openenclave/host.h) and [enclave](https://github.com/openenclave/openenclave/blob/master/include/openenclave/enclave.h). Open Enclave fuzzer targets are built using customized LLVM toolchain to enable fuzzing on enclave binaries.
+
+## OneFuzz CI/CD Integration
+Open Enclave fuzzing infrastructure is an instance of OneFuzz service hosted on Azure DCs series virtual machine scalesets. Onefuzz [workflow](../../../.github/workflows/onefuzz-workflow.yml) is scheduled to [run](../../../src/dynamic/scripts/onefuzz/run.sh) on nightly basis which creates onefuzz job templates and uploads fuzzing artifiacts. Onefuzz instance is configured with an array of SGX virtual machines which are dispatched to run the fuzzer targets and managed by VM scalesets. 
+
+**References**
+* [LibFuzzer](https://llvm.org/docs/LibFuzzer.html)
+* [Address Sanitizer](https://clang.llvm.org/docs/AddressSanitizer.html)
+* [OneFuzz](https://github.com/microsoft/onefuzz)

src/dynamic/fuzzing/common/fuzzsupport.edl

@@ -0,0 +1,51 @@
+// Copyright (c) Open Enclave SDK contributors.
+// Licensed under the MIT License.
+
+enclave {
+    struct oe_rlimit 
+    {
+        uint64_t rlim_cur;
+        uint64_t rlim_max;
+    };
+
+    struct prlimit64_args 
+    {
+        long pid;
+        long resource;
+        struct oe_rlimit* new_limit;
+        struct oe_rlimit* old_limit;
+    };
+
+    struct getrlimit_args 
+    {
+        long resource;
+        struct oe_rlimit* rlim;
+    };
+
+    enum Syscall
+    {
+        OE_OCALL_PRRLIMIT64,
+        OE_OCALL_GETLIMIT
+    };
+
+    untrusted
+    {
+        uint64_t oe_get_tpc_ocall();
+
+        void oe_get_enclave_module_path_ocall([user_check] oe_enclave_t* oe_enclave, [user_check] char* path);
+
+        void oe_get_symbol_ocall([user_check] oe_enclave_t* oe_enclave, uint64_t module_offset, [out] char** symbol);
+
+        void oe_die_ocall();
+
+        oe_result_t oe_get_symbol_offset_ocall(
+            [user_check] oe_enclave_t* oe_enclave,
+            [in, string] const char* name,
+            [out] uint64_t* offset);
+
+        oe_result_t oe_syscall_ocall(
+            uint64_t syscall_id, 
+            [out] uint64_t* return_value, 
+            [user_check] void* args) propagate_errno;
+    };
+};