Pash support for FreeBSD (#309)

* Distro deps for freebsd
* Hacky fix to mktemp for pash tmp files. Someone could fix it better
* Fixes to log parsing on intro, interface, compiler tests
* Fixed head command to match bsd
* Added freebsd implementation of Linux only functions. This is not optimal
* mktemp hacky fixes
* Fixed ps command. Not quite sure if this is the best solution
* Fixed pash setup to support freebsd
* mktemp fixes on bsd [skip ci]
* mktemp fix on bsd

Co-authored-by: Nikos Vasilakis <[email protected]>
Signed-off-by: Dimitris Karnikis <[email protected]>
Signed-off-by: Nikos Vasilakis <[email protected]>

Author: dkarnikis

compiler/pash_ptempfile_name.sh

@@ -1,3 +1,20 @@
 #!/usr/bin/env bash
+if type lsb_release >/dev/null 2>&1 ; then
+    distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+    distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
 
-echo "$(mktemp --tmpdir="$PASH_TMP_PREFIX" -u pash_XXXXXXXXXX)"
+# convert to lowercase
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+    freebsd*)  
+        # bsd is so fun :)
+        tmp=$(TMPDIR=$PASH_TMP_PREFIX mktemp -t pash_XXXXXXXXXX)
+        echo "${tmp}"
+        ;;
+    *)
+        echo "$(mktemp --tmpdir="$PASH_TMP_PREFIX" -u pash_XXXXXXXXXX)"
+        ;;
+esac

compiler/test_evaluation_scripts.sh

@@ -190,6 +190,26 @@ execute_tests "--assert_compiler_success" "${pipeline_microbenchmarks[@]}"
 #cat /tmp/a | sed 's/@/,/' > ${results_time}
 
 
+if type lsb_release >/dev/null 2>&1 ; then
+   distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+   distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
+
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+    freebsd*)  
+        # change sed to gsed
+        alias sed="gsed"
+        sed () {
+            gsed $@
+        }
+        ;;
+    *)
+        ;;
+esac
+
 echo "group,Bash,Pash2,Pash8" > ${results_time}
 paste -d'@' $test_results_dir/results.time_*  | sed 's\,\.\g' | sed 's\:\,\g' | sed 's\@\,\g' >> ${results_time}
 

evaluation/intro/test.sh

@@ -45,10 +45,28 @@ run_test()
 run_test "demo-spell.sh"
 run_test "hello-world.sh"
 
+if type lsb_release >/dev/null 2>&1 ; then
+   distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+   distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
+
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+    freebsd*)  
+        # change sed to gsed
+        sed () {
+            gsed $@
+        }
+        ;;
+    *)
+        ;;
+esac
+
 echo "group,Bash,Pash2" > $output_dir/results.time
 paste $output_dir/results.time_*  | sed 's\,\.\g' | sed 's\:\,\g' | sed 's/\t/,/' >> $output_dir/results.time
 
-
 echo "Below follow the identical outputs:"
 grep --files-with-match "are identical" "$output_dir"/*_distr.time
 

evaluation/tests/input/setup.sh

@@ -4,12 +4,21 @@ PASH_TOP=${PASH_TOP:-$(git rev-parse --show-toplevel)}
 . "$PASH_TOP/scripts/utils.sh"
 cd $(dirname $0)
 
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+head_sz="M"
+# now do different things depending on distro
+case "$distro" in
+   freebsd*) 
+    head_sz="m"
+    ;;
+esac
+
 [ "$1" = "-c" ] && rm-files 1M.txt all_cmds.txt words sorted_words 10M.txt
 
 if [ ! -f ./1M.txt ]; then
   curl -sf 'http://ndr.md/data/dummy/1M.txt' > 1M.txt
   if [ $? -ne 0 ]; then
-    curl -sf 'http://www.gutenberg.org/files/2600/2600-0.txt' | head -c 1M > 1M.txt
+    curl -sf 'http://www.gutenberg.org/files/2600/2600-0.txt' | head -c 1${head_sz} > 1M.txt
     [ $? -ne 0 ] && eexit 'cannot find 1M.txt'
   fi
   append_nl_if_not ./1M.txt

evaluation/tests/interface_tests/run.sh

@@ -107,6 +107,25 @@ else
     done
 fi
 
+if type lsb_release >/dev/null 2>&1 ; then
+   distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+   distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
+
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+    freebsd*)  
+        # change sed to gsed
+        sed () {
+            gsed $@
+        }
+        ;;
+    *)
+        ;;
+esac
+
 echo "group,Bash,Pash-DRY_COMP" > $output_dir/results.time
 paste $output_dir/results.time_*  | sed 's\,\.\g' | sed 's\:\,\g' | sed 's/\t/,/' >> $output_dir/results.time
 

runtime/auto-split.sh

@@ -8,7 +8,24 @@ n_outputs="$#"
 # Set a default DISH_TOP in this directory if it doesn't exist
 PASH_TOP=${PASH_TOP:-$(git rev-parse --show-toplevel)}
 
-temp="$(mktemp --tmpdir -u pash_XXXXXXXXXX)"
+if type lsb_release >/dev/null 2>&1 ; then
+   distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+   distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
+
+# convert to lowercase
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+   freebsd*)  
+    temp="$(TMPDIR=/tmp mktemp pash_XXXXXXXXXX)"
+    ;;
+    *)
+    temp="$(mktemp --tmpdir -u pash_XXXXXXXXXX)"
+    ;;
+esac
+
 
 cat "$input" > "$temp"
 total_lines=$(wc -l $temp | cut -f 1 -d ' ')

runtime/eager_lib.c

@@ -1,5 +1,85 @@
 #include "eager_lib.h"
 
+#ifdef __FreeBSD__
+#define BUF_SIZE 8192
+#define min(a,b) (((a)<(b))?(a):(b))
+void fatal
+(const char *err)
+{
+	perror(err);
+	abort();
+}
+
+int std_copy(int in_fd, int out_fd, int unused, int size)
+{
+    char buf[4096];
+    int err = -1;
+    int bytes;
+    while( (bytes = read(in_fd, buf, size)) > 0 )
+    {
+        if(write(out_fd, buf, bytes) != bytes)
+        {
+            perror("write:");
+            goto out;
+        }
+    }
+    err = 0;
+    out:
+    return err;
+}
+
+
+ssize_t
+send_file(int out_fd, int in_fd, off_t *offset, size_t count)
+{
+    off_t orig = 0;
+
+    if (offset != NULL) {
+        /* Save current file offset and set offset to value in '*offset' */
+        orig = lseek(in_fd, 0, SEEK_CUR);
+        if (orig == -1)
+            return -1;
+        if (lseek(in_fd, *offset, SEEK_SET) == -1)
+            return -1;
+    }
+
+    size_t totSent = 0;
+
+    while (count > 0) {
+        size_t toRead = min(BUF_SIZE, count);
+
+        char buf[BUF_SIZE];
+        ssize_t numRead = read(in_fd, buf, toRead);
+        if (numRead == -1)
+            return -1;
+        if (numRead == 0)
+            break;                      /* EOF */
+
+        ssize_t numSent = write(out_fd, buf, numRead);
+        if (numSent == -1)
+            return -1;
+        if (numSent == 0)               /* Should never happen */
+            fatal("send_file: write() transferred 0 bytes");
+
+        count -= numSent;
+        totSent += numSent;
+    }
+
+    if (offset != NULL) {
+
+        /* Return updated file offset in '*offset', and reset the file offset
+           to the value it had when we were called. */
+
+        *offset = lseek(in_fd, 0, SEEK_CUR);
+        if (*offset == -1)
+            return -1;
+        if (lseek(in_fd, orig, SEEK_SET) == -1)
+            return -1;
+    }
+    return totSent;
+}
+#endif
+
 int safeOpen3(const char *pathname, int flags, mode_t mode) {
     int fd = open(pathname, flags, mode);
     if (fd < 0) {
@@ -77,10 +157,16 @@ int blockOpenOutput(const char *pathname) {
 
 // Returns the number of bytes read, or 0 if the input was done.
 int readInputWriteToFile(int inputFd, int intermediateWriter, int bufferSize) {
-
-    ssize_t res = splice(inputFd, 0, intermediateWriter, 0, bufferSize, 0);
+    ssize_t res = 
+#ifdef __linux__
+    splice(inputFd, 0, intermediateWriter, 0, bufferSize, 0);
+#else
+    // https://man.openbsd.org/sosplice.9
+    // naive implementation to match our needs, maybe we could we use sosplice, somove 
+    std_copy(inputFd, intermediateWriter, 0, bufferSize);
+#endif
     if (res < 0) {
-        printf("Error: Couldn't read from input!\n");
+        printf("Error: Couldn't read from inputaa!\n");
         exit(1);
     }
     return res;
@@ -99,7 +185,7 @@ int bufferedReadInputWriteToFile(int inputFd, int intermediateWriter, int buffer
 
     inputBytesRead = read(inputFd, inputBuf, sizeof(inputBuf));
     if (inputBytesRead < 0) {
-        printf("Error: Couldn't read from input!\n");
+        printf("Error: Couldn't read from input@@!\n");
         exit(1);
     }
     if (inputBytesRead == 0) {
@@ -185,7 +271,14 @@ void bufferedOutputRestIntermediateFile(int outputFd, int intermediateWriter, in
 ssize_t safeWriteOutput(int outputFd, int intermediateReader,
                         off_t intermediateFileDiff, int* doneWriting) {
     ssize_t res;
-    res = sendfile(outputFd, intermediateReader, 0, intermediateFileDiff);
+    res = 
+#ifdef __linux__
+    sendfile
+#else
+    send_file
+#endif
+    (outputFd, intermediateReader, 0, intermediateFileDiff);
+
     if (res < 0 && errno != EAGAIN) {
         printf("ERROR: %s, when outputing %ld bytes!\n", strerror(errno), intermediateFileDiff);
         exit(1);

runtime/eager_lib.h

@@ -10,7 +10,13 @@
 #include <sys/select.h>
 #include <unistd.h>
 #include <sys/time.h>
+#ifdef __linux__
 #include <sys/sendfile.h>
+#else
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+#endif
 
 #ifndef __DEBUG__
 #define __DEBUG__

runtime/wait_for_output_and_sigpipe_rest.sh

@@ -3,8 +3,25 @@
 ## TODO: Give it the output pid as an argument
 
 wait $@
-pids_to_kill="$(ps --ppid $$ |awk '{print $1}' | grep -E '[0-9]')"
+if type lsb_release >/dev/null 2>&1 ; then
+    distro=$(lsb_release -i -s)
+elif [ -e /etc/os-release ] ; then
+    distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+fi
+
+# convert to lowercase
+distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
+# now do different things depending on distro
+case "$distro" in
+    freebsd*)  
+        # not sure at all about this one
+        pids_to_kill="$(ps -efl $$ |awk '{print $1}' | grep -E '[0-9]')"
+        ;;
+    *)
+        pids_to_kill="$(ps --ppid $$ |awk '{print $1}' | grep -E '[0-9]')"
+        ;;
+esac
 for pid in $pids_to_kill
 do
-        (> /dev/null 2>&1 kill -SIGPIPE $pid || true)
+    (> /dev/null 2>&1 kill -SIGPIPE $pid || true)
 done

scripts/distro-deps.sh

@@ -10,15 +10,15 @@ LOG_DIR=$PWD/install_logs
 mkdir -p $LOG_DIR
 
 if [[ $(uname) == 'Darwin' ]]; then
-  echo 'Currently pash can run only on Linux'
-  exit 1
+    echo 'Currently pash can run only on Linux'
+    exit 1
 fi
 
 
 if type lsb_release >/dev/null 2>&1 ; then
-   distro=$(lsb_release -i -s)
+    distro=$(lsb_release -i -s)
 elif [ -e /etc/os-release ] ; then
-   distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
+    distro=$(awk -F= '$1 == "ID" {print $2}' /etc/os-release)
 fi
 
 # convert to lowercase
@@ -52,5 +52,11 @@ case "$distro" in
      echo "|-- running pacman install...."
      yes | pacman -S git libtool m4 automake curl pkg-config python-pip libffi make autoconf gcc10 sudo inetutils bc
      ;;
+   freebsd*)
+     echo "Updating mirros"
+     pkg update &> $LOG_DIR/pkg_update.log
+     echo "|-- running pkg install...."
+     yes | pkg install libtool m4 automake curl libffi py38-pip autoconf gcc gsed gmake
+     ;;
    *)        echo "unknown distro: '$distro'" ; exit 1 ;;
 esac