Make mmap() work better

ELF loading now works properly, by delegating to the underlying mmap()
system call (rather than copying the executable into memory). This
reduces the memory required to emulate GCC by 40mb.

Here's the latest benchmark for running GCC:

    RL: took 416,277µs wall time
    RL: ballooned to 107,028kb in size
    RL: needed 415,958µs cpu (7% kernel)
    RL: caused 21,681 page faults (100% memcpy)
    RL: 3 context switches (33% consensual)
    RL: performed 0 reads and 8 write i/o operations

We're now outperforming Qemu by 33% and we finally have comparable
memory usage. This represents a big improvement compared to yesterday:
https://twitter.com/JustineTunney/status/1604049358340902912

Some fine tuning is also performed by this change, to help ensure error
messages are reliably printed when things fall apart.

Author: jart

blink/assert.c

@@ -16,9 +16,11 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include <errno.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "blink/assert.h"
 #include "blink/debug.h"
@@ -39,7 +41,8 @@ static void PrintBacktraceUsingAsan(void) {
 void AssertFailed(const char *file, int line, const char *msg) {
   _Thread_local static bool noreentry;
   char b[512];
-  snprintf(b, sizeof(b), "%s:%d: assertion failed: %s\n", file, line, msg);
+  snprintf(b, sizeof(b), "%s:%d: assertion failed: %s (%s)\n", file, line, msg,
+           strerror(errno));
   b[sizeof(b) - 1] = 0;
   WriteErrorString(b);
   if (g_machine && !noreentry) {

blink/blink.c

@@ -25,6 +25,7 @@
 
 #include "blink/assert.h"
 #include "blink/case.h"
+#include "blink/debug.h"
 #include "blink/dll.h"
 #include "blink/endian.h"
 #include "blink/jit.h"
@@ -55,6 +56,29 @@ static void OnSigSys(int sig) {
   // do nothing
 }
 
+_Noreturn void TerminateSignal(struct Machine *m, int sig) {
+  int syssig;
+  struct sigaction sa;
+  LOGF("terminating due to signal %s", DescribeSignal(sig));
+  if ((syssig = XlatSignal(sig)) == -1) syssig = SIGTERM;
+  LOCK(&m->system->sig_lock);
+  sa.sa_flags = 0;
+  sa.sa_handler = SIG_DFL;
+  sigemptyset(&sa.sa_mask);
+  unassert(!sigaction(syssig, &sa, 0));
+  unassert(!kill(getpid(), syssig));
+  abort();
+}
+
+static void OnSigSegv(int sig, siginfo_t *si, void *uc) {
+  RestoreIp(g_machine);
+  g_machine->faultaddr = ToGuest(si->si_addr);
+  LOGF("SEGMENTATION FAULT AT ADDRESS %" PRIx64 "\n\t%s", g_machine->faultaddr,
+       GetBacktrace(g_machine));
+  DeliverSignalToUser(g_machine, SIGSEGV_LINUX);
+  siglongjmp(g_machine->onhalt, kMachineSegmentationFault);
+}
+
 static int Exec(char *prog, char **argv, char **envp) {
   struct Machine *old;
   if ((old = g_machine)) KillOtherThreads(old->system);
@@ -79,7 +103,7 @@ static int Exec(char *prog, char **argv, char **envp) {
     FreeMachine(old);
   }
   for (;;) {
-    if (!setjmp(g_machine->onhalt)) {
+    if (!sigsetjmp(g_machine->onhalt, 1)) {
       g_machine->canhalt = true;
       Actor(g_machine);
     }
@@ -119,19 +143,22 @@ static void GetOpts(int argc, char *argv[]) {
   }
 }
 
-static void HandleSigSys(void) {
+static void HandleSigs(void) {
   struct sigaction sa;
   sigfillset(&sa.sa_mask);
   sa.sa_flags = 0;
   sa.sa_handler = OnSigSys;
   unassert(!sigaction(SIGSYS, &sa, 0));
+  sa.sa_sigaction = OnSigSegv;
+  sa.sa_flags = SA_SIGINFO;
+  unassert(!sigaction(SIGSEGV, &sa, 0));
 }
 
 int main(int argc, char *argv[], char **envp) {
   g_blink_path = argc > 0 ? argv[0] : 0;
   GetOpts(argc, argv);
   if (optind_ == argc) PrintUsage(argc, argv, 48, 2);
   WriteErrorInit();
-  HandleSigSys();
+  HandleSigs();
   return Exec(argv[optind_], argv + optind_, envp);
 }

blink/cpucount.c

@@ -17,11 +17,11 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include <stdatomic.h>
+#include <sys/types.h>
 #ifdef __linux
 #include <sched.h>
 #elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__)
 #include <sys/sysctl.h>
-#include <sys/types.h>
 #define HAVE_SYSCTL
 #elif defined(__NetBSD__)
 #include <sys/param.h>

blink/loader.c

@@ -46,7 +46,9 @@ static bool CouldJit(struct Machine *m) {
 }
 
 static i64 LoadElfLoadSegment(struct Machine *m, void *image, size_t imagesize,
-                              const Elf64_Phdr *phdr, i64 last_end) {
+                              const Elf64_Phdr *phdr, i64 last_end, int fd) {
+  i64 bulk;
+  struct System *s = m->system;
   u32 flags = Read32(phdr->p_flags);
   i64 vaddr = Read64(phdr->p_vaddr);
   i64 memsz = Read64(phdr->p_memsz);
@@ -55,6 +57,20 @@ static i64 LoadElfLoadSegment(struct Machine *m, void *image, size_t imagesize,
   long pagesize = GetSystemPageSize();
   i64 start = ROUNDDOWN(vaddr, pagesize);
   i64 end = ROUNDUP(vaddr + memsz, pagesize);
+  long skew = vaddr & (pagesize - 1);
+  u64 key = (flags & PF_R ? PAGE_U : 0) |   //
+            (flags & PF_W ? PAGE_RW : 0) |  //
+            (flags & PF_X ? 0 : PAGE_XD);
+
+  ELF_LOGF("PROGRAM HEADER");
+  ELF_LOGF("  vaddr = %" PRIx64, vaddr);
+  ELF_LOGF("  memsz = %" PRIx64, memsz);
+  ELF_LOGF("  offset = %" PRIx64, offset);
+  ELF_LOGF("  filesz = %" PRIx64, filesz);
+  ELF_LOGF("  pagesize = %" PRIx64, pagesize);
+  ELF_LOGF("  start = %" PRIx64, start);
+  ELF_LOGF("  end = %" PRIx64, end);
+  ELF_LOGF("  skew = %lx", skew);
 
   if (offset > imagesize) {
     LOGF("bad phdr offset");
@@ -72,39 +88,97 @@ static i64 LoadElfLoadSegment(struct Machine *m, void *image, size_t imagesize,
     LOGF("program headers aren't ordered");
     exit(127);
   }
-  if (memsz <= 0) {
-    return last_end;
+  if (skew != (offset & (pagesize - 1))) {
+    LOGF("p_vaddr p_offset skew unequal w.r.t. host page size");
+    exit(127);
   }
 
   // on systems with a page size greater than the elf executable (e.g.
   // apple m1) it's possible for the second program header load to end
   // up overlapping the previous one.
-  if (memsz > 0 && start < last_end) {
+  if (start < last_end) {
     start += last_end - start;
-    memsz = end - start;
+  }
+  if (start >= end) {
+    return last_end;
   }
 
-  if (memsz > 0) {
-    if (ReserveVirtual(m->system, start, end - start,
-                       (flags & PF_R ? PAGE_U : 0) |
-                           (flags & PF_W ? PAGE_RW : 0) |
-                           (flags & PF_X ? 0 : PAGE_XD),
-                       -1, false) == -1) {
-      LOGF("failed to reserve virtual memory for elf program header");
-      exit(127);
+  // mmap() always returns an address that page-size aligned, but elf
+  // program headers can start at any address. therefore the first page
+  // needs to be loaded with special care, if the phdr is skewed. the
+  // only real rule in this situation is that the skew of of the virtual
+  // address and the file offset need to be the same.
+  if (skew) {
+    if (vaddr > start && start + pagesize <= end) {
+      unassert(vaddr < start + pagesize);
+      ELF_LOGF("alloc %" PRIx64 "-%" PRIx64, start, start + pagesize);
+      if (ReserveVirtual(s, start, pagesize, key, -1, 0, 0) == -1) {
+        LOGF("failed to allocate program header skew");
+        exit(127);
+      }
+      start += pagesize;
+    }
+    ELF_LOGF("copy %" PRIx64 "-%" PRIx64 " from %" PRIx64 "-%" PRIx64, vaddr,
+             vaddr + (pagesize - skew), offset,
+             offset + MIN(filesz, pagesize - skew));
+    CopyToUser(m, vaddr, (u8 *)image + offset, MIN(filesz, pagesize - skew));
+    vaddr += pagesize - skew;
+    offset += pagesize - skew;
+    filesz -= MIN(filesz, pagesize - skew);
+  }
+
+  // load the aligned program header.
+  unassert(start <= end);
+  unassert(vaddr == start);
+  unassert(vaddr + filesz <= end);
+  unassert(!(vaddr & (pagesize - 1)));
+  unassert(!(start & (pagesize - 1)));
+  unassert(!(offset & (pagesize - 1)));
+  if (start < end) {
+    bulk = ROUNDDOWN(filesz, pagesize);
+    unassert(bulk >= 0);
+    if (bulk) {
+      // map the bulk of .text directly into memory without copying.
+      ELF_LOGF("load %" PRIx64 "-%" PRIx64 " from %" PRIx64 "-%" PRIx64, start,
+               start + bulk, offset, offset + bulk);
+      if (ReserveVirtual(s, start, bulk, key, fd, offset, 0) == -1) {
+        LOGF("failed to map elf program header file data");
+        exit(127);
+      }
+      if (!HasLinearMapping(m->system) && fd != -1) {
+        SyncVirtual(m, start, bulk, fd, offset);
+      }
+    }
+    start += bulk;
+    offset += bulk;
+    filesz -= bulk;
+    if (start < end) {
+      // allocate .bss zero-initialized memory.
+      ELF_LOGF("alloc %" PRIx64 "-%" PRIx64, start, end);
+      if (ReserveVirtual(s, start, end - start, key, -1, 0, 0) == -1) {
+        LOGF("failed to allocate program header bss");
+        exit(127);
+      }
+      // copy the tail skew.
+      if (filesz) {
+        ELF_LOGF("copy %" PRIx64 "-%" PRIx64 " from %" PRIx64 "-%" PRIx64,
+                 start, start + filesz, offset, offset + filesz);
+        CopyToUser(m, start, (u8 *)image + offset, filesz);
+      }
+    } else {
+      unassert(!filesz);
     }
   }
 
-  CopyToUser(m, vaddr, (u8 *)image + offset, filesz);
-  m->system->brk = MAX(m->system->brk, end);
+  s->brk = MAX(s->brk, end);
 
 #ifdef HAVE_JIT
   if ((flags & PF_X) && CouldJit(m)) {
-    if (!m->system->codesize) {
-      m->system->codestart = vaddr;
-      m->system->codesize = memsz;
-    } else if (vaddr == m->system->codestart + m->system->codesize) {
-      m->system->codesize += memsz;
+    if (!s->codesize) {
+      s->codestart = vaddr;
+      s->codesize = memsz;
+    } else if (vaddr == s->codestart + s->codesize) {
+      s->codesize += memsz;
     } else {
       LOGF("elf has multiple executable program headers at noncontiguous "
            "addresses; only the first region will benefit from jitting");
@@ -129,20 +203,20 @@ bool IsSupportedExecutable(const char *path, void *image) {
 }
 
 static void LoadFlatExecutable(struct Machine *m, intptr_t base,
-                               const char *prog, void *image,
-                               size_t imagesize) {
+                               const char *prog, void *image, size_t imagesize,
+                               int fd) {
   Elf64_Phdr phdr;
   Write32(phdr.p_type, PT_LOAD);
   Write32(phdr.p_flags, PF_X | PF_R | PF_W);
   Write64(phdr.p_offset, 0);
   Write64(phdr.p_vaddr, base);
   Write64(phdr.p_filesz, imagesize);
   Write64(phdr.p_memsz, ROUNDUP(imagesize + kRealSize, 4096));
-  LoadElfLoadSegment(m, image, imagesize, &phdr, 0);
+  LoadElfLoadSegment(m, image, imagesize, &phdr, 0, fd);
   m->ip = base;
 }
 
-static bool LoadElf(struct Machine *m, struct Elf *elf) {
+static bool LoadElf(struct Machine *m, struct Elf *elf, int fd) {
   int i;
   Elf64_Phdr *phdr;
   i64 end = INT64_MIN;
@@ -153,7 +227,7 @@ static bool LoadElf(struct Machine *m, struct Elf *elf) {
     switch (Read32(phdr->p_type)) {
       case PT_LOAD:
         elf->base = MIN(elf->base, (i64)Read64(phdr->p_vaddr));
-        end = LoadElfLoadSegment(m, elf->ehdr, elf->size, phdr, end);
+        end = LoadElfLoadSegment(m, elf->ehdr, elf->size, phdr, end, fd);
         break;
       case PT_GNU_STACK:
         execstack = false;
@@ -270,7 +344,6 @@ void LoadProgram(struct Machine *m, char *prog, char **args, char **vars) {
     WriteErrorString("\n");
     exit(127);
   };
-  unassert(!close(fd));
   if (!IsSupportedExecutable(prog, elf->map)) {
     WriteErrorString("\
 error: unsupported executable; we need:\n\
@@ -290,25 +363,25 @@ error: unsupported executable; we need:\n\
     if (READ32(elf->map) == READ32("\177ELF")) {
       elf->ehdr = (Elf64_Ehdr *)elf->map;
       elf->size = elf->mapsize;
-      execstack = LoadElf(m, elf);
+      execstack = LoadElf(m, elf, fd);
     } else if (READ64(elf->map) == READ64("MZqFpD='") ||
                READ64(elf->map) == READ64("jartsr='")) {
       if (GetElfHeader(ehdr, prog, elf->map) == -1) exit(127);
       memcpy(elf->map, ehdr, 64);
       elf->ehdr = (Elf64_Ehdr *)elf->map;
       elf->size = elf->mapsize;
-      execstack = LoadElf(m, elf);
+      execstack = LoadElf(m, elf, fd);
     } else {
       elf->base = 0x400000;
       elf->ehdr = NULL;
       elf->size = 0;
-      LoadFlatExecutable(m, elf->base, prog, elf->map, elf->mapsize);
+      LoadFlatExecutable(m, elf->base, prog, elf->map, elf->mapsize, fd);
       execstack = true;
     }
     sp = kStackTop;
     Put64(m->sp, sp);
     if (ReserveVirtual(m->system, sp - kStackSize, kStackSize,
-                       PAGE_U | PAGE_RW | (execstack ? 0 : PAGE_XD), -1,
+                       PAGE_U | PAGE_RW | (execstack ? 0 : PAGE_XD), -1, 0,
                        false) == -1) {
       LOGF("failed to reserve stack memory");
       exit(127);
@@ -319,4 +392,5 @@ error: unsupported executable; we need:\n\
   pagesize = sysconf(_SC_PAGESIZE);
   pagesize = MAX(4096, pagesize);
   m->system->brk = ROUNDUP(m->system->brk, pagesize);
+  unassert(!close(fd));
 }

blink/log.h

@@ -16,6 +16,7 @@
 #define LOG_JIX 0
 #define LOG_MEM 0
 #define LOG_THR 0
+#define LOG_ELF 0
 
 #if LOG_ENABLED
 #define LOGF(...) Log(__FILE__, __LINE__, __VA_ARGS__)
@@ -65,6 +66,12 @@
 #define THR_LOGF(...) (void)0
 #endif
 
+#if LOG_ELF
+#define ELF_LOGF(...) Log(__FILE__, __LINE__, "(elf) " __VA_ARGS__)
+#else
+#define ELF_LOGF(...) (void)0
+#endif
+
 #if LOG_ENABLED
 #define LOG_ONCE(x)                                                   \
   do {                                                                \