tools/packer: adapt for Ubuntu 18.04

This change is also a complete refactoring of the generation of the vagrant image

Author: aabadie

Vagrantfile

@@ -5,28 +5,20 @@ require 'fileutils'
 
 RIOTBASE ||= "./"
 
-Vagrant.configure(2) do |config|
-  # For a complete reference, please see the online documentation at https://docs.vagrantup.com.
-
-  config.vm.box = "RIOT/ubuntu1604"
-
-  config.vm.provider "virtualbox" do |vb|
-    vb.name = "RIOT-VM"
-
-    vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-interval", 10000]
-    vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-min-adjust", 100]
-    vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-on-restore", 1]
-    vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
-
-    # additional USB passthrough entries
-    # vb.customize ['usbfilter', 'add', '0', '--target', :id, '--name', '<custom_name>', '--vendorid', '<vID>', '--productid', '<pID>']
-  end
-
-  config.vm.synced_folder RIOTBASE, "/home/vagrant/RIOT"
-
-  if File.exists?(File.join(Dir.home, ".gitconfig"))
-    config.vm.provision "file", source: File.join(Dir.home, ".gitconfig"), destination: ".gitconfig"
-  end
-# use a custom provisioning script
-# config.vm.provision "shell", path: "dist/tools/vagrant/bootstrap.sh"
+Vagrant.configure("2") do |config|
+    config.vm.define "RIOT-VM"
+    config.vm.box = "RIOT/ubuntu1804"
+    config.ssh.username = "user"
+    config.vm.synced_folder RIOTBASE, "/home/user/RIOT"
+
+    if File.exists?(File.join(Dir.home, ".gitconfig"))
+        config.vm.provision "file", source: File.join(Dir.home, ".gitconfig"), destination: ".gitconfig"
+    end
+
+    config.vm.provider :virtualbox do |v, override|
+        v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-on-restore", 0]
+
+        # additional USB passthrough entries
+        # v.customize ['usbfilter', 'add', '0', '--target', :id, '--name', '<custom_name>', '--vendorid', '<vID>', '--productid', '<pID>']
+    end
 end

dist/tools/packer/.gitignore

@@ -1,2 +1,3 @@
 packer_cache
+output-*-virtualbox-iso/
 *.box

dist/tools/packer/README.md

@@ -13,4 +13,4 @@ A simple build of the vagrant box can be done by:
 packer build riot.json
 ```
 This command must be executed in the packer directory [RIOT/dist/tools/packer](https://github.com/RIOT-OS/RIOT/tree/master/dist/tools/packer).
-Note that the build process might take several minutes.
+Note that the build process might take several minutes.

dist/tools/packer/http/preseed.cfg

@@ -0,0 +1,43 @@
+#choose-mirror-bin mirror/http/proxy string
+d-i base-installer/kernel/override-image string linux-server
+d-i clock-setup/utc boolean true
+d-i clock-setup/utc-auto boolean true
+d-i finish-install/reboot_in_progress note
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+d-i partman-auto-lvm/guided_size string max
+d-i partman-auto/choose_recipe select atomic
+d-i partman-auto/method string lvm
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/confirm_write_new_label boolean true
+
+# Default mirror
+d-i mirror/country string FR
+d-i mirror/http/hostname string ftp.fr.debian.org
+d-i mirror/http/directory string /ubuntu
+d-i mirror/http/proxy string
+d-i mirror/http/mirror string ftp.fr.debian.org
+
+# Default user
+d-i passwd/user-fullname string user
+d-i passwd/username string user
+d-i passwd/user-password password user
+d-i passwd/user-password-again password user
+d-i passwd/username string user
+
+# Minimum packages (see postinstall.sh)
+d-i pkgsel/include string openssh-server
+d-i pkgsel/install-language-support boolean true
+d-i pkgsel/update-policy select none
+d-i pkgsel/upgrade select none
+
+d-i time/zone string UTC
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+tasksel tasksel/first multiselect standard, ubuntu-server

dist/tools/packer/preseed.cfg

@@ -1,56 +1,70 @@
 {
-  "_comment": "Build with `packer build riot.json`",
   "builders": [
     {
+      "type": "virtualbox-iso",
       "boot_command": [
-        "{{ user `boot_command_prefix` }}",
-        "/install/vmlinuz noapic ",
-        "initrd=/install/initrd.gz ",
-        "file=/floppy/{{ user `preseed` }} ",
-        "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ",
-        "hostname={{ user `hostname` }} ",
-        "fb=false debconf/frontend=noninteractive ",
-        "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ",
-        "keyboard-configuration/variant=USA console-setup/ask_detect=false ",
-        "passwd/user-fullname={{ user `ssh_fullname` }} ",
-        "passwd/user-password={{ user `ssh_password` }} ",
-        "passwd/user-password-again={{ user `ssh_password` }} ",
-        "passwd/username={{ user `ssh_username` }} ",
-        "-- <enter>"
+        "<esc><wait>",
+        "<esc><wait>",
+        "<enter><wait>",
+        "/install/vmlinuz<wait>",
+        " auto<wait>",
+        " console-setup/ask_detect=false<wait>",
+        " console-setup/layoutcode=us<wait>",
+        " console-setup/modelcode=pc105<wait>",
+        " debconf/frontend=noninteractive<wait>",
+        " debian-installer=en_US<wait>",
+        " fb=false<wait>",
+        " initrd=/install/initrd.gz<wait>",
+        " kbd-chooser/method=us<wait>",
+        " keyboard-configuration/layout=USA<wait>",
+        " keyboard-configuration/variant=USA<wait>",
+        " locale=en_US<wait>",
+        " netcfg/get_domain=vm<wait>",
+        " netcfg/get_hostname={{ user `hostname` }}<wait>",
+        " grub-installer/bootdev=/dev/sda<wait>",
+        " noapic<wait>",
+        " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg<wait>",
+        " -- <wait>",
+        "<enter><wait>"
       ],
+      "boot_wait": "10s",
       "disk_size": "{{ user `disk_size` }}",
-      "floppy_files": [ "{{ user `preseed` }}" ],
       "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
       "guest_os_type": "{{ user `virtualbox_guest_os_type` }}",
       "headless": "{{ user `headless` }}",
       "iso_checksum": "{{ user `iso_checksum` }}",
       "iso_checksum_type": "{{ user `iso_checksum_type` }}",
       "iso_urls": [ "{{ user `iso_path` }}/{{ user `iso_name` }}", "{{ user `iso_url` }}" ],
       "output_directory": "output-{{ user `vm_name` }}-virtualbox-iso",
+      "http_directory": "http",
       "shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now",
-      "ssh_password": "{{ user `ssh_password` }}",
       "ssh_username": "{{ user `ssh_username` }}",
+      "ssh_password": "{{ user `ssh_password` }}",
+      "ssh_port": 22,
       "ssh_wait_timeout": "10000s",
-      "type": "virtualbox-iso",
       "vboxmanage": [
         [ "modifyvm", "{{.Name}}", "--memory", "{{ user `memory` }}" ],
         [ "modifyvm", "{{.Name}}", "--cpus", "{{ user `cpus` }}" ],
         [ "modifyvm", "{{.Name}}", "--usb", "on" ],
-        [ "modifyvm", "{{.Name}}", "--usbxhci", "on" ],
         [ "modifyvm", "{{.Name}}", "--audio", "none"],
         [ "modifyvm", "{{.Name}}", "--ioapic", "on"],
         [ "modifyvm", "{{.Name}}", "--rtcuseutc", "on"],
-        [ "modifyvm", "{{.Name}}", "--clipboard", "bidirectional"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "FTDI USB-TTL", "--vendorid", "0x0403", "--productid",  "0x6001"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "CP2102 USB to UART", "--vendorid", "0x10c4", "--productid",  "0xea60"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "STM32 STLink", "--vendorid", "0x0483", "--productid",  "0x3748"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "STM32 STLink", "--vendorid", "0x0483", "--productid", "0x374b"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "USBasp", "--vendorid", "0x16c0", "--productid",  "0x05dc"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "iotlab-m3", "--vendorid", "0x0403", "--productid",  "0x6010"],
-        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "samr21-xpro", "--vendorid", "0x03eb", "--productid",  "0x2111"],
-        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Arduino Mega 2560", "--vendorid", "0x2341", "--productid",  "0x0042"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "iotlab-m3-cn", "--vendorid", "0x0403", "--productid", "0x6011"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "CMSIS-DAP", "--vendorid", "0x03eb", "--productid",  "0x2111"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Arduino UNO", "--vendorid", "0x2341", "--productid",  "0x0042"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Arduino Mega 2560", "--vendorid", "0x2a03", "--productid",  "0x0042"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Phytec phyWAVE KW22", "--vendorid", "0x0d28", "--productid",  "0x0204"],
         ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Bluepill", "--vendorid", "0x1d50", "--productid",  "0x6017"],
-        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Bluepill", "--vendorid", "0x1d50", "--productid",  "0x6018"]
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Bluepill", "--vendorid", "0x1d50", "--productid",  "0x6018"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "JLink", "--vendorid", "0x1366", "--productid", "0x1015"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "Arduino Zero", "--vendorid", "0x03eb", "--productid", "0x2157"],
+        ["usbfilter", "add", "0", "--target", "{{.Name}}", "--name", "DAPLink", "--vendorid", "0x0d28", "--productid", "0x0204"]
       ],
       "virtualbox_version_file": ".vbox_version",
       "vm_name": "{{user `vm_name`}}"
@@ -68,36 +82,39 @@
   ],
   "provisioners": [
     {
-      "type": "file",
-      "source": "./udev_rules",
-      "destination": "/tmp"
-    },
-    {
+      "type": "shell",
       "environment_vars": [
-        "SSH_USERNAME={{user `ssh_username`}}",
-        "SSH_PASSWORD={{user `ssh_password`}}"
+        "SSH_USERNAME={{ user `ssh_username` }}"
       ],
-      "execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
-      "scripts": [ "setup.sh" ],
-      "type": "shell"
+      "execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -S -E bash '{{.Path}}'",
+      "override": {
+        "virtualbox-iso": {
+          "scripts": [
+            "scripts/base.sh",
+            "scripts/vagrant.sh",
+            "scripts/riot.sh",
+            "scripts/toolchains.sh",
+            "scripts/udev_rules.sh",
+            "scripts/virtualbox.sh",
+            "scripts/cleanup.sh"
+          ]
+        }
+      }
     }
   ],
   "variables": {
-    "boot_command_prefix": "<enter><wait><f6><esc><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
-    "cpus": "1",
-    "disk_size": "65536",
+    "cpus": "2",
+    "disk_size": "10240",
     "headless": "true",
-    "iso_checksum": "a06cd926f5855d4f21fb4bc9978a35312f815fbda0d0ef7fdc846861f4fc4600",
+    "iso_checksum": "e2ecdace33c939527cbc9e8d23576381c493b071107207d2040af72595f8990b",
     "iso_checksum_type": "sha256",
-    "iso_name": "ubuntu-16.04.3-server-amd64.iso",
-    "iso_path": "/Volumes/Storage/software/ubuntu",
-    "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.3-server-amd64.iso",
+    "iso_name": "ubuntu-18.04.4-server-amd64.iso",
+    "iso_path": "iso",
+    "iso_url": "http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/ubuntu-18.04.4-server-amd64.iso",
     "memory": "1024",
-    "preseed" : "preseed.cfg",
-    "hostname": "vagrant",
-    "ssh_fullname": "vagrant",
-    "ssh_password": "vagrant",
-    "ssh_username": "vagrant",
+    "hostname": "riot-vm",
+    "ssh_username": "user",
+    "ssh_password": "user",
     "vagrantfile_template": "vagrantfile.tpl",
     "virtualbox_guest_os_type": "Ubuntu_64",
     "vm_name": "RIOT-VM"

dist/tools/packer/riot.json

@@ -0,0 +1,14 @@
+apt-get update
+apt-get -y upgrade
+apt-get -y install linux-headers-$(uname -r)
+
+# Fix locale setup
+locale-gen
+localectl set-locale LANG="en_US.UTF-8"
+
+# Allow usage of serial port
+adduser ${SSH_USERNAME} dialout
+
+echo '%sudo   ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+echo "UseDNS no" >> /etc/ssh/sshd_config

dist/tools/packer/scripts/base.sh

@@ -0,0 +1,91 @@
+apt-get -y upgrade
+apt-get -y autoremove
+apt-get -y clean
+
+echo "cleaning up guest additions"
+rm -rf VBoxGuestAdditions_*.iso VBoxGuestAdditions_*.iso.?
+
+echo "cleaning permissions"
+chown -R ${SSH_USERNAME}:${SSH_USERNAME} /home/${SSH_USERNAME}
+
+# Remove some packages to get a minimal install
+echo "==> Removing all linux kernels except the current one"
+dpkg --list | awk '{ print $2 }' | grep 'linux-image-*-generic' | grep -v $(uname -r) | xargs apt-get -y purge
+echo "==> Removing linux source"
+dpkg --list | awk '{ print $2 }' | grep linux-source | xargs apt-get -y purge
+echo "==> Removing documentation"
+dpkg --list | awk '{ print $2 }' | grep -- '-doc$' | xargs apt-get -y purge
+echo "==> Removing default system Ruby"
+apt-get -y purge ruby ri doc
+echo "==> Removing obsolete networking components"
+apt-get -y purge ppp pppconfig pppoeconf
+echo "==> Removing other oddities"
+apt-get -y purge popularity-contest installation-report landscape-common wireless-tools wpasupplicant
+
+# Clean up the apt cache
+apt-get -y autoremove --purge
+apt-get -y autoclean
+apt-get -y clean
+
+echo "removing man pages"
+rm -rf /usr/share/man/*
+echo "removing APT files"
+find /var/lib/apt -type f | xargs rm -f
+echo "removing any docs"
+rm -rf /usr/share/doc/*
+echo "removing caches"
+find /var/cache -type f -exec rm -rf {} \;
+
+echo "cleaning up tmp"
+rm -rf /tmp/*
+
+# Remove Bash history
+unset HISTFILE
+rm -f /root/.bash_history
+rm -f ${SSH_USER_HOME}/.bash_history
+
+# Clean up log files
+find /var/log -type f | while read f; do echo -ne '' > "${f}"; done;
+
+echo "clearing last login information"
+>/var/log/lastlog
+>/var/log/wtmp
+>/var/log/btmp
+
+echo "whiteout /"
+count=$(df --sync -kP / | tail -n1  | awk -F ' ' '{print $4}')
+let count--
+dd if=/dev/zero of=/tmp/whitespace bs=1024 count=$count
+rm /tmp/whitespace
+
+echo "whiteout /boot"
+count=$(df --sync -kP /boot | tail -n1 | awk -F ' ' '{print $4}')
+let count--
+dd if=/dev/zero of=/boot/whitespace bs=1024 count=$count
+rm /boot/whitespace
+
+echo "clear out swap and disable until reboot"
+set +e
+swapuuid=$(/sbin/blkid -o value -l -s UUID -t TYPE=swap)
+case "$?" in
+    2|0) ;;
+    *) exit 1 ;;
+esac
+set -e
+if [ "x${swapuuid}" != "x" ]; then
+    # Whiteout the swap partition to reduce box size
+    # Swap is disabled till reboot
+    swappart=$(readlink -f /dev/disk/by-uuid/$swapuuid)
+    /sbin/swapoff "${swappart}"
+    dd if=/dev/zero of="${swappart}" bs=1M || echo "dd exit code $? is suppressed"
+    /sbin/mkswap -U "${swapuuid}" "${swappart}"
+fi
+
+echo "whiteout free space"
+dd if=/dev/zero of=/EMPTY bs=1M  || echo "dd exit code $? is suppressed"
+rm -f /EMPTY
+
+sync
+
+echo "disk usage"
+df -h