RF_prediction_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RF prediction function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Find out more at https://www.synthpop.org.uk/\n",
      "\n",
      "Lade n\"otiges Paket: grid\n",
      "\n",
      "Lade n\"otiges Paket: mvtnorm\n",
      "\n",
      "Lade n\"otiges Paket: modeltools\n",
      "\n",
      "Lade n\"otiges Paket: stats4\n",
      "\n",
      "Lade n\"otiges Paket: strucchange\n",
      "\n",
      "Lade n\"otiges Paket: zoo\n",
      "\n",
      "\n",
      "Attache Paket: 'zoo'\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:base':\n",
      "\n",
      "    as.Date, as.Date.numeric\n",
      "\n",
      "\n",
      "Lade n\"otiges Paket: sandwich\n",
      "\n",
      "\n",
      "Attache Paket: 'dplyr'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:party':\n",
      "\n",
      "    where\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:stats':\n",
      "\n",
      "    filter, lag\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:base':\n",
      "\n",
      "    intersect, setdiff, setequal, union\n",
      "\n",
      "\n",
      "randomForest 4.7-1.1\n",
      "\n",
      "Type rfNews() to see new features/changes/bug fixes.\n",
      "\n",
      "\n",
      "Attache Paket: 'randomForest'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:dplyr':\n",
      "\n",
      "    combine\n",
      "\n",
      "\n",
      "Type 'citation(\"pROC\")' for a citation.\n",
      "\n",
      "\n",
      "Attache Paket: 'pROC'\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:stats':\n",
      "\n",
      "    cov, smooth, var\n",
      "\n",
      "\n",
      "Lade n\"otiges Paket: ggplot2\n",
      "\n",
      "\n",
      "Attache Paket: 'ggplot2'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:randomForest':\n",
      "\n",
      "    margin\n",
      "\n",
      "\n",
      "Lade n\"otiges Paket: lattice\n",
      "\n",
      "here() starts at /Users/emmafoessing/Documents/Master/MA/Code/Master-Thesis\n",
      "\n",
      "\n",
      "Attache Paket: 'Hmisc'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:pracma':\n",
      "\n",
      "    ceil\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:dplyr':\n",
      "\n",
      "    src, summarize\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:base':\n",
      "\n",
      "    format.pval, units\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'purrr'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:pracma':\n",
      "\n",
      "    cross\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:caret':\n",
      "\n",
      "    lift\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'ranger'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:randomForest':\n",
      "\n",
      "    importance\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'xgboost'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:dplyr':\n",
      "\n",
      "    slice\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'data.table'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:purrr':\n",
      "\n",
      "    transpose\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:dplyr':\n",
      "\n",
      "    between, first, last\n",
      "\n",
      "\n",
      "Die folgenden Objekte sind maskiert von 'package:zoo':\n",
      "\n",
      "    yearmon, yearqtr\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'tensorflow'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:caret':\n",
      "\n",
      "    train\n",
      "\n",
      "\n",
      "\n",
      "Attache Paket: 'keras'\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:party':\n",
      "\n",
      "    fit\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:modeltools':\n",
      "\n",
      "    fit\n",
      "\n",
      "\n",
      "Das folgende Objekt ist maskiert 'package:insight':\n",
      "\n",
      "    get_weights\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<ol>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'ranger'</li><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'xgboost'</li><li>'ranger'</li><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'data.table'</li><li>'xgboost'</li><li>'ranger'</li><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'tensorflow'</li><li>'data.table'</li><li>'xgboost'</li><li>'ranger'</li><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "\t<li><style>\n",
       ".list-inline {list-style: none; margin:0; padding: 0}\n",
       ".list-inline>li {display: inline-block}\n",
       ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
       "</style>\n",
       "<ol class=list-inline><li>'keras'</li><li>'tensorflow'</li><li>'data.table'</li><li>'xgboost'</li><li>'ranger'</li><li>'purrr'</li><li>'Hmisc'</li><li>'here'</li><li>'pracma'</li><li>'caret'</li><li>'lattice'</li><li>'ggplot2'</li><li>'pROC'</li><li>'randomForest'</li><li>'rpart.plot'</li><li>'rpart'</li><li>'dplyr'</li><li>'haven'</li><li>'party'</li><li>'strucchange'</li><li>'sandwich'</li><li>'zoo'</li><li>'modeltools'</li><li>'stats4'</li><li>'mvtnorm'</li><li>'grid'</li><li>'insight'</li><li>'synthpop'</li><li>'stats'</li><li>'graphics'</li><li>'grDevices'</li><li>'utils'</li><li>'datasets'</li><li>'methods'</li><li>'base'</li></ol>\n",
       "</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate}\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'ranger'\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'xgboost'\n",
       "\\item 'ranger'\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'data.table'\n",
       "\\item 'xgboost'\n",
       "\\item 'ranger'\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'tensorflow'\n",
       "\\item 'data.table'\n",
       "\\item 'xgboost'\n",
       "\\item 'ranger'\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\item \\begin{enumerate*}\n",
       "\\item 'keras'\n",
       "\\item 'tensorflow'\n",
       "\\item 'data.table'\n",
       "\\item 'xgboost'\n",
       "\\item 'ranger'\n",
       "\\item 'purrr'\n",
       "\\item 'Hmisc'\n",
       "\\item 'here'\n",
       "\\item 'pracma'\n",
       "\\item 'caret'\n",
       "\\item 'lattice'\n",
       "\\item 'ggplot2'\n",
       "\\item 'pROC'\n",
       "\\item 'randomForest'\n",
       "\\item 'rpart.plot'\n",
       "\\item 'rpart'\n",
       "\\item 'dplyr'\n",
       "\\item 'haven'\n",
       "\\item 'party'\n",
       "\\item 'strucchange'\n",
       "\\item 'sandwich'\n",
       "\\item 'zoo'\n",
       "\\item 'modeltools'\n",
       "\\item 'stats4'\n",
       "\\item 'mvtnorm'\n",
       "\\item 'grid'\n",
       "\\item 'insight'\n",
       "\\item 'synthpop'\n",
       "\\item 'stats'\n",
       "\\item 'graphics'\n",
       "\\item 'grDevices'\n",
       "\\item 'utils'\n",
       "\\item 'datasets'\n",
       "\\item 'methods'\n",
       "\\item 'base'\n",
       "\\end{enumerate*}\n",
       "\n",
       "\\end{enumerate}\n"
      ],
      "text/markdown": [
       "1. 1. 'synthpop'\n",
       "2. 'stats'\n",
       "3. 'graphics'\n",
       "4. 'grDevices'\n",
       "5. 'utils'\n",
       "6. 'datasets'\n",
       "7. 'methods'\n",
       "8. 'base'\n",
       "\n",
       "\n",
       "\n",
       "2. 1. 'insight'\n",
       "2. 'synthpop'\n",
       "3. 'stats'\n",
       "4. 'graphics'\n",
       "5. 'grDevices'\n",
       "6. 'utils'\n",
       "7. 'datasets'\n",
       "8. 'methods'\n",
       "9. 'base'\n",
       "\n",
       "\n",
       "\n",
       "3. 1. 'party'\n",
       "2. 'strucchange'\n",
       "3. 'sandwich'\n",
       "4. 'zoo'\n",
       "5. 'modeltools'\n",
       "6. 'stats4'\n",
       "7. 'mvtnorm'\n",
       "8. 'grid'\n",
       "9. 'insight'\n",
       "10. 'synthpop'\n",
       "11. 'stats'\n",
       "12. 'graphics'\n",
       "13. 'grDevices'\n",
       "14. 'utils'\n",
       "15. 'datasets'\n",
       "16. 'methods'\n",
       "17. 'base'\n",
       "\n",
       "\n",
       "\n",
       "4. 1. 'haven'\n",
       "2. 'party'\n",
       "3. 'strucchange'\n",
       "4. 'sandwich'\n",
       "5. 'zoo'\n",
       "6. 'modeltools'\n",
       "7. 'stats4'\n",
       "8. 'mvtnorm'\n",
       "9. 'grid'\n",
       "10. 'insight'\n",
       "11. 'synthpop'\n",
       "12. 'stats'\n",
       "13. 'graphics'\n",
       "14. 'grDevices'\n",
       "15. 'utils'\n",
       "16. 'datasets'\n",
       "17. 'methods'\n",
       "18. 'base'\n",
       "\n",
       "\n",
       "\n",
       "5. 1. 'dplyr'\n",
       "2. 'haven'\n",
       "3. 'party'\n",
       "4. 'strucchange'\n",
       "5. 'sandwich'\n",
       "6. 'zoo'\n",
       "7. 'modeltools'\n",
       "8. 'stats4'\n",
       "9. 'mvtnorm'\n",
       "10. 'grid'\n",
       "11. 'insight'\n",
       "12. 'synthpop'\n",
       "13. 'stats'\n",
       "14. 'graphics'\n",
       "15. 'grDevices'\n",
       "16. 'utils'\n",
       "17. 'datasets'\n",
       "18. 'methods'\n",
       "19. 'base'\n",
       "\n",
       "\n",
       "\n",
       "6. 1. 'rpart'\n",
       "2. 'dplyr'\n",
       "3. 'haven'\n",
       "4. 'party'\n",
       "5. 'strucchange'\n",
       "6. 'sandwich'\n",
       "7. 'zoo'\n",
       "8. 'modeltools'\n",
       "9. 'stats4'\n",
       "10. 'mvtnorm'\n",
       "11. 'grid'\n",
       "12. 'insight'\n",
       "13. 'synthpop'\n",
       "14. 'stats'\n",
       "15. 'graphics'\n",
       "16. 'grDevices'\n",
       "17. 'utils'\n",
       "18. 'datasets'\n",
       "19. 'methods'\n",
       "20. 'base'\n",
       "\n",
       "\n",
       "\n",
       "7. 1. 'rpart.plot'\n",
       "2. 'rpart'\n",
       "3. 'dplyr'\n",
       "4. 'haven'\n",
       "5. 'party'\n",
       "6. 'strucchange'\n",
       "7. 'sandwich'\n",
       "8. 'zoo'\n",
       "9. 'modeltools'\n",
       "10. 'stats4'\n",
       "11. 'mvtnorm'\n",
       "12. 'grid'\n",
       "13. 'insight'\n",
       "14. 'synthpop'\n",
       "15. 'stats'\n",
       "16. 'graphics'\n",
       "17. 'grDevices'\n",
       "18. 'utils'\n",
       "19. 'datasets'\n",
       "20. 'methods'\n",
       "21. 'base'\n",
       "\n",
       "\n",
       "\n",
       "8. 1. 'randomForest'\n",
       "2. 'rpart.plot'\n",
       "3. 'rpart'\n",
       "4. 'dplyr'\n",
       "5. 'haven'\n",
       "6. 'party'\n",
       "7. 'strucchange'\n",
       "8. 'sandwich'\n",
       "9. 'zoo'\n",
       "10. 'modeltools'\n",
       "11. 'stats4'\n",
       "12. 'mvtnorm'\n",
       "13. 'grid'\n",
       "14. 'insight'\n",
       "15. 'synthpop'\n",
       "16. 'stats'\n",
       "17. 'graphics'\n",
       "18. 'grDevices'\n",
       "19. 'utils'\n",
       "20. 'datasets'\n",
       "21. 'methods'\n",
       "22. 'base'\n",
       "\n",
       "\n",
       "\n",
       "9. 1. 'pROC'\n",
       "2. 'randomForest'\n",
       "3. 'rpart.plot'\n",
       "4. 'rpart'\n",
       "5. 'dplyr'\n",
       "6. 'haven'\n",
       "7. 'party'\n",
       "8. 'strucchange'\n",
       "9. 'sandwich'\n",
       "10. 'zoo'\n",
       "11. 'modeltools'\n",
       "12. 'stats4'\n",
       "13. 'mvtnorm'\n",
       "14. 'grid'\n",
       "15. 'insight'\n",
       "16. 'synthpop'\n",
       "17. 'stats'\n",
       "18. 'graphics'\n",
       "19. 'grDevices'\n",
       "20. 'utils'\n",
       "21. 'datasets'\n",
       "22. 'methods'\n",
       "23. 'base'\n",
       "\n",
       "\n",
       "\n",
       "10. 1. 'caret'\n",
       "2. 'lattice'\n",
       "3. 'ggplot2'\n",
       "4. 'pROC'\n",
       "5. 'randomForest'\n",
       "6. 'rpart.plot'\n",
       "7. 'rpart'\n",
       "8. 'dplyr'\n",
       "9. 'haven'\n",
       "10. 'party'\n",
       "11. 'strucchange'\n",
       "12. 'sandwich'\n",
       "13. 'zoo'\n",
       "14. 'modeltools'\n",
       "15. 'stats4'\n",
       "16. 'mvtnorm'\n",
       "17. 'grid'\n",
       "18. 'insight'\n",
       "19. 'synthpop'\n",
       "20. 'stats'\n",
       "21. 'graphics'\n",
       "22. 'grDevices'\n",
       "23. 'utils'\n",
       "24. 'datasets'\n",
       "25. 'methods'\n",
       "26. 'base'\n",
       "\n",
       "\n",
       "\n",
       "11. 1. 'pracma'\n",
       "2. 'caret'\n",
       "3. 'lattice'\n",
       "4. 'ggplot2'\n",
       "5. 'pROC'\n",
       "6. 'randomForest'\n",
       "7. 'rpart.plot'\n",
       "8. 'rpart'\n",
       "9. 'dplyr'\n",
       "10. 'haven'\n",
       "11. 'party'\n",
       "12. 'strucchange'\n",
       "13. 'sandwich'\n",
       "14. 'zoo'\n",
       "15. 'modeltools'\n",
       "16. 'stats4'\n",
       "17. 'mvtnorm'\n",
       "18. 'grid'\n",
       "19. 'insight'\n",
       "20. 'synthpop'\n",
       "21. 'stats'\n",
       "22. 'graphics'\n",
       "23. 'grDevices'\n",
       "24. 'utils'\n",
       "25. 'datasets'\n",
       "26. 'methods'\n",
       "27. 'base'\n",
       "\n",
       "\n",
       "\n",
       "12. 1. 'here'\n",
       "2. 'pracma'\n",
       "3. 'caret'\n",
       "4. 'lattice'\n",
       "5. 'ggplot2'\n",
       "6. 'pROC'\n",
       "7. 'randomForest'\n",
       "8. 'rpart.plot'\n",
       "9. 'rpart'\n",
       "10. 'dplyr'\n",
       "11. 'haven'\n",
       "12. 'party'\n",
       "13. 'strucchange'\n",
       "14. 'sandwich'\n",
       "15. 'zoo'\n",
       "16. 'modeltools'\n",
       "17. 'stats4'\n",
       "18. 'mvtnorm'\n",
       "19. 'grid'\n",
       "20. 'insight'\n",
       "21. 'synthpop'\n",
       "22. 'stats'\n",
       "23. 'graphics'\n",
       "24. 'grDevices'\n",
       "25. 'utils'\n",
       "26. 'datasets'\n",
       "27. 'methods'\n",
       "28. 'base'\n",
       "\n",
       "\n",
       "\n",
       "13. 1. 'Hmisc'\n",
       "2. 'here'\n",
       "3. 'pracma'\n",
       "4. 'caret'\n",
       "5. 'lattice'\n",
       "6. 'ggplot2'\n",
       "7. 'pROC'\n",
       "8. 'randomForest'\n",
       "9. 'rpart.plot'\n",
       "10. 'rpart'\n",
       "11. 'dplyr'\n",
       "12. 'haven'\n",
       "13. 'party'\n",
       "14. 'strucchange'\n",
       "15. 'sandwich'\n",
       "16. 'zoo'\n",
       "17. 'modeltools'\n",
       "18. 'stats4'\n",
       "19. 'mvtnorm'\n",
       "20. 'grid'\n",
       "21. 'insight'\n",
       "22. 'synthpop'\n",
       "23. 'stats'\n",
       "24. 'graphics'\n",
       "25. 'grDevices'\n",
       "26. 'utils'\n",
       "27. 'datasets'\n",
       "28. 'methods'\n",
       "29. 'base'\n",
       "\n",
       "\n",
       "\n",
       "14. 1. 'purrr'\n",
       "2. 'Hmisc'\n",
       "3. 'here'\n",
       "4. 'pracma'\n",
       "5. 'caret'\n",
       "6. 'lattice'\n",
       "7. 'ggplot2'\n",
       "8. 'pROC'\n",
       "9. 'randomForest'\n",
       "10. 'rpart.plot'\n",
       "11. 'rpart'\n",
       "12. 'dplyr'\n",
       "13. 'haven'\n",
       "14. 'party'\n",
       "15. 'strucchange'\n",
       "16. 'sandwich'\n",
       "17. 'zoo'\n",
       "18. 'modeltools'\n",
       "19. 'stats4'\n",
       "20. 'mvtnorm'\n",
       "21. 'grid'\n",
       "22. 'insight'\n",
       "23. 'synthpop'\n",
       "24. 'stats'\n",
       "25. 'graphics'\n",
       "26. 'grDevices'\n",
       "27. 'utils'\n",
       "28. 'datasets'\n",
       "29. 'methods'\n",
       "30. 'base'\n",
       "\n",
       "\n",
       "\n",
       "15. 1. 'purrr'\n",
       "2. 'Hmisc'\n",
       "3. 'here'\n",
       "4. 'pracma'\n",
       "5. 'caret'\n",
       "6. 'lattice'\n",
       "7. 'ggplot2'\n",
       "8. 'pROC'\n",
       "9. 'randomForest'\n",
       "10. 'rpart.plot'\n",
       "11. 'rpart'\n",
       "12. 'dplyr'\n",
       "13. 'haven'\n",
       "14. 'party'\n",
       "15. 'strucchange'\n",
       "16. 'sandwich'\n",
       "17. 'zoo'\n",
       "18. 'modeltools'\n",
       "19. 'stats4'\n",
       "20. 'mvtnorm'\n",
       "21. 'grid'\n",
       "22. 'insight'\n",
       "23. 'synthpop'\n",
       "24. 'stats'\n",
       "25. 'graphics'\n",
       "26. 'grDevices'\n",
       "27. 'utils'\n",
       "28. 'datasets'\n",
       "29. 'methods'\n",
       "30. 'base'\n",
       "\n",
       "\n",
       "\n",
       "16. 1. 'purrr'\n",
       "2. 'Hmisc'\n",
       "3. 'here'\n",
       "4. 'pracma'\n",
       "5. 'caret'\n",
       "6. 'lattice'\n",
       "7. 'ggplot2'\n",
       "8. 'pROC'\n",
       "9. 'randomForest'\n",
       "10. 'rpart.plot'\n",
       "11. 'rpart'\n",
       "12. 'dplyr'\n",
       "13. 'haven'\n",
       "14. 'party'\n",
       "15. 'strucchange'\n",
       "16. 'sandwich'\n",
       "17. 'zoo'\n",
       "18. 'modeltools'\n",
       "19. 'stats4'\n",
       "20. 'mvtnorm'\n",
       "21. 'grid'\n",
       "22. 'insight'\n",
       "23. 'synthpop'\n",
       "24. 'stats'\n",
       "25. 'graphics'\n",
       "26. 'grDevices'\n",
       "27. 'utils'\n",
       "28. 'datasets'\n",
       "29. 'methods'\n",
       "30. 'base'\n",
       "\n",
       "\n",
       "\n",
       "17. 1. 'ranger'\n",
       "2. 'purrr'\n",
       "3. 'Hmisc'\n",
       "4. 'here'\n",
       "5. 'pracma'\n",
       "6. 'caret'\n",
       "7. 'lattice'\n",
       "8. 'ggplot2'\n",
       "9. 'pROC'\n",
       "10. 'randomForest'\n",
       "11. 'rpart.plot'\n",
       "12. 'rpart'\n",
       "13. 'dplyr'\n",
       "14. 'haven'\n",
       "15. 'party'\n",
       "16. 'strucchange'\n",
       "17. 'sandwich'\n",
       "18. 'zoo'\n",
       "19. 'modeltools'\n",
       "20. 'stats4'\n",
       "21. 'mvtnorm'\n",
       "22. 'grid'\n",
       "23. 'insight'\n",
       "24. 'synthpop'\n",
       "25. 'stats'\n",
       "26. 'graphics'\n",
       "27. 'grDevices'\n",
       "28. 'utils'\n",
       "29. 'datasets'\n",
       "30. 'methods'\n",
       "31. 'base'\n",
       "\n",
       "\n",
       "\n",
       "18. 1. 'xgboost'\n",
       "2. 'ranger'\n",
       "3. 'purrr'\n",
       "4. 'Hmisc'\n",
       "5. 'here'\n",
       "6. 'pracma'\n",
       "7. 'caret'\n",
       "8. 'lattice'\n",
       "9. 'ggplot2'\n",
       "10. 'pROC'\n",
       "11. 'randomForest'\n",
       "12. 'rpart.plot'\n",
       "13. 'rpart'\n",
       "14. 'dplyr'\n",
       "15. 'haven'\n",
       "16. 'party'\n",
       "17. 'strucchange'\n",
       "18. 'sandwich'\n",
       "19. 'zoo'\n",
       "20. 'modeltools'\n",
       "21. 'stats4'\n",
       "22. 'mvtnorm'\n",
       "23. 'grid'\n",
       "24. 'insight'\n",
       "25. 'synthpop'\n",
       "26. 'stats'\n",
       "27. 'graphics'\n",
       "28. 'grDevices'\n",
       "29. 'utils'\n",
       "30. 'datasets'\n",
       "31. 'methods'\n",
       "32. 'base'\n",
       "\n",
       "\n",
       "\n",
       "19. 1. 'data.table'\n",
       "2. 'xgboost'\n",
       "3. 'ranger'\n",
       "4. 'purrr'\n",
       "5. 'Hmisc'\n",
       "6. 'here'\n",
       "7. 'pracma'\n",
       "8. 'caret'\n",
       "9. 'lattice'\n",
       "10. 'ggplot2'\n",
       "11. 'pROC'\n",
       "12. 'randomForest'\n",
       "13. 'rpart.plot'\n",
       "14. 'rpart'\n",
       "15. 'dplyr'\n",
       "16. 'haven'\n",
       "17. 'party'\n",
       "18. 'strucchange'\n",
       "19. 'sandwich'\n",
       "20. 'zoo'\n",
       "21. 'modeltools'\n",
       "22. 'stats4'\n",
       "23. 'mvtnorm'\n",
       "24. 'grid'\n",
       "25. 'insight'\n",
       "26. 'synthpop'\n",
       "27. 'stats'\n",
       "28. 'graphics'\n",
       "29. 'grDevices'\n",
       "30. 'utils'\n",
       "31. 'datasets'\n",
       "32. 'methods'\n",
       "33. 'base'\n",
       "\n",
       "\n",
       "\n",
       "20. 1. 'tensorflow'\n",
       "2. 'data.table'\n",
       "3. 'xgboost'\n",
       "4. 'ranger'\n",
       "5. 'purrr'\n",
       "6. 'Hmisc'\n",
       "7. 'here'\n",
       "8. 'pracma'\n",
       "9. 'caret'\n",
       "10. 'lattice'\n",
       "11. 'ggplot2'\n",
       "12. 'pROC'\n",
       "13. 'randomForest'\n",
       "14. 'rpart.plot'\n",
       "15. 'rpart'\n",
       "16. 'dplyr'\n",
       "17. 'haven'\n",
       "18. 'party'\n",
       "19. 'strucchange'\n",
       "20. 'sandwich'\n",
       "21. 'zoo'\n",
       "22. 'modeltools'\n",
       "23. 'stats4'\n",
       "24. 'mvtnorm'\n",
       "25. 'grid'\n",
       "26. 'insight'\n",
       "27. 'synthpop'\n",
       "28. 'stats'\n",
       "29. 'graphics'\n",
       "30. 'grDevices'\n",
       "31. 'utils'\n",
       "32. 'datasets'\n",
       "33. 'methods'\n",
       "34. 'base'\n",
       "\n",
       "\n",
       "\n",
       "21. 1. 'keras'\n",
       "2. 'tensorflow'\n",
       "3. 'data.table'\n",
       "4. 'xgboost'\n",
       "5. 'ranger'\n",
       "6. 'purrr'\n",
       "7. 'Hmisc'\n",
       "8. 'here'\n",
       "9. 'pracma'\n",
       "10. 'caret'\n",
       "11. 'lattice'\n",
       "12. 'ggplot2'\n",
       "13. 'pROC'\n",
       "14. 'randomForest'\n",
       "15. 'rpart.plot'\n",
       "16. 'rpart'\n",
       "17. 'dplyr'\n",
       "18. 'haven'\n",
       "19. 'party'\n",
       "20. 'strucchange'\n",
       "21. 'sandwich'\n",
       "22. 'zoo'\n",
       "23. 'modeltools'\n",
       "24. 'stats4'\n",
       "25. 'mvtnorm'\n",
       "26. 'grid'\n",
       "27. 'insight'\n",
       "28. 'synthpop'\n",
       "29. 'stats'\n",
       "30. 'graphics'\n",
       "31. 'grDevices'\n",
       "32. 'utils'\n",
       "33. 'datasets'\n",
       "34. 'methods'\n",
       "35. 'base'\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[[1]]\n",
       "[1] \"synthpop\"  \"stats\"     \"graphics\"  \"grDevices\" \"utils\"     \"datasets\" \n",
       "[7] \"methods\"   \"base\"     \n",
       "\n",
       "[[2]]\n",
       "[1] \"insight\"   \"synthpop\"  \"stats\"     \"graphics\"  \"grDevices\" \"utils\"    \n",
       "[7] \"datasets\"  \"methods\"   \"base\"     \n",
       "\n",
       "[[3]]\n",
       " [1] \"party\"       \"strucchange\" \"sandwich\"    \"zoo\"         \"modeltools\" \n",
       " [6] \"stats4\"      \"mvtnorm\"     \"grid\"        \"insight\"     \"synthpop\"   \n",
       "[11] \"stats\"       \"graphics\"    \"grDevices\"   \"utils\"       \"datasets\"   \n",
       "[16] \"methods\"     \"base\"       \n",
       "\n",
       "[[4]]\n",
       " [1] \"haven\"       \"party\"       \"strucchange\" \"sandwich\"    \"zoo\"        \n",
       " [6] \"modeltools\"  \"stats4\"      \"mvtnorm\"     \"grid\"        \"insight\"    \n",
       "[11] \"synthpop\"    \"stats\"       \"graphics\"    \"grDevices\"   \"utils\"      \n",
       "[16] \"datasets\"    \"methods\"     \"base\"       \n",
       "\n",
       "[[5]]\n",
       " [1] \"dplyr\"       \"haven\"       \"party\"       \"strucchange\" \"sandwich\"   \n",
       " [6] \"zoo\"         \"modeltools\"  \"stats4\"      \"mvtnorm\"     \"grid\"       \n",
       "[11] \"insight\"     \"synthpop\"    \"stats\"       \"graphics\"    \"grDevices\"  \n",
       "[16] \"utils\"       \"datasets\"    \"methods\"     \"base\"       \n",
       "\n",
       "[[6]]\n",
       " [1] \"rpart\"       \"dplyr\"       \"haven\"       \"party\"       \"strucchange\"\n",
       " [6] \"sandwich\"    \"zoo\"         \"modeltools\"  \"stats4\"      \"mvtnorm\"    \n",
       "[11] \"grid\"        \"insight\"     \"synthpop\"    \"stats\"       \"graphics\"   \n",
       "[16] \"grDevices\"   \"utils\"       \"datasets\"    \"methods\"     \"base\"       \n",
       "\n",
       "[[7]]\n",
       " [1] \"rpart.plot\"  \"rpart\"       \"dplyr\"       \"haven\"       \"party\"      \n",
       " [6] \"strucchange\" \"sandwich\"    \"zoo\"         \"modeltools\"  \"stats4\"     \n",
       "[11] \"mvtnorm\"     \"grid\"        \"insight\"     \"synthpop\"    \"stats\"      \n",
       "[16] \"graphics\"    \"grDevices\"   \"utils\"       \"datasets\"    \"methods\"    \n",
       "[21] \"base\"       \n",
       "\n",
       "[[8]]\n",
       " [1] \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"        \"haven\"       \n",
       " [6] \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"          \"modeltools\"  \n",
       "[11] \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"      \"synthpop\"    \n",
       "[16] \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"        \"datasets\"    \n",
       "[21] \"methods\"      \"base\"        \n",
       "\n",
       "[[9]]\n",
       " [1] \"pROC\"         \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"       \n",
       " [6] \"haven\"        \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"         \n",
       "[11] \"modeltools\"   \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"     \n",
       "[16] \"synthpop\"     \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"       \n",
       "[21] \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[10]]\n",
       " [1] \"caret\"        \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\"\n",
       " [6] \"rpart.plot\"   \"rpart\"        \"dplyr\"        \"haven\"        \"party\"       \n",
       "[11] \"strucchange\"  \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"      \n",
       "[16] \"mvtnorm\"      \"grid\"         \"insight\"      \"synthpop\"     \"stats\"       \n",
       "[21] \"graphics\"     \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"     \n",
       "[26] \"base\"        \n",
       "\n",
       "[[11]]\n",
       " [1] \"pracma\"       \"caret\"        \"lattice\"      \"ggplot2\"      \"pROC\"        \n",
       " [6] \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"        \"haven\"       \n",
       "[11] \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"          \"modeltools\"  \n",
       "[16] \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"      \"synthpop\"    \n",
       "[21] \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"        \"datasets\"    \n",
       "[26] \"methods\"      \"base\"        \n",
       "\n",
       "[[12]]\n",
       " [1] \"here\"         \"pracma\"       \"caret\"        \"lattice\"      \"ggplot2\"     \n",
       " [6] \"pROC\"         \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"       \n",
       "[11] \"haven\"        \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"         \n",
       "[16] \"modeltools\"   \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"     \n",
       "[21] \"synthpop\"     \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"       \n",
       "[26] \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[13]]\n",
       " [1] \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"        \"lattice\"     \n",
       " [6] \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"   \"rpart\"       \n",
       "[11] \"dplyr\"        \"haven\"        \"party\"        \"strucchange\"  \"sandwich\"    \n",
       "[16] \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"      \"grid\"        \n",
       "[21] \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"     \"grDevices\"   \n",
       "[26] \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[14]]\n",
       " [1] \"purrr\"        \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"       \n",
       " [6] \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"  \n",
       "[11] \"rpart\"        \"dplyr\"        \"haven\"        \"party\"        \"strucchange\" \n",
       "[16] \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"     \n",
       "[21] \"grid\"         \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"    \n",
       "[26] \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[15]]\n",
       " [1] \"purrr\"        \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"       \n",
       " [6] \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"  \n",
       "[11] \"rpart\"        \"dplyr\"        \"haven\"        \"party\"        \"strucchange\" \n",
       "[16] \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"     \n",
       "[21] \"grid\"         \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"    \n",
       "[26] \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[16]]\n",
       " [1] \"purrr\"        \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"       \n",
       " [6] \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"  \n",
       "[11] \"rpart\"        \"dplyr\"        \"haven\"        \"party\"        \"strucchange\" \n",
       "[16] \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"     \n",
       "[21] \"grid\"         \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"    \n",
       "[26] \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[17]]\n",
       " [1] \"ranger\"       \"purrr\"        \"Hmisc\"        \"here\"         \"pracma\"      \n",
       " [6] \"caret\"        \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\"\n",
       "[11] \"rpart.plot\"   \"rpart\"        \"dplyr\"        \"haven\"        \"party\"       \n",
       "[16] \"strucchange\"  \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"      \n",
       "[21] \"mvtnorm\"      \"grid\"         \"insight\"      \"synthpop\"     \"stats\"       \n",
       "[26] \"graphics\"     \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"     \n",
       "[31] \"base\"        \n",
       "\n",
       "[[18]]\n",
       " [1] \"xgboost\"      \"ranger\"       \"purrr\"        \"Hmisc\"        \"here\"        \n",
       " [6] \"pracma\"       \"caret\"        \"lattice\"      \"ggplot2\"      \"pROC\"        \n",
       "[11] \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"        \"haven\"       \n",
       "[16] \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"          \"modeltools\"  \n",
       "[21] \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"      \"synthpop\"    \n",
       "[26] \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"        \"datasets\"    \n",
       "[31] \"methods\"      \"base\"        \n",
       "\n",
       "[[19]]\n",
       " [1] \"data.table\"   \"xgboost\"      \"ranger\"       \"purrr\"        \"Hmisc\"       \n",
       " [6] \"here\"         \"pracma\"       \"caret\"        \"lattice\"      \"ggplot2\"     \n",
       "[11] \"pROC\"         \"randomForest\" \"rpart.plot\"   \"rpart\"        \"dplyr\"       \n",
       "[16] \"haven\"        \"party\"        \"strucchange\"  \"sandwich\"     \"zoo\"         \n",
       "[21] \"modeltools\"   \"stats4\"       \"mvtnorm\"      \"grid\"         \"insight\"     \n",
       "[26] \"synthpop\"     \"stats\"        \"graphics\"     \"grDevices\"    \"utils\"       \n",
       "[31] \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[20]]\n",
       " [1] \"tensorflow\"   \"data.table\"   \"xgboost\"      \"ranger\"       \"purrr\"       \n",
       " [6] \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"        \"lattice\"     \n",
       "[11] \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"   \"rpart\"       \n",
       "[16] \"dplyr\"        \"haven\"        \"party\"        \"strucchange\"  \"sandwich\"    \n",
       "[21] \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"      \"grid\"        \n",
       "[26] \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"     \"grDevices\"   \n",
       "[31] \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n",
       "\n",
       "[[21]]\n",
       " [1] \"keras\"        \"tensorflow\"   \"data.table\"   \"xgboost\"      \"ranger\"      \n",
       " [6] \"purrr\"        \"Hmisc\"        \"here\"         \"pracma\"       \"caret\"       \n",
       "[11] \"lattice\"      \"ggplot2\"      \"pROC\"         \"randomForest\" \"rpart.plot\"  \n",
       "[16] \"rpart\"        \"dplyr\"        \"haven\"        \"party\"        \"strucchange\" \n",
       "[21] \"sandwich\"     \"zoo\"          \"modeltools\"   \"stats4\"       \"mvtnorm\"     \n",
       "[26] \"grid\"         \"insight\"      \"synthpop\"     \"stats\"        \"graphics\"    \n",
       "[31] \"grDevices\"    \"utils\"        \"datasets\"     \"methods\"      \"base\"        \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set the library path\n",
    "#.libPaths(\"/user/emma.foessing01/u11969/new_R_libs\")\n",
    "Sys.setenv(\"PKG_CXXFLAGS\"=\"-std=c++14\")\n",
    "\n",
    "# List of required packages\n",
    "list_of_packages <- c(\n",
    "  \"synthpop\", \"jsonlite\", \"codetools\", \"insight\", \"party\", \"haven\", \"dplyr\", \"rpart\", \"rpart.plot\",\n",
    "  \"randomForest\", \"pROC\", \"caret\", \"pracma\", \"here\", \"Hmisc\", \"purrr\",\n",
    "  \"ranger\", \"bnlearn\", \"arulesCBA\", \"network\", \"igraph\", \"xgboost\",\n",
    "  \"data.table\", \"RSNNS\", \"doParallel\", \"sparklyr\"\n",
    ")\n",
    "\n",
    "# Function to load packages and handle errors\n",
    "load_if_installed <- function(p) {\n",
    "  tryCatch({\n",
    "    library(p, character.only = TRUE)\n",
    "  }, error = function(e) {\n",
    "    message(sprintf(\"Package '%s' is not installed.\", p))\n",
    "  })\n",
    "}\n",
    "\n",
    "# Load all required packages\n",
    "lapply(list_of_packages, load_if_installed)\n",
    "\n",
    "# initialize Spark\n",
    "sc <- spark_connect(master = \"local\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "# set path\n",
    "############## adjust to correct directory!\n",
    "directory <- \"/user/emma.foessing01/u11969/Master-Thesis\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "load(file = (paste0(here(), \"/cpspop.RData\")))\n",
    "adult <- read.csv(file = paste0(here(),\"/adult_preprocessed.csv\"))\n",
    "# delete NAs\n",
    "adult[adult == \"?\"] <- NA\n",
    "adult <- na.omit(adult)\n",
    "\n",
    "adult$workclass <- as.factor(adult$workclass)\n",
    "adult$education <- as.factor(adult$education)\n",
    "adult$marital_status <- as.factor(adult$marital_status)\n",
    "adult$relationship <- as.factor(adult$relationship)\n",
    "adult$race <- as.factor(adult$race)\n",
    "adult$sex <- as.factor(adult$sex)\n",
    "adult$native_country <- as.factor(adult$native_country)\n",
    "adult$income <- as.factor(adult$income)\n",
    "adult$occupation <- as.factor(adult$occupation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "## eval cont targets\n",
    "evaluation_metrics_cont <- function(predictions, test_set){\n",
    "  MAE <- mean(abs(predictions - test_set$income))\n",
    "  MSE <- mean((predictions - test_set$income)^2)\n",
    "  RMSE <- sqrt(MSE)\n",
    "\n",
    "  SS_res <- sum((test_set$income - predictions)^2)\n",
    "  SS_tot <- sum((test_set$income - mean(test_set$income))^2)\n",
    "  R_squared <- 1 - (SS_res / SS_tot)\n",
    "\n",
    "  MAPE <- mean(abs((test_set$income - predictions) / test_set$income)) * 100\n",
    "\n",
    "  # Create the dataframe\n",
    "  metrics_df <- data.frame(\n",
    "        MAE = MAE, \n",
    "        MSE = MSE, \n",
    "        RMSE = RMSE,\n",
    "        R_squared = R_squared, \n",
    "        MAPE = MAPE)\n",
    "\n",
    "  return(metrics_df)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "## eval factored targets\n",
    "evaluation_metrics_factor <- function(predictions, test_set){\n",
    "    # confusion matrix for the prediction on original data\n",
    "    cm <- caret::confusionMatrix(predictions, test_set$income,\n",
    "                mode = \"everything\")\n",
    "\n",
    "    # saving evaluation metrics\n",
    "    accuracy <- cm$overall['Accuracy']\n",
    "    f1 <- cm$byClass['F1']\n",
    "    sens <- cm$byClass['Sensitivity']\n",
    "    spec <- cm$byClass['Specificity']\n",
    "\n",
    "    # Create the dataframe\n",
    "    metrics_df <- data.frame(\n",
    "        Accuracy = accuracy, \n",
    "        F1 = f1, \n",
    "        Sensitivity = sens, \n",
    "        Specificity = spec\n",
    "    )\n",
    "    \n",
    "    return(metrics_df)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "rf_pred <- function(data, outer_folds, mtry_steps, ntree_steps, inner_folds) {\n",
    "    # Adjust evaluation metric to fit both numeric and factored targets\n",
    "    summaryFunctionType <- if (is.numeric(data$income)) defaultSummary else multiClassSummary\n",
    "    # Metric: train() uses per default RSME and Accuracy for numeric and factored targets\n",
    "\n",
    "    # Set control args\n",
    "    outer_control <- trainControl(method = \"cv\", number = outer_folds,\n",
    "                                  summaryFunction = summaryFunctionType,\n",
    "                                  verboseIter = FALSE,\n",
    "                                  allowParallel = TRUE)\n",
    "        \n",
    "    inner_control <- trainControl(method = \"cv\", number = inner_folds, \n",
    "                                  summaryFunction = summaryFunctionType,\n",
    "                                  verboseIter = FALSE,\n",
    "                                  allowParallel = TRUE)\n",
    "\n",
    "    # Define the parameter grid for tuning\n",
    "    splitrule_value <- if (is.numeric(data$income)) \"variance\" else \"gini\"\n",
    "    \n",
    "    tunegrid <- expand.grid(mtry = seq(2, ncol(data) - 1, length.out = mtry_steps),\n",
    "                            splitrule = splitrule_value,\n",
    "                            min.node.size = 5)\n",
    "    # You can incorporate ntree_steps into the grid if needed\n",
    "\n",
    "    # Initialize variables to store results\n",
    "    outer_cv_folds <- createFolds(data$income, k = outer_folds)\n",
    "\n",
    "    # Initialize variables to store results\n",
    "    outer_results <- list()\n",
    "\n",
    "    outer_cv_folds <- createFolds(data$income, k = outer_folds)\n",
    "    \n",
    "    # Outer loop: Cross-validation for model evaluation\n",
    "    for (i in seq_along(outer_cv_folds)) {\n",
    "        # Split data into outer folds\n",
    "        outer_test_index <- outer_cv_folds[[i]]\n",
    "        outer_testData <- data[outer_test_index, ]\n",
    "        outer_trainData <- data[-outer_test_index, ]\n",
    "        \n",
    "        # Hyperparameter tuning using inner CV\n",
    "        model <- caret::train(income ~ ., \n",
    "                              data = outer_trainData, \n",
    "                              method = \"ranger\",  \n",
    "                              tuneGrid = tunegrid, \n",
    "                              trControl = inner_control)\n",
    "\n",
    "        # Store the best hyperparameters\n",
    "        best_hyperparameters <- model$bestTune\n",
    "\n",
    "        # Train the final model on the outer training set with the best hyperparameters\n",
    "        final_model <- caret::train(income ~ ., \n",
    "                                    data = outer_trainData, \n",
    "                                    method = \"ranger\", \n",
    "                                    trControl = outer_control, \n",
    "                                    tuneGrid = best_hyperparameters)\n",
    "\n",
    "        # Testing the final model on the outer test set\n",
    "        predictions <- predict(final_model, newdata = outer_testData)\n",
    "        \n",
    "        if (is.numeric(data$income)) {\n",
    "            eval <- evaluation_metrics_cont(predictions, outer_testData)\n",
    "        } else if (is.factor(data$income)) {\n",
    "            eval <- evaluation_metrics_factor(predictions, outer_testData)\n",
    "        } else {\n",
    "            stop(\"The predicted target has to be numeric or factor.\")\n",
    "        }\n",
    "\n",
    "        # Store the evaluation metrics for this outer fold\n",
    "        outer_results[[i]] <- eval\n",
    "    }\n",
    "\n",
    "    # Average the evaluation metrics over the outer folds\n",
    "    eval_avg_outer_folds <- do.call(rbind, outer_results) %>%\n",
    "                            summarise(across(everything(), mean, na.rm = TRUE))\n",
    "\n",
    "    # Return the average evaluation metrics\n",
    "    return(eval_avg_outer_folds)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "adult_res <- rf_pred(adult, 5, 5, 5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in ranger::ranger(dependent.variable.name = \".outcome\", data = x, :\n",
      "\"Unused arguments: control\"\n",
      "Warning message in ranger::ranger(dependent.variable.name = \".outcome\", data = x, :\n",
      "\"Unused arguments: control\"\n",
      "Warning message in ranger::ranger(dependent.variable.name = \".outcome\", data = x, :\n",
      "\"Unused arguments: control\"\n",
      "Warning message:\n",
      "\"model fit failed for Fold1: mtry=5.0, splitrule=variance, min.node.size=5 Error in ranger::ranger(dependent.variable.name = \".outcome\", data = x,  : \n",
      "  User interrupt or internal error.\n",
      "\"\n",
      "Warning message in ranger::ranger(dependent.variable.name = \".outcome\", data = x, :\n",
      "\"Unused arguments: control\"\n",
      "Warning message:\n",
      "\"model fit failed for Fold1: mtry=6.5, splitrule=variance, min.node.size=5 Error in ranger::ranger(dependent.variable.name = \".outcome\", data = x,  : \n",
      "  User interrupt or internal error.\n",
      "\"\n",
      "Warning message in ranger::ranger(dependent.variable.name = \".outcome\", data = x, :\n",
      "\"Unused arguments: control\"\n"
     ]
    }
   ],
   "source": [
    "cps_res <- rf_pred(cpspop, 5, 5, 5, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "r"
    }
   },
   "outputs": [],
   "source": [
    "# Bind results\n",
    "rf_pred_results <- list(cps_res = cps_res, adult_res = adult_res)\n",
    "# Save the results to an RData file \n",
    "save(rf_pred_results, file = paste0(directory, \"/results/rf_pred_results.RData\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "4.4.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}