T12d_Hadoop - Simple Spark script for Test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b54fa2-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55146-495a-11e8-a769-0242ac130002",
     "previous": null
    }
   },
   "source": [
    "# About: Simple Spark script for Test\n",
    "\n",
    "---\n",
    "\n",
    "Sparkの動作確認をしてみる。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55146-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b552b8-495a-11e8-a769-0242ac130002",
     "previous": "98b54fa2-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "## *Operation Note*\n",
    "\n",
    "*This is a cell for your own recording.  ここに経緯を記述*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b552b8-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55420-495a-11e8-a769-0242ac130002",
     "previous": "98b55146-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "# Notebookと環境のBinding\n",
    "\n",
    "Inventory中のgroup名でBind対象を指示する。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55420-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b5557e-495a-11e8-a769-0242ac130002",
     "previous": "98b552b8-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": [
    "target_group = 'hadoop_all_testcluster'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b5557e-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b556dc-495a-11e8-a769-0242ac130002",
     "previous": "98b55420-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "コマンドの動作確認。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b556dc-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55826-495a-11e8-a769-0242ac130002",
     "previous": "98b5557e-495a-11e8-a769-0242ac130002"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;32mXXX.XXX.XXX.72 | SUCCESS | rc=0 >>\r\n",
      "Usage: spark-submit [options] <app jar | python file> [app arguments]\r\n",
      "Usage: spark-submit --kill [submission ID] --master [spark://...]\r\n",
      "Usage: spark-submit --status [submission ID] --master [spark://...]\r\n",
      "\r\n",
      "Options:\r\n",
      "  --master MASTER_URL         spark://host:port, mesos://host:port, yarn, or local.\r\n",
      "  --deploy-mode DEPLOY_MODE   Whether to launch the driver program locally (\"client\") or\r\n",
      "                              on one of the worker machines inside the cluster (\"cluster\")\r\n",
      "                              (Default: client).\r\n",
      "  --class CLASS_NAME          Your application's main class (for Java / Scala apps).\r\n",
      "  --name NAME                 A name of your application.\r\n",
      "  --jars JARS                 Comma-separated list of local jars to include on the driver\r\n",
      "                              and executor classpaths.\r\n",
      "  --packages                  Comma-separated list of maven coordinates of jars to include\r\n",
      "                              on the driver and executor classpaths. Will search the local\r\n",
      "                              maven repo, then maven central and any additional remote\r\n",
      "                              repositories given by --repositories. The format for the\r\n",
      "                              coordinates should be groupId:artifactId:version.\r\n",
      "  --exclude-packages          Comma-separated list of groupId:artifactId, to exclude while\r\n",
      "                              resolving the dependencies provided in --packages to avoid\r\n",
      "                              dependency conflicts.\r\n",
      "  --repositories              Comma-separated list of additional remote repositories to\r\n",
      "                              search for the maven coordinates given with --packages.\r\n",
      "  --py-files PY_FILES         Comma-separated list of .zip, .egg, or .py files to place\r\n",
      "                              on the PYTHONPATH for Python apps.\r\n",
      "  --files FILES               Comma-separated list of files to be placed in the working\r\n",
      "                              directory of each executor.\r\n",
      "\r\n",
      "  --conf PROP=VALUE           Arbitrary Spark configuration property.\r\n",
      "  --properties-file FILE      Path to a file from which to load extra properties. If not\r\n",
      "                              specified, this will look for conf/spark-defaults.conf.\r\n",
      "\r\n",
      "  --driver-memory MEM         Memory for driver (e.g. 1000M, 2G) (Default: 1024M).\r\n",
      "  --driver-java-options       Extra Java options to pass to the driver.\r\n",
      "  --driver-library-path       Extra library path entries to pass to the driver.\r\n",
      "  --driver-class-path         Extra class path entries to pass to the driver. Note that\r\n",
      "                              jars added with --jars are automatically included in the\r\n",
      "                              classpath.\r\n",
      "\r\n",
      "  --executor-memory MEM       Memory per executor (e.g. 1000M, 2G) (Default: 1G).\r\n",
      "\r\n",
      "  --proxy-user NAME           User to impersonate when submitting the application.\r\n",
      "                              This argument does not work with --principal / --keytab.\r\n",
      "\r\n",
      "  --help, -h                  Show this help message and exit\r\n",
      "  --verbose, -v               Print additional debug output\r\n",
      "  --version,                  Print the version of current Spark\r\n",
      "\r\n",
      " Spark standalone with cluster deploy mode only:\r\n",
      "  --driver-cores NUM          Cores for driver (Default: 1).\r\n",
      "\r\n",
      " Spark standalone or Mesos with cluster deploy mode only:\r\n",
      "  --supervise                 If given, restarts the driver on failure.\r\n",
      "  --kill SUBMISSION_ID        If given, kills the driver specified.\r\n",
      "  --status SUBMISSION_ID      If given, requests the status of the driver specified.\r\n",
      "\r\n",
      " Spark standalone and Mesos only:\r\n",
      "  --total-executor-cores NUM  Total cores for all executors.\r\n",
      "\r\n",
      " Spark standalone and YARN only:\r\n",
      "  --executor-cores NUM        Number of cores per executor. (Default: 1 in YARN mode,\r\n",
      "                              or all available cores on the worker in standalone mode)\r\n",
      "\r\n",
      " YARN-only:\r\n",
      "  --driver-cores NUM          Number of cores used by the driver, only in cluster mode\r\n",
      "                              (Default: 1).\r\n",
      "  --queue QUEUE_NAME          The YARN queue to submit to (Default: \"default\").\r\n",
      "  --num-executors NUM         Number of executors to launch (Default: 2).\r\n",
      "  --archives ARCHIVES         Comma separated list of archives to be extracted into the\r\n",
      "                              working directory of each executor.\r\n",
      "  --principal PRINCIPAL       Principal to be used to login to KDC, while running on\r\n",
      "                              secure HDFS.\r\n",
      "  --keytab KEYTAB             The full path to the file that contains the keytab for the\r\n",
      "                              principal specified above. This keytab will be copied to\r\n",
      "                              the node running the Application Master via the Secure\r\n",
      "                              Distributed Cache, for renewing the login tickets and the\r\n",
      "                              delegation tokens periodically.\r\n",
      "      \r\n",
      "\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!ansible -a 'spark-submit --help' hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "lc_cell_meme": {
     "current": "98b55826-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b5597a-495a-11e8-a769-0242ac130002",
     "previous": "98b556dc-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "# Datasetの準備"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b5597a-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55ace-495a-11e8-a769-0242ac130002",
     "previous": "98b55826-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/tmp/tmpDGame6'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tempfile\n",
    "work_dir = tempfile.mkdtemp()\n",
    "work_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55ace-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55c22-495a-11e8-a769-0242ac130002",
     "previous": "98b5597a-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WORK_DIR=/tmp/tmpDGame6\n"
     ]
    }
   ],
   "source": [
    "%env WORK_DIR={work_dir}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55c22-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55d76-495a-11e8-a769-0242ac130002",
     "previous": "98b55ace-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "今回は適当に・・・Iris Data Setを読み込む。 [Iris Data Set - UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/Iris)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55d76-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b55ec0-495a-11e8-a769-0242ac130002",
     "previous": "98b55c22-495a-11e8-a769-0242ac130002"
    },
    "run_control": {
     "marked": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "\r",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r",
      "100  4551  100  4551    0     0   8250      0 --:--:-- --:--:-- --:--:--  8259\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cd ${WORK_DIR}\n",
    "curl -O https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b55ec0-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b5601e-495a-11e8-a769-0242ac130002",
     "previous": "98b55d76-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.1,3.5,1.4,0.2,Iris-setosa\r\n",
      "4.9,3.0,1.4,0.2,Iris-setosa\r\n",
      "4.7,3.2,1.3,0.2,Iris-setosa\r\n",
      "4.6,3.1,1.5,0.2,Iris-setosa\r\n",
      "5.0,3.6,1.4,0.2,Iris-setosa\r\n",
      "5.4,3.9,1.7,0.4,Iris-setosa\r\n",
      "4.6,3.4,1.4,0.3,Iris-setosa\r\n",
      "5.0,3.4,1.5,0.2,Iris-setosa\r\n",
      "4.4,2.9,1.4,0.2,Iris-setosa\r\n",
      "4.9,3.1,1.5,0.1,Iris-setosa\r\n"
     ]
    }
   ],
   "source": [
    "!head {work_dir}/iris.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b5601e-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56172-495a-11e8-a769-0242ac130002",
     "previous": "98b55ec0-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "データをHDFSにアップロードする。\n",
    "\n",
    "まずこのNotebook環境の一時ディレクトリから `hadoop_client` マシンにコピーしてから、 `hadoop fs` コマンドでコピーする。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "lc_cell_meme": {
     "current": "98b56172-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b562bc-495a-11e8-a769-0242ac130002",
     "previous": "98b5601e-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": [
    "hdfs_dataset_dir = '/dataset/iris'\n",
    "hadoop_client_dir = '~/iris'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b562bc-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56406-495a-11e8-a769-0242ac130002",
     "previous": "98b56172-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "このNotebook環境から `hadoop_client` マシンへのコピー・・・"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56406-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b5655a-495a-11e8-a769-0242ac130002",
     "previous": "98b562bc-495a-11e8-a769-0242ac130002"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;33mXXX.XXX.XXX.72 | SUCCESS => {\r\n",
      "    \"changed\": true, \r\n",
      "    \"checksum\": \"d11777fac6574637a4ec5f0effeab8542ae88b65\", \r\n",
      "    \"dest\": \"/home/ansible/iris/iris.data\", \r\n",
      "    \"gid\": 500, \r\n",
      "    \"group\": \"ansible\", \r\n",
      "    \"md5sum\": \"42615765a885ddf54427f12c34a0a070\", \r\n",
      "    \"mode\": \"0664\", \r\n",
      "    \"owner\": \"ansible\", \r\n",
      "    \"size\": 4551, \r\n",
      "    \"src\": \"/home/ansible/.ansible/tmp/ansible-tmp-1472182654.06-124636273342266/source\", \r\n",
      "    \"state\": \"file\", \r\n",
      "    \"uid\": 500\r\n",
      "}\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!ansible -m copy -a 'src={work_dir}/iris.data dest={hadoop_client_dir}/' hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b5655a-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b566a4-495a-11e8-a769-0242ac130002",
     "previous": "98b56406-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "`hadoop fs` コマンドを実行する。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b566a4-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b567ee-495a-11e8-a769-0242ac130002",
     "previous": "98b5655a-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;32mXXX.XXX.XXX.72 | SUCCESS | rc=0 >>\r\n",
      "\r\n",
      "\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!ansible -m shell -a \"chdir={hadoop_client_dir} \\\n",
    "                      hadoop fs -mkdir -p {hdfs_dataset_dir} && \\\n",
    "                      hadoop fs -copyFromLocal iris.data {hdfs_dataset_dir}\" hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b567ee-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56942-495a-11e8-a769-0242ac130002",
     "previous": "98b566a4-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "念のため、HDFSにファイルがアップロードされていることを確認しておく。\n",
    "\n",
    "以下の `dataset_dir` ディレクトリに、 `iris.data` が作成されていればOK。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56942-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56a8c-495a-11e8-a769-0242ac130002",
     "previous": "98b567ee-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;32mXXX.XXX.XXX.72 | SUCCESS | rc=0 >>\r\n",
      "Found 1 items\r\n",
      "-rw-r--r--   3 ansible supergroup       4551 2016-08-26 12:37 /dataset/iris/iris.data\r\n",
      "\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!ansible -a 'hadoop fs -ls {hdfs_dataset_dir}' hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56a8c-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56be0-495a-11e8-a769-0242ac130002",
     "previous": "98b56942-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "データベースおよびテーブルを作成する。まずはHiveのクエリを作成する。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56be0-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56d2a-495a-11e8-a769-0242ac130002",
     "previous": "98b56a8c-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "# アプリケーションの実行\n",
    "\n",
    "サンプルアプリケーションを実行する。参考: http://spark.apache.org/docs/latest/quick-start.html#self-contained-applications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56d2a-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56e74-495a-11e8-a769-0242ac130002",
     "previous": "98b56be0-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56e74-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b56fbe-495a-11e8-a769-0242ac130002",
     "previous": "98b56d2a-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing /tmp/tmpDGame6/SimpleApp.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile {work_dir}/SimpleApp.py\n",
    "# -*- coding:utf-8 -*-\n",
    "\"\"\"SimpleApp.py\"\"\"\n",
    "from pyspark import SparkContext, SparkConf\n",
    "\n",
    "logFile = \"/dataset/iris/iris.data\"  # Should be some file on your system\n",
    "conf = SparkConf().setAppName('test-pyspark').setMaster('yarn-cluster')\n",
    "sc = SparkContext(conf=conf)\n",
    "logData = sc.textFile(logFile).cache()\n",
    "\n",
    "numAs = logData.filter(lambda s: 'a' in s).count()\n",
    "numBs = logData.filter(lambda s: 'b' in s).count()\n",
    "\n",
    "print(\"Lines with a: %i, lines with b: %i\" % (numAs, numBs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b56fbe-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57112-495a-11e8-a769-0242ac130002",
     "previous": "98b56e74-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "クエリを実行する。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b57112-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b5748c-495a-11e8-a769-0242ac130002",
     "previous": "98b56fbe-495a-11e8-a769-0242ac130002"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;33mXXX.XXX.XXX.72 | SUCCESS => {\n",
      "    \"changed\": true, \n",
      "    \"checksum\": \"68346c67549d133e8cd123f9e6dc94fd6c5a3e03\", \n",
      "    \"dest\": \"/home/ansible/iris/SimpleApp.py\", \n",
      "    \"gid\": 500, \n",
      "    \"group\": \"ansible\", \n",
      "    \"md5sum\": \"321e78f262bf625f5cdb840d3672f0cf\", \n",
      "    \"mode\": \"0664\", \n",
      "    \"owner\": \"ansible\", \n",
      "    \"size\": 465, \n",
      "    \"src\": \"/home/ansible/.ansible/tmp/ansible-tmp-1472182691.63-72101184265081/source\", \n",
      "    \"state\": \"file\", \n",
      "    \"uid\": 500\n",
      "}\u001b[0m\n",
      "\u001b[0;32mXXX.XXX.XXX.72 | SUCCESS | rc=0 >>\n",
      "16/08/26 12:38:15 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "16/08/26 12:38:17 INFO TimelineClientImpl: Timeline service address: http://testvm003:8188/ws/v1/timeline/\n",
      "16/08/26 12:38:17 INFO Client: Requesting a new application from cluster with 4 NodeManagers\n",
      "16/08/26 12:38:17 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (4096 MB per container)\n",
      "16/08/26 12:38:17 INFO Client: Will allocate AM container, with 1408 MB memory including 384 MB overhead\n",
      "16/08/26 12:38:17 INFO Client: Setting up container launch context for our AM\n",
      "16/08/26 12:38:17 INFO Client: Setting up the launch environment for our AM container\n",
      "16/08/26 12:38:18 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\n",
      "16/08/26 12:38:18 INFO Client: Preparing resources for our AM container\n",
      "16/08/26 12:38:18 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\n",
      "16/08/26 12:38:18 INFO Client: Uploading resource file:/usr/hdp/XXX.XXX.XXX.0-258/spark/lib/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\n",
      "16/08/26 12:38:21 INFO Client: Uploading resource file:/home/ansible/iris/SimpleApp.py -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/SimpleApp.py\n",
      "16/08/26 12:38:21 INFO Client: Uploading resource file:/usr/hdp/XXX.XXX.XXX.0-258/spark/python/lib/pyspark.zip -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip\n",
      "16/08/26 12:38:22 INFO Client: Uploading resource file:/usr/hdp/XXX.XXX.XXX.0-258/spark/python/lib/py4j-0.9-src.zip -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip\n",
      "16/08/26 12:38:22 INFO Client: Uploading resource file:/tmp/spark-d4075d46-47ed-486a-8e9c-498b04521e06/__spark_conf__7022645124653218094.zip -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip\n",
      "16/08/26 12:38:22 INFO SecurityManager: Changing view acls to: ansible\n",
      "16/08/26 12:38:22 INFO SecurityManager: Changing modify acls to: ansible\n",
      "16/08/26 12:38:22 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(ansible); users with modify permissions: Set(ansible)\n",
      "16/08/26 12:38:22 INFO Client: Submitting application 2 to ResourceManager\n",
      "16/08/26 12:38:23 INFO YarnClientImpl: Submitted application application_1472181274763_0002\n",
      "16/08/26 12:38:24 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:24 INFO Client: \n",
      "\t client token: N/A\n",
      "\t diagnostics: N/A\n",
      "\t ApplicationMaster host: N/A\n",
      "\t ApplicationMaster RPC port: -1\n",
      "\t queue: default\n",
      "\t start time: 1472182702615\n",
      "\t final status: UNDEFINED\n",
      "\t tracking URL: http://testvm001:8088/proxy/application_1472181274763_0002/\n",
      "\t user: ansible\n",
      "16/08/26 12:38:25 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:26 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:27 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:28 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:29 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:30 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:31 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:32 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:33 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:34 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:35 INFO Client: Application report for application_1472181274763_0002 (state: ACCEPTED)\n",
      "16/08/26 12:38:36 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:36 INFO Client: \n",
      "\t client token: N/A\n",
      "\t diagnostics: N/A\n",
      "\t ApplicationMaster host: XXX.XXX.XXX.114\n",
      "\t ApplicationMaster RPC port: 0\n",
      "\t queue: default\n",
      "\t start time: 1472182702615\n",
      "\t final status: UNDEFINED\n",
      "\t tracking URL: http://testvm001:8088/proxy/application_1472181274763_0002/\n",
      "\t user: ansible\n",
      "16/08/26 12:38:37 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:38 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:39 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:40 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:41 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:42 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:43 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:44 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:45 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:46 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:47 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:48 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:49 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:50 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:51 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:52 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:53 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:54 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:55 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:56 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:57 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:58 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:38:59 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:39:00 INFO Client: Application report for application_1472181274763_0002 (state: RUNNING)\n",
      "16/08/26 12:39:01 INFO Client: Application report for application_1472181274763_0002 (state: FINISHED)\n",
      "16/08/26 12:39:01 INFO Client: \n",
      "\t client token: N/A\n",
      "\t diagnostics: N/A\n",
      "\t ApplicationMaster host: XXX.XXX.XXX.114\n",
      "\t ApplicationMaster RPC port: 0\n",
      "\t queue: default\n",
      "\t start time: 1472182702615\n",
      "\t final status: SUCCEEDED\n",
      "\t tracking URL: http://testvm001:8088/proxy/application_1472181274763_0002/\n",
      "\t user: ansible\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Deleting directory /tmp/spark-d4075d46-47ed-486a-8e9c-498b04521e06\n",
      "\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!ansible -m copy -a 'src={work_dir}/SimpleApp.py dest={hadoop_client_dir}/' hadoop_client -l {target_group}\n",
    "!ansible -m shell -a 'spark-submit --master yarn-cluster --num-executors 4 {hadoop_client_dir}/SimpleApp.py' hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b5748c-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57658-495a-11e8-a769-0242ac130002",
     "previous": "98b57112-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "# 実行経過の確認"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b57658-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b577b6-495a-11e8-a769-0242ac130002",
     "previous": "98b5748c-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "## ログの確認\n",
    "\n",
    "ログは、 `yarn logs` コマンドにより確認することができる。この際、Application IDが必要。この情報はログから確認することができる。\n",
    "\n",
    "> `16/08/26 12:38:23 INFO YarnClientImpl: Submitted application application_1472181274763_0002`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b577b6-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57900-495a-11e8-a769-0242ac130002",
     "previous": "98b57658-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": [
    "yarn_application_id = 'application_1472181274763_0002'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "lc_cell_meme": {
     "current": "98b57900-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57a4a-495a-11e8-a769-0242ac130002",
     "previous": "98b577b6-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0;32mXXX.XXX.XXX.72 | SUCCESS | rc=0 >>\r\n",
      "\r\n",
      "\r\n",
      "Container: container_1472181274763_0002_01_000004 on testvm004_45454\r\n",
      "======================================================================\r\n",
      "LogType:directory.info\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:3825\r\n",
      "Log Contents:\r\n",
      "ls -l:\r\n",
      "total 40\r\n",
      "lrwxrwxrwx 1 yarn yarn  128 Aug 26 12:38 __spark__.jar -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\r\n",
      "lrwxrwxrwx 1 yarn yarn  105 Aug 26 12:38 __spark_conf__ -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\r\n",
      "-rw-r--r-- 1 yarn yarn  109 Aug 26 12:38 container_tokens\r\n",
      "-rwx------ 1 yarn yarn  754 Aug 26 12:38 default_container_executor.sh\r\n",
      "-rwx------ 1 yarn yarn  700 Aug 26 12:38 default_container_executor_session.sh\r\n",
      "-rwx------ 1 yarn yarn 6500 Aug 26 12:38 launch_container.sh\r\n",
      "lrwxrwxrwx 1 yarn yarn   84 Aug 26 12:38 py4j-0.9-src.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\r\n",
      "lrwxrwxrwx 1 yarn yarn   79 Aug 26 12:38 pyspark.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\r\n",
      "drwx--x--- 2 yarn yarn 4096 Aug 26 12:38 tmp\r\n",
      "find -L . -maxdepth 5 -ls:\r\n",
      "655647    4 drwx--x---   3 yarn     yarn         4096 Aug 26 12:38 .\r\n",
      "655650    4 -rw-r--r--   1 yarn     yarn           12 Aug 26 12:38 ./.container_tokens.crc\r\n",
      "655652    4 -rw-r--r--   1 yarn     yarn           60 Aug 26 12:38 ./.launch_container.sh.crc\r\n",
      "655620   44 -r-x------   1 yarn     yarn        44846 Aug 26 12:38 ./py4j-0.9-src.zip\r\n",
      "655625    4 drwx------   2 yarn     yarn         4096 Aug 26 12:38 ./__spark_conf__\r\n",
      "655636    4 -r-x------   1 yarn     yarn         2490 Aug 26 12:38 ./__spark_conf__/hadoop-metrics.properties\r\n",
      "655631    8 -r-x------   1 yarn     yarn         4752 Aug 26 12:38 ./__spark_conf__/yarn-site.xml\r\n",
      "655633    8 -r-x------   1 yarn     yarn         4436 Aug 26 12:38 ./__spark_conf__/capacity-scheduler.xml\r\n",
      "655632    4 -r-x------   1 yarn     yarn         2130 Aug 26 12:38 ./__spark_conf__/core-site.xml\r\n",
      "655626    4 -r-x------   1 yarn     yarn          620 Aug 26 12:38 ./__spark_conf__/log4j.properties\r\n",
      "655634    4 -r-x------   1 yarn     yarn         3556 Aug 26 12:38 ./__spark_conf__/hdfs-site.xml\r\n",
      "655635    8 -r-x------   1 yarn     yarn         4567 Aug 26 12:38 ./__spark_conf__/yarn-env.sh\r\n",
      "655639    8 -r-x------   1 yarn     yarn         5308 Aug 26 12:38 ./__spark_conf__/metrics.properties\r\n",
      "655630    4 -r-x------   1 yarn     yarn         2419 Aug 26 12:38 ./__spark_conf__/hadoop-metrics2.properties\r\n",
      "655627    4 -r-x------   1 yarn     yarn            2 Aug 26 12:38 ./__spark_conf__/hosts.exclude\r\n",
      "655628    4 -r-x------   1 yarn     yarn         1921 Aug 26 12:38 ./__spark_conf__/mapred-site.xml\r\n",
      "655640    4 -r-x------   1 yarn     yarn          496 Aug 26 12:38 ./__spark_conf__/__spark_conf__.properties\r\n",
      "655637    4 -r-x------   1 yarn     yarn         1639 Aug 26 12:38 ./__spark_conf__/mapred-env.sh\r\n",
      "655629    8 -r-x------   1 yarn     yarn         4623 Aug 26 12:38 ./__spark_conf__/hadoop-env.sh\r\n",
      "655638    4 -r-x------   1 yarn     yarn           40 Aug 26 12:38 ./__spark_conf__/hosts.list\r\n",
      "655649    4 -rw-r--r--   1 yarn     yarn          109 Aug 26 12:38 ./container_tokens\r\n",
      "655655    4 -rwx------   1 yarn     yarn          754 Aug 26 12:38 ./default_container_executor.sh\r\n",
      "655654    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor_session.sh.crc\r\n",
      "655617  352 -r-x------   1 yarn     yarn       357163 Aug 26 12:38 ./pyspark.zip\r\n",
      "655651    8 -rwx------   1 yarn     yarn         6500 Aug 26 12:38 ./launch_container.sh\r\n",
      "655653    4 -rwx------   1 yarn     yarn          700 Aug 26 12:38 ./default_container_executor_session.sh\r\n",
      "655624 181616 -r-x------   1 yarn     yarn     185971201 Aug 26 12:38 ./__spark__.jar\r\n",
      "655656    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor.sh.crc\r\n",
      "655648    4 drwx--x---   2 yarn     yarn         4096 Aug 26 12:38 ./tmp\r\n",
      "broken symlinks(find -L . -maxdepth 5 -type l -ls):\r\n",
      "End of LogType:directory.info\r\n",
      "\r\n",
      "LogType:launch_container.sh\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:6500\r\n",
      "Log Contents:\r\n",
      "#!/bin/bash\r\n",
      "\r\n",
      "export SPARK_YARN_MODE=\"true\"\r\n",
      "export SPARK_YARN_STAGING_DIR=\".sparkStaging/application_1472181274763_0002\"\r\n",
      "export JAVA_HOME=\"/usr/java/jdk1.7.0_75\"\r\n",
      "export SPARK_YARN_CACHE_FILES_VISIBILITIES=\"PRIVATE,PRIVATE,PRIVATE\"\r\n",
      "export NM_AUX_SERVICE_mapreduce_shuffle=\"AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\r",
      "\r\n",
      "\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\"\r\n",
      "export SPARK_LOG_URL_STDERR=\"http://testvm004:8042/node/containerlogs/container_1472181274763_0002_01_000004/ansible/stderr?start=-4096\"\r\n",
      "export HADOOP_YARN_HOME=\"/usr/hdp/XXX.XXX.XXX.0-258/hadoop-yarn\"\r\n",
      "export NM_HOST=\"testvm004\"\r\n",
      "export PYTHONPATH=\":$PWD/pyspark.zip:$PWD/py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES=\"40968\"\r\n",
      "export JVM_PID=\"$$\"\r\n",
      "export SPARK_YARN_CACHE_FILES_TIME_STAMPS=\"1472182701411,1472182701943,1472182702137\"\r\n",
      "export SPARK_USER=\"ansible\"\r\n",
      "export PWD=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000004\"\r\n",
      "export NM_PORT=\"45454\"\r\n",
      "export LOGNAME=\"ansible\"\r\n",
      "export SPARK_LOG_URL_STDOUT=\"http://testvm004:8042/node/containerlogs/container_1472181274763_0002_01_000004/ansible/stdout?start=-4096\"\r\n",
      "export MALLOC_ARENA_MAX=\"4\"\r\n",
      "export LOG_DIRS=\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004\"\r\n",
      "export SPARK_YARN_CACHE_FILES_FILE_SIZES=\"185971201,357163,44846\"\r\n",
      "export NM_HTTP_PORT=\"8042\"\r\n",
      "export LOCAL_DIRS=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_FILES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS=\"1472182702391\"\r\n",
      "export CLASSPATH=\"$PWD:$PWD/__spark_conf__:$PWD/__spark__.jar:/usr/hdp/current/hadoop-client/*:/usr/hdp/current/hadoop-client/lib/*:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/current/hadoop-mapreduce-client/lib/*:/usr/hdp/current/hadoop-yarn-client/*:/usr/hdp/current/hadoop-yarn-client/lib/*:/etc/tez/conf:/usr/hdp/current/tez-client/*:user/hdp/current/tez-client/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export SPARK_YARN_USER_ENV=\"CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export HADOOP_TOKEN_FILE_LOCATION=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000004/container_tokens\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES=\"PRIVATE\"\r\n",
      "export USER=\"ansible\"\r\n",
      "export CONTAINER_ID=\"container_1472181274763_0002_01_000004\"\r\n",
      "export HOME=\"/home/\"\r\n",
      "export HADOOP_CONF_DIR=\"/usr/hdp/current/hadoop-yarn-nodemanager/../hadoop/conf\"\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\" \"py4j-0.9-src.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" \"__spark__.jar\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\" \"__spark_conf__\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\" \"pyspark.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "# Creating copy of launch script\r\n",
      "cp \"launch_container.sh\" \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/launch_container.sh\"\r\n",
      "chmod 640 \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/launch_container.sh\"\r\n",
      "# Determining directory contents\r\n",
      "echo \"ls -l:\" 1>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "ls -l 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "echo \"find -L . -maxdepth 5 -ls:\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "find -L . -maxdepth 5 -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "find -L . -maxdepth 5 -type l -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/directory.info\"\r\n",
      "exec /bin/bash -c \"$JAVA_HOME/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir=$PWD/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004 -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 3 --hostname testvm004 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/stdout 2> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000004/stderr\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "End of LogType:launch_container.sh\r\n",
      "\r\n",
      "LogType:stderr\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:6169\r\n",
      "Log Contents:\r\n",
      "SLF4J: Class path contains multiple SLF4J bindings.\r\n",
      "SLF4J: Found binding in [jar:file:/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: Found binding in [jar:file:/usr/hdp/XXX.XXX.XXX.0-258/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\r\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\r\n",
      "16/08/26 12:38:40 INFO CoarseGrainedExecutorBackend: Registered signal handlers for [TERM, HUP, INT]\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:45 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:45 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:45 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:46 INFO Slf4jLogger: Slf4jLogger started\r\n",
      "16/08/26 12:38:46 INFO Remoting: Starting remoting\r\n",
      "16/08/26 12:38:47 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkExecutorActorSystem@testvm004:45983]\r\n",
      "16/08/26 12:38:47 INFO Utils: Successfully started service 'sparkExecutorActorSystem' on port 45983.\r\n",
      "16/08/26 12:38:47 INFO DiskBlockManager: Created local directory at /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/blockmgr-552d1b82-142c-442e-8517-fda4279f6c93\r\n",
      "16/08/26 12:38:47 INFO MemoryStore: MemoryStore started with capacity 511.5 MB\r\n",
      "16/08/26 12:38:47 INFO CoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460\r\n",
      "16/08/26 12:38:47 INFO CoarseGrainedExecutorBackend: Successfully registered with driver\r\n",
      "16/08/26 12:38:47 INFO Executor: Starting executor ID 3 on host testvm004\r\n",
      "16/08/26 12:38:47 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 37901.\r\n",
      "16/08/26 12:38:47 INFO NettyBlockTransferService: Server created on 37901\r\n",
      "16/08/26 12:38:47 INFO BlockManagerMaster: Trying to register BlockManager\r\n",
      "16/08/26 12:38:47 INFO BlockManagerMaster: Registered BlockManager\r\n",
      "16/08/26 12:38:50 INFO CoarseGrainedExecutorBackend: Got assigned task 0\r\n",
      "16/08/26 12:38:50 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Started reading broadcast variable 1\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 3.9 KB, free 3.9 KB)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Reading broadcast variable 1 took 216 ms\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 6.8 KB, free 10.7 KB)\r\n",
      "16/08/26 12:38:50 INFO CacheManager: Partition rdd_1_0 not found, computing it\r\n",
      "16/08/26 12:38:50 INFO HadoopRDD: Input split: hdfs://hdfs-cluster/dataset/iris/iris.data:0+2275\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Started reading broadcast variable 0\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 25.3 KB, free 35.9 KB)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Reading broadcast variable 0 took 29 ms\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 321.8 KB, free 357.7 KB)\r\n",
      "16/08/26 12:38:51 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id\r\n",
      "16/08/26 12:38:51 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id\r\n",
      "16/08/26 12:38:51 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap\r\n",
      "16/08/26 12:38:51 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition\r\n",
      "16/08/26 12:38:51 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id\r\n",
      "16/08/26 12:38:52 INFO MemoryStore: Block rdd_1_0 stored as bytes in memory (estimated size 831.0 B, free 358.6 KB)\r\n",
      "16/08/26 12:39:00 INFO PythonRunner: Times: total = 8015, boot = 7995, init = 19, finish = 1\r\n",
      "16/08/26 12:39:00 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2704 bytes result sent to driver\r\n",
      "16/08/26 12:39:00 INFO CoarseGrainedExecutorBackend: Got assigned task 3\r\n",
      "16/08/26 12:39:00 INFO Executor: Running task 0.0 in stage 1.0 (TID 3)\r\n",
      "16/08/26 12:39:00 INFO TorrentBroadcast: Started reading broadcast variable 2\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 3.9 KB, free 362.4 KB)\r\n",
      "16/08/26 12:39:00 INFO TorrentBroadcast: Reading broadcast variable 2 took 284 ms\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 6.8 KB, free 369.3 KB)\r\n",
      "16/08/26 12:39:00 INFO BlockManager: Found block rdd_1_0 locally\r\n",
      "16/08/26 12:39:00 INFO PythonRunner: Times: total = 42, boot = -618, init = 659, finish = 1\r\n",
      "16/08/26 12:39:00 INFO Executor: Finished task 0.0 in stage 1.0 (TID 3). 2124 bytes result sent to driver\r\n",
      "16/08/26 12:39:01 INFO CoarseGrainedExecutorBackend: Driver commanded a shutdown\r\n",
      "16/08/26 12:39:01 INFO MemoryStore: MemoryStore cleared\r\n",
      "16/08/26 12:39:01 INFO BlockManager: BlockManager stopped\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.\r\n",
      "16/08/26 12:39:01 WARN CoarseGrainedExecutorBackend: An unknown (testvm007:45460) driver disconnected.\r\n",
      "16/08/26 12:39:01 ERROR CoarseGrainedExecutorBackend: Driver XXX.XXX.XXX.114:45460 disassociated! Shutting down.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.\r\n",
      "End of LogType:stderr\r\n",
      "\r\n",
      "LogType:stdout\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:0\r\n",
      "Log Contents:\r\n",
      "End of LogType:stdout\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Container: container_1472181274763_0002_01_000003 on testvm005_45454\r\n",
      "======================================================================\r\n",
      "LogType:directory.info\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:3825\r\n",
      "Log Contents:\r\n",
      "ls -l:\r\n",
      "total 40\r\n",
      "lrwxrwxrwx 1 yarn yarn  128 Aug 26 12:38 __spark__.jar -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\r\n",
      "lrwxrwxrwx 1 yarn yarn  105 Aug 26 12:38 __spark_conf__ -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\r\n",
      "-rw-r--r-- 1 yarn yarn  109 Aug 26 12:38 container_tokens\r\n",
      "-rwx------ 1 yarn yarn  754 Aug 26 12:38 default_container_executor.sh\r\n",
      "-rwx------ 1 yarn yarn  700 Aug 26 12:38 default_container_executor_session.sh\r\n",
      "-rwx------ 1 yarn yarn 6500 Aug 26 12:38 launch_container.sh\r\n",
      "lrwxrwxrwx 1 yarn yarn   84 Aug 26 12:38 py4j-0.9-src.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\r\n",
      "lrwxrwxrwx 1 yarn yarn   79 Aug 26 12:38 pyspark.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\r\n",
      "drwx--x--- 2 yarn yarn 4096 Aug 26 12:38 tmp\r\n",
      "find -L . -maxdepth 5 -ls:\r\n",
      "655657    4 drwx--x---   3 yarn     yarn         4096 Aug 26 12:38 .\r\n",
      "655660    4 -rw-r--r--   1 yarn     yarn           12 Aug 26 12:38 ./.container_tokens.crc\r\n",
      "655662    4 -rw-r--r--   1 yarn     yarn           60 Aug 26 12:38 ./.launch_container.sh.crc\r\n",
      "655630   44 -r-x------   1 yarn     yarn        44846 Aug 26 12:38 ./py4j-0.9-src.zip\r\n",
      "655635    4 drwx------   2 yarn     yarn         4096 Aug 26 12:38 ./__spark_conf__\r\n",
      "655646    4 -r-x------   1 yarn     yarn         2490 Aug 26 12:38 ./__spark_conf__/hadoop-metrics.properties\r\n",
      "655641    8 -r-x------   1 yarn     yarn         4752 Aug 26 12:38 ./__spark_conf__/yarn-site.xml\r\n",
      "655643    8 -r-x------   1 yarn     yarn         4436 Aug 26 12:38 ./__spark_conf__/capacity-scheduler.xml\r\n",
      "655642    4 -r-x------   1 yarn     yarn         2130 Aug 26 12:38 ./__spark_conf__/core-site.xml\r\n",
      "655636    4 -r-x------   1 yarn     yarn          620 Aug 26 12:38 ./__spark_conf__/log4j.properties\r\n",
      "655644    4 -r-x------   1 yarn     yarn         3556 Aug 26 12:38 ./__spark_conf__/hdfs-site.xml\r\n",
      "655645    8 -r-x------   1 yarn     yarn         4567 Aug 26 12:38 ./__spark_conf__/yarn-env.sh\r\n",
      "655649    8 -r-x------   1 yarn     yarn         5308 Aug 26 12:38 ./__spark_conf__/metrics.properties\r\n",
      "655640    4 -r-x------   1 yarn     yarn         2419 Aug 26 12:38 ./__spark_conf__/hadoop-metrics2.properties\r\n",
      "655637    4 -r-x------   1 yarn     yarn            2 Aug 26 12:38 ./__spark_conf__/hosts.exclude\r\n",
      "655638    4 -r-x------   1 yarn     yarn         1921 Aug 26 12:38 ./__spark_conf__/mapred-site.xml\r\n",
      "655650    4 -r-x------   1 yarn     yarn          496 Aug 26 12:38 ./__spark_conf__/__spark_conf__.properties\r\n",
      "655647    4 -r-x------   1 yarn     yarn         1639 Aug 26 12:38 ./__spark_conf__/mapred-env.sh\r\n",
      "655639    8 -r-x------   1 yarn     yarn         4623 Aug 26 12:38 ./__spark_conf__/hadoop-env.sh\r\n",
      "655648    4 -r-x------   1 yarn     yarn           40 Aug 26 12:38 ./__spark_conf__/hosts.list\r\n",
      "655659    4 -rw-r--r--   1 yarn     yarn          109 Aug 26 12:38 ./container_tokens\r\n",
      "655665    4 -rwx------   1 yarn     yarn          754 Aug 26 12:38 ./default_container_executor.sh\r\n",
      "655664    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor_session.sh.crc\r\n",
      "655627  352 -r-x------   1 yarn     yarn       357163 Aug 26 12:38 ./pyspark.zip\r\n",
      "655661    8 -rwx------   1 yarn     yarn         6500 Aug 26 12:38 ./launch_container.sh\r\n",
      "655663    4 -rwx------   1 yarn     yarn          700 Aug 26 12:38 ./default_container_executor_session.sh\r\n",
      "655634 181616 -r-x------   1 yarn     yarn     185971201 Aug 26 12:38 ./__spark__.jar\r\n",
      "655666    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor.sh.crc\r\n",
      "655658    4 drwx--x---   2 yarn     yarn         4096 Aug 26 12:38 ./tmp\r\n",
      "broken symlinks(find -L . -maxdepth 5 -type l -ls):\r\n",
      "End of LogType:directory.info\r\n",
      "\r\n",
      "LogType:launch_container.sh\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:6500\r\n",
      "Log Contents:\r\n",
      "#!/bin/bash\r\n",
      "\r\n",
      "export SPARK_YARN_MODE=\"true\"\r\n",
      "export SPARK_YARN_STAGING_DIR=\".sparkStaging/application_1472181274763_0002\"\r\n",
      "export JAVA_HOME=\"/usr/java/jdk1.7.0_75\"\r\n",
      "export SPARK_YARN_CACHE_FILES_VISIBILITIES=\"PRIVATE,PRIVATE,PRIVATE\"\r\n",
      "export NM_AUX_SERVICE_mapreduce_shuffle=\"AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\r",
      "\r\n",
      "\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\"\r\n",
      "export SPARK_LOG_URL_STDERR=\"http://testvm005:8042/node/containerlogs/container_1472181274763_0002_01_000003/ansible/stderr?start=-4096\"\r\n",
      "export HADOOP_YARN_HOME=\"/usr/hdp/XXX.XXX.XXX.0-258/hadoop-yarn\"\r\n",
      "export NM_HOST=\"testvm005\"\r\n",
      "export PYTHONPATH=\":$PWD/pyspark.zip:$PWD/py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES=\"40968\"\r\n",
      "export JVM_PID=\"$$\"\r\n",
      "export SPARK_YARN_CACHE_FILES_TIME_STAMPS=\"1472182701411,1472182701943,1472182702137\"\r\n",
      "export SPARK_USER=\"ansible\"\r\n",
      "export PWD=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000003\"\r\n",
      "export NM_PORT=\"45454\"\r\n",
      "export LOGNAME=\"ansible\"\r\n",
      "export SPARK_LOG_URL_STDOUT=\"http://testvm005:8042/node/containerlogs/container_1472181274763_0002_01_000003/ansible/stdout?start=-4096\"\r\n",
      "export MALLOC_ARENA_MAX=\"4\"\r\n",
      "export LOG_DIRS=\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003\"\r\n",
      "export SPARK_YARN_CACHE_FILES_FILE_SIZES=\"185971201,357163,44846\"\r\n",
      "export NM_HTTP_PORT=\"8042\"\r\n",
      "export LOCAL_DIRS=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_FILES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS=\"1472182702391\"\r\n",
      "export CLASSPATH=\"$PWD:$PWD/__spark_conf__:$PWD/__spark__.jar:/usr/hdp/current/hadoop-client/*:/usr/hdp/current/hadoop-client/lib/*:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/current/hadoop-mapreduce-client/lib/*:/usr/hdp/current/hadoop-yarn-client/*:/usr/hdp/current/hadoop-yarn-client/lib/*:/etc/tez/conf:/usr/hdp/current/tez-client/*:user/hdp/current/tez-client/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export SPARK_YARN_USER_ENV=\"CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export HADOOP_TOKEN_FILE_LOCATION=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000003/container_tokens\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES=\"PRIVATE\"\r\n",
      "export USER=\"ansible\"\r\n",
      "export CONTAINER_ID=\"container_1472181274763_0002_01_000003\"\r\n",
      "export HOME=\"/home/\"\r\n",
      "export HADOOP_CONF_DIR=\"/usr/hdp/current/hadoop-yarn-nodemanager/../hadoop/conf\"\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\" \"py4j-0.9-src.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" \"__spark__.jar\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\" \"__spark_conf__\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\" \"pyspark.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "# Creating copy of launch script\r\n",
      "cp \"launch_container.sh\" \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/launch_container.sh\"\r\n",
      "chmod 640 \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/launch_container.sh\"\r\n",
      "# Determining directory contents\r\n",
      "echo \"ls -l:\" 1>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "ls -l 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "echo \"find -L . -maxdepth 5 -ls:\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "find -L . -maxdepth 5 -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "find -L . -maxdepth 5 -type l -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/directory.info\"\r\n",
      "exec /bin/bash -c \"$JAVA_HOME/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir=$PWD/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003 -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 2 --hostname testvm005 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/stdout 2> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000003/stderr\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "End of LogType:launch_container.sh\r\n",
      "\r\n",
      "LogType:stderr\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:3314\r\n",
      "Log Contents:\r\n",
      "SLF4J: Class path contains multiple SLF4J bindings.\r\n",
      "SLF4J: Found binding in [jar:file:/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: Found binding in [jar:file:/usr/hdp/XXX.XXX.XXX.0-258/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\r\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\r\n",
      "16/08/26 12:38:39 INFO CoarseGrainedExecutorBackend: Registered signal handlers for [TERM, HUP, INT]\r\n",
      "16/08/26 12:38:41 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:41 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:41 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:44 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:44 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:44 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:45 INFO Slf4jLogger: Slf4jLogger started\r\n",
      "16/08/26 12:38:45 INFO Remoting: Starting remoting\r\n",
      "16/08/26 12:38:45 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkExecutorActorSystem@testvm005:38920]\r\n",
      "16/08/26 12:38:45 INFO Utils: Successfully started service 'sparkExecutorActorSystem' on port 38920.\r\n",
      "16/08/26 12:38:45 INFO DiskBlockManager: Created local directory at /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/blockmgr-cd1ff972-22d9-45dc-bf9b-ab709f8e201c\r\n",
      "16/08/26 12:38:45 INFO MemoryStore: MemoryStore started with capacity 511.5 MB\r\n",
      "16/08/26 12:38:46 INFO CoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460\r\n",
      "16/08/26 12:38:46 INFO CoarseGrainedExecutorBackend: Successfully registered with driver\r\n",
      "16/08/26 12:38:46 INFO Executor: Starting executor ID 2 on host testvm005\r\n",
      "16/08/26 12:38:46 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45419.\r\n",
      "16/08/26 12:38:46 INFO NettyBlockTransferService: Server created on 45419\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMaster: Trying to register BlockManager\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMaster: Registered BlockManager\r\n",
      "16/08/26 12:39:01 INFO CoarseGrainedExecutorBackend: Driver commanded a shutdown\r\n",
      "16/08/26 12:39:01 INFO MemoryStore: MemoryStore cleared\r\n",
      "16/08/26 12:39:01 INFO BlockManager: BlockManager stopped\r\n",
      "16/08/26 12:39:01 WARN CoarseGrainedExecutorBackend: An unknown (testvm007:45460) driver disconnected.\r\n",
      "16/08/26 12:39:01 ERROR CoarseGrainedExecutorBackend: Driver XXX.XXX.XXX.114:45460 disassociated! Shutting down.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\r\n",
      "End of LogType:stderr\r\n",
      "\r\n",
      "LogType:stdout\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:0\r\n",
      "Log Contents:\r\n",
      "End of LogType:stdout\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Container: container_1472181274763_0002_01_000002 on testvm006_45454\r\n",
      "======================================================================\r\n",
      "LogType:directory.info\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:3825\r\n",
      "Log Contents:\r\n",
      "ls -l:\r\n",
      "total 40\r\n",
      "lrwxrwxrwx 1 yarn yarn  128 Aug 26 12:38 __spark__.jar -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\r\n",
      "lrwxrwxrwx 1 yarn yarn  105 Aug 26 12:38 __spark_conf__ -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\r\n",
      "-rw-r--r-- 1 yarn yarn  109 Aug 26 12:38 container_tokens\r\n",
      "-rwx------ 1 yarn yarn  754 Aug 26 12:38 default_container_executor.sh\r\n",
      "-rwx------ 1 yarn yarn  700 Aug 26 12:38 default_container_executor_session.sh\r\n",
      "-rwx------ 1 yarn yarn 6500 Aug 26 12:38 launch_container.sh\r\n",
      "lrwxrwxrwx 1 yarn yarn   84 Aug 26 12:38 py4j-0.9-src.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\r\n",
      "lrwxrwxrwx 1 yarn yarn   79 Aug 26 12:38 pyspark.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\r\n",
      "drwx--x--- 2 yarn yarn 4096 Aug 26 12:38 tmp\r\n",
      "find -L . -maxdepth 5 -ls:\r\n",
      "655647    4 drwx--x---   3 yarn     yarn         4096 Aug 26 12:38 .\r\n",
      "655650    4 -rw-r--r--   1 yarn     yarn           12 Aug 26 12:38 ./.container_tokens.crc\r\n",
      "655652    4 -rw-r--r--   1 yarn     yarn           60 Aug 26 12:38 ./.launch_container.sh.crc\r\n",
      "655438   44 -r-x------   1 yarn     yarn        44846 Aug 26 12:38 ./py4j-0.9-src.zip\r\n",
      "655620    4 drwx------   2 yarn     yarn         4096 Aug 26 12:38 ./__spark_conf__\r\n",
      "655636    4 -r-x------   1 yarn     yarn         2490 Aug 26 12:38 ./__spark_conf__/hadoop-metrics.properties\r\n",
      "655631    8 -r-x------   1 yarn     yarn         4752 Aug 26 12:38 ./__spark_conf__/yarn-site.xml\r\n",
      "655633    8 -r-x------   1 yarn     yarn         4436 Aug 26 12:38 ./__spark_conf__/capacity-scheduler.xml\r\n",
      "655632    4 -r-x------   1 yarn     yarn         2130 Aug 26 12:38 ./__spark_conf__/core-site.xml\r\n",
      "655621    4 -r-x------   1 yarn     yarn          620 Aug 26 12:38 ./__spark_conf__/log4j.properties\r\n",
      "655634    4 -r-x------   1 yarn     yarn         3556 Aug 26 12:38 ./__spark_conf__/hdfs-site.xml\r\n",
      "655635    8 -r-x------   1 yarn     yarn         4567 Aug 26 12:38 ./__spark_conf__/yarn-env.sh\r\n",
      "655639    8 -r-x------   1 yarn     yarn         5308 Aug 26 12:38 ./__spark_conf__/metrics.properties\r\n",
      "655630    4 -r-x------   1 yarn     yarn         2419 Aug 26 12:38 ./__spark_conf__/hadoop-metrics2.properties\r\n",
      "655622    4 -r-x------   1 yarn     yarn            2 Aug 26 12:38 ./__spark_conf__/hosts.exclude\r\n",
      "655628    4 -r-x------   1 yarn     yarn         1921 Aug 26 12:38 ./__spark_conf__/mapred-site.xml\r\n",
      "655640    4 -r-x------   1 yarn     yarn          496 Aug 26 12:38 ./__spark_conf__/__spark_conf__.properties\r\n",
      "655637    4 -r-x------   1 yarn     yarn         1639 Aug 26 12:38 ./__spark_conf__/mapred-env.sh\r\n",
      "655629    8 -r-x------   1 yarn     yarn         4623 Aug 26 12:38 ./__spark_conf__/hadoop-env.sh\r\n",
      "655638    4 -r-x------   1 yarn     yarn           40 Aug 26 12:38 ./__spark_conf__/hosts.list\r\n",
      "655649    4 -rw-r--r--   1 yarn     yarn          109 Aug 26 12:38 ./container_tokens\r\n",
      "655655    4 -rwx------   1 yarn     yarn          754 Aug 26 12:38 ./default_container_executor.sh\r\n",
      "655654    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor_session.sh.crc\r\n",
      "655435  352 -r-x------   1 yarn     yarn       357163 Aug 26 12:38 ./pyspark.zip\r\n",
      "655651    8 -rwx------   1 yarn     yarn         6500 Aug 26 12:38 ./launch_container.sh\r\n",
      "655653    4 -rwx------   1 yarn     yarn          700 Aug 26 12:38 ./default_container_executor_session.sh\r\n",
      "655619 181616 -r-x------   1 yarn     yarn     185971201 Aug 26 12:38 ./__spark__.jar\r\n",
      "655656    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor.sh.crc\r\n",
      "655648    4 drwx--x---   2 yarn     yarn         4096 Aug 26 12:38 ./tmp\r\n",
      "broken symlinks(find -L . -maxdepth 5 -type l -ls):\r\n",
      "End of LogType:directory.info\r\n",
      "\r\n",
      "LogType:launch_container.sh\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:6500\r\n",
      "Log Contents:\r\n",
      "#!/bin/bash\r\n",
      "\r\n",
      "export SPARK_YARN_MODE=\"true\"\r\n",
      "export SPARK_YARN_STAGING_DIR=\".sparkStaging/application_1472181274763_0002\"\r\n",
      "export JAVA_HOME=\"/usr/java/jdk1.7.0_75\"\r\n",
      "export SPARK_YARN_CACHE_FILES_VISIBILITIES=\"PRIVATE,PRIVATE,PRIVATE\"\r\n",
      "export NM_AUX_SERVICE_mapreduce_shuffle=\"AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\r",
      "\r\n",
      "\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\"\r\n",
      "export SPARK_LOG_URL_STDERR=\"http://testvm006:8042/node/containerlogs/container_1472181274763_0002_01_000002/ansible/stderr?start=-4096\"\r\n",
      "export HADOOP_YARN_HOME=\"/usr/hdp/XXX.XXX.XXX.0-258/hadoop-yarn\"\r\n",
      "export NM_HOST=\"testvm006\"\r\n",
      "export PYTHONPATH=\":$PWD/pyspark.zip:$PWD/py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES=\"40968\"\r\n",
      "export JVM_PID=\"$$\"\r\n",
      "export SPARK_YARN_CACHE_FILES_TIME_STAMPS=\"1472182701411,1472182701943,1472182702137\"\r\n",
      "export SPARK_USER=\"ansible\"\r\n",
      "export PWD=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000002\"\r\n",
      "export NM_PORT=\"45454\"\r\n",
      "export LOGNAME=\"ansible\"\r\n",
      "export SPARK_LOG_URL_STDOUT=\"http://testvm006:8042/node/containerlogs/container_1472181274763_0002_01_000002/ansible/stdout?start=-4096\"\r\n",
      "export MALLOC_ARENA_MAX=\"4\"\r\n",
      "export LOG_DIRS=\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002\"\r\n",
      "export SPARK_YARN_CACHE_FILES_FILE_SIZES=\"185971201,357163,44846\"\r\n",
      "export NM_HTTP_PORT=\"8042\"\r\n",
      "export LOCAL_DIRS=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_FILES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS=\"1472182702391\"\r\n",
      "export CLASSPATH=\"$PWD:$PWD/__spark_conf__:$PWD/__spark__.jar:/usr/hdp/current/hadoop-client/*:/usr/hdp/current/hadoop-client/lib/*:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/current/hadoop-mapreduce-client/lib/*:/usr/hdp/current/hadoop-yarn-client/*:/usr/hdp/current/hadoop-yarn-client/lib/*:/etc/tez/conf:/usr/hdp/current/tez-client/*:user/hdp/current/tez-client/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export SPARK_YARN_USER_ENV=\"CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export HADOOP_TOKEN_FILE_LOCATION=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000002/container_tokens\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES=\"PRIVATE\"\r\n",
      "export USER=\"ansible\"\r\n",
      "export CONTAINER_ID=\"container_1472181274763_0002_01_000002\"\r\n",
      "export HOME=\"/home/\"\r\n",
      "export HADOOP_CONF_DIR=\"/usr/hdp/current/hadoop-yarn-nodemanager/../hadoop/conf\"\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/py4j-0.9-src.zip\" \"py4j-0.9-src.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" \"__spark__.jar\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/__spark_conf__7022645124653218094.zip\" \"__spark_conf__\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\" \"pyspark.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "# Creating copy of launch script\r\n",
      "cp \"launch_container.sh\" \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/launch_container.sh\"\r\n",
      "chmod 640 \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/launch_container.sh\"\r\n",
      "# Determining directory contents\r\n",
      "echo \"ls -l:\" 1>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "ls -l 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "echo \"find -L . -maxdepth 5 -ls:\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "find -L . -maxdepth 5 -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "find -L . -maxdepth 5 -type l -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/directory.info\"\r\n",
      "exec /bin/bash -c \"$JAVA_HOME/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir=$PWD/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002 -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 1 --hostname testvm006 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/stdout 2> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000002/stderr\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "End of LogType:launch_container.sh\r\n",
      "\r\n",
      "LogType:stderr\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:3179\r\n",
      "Log Contents:\r\n",
      "SLF4J: Class path contains multiple SLF4J bindings.\r\n",
      "SLF4J: Found binding in [jar:file:/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: Found binding in [jar:file:/usr/hdp/XXX.XXX.XXX.0-258/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\r\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\r\n",
      "16/08/26 12:38:40 INFO CoarseGrainedExecutorBackend: Registered signal handlers for [TERM, HUP, INT]\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:43 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:43 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:43 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:44 INFO Slf4jLogger: Slf4jLogger started\r\n",
      "16/08/26 12:38:44 INFO Remoting: Starting remoting\r\n",
      "16/08/26 12:38:45 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkExecutorActorSystem@testvm006:38571]\r\n",
      "16/08/26 12:38:45 INFO Utils: Successfully started service 'sparkExecutorActorSystem' on port 38571.\r\n",
      "16/08/26 12:38:45 INFO DiskBlockManager: Created local directory at /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/blockmgr-2b716de1-b9e1-4731-b2b6-bf0e7f74e307\r\n",
      "16/08/26 12:38:45 INFO MemoryStore: MemoryStore started with capacity 511.5 MB\r\n",
      "16/08/26 12:38:45 INFO CoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460\r\n",
      "16/08/26 12:38:45 INFO CoarseGrainedExecutorBackend: Successfully registered with driver\r\n",
      "16/08/26 12:38:46 INFO Executor: Starting executor ID 1 on host testvm006\r\n",
      "16/08/26 12:38:46 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 43374.\r\n",
      "16/08/26 12:38:46 INFO NettyBlockTransferService: Server created on 43374\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMaster: Trying to register BlockManager\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMaster: Registered BlockManager\r\n",
      "16/08/26 12:39:01 INFO CoarseGrainedExecutorBackend: Driver commanded a shutdown\r\n",
      "16/08/26 12:39:01 INFO MemoryStore: MemoryStore cleared\r\n",
      "16/08/26 12:39:01 INFO BlockManager: BlockManager stopped\r\n",
      "16/08/26 12:39:01 WARN CoarseGrainedExecutorBackend: An unknown (testvm007:45460) driver disconnected.\r\n",
      "16/08/26 12:39:01 ERROR CoarseGrainedExecutorBackend: Driver XXX.XXX.XXX.114:45460 disassociated! Shutting down.\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.\r\n",
      "End of LogType:stderr\r\n",
      "\r\n",
      "LogType:stdout\r\n",
      "Log Upload Time:Fri Aug 26 12:39:02 +0900 2016\r\n",
      "LogLength:0\r\n",
      "Log Contents:\r\n",
      "End of LogType:stdout\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Container: container_1472181274763_0002_01_000001 on testvm007_45454\r\n",
      "======================================================================\r\n",
      "LogType:directory.info\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:4045\r\n",
      "Log Contents:\r\n",
      "ls -l:\r\n",
      "total 44\r\n",
      "lrwxrwxrwx 1 yarn yarn   80 Aug 26 12:38 SimpleApp.py -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/SimpleApp.py\r\n",
      "lrwxrwxrwx 1 yarn yarn  128 Aug 26 12:38 __spark__.jar -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\r\n",
      "lrwxrwxrwx 1 yarn yarn  105 Aug 26 12:38 __spark_conf__ -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/__spark_conf__7022645124653218094.zip\r\n",
      "-rw-r--r-- 1 yarn yarn   74 Aug 26 12:38 container_tokens\r\n",
      "-rwx------ 1 yarn yarn  754 Aug 26 12:38 default_container_executor.sh\r\n",
      "-rwx------ 1 yarn yarn  700 Aug 26 12:38 default_container_executor_session.sh\r\n",
      "-rwx------ 1 yarn yarn 6499 Aug 26 12:38 launch_container.sh\r\n",
      "lrwxrwxrwx 1 yarn yarn   84 Aug 26 12:38 py4j-0.9-src.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/py4j-0.9-src.zip\r\n",
      "lrwxrwxrwx 1 yarn yarn   79 Aug 26 12:38 pyspark.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\r\n",
      "drwx--x--- 2 yarn yarn 4096 Aug 26 12:38 tmp\r\n",
      "find -L . -maxdepth 5 -ls:\r\n",
      "655644    4 drwx--x---   3 yarn     yarn         4096 Aug 26 12:38 .\r\n",
      "655647    4 -rw-r--r--   1 yarn     yarn           12 Aug 26 12:38 ./.container_tokens.crc\r\n",
      "655649    4 -rw-r--r--   1 yarn     yarn           60 Aug 26 12:38 ./.launch_container.sh.crc\r\n",
      "655447   44 -r-x------   1 yarn     yarn        44846 Aug 26 12:38 ./py4j-0.9-src.zip\r\n",
      "655622    4 drwx------   2 yarn     yarn         4096 Aug 26 12:38 ./__spark_conf__\r\n",
      "655633    4 -r-x------   1 yarn     yarn         2490 Aug 26 12:38 ./__spark_conf__/hadoop-metrics.properties\r\n",
      "655628    8 -r-x------   1 yarn     yarn         4752 Aug 26 12:38 ./__spark_conf__/yarn-site.xml\r\n",
      "655630    8 -r-x------   1 yarn     yarn         4436 Aug 26 12:38 ./__spark_conf__/capacity-scheduler.xml\r\n",
      "655629    4 -r-x------   1 yarn     yarn         2130 Aug 26 12:38 ./__spark_conf__/core-site.xml\r\n",
      "655623    4 -r-x------   1 yarn     yarn          620 Aug 26 12:38 ./__spark_conf__/log4j.properties\r\n",
      "655631    4 -r-x------   1 yarn     yarn         3556 Aug 26 12:38 ./__spark_conf__/hdfs-site.xml\r\n",
      "655632    8 -r-x------   1 yarn     yarn         4567 Aug 26 12:38 ./__spark_conf__/yarn-env.sh\r\n",
      "655636    8 -r-x------   1 yarn     yarn         5308 Aug 26 12:38 ./__spark_conf__/metrics.properties\r\n",
      "655627    4 -r-x------   1 yarn     yarn         2419 Aug 26 12:38 ./__spark_conf__/hadoop-metrics2.properties\r\n",
      "655624    4 -r-x------   1 yarn     yarn            2 Aug 26 12:38 ./__spark_conf__/hosts.exclude\r\n",
      "655625    4 -r-x------   1 yarn     yarn         1921 Aug 26 12:38 ./__spark_conf__/mapred-site.xml\r\n",
      "655637    4 -r-x------   1 yarn     yarn          496 Aug 26 12:38 ./__spark_conf__/__spark_conf__.properties\r\n",
      "655634    4 -r-x------   1 yarn     yarn         1639 Aug 26 12:38 ./__spark_conf__/mapred-env.sh\r\n",
      "655626    8 -r-x------   1 yarn     yarn         4623 Aug 26 12:38 ./__spark_conf__/hadoop-env.sh\r\n",
      "655635    4 -r-x------   1 yarn     yarn           40 Aug 26 12:38 ./__spark_conf__/hosts.list\r\n",
      "655646    4 -rw-r--r--   1 yarn     yarn           74 Aug 26 12:38 ./container_tokens\r\n",
      "655652    4 -rwx------   1 yarn     yarn          754 Aug 26 12:38 ./default_container_executor.sh\r\n",
      "655651    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor_session.sh.crc\r\n",
      "655433  352 -r-x------   1 yarn     yarn       357163 Aug 26 12:38 ./pyspark.zip\r\n",
      "655648    8 -rwx------   1 yarn     yarn         6499 Aug 26 12:38 ./launch_container.sh\r\n",
      "655436    4 -r-x------   1 yarn     yarn          465 Aug 26 12:38 ./SimpleApp.py\r\n",
      "655650    4 -rwx------   1 yarn     yarn          700 Aug 26 12:38 ./default_container_executor_session.sh\r\n",
      "655451 181616 -r-x------   1 yarn     yarn     185971201 Aug 26 12:38 ./__spark__.jar\r\n",
      "655653    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor.sh.crc\r\n",
      "655645    4 drwx--x---   2 yarn     yarn         4096 Aug 26 12:38 ./tmp\r\n",
      "broken symlinks(find -L . -maxdepth 5 -type l -ls):\r\n",
      "End of LogType:directory.info\r\n",
      "\r\n",
      "LogType:launch_container.sh\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:6499\r\n",
      "Log Contents:\r\n",
      "#!/bin/bash\r\n",
      "\r\n",
      "export SPARK_YARN_MODE=\"true\"\r\n",
      "export SPARK_YARN_STAGING_DIR=\".sparkStaging/application_1472181274763_0002\"\r\n",
      "export JAVA_HOME=\"/usr/java/jdk1.7.0_75\"\r\n",
      "export SPARK_YARN_CACHE_FILES_VISIBILITIES=\"PRIVATE,PRIVATE,PRIVATE\"\r\n",
      "export NM_AUX_SERVICE_mapreduce_shuffle=\"AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\r",
      "\r\n",
      "\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\"\r\n",
      "export HADOOP_YARN_HOME=\"/usr/hdp/XXX.XXX.XXX.0-258/hadoop-yarn\"\r\n",
      "export NM_HOST=\"testvm007\"\r\n",
      "export PYTHONPATH=\":$PWD/pyspark.zip:$PWD/py4j-0.9-src.zip\"\r\n",
      "export APPLICATION_WEB_PROXY_BASE=\"/proxy/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES=\"40968\"\r\n",
      "export JVM_PID=\"$$\"\r\n",
      "export SPARK_USER=\"ansible\"\r\n",
      "export SPARK_YARN_CACHE_FILES_TIME_STAMPS=\"1472182701411,1472182701943,1472182702137\"\r\n",
      "export PWD=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000001\"\r\n",
      "export NM_PORT=\"45454\"\r\n",
      "export LOGNAME=\"ansible\"\r\n",
      "export APP_SUBMIT_TIME_ENV=\"1472182702615\"\r\n",
      "export MAX_APP_ATTEMPTS=\"2\"\r\n",
      "export MALLOC_ARENA_MAX=\"4\"\r\n",
      "export LOG_DIRS=\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001\"\r\n",
      "export SPARK_YARN_CACHE_FILES_FILE_SIZES=\"185971201,357163,44846\"\r\n",
      "export NM_HTTP_PORT=\"8042\"\r\n",
      "export LOCAL_DIRS=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_FILES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS=\"1472182702391\"\r\n",
      "export CLASSPATH=\"$PWD:$PWD/__spark_conf__:$PWD/__spark__.jar:/usr/hdp/current/hadoop-client/*:/usr/hdp/current/hadoop-client/lib/*:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/current/hadoop-mapreduce-client/lib/*:/usr/hdp/current/hadoop-yarn-client/*:/usr/hdp/current/hadoop-yarn-client/lib/*:/etc/tez/conf:/usr/hdp/current/tez-client/*:user/hdp/current/tez-client/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export SPARK_YARN_USER_ENV=\"CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export HADOOP_TOKEN_FILE_LOCATION=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000001/container_tokens\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES=\"PRIVATE\"\r\n",
      "export USER=\"ansible\"\r\n",
      "export CONTAINER_ID=\"container_1472181274763_0002_01_000001\"\r\n",
      "export HOME=\"/home/\"\r\n",
      "export HADOOP_CONF_DIR=\"/usr/hdp/current/hadoop-yarn-nodemanager/../hadoop/conf\"\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" \"__spark__.jar\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/__spark_conf__7022645124653218094.zip\" \"__spark_conf__\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/11/SimpleApp.py\" \"SimpleApp.py\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\" \"pyspark.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/py4j-0.9-src.zip\" \"py4j-0.9-src.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "# Creating copy of launch script\r\n",
      "cp \"launch_container.sh\" \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/launch_container.sh\"\r\n",
      "chmod 640 \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/launch_container.sh\"\r\n",
      "# Determining directory contents\r\n",
      "echo \"ls -l:\" 1>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "ls -l 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "echo \"find -L . -maxdepth 5 -ls:\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "find -L . -maxdepth 5 -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "find -L . -maxdepth 5 -type l -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/directory.info\"\r\n",
      "exec /bin/bash -c \"$JAVA_HOME/bin/java -server -Xmx1024m -Djava.io.tmpdir=$PWD/tmp -Dhdp.version=XXX.XXX.XXX.0-258 -Dspark.yarn.app.container.log.dir=/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001 -XX:MaxPermSize=256m org.apache.spark.deploy.yarn.ApplicationMaster --class 'org.apache.spark.deploy.PythonRunner' --primary-py-file SimpleApp.py --executor-memory 1024m --executor-cores 1 --properties-file $PWD/__spark_conf__/__spark_conf__.properties 1> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/stdout 2> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000001/stderr\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "End of LogType:launch_container.sh\r\n",
      "\r\n",
      "LogType:stderr\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:37445\r\n",
      "Log Contents:\r\n",
      "SLF4J: Class path contains multiple SLF4J bindings.\r\n",
      "SLF4J: Found binding in [jar:file:/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: Found binding in [jar:file:/usr/hdp/XXX.XXX.XXX.0-258/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\r\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\r\n",
      "16/08/26 12:38:27 INFO ApplicationMaster: Registered signal handlers for [TERM, HUP, INT]\r\n",
      "16/08/26 12:38:29 INFO ApplicationMaster: ApplicationAttemptId: appattempt_1472181274763_0002_000001\r\n",
      "16/08/26 12:38:31 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:31 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:31 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:32 INFO ApplicationMaster: Starting the user application in a separate Thread\r\n",
      "16/08/26 12:38:32 INFO ApplicationMaster: Waiting for spark context initialization\r\n",
      "16/08/26 12:38:32 INFO ApplicationMaster: Waiting for spark context initialization ... \r\n",
      "16/08/26 12:38:33 INFO SparkContext: Running Spark version 1.6.1\r\n",
      "16/08/26 12:38:33 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:33 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:33 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:33 INFO Utils: Successfully started service 'sparkDriver' on port 45460.\r\n",
      "16/08/26 12:38:34 INFO Slf4jLogger: Slf4jLogger started\r\n",
      "16/08/26 12:38:34 INFO Remoting: Starting remoting\r\n",
      "16/08/26 12:38:34 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@XXX.XXX.XXX.114:35006]\r\n",
      "16/08/26 12:38:34 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 35006.\r\n",
      "16/08/26 12:38:34 INFO SparkEnv: Registering MapOutputTracker\r\n",
      "16/08/26 12:38:34 INFO SparkEnv: Registering BlockManagerMaster\r\n",
      "16/08/26 12:38:34 INFO DiskBlockManager: Created local directory at /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/blockmgr-eb0f7305-2099-4295-a74e-3d6df98bb709\r\n",
      "16/08/26 12:38:34 INFO MemoryStore: MemoryStore started with capacity 457.9 MB\r\n",
      "16/08/26 12:38:34 INFO SparkEnv: Registering OutputCommitCoordinator\r\n",
      "16/08/26 12:38:34 INFO JettyUtils: Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\r\n",
      "16/08/26 12:38:34 INFO Server: jetty-8.y.z-SNAPSHOT\r\n",
      "16/08/26 12:38:35 INFO AbstractConnector: Started SelectChannelConnector@XXX.XXX.XXX.0:35817\r\n",
      "16/08/26 12:38:35 INFO Utils: Successfully started service 'SparkUI' on port 35817.\r\n",
      "16/08/26 12:38:35 INFO SparkUI: Bound SparkUI to XXX.XXX.XXX.0, and started at http://XXX.XXX.XXX.114:35817\r\n",
      "16/08/26 12:38:35 INFO YarnClusterScheduler: Created YarnClusterScheduler\r\n",
      "16/08/26 12:38:35 INFO FairSchedulableBuilder: Created default pool default, schedulingMode: FIFO, minShare: 0, weight: 1\r\n",
      "16/08/26 12:38:35 INFO SchedulerExtensionServices: Starting Yarn extension services with app application_1472181274763_0002 and attemptId Some(appattempt_1472181274763_0002_000001)\r\n",
      "16/08/26 12:38:35 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 34603.\r\n",
      "16/08/26 12:38:35 INFO NettyBlockTransferService: Server created on 34603\r\n",
      "16/08/26 12:38:35 INFO BlockManagerMaster: Trying to register BlockManager\r\n",
      "16/08/26 12:38:35 INFO BlockManagerMasterEndpoint: Registering block manager XXX.XXX.XXX.114:34603 with 457.9 MB RAM, BlockManagerId(driver, XXX.XXX.XXX.114, 34603)\r\n",
      "16/08/26 12:38:35 INFO BlockManagerMaster: Registered BlockManager\r\n",
      "16/08/26 12:38:35 INFO EventLoggingListener: Logging events to hdfs://hdfs-cluster/var/log/spark/application_1472181274763_0002_1\r\n",
      "16/08/26 12:38:35 INFO YarnSchedulerBackend$YarnSchedulerEndpoint: ApplicationMaster registered as NettyRpcEndpointRef(spark://YarnAM@XXX.XXX.XXX.114:45460)\r\n",
      "16/08/26 12:38:36 INFO YarnRMClient: Registering the ApplicationMaster\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Will request 4 executor containers, each with 1 cores and 1408 MB memory including 384 MB overhead\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Container request (host: Any, capability: <memory:1408, vCores:1>)\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Container request (host: Any, capability: <memory:1408, vCores:1>)\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Container request (host: Any, capability: <memory:1408, vCores:1>)\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Container request (host: Any, capability: <memory:1408, vCores:1>)\r\n",
      "16/08/26 12:38:36 INFO ApplicationMaster: Started progress reporter thread with (heartbeat : 3000, initial allocation : 200) intervals\r\n",
      "16/08/26 12:38:36 INFO AMRMClientImpl: Received new token for : testvm006:45454\r\n",
      "16/08/26 12:38:36 INFO AMRMClientImpl: Received new token for : testvm005:45454\r\n",
      "16/08/26 12:38:36 INFO AMRMClientImpl: Received new token for : testvm004:45454\r\n",
      "16/08/26 12:38:36 INFO AMRMClientImpl: Received new token for : testvm007:45454\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching container container_1472181274763_0002_01_000002 for on host testvm006\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460,  executorHostname: testvm006\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching container container_1472181274763_0002_01_000003 for on host testvm005\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Starting Executor Container\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460,  executorHostname: testvm005\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching container container_1472181274763_0002_01_000004 for on host testvm004\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460,  executorHostname: testvm004\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching container container_1472181274763_0002_01_000005 for on host testvm007\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460,  executorHostname: testvm007\r\n",
      "16/08/26 12:38:36 INFO YarnAllocator: Received 4 containers from YARN, launching executors on 4 of them.\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Starting Executor Container\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Starting Executor Container\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Starting Executor Container\r\n",
      "16/08/26 12:38:36 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0\r\n",
      "16/08/26 12:38:36 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0\r\n",
      "16/08/26 12:38:36 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Setting up ContainerLaunchContext\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Setting up ContainerLaunchContext\r\n",
      "16/08/26 12:38:36 INFO ContainerManagementProtocolProxy: yarn.client.max-cached-nodemanagers-proxies : 0\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Setting up ContainerLaunchContext\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Setting up ContainerLaunchContext\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Preparing Local resources\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Preparing Local resources\r\n",
      "16/08/26 12:38:36 INFO ExecutorRunnable: Preparing Local resources\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Preparing Local resources\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Prepared Local resources Map(__spark__.jar -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" } size: 185971201 timestamp: 1472182701411 type: FILE visibility: PRIVATE, pyspark.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip\" } size: 357163 timestamp: 1472182701943 type: FILE visibility: PRIVATE, py4j-0.9-src.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip\" } size: 44846 timestamp: 1472182702137 type: FILE visibility: PRIVATE, __spark_conf__ -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip\" } size: 40968 timestamp: 1472182702391 type: ARCHIVE visibility: PRIVATE)\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Prepared Local resources Map(__spark__.jar -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" } size: 185971201 timestamp: 1472182701411 type: FILE visibility: PRIVATE, pyspark.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip\" } size: 357163 timestamp: 1472182701943 type: FILE visibility: PRIVATE, py4j-0.9-src.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip\" } size: 44846 timestamp: 1472182702137 type: FILE visibility: PRIVATE, __spark_conf__ -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip\" } size: 40968 timestamp: 1472182702391 type: ARCHIVE visibility: PRIVATE)\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Prepared Local resources Map(__spark__.jar -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" } size: 185971201 timestamp: 1472182701411 type: FILE visibility: PRIVATE, pyspark.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip\" } size: 357163 timestamp: 1472182701943 type: FILE visibility: PRIVATE, py4j-0.9-src.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip\" } size: 44846 timestamp: 1472182702137 type: FILE visibility: PRIVATE, __spark_conf__ -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip\" } size: 40968 timestamp: 1472182702391 type: ARCHIVE visibility: PRIVATE)\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: Prepared Local resources Map(__spark__.jar -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" } size: 185971201 timestamp: 1472182701411 type: FILE visibility: PRIVATE, pyspark.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip\" } size: 357163 timestamp: 1472182701943 type: FILE visibility: PRIVATE, py4j-0.9-src.zip -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip\" } size: 44846 timestamp: 1472182702137 type: FILE visibility: PRIVATE, __spark_conf__ -> resource { scheme: \"hdfs\" host: \"hdfs-cluster\" port: -1 file: \"/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip\" } size: 40968 timestamp: 1472182702391 type: ARCHIVE visibility: PRIVATE)\r\n",
      "16/08/26 12:38:37 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\r\n",
      "16/08/26 12:38:37 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\r\n",
      "16/08/26 12:38:37 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\r\n",
      "16/08/26 12:38:37 WARN Client: No spark assembly jar for HDP on HDFS, defaultSparkAssembly:hdfs://hdfs-cluster/hdp/apps/XXX.XXX.XXX.0-258/spark/spark-hdp-assembly.jar\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: \r\n",
      "===============================================================================\r\n",
      "YARN executor launch context:\r\n",
      "  env:\r\n",
      "    CLASSPATH -> {{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark__.jar<CPS>/usr/hdp/current/hadoop-client/*<CPS>/usr/hdp/current/hadoop-client/lib/*<CPS>/usr/hdp/current/hadoop-hdfs-client/*<CPS>/usr/hdp/current/hadoop-hdfs-client/lib/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/lib/*<CPS>/usr/hdp/current/hadoop-yarn-client/*<CPS>/usr/hdp/current/hadoop-yarn-client/lib/*<CPS>/etc/tez/conf<CPS>/usr/hdp/current/tez-client/*<CPS>user/hdp/current/tez-client/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*<CPS>/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\r\n",
      "    SPARK_YARN_USER_ENV -> CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_LOG_URL_STDERR -> http://testvm004:8042/node/containerlogs/container_1472181274763_0002_01_000004/ansible/stderr?start=-4096\r\n",
      "    SPARK_YARN_CACHE_FILES_FILE_SIZES -> 185971201,357163,44846\r\n",
      "    SPARK_YARN_STAGING_DIR -> .sparkStaging/application_1472181274763_0002\r\n",
      "    SPARK_YARN_CACHE_FILES_VISIBILITIES -> PRIVATE,PRIVATE,PRIVATE\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES -> 40968\r\n",
      "    SPARK_USER -> ansible\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS -> 1472182702391\r\n",
      "    SPARK_YARN_MODE -> true\r\n",
      "    SPARK_YARN_CACHE_FILES_TIME_STAMPS -> 1472182701411,1472182701943,1472182702137\r\n",
      "    PYTHONPATH -> <CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.9-src.zip\r\n",
      "    SPARK_LOG_URL_STDOUT -> http://testvm004:8042/node/containerlogs/container_1472181274763_0002_01_000004/ansible/stdout?start=-4096\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES -> PRIVATE\r\n",
      "    SPARK_YARN_CACHE_FILES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\r\n",
      "\r\n",
      "  command:\r\n",
      "    {{JAVA_HOME}}/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=<LOG_DIR> -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 3 --hostname testvm004 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> <LOG_DIR>/stdout 2> <LOG_DIR>/stderr\r\n",
      "===============================================================================\r\n",
      "      \r\n",
      "16/08/26 12:38:37 INFO ContainerManagementProtocolProxy: Opening proxy : testvm004:45454\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: \r\n",
      "===============================================================================\r\n",
      "YARN executor launch context:\r\n",
      "  env:\r\n",
      "    CLASSPATH -> {{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark__.jar<CPS>/usr/hdp/current/hadoop-client/*<CPS>/usr/hdp/current/hadoop-client/lib/*<CPS>/usr/hdp/current/hadoop-hdfs-client/*<CPS>/usr/hdp/current/hadoop-hdfs-client/lib/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/lib/*<CPS>/usr/hdp/current/hadoop-yarn-client/*<CPS>/usr/hdp/current/hadoop-yarn-client/lib/*<CPS>/etc/tez/conf<CPS>/usr/hdp/current/tez-client/*<CPS>user/hdp/current/tez-client/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*<CPS>/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\r\n",
      "    SPARK_YARN_USER_ENV -> CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_LOG_URL_STDERR -> http://testvm006:8042/node/containerlogs/container_1472181274763_0002_01_000002/ansible/stderr?start=-4096\r\n",
      "    SPARK_YARN_CACHE_FILES_FILE_SIZES -> 185971201,357163,44846\r\n",
      "    SPARK_YARN_STAGING_DIR -> .sparkStaging/application_1472181274763_0002\r\n",
      "    SPARK_YARN_CACHE_FILES_VISIBILITIES -> PRIVATE,PRIVATE,PRIVATE\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES -> 40968\r\n",
      "    SPARK_USER -> ansible\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS -> 1472182702391\r\n",
      "    SPARK_YARN_MODE -> true\r\n",
      "    SPARK_YARN_CACHE_FILES_TIME_STAMPS -> 1472182701411,1472182701943,1472182702137\r\n",
      "    PYTHONPATH -> <CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.9-src.zip\r\n",
      "    SPARK_LOG_URL_STDOUT -> http://testvm006:8042/node/containerlogs/container_1472181274763_0002_01_000002/ansible/stdout?start=-4096\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES -> PRIVATE\r\n",
      "    SPARK_YARN_CACHE_FILES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\r\n",
      "\r\n",
      "  command:\r\n",
      "    {{JAVA_HOME}}/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=<LOG_DIR> -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 1 --hostname testvm006 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> <LOG_DIR>/stdout 2> <LOG_DIR>/stderr\r\n",
      "===============================================================================\r\n",
      "      \r\n",
      "16/08/26 12:38:37 INFO ContainerManagementProtocolProxy: Opening proxy : testvm006:45454\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: \r\n",
      "===============================================================================\r\n",
      "YARN executor launch context:\r\n",
      "  env:\r\n",
      "    CLASSPATH -> {{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark__.jar<CPS>/usr/hdp/current/hadoop-client/*<CPS>/usr/hdp/current/hadoop-client/lib/*<CPS>/usr/hdp/current/hadoop-hdfs-client/*<CPS>/usr/hdp/current/hadoop-hdfs-client/lib/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/lib/*<CPS>/usr/hdp/current/hadoop-yarn-client/*<CPS>/usr/hdp/current/hadoop-yarn-client/lib/*<CPS>/etc/tez/conf<CPS>/usr/hdp/current/tez-client/*<CPS>user/hdp/current/tez-client/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*<CPS>/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\r\n",
      "    SPARK_YARN_USER_ENV -> CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_LOG_URL_STDERR -> http://testvm007:8042/node/containerlogs/container_1472181274763_0002_01_000005/ansible/stderr?start=-4096\r\n",
      "    SPARK_YARN_CACHE_FILES_FILE_SIZES -> 185971201,357163,44846\r\n",
      "    SPARK_YARN_STAGING_DIR -> .sparkStaging/application_1472181274763_0002\r\n",
      "    SPARK_YARN_CACHE_FILES_VISIBILITIES -> PRIVATE,PRIVATE,PRIVATE\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES -> 40968\r\n",
      "    SPARK_USER -> ansible\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS -> 1472182702391\r\n",
      "    SPARK_YARN_MODE -> true\r\n",
      "    SPARK_YARN_CACHE_FILES_TIME_STAMPS -> 1472182701411,1472182701943,1472182702137\r\n",
      "    PYTHONPATH -> <CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.9-src.zip\r\n",
      "    SPARK_LOG_URL_STDOUT -> http://testvm007:8042/node/containerlogs/container_1472181274763_0002_01_000005/ansible/stdout?start=-4096\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES -> PRIVATE\r\n",
      "    SPARK_YARN_CACHE_FILES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\r\n",
      "\r\n",
      "  command:\r\n",
      "    {{JAVA_HOME}}/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=<LOG_DIR> -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 4 --hostname testvm007 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> <LOG_DIR>/stdout 2> <LOG_DIR>/stderr\r\n",
      "===============================================================================\r\n",
      "      \r\n",
      "16/08/26 12:38:37 INFO ContainerManagementProtocolProxy: Opening proxy : testvm007:45454\r\n",
      "16/08/26 12:38:37 INFO ExecutorRunnable: \r\n",
      "===============================================================================\r\n",
      "YARN executor launch context:\r\n",
      "  env:\r\n",
      "    CLASSPATH -> {{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark__.jar<CPS>/usr/hdp/current/hadoop-client/*<CPS>/usr/hdp/current/hadoop-client/lib/*<CPS>/usr/hdp/current/hadoop-hdfs-client/*<CPS>/usr/hdp/current/hadoop-hdfs-client/lib/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/*<CPS>/usr/hdp/current/hadoop-mapreduce-client/lib/*<CPS>/usr/hdp/current/hadoop-yarn-client/*<CPS>/usr/hdp/current/hadoop-yarn-client/lib/*<CPS>/etc/tez/conf<CPS>/usr/hdp/current/tez-client/*<CPS>user/hdp/current/tez-client/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*<CPS>/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\r\n",
      "    SPARK_YARN_USER_ENV -> CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\r\n",
      "    SPARK_LOG_URL_STDERR -> http://testvm005:8042/node/containerlogs/container_1472181274763_0002_01_000003/ansible/stderr?start=-4096\r\n",
      "    SPARK_YARN_CACHE_FILES_FILE_SIZES -> 185971201,357163,44846\r\n",
      "    SPARK_YARN_STAGING_DIR -> .sparkStaging/application_1472181274763_0002\r\n",
      "    SPARK_YARN_CACHE_FILES_VISIBILITIES -> PRIVATE,PRIVATE,PRIVATE\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES -> 40968\r\n",
      "    SPARK_USER -> ansible\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS -> 1472182702391\r\n",
      "    SPARK_YARN_MODE -> true\r\n",
      "    SPARK_YARN_CACHE_FILES_TIME_STAMPS -> 1472182701411,1472182701943,1472182702137\r\n",
      "    PYTHONPATH -> <CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.9-src.zip\r\n",
      "    SPARK_LOG_URL_STDOUT -> http://testvm005:8042/node/containerlogs/container_1472181274763_0002_01_000003/ansible/stdout?start=-4096\r\n",
      "    SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES -> PRIVATE\r\n",
      "    SPARK_YARN_CACHE_FILES -> hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\r\n",
      "\r\n",
      "  command:\r\n",
      "    {{JAVA_HOME}}/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=<LOG_DIR> -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 2 --hostname testvm005 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> <LOG_DIR>/stdout 2> <LOG_DIR>/stderr\r\n",
      "===============================================================================\r\n",
      "      \r\n",
      "16/08/26 12:38:37 INFO ContainerManagementProtocolProxy: Opening proxy : testvm005:45454\r\n",
      "16/08/26 12:38:45 INFO YarnClusterSchedulerBackend: Registered executor NettyRpcEndpointRef(null) (testvm007:44934) with ID 4\r\n",
      "16/08/26 12:38:45 INFO BlockManagerMasterEndpoint: Registering block manager testvm007:45839 with 511.5 MB RAM, BlockManagerId(4, testvm007, 45839)\r\n",
      "16/08/26 12:38:45 INFO YarnClusterSchedulerBackend: Registered executor NettyRpcEndpointRef(null) (testvm006:49693) with ID 1\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMasterEndpoint: Registering block manager testvm006:43374 with 511.5 MB RAM, BlockManagerId(1, testvm006, 43374)\r\n",
      "16/08/26 12:38:46 INFO YarnClusterSchedulerBackend: Registered executor NettyRpcEndpointRef(null) (testvm005:58981) with ID 2\r\n",
      "16/08/26 12:38:46 INFO BlockManagerMasterEndpoint: Registering block manager testvm005:45419 with 511.5 MB RAM, BlockManagerId(2, testvm005, 45419)\r\n",
      "16/08/26 12:38:47 INFO YarnClusterSchedulerBackend: Registered executor NettyRpcEndpointRef(null) (testvm004:37509) with ID 3\r\n",
      "16/08/26 12:38:47 INFO BlockManagerMasterEndpoint: Registering block manager testvm004:37901 with 511.5 MB RAM, BlockManagerId(3, testvm004, 37901)\r\n",
      "16/08/26 12:38:47 INFO YarnClusterSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8\r\n",
      "16/08/26 12:38:47 INFO YarnClusterScheduler: YarnClusterScheduler.postStartHook done\r\n",
      "16/08/26 12:38:49 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 218.8 KB, free 218.8 KB)\r\n",
      "16/08/26 12:38:49 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 25.3 KB, free 244.1 KB)\r\n",
      "16/08/26 12:38:49 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on XXX.XXX.XXX.114:34603 (size: 25.3 KB, free: 457.9 MB)\r\n",
      "16/08/26 12:38:49 INFO SparkContext: Created broadcast 0 from textFile at NativeMethodAccessorImpl.java:-2\r\n",
      "16/08/26 12:38:49 INFO FileInputFormat: Total input paths to process : 1\r\n",
      "16/08/26 12:38:49 INFO SparkContext: Starting job: count at SimpleApp.py:10\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Got job 0 (count at SimpleApp.py:10) with 2 output partitions\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Final stage: ResultStage 0 (count at SimpleApp.py:10)\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Parents of final stage: List()\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Missing parents: List()\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Submitting ResultStage 0 (PythonRDD[2] at count at SimpleApp.py:10), which has no missing parents\r\n",
      "16/08/26 12:38:49 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 6.8 KB, free 250.9 KB)\r\n",
      "16/08/26 12:38:49 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 3.9 KB, free 254.8 KB)\r\n",
      "16/08/26 12:38:49 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on XXX.XXX.XXX.114:34603 (size: 3.9 KB, free: 457.8 MB)\r\n",
      "16/08/26 12:38:49 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1006\r\n",
      "16/08/26 12:38:49 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 0 (PythonRDD[2] at count at SimpleApp.py:10)\r\n",
      "16/08/26 12:38:49 INFO YarnClusterScheduler: Adding task set 0.0 with 2 tasks\r\n",
      "16/08/26 12:38:50 INFO FairSchedulableBuilder: Added task set TaskSet_0 tasks to pool default\r\n",
      "16/08/26 12:38:50 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, testvm004, partition 0,NODE_LOCAL, 2144 bytes)\r\n",
      "16/08/26 12:38:50 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, testvm007, partition 1,NODE_LOCAL, 2144 bytes)\r\n",
      "16/08/26 12:38:50 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on testvm004:37901 (size: 3.9 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:38:50 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on testvm007:45839 (size: 3.9 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:38:50 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on testvm004:37901 (size: 25.3 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:38:50 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on testvm007:45839 (size: 25.3 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:38:52 INFO BlockManagerInfo: Added rdd_1_0 in memory on testvm004:37901 (size: 831.0 B, free: 511.5 MB)\r\n",
      "16/08/26 12:38:53 INFO BlockManagerInfo: Added rdd_1_1 in memory on testvm007:45839 (size: 833.0 B, free: 511.5 MB)\r\n",
      "16/08/26 12:38:54 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 4003 ms on testvm007 (1/2)\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: ResultStage 0 (count at SimpleApp.py:10) finished in 10.273 s\r\n",
      "16/08/26 12:39:00 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 10268 ms on testvm004 (2/2)\r\n",
      "16/08/26 12:39:00 INFO YarnClusterScheduler: Removed TaskSet 0.0, whose tasks have all completed, from pool default\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Job 0 finished: count at SimpleApp.py:10, took 10.620414 s\r\n",
      "16/08/26 12:39:00 INFO SparkContext: Starting job: count at SimpleApp.py:11\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Got job 1 (count at SimpleApp.py:11) with 2 output partitions\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Final stage: ResultStage 1 (count at SimpleApp.py:11)\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Parents of final stage: List()\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Missing parents: List()\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Submitting ResultStage 1 (PythonRDD[3] at count at SimpleApp.py:11), which has no missing parents\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 6.8 KB, free 261.6 KB)\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 3.9 KB, free 265.5 KB)\r\n",
      "16/08/26 12:39:00 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on XXX.XXX.XXX.114:34603 (size: 3.9 KB, free: 457.8 MB)\r\n",
      "16/08/26 12:39:00 INFO SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1006\r\n",
      "16/08/26 12:39:00 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 1 (PythonRDD[3] at count at SimpleApp.py:11)\r\n",
      "16/08/26 12:39:00 INFO YarnClusterScheduler: Adding task set 1.0 with 2 tasks\r\n",
      "16/08/26 12:39:00 INFO FairSchedulableBuilder: Added task set TaskSet_1 tasks to pool default\r\n",
      "16/08/26 12:39:00 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 2, testvm007, partition 1,PROCESS_LOCAL, 2144 bytes)\r\n",
      "16/08/26 12:39:00 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 3, testvm004, partition 0,PROCESS_LOCAL, 2144 bytes)\r\n",
      "16/08/26 12:39:00 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on testvm004:37901 (size: 3.9 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:39:00 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on testvm007:45839 (size: 3.9 KB, free: 511.5 MB)\r\n",
      "16/08/26 12:39:00 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 3) in 387 ms on testvm004 (1/2)\r\n",
      "16/08/26 12:39:01 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 2) in 470 ms on testvm007 (2/2)\r\n",
      "16/08/26 12:39:01 INFO YarnClusterScheduler: Removed TaskSet 1.0, whose tasks have all completed, from pool default\r\n",
      "16/08/26 12:39:01 INFO DAGScheduler: ResultStage 1 (count at SimpleApp.py:11) finished in 0.473 s\r\n",
      "16/08/26 12:39:01 INFO DAGScheduler: Job 1 finished: count at SimpleApp.py:11, took 0.501739 s\r\n",
      "16/08/26 12:39:01 INFO ApplicationMaster: Final app status: SUCCEEDED, exitCode: 0\r\n",
      "16/08/26 12:39:01 INFO SparkContext: Invoking stop() from shutdown hook\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/metrics/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null}\r\n",
      "16/08/26 12:39:01 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null}\r\n",
      "16/08/26 12:39:01 INFO SparkUI: Stopped Spark web UI at http://XXX.XXX.XXX.114:35817\r\n",
      "16/08/26 12:39:01 INFO YarnAllocator: Driver requested a total number of 0 executor(s).\r\n",
      "16/08/26 12:39:01 INFO YarnClusterSchedulerBackend: Shutting down all executors\r\n",
      "16/08/26 12:39:01 INFO YarnClusterSchedulerBackend: Asking each executor to shut down\r\n",
      "16/08/26 12:39:01 INFO SchedulerExtensionServices: Stopping SchedulerExtensionServices\r\n",
      "(serviceOption=None,\r\n",
      " services=List(),\r\n",
      " started=false)\r\n",
      "16/08/26 12:39:01 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!\r\n",
      "16/08/26 12:39:01 INFO MemoryStore: MemoryStore cleared\r\n",
      "16/08/26 12:39:01 INFO BlockManager: BlockManager stopped\r\n",
      "16/08/26 12:39:01 INFO BlockManagerMaster: BlockManagerMaster stopped\r\n",
      "16/08/26 12:39:01 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.\r\n",
      "16/08/26 12:39:01 INFO SparkContext: Successfully stopped SparkContext\r\n",
      "16/08/26 12:39:01 INFO ApplicationMaster: Unregistering ApplicationMaster with SUCCEEDED\r\n",
      "16/08/26 12:39:01 INFO AMRMClientImpl: Waiting for application to be successfully unregistered.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.\r\n",
      "16/08/26 12:39:01 INFO ApplicationMaster: Deleting staging directory .sparkStaging/application_1472181274763_0002\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Deleting directory /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/spark-30bca635-2d07-46f0-8106-cc0921bf377a/pyspark-98171430-3f25-4097-9194-efe9a754d39d\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Deleting directory /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/spark-30bca635-2d07-46f0-8106-cc0921bf377a\r\n",
      "End of LogType:stderr\r\n",
      "\r\n",
      "LogType:stdout\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:35\r\n",
      "Log Contents:\r\n",
      "Lines with a: 100, lines with b: 0\r\n",
      "End of LogType:stdout\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Container: container_1472181274763_0002_01_000005 on testvm007_45454\r\n",
      "======================================================================\r\n",
      "LogType:directory.info\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:3825\r\n",
      "Log Contents:\r\n",
      "ls -l:\r\n",
      "total 40\r\n",
      "lrwxrwxrwx 1 yarn yarn  128 Aug 26 12:38 __spark__.jar -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\r\n",
      "lrwxrwxrwx 1 yarn yarn  105 Aug 26 12:38 __spark_conf__ -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/__spark_conf__7022645124653218094.zip\r\n",
      "-rw-r--r-- 1 yarn yarn  109 Aug 26 12:38 container_tokens\r\n",
      "-rwx------ 1 yarn yarn  754 Aug 26 12:38 default_container_executor.sh\r\n",
      "-rwx------ 1 yarn yarn  700 Aug 26 12:38 default_container_executor_session.sh\r\n",
      "-rwx------ 1 yarn yarn 6500 Aug 26 12:38 launch_container.sh\r\n",
      "lrwxrwxrwx 1 yarn yarn   84 Aug 26 12:38 py4j-0.9-src.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/py4j-0.9-src.zip\r\n",
      "lrwxrwxrwx 1 yarn yarn   79 Aug 26 12:38 pyspark.zip -> /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\r\n",
      "drwx--x--- 2 yarn yarn 4096 Aug 26 12:38 tmp\r\n",
      "find -L . -maxdepth 5 -ls:\r\n",
      "655671    4 drwx--x---   3 yarn     yarn         4096 Aug 26 12:38 .\r\n",
      "655674    4 -rw-r--r--   1 yarn     yarn           12 Aug 26 12:38 ./.container_tokens.crc\r\n",
      "655676    4 -rw-r--r--   1 yarn     yarn           60 Aug 26 12:38 ./.launch_container.sh.crc\r\n",
      "655447   44 -r-x------   1 yarn     yarn        44846 Aug 26 12:38 ./py4j-0.9-src.zip\r\n",
      "655622    4 drwx------   2 yarn     yarn         4096 Aug 26 12:38 ./__spark_conf__\r\n",
      "655633    4 -r-x------   1 yarn     yarn         2490 Aug 26 12:38 ./__spark_conf__/hadoop-metrics.properties\r\n",
      "655628    8 -r-x------   1 yarn     yarn         4752 Aug 26 12:38 ./__spark_conf__/yarn-site.xml\r\n",
      "655630    8 -r-x------   1 yarn     yarn         4436 Aug 26 12:38 ./__spark_conf__/capacity-scheduler.xml\r\n",
      "655629    4 -r-x------   1 yarn     yarn         2130 Aug 26 12:38 ./__spark_conf__/core-site.xml\r\n",
      "655623    4 -r-x------   1 yarn     yarn          620 Aug 26 12:38 ./__spark_conf__/log4j.properties\r\n",
      "655631    4 -r-x------   1 yarn     yarn         3556 Aug 26 12:38 ./__spark_conf__/hdfs-site.xml\r\n",
      "655632    8 -r-x------   1 yarn     yarn         4567 Aug 26 12:38 ./__spark_conf__/yarn-env.sh\r\n",
      "655636    8 -r-x------   1 yarn     yarn         5308 Aug 26 12:38 ./__spark_conf__/metrics.properties\r\n",
      "655627    4 -r-x------   1 yarn     yarn         2419 Aug 26 12:38 ./__spark_conf__/hadoop-metrics2.properties\r\n",
      "655624    4 -r-x------   1 yarn     yarn            2 Aug 26 12:38 ./__spark_conf__/hosts.exclude\r\n",
      "655625    4 -r-x------   1 yarn     yarn         1921 Aug 26 12:38 ./__spark_conf__/mapred-site.xml\r\n",
      "655637    4 -r-x------   1 yarn     yarn          496 Aug 26 12:38 ./__spark_conf__/__spark_conf__.properties\r\n",
      "655634    4 -r-x------   1 yarn     yarn         1639 Aug 26 12:38 ./__spark_conf__/mapred-env.sh\r\n",
      "655626    8 -r-x------   1 yarn     yarn         4623 Aug 26 12:38 ./__spark_conf__/hadoop-env.sh\r\n",
      "655635    4 -r-x------   1 yarn     yarn           40 Aug 26 12:38 ./__spark_conf__/hosts.list\r\n",
      "655673    4 -rw-r--r--   1 yarn     yarn          109 Aug 26 12:38 ./container_tokens\r\n",
      "655679    4 -rwx------   1 yarn     yarn          754 Aug 26 12:38 ./default_container_executor.sh\r\n",
      "655678    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor_session.sh.crc\r\n",
      "655433  352 -r-x------   1 yarn     yarn       357163 Aug 26 12:38 ./pyspark.zip\r\n",
      "655675    8 -rwx------   1 yarn     yarn         6500 Aug 26 12:38 ./launch_container.sh\r\n",
      "655677    4 -rwx------   1 yarn     yarn          700 Aug 26 12:38 ./default_container_executor_session.sh\r\n",
      "655451 181616 -r-x------   1 yarn     yarn     185971201 Aug 26 12:38 ./__spark__.jar\r\n",
      "655680    4 -rw-r--r--   1 yarn     yarn           16 Aug 26 12:38 ./.default_container_executor.sh.crc\r\n",
      "655672    4 drwx--x---   2 yarn     yarn         4096 Aug 26 12:38 ./tmp\r\n",
      "broken symlinks(find -L . -maxdepth 5 -type l -ls):\r\n",
      "End of LogType:directory.info\r\n",
      "\r\n",
      "LogType:launch_container.sh\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:6500\r\n",
      "Log Contents:\r\n",
      "#!/bin/bash\r\n",
      "\r\n",
      "export SPARK_YARN_MODE=\"true\"\r\n",
      "export SPARK_YARN_STAGING_DIR=\".sparkStaging/application_1472181274763_0002\"\r\n",
      "export JAVA_HOME=\"/usr/java/jdk1.7.0_75\"\r\n",
      "export SPARK_YARN_CACHE_FILES_VISIBILITIES=\"PRIVATE,PRIVATE,PRIVATE\"\r\n",
      "export NM_AUX_SERVICE_mapreduce_shuffle=\"AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\r",
      "\r\n",
      "\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/__spark_conf__7022645124653218094.zip#__spark_conf__\"\r\n",
      "export SPARK_LOG_URL_STDERR=\"http://testvm007:8042/node/containerlogs/container_1472181274763_0002_01_000005/ansible/stderr?start=-4096\"\r\n",
      "export HADOOP_YARN_HOME=\"/usr/hdp/XXX.XXX.XXX.0-258/hadoop-yarn\"\r\n",
      "export NM_HOST=\"testvm007\"\r\n",
      "export PYTHONPATH=\":$PWD/pyspark.zip:$PWD/py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_FILE_SIZES=\"40968\"\r\n",
      "export JVM_PID=\"$$\"\r\n",
      "export SPARK_YARN_CACHE_FILES_TIME_STAMPS=\"1472182701411,1472182701943,1472182702137\"\r\n",
      "export SPARK_USER=\"ansible\"\r\n",
      "export PWD=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000005\"\r\n",
      "export NM_PORT=\"45454\"\r\n",
      "export LOGNAME=\"ansible\"\r\n",
      "export SPARK_LOG_URL_STDOUT=\"http://testvm007:8042/node/containerlogs/container_1472181274763_0002_01_000005/ansible/stdout?start=-4096\"\r\n",
      "export MALLOC_ARENA_MAX=\"4\"\r\n",
      "export LOG_DIRS=\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005\"\r\n",
      "export SPARK_YARN_CACHE_FILES_FILE_SIZES=\"185971201,357163,44846\"\r\n",
      "export NM_HTTP_PORT=\"8042\"\r\n",
      "export LOCAL_DIRS=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002\"\r\n",
      "export SPARK_YARN_CACHE_FILES=\"hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar#__spark__.jar,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/pyspark.zip#pyspark.zip,hdfs://hdfs-cluster/user/ansible/.sparkStaging/application_1472181274763_0002/py4j-0.9-src.zip#py4j-0.9-src.zip\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_TIME_STAMPS=\"1472182702391\"\r\n",
      "export CLASSPATH=\"$PWD:$PWD/__spark_conf__:$PWD/__spark__.jar:/usr/hdp/current/hadoop-client/*:/usr/hdp/current/hadoop-client/lib/*:/usr/hdp/current/hadoop-hdfs-client/*:/usr/hdp/current/hadoop-hdfs-client/lib/*:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/current/hadoop-mapreduce-client/lib/*:/usr/hdp/current/hadoop-yarn-client/*:/usr/hdp/current/hadoop-yarn-client/lib/*:/etc/tez/conf:/usr/hdp/current/tez-client/*:user/hdp/current/tez-client/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export SPARK_YARN_USER_ENV=\"CLASSPATH=/etc/hadoop/conf:/usr/hdp/current/hadoop-client/:/usr/hdp/current/hadoop-client/lib/:/usr/hdp/current/hadoop-hdfs-client/lib:/usr/hdp/current/hadoop-yarn-client/lib/\"\r\n",
      "export HADOOP_TOKEN_FILE_LOCATION=\"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/container_1472181274763_0002_01_000005/container_tokens\"\r\n",
      "export SPARK_YARN_CACHE_ARCHIVES_VISIBILITIES=\"PRIVATE\"\r\n",
      "export USER=\"ansible\"\r\n",
      "export CONTAINER_ID=\"container_1472181274763_0002_01_000005\"\r\n",
      "export HOME=\"/home/\"\r\n",
      "export HADOOP_CONF_DIR=\"/usr/hdp/current/hadoop-yarn-nodemanager/../hadoop/conf\"\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar\" \"__spark__.jar\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/13/__spark_conf__7022645124653218094.zip\" \"__spark_conf__\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/10/pyspark.zip\" \"pyspark.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "ln -sf \"/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/12/py4j-0.9-src.zip\" \"py4j-0.9-src.zip\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "# Creating copy of launch script\r\n",
      "cp \"launch_container.sh\" \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/launch_container.sh\"\r\n",
      "chmod 640 \"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/launch_container.sh\"\r\n",
      "# Determining directory contents\r\n",
      "echo \"ls -l:\" 1>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "ls -l 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "echo \"find -L . -maxdepth 5 -ls:\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "find -L . -maxdepth 5 -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "find -L . -maxdepth 5 -type l -ls 1>>\"/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/directory.info\"\r\n",
      "exec /bin/bash -c \"$JAVA_HOME/bin/java -server -XX:OnOutOfMemoryError='kill %p' -Xms1024m -Xmx1024m -Djava.io.tmpdir=$PWD/tmp '-Dspark.driver.port=45460' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir=/var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005 -XX:MaxPermSize=256m org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460 --executor-id 4 --hostname testvm007 --cores 1 --app-id application_1472181274763_0002 --user-class-path file:$PWD/__app__.jar 1> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/stdout 2> /var/log/hadoop-yarn/userlogs/application_1472181274763_0002/container_1472181274763_0002_01_000005/stderr\"\r\n",
      "hadoop_shell_errorcode=$?\r\n",
      "if [ $hadoop_shell_errorcode -ne 0 ]\r\n",
      "then\r\n",
      "  exit $hadoop_shell_errorcode\r\n",
      "fi\r\n",
      "End of LogType:launch_container.sh\r\n",
      "\r\n",
      "LogType:stderr\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:6172\r\n",
      "Log Contents:\r\n",
      "SLF4J: Class path contains multiple SLF4J bindings.\r\n",
      "SLF4J: Found binding in [jar:file:/hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/filecache/14/spark-assembly-XXX.XXX.XXX.2.4.2.0-258-hadoopXXX.XXX.XXX.2.4.2.0-258.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: Found binding in [jar:file:/usr/hdp/XXX.XXX.XXX.0-258/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]\r\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\r\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\r\n",
      "16/08/26 12:38:39 INFO CoarseGrainedExecutorBackend: Registered signal handlers for [TERM, HUP, INT]\r\n",
      "16/08/26 12:38:40 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:40 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:40 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing view acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: Changing modify acls to: yarn,ansible\r\n",
      "16/08/26 12:38:42 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(yarn, ansible); users with modify permissions: Set(yarn, ansible)\r\n",
      "16/08/26 12:38:44 INFO Slf4jLogger: Slf4jLogger started\r\n",
      "16/08/26 12:38:44 INFO Remoting: Starting remoting\r\n",
      "16/08/26 12:38:44 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkExecutorActorSystem@testvm007:39477]\r\n",
      "16/08/26 12:38:44 INFO Utils: Successfully started service 'sparkExecutorActorSystem' on port 39477.\r\n",
      "16/08/26 12:38:45 INFO DiskBlockManager: Created local directory at /hadoop/tmp/hadoop-yarn/nm-local-dir/usercache/ansible/appcache/application_1472181274763_0002/blockmgr-70eef8cb-21c3-4079-809b-c57dbddc342a\r\n",
      "16/08/26 12:38:45 INFO MemoryStore: MemoryStore started with capacity 511.5 MB\r\n",
      "16/08/26 12:38:45 INFO CoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@XXX.XXX.XXX.114:45460\r\n",
      "16/08/26 12:38:45 INFO CoarseGrainedExecutorBackend: Successfully registered with driver\r\n",
      "16/08/26 12:38:45 INFO Executor: Starting executor ID 4 on host testvm007\r\n",
      "16/08/26 12:38:45 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45839.\r\n",
      "16/08/26 12:38:45 INFO NettyBlockTransferService: Server created on 45839\r\n",
      "16/08/26 12:38:45 INFO BlockManagerMaster: Trying to register BlockManager\r\n",
      "16/08/26 12:38:45 INFO BlockManagerMaster: Registered BlockManager\r\n",
      "16/08/26 12:38:50 INFO CoarseGrainedExecutorBackend: Got assigned task 1\r\n",
      "16/08/26 12:38:50 INFO Executor: Running task 1.0 in stage 0.0 (TID 1)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Started reading broadcast variable 1\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 3.9 KB, free 3.9 KB)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Reading broadcast variable 1 took 250 ms\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 6.8 KB, free 10.7 KB)\r\n",
      "16/08/26 12:38:50 INFO CacheManager: Partition rdd_1_1 not found, computing it\r\n",
      "16/08/26 12:38:50 INFO HadoopRDD: Input split: hdfs://hdfs-cluster/dataset/iris/iris.data:2275+2276\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Started reading broadcast variable 0\r\n",
      "16/08/26 12:38:50 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 25.3 KB, free 35.9 KB)\r\n",
      "16/08/26 12:38:50 INFO TorrentBroadcast: Reading broadcast variable 0 took 19 ms\r\n",
      "16/08/26 12:38:51 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 321.8 KB, free 357.7 KB)\r\n",
      "16/08/26 12:38:52 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id\r\n",
      "16/08/26 12:38:52 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id\r\n",
      "16/08/26 12:38:52 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap\r\n",
      "16/08/26 12:38:52 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition\r\n",
      "16/08/26 12:38:52 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id\r\n",
      "16/08/26 12:38:53 INFO MemoryStore: Block rdd_1_1 stored as bytes in memory (estimated size 833.0 B, free 358.6 KB)\r\n",
      "16/08/26 12:38:54 INFO PythonRunner: Times: total = 254, boot = 230, init = 23, finish = 1\r\n",
      "16/08/26 12:38:54 INFO Executor: Finished task 1.0 in stage 0.0 (TID 1). 2704 bytes result sent to driver\r\n",
      "16/08/26 12:39:00 INFO CoarseGrainedExecutorBackend: Got assigned task 2\r\n",
      "16/08/26 12:39:00 INFO Executor: Running task 1.0 in stage 1.0 (TID 2)\r\n",
      "16/08/26 12:39:00 INFO TorrentBroadcast: Started reading broadcast variable 2\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 3.9 KB, free 362.4 KB)\r\n",
      "16/08/26 12:39:00 INFO TorrentBroadcast: Reading broadcast variable 2 took 354 ms\r\n",
      "16/08/26 12:39:00 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 6.8 KB, free 369.3 KB)\r\n",
      "16/08/26 12:39:00 INFO BlockManager: Found block rdd_1_1 locally\r\n",
      "16/08/26 12:39:00 INFO PythonRunner: Times: total = 43, boot = -6907, init = 6949, finish = 1\r\n",
      "16/08/26 12:39:00 INFO Executor: Finished task 1.0 in stage 1.0 (TID 2). 2124 bytes result sent to driver\r\n",
      "16/08/26 12:39:01 INFO CoarseGrainedExecutorBackend: Driver commanded a shutdown\r\n",
      "16/08/26 12:39:01 INFO MemoryStore: MemoryStore cleared\r\n",
      "16/08/26 12:39:01 INFO BlockManager: BlockManager stopped\r\n",
      "16/08/26 12:39:01 WARN CoarseGrainedExecutorBackend: An unknown (testvm007:45460) driver disconnected.\r\n",
      "16/08/26 12:39:01 ERROR CoarseGrainedExecutorBackend: Driver XXX.XXX.XXX.114:45460 disassociated! Shutting down.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.\r\n",
      "16/08/26 12:39:01 INFO ShutdownHookManager: Shutdown hook called\r\n",
      "16/08/26 12:39:01 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.\r\n",
      "End of LogType:stderr\r\n",
      "\r\n",
      "LogType:stdout\r\n",
      "Log Upload Time:Fri Aug 26 12:39:03 +0900 2016\r\n",
      "LogLength:0\r\n",
      "Log Contents:\r\n",
      "End of LogType:stdout16/08/26 12:40:19 INFO impl.TimelineClientImpl: Timeline service address: http://testvm003:8188/ws/v1/timeline/\r\n",
      "\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!ansible -a 'yarn logs -applicationId {yarn_application_id}' hadoop_client -l {target_group}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lc_cell_meme": {
     "current": "98b57a4a-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57b94-495a-11e8-a769-0242ac130002",
     "previous": "98b57900-495a-11e8-a769-0242ac130002"
    }
   },
   "source": [
    "# 後始末\n",
    "\n",
    "一時ディレクトリを削除する。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "lc_cell_meme": {
     "current": "98b57b94-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": "98b57cde-495a-11e8-a769-0242ac130002",
     "previous": "98b57a4a-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": [
    "!rm -fr {work_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "lc_cell_meme": {
     "current": "98b57cde-495a-11e8-a769-0242ac130002",
     "history": [],
     "next": null,
     "previous": "98b57b94-495a-11e8-a769-0242ac130002"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  },
  "lc_notebook_meme": {
   "current": "98b54cd2-495a-11e8-a769-0242ac130002",
   "history": [],
   "root_cells": [
    "98b54fa2-495a-11e8-a769-0242ac130002",
    "98b55146-495a-11e8-a769-0242ac130002",
    "98b552b8-495a-11e8-a769-0242ac130002",
    "98b55420-495a-11e8-a769-0242ac130002",
    "98b5557e-495a-11e8-a769-0242ac130002",
    "98b556dc-495a-11e8-a769-0242ac130002",
    "98b55826-495a-11e8-a769-0242ac130002",
    "98b5597a-495a-11e8-a769-0242ac130002",
    "98b55ace-495a-11e8-a769-0242ac130002",
    "98b55c22-495a-11e8-a769-0242ac130002",
    "98b55d76-495a-11e8-a769-0242ac130002",
    "98b55ec0-495a-11e8-a769-0242ac130002",
    "98b5601e-495a-11e8-a769-0242ac130002",
    "98b56172-495a-11e8-a769-0242ac130002",
    "98b562bc-495a-11e8-a769-0242ac130002",
    "98b56406-495a-11e8-a769-0242ac130002",
    "98b5655a-495a-11e8-a769-0242ac130002",
    "98b566a4-495a-11e8-a769-0242ac130002",
    "98b567ee-495a-11e8-a769-0242ac130002",
    "98b56942-495a-11e8-a769-0242ac130002",
    "98b56a8c-495a-11e8-a769-0242ac130002",
    "98b56be0-495a-11e8-a769-0242ac130002",
    "98b56d2a-495a-11e8-a769-0242ac130002",
    "98b56e74-495a-11e8-a769-0242ac130002",
    "98b56fbe-495a-11e8-a769-0242ac130002",
    "98b57112-495a-11e8-a769-0242ac130002",
    "98b5748c-495a-11e8-a769-0242ac130002",
    "98b57658-495a-11e8-a769-0242ac130002",
    "98b577b6-495a-11e8-a769-0242ac130002",
    "98b57900-495a-11e8-a769-0242ac130002",
    "98b57a4a-495a-11e8-a769-0242ac130002",
    "98b57b94-495a-11e8-a769-0242ac130002",
    "98b57cde-495a-11e8-a769-0242ac130002"
   ]
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}