PIFuHD_Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WYhlsDkg1Hwb"
      },
      "source": [
        "## Clone PIFuHD repository"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BmpEwdOd1G1z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "610a561d-4e0f-4c48-d94f-5efa5d0bf3d4"
      },
      "source": [
        "!git clone https://github.com/facebookresearch/pifuhd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'pifuhd'...\n",
            "remote: Enumerating objects: 222, done.\u001b[K\n",
            "remote: Counting objects: 100% (9/9), done.\u001b[K\n",
            "remote: Compressing objects: 100% (9/9), done.\u001b[K\n",
            "remote: Total 222 (delta 2), reused 0 (delta 0), pack-reused 213\u001b[K\n",
            "Receiving objects: 100% (222/222), 409.64 KiB | 12.80 MiB/s, done.\n",
            "Resolving deltas: 100% (106/106), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QvQm-A8ESKb2"
      },
      "source": [
        "## Configure input data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xvle9T10fB6g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f4f31eb-de5b-4576-c899-3d44c8cd057c"
      },
      "source": [
        "cd /content/pifuhd/sample_images"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/pifuhd/sample_images\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9SI7Ye1JfIim"
      },
      "source": [
        "**If you want to upload your own picture, run the next cell**. Otherwise, go to the next next cell. Currently PNG, JPEG files are supported."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jaV_7Yi8fM-B",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "9836d6a1-1b7b-4a2c-918d-3602cc37d5c6"
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "filename = list(files.upload().keys())[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-dc3fede3-10f2-42d7-b678-dbc2b7ff2f55\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-dc3fede3-10f2-42d7-b678-dbc2b7ff2f55\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving tree.jpeg to tree.jpeg\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AEzmmB01SOZp"
      },
      "source": [
        "import os\n",
        "\n",
        "try:\n",
        "  image_path = '/content/pifuhd/sample_images/%s' % filename\n",
        "except:\n",
        "  image_path = '/content/pifuhd/sample_images/test.png' # example image\n",
        "image_dir = os.path.dirname(image_path)\n",
        "file_name = os.path.splitext(os.path.basename(image_path))[0]\n",
        "\n",
        "# output pathes\n",
        "obj_path = '/content/pifuhd/results/pifuhd_final/recon/result_%s_256.obj' % file_name\n",
        "out_img_path = '/content/pifuhd/results/pifuhd_final/recon/result_%s_256.png' % file_name\n",
        "video_path = '/content/pifuhd/results/pifuhd_final/recon/result_%s_256.mp4' % file_name\n",
        "video_display_path = '/content/pifuhd/results/pifuhd_final/result_%s_256_display.mp4' % file_name"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "896EC7iQfXkj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c5990969-0c0b-472f-db44-6242699fbf95"
      },
      "source": [
        "cd /content"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbVmda9J5TDL"
      },
      "source": [
        "## Preprocess (for cropping image)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UtMjWGNU5STe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9fea41df-5e52-411c-9317-8b4c55001252"
      },
      "source": [
        "!git clone https://github.com/Daniil-Osokin/lightweight-human-pose-estimation.pytorch.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'lightweight-human-pose-estimation.pytorch'...\n",
            "remote: Enumerating objects: 120, done.\u001b[K\n",
            "remote: Total 120 (delta 0), reused 0 (delta 0), pack-reused 120\u001b[K\n",
            "Receiving objects: 100% (120/120), 222.76 KiB | 7.68 MiB/s, done.\n",
            "Resolving deltas: 100% (53/53), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-vYklhI5dab",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69c74520-f024-4524-ad65-4aee22974704"
      },
      "source": [
        "cd /content/lightweight-human-pose-estimation.pytorch/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/lightweight-human-pose-estimation.pytorch\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dRod9SOu77I6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6e082c16-8d27-4c62-9351-83b09078da3f"
      },
      "source": [
        "!wget https://download.01.org/opencv/openvino_training_extensions/models/human_pose_estimation/checkpoint_iter_370000.pth"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-04-18 02:31:09--  https://download.01.org/opencv/openvino_training_extensions/models/human_pose_estimation/checkpoint_iter_370000.pth\n",
            "Resolving download.01.org (download.01.org)... 23.198.26.238, 2a02:26f0:c900:29d::4b21, 2a02:26f0:c900:296::4b21\n",
            "Connecting to download.01.org (download.01.org)|23.198.26.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 87959810 (84M) [application/octet-stream]\n",
            "Saving to: ‘checkpoint_iter_370000.pth’\n",
            "\n",
            "checkpoint_iter_370 100%[===================>]  83.88M  73.3MB/s    in 1.1s    \n",
            "\n",
            "2023-04-18 02:31:11 (73.3 MB/s) - ‘checkpoint_iter_370000.pth’ saved [87959810/87959810]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PdRcDXe38lHB"
      },
      "source": [
        "import torch\n",
        "import cv2\n",
        "import numpy as np\n",
        "from models.with_mobilenet import PoseEstimationWithMobileNet\n",
        "from modules.keypoints import extract_keypoints, group_keypoints\n",
        "from modules.load_state import load_state\n",
        "from modules.pose import Pose, track_poses\n",
        "import demo\n",
        "\n",
        "def get_rect(net, images, height_size):\n",
        "    net = net.eval()\n",
        "\n",
        "    stride = 8\n",
        "    upsample_ratio = 4\n",
        "    num_keypoints = Pose.num_kpts\n",
        "    previous_poses = []\n",
        "    delay = 33\n",
        "    for image in images:\n",
        "        rect_path = image.replace('.%s' % (image.split('.')[-1]), '_rect.txt')\n",
        "        img = cv2.imread(image, cv2.IMREAD_COLOR)\n",
        "        orig_img = img.copy()\n",
        "        orig_img = img.copy()\n",
        "        heatmaps, pafs, scale, pad = demo.infer_fast(net, img, height_size, stride, upsample_ratio, cpu=False)\n",
        "\n",
        "        total_keypoints_num = 0\n",
        "        all_keypoints_by_type = []\n",
        "        for kpt_idx in range(num_keypoints):  # 19th for bg\n",
        "            total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)\n",
        "\n",
        "        pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs)\n",
        "        for kpt_id in range(all_keypoints.shape[0]):\n",
        "            all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale\n",
        "            all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale\n",
        "        current_poses = []\n",
        "\n",
        "        rects = []\n",
        "        for n in range(len(pose_entries)):\n",
        "            if len(pose_entries[n]) == 0:\n",
        "                continue\n",
        "            pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1\n",
        "            valid_keypoints = []\n",
        "            for kpt_id in range(num_keypoints):\n",
        "                if pose_entries[n][kpt_id] != -1.0:  # keypoint was found\n",
        "                    pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])\n",
        "                    pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])\n",
        "                    valid_keypoints.append([pose_keypoints[kpt_id, 0], pose_keypoints[kpt_id, 1]])\n",
        "            valid_keypoints = np.array(valid_keypoints)\n",
        "            \n",
        "            if pose_entries[n][10] != -1.0 or pose_entries[n][13] != -1.0:\n",
        "              pmin = valid_keypoints.min(0)\n",
        "              pmax = valid_keypoints.max(0)\n",
        "\n",
        "              center = (0.5 * (pmax[:2] + pmin[:2])).astype(np.int)\n",
        "              radius = int(0.65 * max(pmax[0]-pmin[0], pmax[1]-pmin[1]))\n",
        "            elif pose_entries[n][10] == -1.0 and pose_entries[n][13] == -1.0 and pose_entries[n][8] != -1.0 and pose_entries[n][11] != -1.0:\n",
        "              # if leg is missing, use pelvis to get cropping\n",
        "              center = (0.5 * (pose_keypoints[8] + pose_keypoints[11])).astype(np.int)\n",
        "              radius = int(1.45*np.sqrt(((center[None,:] - valid_keypoints)**2).sum(1)).max(0))\n",
        "              center[1] += int(0.05*radius)\n",
        "            else:\n",
        "              center = np.array([img.shape[1]//2,img.shape[0]//2])\n",
        "              radius = max(img.shape[1]//2,img.shape[0]//2)\n",
        "\n",
        "            x1 = center[0] - radius\n",
        "            y1 = center[1] - radius\n",
        "\n",
        "            rects.append([x1, y1, 2*radius, 2*radius])\n",
        "\n",
        "        np.savetxt(rect_path, np.array(rects), fmt='%d')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6cGZD6f6IaY"
      },
      "source": [
        "net = PoseEstimationWithMobileNet()\n",
        "checkpoint = torch.load('checkpoint_iter_370000.pth', map_location='cpu')\n",
        "load_state(net, checkpoint)\n",
        "\n",
        "get_rect(net.cuda(), [image_path], 512)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y0rgMInwTt0s"
      },
      "source": [
        "## Download the Pretrained Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrIcZweSNRFI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d64e509b-5e6b-47b6-d0de-14a49872eaf9"
      },
      "source": [
        "cd /content/pifuhd/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/pifuhd\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3jjm6HuQRk8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f4e1fe3-5415-4f03-aa90-e24b1a734a32"
      },
      "source": [
        "!sh ./scripts/download_trained_model.sh"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+ mkdir -p checkpoints\n",
            "+ cd checkpoints\n",
            "+ wget https://dl.fbaipublicfiles.com/pifuhd/checkpoints/pifuhd.pt pifuhd.pt\n",
            "--2023-04-18 02:31:17--  https://dl.fbaipublicfiles.com/pifuhd/checkpoints/pifuhd.pt\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 13.227.219.59, 13.227.219.70, 13.227.219.10, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|13.227.219.59|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1548375177 (1.4G) [application/octet-stream]\n",
            "Saving to: ‘pifuhd.pt’\n",
            "\n",
            "pifuhd.pt           100%[===================>]   1.44G   260MB/s    in 6.0s    \n",
            "\n",
            "2023-04-18 02:31:23 (248 MB/s) - ‘pifuhd.pt’ saved [1548375177/1548375177]\n",
            "\n",
            "--2023-04-18 02:31:23--  http://pifuhd.pt/\n",
            "Resolving pifuhd.pt (pifuhd.pt)... failed: Name or service not known.\n",
            "wget: unable to resolve host address ‘pifuhd.pt’\n",
            "FINISHED --2023-04-18 02:31:23--\n",
            "Total wall clock time: 6.1s\n",
            "Downloaded: 1 files, 1.4G in 6.0s (248 MB/s)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6heKcA-0QEBw"
      },
      "source": [
        "## Run PIFuHD!\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5995t2PnQTmG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "582d4166-71a6-4d82-f9b9-a682845196ce"
      },
      "source": [
        "# Warning: all images with the corresponding rectangle files under -i will be processed. \n",
        "!python -m apps.simple_test -r 256 --use_rect -i $image_dir\n",
        "\n",
        "# seems that 256 is the maximum resolution that can fit into Google Colab. \n",
        "# If you want to reconstruct a higher-resolution mesh, please try with your own machine. "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Resuming from  ./checkpoints/pifuhd.pt\n",
            "Warning: opt is overwritten.\n",
            "test data size:  1\n",
            "initialize network with normal\n",
            "initialize network with normal\n",
            "generate mesh (test) ...\n",
            "  0% 0/1 [00:00<?, ?it/s]/content/pifuhd/lib/data/EvalDataset.py:87: UserWarning: loadtxt: Empty input file: \"/content/pifuhd/sample_images/tree_rect.txt\"\n",
            "  rects = np.loadtxt(rect_path, dtype=np.int32)\n",
            "  0% 0/1 [00:00<?, ?it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.9/runpy.py\", line 197, in _run_module_as_main\n",
            "    return _run_code(code, main_globals, None,\n",
            "  File \"/usr/lib/python3.9/runpy.py\", line 87, in _run_code\n",
            "    exec(code, run_globals)\n",
            "  File \"/content/pifuhd/apps/simple_test.py\", line 30, in <module>\n",
            "    reconWrapper(cmd, args.use_rect)\n",
            "  File \"/content/pifuhd/apps/recon.py\", line 220, in reconWrapper\n",
            "    recon(opt, use_rect)\n",
            "  File \"/content/pifuhd/apps/recon.py\", line 205, in recon\n",
            "    test_data = test_dataset[i]\n",
            "  File \"/content/pifuhd/lib/data/EvalDataset.py\", line 131, in __getitem__\n",
            "    return self.get_item(index)\n",
            "  File \"/content/pifuhd/lib/data/EvalDataset.py\", line 93, in get_item\n",
            "    im = crop_image(im, rect)\n",
            "  File \"/content/pifuhd/lib/data/EvalDataset.py\", line 17, in crop_image\n",
            "    x, y, w, h = rect\n",
            "ValueError: not enough values to unpack (expected 4, got 0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EUZ8Nt5rNFXZ"
      },
      "source": [
        "## Render the result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xp5s5uiOiDv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fe831f11-8379-465d-ff39-4eea81f8d23a"
      },
      "source": [
        "import sys\n",
        "import torch\n",
        "pyt_version_str=torch.__version__.split(\"+\")[0].replace(\".\", \"\")\n",
        "version_str=\"\".join([\n",
        "    f\"py3{sys.version_info.minor}_cu\",\n",
        "    torch.version.cuda.replace(\".\",\"\"),\n",
        "    f\"_pyt{pyt_version_str}\"\n",
        "])\n",
        "!pip install fvcore iopath\n",
        "!pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting fvcore\n",
            "  Downloading fvcore-0.1.5.post20221221.tar.gz (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.2/50.2 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting iopath\n",
            "  Downloading iopath-0.1.10.tar.gz (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from fvcore) (1.22.4)\n",
            "Collecting yacs>=0.1.6\n",
            "  Downloading yacs-0.1.8-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from fvcore) (6.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from fvcore) (4.65.0)\n",
            "Requirement already satisfied: termcolor>=1.1 in /usr/local/lib/python3.9/dist-packages (from fvcore) (2.2.0)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.9/dist-packages (from fvcore) (8.4.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/dist-packages (from fvcore) (0.8.10)\n",
            "Requirement already satisfied: typing_extensions in /usr/local/lib/python3.9/dist-packages (from iopath) (4.5.0)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n",
            "Building wheels for collected packages: fvcore, iopath\n",
            "  Building wheel for fvcore (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fvcore: filename=fvcore-0.1.5.post20221221-py3-none-any.whl size=61429 sha256=933588a3e84a5c1d8c723b8c4f99cc96e63fe2c891464b62b61582264b9127c3\n",
            "  Stored in directory: /root/.cache/pip/wheels/83/42/02/66178d16e5c44dc26d309931834956baeda371956e86fbd876\n",
            "  Building wheel for iopath (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for iopath: filename=iopath-0.1.10-py3-none-any.whl size=31547 sha256=8d48f035b1fc8c3240544f887d9a139481cc24fb20860f7720c6a19f8477d96a\n",
            "  Stored in directory: /root/.cache/pip/wheels/c1/13/6d/441d8f2af76ee6d2a3e67eebb1d0c556fefcee0a8b32266a8e\n",
            "Successfully built fvcore iopath\n",
            "Installing collected packages: yacs, portalocker, iopath, fvcore\n",
            "Successfully installed fvcore-0.1.5.post20221221 iopath-0.1.10 portalocker-2.7.0 yacs-0.1.8\n",
            "Looking in links: https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py39_cu118_pyt200/download.html\n",
            "Collecting pytorch3d\n",
            "  Downloading https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py39_cu118_pyt200/pytorch3d-0.7.3-cp39-cp39-linux_x86_64.whl (20.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.1/20.1 MB\u001b[0m \u001b[31m178.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: iopath in /usr/local/lib/python3.9/dist-packages (from pytorch3d) (0.1.10)\n",
            "Requirement already satisfied: fvcore in /usr/local/lib/python3.9/dist-packages (from pytorch3d) (0.1.5.post20221221)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (6.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (1.22.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (4.65.0)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (8.4.0)\n",
            "Requirement already satisfied: yacs>=0.1.6 in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (0.1.8)\n",
            "Requirement already satisfied: termcolor>=1.1 in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (2.2.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/dist-packages (from fvcore->pytorch3d) (0.8.10)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from iopath->pytorch3d) (4.5.0)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.9/dist-packages (from iopath->pytorch3d) (2.7.0)\n",
            "Installing collected packages: pytorch3d\n",
            "Successfully installed pytorch3d-0.7.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "afwL_-ROCmDf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "459ec229-c5b0-4867-9475-c4150c0eb499"
      },
      "source": [
        "from lib.colab_util import generate_video_from_obj, set_renderer, video\n",
        "\n",
        "renderer = set_renderer()\n",
        "generate_video_from_obj(obj_path, out_img_path, video_path, renderer)\n",
        "\n",
        "# we cannot play a mp4 video generated by cv2\n",
        "!ffmpeg -i $video_path -vcodec libx264 $video_display_path -y -loglevel quiet\n",
        "video(video_display_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-22614b7ef849>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mrenderer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset_renderer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgenerate_video_from_obj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_img_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvideo_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# we cannot play a mp4 video generated by cv2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/pifuhd/lib/colab_util.py\u001b[0m in \u001b[0;36mgenerate_video_from_obj\u001b[0;34m(obj_path, image_path, video_path, renderer)\u001b[0m\n\u001b[1;32m     99\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgenerate_video_from_obj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimage_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvideo_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0minput_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m     \u001b[0minput_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_image\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0minput_image\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m//\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m     \u001b[0minput_image\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_image\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'shape'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1CBxbdrM9F-9"
      },
      "source": [
        "import IPython\n",
        "IPython.display.HTML('<a href=\"https://twitter.com/intent/tweet?button_hashtag=pifuhd&ref_src=twsrc%5Etfw\" class=\"twitter-hashtag-button\" data-size=\"large\" data-text=\"Google Colab Link: \" data-url=\"https://bit.ly/37sfogZ\" data-show-count=\"false\">Tweet #pifuhd</a><script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script>  (Don\\'t forget to add your result to the tweet!)')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68JDAYJFSFMV"
      },
      "source": [
        "IPython.display.HTML('<h2>Rigging (Mixamo) + Photoreal Rendering (Blender)</h2><blockquote class=\"twitter-tweet\"><p lang=\"pt\" dir=\"ltr\">vcs ainda tem a PACHORRA de me dizer que eu não sei dançar<a href=\"https://twitter.com/hashtag/b3d?src=hash&amp;ref_src=twsrc%5Etfw\">#b3d</a> <a href=\"https://twitter.com/hashtag/pifuhd?src=hash&amp;ref_src=twsrc%5Etfw\">#pifuhd</a> <a href=\"https://t.co/kHCnLh6zxH\">pic.twitter.com/kHCnLh6zxH</a></p>&mdash; lukas arendero (@lukazvd) <a href=\"https://twitter.com/lukazvd/status/1274810484798128131?ref_src=twsrc%5Etfw\">June 21, 2020</a></blockquote> <script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script><h2>FaceApp + Rigging (Mixamo)</h2><blockquote class=\"twitter-tweet\"><p lang=\"ja\" dir=\"ltr\">カツラかぶってる自分に見える <a href=\"https://twitter.com/hashtag/pifuhd?src=hash&amp;ref_src=twsrc%5Etfw\">#pifuhd</a> <a href=\"https://t.co/V8o7VduTiG\">pic.twitter.com/V8o7VduTiG</a></p>&mdash; Shuhei Tsuchida (@shuhei2306) <a href=\"https://twitter.com/shuhei2306/status/1274507242910314498?ref_src=twsrc%5Etfw\">June 21, 2020</a></blockquote> <script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script><h2>Rigging (Mixamo) + AR (Adobe Aero)</AR><blockquote class=\"twitter-tweet\"><p lang=\"ja\" dir=\"ltr\">写真→PIFuHD→Mixamo→AdobeAeroでサウンド付きARを作成。Zip化してLINEでARコンテンツを共有。<br>写真が1枚あれば簡単にARの3Dアニメーションが作れる時代…凄い。<a href=\"https://twitter.com/hashtag/PIFuHD?src=hash&amp;ref_src=twsrc%5Etfw\">#PIFuHD</a> <a href=\"https://twitter.com/hashtag/AdobeAero?src=hash&amp;ref_src=twsrc%5Etfw\">#AdobeAero</a> <a href=\"https://twitter.com/hashtag/Mixamo?src=hash&amp;ref_src=twsrc%5Etfw\">#Mixamo</a> <a href=\"https://t.co/CbiMi4gZ0K\">pic.twitter.com/CbiMi4gZ0K</a></p>&mdash; モジョン (@mojon1) <a href=\"https://twitter.com/mojon1/status/1273217947872317441?ref_src=twsrc%5Etfw\">June 17, 2020</a></blockquote> <script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script><h2>3D Printing</h2><blockquote class=\"twitter-tweet\"><p lang=\"ja\" dir=\"ltr\"><a href=\"https://twitter.com/hashtag/pifuhd?src=hash&amp;ref_src=twsrc%5Etfw\">#pifuhd</a> 楽しい〜<br>小さい自分プリントした <a href=\"https://t.co/4qyWuij0Hs\">pic.twitter.com/4qyWuij0Hs</a></p>&mdash; isb (@vxzxzxzxv) <a href=\"https://twitter.com/vxzxzxzxv/status/1273136266406694913?ref_src=twsrc%5Etfw\">June 17, 2020</a></blockquote> <script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script>')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lX5CTTW_KWhQ"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    }
  ]
}