1.Download_All.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download All Files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import shutil\n",
    "import gzip\n",
    "import tarfile\n",
    "import re\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Z-C7OTd7nwnz",
    "outputId": "407c1656-7f57-450c-b608-a2d6b275a891"
   },
   "outputs": [],
   "source": [
    "use_google_drive = False\n",
    "\n",
    "try:\n",
    "  import google.colab\n",
    "  from google.colab import drive\n",
    "  use_google_drive = True\n",
    "except Exception:\n",
    "  pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "2bpgy4bTnktG"
   },
   "outputs": [],
   "source": [
    "if use_google_drive:\n",
    "  metadata_folder = \"/content/gdrive/MyDrive/ColabData/amazon/\"\n",
    "  metadata_source = \"/content/gdrive/MyDrive/ColabData/amazon/metadata.csv\"\n",
    "  tar_dataset = \"/content/gdrive/MyDrive/ColabData/amazon/abo-images-small.tar\"\n",
    "\n",
    "  drive.mount(\"/content/gdrive\")\n",
    "else:\n",
    "  metadata_folder = \"metadata\\\\\"\n",
    "  metadata_source = \"metadata\\\\metadata.csv\"\n",
    "  tar_dataset = \"dataset\\\\abo-images-small.tar\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://stackoverflow.com/a/57923425/1930773\n",
    "\n",
    "def gzip_file(gzip_file, work_dir):\n",
    "  filename = os.path.split(gzip_file)[-1]\n",
    "  filename = re.sub(r\"\\.gz$\", \"\", filename, flags=re.IGNORECASE)\n",
    "\n",
    "  with gzip.open(gzip_file, 'rb') as f_in:\n",
    "    with open(os.path.join(work_dir, filename), 'wb') as f_out:\n",
    "      shutil.copyfileobj(f_in, f_out)\n",
    "\n",
    "\n",
    "if not 'gz' in [format[0] for format in shutil.get_unpack_formats()]:\n",
    "  shutil.register_unpack_format('gz', ['.gz', ], gzip_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(f\"{metadata_folder}\\\\abo-listings.tar\"):\n",
    "  urllib.request.urlretrieve(\"https://amazon-berkeley-objects.s3.amazonaws.com/archives/abo-listings.tar\", f\"{metadata_folder}\\\\abo-listings.tar\")\n",
    "\n",
    "if not os.path.exists(tar_dataset):\n",
    "  urllib.request.urlretrieve(\"https://amazon-berkeley-objects.s3.amazonaws.com/archives/abo-images-small.tar\", tar_dataset)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tar = tarfile.open(tar_dataset)\n",
    "tar.extract(tar.getmember(\"images/metadata/images.csv.gz\"), path=metadata_folder)\n",
    "shutil.unpack_archive(os.path.join(metadata_folder, \"images\", \"metadata\", \"images.csv.gz\"), metadata_folder)\n",
    "shutil.rmtree(os.path.join(metadata_folder, \"images\"), ignore_errors=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "shutil.unpack_archive(os.path.join(metadata_folder, \"abo-listings.tar\"), metadata_folder)\n",
    "\n",
    "for gzfile in os.listdir(os.path.join(metadata_folder, \"listings\", \"metadata\")):\n",
    "  shutil.unpack_archive(os.path.join(metadata_folder, \"listings\", \"metadata\", gzfile), metadata_folder)\n",
    "\n",
    "shutil.rmtree(os.path.join(metadata_folder, \"listings\"), ignore_errors=True)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "26de051ba29f2982a8de78e945f0abaf191376122a1563185a90213a26c5da77"
  },
  "kernelspec": {
   "display_name": "Python 3.10.1 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.1 (tags/v3.10.1:2cd268a, Dec  6 2021, 19:10:37) [MSC v.1929 64 bit (AMD64)]"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}