Captioning Google Photos Blip 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f7e168e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google_auth_oauthlib.flow import InstalledAppFlow\n",
    "from google.auth.transport.requests import Request\n",
    "import os\n",
    "import pickle\n",
    "import requests\n",
    "from PIL import Image\n",
    "from IPython.display import display\n",
    "\n",
    "# import and set Processor and Model\n",
    "\n",
    "from transformers import BlipProcessor, BlipForConditionalGeneration\n",
    "\n",
    "processor = BlipProcessor.from_pretrained(\"Salesforce/blip-image-captioning-base\")\n",
    "model = BlipForConditionalGeneration.from_pretrained(\"Salesforce/blip-image-captioning-base\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35633a26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# defined captioning function\n",
    "\n",
    "def caption(img):\n",
    "    raw_image = img.convert('RGB')\n",
    "    inputs = processor(raw_image, return_tensors=\"pt\")\n",
    "\n",
    "    out = model.generate(**inputs)\n",
    "    return processor.decode(out[0], skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dbef584",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set location of client secret file you got from Google Cloud Console and required API scopes\n",
    "\n",
    "CLIENT_SECRET = 'client_secret.json'\n",
    "SCOPES = ['https://www.googleapis.com/auth/photoslibrary']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dff915b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#authenticate and authorize, save token in a .pickle\n",
    "\n",
    "creds = None\n",
    "if os.path.exists('token.pickle'):\n",
    "    with open('token.pickle', 'rb') as token:\n",
    "        creds = pickle.load(token)\n",
    "\n",
    "if not creds or not creds.valid:\n",
    "    if creds and creds.expired and creds.refresh_token:\n",
    "        creds.refresh(Request())\n",
    "    else:\n",
    "        flow = InstalledAppFlow.from_client_secrets_file(\n",
    "            CLIENT_SECRET, SCOPES)\n",
    "        creds = flow.run_local_server(port=0)\n",
    "    with open('token.pickle', 'wb') as token:\n",
    "        pickle.dump(creds, token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51e76c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#get all media item info\n",
    "\n",
    "response = requests.get('https://photoslibrary.googleapis.com/v1/mediaItems', headers={'Authorization': f'Bearer {creds.token}'})\n",
    "mediaItems = response.json()['mediaItems']\n",
    "\n",
    "#loop through all the media items to get each base url and feed it to the captioning model function\n",
    "\n",
    "baseUrls = []\n",
    "\n",
    "for i in mediaItems:\n",
    "    # baseUrls.append(i['baseUrl'])\n",
    "    img = Image.open(requests.get(i['baseUrl'], stream=True).raw)\n",
    "    display(img)\n",
    "    print(caption(img))\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}