ConvertForBLINK.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from pathlib import Path\n",
    "import urllib\n",
    "from tqdm import tqdm\n",
    "import paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blink_data_folder_path = paths.OUTPUT_DATASETS_PATH / 'melart_blink'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_folder = paths.CANDIDATES_FOLDER_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "candidate_types_dict = {}\n",
    "\n",
    "with open(paths.CANDIDATE_TYPES_DICT_PATH, 'r') as f:\n",
    "    candidate_types_dict = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read all the candidate json files from el_candidates and put them in a dictionary\n",
    "candidate_counter=0\n",
    "candidate_objects=[]\n",
    "candidate_index={}\n",
    "qid2id={}\n",
    "for file in tqdm(list(candidates_folder.iterdir())):\n",
    "    if not file.name.endswith('.json'):\n",
    "        continue\n",
    "    with open(file, 'r') as f:\n",
    "        candidate = json.load(f)\n",
    "        qid = file.name.split('.')[0]\n",
    "        types_qid_list=[]\n",
    "        if candidate.get('statements', {}).get('P31'):\n",
    "            for entity_type in candidate['statements']['P31']:\n",
    "                try:\n",
    "                    types_qid_list.append(entity_type[\"value\"][\"content\"])\n",
    "                except:\n",
    "                    pass\n",
    "        \"\"\"\n",
    "        build an object like this\n",
    "        {\n",
    "            \"title\": \"Elon Musk\",\n",
    "            \"text\": \"Elon Reeve Musk (; born June 28, 1971) is an entrepreneur and business magnate. He is the founder, CEO and chief engineer at SpaceX; early stage investor, CEO, and product architect of Tesla, Inc.; founder of The Boring Company; and co-founder of Neuralink and OpenAI. A centibillionaire, Musk is one of the richest people in the world.\\nMusk was born to a Canadian mother and South African father and raised in Pretoria, South Africa. He briefly attended the University of Pretoria before moving to Canada aged 17 to attend Queen's University. He transferred to the University of Pennsylvania two years later, where he received bachelors' degrees in economics and physics. He moved to California in 1995 to attend Stanford University but decided instead to pursue a business career, co-founding\",\n",
    "            \"document_id\": 909036\n",
    "        }\n",
    "        \"\"\"\n",
    "        obj={}\n",
    "        obj['document_id']=int(qid[1:])\n",
    "        obj['title']=candidate['labels']['en'] if candidate['labels'].get('en') else ''\n",
    "        desc_text=candidate['descriptions']['en'] if candidate['descriptions'].get('en') else ''\n",
    "        entity_types=[]\n",
    "        for type_qid in types_qid_list:\n",
    "            if type_qid in candidate_types_dict:\n",
    "                entity_types.append(candidate_types_dict[type_qid])\n",
    "        types_str=\", \".join(entity_types)\n",
    "        obj['text']=f\"{types_str}. {desc_text}\"\n",
    "        obj['type']=types_str\n",
    "        candidate_objects.append(obj)\n",
    "        candidate_index[qid]=obj\n",
    "        qid2id[qid]=candidate_counter\n",
    "        candidate_counter+=1\n",
    "\n",
    "qid2line_number={}\n",
    "\n",
    "# write the candidate objects to a json file\n",
    "documents_folder_path = blink_data_folder_path / 'documents'\n",
    "documents_folder_path.mkdir(parents=True, exist_ok=True)\n",
    "counter=0\n",
    "with open(documents_folder_path / 'documents.jsonl', 'w') as f:\n",
    "    for obj in candidate_objects:\n",
    "        qid2line_number[obj['document_id']]=counter\n",
    "        f.write(json.dumps(obj))\n",
    "        f.write('\\n')\n",
    "        counter+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "candidate_counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the file with the paintings and the sentence mentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paintings_file = paths.COMBINED_ANNOTATIONS_PATH\n",
    "paintings=None\n",
    "with open(paintings_file, 'r') as f:\n",
    "    paintings = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "painting_images_path = paths.ARTPEDIA_IMAGES_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mention_objects=[]\n",
    "\n",
    "paintings_mentions_index={}\n",
    "\n",
    "for qid,paiting_obj in tqdm(paintings.items()):\n",
    "    counter=0\n",
    "    img_url=paiting_obj.get(\"img_url\",None)\n",
    "    new_img_file_path=None\n",
    "    if img_url: # this check is to make the MIMIC and BLINK data compatible\n",
    "        img_file_name=img_url.split('/')[-1]\n",
    "        img_path=Path(urllib.parse.unquote(img_file_name).replace('_',' '))\n",
    "        img_path= painting_images_path / img_path\n",
    "        if not img_path.exists():\n",
    "            print(f'no image in dict for {qid} ({img_file_name})')\n",
    "            continue\n",
    "    else:\n",
    "        print(f'no image for {qid}')\n",
    "        continue\n",
    "    for field in [\"visual_el_matches\",\"contextual_el_matches\"]:\n",
    "        for i,el_matches in enumerate(paiting_obj[field]):\n",
    "            if len(el_matches)>0:\n",
    "                sentence=paiting_obj[\"visual_sentences\"][i] if field==\"visual_el_matches\" else paiting_obj[\"contextual_sentences\"][i]\n",
    "                for el_match in el_matches:\n",
    "                    match_qid=el_match['qid'].split('/')[-1]\n",
    "                    #match_id=int(match_qid[1:])\n",
    "                    obj={}\n",
    "                    try:\n",
    "                        obj[\"label_id\"]=qid2id[match_qid]\n",
    "                    except:\n",
    "                        print(f'no candidate for {qid} trying to match {match_qid}')\n",
    "                        continue\n",
    "                    obj['mention']=el_match['text']\n",
    "                    obj['label']=candidate_index[match_qid]['text']\n",
    "                    obj['label_title']=candidate_index[match_qid]['title']\n",
    "                    start_index=el_match[\"start\"]\n",
    "                    end_index=el_match[\"end\"]\n",
    "                    left_context=sentence[:start_index]\n",
    "                    right_context=sentence[end_index:]\n",
    "                    obj['context_left']=left_context\n",
    "                    obj['context_right']=right_context\n",
    "                    obj['world']=\"undefined\"\n",
    "                    mention_objects.append(obj)\n",
    "                    painting_mentions=paintings_mentions_index.get(qid,[])\n",
    "                    painting_mentions.append(obj)\n",
    "                    paintings_mentions_index[qid]=painting_mentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the paintings according to the split property in the painting objects\n",
    "train_paintings = {}\n",
    "dev_paintings = {}\n",
    "test_paintings = {}\n",
    "\n",
    "for qid, painting_obj in paintings.items():\n",
    "    if painting_obj['split'] == 'train':\n",
    "        train_paintings[qid] = painting_obj\n",
    "    elif painting_obj['split'] == 'val':\n",
    "        dev_paintings[qid] = painting_obj\n",
    "    elif painting_obj['split'] == 'test':\n",
    "        test_paintings[qid] = painting_obj\n",
    "\n",
    "# report sizes\n",
    "print(f'train: {len(train_paintings)}, dev: {len(dev_paintings)}, test: {len(test_paintings)}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write 3 json files called MELART_train.json, MELART_dev.json, MELART_test.json inside the MELART folder, using the paintings_mentions_index\n",
    "blink_format_folder_path = blink_data_folder_path / 'blink_format'\n",
    "blink_format_folder_path.mkdir(parents=True, exist_ok=True)\n",
    "with open(blink_format_folder_path / 'train.jsonl', 'w') as f:\n",
    "    #find all mentions for the train paintings\n",
    "    train_mentions=[]\n",
    "    counter=0\n",
    "    for qid,painting_obj in train_paintings.items():\n",
    "        mentions=paintings_mentions_index.get(qid,[])\n",
    "        if len(mentions)>0:\n",
    "            train_mentions.extend(mentions)\n",
    "            counter+=1\n",
    "    print(f'found {counter} paintings with mentions for train')\n",
    "    for mention in train_mentions:\n",
    "        f.write(json.dumps(mention))\n",
    "        f.write('\\n')\n",
    "with open(blink_format_folder_path / 'valid.jsonl', 'w') as f:\n",
    "    #find all mentions for the dev paintings\n",
    "    dev_mentions=[]\n",
    "    counter=0\n",
    "    for qid,painting_obj in dev_paintings.items():\n",
    "        mentions=paintings_mentions_index.get(qid,[])\n",
    "        if len(mentions)>0:\n",
    "            dev_mentions.extend(mentions)\n",
    "            counter+=1\n",
    "    print(f'found {counter} paintings with mentions for dev')\n",
    "    for mention in dev_mentions:\n",
    "        f.write(json.dumps(mention))\n",
    "        f.write('\\n')\n",
    "with open(blink_format_folder_path / 'test.jsonl', 'w') as f:\n",
    "    #find all mentions for the test paintings\n",
    "    test_mentions=[]\n",
    "    counter=0\n",
    "    for qid,painting_obj in test_paintings.items():\n",
    "        mentions=paintings_mentions_index.get(qid,[])\n",
    "        if len(mentions)>0:\n",
    "            test_mentions.extend(mentions)\n",
    "            counter+=1\n",
    "    print(f'found {counter} paintings with mentions for test')\n",
    "    for mention in test_mentions:\n",
    "        f.write(json.dumps(mention))\n",
    "        f.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print sizes\n",
    "print(f'train: {len(train_mentions)}, dev: {len(dev_mentions)}, test: {len(test_mentions)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "wikid_filter",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}