name change belar => ragas (#15)

* readme init

* new logo

* new logo

* new logo

* fix height

* change belar -> ragas

* fix ci

* Update README.md

* added quickstart

* lint changes

---------

Co-authored-by: Jithin James <[email protected]>

.gitignore

@@ -160,3 +160,4 @@ cython_debug/
 #.idea/
 belar/_version.py
 **/tmp.ipynb
+ragas/_version.py

Makefile

@@ -5,20 +5,20 @@ help: ## Show all Makefile targets
 
 .PHONY: format lint type style clean run-benchmarks
 format: ## Running code formatter: black and isort
+	@echo "(isort) Ordering imports..."
+	@isort .
 	@echo "(black) Formatting codebase..."
-	@black --config pyproject.toml belar tests examples
+	@black --config pyproject.toml ragas tests examples
 	@echo "(black) Formatting stubs..."
-	@find belar -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
-	@echo "(isort) Reordering imports..."
-	@isort .
+	@find ragas -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
 	@echo "(ruff) Running fix only..."
-	@ruff check belar examples tests --fix-only
+	@ruff check ragas examples tests --fix-only
 lint: ## Running lint checker: ruff
 	@echo "(ruff) Linting development project..."
-	@ruff check belar examples tests
+	@ruff check ragas examples tests
 type: ## Running type checker: pyright
 	@echo "(pyright) Typechecking codebase..."
-	@pyright -p belar
+	@pyright -p ragas
 clean: ## Clean all generated files
 	@echo "Cleaning all generated files..."
 	@cd $(GIT_ROOT)/docs && make clean

README.md

@@ -1,2 +1,36 @@
-# BeLAR
-Benchmarking LLM Augmented Retrieval
+<h1 align="center">
+  <img style="vertical-align:middle" height="200"
+  src="./docs/assets/logo.png">
+</h1>
+<p align="center">
+  <i>SOTA metrics for evaluating Retrieval Augmented Generation (RAG)</i>
+</p>
+
+<p align="center">
+    <a href="https://github.com/beir-cellar/beir/releases">
+        <img alt="GitHub release" src="https://img.shields.io/github/release/beir-cellar/beir.svg">
+    </a>
+    <a href="https://www.python.org/">
+            <img alt="Build" src="https://img.shields.io/badge/Made%20with-Python-1f425f.svg?color=purple">
+    </a>
+    <a href="https://github.com/beir-cellar/beir/blob/master/LICENSE">
+        <img alt="License" src="https://img.shields.io/github/license/beir-cellar/beir.svg?color=green">
+    </a>
+    <a href="https://colab.research.google.com/drive/1HfutiEhHMJLXiWGT8pcipxT5L2TpYEdt?usp=sharing">
+        <img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+    <a href="https://github.com/beir-cellar/beir/">
+        <img alt="Downloads" src="https://badges.frapsoft.com/os/v1/open-source.svg?v=103">
+    </a>
+</p>
+
+<h4 align="center">
+    <p>
+        <a href="#beers-installation">Installation</a> |
+        <a href="#beers-quick-example">Quick Example</a> |
+        <a href="https://huggingface.co/BeIR">Hugging Face</a>
+    <p>
+</h4>
+
+## What
+

examples/quickstart.ipynb

@@ -10,24 +10,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 1,
    "id": "22c7dd25",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
+    "# only run this if your have an editable install\n",
     "%load_ext autoreload\n",
     "%autoreload 2"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "40258397",
+   "metadata": {},
+   "source": [
+    "## Load the Data\n",
+    "\n",
+    "For this quickstart we are going to be using a dataset that we prepared from [eli5](https://huggingface.co/datasets/eli5) dataset with the models response. \n",
+    "\n",
+    "prompt: str\n",
+    "context: str\n",
+    "references: list[str]\n",
+    "ground_truth: list[str]\n",
+    "generated_text: str"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -85,7 +93,9 @@
    "cell_type": "code",
    "execution_count": 28,
    "id": "a77c805d",
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "name": "stderr",

pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "belar"
+name = "ragas"
 dependencies = [
     "Levenshtein",
     "rouge-score",
@@ -19,4 +19,4 @@ readme = {file = ["README.md"], content-type = "text/plain"}
 requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 [tool.setuptools_scm]
-write_to = "belar/_version.py"
+write_to = "ragas/_version.py"

ragas/metrics/__init__.py

@@ -0,0 +1,17 @@
+from ragas.metrics.base import Evaluation, Metric
+from ragas.metrics.factual import EntailmentScore
+from ragas.metrics.similarity import SBERTScore
+from ragas.metrics.simple import BLUE, EditDistance, EditRatio, Rouge1, Rouge2, RougeL
+
+__all__ = [
+    "Evaluation",
+    "Metric",
+    "EntailmentScore",
+    "SBERTScore",
+    "BLUE",
+    "EditDistance",
+    "EditRatio",
+    "RougeL",
+    "Rouge1",
+    "Rouge2",
+]

belar/metrics/factual.py → ragas/metrics/factual.py

@@ -9,18 +9,23 @@
 import numpy as np
 import spacy
 import transformers
-from transformers import (AutoConfig, AutoModelForSequenceClassification,
-                          AutoTokenizer, PreTrainedModel)
+from transformers import (
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    PreTrainedModel,
+)
 
-from belar.metrics import Metric
-from belar.utils import device_check
+from ragas.metrics import Metric
+from ragas.utils import device_check
 
 if t.TYPE_CHECKING:
     from torch import device as Device
 
 from transformers.models.auto.modeling_auto import (
     MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
-    MODEL_WITH_LM_HEAD_MAPPING_NAMES)
+    MODEL_WITH_LM_HEAD_MAPPING_NAMES,
+)
 
 MODEL_MAPPINGS_NAMES = [
     MODEL_WITH_LM_HEAD_MAPPING_NAMES,

belar/metrics/similarity.py → ragas/metrics/similarity.py

@@ -7,7 +7,7 @@
 from numpy.linalg import norm
 from sentence_transformers import SentenceTransformer
 
-from belar.metrics.base import Metric
+from ragas.metrics.base import Metric
 
 if t.TYPE_CHECKING:
     from torch import Tensor

belar/metrics/simple.py → ragas/metrics/simple.py

@@ -8,7 +8,7 @@
 from nltk.translate.bleu_score import sentence_bleu
 from rouge_score import rouge_scorer
 
-from belar.metrics.base import Metric
+from ragas.metrics.base import Metric
 
 ROUGE_TYPES = t.Literal["rouge1", "rouge2", "rougeL"]
 

tests/benchmarks/benchmark.py

@@ -5,8 +5,16 @@
 from tqdm import tqdm
 from utils import print_table, timeit
 
-from belar.metrics import (EditDistance, EditRatio, EntailmentScore,
-                           Evaluation, Rouge1, Rouge2, RougeL, SBERTScore)
+from ragas.metrics import (
+    EditDistance,
+    EditRatio,
+    EntailmentScore,
+    Evaluation,
+    Rouge1,
+    Rouge2,
+    RougeL,
+    SBERTScore,
+)
 
 DEVICE = "cuda" if is_available() else "cpu"
 BATCHES = [0, 1]

tests/unit/test_simple.py

@@ -1,4 +1,4 @@
 def test_import():
-    import belar
+    import ragas
 
-    assert belar is not None
+    assert ragas is not None