Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reading ease name #258

Merged
merged 4 commits into from
Mar 10, 2024
+56 −55
Merged

Reading ease name #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fc9a1c9
Change name of reading ease back to flesch_reading_ease
Mar 10, 2024
5144987
bump version
Mar 10, 2024
1d23984
Fix toxicity name
Mar 10, 2024
38b9baf
bump version
Mar 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bumpversion.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.0.88
current_version = 0.0.90
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?
serialize =
Expand Down
3 changes: 2 additions & 1 deletion langkit/core/workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ def init(self) -> None:
targets = validator.get_target_metric_names()
if not set(targets).issubset(metric_names):
raise ValueError(
f"Validator {validator} has target metric names ({targets}) that are not in the list of metrics: {metric_names}"
f"Validator {validator} has target metric names ({targets}) but this workflow is "
"only generating metrics for these: {metric_names}"
)

def _condense_metric_results(self, metric_results: Dict[str, SingleMetricResult]) -> pd.DataFrame:
Expand Down
8 changes: 4 additions & 4 deletions langkit/metrics/library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def recommended(prompt: bool = True, response: bool = True) -> MetricCreator:
- response.pii.*
- response.stats.token_count
- response.stats.char_count
- response.stats.reading_ease
- response.stats.flesch_reading_ease
- response.sentiment.sentiment_score
- response.toxicity.toxicity_score
- response.similarity.refusal
Expand All @@ -73,7 +73,7 @@ def recommended(prompt: bool = True, response: bool = True) -> MetricCreator:
lib.response.pii,
lib.response.stats.token_count,
lib.response.stats.char_count,
lib.response.stats.reading_ease,
lib.response.stats.flesch_reading_ease,
lib.response.sentiment.sentiment_score,
lib.response.toxicity.toxicity_score,
lib.response.similarity.refusal,
Expand Down Expand Up @@ -130,7 +130,7 @@ def char_count() -> MetricCreator:
return prompt_char_count_metric

@staticmethod
def reading_ease() -> MetricCreator:
def flesch_reading_ease() -> MetricCreator:
from langkit.metrics.text_statistics import prompt_reading_ease_metric

return prompt_reading_ease_metric
Expand Down Expand Up @@ -312,7 +312,7 @@ def char_count() -> MetricCreator:
return response_char_count_metric

@staticmethod
def reading_ease() -> MetricCreator:
def flesch_reading_ease() -> MetricCreator:
from langkit.metrics.text_statistics import response_reading_ease_metric

return response_reading_ease_metric
Expand Down
2 changes: 1 addition & 1 deletion langkit/metrics/toxicity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def udf(text: pd.DataFrame) -> SingleMetricResult:
metrics = __toxicity(_pipeline, max_length, col)
return SingleMetricResult(metrics=metrics)

return SingleMetric(name=f"{column_name}.toxicity", input_name=column_name, evaluate=udf, init=init)
return SingleMetric(name=f"{column_name}.toxicity.toxicity_score", input_name=column_name, evaluate=udf, init=init)


prompt_toxicity_metric = partial(toxicity_metric, "prompt")
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "langkit"
version = "0.0.88"
version = "0.0.90"
description = "A language toolkit for monitoring LLM interactions"
authors = ["WhyLabs.ai <[email protected]>"]
homepage = "https://docs.whylabs.ai/docs/large-language-model-monitoring"
Expand Down
2 changes: 1 addition & 1 deletion tests/langkit/metrics/test_library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def test_recommended():
"response.stats.char_count",
"response.stats.flesch_reading_ease",
"response.sentiment.sentiment_score",
"response.toxicity",
"response.toxicity.toxicity_score",
"response.similarity.refusal",
"id",
]
92 changes: 46 additions & 46 deletions tests/langkit/metrics/test_toxicity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,12 @@ def test_prompt_toxicity_row_non_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] < 0.1


def test_prompt_toxicity_df_non_toxic():
Expand All @@ -89,12 +89,12 @@ def test_prompt_toxicity_df_non_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] < 0.1


def test_prompt_toxicity_row_toxic():
Expand All @@ -107,12 +107,12 @@ def test_prompt_toxicity_row_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7


def test_prompt_toxicity_df_toxic():
Expand Down Expand Up @@ -140,12 +140,12 @@ def test_prompt_toxicity_df_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7


def test_prompt_toxicity_df_mixed():
Expand Down Expand Up @@ -173,13 +173,13 @@ def test_prompt_toxicity_df_mixed():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/min"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["prompt.toxicity.toxicity_score"] < 0.1


def test_response_toxicity_row_non_toxic():
Expand All @@ -191,11 +191,11 @@ def test_response_toxicity_row_non_toxic():
expected_columns = [
"prompt",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] < 0.1


def test_response_toxicity_df_non_toxic():
Expand All @@ -207,11 +207,11 @@ def test_response_toxicity_df_non_toxic():
expected_columns = [
"prompt",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] < 0.1


def test_response_toxicity_row_toxic():
Expand All @@ -225,11 +225,11 @@ def test_response_toxicity_row_toxic():
expected_columns = [
"prompt",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7


def test_response_toxicity_df_toxic():
Expand Down Expand Up @@ -258,11 +258,11 @@ def test_response_toxicity_df_toxic():
expected_columns = [
"prompt",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7


def test_response_toxicity_df_mixed():
Expand Down Expand Up @@ -291,12 +291,12 @@ def test_response_toxicity_df_mixed():
expected_columns = [
"prompt",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/min"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["response.toxicity.toxicity_score"] < 0.1


def test_prompt_response_toxicity_row_non_toxic():
Expand All @@ -307,14 +307,14 @@ def test_prompt_response_toxicity_row_non_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] < 0.1


def test_prompt_response_toxicity_df_non_toxic():
Expand All @@ -325,14 +325,14 @@ def test_prompt_response_toxicity_df_non_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] < 0.1


def test_prompt_response_toxicity_row_toxic():
Expand All @@ -345,14 +345,14 @@ def test_prompt_response_toxicity_row_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7


def test_prompt_response_toxicity_df_toxic():
Expand Down Expand Up @@ -380,16 +380,16 @@ def test_prompt_response_toxicity_df_toxic():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/min"]["prompt.toxicity"] > 0.7
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/min"]["response.toxicity"] > 0.7
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["prompt.toxicity.toxicity_score"] > 0.7
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["response.toxicity.toxicity_score"] > 0.7


def test_prompt_response_toxicity_df_mixed():
Expand Down Expand Up @@ -417,13 +417,13 @@ def test_prompt_response_toxicity_df_mixed():

expected_columns = [
"prompt",
"prompt.toxicity",
"prompt.toxicity.toxicity_score",
"response",
"response.toxicity",
"response.toxicity.toxicity_score",
]

assert actual.index.tolist() == expected_columns
assert actual["distribution/max"]["prompt.toxicity"] > 0.7
assert actual["distribution/min"]["prompt.toxicity"] < 0.1
assert actual["distribution/max"]["response.toxicity"] > 0.7
assert actual["distribution/min"]["response.toxicity"] < 0.1
assert actual["distribution/max"]["prompt.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["prompt.toxicity.toxicity_score"] < 0.1
assert actual["distribution/max"]["response.toxicity.toxicity_score"] > 0.7
assert actual["distribution/min"]["response.toxicity.toxicity_score"] < 0.1
Loading