From 75701195902d8b5c1f18c590851ef8f7e7c09982 Mon Sep 17 00:00:00 2001 From: John Leung Date: Sat, 7 Sep 2024 19:59:34 -0700 Subject: [PATCH 1/2] Changed submission default from topn=3 to 2 according to new rule --- arckit/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arckit/data.py b/arckit/data.py index 0380922..67008aa 100644 --- a/arckit/data.py +++ b/arckit/data.py @@ -185,7 +185,7 @@ def __iter__(self): def __repr__(self): return f"" - def score_submission(self,fn: str, topn=3, return_correct=False) -> int: + def score_submission(self,fn: str, topn=2, return_correct=False) -> int: """ Score a submission file, in Kaggle csv format Two columns: output_id,output @@ -303,4 +303,4 @@ def load_single(id: str, version='latest') -> Task: for i in range(10): train_tasks[i].show() - print(train_tasks["08ed6ac7"].gpt_prompt("gpt3")) \ No newline at end of file + print(train_tasks["08ed6ac7"].gpt_prompt("gpt3")) From f5b907f39801dacd6fa658253250d0a95baeaec4 Mon Sep 17 00:00:00 2001 From: John Leung Date: Sat, 7 Sep 2024 20:01:06 -0700 Subject: [PATCH 2/2] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de175c7..f2da4eb 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ To evaluate a submission in [Kaggle ARC format](https://www.kaggle.com/competiti ```python >>> eval_set.score_submission( 'submission.csv', # Submission with two columns output_id,output in Kaggle fomrat - topn=3, # How many predictions to consider (default: 3) + topn=2, # How many predictions to consider (default: 2) return_correct=False # Whether to return a list of which tasks were solved ) ```