From 8f727920a342156fd0341f811ad7b0b83876d334 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 11:45:06 +0200 Subject: [PATCH 1/7] add alternative benchmark for gpu --- nonasv/pipeline.py | 169 +++++++++++ .../complex/fb15k/trouillon2016/metadata.json | 20 ++ .../complex/fb15k/trouillon2016/results.json | 268 ++++++++++++++++++ .../complex/wn18/trouillon2016/metadata.json | 20 ++ .../complex/wn18/trouillon2016/results.json | 268 ++++++++++++++++++ .../conve/fb15k/dettmers2018/metadata.json | 20 ++ .../conve/fb15k/dettmers2018/results.json | 268 ++++++++++++++++++ .../conve/fb15k237/dettmers2018/metadata.json | 20 ++ .../conve/fb15k237/dettmers2018/results.json | 268 ++++++++++++++++++ .../conve/wn18/dettmers2018/metadata.json | 20 ++ .../cuda/conve/wn18/dettmers2018/results.json | 268 ++++++++++++++++++ .../conve/wn18rr/dettmers2018/metadata.json | 20 ++ .../conve/wn18rr/dettmers2018/results.json | 268 ++++++++++++++++++ .../convkb/fb15k237/nguyen2018/metadata.json | 20 ++ .../convkb/fb15k237/nguyen2018/results.json | 268 ++++++++++++++++++ .../convkb/wn18rr/nguyen2018/metadata.json | 20 ++ .../convkb/wn18rr/nguyen2018/results.json | 268 ++++++++++++++++++ .../distmult/fb15k/yang2014/metadata.json | 20 ++ .../cuda/distmult/fb15k/yang2014/results.json | 268 ++++++++++++++++++ .../cuda/distmult/wn18/yang2014/metadata.json | 20 ++ .../cuda/distmult/wn18/yang2014/results.json | 268 ++++++++++++++++++ .../cuda/hole/fb15k/nickel2016/metadata.json | 20 ++ .../cuda/hole/fb15k/nickel2016/results.json | 268 ++++++++++++++++++ .../cuda/hole/wn18/nickel2016/metadata.json | 20 ++ .../cuda/hole/wn18/nickel2016/results.json | 268 ++++++++++++++++++ .../cuda/kg2e/fb15k/he2015/metadata.json | 19 ++ .../cuda/kg2e/fb15k/he2015/results.json | 268 ++++++++++++++++++ .../cuda/kg2e/wn18/he2015/metadata.json | 19 ++ .../cuda/kg2e/wn18/he2015/results.json | 268 ++++++++++++++++++ .../mure/fb15k237/balazevic2019/metadata.json | 20 ++ .../mure/fb15k237/balazevic2019/results.json | 268 ++++++++++++++++++ .../cuda/quate/fb15k/zhang2019/metadata.json | 19 ++ .../cuda/quate/fb15k/zhang2019/results.json | 268 ++++++++++++++++++ .../quate/fb15k237/zhang2019/metadata.json | 19 ++ .../quate/fb15k237/zhang2019/results.json | 268 ++++++++++++++++++ .../cuda/quate/wn18/zhang2019/metadata.json | 19 ++ .../cuda/quate/wn18/zhang2019/results.json | 268 ++++++++++++++++++ .../cuda/quate/wn18rr/zhang2019/metadata.json | 19 ++ .../cuda/quate/wn18rr/zhang2019/results.json | 268 ++++++++++++++++++ .../cuda/rotate/fb15k/sun2019/metadata.json | 20 ++ .../cuda/rotate/fb15k/sun2019/results.json | 268 ++++++++++++++++++ .../rotate/fb15k237/sun2019/metadata.json | 20 ++ .../cuda/rotate/fb15k237/sun2019/results.json | 268 ++++++++++++++++++ .../cuda/rotate/wn18/sun2019/metadata.json | 20 ++ .../cuda/rotate/wn18/sun2019/results.json | 268 ++++++++++++++++++ .../cuda/rotate/wn18rr/sun2019/metadata.json | 20 ++ .../cuda/rotate/wn18rr/sun2019/results.json | 268 ++++++++++++++++++ .../simple/fb15k/kazemi2018/metadata.json | 20 ++ .../cuda/simple/fb15k/kazemi2018/results.json | 268 ++++++++++++++++++ .../cuda/simple/wn18/kazemi2018/metadata.json | 20 ++ .../cuda/simple/wn18/kazemi2018/results.json | 268 ++++++++++++++++++ .../cuda/transd/fb15k/ji2015/metadata.json | 20 ++ .../cuda/transd/fb15k/ji2015/results.json | 268 ++++++++++++++++++ .../cuda/transd/wn18/ji2015/metadata.json | 20 ++ .../cuda/transd/wn18/ji2015/results.json | 268 ++++++++++++++++++ .../transe/fb15k/bordes2013/metadata.json | 20 ++ .../cuda/transe/fb15k/bordes2013/results.json | 268 ++++++++++++++++++ .../transe/fb15k237/nguyen2018/metadata.json | 20 ++ .../transe/fb15k237/nguyen2018/results.json | 268 ++++++++++++++++++ .../cuda/transe/wn18/bordes2013/metadata.json | 20 ++ .../cuda/transe/wn18/bordes2013/results.json | 268 ++++++++++++++++++ .../cuda/transh/fb15k/wang2014/metadata.json | 19 ++ .../cuda/transh/fb15k/wang2014/results.json | 268 ++++++++++++++++++ .../cuda/transh/wn18/wang2014/metadata.json | 19 ++ .../cuda/transh/wn18/wang2014/results.json | 268 ++++++++++++++++++ .../cuda/transr/fb15k/li2015/metadata.json | 19 ++ .../cuda/transr/fb15k/li2015/results.json | 268 ++++++++++++++++++ .../cuda/transr/wn18/li2015/metadata.json | 19 ++ .../cuda/transr/wn18/li2015/results.json | 268 ++++++++++++++++++ .../tucker/fb15k/balazevic2019/metadata.json | 20 ++ .../tucker/fb15k/balazevic2019/results.json | 268 ++++++++++++++++++ .../fb15k237/balazevic2019/metadata.json | 20 ++ .../fb15k237/balazevic2019/results.json | 268 ++++++++++++++++++ .../tucker/wn18/balazevic2019/metadata.json | 20 ++ .../tucker/wn18/balazevic2019/results.json | 268 ++++++++++++++++++ .../tucker/wn18rr/balazevic2019/metadata.json | 20 ++ .../tucker/wn18rr/balazevic2019/results.json | 268 ++++++++++++++++++ nonasv/results/27c9fe15/summary.tsv | 39 +++ 78 files changed, 11142 insertions(+) create mode 100644 nonasv/pipeline.py create mode 100644 nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json create mode 100644 nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json create mode 100644 nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json create mode 100644 nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json create mode 100644 nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json create mode 100644 nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json create mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json create mode 100644 nonasv/results/27c9fe15/summary.tsv diff --git a/nonasv/pipeline.py b/nonasv/pipeline.py new file mode 100644 index 0000000..ad8ed5e --- /dev/null +++ b/nonasv/pipeline.py @@ -0,0 +1,169 @@ +import json +import logging +import pathlib +import pprint +from typing import Any, Mapping +import platform +import click +import more_click +import pandas +import torch + +from pykeen.experiments.cli import HERE +from pykeen.pipeline.api import pipeline_from_config +from pykeen.utils import ( + CONFIGURATION_FILE_FORMATS, + format_relative_comparison, + load_configuration, + resolve_device, +) +from pykeen.version import get_git_hash + +logger = logging.getLogger(__name__) + + +def _collect_system_information() -> Mapping[str, Any]: + uname = platform.uname() + result = dict( + system=uname.system, + release=uname.release, + machine=uname.machine, + torch_version=torch.__version__, + ) + if torch.cuda.is_available(): + # e.g. _CudaDeviceProperties(name='Quadro RTX 8000', major=7, minor=5, total_memory=48601MB, multi_processor_count=72) + properties = torch.cuda.get_device_properties(device=torch.device("cuda")) + result["gpu"] = dict( + name=properties.name, + total_memory=properties.total_memory, + compute_capability=(properties.major, properties.minor), + cuda=torch.version.cuda, + cudnn=torch.backends.cudnn.version(), + ) + return result + + +@click.command() +@click.option("-c", "--configuration-root", type=pathlib.Path, default=HERE) +@click.option( + "-o", + "--output-root", + type=pathlib.Path, + default=pathlib.Path(__file__).parent.joinpath("results", get_git_hash()), +) +@click.option("-e", "--num-epochs", type=int, default=5) +@click.option("--debug", is_flag=True) +@more_click.log_level_option() +def main( + configuration_root: pathlib.Path, + output_root: pathlib.Path, + num_epochs: int, + log_level: str, + debug: bool, +): + """""" + logging.basicConfig(level=log_level) + + device = resolve_device(device=None) + logging.info(f"Running on device: {device}") + + system_information = _collect_system_information() + + configuration_paths = sorted( + path + for ext in CONFIGURATION_FILE_FORMATS + for path in configuration_root.rglob(f"*{ext}") + ) + logger.info( + f"Found {len(configuration_paths)} configurations under {configuration_root}" + ) + + for i, path in enumerate(configuration_paths, start=1): + logger.info( + f"Progress: {format_relative_comparison(part=i, total=len(configuration_paths))}" + ) + reference, model, dataset, *remainder = path.stem.split("_") + if model in { + "nodepiece", # no precomputed anchors... + "rgcn", # too slow + "boxe", + }: + logger.warning(f"Skipping {path} due to explicit model ignore rule") + continue + + output_path = output_root.joinpath( + device.type, model, dataset, "_".join((reference, *remainder)) + ) + if output_path.exists(): + logger.debug( + f"Skipping configuration {path} since output path exists {output_path}" + ) + continue + + # load configuration + configuration = dict(load_configuration(path)) + # reduce number of training epochs + configuration["pipeline"]["training_kwargs"]["num_epochs"] = num_epochs + # discard results + configuration.pop("results", None) + # add system information to metadata + configuration.setdefault("metadata", {}) + configuration["metadata"]["system"] = system_information + + logger.info(f"Running configuration from {path}") + logger.debug(pprint.pformat(configuration, indent=2, sort_dicts=True)) + try: + result = pipeline_from_config(config=configuration) + except TypeError as error: + logger.error("Could not run pipeline", exc_info=error) + continue + + # save results + result.save_to_directory( + directory=output_path, + save_metadata=True, + save_replicates=False, + save_training=False, + ) + + if debug: + break + + data = [] + for i, path in enumerate(configuration_paths, start=1): + reference, model, dataset, *remainder = path.stem.split("_") + output_path = output_root.joinpath( + device.type, + model, + dataset, + "_".join((reference, *remainder)), + "results.json", + ) + if not output_path.exists(): + logger.warning(f"{output_path} is not existing") + continue + + results = json.loads(output_path.read_text()) + data.append( + { + "training": results["times"]["training"] / num_epochs, + "evaluation": results["times"]["evaluation"], + "batch_size.evaluation": results["evaluator_metadata"]["batch_size"], + "path": path.relative_to(configuration_root).as_posix(), + } + ) + df = pandas.DataFrame.from_records(data=data) + df = df[sorted(df.columns)] + df = df.sort_values(by="path") + output_path = output_root.joinpath("summary.tsv") + df.to_csv(output_path, sep="\t", index=False) + logger.info(f"Written summary to {output_path}") + + # print + print(f"{'path':50} training evaluation") + for _, row in df.iterrows(): + print(f"{row['path']:50} {row['training']:8.2f}s {row['evaluation']:8.2f}s") + + +if __name__ == "__main__": + main() diff --git a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json new file mode 100644 index 0000000..1307ab5 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json new file mode 100644 index 0000000..fb86d08 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0003799774137384096, + 0.0001661697653289569, + 0.00013809030711660472, + 0.00012140296024768122, + 0.00010931310573704068 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.10869346503978508, + "adjusted_arithmetic_mean_rank_index": 0.8914272125051246, + "adjusted_geometric_mean_rank_index": 0.9861409524067312, + "adjusted_hits_at_k": 0.25224795638091657, + "adjusted_inverse_harmonic_mean_rank": 0.11382815690449287, + "arithmetic_mean_rank": 802.9025833319226, + "count": 118142.0, + "geometric_mean_rank": 76.26307316764641, + "harmonic_mean_rank": 8.738223046910447, + "hits_at_1": 0.0471635828071304, + "hits_at_10": 0.2527551590458939, + "hits_at_3": 0.12581469756733424, + "hits_at_5": 0.17230112914966736, + "inverse_arithmetic_mean_rank": 0.0012454811091155709, + "inverse_geometric_mean_rank": 0.013112505941135305, + "inverse_harmonic_mean_rank": 0.11443974302688092, + "inverse_median_rank": 0.012987012987012988, + "median_absolute_deviation": 111.19516638792014, + "median_rank": 77.0, + "standard_deviation": 1986.0993868995192, + "variance": 3944590.774642646, + "z_arithmetic_mean_rank": 530.2363209420166, + "z_geometric_mean_rank": 339.5423969620626, + "z_hits_at_k": 3327.9048141920634, + "z_inverse_harmonic_mean_rank": 3709.417282720864 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1086934822278711, + "adjusted_arithmetic_mean_rank_index": 0.8914271953147114, + "adjusted_geometric_mean_rank_index": 0.9861409491774481, + "adjusted_hits_at_k": 0.25224795638091657, + "adjusted_inverse_harmonic_mean_rank": 0.11382815277237188, + "arithmetic_mean_rank": 802.9027102977773, + "count": 118142.0, + "geometric_mean_rank": 76.26309070462126, + "harmonic_mean_rank": 8.738223362207178, + "hits_at_1": 0.0471635828071304, + "hits_at_10": 0.2527551590458939, + "hits_at_3": 0.12581469756733424, + "hits_at_5": 0.17230112914966736, + "inverse_arithmetic_mean_rank": 0.0012454809121632234, + "inverse_geometric_mean_rank": 0.013112502925866912, + "inverse_harmonic_mean_rank": 0.11443973889761169, + "inverse_median_rank": 0.012987012987012988, + "median_absolute_deviation": 111.19516638792014, + "median_rank": 77.0, + "standard_deviation": 1986.0996845871812, + "variance": 3944591.9571173005, + "z_arithmetic_mean_rank": 530.2363107168617, + "z_geometric_mean_rank": 339.5423958501744, + "z_hits_at_k": 3327.9048141920634, + "z_inverse_harmonic_mean_rank": 3709.417148063839 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.10869349044494561, + "adjusted_arithmetic_mean_rank_index": 0.8914271870965242, + "adjusted_geometric_mean_rank_index": 0.9861409447167004, + "adjusted_hits_at_k": 0.25224795638091657, + "adjusted_inverse_harmonic_mean_rank": 0.11382816117618733, + "arithmetic_mean_rank": 802.9027709960938, + "count": 118142.0, + "geometric_mean_rank": 76.26311492919922, + "harmonic_mean_rank": 8.738222720963748, + "hits_at_1": 0.0471635828071304, + "hits_at_10": 0.2527551590458939, + "hits_at_3": 0.12581469756733424, + "hits_at_5": 0.17230112914966736, + "inverse_arithmetic_mean_rank": 0.001245480845682323, + "inverse_geometric_mean_rank": 0.013112498447299004, + "inverse_harmonic_mean_rank": 0.1144397472956273, + "inverse_median_rank": 0.012987012974917889, + "median_absolute_deviation": 111.19516638792014, + "median_rank": 77.0, + "standard_deviation": 1986.099609375, + "variance": 3944591.5, + "z_arithmetic_mean_rank": 530.2363058285418, + "z_geometric_mean_rank": 339.5423943142753, + "z_hits_at_k": 3327.9048141920634, + "z_inverse_harmonic_mean_rank": 3709.4174219262914 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.12324713991639374, + "adjusted_arithmetic_mean_rank_index": 0.8768718911367203, + "adjusted_geometric_mean_rank_index": 0.9840322542018902, + "adjusted_hits_at_k": 0.23782172866273546, + "adjusted_inverse_harmonic_mean_rank": 0.10889755510344874, + "arithmetic_mean_rank": 907.9307443584838, + "count": 59071.0, + "geometric_mean_rank": 87.46514480716856, + "harmonic_mean_rank": 9.131233166899994, + "hits_at_1": 0.045250630597078095, + "hits_at_10": 0.23834030234802187, + "hits_at_3": 0.12136242826429212, + "hits_at_5": 0.16500482470247668, + "inverse_arithmetic_mean_rank": 0.0011014055931177543, + "inverse_geometric_mean_rank": 0.011433125757748028, + "inverse_harmonic_mean_rank": 0.10951423337046323, + "inverse_median_rank": 0.01098901098901099, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 91.0, + "standard_deviation": 2127.478091029021, + "variance": 4526163.027808487, + "z_arithmetic_mean_rank": 368.75830553111393, + "z_geometric_mean_rank": 239.5627482369506, + "z_hits_at_k": 2215.207348551444, + "z_inverse_harmonic_mean_rank": 2505.5085253778548 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12324716749237978, + "adjusted_arithmetic_mean_rank_index": 0.8768718635569903, + "adjusted_geometric_mean_rank_index": 0.9840322468993766, + "adjusted_hits_at_k": 0.23782172866273546, + "adjusted_inverse_harmonic_mean_rank": 0.1088975468405954, + "arithmetic_mean_rank": 907.9309475038513, + "count": 59071.0, + "geometric_mean_rank": 87.46518435018889, + "harmonic_mean_rank": 9.131233855375177, + "hits_at_1": 0.045250630597078095, + "hits_at_10": 0.23834030234802187, + "hits_at_3": 0.12136242826429212, + "hits_at_5": 0.16500482470247668, + "inverse_arithmetic_mean_rank": 0.001101405346683326, + "inverse_geometric_mean_rank": 0.011433120588830501, + "inverse_harmonic_mean_rank": 0.1095142251133281, + "inverse_median_rank": 0.01098901098901099, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 91.0, + "standard_deviation": 2127.478475678952, + "variance": 4526164.664477237, + "z_arithmetic_mean_rank": 368.7582939327784, + "z_geometric_mean_rank": 239.56274645915295, + "z_hits_at_k": 2215.207348551444, + "z_inverse_harmonic_mean_rank": 2505.508335266612 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.12324715387227646, + "adjusted_arithmetic_mean_rank_index": 0.8768718771789428, + "adjusted_geometric_mean_rank_index": 0.9840322534412952, + "adjusted_hits_at_k": 0.23782172866273546, + "adjusted_inverse_harmonic_mean_rank": 0.10889755434025136, + "arithmetic_mean_rank": 907.9308471679688, + "count": 59071.0, + "geometric_mean_rank": 87.46514892578125, + "harmonic_mean_rank": 9.131233230490912, + "hits_at_1": 0.045250630597078095, + "hits_at_10": 0.23834030234802187, + "hits_at_3": 0.12136242826429212, + "hits_at_5": 0.16500482470247668, + "inverse_arithmetic_mean_rank": 0.001101405476219952, + "inverse_geometric_mean_rank": 0.011433125473558903, + "inverse_harmonic_mean_rank": 0.10951423260779401, + "inverse_median_rank": 0.010989011265337467, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 91.0, + "standard_deviation": 2127.478271484375, + "variance": 4526164.0, + "z_arithmetic_mean_rank": 368.7582996613325, + "z_geometric_mean_rank": 239.56274805178367, + "z_hits_at_k": 2215.207348551444, + "z_inverse_harmonic_mean_rank": 2505.508507818254 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0942187911539081, + "adjusted_arithmetic_mean_rank_index": 0.9059035132767959, + "adjusted_geometric_mean_rank_index": 0.9879743242702755, + "adjusted_hits_at_k": 0.2666741240328079, + "adjusted_inverse_harmonic_mean_rank": 0.11875873999814945, + "arithmetic_mean_rank": 697.8744223053613, + "count": 59071.0, + "geometric_mean_rank": 66.49570342330368, + "harmonic_mean_rank": 8.377647410115342, + "hits_at_1": 0.04907653501718271, + "hits_at_10": 0.267170015743766, + "hits_at_3": 0.13026696687037634, + "hits_at_5": 0.179597433596858, + "inverse_arithmetic_mean_rank": 0.00143292255460029, + "inverse_geometric_mean_rank": 0.015038565629332768, + "inverse_harmonic_mean_rank": 0.1193652526832986, + "inverse_median_rank": 0.015384615384615385, + "median_absolute_deviation": 93.40393976585293, + "median_rank": 65.0, + "standard_deviation": 1827.8284088665332, + "variance": 3340956.692259562, + "z_arithmetic_mean_rank": 381.07985853865824, + "z_geometric_mean_rank": 240.52049145945, + "z_hits_at_k": 2491.5898231805772, + "z_inverse_harmonic_mean_rank": 2740.7652003039234 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.09421879801048227, + "adjusted_arithmetic_mean_rank_index": 0.905903506419296, + "adjusted_geometric_mean_rank_index": 0.9879743241749565, + "adjusted_hits_at_k": 0.2666741240328079, + "adjusted_inverse_harmonic_mean_rank": 0.11875873999674515, + "arithmetic_mean_rank": 697.8744730917032, + "count": 59071.0, + "geometric_mean_rank": 66.49570394244209, + "harmonic_mean_rank": 8.377647410213836, + "hits_at_1": 0.04907653501718271, + "hits_at_10": 0.267170015743766, + "hits_at_3": 0.13026696687037634, + "hits_at_5": 0.179597433596858, + "inverse_arithmetic_mean_rank": 0.0014329224503223754, + "inverse_geometric_mean_rank": 0.015038565511925226, + "inverse_harmonic_mean_rank": 0.11936525268189527, + "inverse_median_rank": 0.015384615384615385, + "median_absolute_deviation": 93.40393976585293, + "median_rank": 65.0, + "standard_deviation": 1827.828599332042, + "variance": 3340957.388536134, + "z_arithmetic_mean_rank": 381.0798556539635, + "z_geometric_mean_rank": 240.52049143624475, + "z_hits_at_k": 2491.5898231805772, + "z_inverse_harmonic_mean_rank": 2740.7652002715145 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.09421880322545154, + "adjusted_arithmetic_mean_rank_index": 0.9059035012036225, + "adjusted_geometric_mean_rank_index": 0.9879743240448803, + "adjusted_hits_at_k": 0.2666741240328079, + "adjusted_inverse_harmonic_mean_rank": 0.11875874103301752, + "arithmetic_mean_rank": 697.87451171875, + "count": 59071.0, + "geometric_mean_rank": 66.4957046508789, + "harmonic_mean_rank": 8.377647337533142, + "hits_at_1": 0.04907653501718271, + "hits_at_10": 0.267170015743766, + "hits_at_3": 0.13026696687037634, + "hits_at_5": 0.179597433596858, + "inverse_arithmetic_mean_rank": 0.001432922319509089, + "inverse_geometric_mean_rank": 0.0150385657325387, + "inverse_harmonic_mean_rank": 0.11936525371745442, + "inverse_median_rank": 0.015384615398943424, + "median_absolute_deviation": 93.40393976585293, + "median_rank": 65.0, + "standard_deviation": 1827.828369140625, + "variance": 3340956.75, + "z_arithmetic_mean_rank": 381.079853459924, + "z_geometric_mean_rank": 240.52049140457794, + "z_hits_at_k": 2491.5898231805772, + "z_inverse_harmonic_mean_rank": 2740.7652241870533 + } + } + }, + "times": { + "evaluation": 31.021438598632812, + "training": 9.139031410217285 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json new file mode 100644 index 0000000..11968d1 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json new file mode 100644 index 0000000..017cd20 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.0004529083935333639, + 0.00038640364365897227, + 0.00035145446217553473, + 0.0003258956506732659, + 0.0003048123034097779 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0818915906396609, + "adjusted_arithmetic_mean_rank_index": 0.9181532777971985, + "adjusted_geometric_mean_rank_index": 0.9959612239406452, + "adjusted_hits_at_k": 0.3173331922397517, + "adjusted_inverse_harmonic_mean_rank": 0.1765106114247664, + "arithmetic_mean_rank": 1675.7667, + "count": 10000.0, + "geometric_mean_rank": 61.814524182048764, + "harmonic_mean_rank": 5.658159708335188, + "hits_at_1": 0.1071, + "hits_at_10": 0.3175, + "hits_at_3": 0.1959, + "hits_at_5": 0.2414, + "inverse_arithmetic_mean_rank": 0.0005967417779575165, + "inverse_geometric_mean_rank": 0.016177427768511478, + "inverse_harmonic_mean_rank": 0.17673590912021675, + "inverse_median_rank": 0.0196078431372549, + "median_absolute_deviation": 74.13011092528009, + "median_rank": 51.0, + "standard_deviation": 5371.088487231532, + "variance": 28848591.537671108, + "z_arithmetic_mean_rank": 159.02472772888677, + "z_geometric_mean_rank": 99.6766781576916, + "z_hits_at_k": 2029.8254949907523, + "z_inverse_harmonic_mean_rank": 2786.0057005980207 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08189159552647424, + "adjusted_arithmetic_mean_rank_index": 0.9181532729101463, + "adjusted_geometric_mean_rank_index": 0.9959612237297443, + "adjusted_hits_at_k": 0.3173331922397517, + "adjusted_inverse_harmonic_mean_rank": 0.17651061139836607, + "arithmetic_mean_rank": 1675.7668, + "count": 10000.0, + "geometric_mean_rank": 61.814527357724415, + "harmonic_mean_rank": 5.658159709180158, + "hits_at_1": 0.1071, + "hits_at_10": 0.3175, + "hits_at_3": 0.1959, + "hits_at_5": 0.2414, + "inverse_arithmetic_mean_rank": 0.0005967417423474435, + "inverse_geometric_mean_rank": 0.016177426937408087, + "inverse_harmonic_mean_rank": 0.17673590909382364, + "inverse_median_rank": 0.0196078431372549, + "median_absolute_deviation": 74.13011092528009, + "median_rank": 51.0, + "standard_deviation": 5371.088492272098, + "variance": 28848591.591817763, + "z_arithmetic_mean_rank": 159.02472688244623, + "z_geometric_mean_rank": 99.67667813658444, + "z_hits_at_k": 2029.8254949907523, + "z_inverse_harmonic_mean_rank": 2786.005700181324 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08189159775990064, + "adjusted_arithmetic_mean_rank_index": 0.9181532706766107, + "adjusted_geometric_mean_rank_index": 0.9959612240995342, + "adjusted_hits_at_k": 0.3173331922397517, + "adjusted_inverse_harmonic_mean_rank": 0.17651061539157242, + "arithmetic_mean_rank": 1675.766845703125, + "count": 10000.0, + "geometric_mean_rank": 61.81452178955078, + "harmonic_mean_rank": 5.658159581373549, + "hits_at_1": 0.1071, + "hits_at_10": 0.3175, + "hits_at_3": 0.1959, + "hits_at_5": 0.2414, + "inverse_arithmetic_mean_rank": 0.0005967417382635176, + "inverse_geometric_mean_rank": 0.016177428886294365, + "inverse_harmonic_mean_rank": 0.1767359130859375, + "inverse_median_rank": 0.019607843831181526, + "median_absolute_deviation": 74.13011092528009, + "median_rank": 51.0, + "standard_deviation": 5371.08837890625, + "variance": 28848590.0, + "z_arithmetic_mean_rank": 159.02472649559647, + "z_geometric_mean_rank": 99.67667817359336, + "z_hits_at_k": 2029.8254949907523, + "z_inverse_harmonic_mean_rank": 2786.005763209238 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0814552638078817, + "adjusted_arithmetic_mean_rank_index": 0.9185896269932536, + "adjusted_geometric_mean_rank_index": 0.9960286009146595, + "adjusted_hits_at_k": 0.31803335943219535, + "adjusted_inverse_harmonic_mean_rank": 0.1759327156692569, + "arithmetic_mean_rank": 1666.7994, + "count": 5000.0, + "geometric_mean_rank": 60.801578002011404, + "harmonic_mean_rank": 5.676716353502426, + "hits_at_1": 0.1036, + "hits_at_10": 0.3182, + "hits_at_3": 0.1976, + "hits_at_5": 0.2428, + "inverse_arithmetic_mean_rank": 0.0005999522198052147, + "inverse_geometric_mean_rank": 0.016446941557452976, + "inverse_harmonic_mean_rank": 0.17615817626382177, + "inverse_median_rank": 0.02040816326530612, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 49.0, + "standard_deviation": 5358.16252366048, + "variance": 28709905.62995964, + "z_arithmetic_mean_rank": 112.50089413340184, + "z_geometric_mean_rank": 70.49203961549347, + "z_hits_at_k": 1438.4534399644135, + "z_inverse_harmonic_mean_rank": 1963.5308287567036 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0814552638078817, + "adjusted_arithmetic_mean_rank_index": 0.9185896269932536, + "adjusted_geometric_mean_rank_index": 0.9960286009146595, + "adjusted_hits_at_k": 0.31803335943219535, + "adjusted_inverse_harmonic_mean_rank": 0.1759327156692569, + "arithmetic_mean_rank": 1666.7994, + "count": 5000.0, + "geometric_mean_rank": 60.801578002011404, + "harmonic_mean_rank": 5.676716353502426, + "hits_at_1": 0.1036, + "hits_at_10": 0.3182, + "hits_at_3": 0.1976, + "hits_at_5": 0.2428, + "inverse_arithmetic_mean_rank": 0.0005999522198052147, + "inverse_geometric_mean_rank": 0.016446941557452976, + "inverse_harmonic_mean_rank": 0.17615817626382177, + "inverse_median_rank": 0.02040816326530612, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 49.0, + "standard_deviation": 5358.16252366048, + "variance": 28709905.62995964, + "z_arithmetic_mean_rank": 112.50089413340184, + "z_geometric_mean_rank": 70.49203961549347, + "z_hits_at_k": 1438.4534399644135, + "z_inverse_harmonic_mean_rank": 1963.5308287567036 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08145526568820308, + "adjusted_arithmetic_mean_rank_index": 0.9185896251128404, + "adjusted_geometric_mean_rank_index": 0.9960286018934754, + "adjusted_hits_at_k": 0.31803335943219535, + "adjusted_inverse_harmonic_mean_rank": 0.1759327181170423, + "arithmetic_mean_rank": 1666.7994384765625, + "count": 5000.0, + "geometric_mean_rank": 60.80156326293945, + "harmonic_mean_rank": 5.676716274643858, + "hits_at_1": 0.1036, + "hits_at_10": 0.3182, + "hits_at_3": 0.1976, + "hits_at_5": 0.2428, + "inverse_arithmetic_mean_rank": 0.0005999521818011999, + "inverse_geometric_mean_rank": 0.016446946188807487, + "inverse_harmonic_mean_rank": 0.1761581787109375, + "inverse_median_rank": 0.020408162847161293, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 49.0, + "standard_deviation": 5358.16259765625, + "variance": 28709906.0, + "z_arithmetic_mean_rank": 112.50089390310512, + "z_geometric_mean_rank": 70.4920396847673, + "z_hits_at_k": 1438.4534399644135, + "z_inverse_harmonic_mean_rank": 1963.5308560756825 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08232789724114213, + "adjusted_arithmetic_mean_rank_index": 0.9177169488334667, + "adjusted_geometric_mean_rank_index": 0.9958931326163891, + "adjusted_hits_at_k": 0.31663302505530083, + "adjusted_inverse_harmonic_mean_rank": 0.17708850717355146, + "arithmetic_mean_rank": 1684.734, + "count": 5000.0, + "geometric_mean_rank": 62.84434591034264, + "harmonic_mean_rank": 5.6397239876890195, + "hits_at_1": 0.1106, + "hits_at_10": 0.3168, + "hits_at_3": 0.1942, + "hits_at_5": 0.24, + "inverse_arithmetic_mean_rank": 0.0005935655124191712, + "inverse_geometric_mean_rank": 0.01591233046528414, + "inverse_harmonic_mean_rank": 0.17731364197661176, + "inverse_median_rank": 0.018518518518518517, + "median_absolute_deviation": 78.5779175807969, + "median_rank": 54.0, + "standard_deviation": 5383.968482489845, + "variance": 28987116.620444, + "z_arithmetic_mean_rank": 112.39403510187431, + "z_geometric_mean_rank": 70.48245014898455, + "z_hits_at_k": 1432.1532309511042, + "z_inverse_harmonic_mean_rank": 1976.4763690695154 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08232790701454225, + "adjusted_arithmetic_mean_rank_index": 0.9177169390595888, + "adjusted_geometric_mean_rank_index": 0.9958931321875913, + "adjusted_hits_at_k": 0.31663302505530083, + "adjusted_inverse_harmonic_mean_rank": 0.17708850712075072, + "arithmetic_mean_rank": 1684.7342, + "count": 5000.0, + "geometric_mean_rank": 62.844352367506836, + "harmonic_mean_rank": 5.639723989367966, + "hits_at_1": 0.1106, + "hits_at_10": 0.3168, + "hits_at_3": 0.1942, + "hits_at_5": 0.24, + "inverse_arithmetic_mean_rank": 0.000593565441955176, + "inverse_geometric_mean_rank": 0.015912328830315736, + "inverse_harmonic_mean_rank": 0.17731364192382545, + "inverse_median_rank": 0.018518518518518517, + "median_absolute_deviation": 78.5779175807969, + "median_rank": 54.0, + "standard_deviation": 5383.968492213747, + "variance": 28987116.72515036, + "z_arithmetic_mean_rank": 112.39403390485428, + "z_geometric_mean_rank": 70.4824501186372, + "z_hits_at_k": 1432.1532309511042, + "z_inverse_harmonic_mean_rank": 1976.4763684802087 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08232789767063727, + "adjusted_arithmetic_mean_rank_index": 0.9177169484039505, + "adjusted_geometric_mean_rank_index": 0.9958931331773225, + "adjusted_hits_at_k": 0.31663302505530083, + "adjusted_inverse_harmonic_mean_rank": 0.17708850044900623, + "arithmetic_mean_rank": 1684.7340087890625, + "count": 5000.0, + "geometric_mean_rank": 62.844337463378906, + "harmonic_mean_rank": 5.639724201514671, + "hits_at_1": 0.1106, + "hits_at_10": 0.3168, + "hits_at_3": 0.1942, + "hits_at_5": 0.24, + "inverse_arithmetic_mean_rank": 0.0005935655208304524, + "inverse_geometric_mean_rank": 0.015912333503365517, + "inverse_harmonic_mean_rank": 0.17731363525390625, + "inverse_median_rank": 0.018518518656492233, + "median_absolute_deviation": 78.5779175807969, + "median_rank": 54.0, + "standard_deviation": 5383.96826171875, + "variance": 28987116.0, + "z_arithmetic_mean_rank": 112.39403504927088, + "z_geometric_mean_rank": 70.48245018868356, + "z_hits_at_k": 1432.1532309511042, + "z_inverse_harmonic_mean_rank": 1976.4762940171875 + } + } + }, + "times": { + "evaluation": 4.746052265167236, + "training": 1.822469711303711 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json new file mode 100644 index 0000000..91b1a21 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json new file mode 100644 index 0000000..a840c4c --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 9.350439590000361e-05, + 1.3484684583750924e-05, + 1.1116209103459396e-05, + 9.792114026757163e-06, + 8.904646023875852e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.061309328820577665, + "adjusted_arithmetic_mean_rank_index": 0.9388177642514287, + "adjusted_geometric_mean_rank_index": 0.9926356631379782, + "adjusted_hits_at_k": 0.34970533694843586, + "adjusted_inverse_harmonic_mean_rank": 0.23031375805864562, + "arithmetic_mean_rank": 452.8829628751841, + "count": 118142.0, + "geometric_mean_rank": 40.99283647360712, + "harmonic_mean_rank": 4.331911916722617, + "hits_at_1": 0.16881380034196136, + "hits_at_10": 0.3501464339523624, + "hits_at_3": 0.24841292681688137, + "hits_at_5": 0.2896345076264157, + "inverse_arithmetic_mean_rank": 0.002208075997496958, + "inverse_geometric_mean_rank": 0.02439450611435101, + "inverse_harmonic_mean_rank": 0.23084495234994698, + "inverse_median_rank": 0.022727272727272728, + "median_absolute_deviation": 63.75189539574088, + "median_rank": 44.0, + "standard_deviation": 1078.3816682039305, + "variance": 1162907.0223182922, + "z_arithmetic_mean_rank": 558.4250406185861, + "z_geometric_mean_rank": 341.77861851221815, + "z_hits_at_k": 4613.659080043995, + "z_inverse_harmonic_mean_rank": 7505.435015590688 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.061309349446280914, + "adjusted_arithmetic_mean_rank_index": 0.9388177436229329, + "adjusted_geometric_mean_rank_index": 0.9926356616147918, + "adjusted_hits_at_k": 0.34970533694843586, + "adjusted_inverse_harmonic_mean_rank": 0.2303137574763396, + "arithmetic_mean_rank": 452.8831152342097, + "count": 118142.0, + "geometric_mean_rank": 40.99284474543766, + "harmonic_mean_rank": 4.331911927642317, + "hits_at_1": 0.16881380034196136, + "hits_at_10": 0.3501464339523624, + "hits_at_3": 0.24841292681688137, + "hits_at_5": 0.2896345076264157, + "inverse_arithmetic_mean_rank": 0.002208075254655602, + "inverse_geometric_mean_rank": 0.024394501191852415, + "inverse_harmonic_mean_rank": 0.23084495176804282, + "inverse_median_rank": 0.022727272727272728, + "median_absolute_deviation": 63.75189539574088, + "median_rank": 44.0, + "standard_deviation": 1078.3819740001402, + "variance": 1162907.681848439, + "z_arithmetic_mean_rank": 558.4250283484, + "z_geometric_mean_rank": 341.77861798776337, + "z_hits_at_k": 4613.659080043995, + "z_inverse_harmonic_mean_rank": 7505.434996614574 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.061309345645473065, + "adjusted_arithmetic_mean_rank_index": 0.9388177474242554, + "adjusted_geometric_mean_rank_index": 0.9926356611181276, + "adjusted_hits_at_k": 0.34970533694843586, + "adjusted_inverse_harmonic_mean_rank": 0.23031375818123168, + "arithmetic_mean_rank": 452.8830871582031, + "count": 118142.0, + "geometric_mean_rank": 40.99284744262695, + "harmonic_mean_rank": 4.331911914423821, + "hits_at_1": 0.16881380034196136, + "hits_at_10": 0.3501464339523624, + "hits_at_3": 0.24841292681688137, + "hits_at_5": 0.2896345076264157, + "inverse_arithmetic_mean_rank": 0.002208075486123562, + "inverse_geometric_mean_rank": 0.024394499137997627, + "inverse_harmonic_mean_rank": 0.23084495247244843, + "inverse_median_rank": 0.022727273404598236, + "median_absolute_deviation": 63.75189539574088, + "median_rank": 44.0, + "standard_deviation": 1078.3818359375, + "variance": 1162907.5, + "z_arithmetic_mean_rank": 558.4250306094924, + "z_geometric_mean_rank": 341.7786178167548, + "z_hits_at_k": 4613.659080043995, + "z_inverse_harmonic_mean_rank": 7505.435019585506 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0681778728311803, + "adjusted_arithmetic_mean_rank_index": 0.9319486346193927, + "adjusted_geometric_mean_rank_index": 0.9906846783036631, + "adjusted_hits_at_k": 0.31298586873037954, + "adjusted_inverse_harmonic_mean_rank": 0.19818517693798296, + "arithmetic_mean_rank": 502.2492762946285, + "count": 59071.0, + "geometric_mean_rank": 51.44235107340421, + "harmonic_mean_rank": 5.031698071957418, + "hits_at_1": 0.13917150547646054, + "hits_at_10": 0.3134533019586599, + "hits_at_3": 0.21396285825531988, + "hits_at_5": 0.2538809229571194, + "inverse_arithmetic_mean_rank": 0.0019910431875134887, + "inverse_geometric_mean_rank": 0.019439235943417094, + "inverse_harmonic_mean_rank": 0.19874006462613178, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1139.641318600422, + "variance": 1298782.3350613085, + "z_arithmetic_mean_rank": 391.9201913278106, + "z_geometric_mean_rank": 241.18228153319507, + "z_hits_at_k": 2915.329058883141, + "z_inverse_harmonic_mean_rank": 4559.832862637971 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.068177891215171, + "adjusted_arithmetic_mean_rank_index": 0.9319486162329061, + "adjusted_geometric_mean_rank_index": 0.9906846771163226, + "adjusted_hits_at_k": 0.31298586873037954, + "adjusted_inverse_harmonic_mean_rank": 0.19818517668041305, + "arithmetic_mean_rank": 502.2494117248735, + "count": 59071.0, + "geometric_mean_rank": 51.44235750283864, + "harmonic_mean_rank": 5.031698078474057, + "hits_at_1": 0.13917150547646054, + "hits_at_10": 0.3134533019586599, + "hits_at_3": 0.21396285825531988, + "hits_at_5": 0.2538809229571194, + "inverse_arithmetic_mean_rank": 0.001991042650633882, + "inverse_geometric_mean_rank": 0.019439233513837677, + "inverse_harmonic_mean_rank": 0.19874006436874012, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1139.6416993059368, + "variance": 1298783.2027969232, + "z_arithmetic_mean_rank": 391.9201835955869, + "z_geometric_mean_rank": 241.18228124413687, + "z_hits_at_k": 2915.329058883141, + "z_inverse_harmonic_mean_rank": 4559.832856711818 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.06817788407579185, + "adjusted_arithmetic_mean_rank_index": 0.9319486233732546, + "adjusted_geometric_mean_rank_index": 0.9906846794870225, + "adjusted_hits_at_k": 0.31298586873037954, + "adjusted_inverse_harmonic_mean_rank": 0.19818519443912916, + "arithmetic_mean_rank": 502.2493591308594, + "count": 59071.0, + "geometric_mean_rank": 51.442344665527344, + "harmonic_mean_rank": 5.03169762917033, + "hits_at_1": 0.13917150547646054, + "hits_at_10": 0.3134533019586599, + "hits_at_3": 0.21396285825531988, + "hits_at_5": 0.2538809229571194, + "inverse_arithmetic_mean_rank": 0.0019910428673028946, + "inverse_geometric_mean_rank": 0.01943923905491829, + "inverse_harmonic_mean_rank": 0.1987400821151665, + "inverse_median_rank": 0.01666666753590107, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1139.6414794921875, + "variance": 1298782.75, + "z_arithmetic_mean_rank": 391.9201865983776, + "z_geometric_mean_rank": 241.18228182128405, + "z_hits_at_k": 2915.329058883141, + "z_inverse_harmonic_mean_rank": 4559.83326530331 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.054478068986399376, + "adjusted_arithmetic_mean_rank_index": 0.9456496014933604, + "adjusted_geometric_mean_rank_index": 0.9941858125013604, + "adjusted_hits_at_k": 0.38642465227813766, + "adjusted_inverse_harmonic_mean_rank": 0.26244221727901273, + "arithmetic_mean_rank": 403.5166494557397, + "count": 59071.0, + "geometric_mean_rank": 32.66593783309161, + "harmonic_mean_rank": 3.803006686444386, + "hits_at_1": 0.1984560952074622, + "hits_at_10": 0.3868395659460649, + "hits_at_3": 0.2828629953784429, + "hits_at_5": 0.3253880922957119, + "inverse_arithmetic_mean_rank": 0.0024782124885027487, + "inverse_geometric_mean_rank": 0.0306129279100926, + "inverse_harmonic_mean_rank": 0.2629498400737623, + "inverse_median_rank": 0.03125, + "median_absolute_deviation": 45.96066877367366, + "median_rank": 32.0, + "standard_deviation": 1011.0181223766575, + "variance": 1022157.643774022, + "z_arithmetic_mean_rank": 397.799557108152, + "z_geometric_mean_rank": 242.03266658924255, + "z_hits_at_k": 3610.4430249252455, + "z_inverse_harmonic_mean_rank": 6056.754191061033 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.054478091841646606, + "adjusted_arithmetic_mean_rank_index": 0.9456495786350271, + "adjusted_geometric_mean_rank_index": 0.9941858108304271, + "adjusted_hits_at_k": 0.38642465227813766, + "adjusted_inverse_harmonic_mean_rank": 0.2624422163719718, + "arithmetic_mean_rank": 403.5168187435459, + "count": 59071.0, + "geometric_mean_rank": 32.6659469335329, + "harmonic_mean_rank": 3.8030066995537632, + "hits_at_1": 0.1984560952074622, + "hits_at_10": 0.3868395659460649, + "hits_at_3": 0.2828629953784429, + "hits_at_5": 0.3253880922957119, + "inverse_arithmetic_mean_rank": 0.002478211448815836, + "inverse_geometric_mean_rank": 0.03061291938160409, + "inverse_harmonic_mean_rank": 0.26294983916734566, + "inverse_median_rank": 0.03125, + "median_absolute_deviation": 45.96066877367366, + "median_rank": 32.0, + "standard_deviation": 1011.0183472329007, + "variance": 1022158.0984415463, + "z_arithmetic_mean_rank": 397.79954749250277, + "z_geometric_mean_rank": 242.03266618245698, + "z_hits_at_k": 3610.4430249252455, + "z_inverse_harmonic_mean_rank": 6056.754170127952 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.054478079000914746, + "adjusted_arithmetic_mean_rank_index": 0.9456495914774928, + "adjusted_geometric_mean_rank_index": 0.9941858129267364, + "adjusted_hits_at_k": 0.38642465227813766, + "adjusted_inverse_harmonic_mean_rank": 0.2624422165665027, + "arithmetic_mean_rank": 403.5167236328125, + "count": 59071.0, + "geometric_mean_rank": 32.66593551635742, + "harmonic_mean_rank": 3.8030066967422265, + "hits_at_1": 0.1984560952074622, + "hits_at_10": 0.3868395659460649, + "hits_at_3": 0.2828629953784429, + "hits_at_5": 0.3253880922957119, + "inverse_arithmetic_mean_rank": 0.0024782121181488037, + "inverse_geometric_mean_rank": 0.03061293065547943, + "inverse_harmonic_mean_rank": 0.26294983936174265, + "inverse_median_rank": 0.03125, + "median_absolute_deviation": 45.96066877367366, + "median_rank": 32.0, + "standard_deviation": 1011.0182495117188, + "variance": 1022157.875, + "z_arithmetic_mean_rank": 397.7995528948496, + "z_geometric_mean_rank": 242.03266669279952, + "z_hits_at_k": 3610.4430249252455, + "z_inverse_harmonic_mean_rank": 6056.75417461742 + } + } + }, + "times": { + "evaluation": 15.175896167755127, + "training": 160.1173620223999 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json new file mode 100644 index 0000000..991276c --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json new file mode 100644 index 0000000..375b5c8 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 4096, + "slice_size": null + }, + "losses": [ + 0.0001553554180276956, + 1.53616951009097e-05, + 1.370961714377088e-05, + 1.2225620274735016e-05, + 1.108662612753889e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.18175210325712907, + "adjusted_arithmetic_mean_rank_index": 0.8183625829185686, + "adjusted_geometric_mean_rank_index": 0.9847747167425137, + "adjusted_hits_at_k": 0.3200769143496645, + "adjusted_inverse_harmonic_mean_rank": 0.2096363216603547, + "arithmetic_mean_rank": 1296.9228398082005, + "count": 40876.0, + "geometric_mean_rank": 80.83336590952712, + "harmonic_mean_rank": 4.757383657725782, + "hits_at_1": 0.1545894901653782, + "hits_at_10": 0.32055484881103824, + "hits_at_3": 0.22717487033956354, + "hits_at_5": 0.2661463939720129, + "inverse_arithmetic_mean_rank": 0.0007710558942333748, + "inverse_geometric_mean_rank": 0.012371129034998391, + "inverse_harmonic_mean_rank": 0.21019957017257668, + "inverse_median_rank": 0.011235955056179775, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 89.0, + "standard_deviation": 2518.7222644431486, + "variance": 6343961.845401622, + "z_arithmetic_mean_rank": 286.20881146568547, + "z_geometric_mean_rank": 199.44238519906037, + "z_hits_at_k": 2439.957034486236, + "z_inverse_harmonic_mean_rank": 3947.578789367722 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.18175216154065776, + "adjusted_arithmetic_mean_rank_index": 0.8183625246268708, + "adjusted_geometric_mean_rank_index": 0.9847747097533328, + "adjusted_hits_at_k": 0.3200769143496645, + "adjusted_inverse_harmonic_mean_rank": 0.2096363193214703, + "arithmetic_mean_rank": 1296.9232557001665, + "count": 40876.0, + "geometric_mean_rank": 80.83340255711062, + "harmonic_mean_rank": 4.757383710623325, + "hits_at_1": 0.1545894901653782, + "hits_at_10": 0.32055484881103824, + "hits_at_3": 0.22717487033956354, + "hits_at_5": 0.2661463939720129, + "inverse_arithmetic_mean_rank": 0.0007710556469743714, + "inverse_geometric_mean_rank": 0.012371123426277615, + "inverse_harmonic_mean_rank": 0.2101995678353591, + "inverse_median_rank": 0.011235955056179775, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 89.0, + "standard_deviation": 2518.7227934087005, + "variance": 6343964.510036528, + "z_arithmetic_mean_rank": 286.20879107912594, + "z_geometric_mean_rank": 199.44238378357022, + "z_hits_at_k": 2439.957034486236, + "z_inverse_harmonic_mean_rank": 3947.578745325114 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1817521391185075, + "adjusted_arithmetic_mean_rank_index": 0.8183625470521637, + "adjusted_geometric_mean_rank_index": 0.9847747108300027, + "adjusted_hits_at_k": 0.3200769143496645, + "adjusted_inverse_harmonic_mean_rank": 0.20963631081804915, + "arithmetic_mean_rank": 1296.923095703125, + "count": 40876.0, + "geometric_mean_rank": 80.8333969116211, + "harmonic_mean_rank": 4.757383902941559, + "hits_at_1": 0.1545894901653782, + "hits_at_10": 0.32055484881103824, + "hits_at_3": 0.22717487033956354, + "hits_at_5": 0.2661463939720129, + "inverse_arithmetic_mean_rank": 0.0007710557547397912, + "inverse_geometric_mean_rank": 0.0123711246997118, + "inverse_harmonic_mean_rank": 0.21019955933799786, + "inverse_median_rank": 0.01123595517128706, + "median_absolute_deviation": 130.46899522849296, + "median_rank": 89.0, + "standard_deviation": 2518.72265625, + "variance": 6343963.5, + "z_arithmetic_mean_rank": 286.2087989220022, + "z_geometric_mean_rank": 199.44238400162374, + "z_hits_at_k": 2439.957034486236, + "z_inverse_harmonic_mean_rank": 3947.5785852005483 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23954731895542877, + "adjusted_arithmetic_mean_rank_index": 0.7605607668569763, + "adjusted_geometric_mean_rank_index": 0.9634020053121783, + "adjusted_hits_at_k": 0.19567726258469004, + "adjusted_inverse_harmonic_mean_rank": 0.11268113063070488, + "arithmetic_mean_rank": 1685.6078383403465, + "count": 20438.0, + "geometric_mean_rank": 190.0114797272476, + "harmonic_mean_rank": 8.824342686842055, + "hits_at_1": 0.07123984734318427, + "hits_at_10": 0.19625207945982973, + "hits_at_3": 0.12036402779136902, + "hits_at_5": 0.15104217633819356, + "inverse_arithmetic_mean_rank": 0.0005932578012834838, + "inverse_geometric_mean_rank": 0.005262839915964299, + "inverse_harmonic_mean_rank": 0.11332288822952177, + "inverse_median_rank": 0.003125, + "median_absolute_deviation": 471.46750548478144, + "median_rank": 320.0, + "standard_deviation": 2761.296311585376, + "variance": 7624757.320375001, + "z_arithmetic_mean_rank": 187.88604932091593, + "z_geometric_mean_rank": 137.97311438795538, + "z_hits_at_k": 1046.058168464544, + "z_inverse_harmonic_mean_rank": 1488.0438353149812 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23954740934945482, + "adjusted_arithmetic_mean_rank_index": 0.7605606764501023, + "adjusted_geometric_mean_rank_index": 0.9634019815009212, + "adjusted_hits_at_k": 0.19567726258469004, + "adjusted_inverse_harmonic_mean_rank": 0.11268112698425874, + "arithmetic_mean_rank": 1685.6084744104119, + "count": 20438.0, + "geometric_mean_rank": 190.0116027012244, + "harmonic_mean_rank": 8.8243429705819, + "hits_at_1": 0.07123984734318427, + "hits_at_10": 0.19625207945982973, + "hits_at_3": 0.12036402779136902, + "hits_at_5": 0.15104217633819356, + "inverse_arithmetic_mean_rank": 0.0005932575774156438, + "inverse_geometric_mean_rank": 0.0052628365098967515, + "inverse_harmonic_mean_rank": 0.11332288458571294, + "inverse_median_rank": 0.003125, + "median_absolute_deviation": 471.46750548478144, + "median_rank": 320.0, + "standard_deviation": 2761.296903529542, + "variance": 7624760.589441836, + "z_arithmetic_mean_rank": 187.8860269871444, + "z_geometric_mean_rank": 137.97311097783862, + "z_hits_at_k": 1046.058168464544, + "z_inverse_harmonic_mean_rank": 1488.0437871607633 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2395473638570587, + "adjusted_arithmetic_mean_rank_index": 0.7605607219489643, + "adjusted_geometric_mean_rank_index": 0.9634019826668618, + "adjusted_hits_at_k": 0.19567726258469004, + "adjusted_inverse_harmonic_mean_rank": 0.11268113416020661, + "arithmetic_mean_rank": 1685.608154296875, + "count": 20438.0, + "geometric_mean_rank": 190.0115966796875, + "harmonic_mean_rank": 8.824342412201986, + "hits_at_1": 0.07123984734318427, + "hits_at_10": 0.19625207945982973, + "hits_at_3": 0.12036402779136902, + "hits_at_5": 0.15104217633819356, + "inverse_arithmetic_mean_rank": 0.000593257718719542, + "inverse_geometric_mean_rank": 0.005262836813926697, + "inverse_harmonic_mean_rank": 0.11332289175647078, + "inverse_median_rank": 0.0031250000465661287, + "median_absolute_deviation": 471.46750548478144, + "median_rank": 320.0, + "standard_deviation": 2761.296630859375, + "variance": 7624758.5, + "z_arithmetic_mean_rank": 187.88603822701086, + "z_geometric_mean_rank": 137.9731111448182, + "z_hits_at_k": 1046.058168464544, + "z_inverse_harmonic_mean_rank": 1488.0438819248552 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.12553912270443726, + "adjusted_arithmetic_mean_rank_index": 0.8745817643831122, + "adjusted_geometric_mean_rank_index": 0.9937285636670846, + "adjusted_hits_at_k": 0.4444736443896051, + "adjusted_inverse_harmonic_mean_rank": 0.30658945384952835, + "arithmetic_mean_rank": 908.2378412760544, + "count": 20438.0, + "geometric_mean_rank": 34.38756991757967, + "harmonic_mean_rank": 3.2565201415296796, + "hits_at_1": 0.23793913298757216, + "hits_at_10": 0.4448576181622468, + "hits_at_3": 0.3339857128877581, + "hits_at_5": 0.3812506116058323, + "inverse_arithmetic_mean_rank": 0.0011010331815673104, + "inverse_geometric_mean_rank": 0.02908027529705664, + "inverse_harmonic_mean_rank": 0.30707625211563155, + "inverse_median_rank": 0.05, + "median_absolute_deviation": 28.169442151606436, + "median_rank": 20.0, + "standard_deviation": 2181.974865634427, + "variance": 4761014.314260376, + "z_arithmetic_mean_rank": 216.54184149302714, + "z_geometric_mean_rank": 142.31002549861495, + "z_hits_at_k": 2416.1095916884306, + "z_inverse_harmonic_mean_rank": 4116.719306532869 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12553914975654407, + "adjusted_arithmetic_mean_rank_index": 0.8745817373272656, + "adjusted_geometric_mean_rank_index": 0.9937285619905694, + "adjusted_hits_at_k": 0.4444736443896051, + "adjusted_inverse_harmonic_mean_rank": 0.30658945281817807, + "arithmetic_mean_rank": 908.2380369899207, + "count": 20438.0, + "geometric_mean_rank": 34.38757884293027, + "harmonic_mean_rank": 3.256520152459392, + "hits_at_1": 0.23793913298757216, + "hits_at_10": 0.4448576181622468, + "hits_at_3": 0.3339857128877581, + "hits_at_5": 0.3812506116058323, + "inverse_arithmetic_mean_rank": 0.0011010329443085169, + "inverse_geometric_mean_rank": 0.029080267749225088, + "inverse_harmonic_mean_rank": 0.3070762510850053, + "inverse_median_rank": 0.05, + "median_absolute_deviation": 28.169442151606436, + "median_rank": 20.0, + "standard_deviation": 2181.9752592876844, + "variance": 4761016.032143558, + "z_arithmetic_mean_rank": 216.54183479414195, + "z_geometric_mean_rank": 142.31002525852432, + "z_hits_at_k": 2416.1095916884306, + "z_inverse_harmonic_mean_rank": 4116.719292684448 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.12553914133660884, + "adjusted_arithmetic_mean_rank_index": 0.8745817457483649, + "adjusted_geometric_mean_rank_index": 0.9937285608929696, + "adjusted_hits_at_k": 0.4444736443896051, + "adjusted_inverse_harmonic_mean_rank": 0.3065894525433575, + "arithmetic_mean_rank": 908.2379760742188, + "count": 20438.0, + "geometric_mean_rank": 34.3875846862793, + "harmonic_mean_rank": 3.2565201553717964, + "hits_at_1": 0.23793913298757216, + "hits_at_10": 0.4448576181622468, + "hits_at_3": 0.3339857128877581, + "hits_at_5": 0.3812506116058323, + "inverse_arithmetic_mean_rank": 0.0011010330636054277, + "inverse_geometric_mean_rank": 0.029080262407660484, + "inverse_harmonic_mean_rank": 0.3070762508103777, + "inverse_median_rank": 0.05000000074505806, + "median_absolute_deviation": 28.169442151606436, + "median_rank": 20.0, + "standard_deviation": 2181.97509765625, + "variance": 4761015.5, + "z_arithmetic_mean_rank": 216.5418368791618, + "z_geometric_mean_rank": 142.31002510133908, + "z_hits_at_k": 2416.1095916884306, + "z_inverse_harmonic_mean_rank": 4116.719288994305 + } + } + }, + "times": { + "evaluation": 6.952138900756836, + "training": 87.22090458869934 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json new file mode 100644 index 0000000..a3fef32 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json new file mode 100644 index 0000000..59250d4 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 8192, + "slice_size": null + }, + "losses": [ + 0.00012069320004326263, + 3.8810777744959124e-06, + 3.485230710315089e-06, + 3.3256080524316043e-06, + 3.18438816935551e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6801818998982223, + "adjusted_arithmetic_mean_rank_index": 0.3198337297791498, + "adjusted_geometric_mean_rank_index": 0.7519856699805167, + "adjusted_hits_at_k": 0.04796737344145887, + "adjusted_inverse_harmonic_mean_rank": 0.02738208123845674, + "arithmetic_mean_rank": 13918.7207, + "count": 10000.0, + "geometric_mean_rank": 3735.5159149214624, + "harmonic_mean_rank": 36.1687474865135, + "hits_at_1": 0.0169, + "hits_at_10": 0.0482, + "hits_at_3": 0.0272, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.18456833464587e-05, + "inverse_geometric_mean_rank": 0.0002677006396908965, + "inverse_harmonic_mean_rank": 0.027648178869696196, + "inverse_median_rank": 9.786651008025054e-05, + "median_absolute_deviation": 14536.173451338174, + "median_rank": 10218.0, + "standard_deviation": 13115.026157339204, + "variance": 172003911.1076915, + "z_arithmetic_mean_rank": 55.39540404263296, + "z_geometric_mean_rank": 75.25938942609984, + "z_hits_at_k": 306.823868162061, + "z_inverse_harmonic_mean_rank": 432.1929078869801 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6801822859564761, + "adjusted_arithmetic_mean_rank_index": 0.3198333437020291, + "adjusted_geometric_mean_rank_index": 0.7519855443955777, + "adjusted_hits_at_k": 0.04796737344145887, + "adjusted_inverse_harmonic_mean_rank": 0.027382081193512736, + "arithmetic_mean_rank": 13918.7286, + "count": 10000.0, + "geometric_mean_rank": 3735.5178059370073, + "harmonic_mean_rank": 36.168747545292184, + "hits_at_1": 0.0169, + "hits_at_10": 0.0482, + "hits_at_3": 0.0272, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.184564256824434e-05, + "inverse_geometric_mean_rank": 0.00026770050417392207, + "inverse_harmonic_mean_rank": 0.02764817882476449, + "inverse_median_rank": 9.786651008025054e-05, + "median_absolute_deviation": 14536.173451338174, + "median_rank": 10218.0, + "standard_deviation": 13115.03010968492, + "variance": 172004014.77794203, + "z_arithmetic_mean_rank": 55.39533717383175, + "z_geometric_mean_rank": 75.25937685744833, + "z_hits_at_k": 306.823868162061, + "z_inverse_harmonic_mean_rank": 432.1929071775936 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6801820909420813, + "adjusted_arithmetic_mean_rank_index": 0.3198335387259543, + "adjusted_geometric_mean_rank_index": 0.7519856568071709, + "adjusted_hits_at_k": 0.04796737344145887, + "adjusted_inverse_harmonic_mean_rank": 0.027382083521728977, + "arithmetic_mean_rank": 13918.724609375, + "count": 10000.0, + "geometric_mean_rank": 3735.51611328125, + "harmonic_mean_rank": 36.16874450040376, + "hits_at_1": 0.0169, + "hits_at_10": 0.0482, + "hits_at_3": 0.0272, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.184566493378952e-05, + "inverse_geometric_mean_rank": 0.0002677006123121828, + "inverse_harmonic_mean_rank": 0.027648181152343753, + "inverse_median_rank": 9.78665120783262e-05, + "median_absolute_deviation": 14536.173451338174, + "median_rank": 10218.0, + "standard_deviation": 13115.02734375, + "variance": 172003952.0, + "z_arithmetic_mean_rank": 55.395370952098816, + "z_geometric_mean_rank": 75.25938810769976, + "z_hits_at_k": 306.823868162061, + "z_inverse_harmonic_mean_rank": 432.19294392566024 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7005482046785227, + "adjusted_arithmetic_mean_rank_index": 0.2994664300262051, + "adjusted_geometric_mean_rank_index": 0.7317236995864804, + "adjusted_hits_at_k": 0.0445665370044485, + "adjusted_inverse_harmonic_mean_rank": 0.025789613592086494, + "arithmetic_mean_rank": 14335.1488, + "count": 5000.0, + "geometric_mean_rank": 4040.7214584881763, + "harmonic_mean_rank": 38.37865153940608, + "hits_at_1": 0.0154, + "hits_at_10": 0.0448, + "hits_at_3": 0.0266, + "hits_at_5": 0.0326, + "inverse_arithmetic_mean_rank": 6.975860620295759e-05, + "inverse_geometric_mean_rank": 0.0002474805576858908, + "inverse_harmonic_mean_rank": 0.026056152571520892, + "inverse_median_rank": 9.11410864017499e-05, + "median_absolute_deviation": 15384.963221432632, + "median_rank": 10972.0, + "standard_deviation": 13176.422049504126, + "variance": 173618098.02665854, + "z_arithmetic_mean_rank": 36.676052233641556, + "z_geometric_mean_rank": 51.78636032286495, + "z_hits_at_k": 201.57284309986937, + "z_inverse_harmonic_mean_rank": 287.82993064793305 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7005485956326534, + "adjusted_arithmetic_mean_rank_index": 0.29946603905296787, + "adjusted_geometric_mean_rank_index": 0.7317235641683568, + "adjusted_hits_at_k": 0.0445665370044485, + "adjusted_inverse_harmonic_mean_rank": 0.0257896135490425, + "arithmetic_mean_rank": 14335.1568, + "count": 5000.0, + "geometric_mean_rank": 4040.723497622826, + "harmonic_mean_rank": 38.378651602789134, + "hits_at_1": 0.0154, + "hits_at_10": 0.0448, + "hits_at_3": 0.0266, + "hits_at_5": 0.0326, + "inverse_arithmetic_mean_rank": 6.975856727287419e-05, + "inverse_geometric_mean_rank": 0.0002474804327958357, + "inverse_harmonic_mean_rank": 0.026056152528488676, + "inverse_median_rank": 9.11410864017499e-05, + "median_absolute_deviation": 15384.963221432632, + "median_rank": 10972.0, + "standard_deviation": 13176.425496128066, + "variance": 173618188.85501376, + "z_arithmetic_mean_rank": 36.67600435062886, + "z_geometric_mean_rank": 51.78635073890348, + "z_hits_at_k": 201.57284309986937, + "z_inverse_harmonic_mean_rank": 287.8299301675323 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7005483778589853, + "adjusted_arithmetic_mean_rank_index": 0.2994662568372789, + "adjusted_geometric_mean_rank_index": 0.7317235227638998, + "adjusted_hits_at_k": 0.0445665370044485, + "adjusted_inverse_harmonic_mean_rank": 0.02578961336425327, + "arithmetic_mean_rank": 14335.15234375, + "count": 5000.0, + "geometric_mean_rank": 4040.72412109375, + "harmonic_mean_rank": 38.37865187489459, + "hits_at_1": 0.0154, + "hits_at_10": 0.0448, + "hits_at_3": 0.0266, + "hits_at_5": 0.0326, + "inverse_arithmetic_mean_rank": 6.975858559599146e-05, + "inverse_geometric_mean_rank": 0.000247480405960232, + "inverse_harmonic_mean_rank": 0.026056152343750003, + "inverse_median_rank": 9.114108979701996e-05, + "median_absolute_deviation": 15384.963221432632, + "median_rank": 10972.0, + "standard_deviation": 13176.423828125, + "variance": 173618144.0, + "z_arithmetic_mean_rank": 36.676031022963286, + "z_geometric_mean_rank": 51.78634780858139, + "z_hits_at_k": 201.57284309986937, + "z_inverse_harmonic_mean_rank": 287.8299281051567 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6598165394018486, + "adjusted_arithmetic_mean_rank_index": 0.34020008515588696, + "adjusted_geometric_mean_rank_index": 0.7707405910086426, + "adjusted_hits_at_k": 0.05136820983964757, + "adjusted_inverse_harmonic_mean_rank": 0.02897454886629688, + "arithmetic_mean_rank": 13502.2926, + "count": 5000.0, + "geometric_mean_rank": 3453.363290191359, + "harmonic_mean_rank": 34.19948643516285, + "hits_at_1": 0.0184, + "hits_at_10": 0.0516, + "hits_at_3": 0.0278, + "hits_at_5": 0.035, + "inverse_arithmetic_mean_rank": 7.406149678610875e-05, + "inverse_geometric_mean_rank": 0.000289572777599251, + "inverse_harmonic_mean_rank": 0.029240205167871503, + "inverse_median_rank": 0.00010305559849538827, + "median_absolute_deviation": 13846.022118623816, + "median_rank": 9703.5, + "standard_deviation": 13040.049825970193, + "variance": 170042899.46378523, + "z_arithmetic_mean_rank": 41.664764240515254, + "z_geometric_mean_rank": 54.54770547603589, + "z_hits_at_k": 232.341991733733, + "z_inverse_harmonic_mean_rank": 323.3835557863843 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6598169205644525, + "adjusted_arithmetic_mean_rank_index": 0.3401997039746557, + "adjusted_geometric_mean_rank_index": 0.7707404745556267, + "adjusted_hits_at_k": 0.05136820983964757, + "adjusted_inverse_harmonic_mean_rank": 0.028974548819452866, + "arithmetic_mean_rank": 13502.3004, + "count": 5000.0, + "geometric_mean_rank": 3453.36504382982, + "harmonic_mean_rank": 34.199486489936845, + "hits_at_1": 0.0184, + "hits_at_10": 0.0516, + "hits_at_3": 0.0278, + "hits_at_5": 0.035, + "inverse_arithmetic_mean_rank": 7.406145400231208e-05, + "inverse_geometric_mean_rank": 0.0002895726305525433, + "inverse_harmonic_mean_rank": 0.029240205121040305, + "inverse_median_rank": 0.00010305559849538827, + "median_absolute_deviation": 13846.022118623816, + "median_rank": 9703.5, + "standard_deviation": 13040.054287055704, + "variance": 170043015.80935985, + "z_arithmetic_mean_rank": 41.66471755673467, + "z_geometric_mean_rank": 54.54769723429426, + "z_hits_at_k": 232.341991733733, + "z_inverse_harmonic_mean_rank": 323.3835552635605 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6598167005865956, + "adjusted_arithmetic_mean_rank_index": 0.34019992396326293, + "adjusted_geometric_mean_rank_index": 0.7707405105397687, + "adjusted_hits_at_k": 0.05136820983964757, + "adjusted_inverse_harmonic_mean_rank": 0.028974547555488634, + "arithmetic_mean_rank": 13502.2958984375, + "count": 5000.0, + "geometric_mean_rank": 3453.364501953125, + "harmonic_mean_rank": 34.199487967871185, + "hits_at_1": 0.0184, + "hits_at_10": 0.0516, + "hits_at_3": 0.0278, + "hits_at_5": 0.035, + "inverse_arithmetic_mean_rank": 7.406147778965533e-05, + "inverse_geometric_mean_rank": 0.0002895726647693664, + "inverse_harmonic_mean_rank": 0.029240203857421872, + "inverse_median_rank": 0.00010305560135748237, + "median_absolute_deviation": 13846.022118623816, + "median_rank": 9703.5, + "standard_deviation": 13040.0517578125, + "variance": 170042960.0, + "z_arithmetic_mean_rank": 41.66474449903673, + "z_geometric_mean_rank": 54.54769978100375, + "z_hits_at_k": 232.341991733733, + "z_inverse_harmonic_mean_rank": 323.38354115651606 + } + } + }, + "times": { + "evaluation": 3.979680061340332, + "training": 160.42055320739746 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json new file mode 100644 index 0000000..feb9a6f --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json new file mode 100644 index 0000000..d67452d --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 4096, + "slice_size": null + }, + "losses": [ + 0.00019061330150767425, + 4.929431331911384e-06, + 3.852578089436065e-06, + 3.58928681582103e-06, + 3.4748540160042394e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7533475301717399, + "adjusted_arithmetic_mean_rank_index": 0.24666463724982302, + "adjusted_geometric_mean_rank_index": 0.6438260194484903, + "adjusted_hits_at_k": 0.04046100837778324, + "adjusted_inverse_harmonic_mean_rank": 0.022035505980316127, + "arithmetic_mean_rank": 15272.273939808481, + "count": 5848.0, + "geometric_mean_rank": 5314.362994937839, + "harmonic_mean_rank": 44.83227908893811, + "hits_at_1": 0.013508891928864569, + "hits_at_10": 0.040697674418604654, + "hits_at_3": 0.022058823529411766, + "hits_at_5": 0.028214774281805747, + "inverse_arithmetic_mean_rank": 6.547813403172497e-05, + "inverse_geometric_mean_rank": 0.0001881693066417449, + "inverse_harmonic_mean_rank": 0.022305357218538983, + "inverse_median_rank": 7.543184732594101e-05, + "median_absolute_deviation": 16475.417153143502, + "median_rank": 13257.0, + "standard_deviation": 12660.061963371547, + "variance": 160277168.91640705, + "z_arithmetic_mean_rank": 32.67081918717673, + "z_geometric_mean_rank": 49.27749924271126, + "z_hits_at_k": 196.99271243043611, + "z_inverse_harmonic_mean_rank": 264.73240390539024 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7533477747862963, + "adjusted_arithmetic_mean_rank_index": 0.24666439262319972, + "adjusted_geometric_mean_rank_index": 0.6438258452392241, + "adjusted_hits_at_k": 0.04046100837778324, + "adjusted_inverse_harmonic_mean_rank": 0.022035505777574325, + "arithmetic_mean_rank": 15272.278898768809, + "count": 5848.0, + "geometric_mean_rank": 5314.365593771889, + "harmonic_mean_rank": 44.83227949632316, + "hits_at_1": 0.013508891928864569, + "hits_at_10": 0.040697674418604654, + "hits_at_3": 0.022058823529411766, + "hits_at_5": 0.028214774281805747, + "inverse_arithmetic_mean_rank": 6.54781127707546e-05, + "inverse_geometric_mean_rank": 0.0001881692146230848, + "inverse_harmonic_mean_rank": 0.022305357015853126, + "inverse_median_rank": 7.543184732594101e-05, + "median_absolute_deviation": 16475.417153143502, + "median_rank": 13257.0, + "standard_deviation": 12660.064268633454, + "variance": 160277227.28592953, + "z_arithmetic_mean_rank": 32.67078678629322, + "z_geometric_mean_rank": 49.27748590898953, + "z_hits_at_k": 196.99271243043611, + "z_inverse_harmonic_mean_rank": 264.7324014696702 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7533476499089838, + "adjusted_arithmetic_mean_rank_index": 0.2466645175066725, + "adjusted_geometric_mean_rank_index": 0.6438258365996382, + "adjusted_hits_at_k": 0.04046100837778324, + "adjusted_inverse_harmonic_mean_rank": 0.022035505581874164, + "arithmetic_mean_rank": 15272.2763671875, + "count": 5848.0, + "geometric_mean_rank": 5314.36572265625, + "harmonic_mean_rank": 44.8322798895589, + "hits_at_1": 0.013508891928864569, + "hits_at_10": 0.040697674418604654, + "hits_at_3": 0.022058823529411766, + "hits_at_5": 0.028214774281805747, + "inverse_arithmetic_mean_rank": 6.547812517965212e-05, + "inverse_geometric_mean_rank": 0.00018816921510733664, + "inverse_harmonic_mean_rank": 0.022305356820206963, + "inverse_median_rank": 7.543184619862586e-05, + "median_absolute_deviation": 16475.417153143502, + "median_rank": 13257.0, + "standard_deviation": 12660.0625, + "variance": 160277184.0, + "z_arithmetic_mean_rank": 32.67080332715375, + "z_geometric_mean_rank": 49.2774852477283, + "z_hits_at_k": 196.99271243043611, + "z_inverse_harmonic_mean_rank": 264.7323991185477 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8759271251000791, + "adjusted_arithmetic_mean_rank_index": 0.12407899680084766, + "adjusted_geometric_mean_rank_index": 0.40379021336988474, + "adjusted_hits_at_k": 0.026093526032368673, + "adjusted_inverse_harmonic_mean_rank": 0.015624904836206868, + "arithmetic_mean_rank": 17753.335157318743, + "count": 2924.0, + "geometric_mean_rank": 8893.970565775853, + "harmonic_mean_rank": 62.906612149750195, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.026333789329685362, + "hits_at_3": 0.01744186046511628, + "hits_at_5": 0.02051983584131327, + "inverse_arithmetic_mean_rank": 5.6327444456978774e-05, + "inverse_geometric_mean_rank": 0.00011243572177403158, + "inverse_harmonic_mean_rank": 0.015896580118151715, + "inverse_median_rank": 5.901619994688542e-05, + "median_absolute_deviation": 16516.93001526166, + "median_rank": 16944.5, + "standard_deviation": 12453.794421336042, + "variance": 155096995.48890075, + "z_arithmetic_mean_rank": 11.62080349509435, + "z_geometric_mean_rank": 21.856266425692148, + "z_hits_at_k": 89.82201361985973, + "z_inverse_harmonic_mean_rank": 132.72052156271272 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8759273444583465, + "adjusted_arithmetic_mean_rank_index": 0.12407877743175677, + "adjusted_geometric_mean_rank_index": 0.40379006269602624, + "adjusted_hits_at_k": 0.026093526032368673, + "adjusted_inverse_harmonic_mean_rank": 0.015624904816403959, + "arithmetic_mean_rank": 17753.339603283173, + "count": 2924.0, + "geometric_mean_rank": 8893.972813203216, + "harmonic_mean_rank": 62.906612228093465, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.026333789329685362, + "hits_at_3": 0.01744186046511628, + "hits_at_5": 0.02051983584131327, + "inverse_arithmetic_mean_rank": 5.6327430350910836e-05, + "inverse_geometric_mean_rank": 0.00011243569336253055, + "inverse_harmonic_mean_rank": 0.01589658009835427, + "inverse_median_rank": 5.901619994688542e-05, + "median_absolute_deviation": 16516.93001526166, + "median_rank": 16944.5, + "standard_deviation": 12453.796314877382, + "variance": 155097042.65245345, + "z_arithmetic_mean_rank": 11.620782949754982, + "z_geometric_mean_rank": 21.856258270051214, + "z_hits_at_k": 89.82201361985973, + "z_inverse_harmonic_mean_rank": 132.72052139450352 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8759272599579563, + "adjusted_arithmetic_mean_rank_index": 0.12407886193631645, + "adjusted_geometric_mean_rank_index": 0.40379000774711216, + "adjusted_hits_at_k": 0.026093526032368673, + "adjusted_inverse_harmonic_mean_rank": 0.015624902722154796, + "arithmetic_mean_rank": 17753.337890625, + "count": 2924.0, + "geometric_mean_rank": 8893.9736328125, + "harmonic_mean_rank": 62.90662051325779, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.026333789329685362, + "hits_at_3": 0.01744186046511628, + "hits_at_5": 0.02051983584131327, + "inverse_arithmetic_mean_rank": 5.632743705064058e-05, + "inverse_geometric_mean_rank": 0.00011243568587815389, + "inverse_harmonic_mean_rank": 0.015896578004683092, + "inverse_median_rank": 5.901620170334354e-05, + "median_absolute_deviation": 16516.93001526166, + "median_rank": 16944.5, + "standard_deviation": 12453.7958984375, + "variance": 155097024.0, + "z_arithmetic_mean_rank": 11.620790864155552, + "z_geometric_mean_rank": 21.856255295788664, + "z_hits_at_k": 89.82201361985973, + "z_inverse_harmonic_mean_rank": 132.72050360560533 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6308223124201837, + "adjusted_arithmetic_mean_rank_index": 0.36919589515820417, + "adjusted_geometric_mean_rank_index": 0.7872698764693477, + "adjusted_hits_at_k": 0.054828489141919325, + "adjusted_inverse_harmonic_mean_rank": 0.0284461064058264, + "arithmetic_mean_rank": 12791.212722298222, + "count": 2924.0, + "geometric_mean_rank": 3175.460704878214, + "harmonic_mean_rank": 34.826054266273786, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.05506155950752394, + "hits_at_3": 0.02667578659370725, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 7.817867013162518e-05, + "inverse_geometric_mean_rank": 0.00031491493453651546, + "inverse_harmonic_mean_rank": 0.02871413431892625, + "inverse_median_rank": 0.00010870155986738409, + "median_absolute_deviation": 13220.363982414452, + "median_rank": 9199.5, + "standard_deviation": 12375.217687533675, + "variance": 153146012.81384635, + "z_arithmetic_mean_rank": 34.57763373270559, + "z_geometric_mean_rank": 42.6131565673655, + "z_hits_at_k": 188.77878751881445, + "z_inverse_harmonic_mean_rank": 241.679769823527 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6308225822798252, + "adjusted_arithmetic_mean_rank_index": 0.3691956252852534, + "adjusted_geometric_mean_rank_index": 0.7872697221168297, + "adjusted_hits_at_k": 0.054828489141919325, + "adjusted_inverse_harmonic_mean_rank": 0.028446106020145728, + "arithmetic_mean_rank": 12791.218194254447, + "count": 2924.0, + "geometric_mean_rank": 3175.463008200353, + "harmonic_mean_rank": 34.82605473391911, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.05506155950752394, + "hits_at_3": 0.02667578659370725, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 7.817863668756581e-05, + "inverse_geometric_mean_rank": 0.0003149147061129631, + "inverse_harmonic_mean_rank": 0.02871413393335198, + "inverse_median_rank": 0.00010870155986738409, + "median_absolute_deviation": 13220.363982414452, + "median_rank": 9199.5, + "standard_deviation": 12375.220704312107, + "variance": 153146087.48043504, + "z_arithmetic_mean_rank": 34.57760845732153, + "z_geometric_mean_rank": 42.613148212609126, + "z_hits_at_k": 188.77878751881445, + "z_inverse_harmonic_mean_rank": 241.67976654676164 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6308224652044876, + "adjusted_arithmetic_mean_rank_index": 0.3691957423663652, + "adjusted_geometric_mean_rank_index": 0.7872697463565012, + "adjusted_hits_at_k": 0.054828489141919325, + "adjusted_inverse_harmonic_mean_rank": 0.02844610641801832, + "arithmetic_mean_rank": 12791.2158203125, + "count": 2924.0, + "geometric_mean_rank": 3175.462646484375, + "harmonic_mean_rank": 34.826054251490845, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.05506155950752394, + "hits_at_3": 0.02667578659370725, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 7.817865116521716e-05, + "inverse_geometric_mean_rank": 0.0003149147378280759, + "inverse_harmonic_mean_rank": 0.028714134331114805, + "inverse_median_rank": 0.00010870156256714836, + "median_absolute_deviation": 13220.363982414452, + "median_rank": 9199.5, + "standard_deviation": 12375.21875, + "variance": 153146048.0, + "z_arithmetic_mean_rank": 34.57761942274084, + "z_geometric_mean_rank": 42.6131495246483, + "z_hits_at_k": 188.77878751881445, + "z_inverse_harmonic_mean_rank": 241.67976992711024 + } + } + }, + "times": { + "evaluation": 2.390557289123535, + "training": 96.70088291168213 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..68117f3 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..b66b703 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 32, + "slice_size": null + }, + "losses": [ + 0.0027138416286498156, + 0.002708068411666065, + 0.00270294341827581, + 0.0026974449028830463, + 0.0026920965323165968 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9022145800402283, + "adjusted_arithmetic_mean_rank_index": 0.09779912563002002, + "adjusted_geometric_mean_rank_index": 0.3132169673715255, + "adjusted_hits_at_k": 0.025222428209433437, + "adjusted_inverse_harmonic_mean_rank": 0.017172625662514082, + "arithmetic_mean_rank": 6437.904565025932, + "count": 40876.0, + "geometric_mean_rank": 3602.1284793223763, + "harmonic_mean_rank": 55.950214649740836, + "hits_at_1": 0.012550151678246404, + "hits_at_10": 0.025907623055093452, + "hits_at_3": 0.02030531363147079, + "hits_at_5": 0.022971915060182015, + "inverse_arithmetic_mean_rank": 0.0001553300441004552, + "inverse_geometric_mean_rank": 0.0002776136403074989, + "inverse_harmonic_mean_rank": 0.017873032413909282, + "inverse_median_rank": 0.00016079755587715066, + "median_absolute_deviation": 5565.68872827003, + "median_rank": 6219.0, + "standard_deviation": 4305.144839101484, + "variance": 18534272.085642144, + "z_arithmetic_mean_rank": 34.203630631700776, + "z_geometric_mean_rank": 63.43454801929772, + "z_hits_at_k": 192.27141470503148, + "z_inverse_harmonic_mean_rank": 323.370932509128 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9022146588944141, + "adjusted_arithmetic_mean_rank_index": 0.09779904676478202, + "adjusted_geometric_mean_rank_index": 0.3132168544257061, + "adjusted_hits_at_k": 0.025222428209433437, + "adjusted_inverse_harmonic_mean_rank": 0.01717262556907924, + "arithmetic_mean_rank": 6437.905127703298, + "count": 40876.0, + "geometric_mean_rank": 3602.129071550762, + "harmonic_mean_rank": 55.950214942023315, + "hits_at_1": 0.012550151678246404, + "hits_at_10": 0.025907623055093452, + "hits_at_3": 0.02030531363147079, + "hits_at_5": 0.022971915060182015, + "inverse_arithmetic_mean_rank": 0.0001553300305245018, + "inverse_geometric_mean_rank": 0.0002776135946648595, + "inverse_harmonic_mean_rank": 0.017873032320541025, + "inverse_median_rank": 0.00016079755587715066, + "median_absolute_deviation": 5565.68872827003, + "median_rank": 6219.0, + "standard_deviation": 4305.144722351568, + "variance": 18534271.080391556, + "z_arithmetic_mean_rank": 34.20360304988492, + "z_geometric_mean_rank": 63.43452514484409, + "z_hits_at_k": 192.27141470503148, + "z_inverse_harmonic_mean_rank": 323.3709307496934 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9022145424612803, + "adjusted_arithmetic_mean_rank_index": 0.0977991632142351, + "adjusted_geometric_mean_rank_index": 0.3132167462681771, + "adjusted_hits_at_k": 0.025222428209433437, + "adjusted_inverse_harmonic_mean_rank": 0.01717262544251896, + "arithmetic_mean_rank": 6437.904296875, + "count": 40876.0, + "geometric_mean_rank": 3602.129638671875, + "harmonic_mean_rank": 55.950215337928626, + "hits_at_1": 0.012550151678246404, + "hits_at_10": 0.025907623055093452, + "hits_at_3": 0.02030531363147079, + "hits_at_5": 0.022971915060182015, + "inverse_arithmetic_mean_rank": 0.00015533005353063345, + "inverse_geometric_mean_rank": 0.000277613551588729, + "inverse_harmonic_mean_rank": 0.01787303219407094, + "inverse_median_rank": 0.00016079755732789636, + "median_absolute_deviation": 5565.68872827003, + "median_rank": 6219.0, + "standard_deviation": 4305.14501953125, + "variance": 18534272.0, + "z_arithmetic_mean_rank": 34.20364377615988, + "z_geometric_mean_rank": 63.43450324014324, + "z_hits_at_k": 192.27141470503148, + "z_inverse_harmonic_mean_rank": 323.3709283664868 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9861542601103468, + "adjusted_arithmetic_mean_rank_index": 0.013847707833329004, + "adjusted_geometric_mean_rank_index": 0.023761321767014065, + "adjusted_hits_at_k": 0.00011720820855638033, + "adjusted_inverse_harmonic_mean_rank": 0.00011582849847661622, + "arithmetic_mean_rank": 6939.210832762501, + "count": 20438.0, + "geometric_mean_rank": 5042.814960457132, + "harmonic_mean_rank": 1191.8958402433716, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0008317839318915745, + "hits_at_3": 0.0002446423329092866, + "hits_at_5": 0.0004403561992367159, + "inverse_arithmetic_mean_rank": 0.00014410860602168788, + "inverse_geometric_mean_rank": 0.00019830194203860891, + "inverse_harmonic_mean_rank": 0.0008389994882403578, + "inverse_median_rank": 0.00014624159110851126, + "median_absolute_deviation": 5238.774939089544, + "median_rank": 6838.0, + "standard_deviation": 4115.020282699011, + "variance": 16933391.92702425, + "z_arithmetic_mean_rank": 3.4208852603670437, + "z_geometric_mean_rank": 3.4029652710831897, + "z_hits_at_k": 0.6265756294420397, + "z_inverse_harmonic_mean_rank": 1.529607327750341 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.986154364411146, + "adjusted_arithmetic_mean_rank_index": 0.013847603517705087, + "adjusted_geometric_mean_rank_index": 0.023761114601180355, + "adjusted_hits_at_k": 0.00011720820855638033, + "adjusted_inverse_harmonic_mean_rank": 0.00011582837630638547, + "arithmetic_mean_rank": 6939.2115666895, + "count": 20438.0, + "geometric_mean_rank": 5042.816030371511, + "harmonic_mean_rank": 1191.896013674818, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0008317839318915745, + "hits_at_3": 0.0002446423329092866, + "hits_at_5": 0.0004403561992367159, + "inverse_arithmetic_mean_rank": 0.00014410859078001445, + "inverse_geometric_mean_rank": 0.00019830189996566832, + "inverse_harmonic_mean_rank": 0.0008389993661584872, + "inverse_median_rank": 0.00014624159110851126, + "median_absolute_deviation": 5238.774939089544, + "median_rank": 6838.0, + "standard_deviation": 4115.0200287993475, + "variance": 16933389.837419786, + "z_arithmetic_mean_rank": 3.4208594906306473, + "z_geometric_mean_rank": 3.4029356019366617, + "z_hits_at_k": 0.6265756294420397, + "z_inverse_harmonic_mean_rank": 1.5296057143953998 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9861543443862074, + "adjusted_arithmetic_mean_rank_index": 0.013847623545489851, + "adjusted_geometric_mean_rank_index": 0.023761420000934508, + "adjusted_hits_at_k": 0.00011720820855638033, + "adjusted_inverse_harmonic_mean_rank": 0.00011582844626088513, + "arithmetic_mean_rank": 6939.21142578125, + "count": 20438.0, + "geometric_mean_rank": 5042.814453125, + "harmonic_mean_rank": 1191.8959143682134, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0008317839318915745, + "hits_at_3": 0.0002446423329092866, + "hits_at_5": 0.0004403561992367159, + "inverse_arithmetic_mean_rank": 0.00014410859148483723, + "inverse_geometric_mean_rank": 0.00019830196106340736, + "inverse_harmonic_mean_rank": 0.0008389994360623919, + "inverse_median_rank": 0.00014624159666709602, + "median_absolute_deviation": 5238.774939089544, + "median_rank": 6838.0, + "standard_deviation": 4115.02001953125, + "variance": 16933390.0, + "z_arithmetic_mean_rank": 3.4208644382187003, + "z_geometric_mean_rank": 3.4029793396027515, + "z_hits_at_k": 0.6265756294420397, + "z_inverse_harmonic_mean_rank": 1.5296066382001356 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8205728810454175, + "adjusted_arithmetic_mean_rank_index": 0.17945192329106108, + "adjusted_geometric_mean_rank_index": 0.5168758436756876, + "adjusted_hits_at_k": 0.050327058574015544, + "adjusted_inverse_harmonic_mean_rank": 0.034229060625992506, + "arithmetic_mean_rank": 5936.598297289363, + "count": 20438.0, + "geometric_mean_rank": 2573.033054611455, + "harmonic_mean_rank": 28.647495579245472, + "hits_at_1": 0.024953517956747235, + "hits_at_10": 0.05098346217829533, + "hits_at_3": 0.040365984930032295, + "hits_at_5": 0.04550347392112731, + "inverse_arithmetic_mean_rank": 0.00016844663390086503, + "inverse_geometric_mean_rank": 0.0003886463868809515, + "inverse_harmonic_mean_rank": 0.03490706533957822, + "inverse_median_rank": 0.00017921146953405018, + "median_absolute_deviation": 5862.20917197115, + "median_rank": 5580.0, + "standard_deviation": 4430.861800611265, + "variance": 19632536.296116095, + "z_arithmetic_mean_rank": 44.43135166020866, + "z_geometric_mean_rank": 74.02083142469466, + "z_hits_at_k": 273.57232645173275, + "z_inverse_harmonic_mean_rank": 459.60952979376 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8205729351496311, + "adjusted_arithmetic_mean_rank_index": 0.17945186917936795, + "adjusted_geometric_mean_rank_index": 0.5168757872948017, + "adjusted_hits_at_k": 0.050327058574015544, + "adjusted_inverse_harmonic_mean_rank": 0.03422906056129243, + "arithmetic_mean_rank": 5936.598688717096, + "count": 20438.0, + "geometric_mean_rank": 2573.0333547692935, + "harmonic_mean_rank": 28.647495632306196, + "hits_at_1": 0.024953517956747235, + "hits_at_10": 0.05098346217829533, + "hits_at_3": 0.040365984930032295, + "hits_at_5": 0.04550347392112731, + "inverse_arithmetic_mean_rank": 0.0001684466227943902, + "inverse_geometric_mean_rank": 0.00038864634154331173, + "inverse_harmonic_mean_rank": 0.034907065274923564, + "inverse_median_rank": 0.00017921146953405018, + "median_absolute_deviation": 5862.20917197115, + "median_rank": 5580.0, + "standard_deviation": 4430.861770787402, + "variance": 19632536.031825267, + "z_arithmetic_mean_rank": 44.43133826243826, + "z_geometric_mean_rank": 74.02082335049249, + "z_hits_at_k": 273.57232645173275, + "z_inverse_harmonic_mean_rank": 459.60952892500194 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8205729274223448, + "adjusted_arithmetic_mean_rank_index": 0.17945187690772257, + "adjusted_geometric_mean_rank_index": 0.5168757240617707, + "adjusted_hits_at_k": 0.050327058574015544, + "adjusted_inverse_harmonic_mean_rank": 0.034229060705167576, + "arithmetic_mean_rank": 5936.5986328125, + "count": 20438.0, + "geometric_mean_rank": 2573.03369140625, + "harmonic_mean_rank": 28.64749551431377, + "hits_at_1": 0.024953517956747235, + "hits_at_10": 0.05098346217829533, + "hits_at_3": 0.040365984930032295, + "hits_at_5": 0.04550347392112731, + "inverse_arithmetic_mean_rank": 0.0001684466260485351, + "inverse_geometric_mean_rank": 0.0003886462945956737, + "inverse_harmonic_mean_rank": 0.03490706541869771, + "inverse_median_rank": 0.0001792114635463804, + "median_absolute_deviation": 5862.20917197115, + "median_rank": 5580.0, + "standard_deviation": 4430.86181640625, + "variance": 19632536.0, + "z_arithmetic_mean_rank": 44.431340175938175, + "z_geometric_mean_rank": 74.02081429500736, + "z_hits_at_k": 273.57232645173275, + "z_inverse_harmonic_mean_rank": 459.60953085688055 + } + } + }, + "times": { + "evaluation": 113.86807107925415, + "training": 28.116236209869385 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json new file mode 100644 index 0000000..eab649f --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json new file mode 100644 index 0000000..840c2db --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 4, + "slice_size": null + }, + "losses": [ + 0.002721442517725209, + 0.002717859683797624, + 0.0027141772589411987, + 0.0027092453467933092, + 0.00270177292591784 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9368993007191023, + "adjusted_arithmetic_mean_rank_index": 0.06310381205252158, + "adjusted_geometric_mean_rank_index": 0.17085137097247383, + "adjusted_hits_at_k": 0.005397640591231649, + "adjusted_inverse_harmonic_mean_rank": 0.003779893635427336, + "arithmetic_mean_rank": 18993.336012311902, + "count": 5848.0, + "geometric_mean_rank": 12370.145090151156, + "harmonic_mean_rank": 246.62237264877194, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.005642954856361149, + "hits_at_3": 0.00393296853625171, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 5.265004522384997e-05, + "inverse_geometric_mean_rank": 8.083979554905775e-05, + "inverse_harmonic_mean_rank": 0.004054782172678848, + "inverse_median_rank": 5.4449919686368463e-05, + "median_absolute_deviation": 15544.342959921983, + "median_rank": 18365.5, + "standard_deviation": 12003.289451468483, + "variance": 144078957.65573457, + "z_arithmetic_mean_rank": 8.358122415015941, + "z_geometric_mean_rank": 13.07671335017524, + "z_hits_at_k": 26.279519552834255, + "z_inverse_harmonic_mean_rank": 45.411270769422465 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9368993260240565, + "adjusted_arithmetic_mean_rank_index": 0.06310378674631922, + "adjusted_geometric_mean_rank_index": 0.17085134895810905, + "adjusted_hits_at_k": 0.005397640591231649, + "adjusted_inverse_harmonic_mean_rank": 0.0037798936340233966, + "arithmetic_mean_rank": 18993.336525307797, + "count": 5848.0, + "geometric_mean_rank": 12370.14541855895, + "harmonic_mean_rank": 246.62237273413962, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.005642954856361149, + "hits_at_3": 0.00393296853625171, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 5.2650043801811407e-05, + "inverse_geometric_mean_rank": 8.083979340288905e-05, + "inverse_harmonic_mean_rank": 0.004054782171275296, + "inverse_median_rank": 5.4449919686368463e-05, + "median_absolute_deviation": 15544.342959921983, + "median_rank": 18365.5, + "standard_deviation": 12003.289482275293, + "variance": 144078958.39530066, + "z_arithmetic_mean_rank": 8.358119063200405, + "z_geometric_mean_rank": 13.076711665228046, + "z_hits_at_k": 26.279519552834255, + "z_inverse_harmonic_mean_rank": 45.41127075255568 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9368992970287965, + "adjusted_arithmetic_mean_rank_index": 0.0631038157430095, + "adjusted_geometric_mean_rank_index": 0.170851277512596, + "adjusted_hits_at_k": 0.005397640591231649, + "adjusted_inverse_harmonic_mean_rank": 0.0037798933020507103, + "arithmetic_mean_rank": 18993.3359375, + "count": 5848.0, + "geometric_mean_rank": 12370.146484375, + "harmonic_mean_rank": 246.62239292001001, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.005642954856361149, + "hits_at_3": 0.00393296853625171, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 5.265004438115284e-05, + "inverse_geometric_mean_rank": 8.083978900685906e-05, + "inverse_harmonic_mean_rank": 0.004054781839394211, + "inverse_median_rank": 5.4449919844046235e-05, + "median_absolute_deviation": 15544.342959921983, + "median_rank": 18365.5, + "standard_deviation": 12003.2900390625, + "variance": 144078976.0, + "z_arithmetic_mean_rank": 8.358122903822377, + "z_geometric_mean_rank": 13.07670619689327, + "z_hits_at_k": 26.279519552834255, + "z_inverse_harmonic_mean_rank": 45.41126676426843 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9748546833736785, + "adjusted_arithmetic_mean_rank_index": 0.025146557325686825, + "adjusted_geometric_mean_rank_index": 0.08085733565234698, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.004117111631119762, + "arithmetic_mean_rank": 19758.40389876881, + "count": 2924.0, + "geometric_mean_rank": 13710.786124097143, + "harmonic_mean_rank": 227.68862534181577, + "hits_at_1": 0.0030779753761969904, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.005129958960328317, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.0611375550547997e-05, + "inverse_geometric_mean_rank": 7.293527817799361e-05, + "inverse_harmonic_mean_rank": 0.004391962920847529, + "inverse_median_rank": 5.133074968559916e-05, + "median_absolute_deviation": 15015.795269024737, + "median_rank": 19481.5, + "standard_deviation": 11717.673398597328, + "variance": 137303869.87619546, + "z_arithmetic_mean_rank": 2.355138329567282, + "z_geometric_mean_rank": 4.376627793280527, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 34.97142598577112 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9748547339948171, + "adjusted_arithmetic_mean_rank_index": 0.02514650670205054, + "adjusted_geometric_mean_rank_index": 0.08085728684521754, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.004117111628311883, + "arithmetic_mean_rank": 19758.4049247606, + "count": 2924.0, + "geometric_mean_rank": 13710.786852096542, + "harmonic_mean_rank": 227.688625487342, + "hits_at_1": 0.0030779753761969904, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.005129958960328317, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.061137292245853e-05, + "inverse_geometric_mean_rank": 7.293527430536112e-05, + "inverse_harmonic_mean_rank": 0.004391962918040425, + "inverse_median_rank": 5.133074968559916e-05, + "median_absolute_deviation": 15015.795269024737, + "median_rank": 19481.5, + "standard_deviation": 11717.6733947239, + "variance": 137303869.78542036, + "z_arithmetic_mean_rank": 2.3551335883351356, + "z_geometric_mean_rank": 4.37662515145913, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 34.97142596192053 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.974854703015735, + "adjusted_arithmetic_mean_rank_index": 0.025146537682661285, + "adjusted_geometric_mean_rank_index": 0.0808574005395446, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.004117111497430378, + "arithmetic_mean_rank": 19758.404296875, + "count": 2924.0, + "geometric_mean_rank": 13710.78515625, + "harmonic_mean_rank": 227.68863227064298, + "hits_at_1": 0.0030779753761969904, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.005129958960328317, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.061137562734075e-05, + "inverse_geometric_mean_rank": 7.293528324225917e-05, + "inverse_harmonic_mean_rank": 0.004391962787195041, + "inverse_median_rank": 5.1330749556655064e-05, + "median_absolute_deviation": 15015.795269024737, + "median_rank": 19481.5, + "standard_deviation": 11717.673828125, + "variance": 137303888.0, + "z_arithmetic_mean_rank": 2.355136489870432, + "z_geometric_mean_rank": 4.3766313054802595, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 34.97142485019141 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8989607553411842, + "adjusted_arithmetic_mean_rank_index": 0.10104422784179035, + "adjusted_geometric_mean_rank_index": 0.2521623042350343, + "adjusted_hits_at_k": 0.005226654514911831, + "adjusted_inverse_harmonic_mean_rank": 0.0034426756775355026, + "arithmetic_mean_rank": 18228.268125854993, + "count": 2924.0, + "geometric_mean_rank": 11160.591972363447, + "harmonic_mean_rank": 268.99064364646364, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.005471956224350205, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.003761969904240766, + "inverse_arithmetic_mean_rank": 5.485984697479839e-05, + "inverse_geometric_mean_rank": 8.960098196191226e-05, + "inverse_harmonic_mean_rank": 0.003717601424510167, + "inverse_median_rank": 5.9044076403034865e-05, + "median_absolute_deviation": 15788.231024866154, + "median_rank": 16936.5, + "standard_deviation": 12234.51624272486, + "variance": 149683387.6934984, + "z_arithmetic_mean_rank": 9.463459228386938, + "z_geometric_mean_rank": 13.648981209524969, + "z_hits_at_k": 17.99578134554909, + "z_inverse_harmonic_mean_rank": 29.24917222250908 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8989607553411842, + "adjusted_arithmetic_mean_rank_index": 0.10104422784179035, + "adjusted_geometric_mean_rank_index": 0.2521623042350343, + "adjusted_hits_at_k": 0.005226654514911831, + "adjusted_inverse_harmonic_mean_rank": 0.0034426756775355026, + "arithmetic_mean_rank": 18228.268125854993, + "count": 2924.0, + "geometric_mean_rank": 11160.591972363447, + "harmonic_mean_rank": 268.99064364646364, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.005471956224350205, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.003761969904240766, + "inverse_arithmetic_mean_rank": 5.485984697479839e-05, + "inverse_geometric_mean_rank": 8.960098196191226e-05, + "inverse_harmonic_mean_rank": 0.003717601424510167, + "inverse_median_rank": 5.9044076403034865e-05, + "median_absolute_deviation": 15788.231024866154, + "median_rank": 16936.5, + "standard_deviation": 12234.51624272486, + "variance": 149683387.6934984, + "z_arithmetic_mean_rank": 9.463459228386938, + "z_geometric_mean_rank": 13.648981209524969, + "z_hits_at_k": 17.99578134554909, + "z_inverse_harmonic_mean_rank": 29.24917222250908 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8989607283288666, + "adjusted_arithmetic_mean_rank_index": 0.10104425485544011, + "adjusted_geometric_mean_rank_index": 0.2521627740916954, + "adjusted_hits_at_k": 0.005226654514911831, + "adjusted_inverse_harmonic_mean_rank": 0.0034426757969596754, + "arithmetic_mean_rank": 18228.267578125, + "count": 2924.0, + "geometric_mean_rank": 11160.5849609375, + "harmonic_mean_rank": 268.99063500779636, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.005471956224350205, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.003761969904240766, + "inverse_arithmetic_mean_rank": 5.485984729602933e-05, + "inverse_geometric_mean_rank": 8.960103878052905e-05, + "inverse_harmonic_mean_rank": 0.0037176015439013936, + "inverse_median_rank": 5.904407589696348e-05, + "median_absolute_deviation": 15788.231024866154, + "median_rank": 16936.5, + "standard_deviation": 12234.5166015625, + "variance": 149683392.0, + "z_arithmetic_mean_rank": 9.463461758393635, + "z_geometric_mean_rank": 13.649006641814537, + "z_hits_at_k": 17.99578134554909, + "z_inverse_harmonic_mean_rank": 29.249173237143776 + } + } + }, + "times": { + "evaluation": 229.65697264671326, + "training": 11.373437881469727 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json new file mode 100644 index 0000000..bd530c9 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult creates two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json new file mode 100644 index 0000000..fa61601 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 1.9813735472315636e-05, + 1.0421016274152843e-05, + 6.530047764306711e-06, + 4.6684656685330165e-06, + 3.782523409532725e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04563974028451357, + "adjusted_arithmetic_mean_rank_index": 0.9544894743555506, + "adjusted_geometric_mean_rank_index": 0.9927695844264502, + "adjusted_hits_at_k": 0.3147999080140758, + "adjusted_inverse_harmonic_mean_rank": 0.18212225677789062, + "arithmetic_mean_rank": 337.134025156168, + "count": 118142.0, + "geometric_mean_rank": 40.26556227491919, + "harmonic_mean_rank": 5.473851925109878, + "hits_at_1": 0.11285571600277632, + "hits_at_10": 0.31526468148499265, + "hits_at_3": 0.19793130300824432, + "hits_at_5": 0.24297032384757325, + "inverse_arithmetic_mean_rank": 0.0029661793986435446, + "inverse_geometric_mean_rank": 0.02483511823757357, + "inverse_harmonic_mean_rank": 0.18268671014149268, + "inverse_median_rank": 0.023809523809523808, + "median_absolute_deviation": 60.78669095872968, + "median_rank": 42.0, + "standard_deviation": 889.735761711197, + "variance": 791629.7256678039, + "z_arithmetic_mean_rank": 567.7468447904905, + "z_geometric_mean_rank": 341.824729522192, + "z_hits_at_k": 4153.1520985060315, + "z_inverse_harmonic_mean_rank": 5934.9765930662215 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04563978153592007, + "adjusted_arithmetic_mean_rank_index": 0.9544894330985589, + "adjusted_geometric_mean_rank_index": 0.9927695715296923, + "adjusted_hits_at_k": 0.3147999080140758, + "adjusted_inverse_harmonic_mean_rank": 0.18212213620472156, + "arithmetic_mean_rank": 337.13432987421913, + "count": 118142.0, + "geometric_mean_rank": 40.265632312173224, + "harmonic_mean_rank": 5.473855535359432, + "hits_at_1": 0.11285571600277632, + "hits_at_10": 0.31526468148499265, + "hits_at_3": 0.19793130300824432, + "hits_at_5": 0.24297032384757325, + "inverse_arithmetic_mean_rank": 0.0029661767176694474, + "inverse_geometric_mean_rank": 0.0248350750398542, + "inverse_harmonic_mean_rank": 0.18268658965153647, + "inverse_median_rank": 0.023809523809523808, + "median_absolute_deviation": 60.78669095872968, + "median_rank": 42.0, + "standard_deviation": 889.7358922933918, + "variance": 791629.958035118, + "z_arithmetic_mean_rank": 567.7468202501184, + "z_geometric_mean_rank": 341.82472508165426, + "z_hits_at_k": 4153.1520985060315, + "z_inverse_harmonic_mean_rank": 5934.972663843354 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.045639757899224476, + "adjusted_arithmetic_mean_rank_index": 0.9544894567384548, + "adjusted_geometric_mean_rank_index": 0.9927695770908282, + "adjusted_hits_at_k": 0.3147999080140758, + "adjusted_inverse_harmonic_mean_rank": 0.18212220161411308, + "arithmetic_mean_rank": 337.1341552734375, + "count": 118142.0, + "geometric_mean_rank": 40.265602111816406, + "harmonic_mean_rank": 5.47385357684495, + "hits_at_1": 0.11285571600277632, + "hits_at_10": 0.31526468148499265, + "hits_at_3": 0.19793130300824432, + "hits_at_5": 0.24297032384757325, + "inverse_arithmetic_mean_rank": 0.00296617834828794, + "inverse_geometric_mean_rank": 0.024835092946887016, + "inverse_harmonic_mean_rank": 0.18268665501578607, + "inverse_median_rank": 0.02380952425301075, + "median_absolute_deviation": 60.78669095872968, + "median_rank": 42.0, + "standard_deviation": 889.73583984375, + "variance": 791629.875, + "z_arithmetic_mean_rank": 567.7468343115373, + "z_geometric_mean_rank": 341.82472699643273, + "z_hits_at_k": 4153.1520985060315, + "z_inverse_harmonic_mean_rank": 5934.9747953961605 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0492053071915437, + "adjusted_arithmetic_mean_rank_index": 0.950923776041295, + "adjusted_geometric_mean_rank_index": 0.9914887878798211, + "adjusted_hits_at_k": 0.28574585583230167, + "adjusted_inverse_harmonic_mean_rank": 0.1620356021690724, + "arithmetic_mean_rank": 362.48314739889287, + "count": 59071.0, + "geometric_mean_rank": 47.08810772419198, + "harmonic_mean_rank": 6.149475038945622, + "hits_at_1": 0.09717120075840938, + "hits_at_10": 0.28623182272180936, + "hits_at_3": 0.1740786511147602, + "hits_at_5": 0.21628210120025054, + "inverse_arithmetic_mean_rank": 0.0027587489437117326, + "inverse_geometric_mean_rank": 0.021236784579607138, + "inverse_harmonic_mean_rank": 0.16261550679803366, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 902.9855077973994, + "variance": 815382.8272921271, + "z_arithmetic_mean_rank": 399.89996701532084, + "z_geometric_mean_rank": 241.37804208791823, + "z_hits_at_k": 2661.6000279583345, + "z_inverse_harmonic_mean_rank": 3728.105578345422 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.049205341661526236, + "adjusted_arithmetic_mean_rank_index": 0.9509237415666327, + "adjusted_geometric_mean_rank_index": 0.9914887809523989, + "adjusted_hits_at_k": 0.28574585583230167, + "adjusted_inverse_harmonic_mean_rank": 0.16203559831201408, + "arithmetic_mean_rank": 362.48340133060213, + "count": 59071.0, + "geometric_mean_rank": 47.088145236097084, + "harmonic_mean_rank": 6.149475184703368, + "hits_at_1": 0.09717120075840938, + "hits_at_10": 0.28623182272180936, + "hits_at_3": 0.1740786511147602, + "hits_at_5": 0.21628210120025054, + "inverse_arithmetic_mean_rank": 0.00275874701111611, + "inverse_geometric_mean_rank": 0.021236767661713177, + "inverse_harmonic_mean_rank": 0.1626155029436446, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 902.9855805051617, + "variance": 815382.9586002439, + "z_arithmetic_mean_rank": 399.89995251740135, + "z_geometric_mean_rank": 241.37804040143664, + "z_hits_at_k": 2661.6000279583345, + "z_inverse_harmonic_mean_rank": 3728.1054896024525 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04920532469835637, + "adjusted_arithmetic_mean_rank_index": 0.9509237585321055, + "adjusted_geometric_mean_rank_index": 0.9914887835904787, + "adjusted_hits_at_k": 0.28574585583230167, + "adjusted_inverse_harmonic_mean_rank": 0.16203561446886694, + "arithmetic_mean_rank": 362.4832763671875, + "count": 59071.0, + "geometric_mean_rank": 47.088130950927734, + "harmonic_mean_rank": 6.149474574137982, + "hits_at_1": 0.09717120075840938, + "hits_at_10": 0.28623182272180936, + "hits_at_3": 0.1740786511147602, + "hits_at_5": 0.21628210120025054, + "inverse_arithmetic_mean_rank": 0.002758747898042202, + "inverse_geometric_mean_rank": 0.02123677358031273, + "inverse_harmonic_mean_rank": 0.16261551908931626, + "inverse_median_rank": 0.019999999552965164, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 902.9854736328125, + "variance": 815382.8125, + "z_arithmetic_mean_rank": 399.8999596520342, + "z_geometric_mean_rank": 241.3780410436774, + "z_hits_at_k": 2661.6000279583345, + "z_inverse_harmonic_mean_rank": 3728.10586133837 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0420935281672918, + "adjusted_arithmetic_mean_rank_index": 0.9580358145531809, + "adjusted_geometric_mean_rank_index": 0.9938616322089793, + "adjusted_hits_at_k": 0.3438538392238848, + "adjusted_inverse_harmonic_mean_rank": 0.2022088351751535, + "arithmetic_mean_rank": 311.78490291344315, + "count": 59071.0, + "geometric_mean_rank": 34.43152812195992, + "harmonic_mean_rank": 4.931989991474332, + "hits_at_1": 0.12854023124714326, + "hits_at_10": 0.34429754024817594, + "hits_at_3": 0.22178395490172842, + "hits_at_5": 0.26965854649489596, + "inverse_arithmetic_mean_rank": 0.003207339388968481, + "inverse_geometric_mean_rank": 0.029043148955163994, + "inverse_harmonic_mean_rank": 0.20275791348495165, + "inverse_median_rank": 0.029411764705882353, + "median_absolute_deviation": 48.92587321068486, + "median_rank": 34.0, + "standard_deviation": 875.5520932797359, + "variance": 766591.4680465274, + "z_arithmetic_mean_rank": 403.00997549321005, + "z_geometric_mean_rank": 241.95374550665005, + "z_hits_at_k": 3212.6953808476746, + "z_inverse_harmonic_mean_rank": 4666.662332815998 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.042093576163310976, + "adjusted_arithmetic_mean_rank_index": 0.9580357665506809, + "adjusted_geometric_mean_rank_index": 0.9938616152525771, + "adjusted_hits_at_k": 0.3438538392238848, + "adjusted_inverse_harmonic_mean_rank": 0.20220859788631668, + "arithmetic_mean_rank": 311.7852584178362, + "count": 59071.0, + "geometric_mean_rank": 34.43162047198766, + "harmonic_mean_rank": 4.9319957594468615, + "hits_at_1": 0.12854023124714326, + "hits_at_10": 0.34429754024817594, + "hits_at_3": 0.22178395490172842, + "hits_at_5": 0.26965854649489596, + "inverse_arithmetic_mean_rank": 0.0032073357318897324, + "inverse_geometric_mean_rank": 0.02904307105770884, + "inverse_harmonic_mean_rank": 0.2027576763594284, + "inverse_median_rank": 0.029411764705882353, + "median_absolute_deviation": 48.92587321068486, + "median_rank": 34.0, + "standard_deviation": 875.5522866297537, + "variance": 766591.8066225903, + "z_arithmetic_mean_rank": 403.00995530034663, + "z_geometric_mean_rank": 241.95374137864584, + "z_hits_at_k": 3212.6953808476746, + "z_inverse_harmonic_mean_rank": 4666.656856562322 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04209355000943946, + "adjusted_arithmetic_mean_rank_index": 0.9580357927080839, + "adjusted_geometric_mean_rank_index": 0.9938616248602049, + "adjusted_hits_at_k": 0.3438538392238848, + "adjusted_inverse_harmonic_mean_rank": 0.20220871254805994, + "arithmetic_mean_rank": 311.7850646972656, + "count": 59071.0, + "geometric_mean_rank": 34.43156814575195, + "harmonic_mean_rank": 4.9319929722690325, + "hits_at_1": 0.12854023124714326, + "hits_at_10": 0.34429754024817594, + "hits_at_3": 0.22178395490172842, + "hits_at_5": 0.26965854649489596, + "inverse_arithmetic_mean_rank": 0.003207337809726596, + "inverse_geometric_mean_rank": 0.02904311567544937, + "inverse_harmonic_mean_rank": 0.20275779094225593, + "inverse_median_rank": 0.029411764815449715, + "median_absolute_deviation": 48.92587321068486, + "median_rank": 34.0, + "standard_deviation": 875.55224609375, + "variance": 766591.75, + "z_arithmetic_mean_rank": 403.0099663037916, + "z_geometric_mean_rank": 241.95374371760477, + "z_hits_at_k": 3212.6953808476746, + "z_inverse_harmonic_mean_rank": 4666.659502775273 + } + } + }, + "times": { + "evaluation": 19.562544584274292, + "training": 17.731594562530518 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json new file mode 100644 index 0000000..e4b25e5 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult create two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json new file mode 100644 index 0000000..7db2211 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 2.0877770193848152e-05, + 1.682386126120239e-05, + 1.3304294049895757e-05, + 1.077110180917705e-05, + 8.764388467831282e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.043555229029547775, + "adjusted_arithmetic_mean_rank_index": 0.9564915129253, + "adjusted_geometric_mean_rank_index": 0.9997352943268814, + "adjusted_hits_at_k": 0.7667430042934947, + "adjusted_inverse_harmonic_mean_rank": 0.6225622001033762, + "arithmetic_mean_rank": 891.2808, + "count": 10000.0, + "geometric_mean_rank": 4.985848515101419, + "harmonic_mean_rank": 1.6059988219783015, + "hits_at_1": 0.5354, + "hits_at_10": 0.7668, + "hits_at_3": 0.6879, + "hits_at_5": 0.7253, + "inverse_arithmetic_mean_rank": 0.0011219808616992535, + "inverse_geometric_mean_rank": 0.20056766605947685, + "inverse_harmonic_mean_rank": 0.6226654629597922, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4327.1181617043185, + "variance": 18723951.585351363, + "z_arithmetic_mean_rank": 165.66493427204637, + "z_geometric_mean_rank": 100.05439045229771, + "z_hits_at_k": 4904.480641422729, + "z_inverse_harmonic_mean_rank": 9826.388478656007 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04355523391636111, + "adjusted_arithmetic_mean_rank_index": 0.9564915080382479, + "adjusted_geometric_mean_rank_index": 0.999735294324712, + "adjusted_hits_at_k": 0.7667430042934947, + "adjusted_inverse_harmonic_mean_rank": 0.6225622001029469, + "arithmetic_mean_rank": 891.2809, + "count": 10000.0, + "geometric_mean_rank": 4.985848547766591, + "harmonic_mean_rank": 1.6059988219794084, + "hits_at_1": 0.5354, + "hits_at_10": 0.7668, + "hits_at_3": 0.6879, + "hits_at_5": 0.7253, + "inverse_arithmetic_mean_rank": 0.0011219807358151621, + "inverse_geometric_mean_rank": 0.2005676647454423, + "inverse_harmonic_mean_rank": 0.6226654629593631, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4327.118493847283, + "variance": 18723954.459795184, + "z_arithmetic_mean_rank": 165.66493342560585, + "z_geometric_mean_rank": 100.0543904520806, + "z_hits_at_k": 4904.480641422729, + "z_inverse_harmonic_mean_rank": 9826.388478649233 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.043555230141488696, + "adjusted_arithmetic_mean_rank_index": 0.9564915118133047, + "adjusted_geometric_mean_rank_index": 0.9997352943327442, + "adjusted_hits_at_k": 0.7667430042934947, + "adjusted_inverse_harmonic_mean_rank": 0.622562215663466, + "arithmetic_mean_rank": 891.2808227539062, + "count": 10000.0, + "geometric_mean_rank": 4.985848426818848, + "harmonic_mean_rank": 1.6059987818561974, + "hits_at_1": 0.5354, + "hits_at_10": 0.7668, + "hits_at_3": 0.6879, + "hits_at_5": 0.7253, + "inverse_arithmetic_mean_rank": 0.0011219808366149664, + "inverse_geometric_mean_rank": 0.20056766271591187, + "inverse_harmonic_mean_rank": 0.622665478515625, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4327.1181640625, + "variance": 18723950.0, + "z_arithmetic_mean_rank": 165.6649340794481, + "z_geometric_mean_rank": 100.05439045288448, + "z_hits_at_k": 4904.480641422729, + "z_inverse_harmonic_mean_rank": 9826.388724253135 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.043580116319214164, + "adjusted_arithmetic_mean_rank_index": 0.9564666255030494, + "adjusted_geometric_mean_rank_index": 0.9997343914817656, + "adjusted_hits_at_k": 0.766142856314531, + "adjusted_inverse_harmonic_mean_rank": 0.6197832789019377, + "arithmetic_mean_rank": 891.7694, + "count": 5000.0, + "geometric_mean_rank": 4.999549826111161, + "harmonic_mean_rank": 1.6131964522263165, + "hits_at_1": 0.5318, + "hits_at_10": 0.7662, + "hits_at_3": 0.6866, + "hits_at_5": 0.7258, + "inverse_arithmetic_mean_rank": 0.0011213661289566563, + "inverse_geometric_mean_rank": 0.2000180085769518, + "inverse_harmonic_mean_rank": 0.6198873042523337, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4331.006425096093, + "variance": 18757616.654223643, + "z_arithmetic_mean_rank": 117.1397405499343, + "z_geometric_mean_rank": 70.75430993104793, + "z_hits_at_k": 3465.2365686963662, + "z_inverse_harmonic_mean_rank": 6917.210199606586 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.043580116319214164, + "adjusted_arithmetic_mean_rank_index": 0.9564666255030494, + "adjusted_geometric_mean_rank_index": 0.9997343914817656, + "adjusted_hits_at_k": 0.766142856314531, + "adjusted_inverse_harmonic_mean_rank": 0.6197832789019377, + "arithmetic_mean_rank": 891.7694, + "count": 5000.0, + "geometric_mean_rank": 4.999549826111161, + "harmonic_mean_rank": 1.6131964522263165, + "hits_at_1": 0.5318, + "hits_at_10": 0.7662, + "hits_at_3": 0.6866, + "hits_at_5": 0.7258, + "inverse_arithmetic_mean_rank": 0.0011213661289566563, + "inverse_geometric_mean_rank": 0.2000180085769518, + "inverse_harmonic_mean_rank": 0.6198873042523337, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4331.006425096093, + "variance": 18757616.654223643, + "z_arithmetic_mean_rank": 117.1397405499343, + "z_geometric_mean_rank": 70.75430993104793, + "z_hits_at_k": 3465.2365686963662, + "z_inverse_harmonic_mean_rank": 6917.210199606586 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04358011676781876, + "adjusted_arithmetic_mean_rank_index": 0.9564666250544229, + "adjusted_geometric_mean_rank_index": 0.9997343915108016, + "adjusted_hits_at_k": 0.766142856314531, + "adjusted_inverse_harmonic_mean_rank": 0.6197832793372231, + "arithmetic_mean_rank": 891.7694091796875, + "count": 5000.0, + "geometric_mean_rank": 4.999549388885498, + "harmonic_mean_rank": 1.6131964510938386, + "hits_at_1": 0.5318, + "hits_at_10": 0.7662, + "hits_at_3": 0.6866, + "hits_at_5": 0.7258, + "inverse_arithmetic_mean_rank": 0.0011213661637157202, + "inverse_geometric_mean_rank": 0.2000180333852768, + "inverse_harmonic_mean_rank": 0.6198873046875, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4331.00634765625, + "variance": 18757616.0, + "z_arithmetic_mean_rank": 117.13974049499042, + "z_geometric_mean_rank": 70.7543099331029, + "z_hits_at_k": 3465.2365686963662, + "z_inverse_harmonic_mean_rank": 6917.210204464671 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04353034289378082, + "adjusted_arithmetic_mean_rank_index": 0.9565163991935886, + "adjusted_geometric_mean_rank_index": 0.9997362210601108, + "adjusted_hits_at_k": 0.7673431522656075, + "adjusted_inverse_harmonic_mean_rank": 0.625341121272479, + "arithmetic_mean_rank": 890.7922, + "count": 5000.0, + "geometric_mean_rank": 4.972184752657029, + "harmonic_mean_rank": 1.5988651340536348, + "hits_at_1": 0.539, + "hits_at_10": 0.7674, + "hits_at_3": 0.6892, + "hits_at_5": 0.7248, + "inverse_arithmetic_mean_rank": 0.0011225962688043295, + "inverse_geometric_mean_rank": 0.20111883402273448, + "inverse_harmonic_mean_rank": 0.6254436216672509, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4323.226346031302, + "variance": 18690286.03901916, + "z_arithmetic_mean_rank": 117.14585622847781, + "z_geometric_mean_rank": 70.75443745443081, + "z_hits_at_k": 3470.7465355941936, + "z_inverse_harmonic_mean_rank": 6979.402382059741 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04353035266718092, + "adjusted_arithmetic_mean_rank_index": 0.9565163894197108, + "adjusted_geometric_mean_rank_index": 0.9997362210557844, + "adjusted_hits_at_k": 0.7673431522656075, + "adjusted_inverse_harmonic_mean_rank": 0.6253411212716203, + "arithmetic_mean_rank": 890.7924, + "count": 5000.0, + "geometric_mean_rank": 4.972184817808335, + "harmonic_mean_rank": 1.5988651340558293, + "hits_at_1": 0.539, + "hits_at_10": 0.7674, + "hits_at_3": 0.6892, + "hits_at_5": 0.7248, + "inverse_arithmetic_mean_rank": 0.0011225960167599094, + "inverse_geometric_mean_rank": 0.2011188313874433, + "inverse_harmonic_mean_rank": 0.6254436216663924, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4323.227010937807, + "variance": 18690291.788102243, + "z_arithmetic_mean_rank": 117.1458550314578, + "z_geometric_mean_rank": 70.75443745412461, + "z_hits_at_k": 3470.7465355941936, + "z_inverse_harmonic_mean_rank": 6979.402382050157 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.043530347651632335, + "adjusted_arithmetic_mean_rank_index": 0.9565163944355045, + "adjusted_geometric_mean_rank_index": 0.9997362210347283, + "adjusted_hits_at_k": 0.7673431522656075, + "adjusted_inverse_harmonic_mean_rank": 0.6253411031158858, + "arithmetic_mean_rank": 890.7922973632812, + "count": 5000.0, + "geometric_mean_rank": 4.972185134887695, + "harmonic_mean_rank": 1.5988651804559029, + "hits_at_1": 0.539, + "hits_at_10": 0.7674, + "hits_at_3": 0.6892, + "hits_at_5": 0.7248, + "inverse_arithmetic_mean_rank": 0.0011225960915908217, + "inverse_geometric_mean_rank": 0.2011188119649887, + "inverse_harmonic_mean_rank": 0.625443603515625, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4323.2265625, + "variance": 18690288.0, + "z_arithmetic_mean_rank": 117.14585564574882, + "z_geometric_mean_rank": 70.7544374526344, + "z_hits_at_k": 3470.7465355941936, + "z_inverse_harmonic_mean_rank": 6979.402179414871 + } + } + }, + "times": { + "evaluation": 4.716632843017578, + "training": 6.86816668510437 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json new file mode 100644 index 0000000..69aa27e --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json new file mode 100644 index 0000000..9ec0e43 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 512, + "slice_size": null + }, + "losses": [ + 1.8794194563786133e-05, + 1.6593579347000567e-06, + 1.1399794914663178e-06, + 9.579132348384458e-07, + 8.295052289118864e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.03301642030053835, + "adjusted_arithmetic_mean_rank_index": 0.9671145034611406, + "adjusted_geometric_mean_rank_index": 0.9959516506556431, + "adjusted_hits_at_k": 0.4080815117156307, + "adjusted_inverse_harmonic_mean_rank": 0.2193426474691437, + "arithmetic_mean_rank": 243.88742360887744, + "count": 118142.0, + "geometric_mean_rank": 22.985003721360656, + "harmonic_mean_rank": 4.547906003106001, + "hits_at_1": 0.12495979414602766, + "hits_at_10": 0.4084830119686479, + "hits_at_3": 0.2488022887711398, + "hits_at_5": 0.3122682873152647, + "inverse_arithmetic_mean_rank": 0.004100252424674842, + "inverse_geometric_mean_rank": 0.04350662771790939, + "inverse_harmonic_mean_rank": 0.21988141340587253, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 880.602477092814, + "variance": 775460.7226619999, + "z_arithmetic_mean_rank": 575.2564304199459, + "z_geometric_mean_rank": 342.92036031626384, + "z_hits_at_k": 5383.815381126172, + "z_inverse_harmonic_mean_rank": 7147.909879999795 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.03301642946751757, + "adjusted_arithmetic_mean_rank_index": 0.9671144942929202, + "adjusted_geometric_mean_rank_index": 0.9959516461528362, + "adjusted_hits_at_k": 0.4080815117156307, + "adjusted_inverse_harmonic_mean_rank": 0.2193426163386077, + "arithmetic_mean_rank": 243.88749132399994, + "count": 118142.0, + "geometric_mean_rank": 22.985028174346066, + "harmonic_mean_rank": 4.547906646548572, + "hits_at_1": 0.12495979414602766, + "hits_at_10": 0.4084830119686479, + "hits_at_3": 0.2488022887711398, + "hits_at_5": 0.3122682873152647, + "inverse_arithmetic_mean_rank": 0.004100251286243782, + "inverse_geometric_mean_rank": 0.04350658143269604, + "inverse_harmonic_mean_rank": 0.21988138229682105, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 880.6024965562463, + "variance": 775460.7569410936, + "z_arithmetic_mean_rank": 575.2564249665298, + "z_geometric_mean_rank": 342.9203587658832, + "z_hits_at_k": 5383.815381126172, + "z_inverse_harmonic_mean_rank": 7147.908865521905 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.033016424031897265, + "adjusted_arithmetic_mean_rank_index": 0.9671144997292764, + "adjusted_geometric_mean_rank_index": 0.9959516491211775, + "adjusted_hits_at_k": 0.4080815117156307, + "adjusted_inverse_harmonic_mean_rank": 0.2193426353823576, + "arithmetic_mean_rank": 243.887451171875, + "count": 118142.0, + "geometric_mean_rank": 22.98501205444336, + "harmonic_mean_rank": 4.547906252929906, + "hits_at_1": 0.12495979414602766, + "hits_at_10": 0.4084830119686479, + "hits_at_3": 0.2488022887711398, + "hits_at_5": 0.3122682873152647, + "inverse_arithmetic_mean_rank": 0.004100251942873001, + "inverse_geometric_mean_rank": 0.04350661113858223, + "inverse_harmonic_mean_rank": 0.21988140132742803, + "inverse_median_rank": 0.05263157933950424, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 880.6024780273438, + "variance": 775460.75, + "z_arithmetic_mean_rank": 575.2564282001686, + "z_geometric_mean_rank": 342.92035978792546, + "z_hits_at_k": 5383.815381126172, + "z_inverse_harmonic_mean_rank": 7147.909486117169 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.03559929270353428, + "adjusted_arithmetic_mean_rank_index": 0.9645316377297036, + "adjusted_geometric_mean_rank_index": 0.9953978865707077, + "adjusted_hits_at_k": 0.38825165063719563, + "adjusted_inverse_harmonic_mean_rank": 0.20451332513669285, + "arithmetic_mean_rank": 262.2510538165936, + "count": 59071.0, + "geometric_mean_rank": 25.920386954674456, + "harmonic_mean_rank": 4.876530312323604, + "hits_at_1": 0.11244096087758798, + "hits_at_10": 0.3886678742530176, + "hits_at_3": 0.23251680181476528, + "hits_at_5": 0.29401906180697807, + "inverse_arithmetic_mean_rank": 0.003813140063487998, + "inverse_geometric_mean_rank": 0.03857967096512273, + "inverse_harmonic_mean_rank": 0.2050638334950722, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 897.8766380331401, + "variance": 806182.4571256945, + "z_arithmetic_mean_rank": 405.62259545037574, + "z_geometric_mean_rank": 242.3297125453849, + "z_hits_at_k": 3616.39751933023, + "z_inverse_harmonic_mean_rank": 4705.430523179204 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.03559929729953194, + "adjusted_arithmetic_mean_rank_index": 0.964531633133082, + "adjusted_geometric_mean_rank_index": 0.9953978825519514, + "adjusted_hits_at_k": 0.38825165063719563, + "adjusted_inverse_harmonic_mean_rank": 0.20451329120962963, + "arithmetic_mean_rank": 262.2510876741548, + "count": 59071.0, + "geometric_mean_rank": 25.920408716190114, + "harmonic_mean_rank": 4.876531118569549, + "hits_at_1": 0.11244096087758798, + "hits_at_10": 0.3886678742530176, + "hits_at_3": 0.23251680181476528, + "hits_at_5": 0.29401906180697807, + "inverse_arithmetic_mean_rank": 0.003813139571197864, + "inverse_geometric_mean_rank": 0.038579638575505615, + "inverse_harmonic_mean_rank": 0.20506379959148785, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 897.8766322354212, + "variance": 806182.4467144217, + "z_arithmetic_mean_rank": 405.6225935173198, + "z_geometric_mean_rank": 242.32971156701828, + "z_hits_at_k": 3616.39751933023, + "z_inverse_harmonic_mean_rank": 4705.429742587335 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.035599294644504544, + "adjusted_arithmetic_mean_rank_index": 0.9645316357884699, + "adjusted_geometric_mean_rank_index": 0.9953978847516574, + "adjusted_hits_at_k": 0.38825165063719563, + "adjusted_inverse_harmonic_mean_rank": 0.20451331644494522, + "arithmetic_mean_rank": 262.2510681152344, + "count": 59071.0, + "geometric_mean_rank": 25.92039680480957, + "harmonic_mean_rank": 4.876530518875093, + "hits_at_1": 0.11244096087758798, + "hits_at_10": 0.3886678742530176, + "hits_at_3": 0.23251680181476528, + "hits_at_5": 0.29401906180697807, + "inverse_arithmetic_mean_rank": 0.0038131398614495993, + "inverse_geometric_mean_rank": 0.038579657673835754, + "inverse_harmonic_mean_rank": 0.2050638248093396, + "inverse_median_rank": 0.04545454680919647, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 897.8766479492188, + "variance": 806182.5, + "z_arithmetic_mean_rank": 405.62259463401233, + "z_geometric_mean_rank": 242.32971210253692, + "z_hits_at_k": 3616.39751933023, + "z_inverse_harmonic_mean_rank": 4705.4303231999875 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.030447568375566038, + "adjusted_arithmetic_mean_rank_index": 0.9696833468576376, + "adjusted_geometric_mean_rank_index": 0.9964412605584122, + "adjusted_hits_at_k": 0.4279112902287407, + "adjusted_inverse_harmonic_mean_rank": 0.23417191353708747, + "arithmetic_mean_rank": 225.52379340116133, + "count": 59071.0, + "geometric_mean_rank": 20.3820412478714, + "harmonic_mean_rank": 4.260776690468067, + "hits_at_1": 0.13747862741446734, + "hits_at_10": 0.42829814968427826, + "hits_at_3": 0.26508777572751435, + "hits_at_5": 0.3305175128235513, + "inverse_arithmetic_mean_rank": 0.0044341219386160365, + "inverse_geometric_mean_rank": 0.049062799345695324, + "inverse_harmonic_mean_rank": 0.2346989933166728, + "inverse_median_rank": 0.058823529411764705, + "median_absolute_deviation": 23.721635496089633, + "median_rank": 17.0, + "standard_deviation": 862.5917588114866, + "variance": 744064.5423694939, + "z_arithmetic_mean_rank": 407.9096584040882, + "z_geometric_mean_rank": 242.5817511775936, + "z_hits_at_k": 3998.0610035746577, + "z_inverse_harmonic_mean_rank": 5404.319981173845 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.030447582088714377, + "adjusted_arithmetic_mean_rank_index": 0.9696833331426375, + "adjusted_geometric_mean_rank_index": 0.9964412557375856, + "adjusted_hits_at_k": 0.4279112902287407, + "adjusted_inverse_harmonic_mean_rank": 0.23417188520306811, + "arithmetic_mean_rank": 225.52389497384505, + "count": 59071.0, + "geometric_mean_rank": 20.382067503646027, + "harmonic_mean_rank": 4.260777204496071, + "hits_at_1": 0.13747862741446734, + "hits_at_10": 0.42829814968427826, + "hits_at_3": 0.26508777572751435, + "hits_at_5": 0.3305175128235513, + "inverse_arithmetic_mean_rank": 0.004434119941551977, + "inverse_geometric_mean_rank": 0.049062736143971455, + "inverse_harmonic_mean_rank": 0.23469896500215426, + "inverse_median_rank": 0.058823529411764705, + "median_absolute_deviation": 23.721635496089633, + "median_rank": 17.0, + "standard_deviation": 862.5918060275918, + "variance": 744064.6238259425, + "z_arithmetic_mean_rank": 407.90965263469866, + "z_geometric_mean_rank": 242.58175000397242, + "z_hits_at_k": 3998.0610035746577, + "z_inverse_harmonic_mean_rank": 5404.319327269179 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.030447575947657034, + "adjusted_arithmetic_mean_rank_index": 0.9696833392845241, + "adjusted_geometric_mean_rank_index": 0.9964412569308693, + "adjusted_hits_at_k": 0.4279112902287407, + "adjusted_inverse_harmonic_mean_rank": 0.23417189805527577, + "arithmetic_mean_rank": 225.5238494873047, + "count": 59071.0, + "geometric_mean_rank": 20.382061004638672, + "harmonic_mean_rank": 4.260776971334831, + "hits_at_1": 0.13747862741446734, + "hits_at_10": 0.42829814968427826, + "hits_at_3": 0.26508777572751435, + "hits_at_5": 0.3305175128235513, + "inverse_arithmetic_mean_rank": 0.004434120841324329, + "inverse_geometric_mean_rank": 0.04906275123357773, + "inverse_harmonic_mean_rank": 0.2346989778455164, + "inverse_median_rank": 0.05882352963089943, + "median_absolute_deviation": 23.721635496089633, + "median_rank": 17.0, + "standard_deviation": 862.591796875, + "variance": 744064.625, + "z_arithmetic_mean_rank": 407.9096552183615, + "z_geometric_mean_rank": 242.58175029447509, + "z_hits_at_k": 3998.0610035746577, + "z_inverse_harmonic_mean_rank": 5404.3196238779465 + } + } + }, + "times": { + "evaluation": 34.93160533905029, + "training": 4.057121515274048 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json new file mode 100644 index 0000000..83e9544 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json new file mode 100644 index 0000000..d1cbe0f --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 8.82791924415056e-05, + 2.338981202127039e-06, + 2.670299490484847e-07, + 2.541239686084076e-07, + 2.686744262802388e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04818624212508355, + "adjusted_arithmetic_mean_rank_index": 0.9518602735097387, + "adjusted_geometric_mean_rank_index": 0.9998959797743731, + "adjusted_hits_at_k": 0.8902731885548729, + "adjusted_inverse_harmonic_mean_rank": 0.771540162192838, + "arithmetic_mean_rank": 986.0463, + "count": 10000.0, + "geometric_mean_rank": 2.566301382855976, + "harmonic_mean_rank": 1.2960038160389173, + "hits_at_1": 0.7075, + "hits_at_10": 0.8903, + "hits_at_3": 0.8156, + "hits_at_5": 0.8519, + "inverse_arithmetic_mean_rank": 0.0010141511610560275, + "inverse_geometric_mean_rank": 0.3896658462176113, + "inverse_harmonic_mean_rank": 0.7716026663072505, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5035.254535130901, + "variance": 25353788.233556304, + "z_arithmetic_mean_rank": 164.86280067963153, + "z_geometric_mean_rank": 100.07047199367629, + "z_hits_at_k": 5694.642917372759, + "z_inverse_harmonic_mean_rank": 12177.824736762357 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.048186247011896895, + "adjusted_arithmetic_mean_rank_index": 0.9518602686226866, + "adjusted_geometric_mean_rank_index": 0.9998959797654242, + "adjusted_hits_at_k": 0.8902731885548729, + "adjusted_inverse_harmonic_mean_rank": 0.7715401621652603, + "arithmetic_mean_rank": 986.0464, + "count": 10000.0, + "geometric_mean_rank": 2.5663015176053334, + "harmonic_mean_rank": 1.296003816085225, + "hits_at_1": 0.7075, + "hits_at_10": 0.8903, + "hits_at_3": 0.8156, + "hits_at_5": 0.8519, + "inverse_arithmetic_mean_rank": 0.0010141510582057803, + "inverse_geometric_mean_rank": 0.38966582575734116, + "inverse_harmonic_mean_rank": 0.7716026662796803, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5035.254553371363, + "variance": 25353788.41724704, + "z_arithmetic_mean_rank": 164.862799833191, + "z_geometric_mean_rank": 100.07047199278067, + "z_hits_at_k": 5694.642917372759, + "z_inverse_harmonic_mean_rank": 12177.824736327077 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.048186246362866995, + "adjusted_arithmetic_mean_rank_index": 0.9518602692717482, + "adjusted_geometric_mean_rank_index": 0.9998959797609986, + "adjusted_hits_at_k": 0.8902731885548729, + "adjusted_inverse_harmonic_mean_rank": 0.7715401325962403, + "arithmetic_mean_rank": 986.04638671875, + "count": 10000.0, + "geometric_mean_rank": 2.5663015842437744, + "harmonic_mean_rank": 1.2960038657365307, + "hits_at_1": 0.7075, + "hits_at_10": 0.8903, + "hits_at_3": 0.8156, + "hits_at_5": 0.8519, + "inverse_arithmetic_mean_rank": 0.0010141510283574462, + "inverse_geometric_mean_rank": 0.389665812253952, + "inverse_harmonic_mean_rank": 0.7716026367187501, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5035.25439453125, + "variance": 25353788.0, + "z_arithmetic_mean_rank": 164.86279994560888, + "z_geometric_mean_rank": 100.07047199233776, + "z_hits_at_k": 5694.642917372759, + "z_inverse_harmonic_mean_rank": 12177.824269615996 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04813570932708946, + "adjusted_arithmetic_mean_rank_index": 0.9519108098558059, + "adjusted_geometric_mean_rank_index": 0.9998956496843767, + "adjusted_hits_at_k": 0.8889728701920999, + "adjusted_inverse_harmonic_mean_rank": 0.7717651857990683, + "arithmetic_mean_rank": 984.9894, + "count": 5000.0, + "geometric_mean_rank": 2.5713136366255833, + "harmonic_mean_rank": 1.2956260718711086, + "hits_at_1": 0.7092, + "hits_at_10": 0.889, + "hits_at_3": 0.8126, + "hits_at_5": 0.8516, + "inverse_arithmetic_mean_rank": 0.0010152393518143444, + "inverse_geometric_mean_rank": 0.3889062717811164, + "inverse_harmonic_mean_rank": 0.7718276296769999, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5033.722594947763, + "variance": 25338363.16288764, + "z_arithmetic_mean_rank": 116.5817837444569, + "z_geometric_mean_rank": 70.76572267521647, + "z_hits_at_k": 4020.7923013041427, + "z_inverse_harmonic_mean_rank": 8613.433431712892 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04813571910094273, + "adjusted_arithmetic_mean_rank_index": 0.9519108000814749, + "adjusted_geometric_mean_rank_index": 0.9998956496664444, + "adjusted_hits_at_k": 0.8889728701920999, + "adjusted_inverse_harmonic_mean_rank": 0.7717651857439131, + "arithmetic_mean_rank": 984.9896, + "count": 5000.0, + "geometric_mean_rank": 2.571313906650664, + "harmonic_mean_rank": 1.2956260719636692, + "hits_at_1": 0.7092, + "hits_at_10": 0.889, + "hits_at_3": 0.8126, + "hits_at_5": 0.8516, + "inverse_arithmetic_mean_rank": 0.0010152391456721979, + "inverse_geometric_mean_rank": 0.38890623094034354, + "inverse_harmonic_mean_rank": 0.7718276296218598, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5033.722631481778, + "variance": 25338363.53069184, + "z_arithmetic_mean_rank": 116.58178254738158, + "z_geometric_mean_rank": 70.76572267394735, + "z_hits_at_k": 4020.7923013041427, + "z_inverse_harmonic_mean_rank": 8613.433431097323 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.048135714309463874, + "adjusted_arithmetic_mean_rank_index": 0.9519108048731879, + "adjusted_geometric_mean_rank_index": 0.9998956496696731, + "adjusted_hits_at_k": 0.8889728701920999, + "adjusted_inverse_harmonic_mean_rank": 0.7717651928427455, + "arithmetic_mean_rank": 984.989501953125, + "count": 5000.0, + "geometric_mean_rank": 2.5713138580322266, + "harmonic_mean_rank": 1.2956260600504965, + "hits_at_1": 0.7092, + "hits_at_10": 0.889, + "hits_at_3": 0.8126, + "hits_at_5": 0.8516, + "inverse_arithmetic_mean_rank": 0.0010152392787858844, + "inverse_geometric_mean_rank": 0.38890624046325684, + "inverse_harmonic_mean_rank": 0.77182763671875, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5033.72265625, + "variance": 25338364.0, + "z_arithmetic_mean_rank": 116.58178313422906, + "z_geometric_mean_rank": 70.76572267417586, + "z_hits_at_k": 4020.7923013041427, + "z_inverse_harmonic_mean_rank": 8613.433510325203 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.048236772580123974, + "adjusted_arithmetic_mean_rank_index": 0.9518097395069041, + "adjusted_geometric_mean_rank_index": 0.9998963195837571, + "adjusted_hits_at_k": 0.8915735069028025, + "adjusted_inverse_harmonic_mean_rank": 0.7713151385892261, + "arithmetic_mean_rank": 987.1032, + "count": 5000.0, + "geometric_mean_rank": 2.5612988994572374, + "harmonic_mean_rank": 1.2963817805361464, + "hits_at_1": 0.7058, + "hits_at_10": 0.8916, + "hits_at_3": 0.8186, + "hits_at_5": 0.8522, + "inverse_arithmetic_mean_rank": 0.0010130653005683702, + "inverse_geometric_mean_rank": 0.39042690418205744, + "inverse_harmonic_mean_rank": 0.7713777029375009, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5036.785787598055, + "variance": 25369211.07014976, + "z_arithmetic_mean_rank": 116.5694252551693, + "z_geometric_mean_rank": 70.76576812420075, + "z_hits_at_k": 4032.649084277442, + "z_inverse_harmonic_mean_rank": 8608.611416172516 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.048236772580123974, + "adjusted_arithmetic_mean_rank_index": 0.9518097395069041, + "adjusted_geometric_mean_rank_index": 0.9998963195837571, + "adjusted_hits_at_k": 0.8915735069028025, + "adjusted_inverse_harmonic_mean_rank": 0.7713151385892261, + "arithmetic_mean_rank": 987.1032, + "count": 5000.0, + "geometric_mean_rank": 2.5612988994572374, + "harmonic_mean_rank": 1.2963817805361464, + "hits_at_1": 0.7058, + "hits_at_10": 0.8916, + "hits_at_3": 0.8186, + "hits_at_5": 0.8522, + "inverse_arithmetic_mean_rank": 0.0010130653005683702, + "inverse_geometric_mean_rank": 0.39042690418205744, + "inverse_harmonic_mean_rank": 0.7713777029375009, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5036.785787598055, + "variance": 25369211.07014976, + "z_arithmetic_mean_rank": 116.5694252551693, + "z_geometric_mean_rank": 70.76576812420075, + "z_hits_at_k": 4032.649084277442, + "z_inverse_harmonic_mean_rank": 8608.611416172516 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04823677309074595, + "adjusted_arithmetic_mean_rank_index": 0.9518097389962572, + "adjusted_geometric_mean_rank_index": 0.9998963195713949, + "adjusted_hits_at_k": 0.8915735069028025, + "adjusted_inverse_harmonic_mean_rank": 0.7713151700353283, + "arithmetic_mean_rank": 987.1032104492188, + "count": 5000.0, + "geometric_mean_rank": 2.5612990856170654, + "harmonic_mean_rank": 1.2963817277021077, + "hits_at_1": 0.7058, + "hits_at_10": 0.8916, + "hits_at_3": 0.8186, + "hits_at_5": 0.8522, + "inverse_arithmetic_mean_rank": 0.0010130653390660882, + "inverse_geometric_mean_rank": 0.3904268741607666, + "inverse_harmonic_mean_rank": 0.771377734375, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5036.78564453125, + "variance": 25369212.0, + "z_arithmetic_mean_rank": 116.56942519262968, + "z_geometric_mean_rank": 70.76576812332583, + "z_hits_at_k": 4032.649084277442, + "z_inverse_harmonic_mean_rank": 8608.611767140961 + } + } + }, + "times": { + "evaluation": 7.370163202285767, + "training": 2.276630163192749 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json new file mode 100644 index 0000000..e88fb54 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json new file mode 100644 index 0000000..681a02a --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 0.007800171632286395, + 0.006663566764337815, + 0.006155564227366854, + 0.005809586482315281, + 0.005533811495539998 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9175757107818011, + "adjusted_arithmetic_mean_rank_index": 0.08243544897129595, + "adjusted_geometric_mean_rank_index": 0.12309208616904688, + "adjusted_hits_at_k": -0.00029760758581000226, + "adjusted_inverse_harmonic_mean_rank": -3.625504840630545e-05, + "arithmetic_mean_rank": 6777.996343383386, + "count": 118142.0, + "geometric_mean_rank": 4763.144298573742, + "harmonic_mean_rank": 1529.2532991391572, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0003808975639484688, + "hits_at_3": 6.771512247972779e-05, + "hits_at_5": 0.0001269658546494896, + "inverse_arithmetic_mean_rank": 0.00014753622594916124, + "inverse_geometric_mean_rank": 0.0002099453506582692, + "inverse_harmonic_mean_rank": 0.0006539139072401656, + "inverse_median_rank": 0.00015420200462606013, + "median_absolute_deviation": 5329.954975527639, + "median_rank": 6485.0, + "standard_deviation": 4252.917905988223, + "variance": 18087310.715075247, + "z_arithmetic_mean_rank": 49.034030557477536, + "z_geometric_mean_rank": 42.38236114512432, + "z_hits_at_k": -3.926333960309983, + "z_inverse_harmonic_mean_rank": -1.1814748371711765 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9175761049619076, + "adjusted_arithmetic_mean_rank_index": 0.08243505473781987, + "adjusted_geometric_mean_rank_index": 0.12309148448714802, + "adjusted_hits_at_k": -0.00029760758581000226, + "adjusted_inverse_harmonic_mean_rank": -3.6256189824248686e-05, + "arithmetic_mean_rank": 6777.999255133653, + "count": 118142.0, + "geometric_mean_rank": 4763.147566073076, + "harmonic_mean_rank": 1529.2559666394513, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0003808975639484688, + "hits_at_3": 6.771512247972779e-05, + "hits_at_5": 0.0001269658546494896, + "inverse_arithmetic_mean_rank": 0.00014753616256930104, + "inverse_geometric_mean_rank": 0.00020994520663663563, + "inverse_harmonic_mean_rank": 0.000653912766609965, + "inverse_median_rank": 0.00015420200462606013, + "median_absolute_deviation": 5329.954975527639, + "median_rank": 6485.0, + "standard_deviation": 4252.918495996746, + "variance": 18087315.733591225, + "z_arithmetic_mean_rank": 49.03379606058866, + "z_geometric_mean_rank": 42.3821539774637, + "z_hits_at_k": -3.926333960309983, + "z_inverse_harmonic_mean_rank": -1.181512033551762 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9175758752917029, + "adjusted_arithmetic_mean_rank_index": 0.0824352844391204, + "adjusted_geometric_mean_rank_index": 0.12309132402137501, + "adjusted_hits_at_k": -0.00029760758581000226, + "adjusted_inverse_harmonic_mean_rank": -3.6255670257842945e-05, + "arithmetic_mean_rank": 6777.99755859375, + "count": 118142.0, + "geometric_mean_rank": 4763.1484375, + "harmonic_mean_rank": 1529.254752408621, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0003808975639484688, + "hits_at_3": 6.771512247972779e-05, + "hits_at_5": 0.0001269658546494896, + "inverse_arithmetic_mean_rank": 0.00014753619325347245, + "inverse_geometric_mean_rank": 0.0002099451667163521, + "inverse_harmonic_mean_rank": 0.0006539132858177951, + "inverse_median_rank": 0.00015420200361404568, + "median_absolute_deviation": 5329.954975527639, + "median_rank": 6485.0, + "standard_deviation": 4252.91845703125, + "variance": 18087316.0, + "z_arithmetic_mean_rank": 49.03393269089429, + "z_geometric_mean_rank": 42.38209872680903, + "z_hits_at_k": -3.926333960309983, + "z_inverse_harmonic_mean_rank": -1.1814951019888085 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9418988796405415, + "adjusted_arithmetic_mean_rank_index": 0.058109008371985404, + "adjusted_geometric_mean_rank_index": 0.07097860130274114, + "adjusted_hits_at_k": -0.0002573392909877556, + "adjusted_inverse_harmonic_mean_rank": -8.08950121906492e-05, + "arithmetic_mean_rank": 6938.732626838889, + "count": 59071.0, + "geometric_mean_rank": 5031.639313961612, + "harmonic_mean_rank": 1636.1238648057067, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.00042321951549829863, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014411853774737172, + "inverse_geometric_mean_rank": 0.00019874238545382933, + "inverse_harmonic_mean_rank": 0.0006112006685500869, + "inverse_median_rank": 0.00014949917775452235, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 6689.0, + "standard_deviation": 4216.167552085728, + "variance": 17776068.827260558, + "z_arithmetic_mean_rank": 24.437069633476995, + "z_geometric_mean_rank": 17.279747408168674, + "z_hits_at_k": -2.3970050662423676, + "z_inverse_harmonic_mean_rank": -1.861227669543867 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9418992771943402, + "adjusted_arithmetic_mean_rank_index": 0.058108610764213586, + "adjusted_geometric_mean_rank_index": 0.07097804978650235, + "adjusted_hits_at_k": -0.0002573392909877556, + "adjusted_inverse_harmonic_mean_rank": -8.089554686966131e-05, + "arithmetic_mean_rank": 6938.735555517936, + "count": 59071.0, + "geometric_mean_rank": 5031.642300415183, + "harmonic_mean_rank": 1636.1252950993955, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.00042321951549829863, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014411847691828572, + "inverse_geometric_mean_rank": 0.00019874226749335612, + "inverse_harmonic_mean_rank": 0.0006112001342410939, + "inverse_median_rank": 0.00014949917775452235, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 6689.0, + "standard_deviation": 4216.168227987367, + "variance": 17776074.526690133, + "z_arithmetic_mean_rank": 24.436902424139248, + "z_geometric_mean_rank": 17.27961314148652, + "z_hits_at_k": -2.3970050662423676, + "z_inverse_harmonic_mean_rank": -1.8612399714070431 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9418990506630458, + "adjusted_arithmetic_mean_rank_index": 0.0581088373262626, + "adjusted_geometric_mean_rank_index": 0.07097745746800044, + "adjusted_hits_at_k": -0.0002573392909877556, + "adjusted_inverse_harmonic_mean_rank": -8.089523831127923e-05, + "arithmetic_mean_rank": 6938.73388671875, + "count": 59071.0, + "geometric_mean_rank": 5031.6455078125, + "harmonic_mean_rank": 1636.1244696896456, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.00042321951549829863, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014411851589102298, + "inverse_geometric_mean_rank": 0.00019874214194715023, + "inverse_harmonic_mean_rank": 0.0006112004425859413, + "inverse_median_rank": 0.000149499173858203, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 6689.0, + "standard_deviation": 4216.16796875, + "variance": 17776072.0, + "z_arithmetic_mean_rank": 24.436997702181714, + "z_geometric_mean_rank": 17.27946894148951, + "z_hits_at_k": -2.3970050662423676, + "z_inverse_harmonic_mean_rank": -1.8612328721139875 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8933845741729127, + "adjusted_arithmetic_mean_rank_index": 0.10662982173068158, + "adjusted_geometric_mean_rank_index": 0.17228976563296583, + "adjusted_hits_at_k": -0.00033787571296769276, + "adjusted_inverse_harmonic_mean_rank": 8.384746007820249e-06, + "arithmetic_mean_rank": 6617.260059927883, + "count": 59071.0, + "geometric_mean_rank": 4508.976536947512, + "harmonic_mean_rank": 1435.4881314087258, + "hits_at_1": 0.0, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 3.3857561239863894e-05, + "hits_at_5": 6.771512247972779e-05, + "inverse_arithmetic_mean_rank": 0.0001511199485804852, + "inverse_geometric_mean_rank": 0.00022177981894689128, + "inverse_harmonic_mean_rank": 0.0006966271459302441, + "inverse_median_rank": 0.0001589067217543302, + "median_absolute_deviation": 5362.572224334762, + "median_rank": 6293.0, + "standard_deviation": 4283.325845327644, + "variance": 18346880.297251772, + "z_arithmetic_mean_rank": 44.85519350085002, + "z_geometric_mean_rank": 41.943619470153436, + "z_hits_at_k": -3.156840548303876, + "z_inverse_harmonic_mean_rank": 0.19350676903425246 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8933849649976404, + "adjusted_arithmetic_mean_rank_index": 0.10662943085318222, + "adjusted_geometric_mean_rank_index": 0.17228912115264017, + "adjusted_hits_at_k": -0.00033787571296769276, + "adjusted_inverse_harmonic_mean_rank": 8.382997853248193e-06, + "arithmetic_mean_rank": 6617.26295474937, + "count": 59071.0, + "geometric_mean_rank": 4508.980046994949, + "harmonic_mean_rank": 1435.4917312315515, + "hits_at_1": 0.0, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 3.3857561239863894e-05, + "hits_at_5": 6.771512247972779e-05, + "inverse_arithmetic_mean_rank": 0.00015111988247078437, + "inverse_geometric_mean_rank": 0.00022177964630082123, + "inverse_harmonic_mean_rank": 0.0006966253989788363, + "inverse_median_rank": 0.0001589067217543302, + "median_absolute_deviation": 5362.572224334762, + "median_rank": 6293.0, + "standard_deviation": 4283.326350392866, + "variance": 18346884.623969864, + "z_arithmetic_mean_rank": 44.855029073248204, + "z_geometric_mean_rank": 41.94346257263001, + "z_hits_at_k": -3.156840548303876, + "z_inverse_harmonic_mean_rank": 0.19346642437233008 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8933847322055644, + "adjusted_arithmetic_mean_rank_index": 0.10662966367669136, + "adjusted_geometric_mean_rank_index": 0.1722885954471729, + "adjusted_hits_at_k": -0.00033787571296769276, + "adjusted_inverse_harmonic_mean_rank": 8.383922294825192e-06, + "arithmetic_mean_rank": 6617.26123046875, + "count": 59071.0, + "geometric_mean_rank": 4508.98291015625, + "harmonic_mean_rank": 1435.4898276070808, + "hits_at_1": 0.0, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 3.3857561239863894e-05, + "hits_at_5": 6.771512247972779e-05, + "inverse_arithmetic_mean_rank": 0.0001511199225205928, + "inverse_geometric_mean_rank": 0.00022177951177582145, + "inverse_harmonic_mean_rank": 0.0006966263227841681, + "inverse_median_rank": 0.00015890672511886805, + "median_absolute_deviation": 5362.572224334762, + "median_rank": 6293.0, + "standard_deviation": 4283.326171875, + "variance": 18346882.0, + "z_arithmetic_mean_rank": 44.855127013424635, + "z_geometric_mean_rank": 41.943334590622555, + "z_hits_at_k": -3.156840548303876, + "z_inverse_harmonic_mean_rank": 0.19348775903202942 + } + } + }, + "times": { + "evaluation": 28.49927306175232, + "training": 13.013453960418701 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json new file mode 100644 index 0000000..edae5ec --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json new file mode 100644 index 0000000..cab0e2e --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.18607304607422293, + 0.14845974718735003, + 0.13420732638760424, + 0.12613810872870132, + 0.11995924184578269 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6805475312723067, + "adjusted_arithmetic_mean_rank_index": 0.31946808053646936, + "adjusted_geometric_mean_rank_index": 0.6303612792466775, + "adjusted_hits_at_k": 0.020160578927561585, + "adjusted_inverse_harmonic_mean_rank": 0.011663063971064072, + "arithmetic_mean_rank": 13926.2027, + "count": 10000.0, + "geometric_mean_rank": 5566.894863075257, + "harmonic_mean_rank": 83.79797808279572, + "hits_at_1": 0.0065, + "hits_at_10": 0.0204, + "hits_at_3": 0.0118, + "hits_at_5": 0.0149, + "inverse_arithmetic_mean_rank": 7.180708349161111e-05, + "inverse_geometric_mean_rank": 0.0001796333547868697, + "inverse_harmonic_mean_rank": 0.01193346215360901, + "inverse_median_rank": 8.941744534358654e-05, + "median_absolute_deviation": 13907.550110691798, + "median_rank": 11183.5, + "standard_deviation": 11967.17692409587, + "variance": 143213323.53261268, + "z_arithmetic_mean_rank": 55.33207336281332, + "z_geometric_mean_rank": 63.08711307648896, + "z_hits_at_k": 128.95738013444262, + "z_inverse_harmonic_mean_rank": 184.08730470957238 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.680547995519574, + "adjusted_arithmetic_mean_rank_index": 0.3194676162665141, + "adjusted_geometric_mean_rank_index": 0.6303610204221396, + "adjusted_hits_at_k": 0.020160578927561585, + "adjusted_inverse_harmonic_mean_rank": 0.011663063881889167, + "arithmetic_mean_rank": 13926.2122, + "count": 10000.0, + "geometric_mean_rank": 5566.898760367626, + "harmonic_mean_rank": 83.79797870881961, + "hits_at_1": 0.0065, + "hits_at_10": 0.0204, + "hits_at_3": 0.0118, + "hits_at_5": 0.0149, + "inverse_arithmetic_mean_rank": 7.180703450720074e-05, + "inverse_geometric_mean_rank": 0.00017963322902857356, + "inverse_harmonic_mean_rank": 0.011933462064458502, + "inverse_median_rank": 8.941744534358654e-05, + "median_absolute_deviation": 13907.550110691798, + "median_rank": 11183.5, + "standard_deviation": 11967.181705713803, + "variance": 143213437.97757113, + "z_arithmetic_mean_rank": 55.33199295096378, + "z_geometric_mean_rank": 63.08708717310082, + "z_hits_at_k": 128.95738013444262, + "z_inverse_harmonic_mean_rank": 184.08730330205475 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.680547790655196, + "adjusted_arithmetic_mean_rank_index": 0.3194678211409039, + "adjusted_geometric_mean_rank_index": 0.6303612364287918, + "adjusted_hits_at_k": 0.020160578927561585, + "adjusted_inverse_harmonic_mean_rank": 0.011663063578822358, + "arithmetic_mean_rank": 13926.2080078125, + "count": 10000.0, + "geometric_mean_rank": 5566.8955078125, + "harmonic_mean_rank": 83.79798083640323, + "hits_at_1": 0.0065, + "hits_at_10": 0.0204, + "hits_at_3": 0.0118, + "hits_at_5": 0.0149, + "inverse_arithmetic_mean_rank": 7.180705870268866e-05, + "inverse_geometric_mean_rank": 0.00017963333812076598, + "inverse_harmonic_mean_rank": 0.011933461761474608, + "inverse_median_rank": 8.941744454205036e-05, + "median_absolute_deviation": 13907.550110691798, + "median_rank": 11183.5, + "standard_deviation": 11967.1796875, + "variance": 143213392.0, + "z_arithmetic_mean_rank": 55.332028435337506, + "z_geometric_mean_rank": 63.08710879123718, + "z_hits_at_k": 128.95738013444262, + "z_inverse_harmonic_mean_rank": 184.08729851851314 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6931484301437246, + "adjusted_arithmetic_mean_rank_index": 0.3068665662002288, + "adjusted_geometric_mean_rank_index": 0.6209297388690878, + "adjusted_hits_at_k": 0.020160573334964144, + "adjusted_inverse_harmonic_mean_rank": 0.011987723703128655, + "arithmetic_mean_rank": 14183.729, + "count": 5000.0, + "geometric_mean_rank": 5709.0639095025035, + "harmonic_mean_rank": 81.57911845258644, + "hits_at_1": 0.0062, + "hits_at_10": 0.0204, + "hits_at_3": 0.0128, + "hits_at_5": 0.0162, + "inverse_arithmetic_mean_rank": 7.05033210941918e-05, + "inverse_geometric_mean_rank": 0.00017516006404054101, + "inverse_harmonic_mean_rank": 0.012258038809051328, + "inverse_median_rank": 8.567879021548216e-05, + "median_absolute_deviation": 14252.996427603604, + "median_rank": 11671.5, + "standard_deviation": 12023.017353042413, + "variance": 144552946.27155897, + "z_arithmetic_mean_rank": 37.582356759430326, + "z_geometric_mean_rank": 43.94512738951765, + "z_hits_at_k": 91.18554769571841, + "z_inverse_harmonic_mean_rank": 133.79129042696715 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6931488504194151, + "adjusted_arithmetic_mean_rank_index": 0.30686614590399874, + "adjusted_geometric_mean_rank_index": 0.6209294933217391, + "adjusted_hits_at_k": 0.020160573334964144, + "adjusted_inverse_harmonic_mean_rank": 0.011987723620118795, + "arithmetic_mean_rank": 14183.7376, + "count": 5000.0, + "geometric_mean_rank": 5709.067606969973, + "harmonic_mean_rank": 81.57911900487858, + "hits_at_1": 0.0062, + "hits_at_10": 0.0204, + "hits_at_3": 0.0128, + "hits_at_5": 0.0162, + "inverse_arithmetic_mean_rank": 7.050327834604047e-05, + "inverse_geometric_mean_rank": 0.00017515995059843746, + "inverse_harmonic_mean_rank": 0.012258038726064182, + "inverse_median_rank": 8.567879021548216e-05, + "median_absolute_deviation": 14252.996427603604, + "median_rank": 11671.5, + "standard_deviation": 12023.021595936118, + "variance": 144553048.29634625, + "z_arithmetic_mean_rank": 37.58230528519168, + "z_geometric_mean_rank": 43.945110011368655, + "z_hits_at_k": 91.18554769571841, + "z_inverse_harmonic_mean_rank": 133.79128950051972 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6931486928160312, + "adjusted_arithmetic_mean_rank_index": 0.30686630351508504, + "adjusted_geometric_mean_rank_index": 0.6209295406345121, + "adjusted_hits_at_k": 0.020160573334964144, + "adjusted_inverse_harmonic_mean_rank": 0.01198772322402437, + "arithmetic_mean_rank": 14183.734375, + "count": 5000.0, + "geometric_mean_rank": 5709.06689453125, + "harmonic_mean_rank": 81.57912164022632, + "hits_at_1": 0.0062, + "hits_at_10": 0.0204, + "hits_at_3": 0.0128, + "hits_at_5": 0.0162, + "inverse_arithmetic_mean_rank": 7.050329440971836e-05, + "inverse_geometric_mean_rank": 0.00017515997751615942, + "inverse_harmonic_mean_rank": 0.012258038330078125, + "inverse_median_rank": 8.567878830945119e-05, + "median_absolute_deviation": 14252.996427603604, + "median_rank": 11671.5, + "standard_deviation": 12023.01953125, + "variance": 144552992.0, + "z_arithmetic_mean_rank": 37.58232458803117, + "z_geometric_mean_rank": 43.94511335984065, + "z_hits_at_k": 91.18554769571841, + "z_inverse_harmonic_mean_rank": 133.79128507983188 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6679472166416922, + "adjusted_arithmetic_mean_rank_index": 0.3320690105748175, + "adjusted_geometric_mean_rank_index": 0.6395941772795828, + "adjusted_hits_at_k": 0.020160584520158967, + "adjusted_inverse_harmonic_mean_rank": 0.01133840424277726, + "arithmetic_mean_rank": 13668.6764, + "count": 5000.0, + "geometric_mean_rank": 5428.2661584767875, + "harmonic_mean_rank": 86.14091336829217, + "hits_at_1": 0.0068, + "hits_at_10": 0.0204, + "hits_at_3": 0.0108, + "hits_at_5": 0.0136, + "inverse_arithmetic_mean_rank": 7.315997326558993e-05, + "inverse_geometric_mean_rank": 0.00018422088578659665, + "inverse_harmonic_mean_rank": 0.011608885498166688, + "inverse_median_rank": 9.337940050424876e-05, + "median_absolute_deviation": 13553.949481578213, + "median_rank": 10709.0, + "standard_deviation": 11905.505499695637, + "variance": 141741061.20328304, + "z_arithmetic_mean_rank": 40.668940546682066, + "z_geometric_mean_rank": 45.26606644756686, + "z_hits_at_k": 91.18772829639525, + "z_inverse_harmonic_mean_rank": 126.54738811957063 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6679477248584975, + "adjusted_arithmetic_mean_rank_index": 0.332068502333176, + "adjusted_geometric_mean_rank_index": 0.6395939060174631, + "adjusted_hits_at_k": 0.020160584520158967, + "adjusted_inverse_harmonic_mean_rank": 0.011338404147437312, + "arithmetic_mean_rank": 13668.6868, + "count": 5000.0, + "geometric_mean_rank": 5428.270243348978, + "harmonic_mean_rank": 86.14091407554555, + "hits_at_1": 0.0068, + "hits_at_10": 0.0204, + "hits_at_3": 0.0108, + "hits_at_5": 0.0136, + "inverse_arithmetic_mean_rank": 7.315991760086273e-05, + "inverse_geometric_mean_rank": 0.000184220747157026, + "inverse_harmonic_mean_rank": 0.011608885402852823, + "inverse_median_rank": 9.337940050424876e-05, + "median_absolute_deviation": 13553.949481578213, + "median_rank": 10709.0, + "standard_deviation": 11905.51086663255, + "variance": 141741188.99550575, + "z_arithmetic_mean_rank": 40.6688783016413, + "z_geometric_mean_rank": 45.26604724950413, + "z_hits_at_k": 91.18772829639525, + "z_inverse_harmonic_mean_rank": 126.54738705548604 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6679474727353167, + "adjusted_arithmetic_mean_rank_index": 0.3320687544686779, + "adjusted_geometric_mean_rank_index": 0.6395942775560269, + "adjusted_hits_at_k": 0.020160584520158967, + "adjusted_inverse_harmonic_mean_rank": 0.01133840393739812, + "arithmetic_mean_rank": 13668.681640625, + "count": 5000.0, + "geometric_mean_rank": 5428.2646484375, + "harmonic_mean_rank": 86.140915633664, + "hits_at_1": 0.0068, + "hits_at_10": 0.0204, + "hits_at_3": 0.0108, + "hits_at_5": 0.0136, + "inverse_arithmetic_mean_rank": 7.315994298551232e-05, + "inverse_geometric_mean_rank": 0.00018422093125991523, + "inverse_harmonic_mean_rank": 0.011608885192871093, + "inverse_median_rank": 9.337939991382882e-05, + "median_absolute_deviation": 13553.949481578213, + "median_rank": 10709.0, + "standard_deviation": 11905.5087890625, + "variance": 141741136.0, + "z_arithmetic_mean_rank": 40.668909181016986, + "z_geometric_mean_rank": 45.266073544442264, + "z_hits_at_k": 91.18772829639525, + "z_inverse_harmonic_mean_rank": 126.54738471124864 + } + } + }, + "times": { + "evaluation": 7.71472430229187, + "training": 190.2168550491333 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..5922760 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "The number of epochs has been inferred from the convergence experiment plot, Figure 2b. The publication favours SGD optimizer over others to ensure comaparabiltiy to RSGD, which is used to optimize the hyperbolic models studied in the same paper. The evaluation uses the non-deterministic variant of rank computation. While it is not guaranteed to yield the same results as average ranking definition due to the unknown properties of the employed sort algorithm (e.g. stability), it is closest to the average rank definition.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with MuRE as described by Bala\u017eevi\u0107 et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..c01d201 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0009751350673539609, + 0.0007105607929926378, + 0.0006821246302445668, + 0.000649823348379108, + 0.0006163581072503264 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.12330314877380849, + "adjusted_arithmetic_mean_rank_index": 0.876819729646672, + "adjusted_geometric_mean_rank_index": 0.9872007499946741, + "adjusted_hits_at_k": 0.30210753600136414, + "adjusted_inverse_harmonic_mean_rank": 0.18525654778353684, + "arithmetic_mean_rank": 879.8504501418926, + "count": 40876.0, + "geometric_mean_rank": 68.11252538045748, + "harmonic_mean_rank": 5.3810548114598244, + "hits_at_1": 0.12488991095019082, + "hits_at_10": 0.3025981015754966, + "hits_at_3": 0.20099814071826988, + "hits_at_5": 0.24297876504550348, + "inverse_arithmetic_mean_rank": 0.0011365567862569497, + "inverse_geometric_mean_rank": 0.014681587482100834, + "inverse_harmonic_mean_rank": 0.18583717041319459, + "inverse_median_rank": 0.014084507042253521, + "median_absolute_deviation": 103.78215529539213, + "median_rank": 71.0, + "standard_deviation": 1980.2107549290874, + "variance": 3921234.633936826, + "z_arithmetic_mean_rank": 306.65323406753174, + "z_geometric_mean_rank": 199.93371976539004, + "z_hits_at_k": 2302.975861709799, + "z_inverse_harmonic_mean_rank": 3488.4928948840657 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12330350876030921, + "adjusted_arithmetic_mean_rank_index": 0.8768193696097153, + "adjusted_geometric_mean_rank_index": 0.9872007362227379, + "adjusted_hits_at_k": 0.30210753600136414, + "adjusted_inverse_harmonic_mean_rank": 0.1852565465306213, + "arithmetic_mean_rank": 879.8530188863881, + "count": 40876.0, + "geometric_mean_rank": 68.11259759323725, + "harmonic_mean_rank": 5.381054847713081, + "hits_at_1": 0.12488991095019082, + "hits_at_10": 0.3025981015754966, + "hits_at_3": 0.20099814071826988, + "hits_at_5": 0.24297876504550348, + "inverse_arithmetic_mean_rank": 0.0011365534680618354, + "inverse_geometric_mean_rank": 0.014681571916724076, + "inverse_harmonic_mean_rank": 0.1858371691611719, + "inverse_median_rank": 0.014084507042253521, + "median_absolute_deviation": 103.78215529539213, + "median_rank": 71.0, + "standard_deviation": 1980.216275380783, + "variance": 3921256.4972829404, + "z_arithmetic_mean_rank": 306.6531081505463, + "z_geometric_mean_rank": 199.93371697621626, + "z_hits_at_k": 2302.975861709799, + "z_inverse_harmonic_mean_rank": 3488.4928712909077 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.12330333032140904, + "adjusted_arithmetic_mean_rank_index": 0.8768195480736256, + "adjusted_geometric_mean_rank_index": 0.9872007411578119, + "adjusted_hits_at_k": 0.30210753600136414, + "adjusted_inverse_harmonic_mean_rank": 0.185256550124665, + "arithmetic_mean_rank": 879.8517456054688, + "count": 40876.0, + "geometric_mean_rank": 68.1125717163086, + "harmonic_mean_rank": 5.381054743719011, + "hits_at_1": 0.12488991095019082, + "hits_at_10": 0.3025981015754966, + "hits_at_3": 0.20099814071826988, + "hits_at_5": 0.24297876504550348, + "inverse_arithmetic_mean_rank": 0.001136555103585124, + "inverse_geometric_mean_rank": 0.014681577682495117, + "inverse_harmonic_mean_rank": 0.18583717275265435, + "inverse_median_rank": 0.014084506779909134, + "median_absolute_deviation": 103.78215529539213, + "median_rank": 71.0, + "standard_deviation": 1980.21337890625, + "variance": 3921245.25, + "z_arithmetic_mean_rank": 306.6531705653545, + "z_geometric_mean_rank": 199.93371797569654, + "z_hits_at_k": 2302.975861709799, + "z_inverse_harmonic_mean_rank": 3488.4929389689255 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.17396412366101632, + "adjusted_arithmetic_mean_rank_index": 0.8261532837214485, + "adjusted_geometric_mean_rank_index": 0.9696588363086467, + "adjusted_hits_at_k": 0.18368121518304942, + "adjusted_inverse_harmonic_mean_rank": 0.1040972149746863, + "arithmetic_mean_rank": 1224.122614737254, + "count": 20438.0, + "geometric_mean_rank": 157.6978818065583, + "harmonic_mean_rank": 9.54697859231374, + "hits_at_1": 0.06306879342401409, + "hits_at_10": 0.1842646051472747, + "hits_at_3": 0.10822976807906841, + "hits_at_5": 0.13822291809374695, + "inverse_arithmetic_mean_rank": 0.0008169116295712258, + "inverse_geometric_mean_rank": 0.006341239264244906, + "inverse_harmonic_mean_rank": 0.10474518093138897, + "inverse_median_rank": 0.00423728813559322, + "median_absolute_deviation": 346.9289191303109, + "median_rank": 236.0, + "standard_deviation": 2284.762164276684, + "variance": 5220138.147310277, + "z_arithmetic_mean_rank": 204.08977609163793, + "z_geometric_mean_rank": 138.86918316715838, + "z_hits_at_k": 981.929290085826, + "z_inverse_harmonic_mean_rank": 1374.6864106662656 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.17396472165226567, + "adjusted_arithmetic_mean_rank_index": 0.8261526856452046, + "adjusted_geometric_mean_rank_index": 0.9696587831390359, + "adjusted_hits_at_k": 0.18368121518304942, + "adjusted_inverse_harmonic_mean_rank": 0.1040972127486539, + "arithmetic_mean_rank": 1224.12682258538, + "count": 20438.0, + "geometric_mean_rank": 157.69815640266404, + "harmonic_mean_rank": 9.546978795058282, + "hits_at_1": 0.06306879342401409, + "hits_at_10": 0.1842646051472747, + "hits_at_3": 0.10822976807906841, + "hits_at_5": 0.13822291809374695, + "inverse_arithmetic_mean_rank": 0.0008169088214961095, + "inverse_geometric_mean_rank": 0.006341228222393516, + "inverse_harmonic_mean_rank": 0.10474517870696656, + "inverse_median_rank": 0.00423728813559322, + "median_absolute_deviation": 346.9289191303109, + "median_rank": 236.0, + "standard_deviation": 2284.7695923634533, + "variance": 5220172.090188661, + "z_arithmetic_mean_rank": 204.08962834514932, + "z_geometric_mean_rank": 138.8691755525004, + "z_hits_at_k": 981.929290085826, + "z_inverse_harmonic_mean_rank": 1374.6863812697395 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.17396442794298037, + "adjusted_arithmetic_mean_rank_index": 0.8261529793962359, + "adjusted_geometric_mean_rank_index": 0.9696588108466823, + "adjusted_hits_at_k": 0.18368121518304942, + "adjusted_inverse_harmonic_mean_rank": 0.10409721372836692, + "arithmetic_mean_rank": 1224.124755859375, + "count": 20438.0, + "geometric_mean_rank": 157.69801330566406, + "harmonic_mean_rank": 9.546978705827115, + "hits_at_1": 0.06306879342401409, + "hits_at_10": 0.1842646051472747, + "hits_at_3": 0.10822976807906841, + "hits_at_5": 0.13822291809374695, + "inverse_arithmetic_mean_rank": 0.0008169101784005761, + "inverse_geometric_mean_rank": 0.006341233849525452, + "inverse_harmonic_mean_rank": 0.104745179685971, + "inverse_median_rank": 0.0042372881434857845, + "median_absolute_deviation": 346.9289191303109, + "median_rank": 236.0, + "standard_deviation": 2284.765869140625, + "variance": 5220155.0, + "z_arithmetic_mean_rank": 204.08970091229077, + "z_geometric_mean_rank": 138.86917952063646, + "z_hits_at_k": 981.929290085826, + "z_inverse_harmonic_mean_rank": 1374.6863942076282 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07402909794266327, + "adjusted_arithmetic_mean_rank_index": 0.9260989099858921, + "adjusted_geometric_mean_rank_index": 0.9946618446851297, + "adjusted_hits_at_k": 0.4205310753878879, + "adjusted_inverse_harmonic_mean_rank": 0.26641415717647093, + "arithmetic_mean_rank": 535.578285546531, + "count": 20438.0, + "geometric_mean_rank": 29.41901349946047, + "harmonic_mean_rank": 3.7463123189439558, + "hits_at_1": 0.18671102847636756, + "hits_at_10": 0.42093159800371854, + "hits_at_3": 0.2937665133574714, + "hits_at_5": 0.34773461199726, + "inverse_arithmetic_mean_rank": 0.0018671406720299532, + "inverse_geometric_mean_rank": 0.03399162245934384, + "inverse_harmonic_mean_rank": 0.26692915989500016, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1544.4366202382741, + "variance": 2385284.473933023, + "z_arithmetic_mean_rank": 229.29721558336044, + "z_geometric_mean_rank": 142.44367894315846, + "z_hits_at_k": 2285.960433588959, + "z_inverse_harmonic_mean_rank": 3577.2668975115416 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07402922644017056, + "adjusted_arithmetic_mean_rank_index": 0.926098781470621, + "adjusted_geometric_mean_rank_index": 0.99466184259013, + "adjusted_hits_at_k": 0.4205310753878879, + "adjusted_inverse_harmonic_mean_rank": 0.2664141568966516, + "arithmetic_mean_rank": 535.579215187396, + "count": 20438.0, + "geometric_mean_rank": 29.419024652718747, + "harmonic_mean_rank": 3.7463123228684223, + "hits_at_1": 0.18671102847636756, + "hits_at_10": 0.42093159800371854, + "hits_at_3": 0.2937665133574714, + "hits_at_5": 0.34773461199726, + "inverse_arithmetic_mean_rank": 0.0018671374311083112, + "inverse_geometric_mean_rank": 0.03399160957253508, + "inverse_harmonic_mean_rank": 0.2669291596153773, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1544.439056923411, + "variance": 2385292.0005504754, + "z_arithmetic_mean_rank": 229.2971837636558, + "z_geometric_mean_rank": 142.44367864313745, + "z_hits_at_k": 2285.960433588959, + "z_inverse_harmonic_mean_rank": 3577.266893754277 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07402916011594705, + "adjusted_arithmetic_mean_rank_index": 0.9260988478040133, + "adjusted_geometric_mean_rank_index": 0.9946618435206058, + "adjusted_hits_at_k": 0.4205310753878879, + "adjusted_inverse_harmonic_mean_rank": 0.26641415115115213, + "arithmetic_mean_rank": 535.5787353515625, + "count": 20438.0, + "geometric_mean_rank": 29.41901969909668, + "harmonic_mean_rank": 3.746312403449072, + "hits_at_1": 0.18671102847636756, + "hits_at_10": 0.42093159800371854, + "hits_at_3": 0.2937665133574714, + "hits_at_5": 0.34773461199726, + "inverse_arithmetic_mean_rank": 0.0018671391298994422, + "inverse_geometric_mean_rank": 0.03399161621928215, + "inverse_harmonic_mean_rank": 0.26692915387391136, + "inverse_median_rank": 0.0476190485060215, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1544.437744140625, + "variance": 2385288.0, + "z_arithmetic_mean_rank": 229.29720018745456, + "z_geometric_mean_rank": 142.44367877638916, + "z_hits_at_k": 2285.960433588959, + "z_inverse_harmonic_mean_rank": 3577.2668166067815 + } + } + }, + "times": { + "evaluation": 15.51103925704956, + "training": 64.47068643569946 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json new file mode 100644 index 0000000..0621d95 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json new file mode 100644 index 0000000..cbefaad --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.00029434056921607014, + 0.00026399555195481854, + 0.00024915793555821663, + 0.0002383932134910319, + 0.00023161682828993547 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.094754230917993, + "adjusted_arithmetic_mean_rank_index": 0.9053683339153923, + "adjusted_geometric_mean_rank_index": 0.9882856801373995, + "adjusted_hits_at_k": 0.28365521929977294, + "adjusted_inverse_harmonic_mean_rank": 0.1615919542527177, + "arithmetic_mean_rank": 699.9355182746186, + "count": 118142.0, + "geometric_mean_rank": 64.6158947427477, + "harmonic_mean_rank": 6.166346705456983, + "hits_at_1": 0.09879636369792284, + "hits_at_10": 0.28414111831524774, + "hits_at_3": 0.1777352677286655, + "hits_at_5": 0.2195747490308273, + "inverse_arithmetic_mean_rank": 0.0014287030360526033, + "inverse_geometric_mean_rank": 0.015476068295289483, + "inverse_harmonic_mean_rank": 0.16217057648007982, + "inverse_median_rank": 0.013888888888888888, + "median_absolute_deviation": 105.26475751389773, + "median_rank": 72.0, + "standard_deviation": 1722.2560768708345, + "variance": 2966165.9943185174, + "z_arithmetic_mean_rank": 538.5287410327303, + "z_geometric_mean_rank": 340.28085731372414, + "z_hits_at_k": 3742.2605258015683, + "z_inverse_harmonic_mean_rank": 5265.937744705862 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.09475424810607905, + "adjusted_arithmetic_mean_rank_index": 0.9053683167249791, + "adjusted_geometric_mean_rank_index": 0.9882856765779632, + "adjusted_hits_at_k": 0.28365521929977294, + "adjusted_inverse_harmonic_mean_rank": 0.16159195298409093, + "arithmetic_mean_rank": 699.9356452404734, + "count": 118142.0, + "geometric_mean_rank": 64.61591407265558, + "harmonic_mean_rank": 6.166346753661744, + "hits_at_1": 0.09879636369792284, + "hits_at_10": 0.28414111831524774, + "hits_at_3": 0.1777352677286655, + "hits_at_5": 0.2195747490308273, + "inverse_arithmetic_mean_rank": 0.0014287027768909168, + "inverse_geometric_mean_rank": 0.015476063665609954, + "inverse_harmonic_mean_rank": 0.16217057521232858, + "inverse_median_rank": 0.013888888888888888, + "median_absolute_deviation": 105.26475751389773, + "median_rank": 72.0, + "standard_deviation": 1722.2561085189475, + "variance": 2966166.1033308282, + "z_arithmetic_mean_rank": 538.5287308075752, + "z_geometric_mean_rank": 340.28085608815945, + "z_hits_at_k": 3742.2605258015683, + "z_inverse_harmonic_mean_rank": 5265.937703364017 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.09475423478978842, + "adjusted_arithmetic_mean_rank_index": 0.9053683300430727, + "adjusted_geometric_mean_rank_index": 0.9882856809181254, + "adjusted_hits_at_k": 0.28365521929977294, + "adjusted_inverse_harmonic_mean_rank": 0.1615919542822823, + "arithmetic_mean_rank": 699.935546875, + "count": 118142.0, + "geometric_mean_rank": 64.61589050292969, + "harmonic_mean_rank": 6.1663467043336, + "hits_at_1": 0.09879636369792284, + "hits_at_10": 0.28414111831524774, + "hits_at_3": 0.1777352677286655, + "hits_at_5": 0.2195747490308273, + "inverse_arithmetic_mean_rank": 0.0014287029625847936, + "inverse_geometric_mean_rank": 0.015476069413125515, + "inverse_harmonic_mean_rank": 0.16217057650962402, + "inverse_median_rank": 0.013888888992369175, + "median_absolute_deviation": 105.26475751389773, + "median_rank": 72.0, + "standard_deviation": 1722.256103515625, + "variance": 2966166.0, + "z_arithmetic_mean_rank": 538.5287387294077, + "z_geometric_mean_rank": 340.2808575825392, + "z_hits_at_k": 3742.2605258015683, + "z_inverse_harmonic_mean_rank": 5265.937745669309 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.09877014319497167, + "adjusted_arithmetic_mean_rank_index": 0.9013522109413388, + "adjusted_geometric_mean_rank_index": 0.9865650476892843, + "adjusted_hits_at_k": 0.26121629197880863, + "adjusted_inverse_harmonic_mean_rank": 0.14501653728384634, + "arithmetic_mean_rank": 727.6148533121159, + "count": 59071.0, + "geometric_mean_rank": 73.75009958894451, + "harmonic_mean_rank": 6.867744152780259, + "hits_at_1": 0.08549034213065633, + "hits_at_10": 0.2617189483841479, + "hits_at_3": 0.15826717001574375, + "hits_at_5": 0.19796516056948418, + "inverse_arithmetic_mean_rank": 0.0013743534720985725, + "inverse_geometric_mean_rank": 0.013559303723976323, + "inverse_harmonic_mean_rank": 0.1456082197813341, + "inverse_median_rank": 0.011627906976744186, + "median_absolute_deviation": 126.02118857297617, + "median_rank": 86.0, + "standard_deviation": 1733.651730374744, + "variance": 3005548.3222313435, + "z_arithmetic_mean_rank": 379.0532201489248, + "z_geometric_mean_rank": 240.17935705842558, + "z_hits_at_k": 2433.117666777289, + "z_inverse_harmonic_mean_rank": 3336.531937198153 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.09877016157896236, + "adjusted_arithmetic_mean_rank_index": 0.9013521925548522, + "adjusted_geometric_mean_rank_index": 0.9865650431569083, + "adjusted_hits_at_k": 0.26121629197880863, + "adjusted_inverse_harmonic_mean_rank": 0.14501653567957498, + "arithmetic_mean_rank": 727.6149887423609, + "count": 59071.0, + "geometric_mean_rank": 73.7501241317043, + "harmonic_mean_rank": 6.8677442283948125, + "hits_at_1": 0.08549034213065633, + "hits_at_10": 0.2617189483841479, + "hits_at_3": 0.15826717001574375, + "hits_at_5": 0.19796516056948418, + "inverse_arithmetic_mean_rank": 0.001374353216291545, + "inverse_geometric_mean_rank": 0.013559299211675658, + "inverse_harmonic_mean_rank": 0.14560821817817296, + "inverse_median_rank": 0.011627906976744186, + "median_absolute_deviation": 126.02118857297617, + "median_rank": 86.0, + "standard_deviation": 1733.6517922746639, + "variance": 3005548.536857154, + "z_arithmetic_mean_rank": 379.0532124167011, + "z_geometric_mean_rank": 240.1793559550182, + "z_hits_at_k": 2433.117666777289, + "z_inverse_harmonic_mean_rank": 3336.5319002871724 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.09877015349627322, + "adjusted_arithmetic_mean_rank_index": 0.9013522006386387, + "adjusted_geometric_mean_rank_index": 0.9865650505822927, + "adjusted_hits_at_k": 0.26121629197880863, + "adjusted_inverse_harmonic_mean_rank": 0.14501652890756012, + "arithmetic_mean_rank": 727.6149291992188, + "count": 59071.0, + "geometric_mean_rank": 73.75008392333984, + "harmonic_mean_rank": 6.867744547582034, + "hits_at_1": 0.08549034213065633, + "hits_at_10": 0.2617189483841479, + "hits_at_3": 0.15826717001574375, + "hits_at_5": 0.19796516056948418, + "inverse_arithmetic_mean_rank": 0.0013743533054366708, + "inverse_geometric_mean_rank": 0.013559306971728802, + "inverse_harmonic_mean_rank": 0.1456082114108446, + "inverse_median_rank": 0.011627906933426857, + "median_absolute_deviation": 126.02118857297617, + "median_rank": 86.0, + "standard_deviation": 1733.6517333984375, + "variance": 3005548.25, + "z_arithmetic_mean_rank": 379.0532158162438, + "z_geometric_mean_rank": 240.17935776272876, + "z_hits_at_k": 2433.117666777289, + "z_inverse_harmonic_mean_rank": 3336.531744477054 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.09076011801823436, + "adjusted_arithmetic_mean_rank_index": 0.9093626534248616, + "adjusted_geometric_mean_rank_index": 0.9897888902290393, + "adjusted_hits_at_k": 0.3060940531920783, + "adjusted_inverse_harmonic_mean_rank": 0.17816730833215674, + "arithmetic_mean_rank": 672.2561832371214, + "count": 59071.0, + "geometric_mean_rank": 56.61299275088353, + "harmonic_mean_rank": 5.594939792094619, + "hits_at_1": 0.11210238526518934, + "hits_at_10": 0.3065632882463476, + "hits_at_3": 0.19720336544158723, + "hits_at_5": 0.24118433749217044, + "inverse_arithmetic_mean_rank": 0.0014875281550921417, + "inverse_geometric_mean_rank": 0.017663789731101852, + "inverse_harmonic_mean_rank": 0.1787329331788256, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1710.3366262901154, + "variance": 2925251.375229454, + "z_arithmetic_mean_rank": 382.5349899284484, + "z_geometric_mean_rank": 240.96224413000627, + "z_hits_at_k": 2859.8981271075604, + "z_inverse_harmonic_mean_rank": 4111.821652168115 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.09076013401690743, + "adjusted_arithmetic_mean_rank_index": 0.9093626374240282, + "adjusted_geometric_mean_rank_index": 0.9897888874690414, + "adjusted_hits_at_k": 0.3060940531920783, + "adjusted_inverse_harmonic_mean_rank": 0.17816730739917327, + "arithmetic_mean_rank": 672.2563017385858, + "count": 59071.0, + "geometric_mean_rank": 56.613007782720985, + "harmonic_mean_rank": 5.594939821280028, + "hits_at_1": 0.11210238526518934, + "hits_at_10": 0.3065632882463476, + "hits_at_3": 0.19720336544158723, + "hits_at_5": 0.24118433749217044, + "inverse_arithmetic_mean_rank": 0.0014875278928792562, + "inverse_geometric_mean_rank": 0.017663785041027495, + "inverse_harmonic_mean_rank": 0.17873293224648426, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1710.3366270097567, + "variance": 2925251.377691112, + "z_arithmetic_mean_rank": 382.53498319749394, + "z_geometric_mean_rank": 240.96224345808997, + "z_hits_at_k": 2859.8981271075604, + "z_inverse_harmonic_mean_rank": 4111.821630636322 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.0907601237356731, + "adjusted_arithmetic_mean_rank_index": 0.9093626477066508, + "adjusted_geometric_mean_rank_index": 0.9897888890885413, + "adjusted_hits_at_k": 0.3060940531920783, + "adjusted_inverse_harmonic_mean_rank": 0.17816731676754022, + "arithmetic_mean_rank": 672.2562255859375, + "count": 59071.0, + "geometric_mean_rank": 56.612998962402344, + "harmonic_mean_rank": 5.594939528220595, + "hits_at_1": 0.11210238526518934, + "hits_at_10": 0.3065632882463476, + "hits_at_3": 0.19720336544158723, + "hits_at_5": 0.24118433749217044, + "inverse_arithmetic_mean_rank": 0.001487528090365231, + "inverse_geometric_mean_rank": 0.01766378805041313, + "inverse_harmonic_mean_rank": 0.17873294160840345, + "inverse_median_rank": 0.01666666753590107, + "median_absolute_deviation": 87.47353089183052, + "median_rank": 60.0, + "standard_deviation": 1710.336669921875, + "variance": 2925251.5, + "z_arithmetic_mean_rank": 382.53498752301016, + "z_geometric_mean_rank": 240.96224385235422, + "z_hits_at_k": 2859.8981271075604, + "z_inverse_harmonic_mean_rank": 4111.821846843517 + } + } + }, + "times": { + "evaluation": 144.05696630477905, + "training": 7.538693904876709 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json new file mode 100644 index 0000000..7a7a296 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K-237 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json new file mode 100644 index 0000000..5574951 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 512, + "slice_size": null + }, + "losses": [ + 0.0007930459803240636, + 0.0007622064488433046, + 0.0007567433983401318, + 0.0007538720628881989, + 0.0007519833328350528 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9998827472540538, + "adjusted_arithmetic_mean_rank_index": 0.00011726918016985444, + "adjusted_geometric_mean_rank_index": -0.04594329569284983, + "adjusted_hits_at_k": -0.0004586043346634101, + "adjusted_inverse_harmonic_mean_rank": -0.00020229649213419107, + "arithmetic_mean_rank": 7134.832273216557, + "count": 40876.0, + "geometric_mean_rank": 5485.375721194342, + "harmonic_mean_rank": 1958.8929986398573, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0002446423329092866, + "hits_at_3": 9.785693316371466e-05, + "hits_at_5": 0.00017124963303650063, + "inverse_arithmetic_mean_rank": 0.00014015746435328263, + "inverse_geometric_mean_rank": 0.00018230291794529399, + "inverse_harmonic_mean_rank": 0.0005104924060141838, + "inverse_median_rank": 0.00014019346698443852, + "median_absolute_deviation": 4894.069923286992, + "median_rank": 7133.0, + "standard_deviation": 3938.004768056505, + "variance": 15507881.553235766, + "z_arithmetic_mean_rank": 0.04101296097667273, + "z_geometric_mean_rank": -9.304707280866893, + "z_hits_at_k": -3.4959561975327444, + "z_inverse_harmonic_mean_rank": -3.8093653580044204 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9998827781100396, + "adjusted_arithmetic_mean_rank_index": 0.0001172383198593252, + "adjusted_geometric_mean_rank_index": -0.04594334860250249, + "adjusted_hits_at_k": -0.0004586043346634101, + "adjusted_inverse_harmonic_mean_rank": -0.00020229651184680824, + "arithmetic_mean_rank": 7134.832493394657, + "count": 40876.0, + "geometric_mean_rank": 5485.375998624691, + "harmonic_mean_rank": 1958.8930742284265, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0002446423329092866, + "hits_at_3": 9.785693316371466e-05, + "hits_at_5": 0.00017124963303650063, + "inverse_arithmetic_mean_rank": 0.00014015746002807888, + "inverse_geometric_mean_rank": 0.00018230290872507605, + "inverse_harmonic_mean_rank": 0.0005104923863156147, + "inverse_median_rank": 0.00014019346698443852, + "median_absolute_deviation": 4894.069923286992, + "median_rank": 7133.0, + "standard_deviation": 3938.004705572819, + "variance": 15507881.061113667, + "z_arithmetic_mean_rank": 0.04100216809217727, + "z_geometric_mean_rank": -9.304717996441866, + "z_hits_at_k": -3.4959561975327444, + "z_inverse_harmonic_mean_rank": -3.8093657292049308 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9998827133446111, + "adjusted_arithmetic_mean_rank_index": 0.00011730309436530639, + "adjusted_geometric_mean_rank_index": -0.04594287878634984, + "adjusted_hits_at_k": -0.0004586043346634101, + "adjusted_inverse_harmonic_mean_rank": -0.00020229652761597373, + "arithmetic_mean_rank": 7134.83203125, + "count": 40876.0, + "geometric_mean_rank": 5485.37353515625, + "harmonic_mean_rank": 1958.8931346957265, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0002446423329092866, + "hits_at_3": 9.785693316371466e-05, + "hits_at_5": 0.00017124963303650063, + "inverse_arithmetic_mean_rank": 0.0001401574700139463, + "inverse_geometric_mean_rank": 0.00018230298883281648, + "inverse_harmonic_mean_rank": 0.000510492370557687, + "inverse_median_rank": 0.00014019347145222127, + "median_absolute_deviation": 4894.069923286992, + "median_rank": 7133.0, + "standard_deviation": 3938.0048828125, + "variance": 15507882.0, + "z_arithmetic_mean_rank": 0.04102482190694445, + "z_geometric_mean_rank": -9.304622846503037, + "z_hits_at_k": -3.4959561975327444, + "z_inverse_harmonic_mean_rank": -3.809366026147858 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9989789751387469, + "adjusted_arithmetic_mean_rank_index": 0.0010211699831054188, + "adjusted_geometric_mean_rank_index": -0.029203457964425805, + "adjusted_hits_at_k": -0.00042138983804789365, + "adjusted_inverse_harmonic_mean_rank": -0.00014306071821701925, + "arithmetic_mean_rank": 7029.453713670614, + "count": 20438.0, + "geometric_mean_rank": 5316.3532096388135, + "harmonic_mean_rank": 1723.253977787341, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.00029357079949114397, + "hits_at_3": 0.00014678539974557198, + "hits_at_5": 0.0002446423329092866, + "inverse_arithmetic_mean_rank": 0.00014225856527872685, + "inverse_geometric_mean_rank": 0.00018809886412117052, + "inverse_harmonic_mean_rank": 0.000580297514405857, + "inverse_median_rank": 0.00014240956992309884, + "median_absolute_deviation": 4910.378547690553, + "median_rank": 7022.0, + "standard_deviation": 3964.421908029607, + "variance": 15716641.06486511, + "z_arithmetic_mean_rank": 0.25226596239463106, + "z_geometric_mean_rank": -4.182357960676967, + "z_hits_at_k": -2.2526801345004044, + "z_inverse_harmonic_mean_rank": -1.8892304206305193 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9989790029522932, + "adjusted_arithmetic_mean_rank_index": 0.0010211421656057817, + "adjusted_geometric_mean_rank_index": -0.02920352481459032, + "adjusted_hits_at_k": -0.00042138983804789365, + "adjusted_inverse_harmonic_mean_rank": -0.00014306075211986604, + "arithmetic_mean_rank": 7029.45390938448, + "count": 20438.0, + "geometric_mean_rank": 5316.353554888564, + "harmonic_mean_rank": 1723.25407839257, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.00029357079949114397, + "hits_at_3": 0.00014678539974557198, + "hits_at_5": 0.0002446423329092866, + "inverse_arithmetic_mean_rank": 0.00014225856131796773, + "inverse_geometric_mean_rank": 0.00018809885190582682, + "inverse_harmonic_mean_rank": 0.0005802974805275305, + "inverse_median_rank": 0.00014240956992309884, + "median_absolute_deviation": 4910.378547690553, + "median_rank": 7022.0, + "standard_deviation": 3964.4218002748275, + "variance": 15716640.210494306, + "z_arithmetic_mean_rank": 0.2522590904649462, + "z_geometric_mean_rank": -4.182367534588337, + "z_hits_at_k": -2.2526801345004044, + "z_inverse_harmonic_mean_rank": -1.8892308683445416 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9989789608720815, + "adjusted_arithmetic_mean_rank_index": 0.0010211842517985348, + "adjusted_geometric_mean_rank_index": -0.029203233576663656, + "adjusted_hits_at_k": -0.00042138983804789365, + "adjusted_inverse_harmonic_mean_rank": -0.000143060756398357, + "arithmetic_mean_rank": 7029.45361328125, + "count": 20438.0, + "geometric_mean_rank": 5316.35205078125, + "harmonic_mean_rank": 1723.2540910888076, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.00029357079949114397, + "hits_at_3": 0.00014678539974557198, + "hits_at_5": 0.0002446423329092866, + "inverse_arithmetic_mean_rank": 0.00014225856284610927, + "inverse_geometric_mean_rank": 0.00018809890025295317, + "inverse_harmonic_mean_rank": 0.000580297476252134, + "inverse_median_rank": 0.00014240956807043403, + "median_absolute_deviation": 4910.378547690553, + "median_rank": 7022.0, + "standard_deviation": 3964.421875, + "variance": 15716641.0, + "z_arithmetic_mean_rank": 0.25226948727851095, + "z_geometric_mean_rank": -4.182325825100975, + "z_hits_at_k": -2.2526801345004044, + "z_inverse_harmonic_mean_rank": -1.889230924845414 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.000761777182881, + "adjusted_arithmetic_mean_rank_index": -0.0007618824923663059, + "adjusted_geometric_mean_rank_index": -0.06292935156569657, + "adjusted_hits_at_k": -0.0004958179572368788, + "adjusted_inverse_harmonic_mean_rank": -0.00026153100818154026, + "arithmetic_mean_rank": 7240.210832762501, + "count": 20438.0, + "geometric_mean_rank": 5659.771955729911, + "harmonic_mean_rank": 2269.1827184376725, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0001957138663274293, + "hits_at_3": 4.892846658185733e-05, + "hits_at_5": 9.785693316371466e-05, + "inverse_arithmetic_mean_rank": 0.0001381175249034081, + "inverse_geometric_mean_rank": 0.00017668556398064899, + "inverse_harmonic_mean_rank": 0.0004406872976225105, + "inverse_median_rank": 0.00013803575125957622, + "median_absolute_deviation": 4854.039663387341, + "median_rank": 7244.5, + "standard_deviation": 3908.5691448383595, + "variance": 15276912.759982465, + "z_arithmetic_mean_rank": -0.18863809493518602, + "z_geometric_mean_rank": -9.011995783715662, + "z_hits_at_k": -2.695211599906065, + "z_inverse_harmonic_mean_rank": -3.5116985829733194 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0007618109980143, + "adjusted_arithmetic_mean_rank_index": -0.0007619163121743711, + "adjusted_geometric_mean_rank_index": -0.06292939006290288, + "adjusted_hits_at_k": -0.0004958179572368788, + "adjusted_inverse_harmonic_mean_rank": -0.00026153101370422917, + "arithmetic_mean_rank": 7240.211077404834, + "count": 20438.0, + "geometric_mean_rank": 5659.7721606794785, + "harmonic_mean_rank": 2269.1827468550846, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0001957138663274293, + "hits_at_3": 4.892846658185733e-05, + "hits_at_5": 9.785693316371466e-05, + "inverse_arithmetic_mean_rank": 0.00013811752023650087, + "inverse_geometric_mean_rank": 0.00017668555758257695, + "inverse_harmonic_mean_rank": 0.0004406872921036987, + "inverse_median_rank": 0.00013803575125957622, + "median_absolute_deviation": 4854.039663387341, + "median_rank": 7244.5, + "standard_deviation": 3908.5691269052727, + "variance": 15276912.619797047, + "z_arithmetic_mean_rank": -0.1886464685416911, + "z_geometric_mean_rank": -9.012001296829226, + "z_hits_at_k": -2.695211599906065, + "z_inverse_harmonic_mean_rank": -3.511698657129034 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0007617241684128, + "adjusted_arithmetic_mean_rank_index": -0.0007618294705693529, + "adjusted_geometric_mean_rank_index": -0.06292953818011693, + "adjusted_hits_at_k": -0.0004958179572368788, + "adjusted_inverse_harmonic_mean_rank": -0.0002615309942692217, + "arithmetic_mean_rank": 7240.21044921875, + "count": 20438.0, + "geometric_mean_rank": 5659.77294921875, + "harmonic_mean_rank": 2269.182646850792, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.0001957138663274293, + "hits_at_3": 4.892846658185733e-05, + "hits_at_5": 9.785693316371466e-05, + "inverse_arithmetic_mean_rank": 0.00013811753888148814, + "inverse_geometric_mean_rank": 0.00017668552754912525, + "inverse_harmonic_mean_rank": 0.00044068731152506214, + "inverse_median_rank": 0.00013803575711790472, + "median_absolute_deviation": 4854.039663387341, + "median_rank": 7244.5, + "standard_deviation": 3908.569091796875, + "variance": 15276912.0, + "z_arithmetic_mean_rank": -0.1886249670173754, + "z_geometric_mean_rank": -9.012022508420841, + "z_hits_at_k": -2.695211599906065, + "z_inverse_harmonic_mean_rank": -3.5116983961661417 + } + } + }, + "times": { + "evaluation": 31.55549693107605, + "training": 3.1638309955596924 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json new file mode 100644 index 0000000..8801287 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json new file mode 100644 index 0000000..d28be8e --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 64, + "slice_size": null + }, + "losses": [ + 0.0003714370767319881, + 0.00035896927278014634, + 0.00035071555033651267, + 0.0003477621583505325, + 0.0003448637009772764 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7635239113975782, + "adjusted_arithmetic_mean_rank_index": 0.23648764531221989, + "adjusted_geometric_mean_rank_index": 0.5844441843224377, + "adjusted_hits_at_k": 0.013458941400830942, + "adjusted_inverse_harmonic_mean_rank": 0.007385369297529207, + "arithmetic_mean_rank": 15624.1677, + "count": 10000.0, + "geometric_mean_rank": 6258.298951492494, + "harmonic_mean_rank": 130.6005122071634, + "hits_at_1": 0.0036, + "hits_at_10": 0.0137, + "hits_at_3": 0.0074, + "hits_at_5": 0.0098, + "inverse_arithmetic_mean_rank": 6.400340928240292e-05, + "inverse_geometric_mean_rank": 0.00015978782856985725, + "inverse_harmonic_mean_rank": 0.007656937810578895, + "inverse_median_rank": 9.570293808019906e-05, + "median_absolute_deviation": 13722.966134487851, + "median_rank": 10449.0, + "standard_deviation": 14467.476928558646, + "variance": 209307888.6783767, + "z_arithmetic_mean_rank": 40.9598095616972, + "z_geometric_mean_rank": 58.49168969786516, + "z_hits_at_k": 86.09027690476483, + "z_inverse_harmonic_mean_rank": 116.56908781774791 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7635239358316449, + "adjusted_arithmetic_mean_rank_index": 0.23648762087695907, + "adjusted_geometric_mean_rank_index": 0.5844441164415516, + "adjusted_hits_at_k": 0.013458941400830942, + "adjusted_inverse_harmonic_mean_rank": 0.007385369198125054, + "arithmetic_mean_rank": 15624.1682, + "count": 10000.0, + "geometric_mean_rank": 6258.2999736199245, + "harmonic_mean_rank": 130.6005139021859, + "hits_at_1": 0.0036, + "hits_at_10": 0.0137, + "hits_at_3": 0.0074, + "hits_at_5": 0.0098, + "inverse_arithmetic_mean_rank": 6.400340723418479e-05, + "inverse_geometric_mean_rank": 0.0001597878024727505, + "inverse_harmonic_mean_rank": 0.007656937711201937, + "inverse_median_rank": 9.570293808019906e-05, + "median_absolute_deviation": 13722.966134487851, + "median_rank": 10449.0, + "standard_deviation": 14467.47674622319, + "variance": 209307883.40250874, + "z_arithmetic_mean_rank": 40.95980532949457, + "z_geometric_mean_rank": 58.49168290428617, + "z_hits_at_k": 86.09027690476483, + "z_inverse_harmonic_mean_rank": 116.56908624877403 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7635239245308891, + "adjusted_arithmetic_mean_rank_index": 0.23648763217826718, + "adjusted_geometric_mean_rank_index": 0.5844440952331575, + "adjusted_hits_at_k": 0.013458941400830942, + "adjusted_inverse_harmonic_mean_rank": 0.007385369658435771, + "arithmetic_mean_rank": 15624.16796875, + "count": 10000.0, + "geometric_mean_rank": 6258.30029296875, + "harmonic_mean_rank": 130.6005060530473, + "hits_at_1": 0.0036, + "hits_at_10": 0.0137, + "hits_at_3": 0.0074, + "hits_at_5": 0.0098, + "inverse_arithmetic_mean_rank": 6.400340498657897e-05, + "inverse_geometric_mean_rank": 0.00015978779993019998, + "inverse_harmonic_mean_rank": 0.0076569381713867185, + "inverse_median_rank": 9.570294059813023e-05, + "median_absolute_deviation": 13722.966134487851, + "median_rank": 10449.0, + "standard_deviation": 14467.4765625, + "variance": 209307888.0, + "z_arithmetic_mean_rank": 40.95980728688829, + "z_geometric_mean_rank": 58.49168078173138, + "z_hits_at_k": 86.09027690476483, + "z_inverse_harmonic_mean_rank": 116.56909351421984 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7621333909495468, + "adjusted_arithmetic_mean_rank_index": 0.23787823398522556, + "adjusted_geometric_mean_rank_index": 0.5880241496733576, + "adjusted_hits_at_k": 0.01435915574139819, + "adjusted_inverse_harmonic_mean_rank": 0.007780013358221885, + "arithmetic_mean_rank": 15595.3516, + "count": 5000.0, + "geometric_mean_rank": 6204.558347786061, + "harmonic_mean_rank": 124.20077311450835, + "hits_at_1": 0.0036, + "hits_at_10": 0.0146, + "hits_at_3": 0.0088, + "hits_at_5": 0.0114, + "inverse_arithmetic_mean_rank": 6.412167071629215e-05, + "inverse_geometric_mean_rank": 0.0001611718262520369, + "inverse_harmonic_mean_rank": 0.00805147967217594, + "inverse_median_rank": 9.573042312847023e-05, + "median_absolute_deviation": 13729.637844471126, + "median_rank": 10446.0, + "standard_deviation": 14485.4557773229, + "variance": 209828429.07677743, + "z_arithmetic_mean_rank": 29.133263899145902, + "z_geometric_mean_rank": 41.61629657579756, + "z_hits_at_k": 64.94594468981366, + "z_inverse_harmonic_mean_rank": 86.83033180551921 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7621334398188131, + "adjusted_arithmetic_mean_rank_index": 0.23787818511357084, + "adjusted_geometric_mean_rank_index": 0.5880240150809598, + "adjusted_hits_at_k": 0.01435915574139819, + "adjusted_inverse_harmonic_mean_rank": 0.007780013159413577, + "arithmetic_mean_rank": 15595.3526, + "count": 5000.0, + "geometric_mean_rank": 6204.560374486879, + "harmonic_mean_rank": 124.20077618045295, + "hits_at_1": 0.0036, + "hits_at_10": 0.0146, + "hits_at_3": 0.0088, + "hits_at_5": 0.0114, + "inverse_arithmetic_mean_rank": 6.412166660470376e-05, + "inverse_geometric_mean_rank": 0.00016117177360574892, + "inverse_harmonic_mean_rank": 0.008051479473422024, + "inverse_median_rank": 9.573042312847023e-05, + "median_absolute_deviation": 13729.637844471126, + "median_rank": 10446.0, + "standard_deviation": 14485.455415093902, + "variance": 209828418.58267325, + "z_arithmetic_mean_rank": 29.133257913769313, + "z_geometric_mean_rank": 41.616287050275275, + "z_hits_at_k": 64.94594468981366, + "z_inverse_harmonic_mean_rank": 86.83032958668093 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7621333891169493, + "adjusted_arithmetic_mean_rank_index": 0.23787823581791256, + "adjusted_geometric_mean_rank_index": 0.588024003632733, + "adjusted_hits_at_k": 0.01435915574139819, + "adjusted_inverse_harmonic_mean_rank": 0.007780013788702784, + "arithmetic_mean_rank": 15595.3515625, + "count": 5000.0, + "geometric_mean_rank": 6204.560546875, + "harmonic_mean_rank": 124.20076647579945, + "hits_at_1": 0.0036, + "hits_at_10": 0.0146, + "hits_at_3": 0.0088, + "hits_at_5": 0.0114, + "inverse_arithmetic_mean_rank": 6.412166840163991e-05, + "inverse_geometric_mean_rank": 0.00016117177437990904, + "inverse_harmonic_mean_rank": 0.008051480102539062, + "inverse_median_rank": 9.5730421890039e-05, + "median_absolute_deviation": 13729.637844471126, + "median_rank": 10446.0, + "standard_deviation": 14485.455078125, + "variance": 209828416.0, + "z_arithmetic_mean_rank": 29.133264123597513, + "z_geometric_mean_rank": 41.61628624004869, + "z_hits_at_k": 64.94594468981366, + "z_inverse_harmonic_mean_rank": 86.83033660998389 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7649143673741152, + "adjusted_arithmetic_mean_rank_index": 0.23509712111702274, + "adjusted_geometric_mean_rank_index": 0.5808749569746401, + "adjusted_hits_at_k": 0.012558727070539942, + "adjusted_inverse_harmonic_mean_rank": 0.006990725241428644, + "arithmetic_mean_rank": 15652.9838, + "count": 5000.0, + "geometric_mean_rank": 6312.505027892547, + "harmonic_mean_rank": 137.69560445684527, + "hits_at_1": 0.0036, + "hits_at_10": 0.0128, + "hits_at_3": 0.006, + "hits_at_5": 0.0082, + "inverse_arithmetic_mean_rank": 6.388558327135048e-05, + "inverse_geometric_mean_rank": 0.00015841571540638498, + "inverse_harmonic_mean_rank": 0.007262395948981848, + "inverse_median_rank": 9.563886763580719e-05, + "median_absolute_deviation": 13704.433606756531, + "median_rank": 10456.0, + "standard_deviation": 14449.418242432375, + "variance": 208785687.54473752, + "z_arithmetic_mean_rank": 28.79266217842425, + "z_geometric_mean_rank": 41.11032485001479, + "z_hits_at_k": 56.803997459095015, + "z_inverse_harmonic_mean_rank": 78.02315047356781 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7649143673741152, + "adjusted_arithmetic_mean_rank_index": 0.23509712111702274, + "adjusted_geometric_mean_rank_index": 0.5808749569746401, + "adjusted_hits_at_k": 0.012558727070539942, + "adjusted_inverse_harmonic_mean_rank": 0.006990725241428644, + "arithmetic_mean_rank": 15652.9838, + "count": 5000.0, + "geometric_mean_rank": 6312.505027892547, + "harmonic_mean_rank": 137.69560445684527, + "hits_at_1": 0.0036, + "hits_at_10": 0.0128, + "hits_at_3": 0.006, + "hits_at_5": 0.0082, + "inverse_arithmetic_mean_rank": 6.388558327135048e-05, + "inverse_geometric_mean_rank": 0.00015841571540638498, + "inverse_harmonic_mean_rank": 0.007262395948981848, + "inverse_median_rank": 9.563886763580719e-05, + "median_absolute_deviation": 13704.433606756531, + "median_rank": 10456.0, + "standard_deviation": 14449.418242432375, + "variance": 208785687.54473752, + "z_arithmetic_mean_rank": 28.79266217842425, + "z_geometric_mean_rank": 41.11032485001479, + "z_hits_at_k": 56.803997459095015, + "z_inverse_harmonic_mean_rank": 78.02315047356781 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7649143000292802, + "adjusted_arithmetic_mean_rank_index": 0.23509718846514893, + "adjusted_geometric_mean_rank_index": 0.5808749017587895, + "adjusted_hits_at_k": 0.012558727070539942, + "adjusted_inverse_harmonic_mean_rank": 0.006990725532760876, + "arithmetic_mean_rank": 15652.982421875, + "count": 5000.0, + "geometric_mean_rank": 6312.505859375, + "harmonic_mean_rank": 137.69559893467442, + "hits_at_1": 0.0036, + "hits_at_10": 0.0128, + "hits_at_3": 0.006, + "hits_at_5": 0.0082, + "inverse_arithmetic_mean_rank": 6.38855854049325e-05, + "inverse_geometric_mean_rank": 0.00015841569984331727, + "inverse_harmonic_mean_rank": 0.007262396240234376, + "inverse_median_rank": 9.563886851537973e-05, + "median_absolute_deviation": 13704.433606756531, + "median_rank": 10456.0, + "standard_deviation": 14449.4189453125, + "variance": 208785696.0, + "z_arithmetic_mean_rank": 28.7926704266403, + "z_geometric_mean_rank": 41.110320942217555, + "z_hits_at_k": 56.803997459095015, + "z_inverse_harmonic_mean_rank": 78.02315372511293 + } + } + }, + "times": { + "evaluation": 49.746681928634644, + "training": 8.512414455413818 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json new file mode 100644 index 0000000..8eb29b5 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18RR Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json new file mode 100644 index 0000000..175356f --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 128, + "slice_size": null + }, + "losses": [ + 0.0003800833376947446, + 0.00037256451326309726, + 0.0003562385337013802, + 0.000347856839113493, + 0.0003418508737124453 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0166994688460609, + "adjusted_arithmetic_mean_rank_index": -0.01670029263459827, + "adjusted_geometric_mean_rank_index": 0.22735517405740358, + "adjusted_hits_at_k": 0.01155511005618705, + "adjusted_inverse_harmonic_mean_rank": 0.006916144054771029, + "arithmetic_mean_rank": 20611.088748290014, + "count": 5848.0, + "geometric_mean_rank": 11527.227772272285, + "harmonic_mean_rank": 139.07882417125074, + "hits_at_1": 0.004274965800273598, + "hits_at_10": 0.01179890560875513, + "hits_at_3": 0.0070109439124487, + "hits_at_5": 0.008891928864569083, + "inverse_arithmetic_mean_rank": 4.851757285664807e-05, + "inverse_geometric_mean_rank": 8.675112696266926e-05, + "inverse_harmonic_mean_rank": 0.007190167201648746, + "inverse_median_rank": 4.828585224529213e-05, + "median_absolute_deviation": 17863.132829664744, + "median_rank": 20710.0, + "standard_deviation": 13176.818342055025, + "variance": 173628541.6195177, + "z_arithmetic_mean_rank": -2.21195971632244, + "z_geometric_mean_rank": 17.401431565374203, + "z_hits_at_k": 56.25842171670579, + "z_inverse_harmonic_mean_rank": 83.08987517740859 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0166994857160303, + "adjusted_arithmetic_mean_rank_index": -0.016700309505399913, + "adjusted_geometric_mean_rank_index": 0.2273551670159204, + "adjusted_hits_at_k": 0.01155511005618705, + "adjusted_inverse_harmonic_mean_rank": 0.0069161440545278995, + "arithmetic_mean_rank": 20611.089090287278, + "count": 5848.0, + "geometric_mean_rank": 11527.22787731633, + "harmonic_mean_rank": 139.07882417595226, + "hits_at_1": 0.004274965800273598, + "hits_at_10": 0.01179890560875513, + "hits_at_3": 0.0070109439124487, + "hits_at_5": 0.008891928864569083, + "inverse_arithmetic_mean_rank": 4.851757205160195e-05, + "inverse_geometric_mean_rank": 8.675112617213319e-05, + "inverse_harmonic_mean_rank": 0.007190167201405685, + "inverse_median_rank": 4.828585224529213e-05, + "median_absolute_deviation": 17863.132829664744, + "median_rank": 20710.0, + "standard_deviation": 13176.818782175542, + "variance": 173628553.2182941, + "z_arithmetic_mean_rank": -2.21196195086614, + "z_geometric_mean_rank": 17.401431026429407, + "z_hits_at_k": 56.25842171670579, + "z_inverse_harmonic_mean_rank": 83.08987517448767 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0166995228826816, + "adjusted_arithmetic_mean_rank_index": -0.016700346673884647, + "adjusted_geometric_mean_rank_index": 0.22735558246544507, + "adjusted_hits_at_k": 0.01155511005618705, + "adjusted_inverse_harmonic_mean_rank": 0.00691614434941122, + "arithmetic_mean_rank": 20611.08984375, + "count": 5848.0, + "geometric_mean_rank": 11527.2216796875, + "harmonic_mean_rank": 139.0788184736221, + "hits_at_1": 0.004274965800273598, + "hits_at_10": 0.01179890560875513, + "hits_at_3": 0.0070109439124487, + "hits_at_5": 0.008891928864569083, + "inverse_arithmetic_mean_rank": 4.851756966672838e-05, + "inverse_geometric_mean_rank": 8.675116987433285e-05, + "inverse_harmonic_mean_rank": 0.007190167496207637, + "inverse_median_rank": 4.828585224458948e-05, + "median_absolute_deviation": 17863.132829664744, + "median_rank": 20710.0, + "standard_deviation": 13176.8193359375, + "variance": 173628560.0, + "z_arithmetic_mean_rank": -2.211966873845199, + "z_geometric_mean_rank": 17.4014628243267, + "z_hits_at_k": 56.25842171670579, + "z_inverse_harmonic_mean_rank": 83.0898787171869 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0044911327331056, + "adjusted_arithmetic_mean_rank_index": -0.004491354330853348, + "adjusted_geometric_mean_rank_index": 0.2916570535589309, + "adjusted_hits_at_k": 0.014462749736302826, + "adjusted_inverse_harmonic_mean_rank": 0.009295211355255004, + "arithmetic_mean_rank": 20359.076949384405, + "count": 2924.0, + "geometric_mean_rank": 10566.530983280216, + "harmonic_mean_rank": 104.50813006584458, + "hits_at_1": 0.006155950752393981, + "hits_at_10": 0.014705882352941176, + "hits_at_3": 0.009575923392612859, + "hits_at_5": 0.012311901504787962, + "inverse_arithmetic_mean_rank": 4.9118140399299235e-05, + "inverse_geometric_mean_rank": 9.463843919847812e-05, + "inverse_harmonic_mean_rank": 0.009568633553867601, + "inverse_median_rank": 4.81278275098662e-05, + "median_absolute_deviation": 19138.912038688817, + "median_rank": 20778.0, + "standard_deviation": 13625.52007285251, + "variance": 185654797.2557067, + "z_arithmetic_mean_rank": -0.4206444882002103, + "z_geometric_mean_rank": 15.786747812228626, + "z_hits_at_k": 49.785272491856105, + "z_inverse_harmonic_mean_rank": 78.95506001715788 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0044911496068185, + "adjusted_arithmetic_mean_rank_index": -0.004491371205398886, + "adjusted_geometric_mean_rank_index": 0.29165704689618044, + "adjusted_hits_at_k": 0.014462749736302826, + "adjusted_inverse_harmonic_mean_rank": 0.009295211354996282, + "arithmetic_mean_rank": 20359.07729138167, + "count": 2924.0, + "geometric_mean_rank": 10566.531082660744, + "harmonic_mean_rank": 104.50813006866956, + "hits_at_1": 0.006155950752393981, + "hits_at_10": 0.014705882352941176, + "hits_at_3": 0.009575923392612859, + "hits_at_5": 0.012311901504787962, + "inverse_arithmetic_mean_rank": 4.911813957419948e-05, + "inverse_geometric_mean_rank": 9.463843830838298e-05, + "inverse_harmonic_mean_rank": 0.00956863355360895, + "inverse_median_rank": 4.81278275098662e-05, + "median_absolute_deviation": 19138.912038688817, + "median_rank": 20778.0, + "standard_deviation": 13625.520474534556, + "variance": 185654808.20196038, + "z_arithmetic_mean_rank": -0.420646068610921, + "z_geometric_mean_rank": 15.78674745158876, + "z_hits_at_k": 49.785272491856105, + "z_inverse_harmonic_mean_rank": 78.95506001496025 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.004491094371774, + "adjusted_arithmetic_mean_rank_index": -0.004491315967628973, + "adjusted_geometric_mean_rank_index": 0.29165651190528796, + "adjusted_hits_at_k": 0.014462749736302826, + "adjusted_inverse_harmonic_mean_rank": 0.009295210999538096, + "arithmetic_mean_rank": 20359.076171875, + "count": 2924.0, + "geometric_mean_rank": 10566.5390625, + "harmonic_mean_rank": 104.5081339498945, + "hits_at_1": 0.006155950752393981, + "hits_at_10": 0.014705882352941176, + "hits_at_3": 0.009575923392612859, + "hits_at_5": 0.012311901504787962, + "inverse_arithmetic_mean_rank": 4.9118141760118306e-05, + "inverse_geometric_mean_rank": 9.463836613576859e-05, + "inverse_harmonic_mean_rank": 0.009568633198248867, + "inverse_median_rank": 4.812782572116703e-05, + "median_absolute_deviation": 19138.912038688817, + "median_rank": 20778.0, + "standard_deviation": 13625.5205078125, + "variance": 185654800.0, + "z_arithmetic_mean_rank": -0.4206408952352092, + "z_geometric_mean_rank": 15.786718493721295, + "z_hits_at_k": 49.785272491856105, + "z_inverse_harmonic_mean_rank": 78.95505699563974 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.028902389254621, + "adjusted_arithmetic_mean_rank_index": -0.028903814699699826, + "adjusted_geometric_mean_rank_index": 0.15736049054991996, + "adjusted_hits_at_k": 0.00864747069608476, + "adjusted_inverse_harmonic_mean_rank": 0.004537077020969589, + "arithmetic_mean_rank": 20863.100547195623, + "count": 2924.0, + "geometric_mean_rank": 12575.27000338155, + "harmonic_mean_rank": 207.82671892798226, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.008891928864569083, + "hits_at_3": 0.004445964432284542, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 4.7931514193580305e-05, + "inverse_geometric_mean_rank": 7.952115538919603e-05, + "inverse_harmonic_mean_rank": 0.004811700849429894, + "inverse_median_rank": 4.8497781226508885e-05, + "median_absolute_deviation": 16431.680387697586, + "median_rank": 20619.5, + "standard_deviation": 12707.291847193621, + "variance": 161475266.08975345, + "z_arithmetic_mean_rank": -2.7070331259667455, + "z_geometric_mean_rank": 8.517571193494355, + "z_hits_at_k": 29.773919702326012, + "z_inverse_harmonic_mean_rank": 38.54726950873501 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0289024061208485, + "adjusted_arithmetic_mean_rank_index": -0.02890383156675913, + "adjusted_geometric_mean_rank_index": 0.1573604831171146, + "adjusted_hits_at_k": 0.00864747069608476, + "adjusted_inverse_harmonic_mean_rank": 0.004537077020742053, + "arithmetic_mean_rank": 20863.100889192887, + "count": 2924.0, + "geometric_mean_rank": 12575.27011429743, + "harmonic_mean_rank": 207.82671893780727, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.008891928864569083, + "hits_at_3": 0.004445964432284542, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 4.793151340786552e-05, + "inverse_geometric_mean_rank": 7.952115468780682e-05, + "inverse_harmonic_mean_rank": 0.0048117008492024205, + "inverse_median_rank": 4.8497781226508885e-05, + "median_absolute_deviation": 16431.680387697586, + "median_rank": 20619.5, + "standard_deviation": 12707.292329251442, + "variance": 161475278.34105256, + "z_arithmetic_mean_rank": -2.7070347056782538, + "z_geometric_mean_rank": 8.517570791173235, + "z_hits_at_k": 29.773919702326012, + "z_inverse_harmonic_mean_rank": 38.547269506801854 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0289023430042623, + "adjusted_arithmetic_mean_rank_index": -0.02890376844706033, + "adjusted_geometric_mean_rank_index": 0.15736052218886976, + "adjusted_hits_at_k": 0.00864747069608476, + "adjusted_inverse_harmonic_mean_rank": 0.004537077313478786, + "arithmetic_mean_rank": 20863.099609375, + "count": 2924.0, + "geometric_mean_rank": 12575.26953125, + "harmonic_mean_rank": 207.82670629742728, + "hits_at_1": 0.0023939808481532147, + "hits_at_10": 0.008891928864569083, + "hits_at_3": 0.004445964432284542, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 4.793151674675755e-05, + "inverse_geometric_mean_rank": 7.952115993248299e-05, + "inverse_harmonic_mean_rank": 0.004811701141858395, + "inverse_median_rank": 4.8497782699996606e-05, + "median_absolute_deviation": 16431.680387697586, + "median_rank": 20619.5, + "standard_deviation": 12707.2919921875, + "variance": 161475264.0, + "z_arithmetic_mean_rank": -2.7070287941016034, + "z_geometric_mean_rank": 8.51757290603991, + "z_hits_at_k": 29.773919702326012, + "z_inverse_harmonic_mean_rank": 38.547271993910115 + } + } + }, + "times": { + "evaluation": 13.313127517700195, + "training": 2.919541358947754 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json new file mode 100644 index 0000000..fafc4ec --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json new file mode 100644 index 0000000..c271294 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.0029586357451772804, + 0.0027738664607501995, + 0.002592642678937381, + 0.0024149229981592704, + 0.0022405725565874963 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.46055182855559323, + "adjusted_arithmetic_mean_rank_index": 0.5395212094828172, + "adjusted_geometric_mean_rank_index": 0.8576850966163081, + "adjusted_hits_at_k": 0.10618046742846822, + "adjusted_inverse_harmonic_mean_rank": 0.06260713816785825, + "arithmetic_mean_rank": 3402.0283811007093, + "count": 118142.0, + "geometric_mean_rank": 773.8566421415263, + "harmonic_mean_rank": 15.80925834723783, + "hits_at_1": 0.04050210763318718, + "hits_at_10": 0.10678674815053071, + "hits_at_3": 0.06778283760220752, + "hits_at_5": 0.08225694503224933, + "inverse_arithmetic_mean_rank": 0.0002939422861829433, + "inverse_geometric_mean_rank": 0.0012922290066964567, + "inverse_harmonic_mean_rank": 0.06325407416564348, + "inverse_median_rank": 0.0005046681806712087, + "median_absolute_deviation": 2890.333024976671, + "median_rank": 1981.5, + "standard_deviation": 3716.0480418830743, + "variance": 13809013.04958303, + "z_arithmetic_mean_rank": 320.9165450339127, + "z_geometric_mean_rank": 295.3132134234969, + "z_hits_at_k": 1400.8378652422487, + "z_inverse_harmonic_mean_rank": 2040.2333364354065 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.46055483303303296, + "adjusted_arithmetic_mean_rank_index": 0.5395182045985893, + "adjusted_geometric_mean_rank_index": 0.8576832347776194, + "adjusted_hits_at_k": 0.10618046742846822, + "adjusted_inverse_harmonic_mean_rank": 0.06260632151148991, + "arithmetic_mean_rank": 3402.050574732102, + "count": 118142.0, + "geometric_mean_rank": 773.8667530600802, + "harmonic_mean_rank": 15.809462318094637, + "hits_at_1": 0.04050210763318718, + "hits_at_10": 0.10678674815053071, + "hits_at_3": 0.06778283760220752, + "hits_at_5": 0.08224848064193936, + "inverse_arithmetic_mean_rank": 0.00029394036861981276, + "inverse_geometric_mean_rank": 0.0012922121231410022, + "inverse_harmonic_mean_rank": 0.06325325807288558, + "inverse_median_rank": 0.0005045408678102926, + "median_absolute_deviation": 2891.074326085924, + "median_rank": 1982.0, + "standard_deviation": 3716.0616388245403, + "variance": 13809114.103543328, + "z_arithmetic_mean_rank": 320.9147576768122, + "z_geometric_mean_rank": 295.3125723658777, + "z_hits_at_k": 1400.8378652422487, + "z_inverse_harmonic_mean_rank": 2040.2067233431 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4605533406584393, + "adjusted_arithmetic_mean_rank_index": 0.5395196971752416, + "adjusted_geometric_mean_rank_index": 0.8576842786891514, + "adjusted_hits_at_k": 0.10618046742846822, + "adjusted_inverse_harmonic_mean_rank": 0.0626066837363943, + "arithmetic_mean_rank": 3402.03955078125, + "count": 118142.0, + "geometric_mean_rank": 773.861083984375, + "harmonic_mean_rank": 15.809371846927815, + "hits_at_1": 0.04050210763318718, + "hits_at_10": 0.10678674815053071, + "hits_at_3": 0.06778283760220752, + "hits_at_5": 0.08224848064193936, + "inverse_arithmetic_mean_rank": 0.00029394132434390485, + "inverse_geometric_mean_rank": 0.0012922215973958373, + "inverse_harmonic_mean_rank": 0.06325362004780265, + "inverse_median_rank": 0.0005046045407652855, + "median_absolute_deviation": 2890.7036755312974, + "median_rank": 1981.75, + "standard_deviation": 3716.054931640625, + "variance": 13809065.0, + "z_arithmetic_mean_rank": 320.9156454872152, + "z_geometric_mean_rank": 295.31293179950933, + "z_hits_at_k": 1400.8378652422487, + "z_inverse_harmonic_mean_rank": 2040.2185274815236 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.49774717752314473, + "adjusted_arithmetic_mean_rank_index": 0.5023210100871964, + "adjusted_geometric_mean_rank_index": 0.8015392564962838, + "adjusted_hits_at_k": 0.07273844154098844, + "adjusted_inverse_harmonic_mean_rank": 0.03726819296769626, + "arithmetic_mean_rank": 3666.7785207631496, + "count": 59071.0, + "geometric_mean_rank": 1075.6624565891095, + "harmonic_mean_rank": 26.36126849123481, + "hits_at_1": 0.0215672665097933, + "hits_at_10": 0.07336933520678506, + "hits_at_3": 0.037124815899510756, + "hits_at_5": 0.05058319649235665, + "inverse_arithmetic_mean_rank": 0.0002727189532548791, + "inverse_geometric_mean_rank": 0.0009296596658871663, + "inverse_harmonic_mean_rank": 0.03793444159686407, + "inverse_median_rank": 0.00041999160016799666, + "median_absolute_deviation": 3337.33759385611, + "median_rank": 2381.0, + "standard_deviation": 3753.600259522372, + "variance": 14089514.90828642, + "z_arithmetic_mean_rank": 211.24527583191812, + "z_geometric_mean_rank": 195.13481015090386, + "z_hits_at_k": 677.5273694704455, + "z_inverse_harmonic_mean_rank": 857.4643858375553 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4977502821195721, + "adjusted_arithmetic_mean_rank_index": 0.5023179050692781, + "adjusted_geometric_mean_rank_index": 0.8015365635075814, + "adjusted_hits_at_k": 0.07273844154098844, + "adjusted_inverse_harmonic_mean_rank": 0.03726745035530307, + "arithmetic_mean_rank": 3666.801391545767, + "count": 59071.0, + "geometric_mean_rank": 1075.6770390894235, + "harmonic_mean_rank": 26.36178419778149, + "hits_at_1": 0.0215672665097933, + "hits_at_10": 0.07336933520678506, + "hits_at_3": 0.037124815899510756, + "hits_at_5": 0.05056626771173672, + "inverse_arithmetic_mean_rank": 0.00027271725223668104, + "inverse_geometric_mean_rank": 0.0009296470628828471, + "inverse_harmonic_mean_rank": 0.03793369949838814, + "inverse_median_rank": 0.00041999160016799666, + "median_absolute_deviation": 3337.33759385611, + "median_rank": 2381.0, + "standard_deviation": 3753.612967564641, + "variance": 14089610.310269432, + "z_arithmetic_mean_rank": 211.24397005263864, + "z_geometric_mean_rank": 195.13415454254167, + "z_hits_at_k": 677.5273694704455, + "z_inverse_harmonic_mean_rank": 857.4472998554097 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.497748741075718, + "adjusted_arithmetic_mean_rank_index": 0.5023194463223497, + "adjusted_geometric_mean_rank_index": 0.8015379454914331, + "adjusted_hits_at_k": 0.07273844154098844, + "adjusted_inverse_harmonic_mean_rank": 0.03726779246443373, + "arithmetic_mean_rank": 3666.7900390625, + "count": 59071.0, + "geometric_mean_rank": 1075.6695556640625, + "harmonic_mean_rank": 26.361546617879256, + "hits_at_1": 0.0215672665097933, + "hits_at_10": 0.07336933520678506, + "hits_at_3": 0.037124815899510756, + "hits_at_5": 0.05056626771173672, + "inverse_arithmetic_mean_rank": 0.0002727180835790932, + "inverse_geometric_mean_rank": 0.0009296535281464458, + "inverse_harmonic_mean_rank": 0.03793404137076569, + "inverse_median_rank": 0.0004199915856588632, + "median_absolute_deviation": 3337.33759385611, + "median_rank": 2381.0, + "standard_deviation": 3753.6064453125, + "variance": 14089562.0, + "z_arithmetic_mean_rank": 211.24461820874544, + "z_geometric_mean_rank": 195.13449098664515, + "z_hits_at_k": 677.5273694704455, + "z_inverse_harmonic_mean_rank": 857.4551710821112 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.42355838525406514, + "adjusted_arithmetic_mean_rank_index": 0.5765194496169137, + "adjusted_geometric_mean_rank_index": 0.8979622403295612, + "adjusted_hits_at_k": 0.13962235407383589, + "adjusted_inverse_harmonic_mean_rank": 0.08794598722854557, + "arithmetic_mean_rank": 3137.278241438269, + "count": 59071.0, + "geometric_mean_rank": 556.7305049257791, + "harmonic_mean_rank": 11.290032187524611, + "hits_at_1": 0.059436948756581065, + "hits_at_10": 0.14020416109427639, + "hits_at_3": 0.09844085930490427, + "hits_at_5": 0.113930693572142, + "inverse_arithmetic_mean_rank": 0.00031874762869026085, + "inverse_geometric_mean_rank": 0.0017962011981601685, + "inverse_harmonic_mean_rank": 0.0885737067344229, + "inverse_median_rank": 0.0006406149903907751, + "median_absolute_deviation": 2303.9638475577053, + "median_rank": 1561.0, + "standard_deviation": 3659.00613800757, + "variance": 13388325.917977072, + "z_arithmetic_mean_rank": 242.52025418259967, + "z_geometric_mean_rank": 218.6072188825533, + "z_hits_at_k": 1304.519655818148, + "z_inverse_harmonic_mean_rank": 2029.655259950779 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.42356129015598803, + "adjusted_arithmetic_mean_rank_index": 0.5765165443227522, + "adjusted_geometric_mean_rank_index": 0.897960954948751, + "adjusted_hits_at_k": 0.13962235407383589, + "adjusted_inverse_harmonic_mean_rank": 0.08794509652848301, + "arithmetic_mean_rank": 3137.299757918437, + "count": 59071.0, + "geometric_mean_rank": 556.7375055236851, + "harmonic_mean_rank": 11.290145643455114, + "hits_at_1": 0.059436948756581065, + "hits_at_10": 0.14020416109427639, + "hits_at_3": 0.09844085930490427, + "hits_at_5": 0.113930693572142, + "inverse_arithmetic_mean_rank": 0.0003187454426297756, + "inverse_geometric_mean_rank": 0.001796178612143919, + "inverse_harmonic_mean_rank": 0.08857281664738302, + "inverse_median_rank": 0.0006406149903907751, + "median_absolute_deviation": 2303.9638475577053, + "median_rank": 1561.0, + "standard_deviation": 3659.020621260588, + "variance": 13388431.90681022, + "z_arithmetic_mean_rank": 242.51903203358285, + "z_geometric_mean_rank": 218.60690595901207, + "z_hits_at_k": 1304.519655818148, + "z_inverse_harmonic_mean_rank": 2029.6347039922414 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.42355984618646597, + "adjusted_arithmetic_mean_rank_index": 0.5765179884872483, + "adjusted_geometric_mean_rank_index": 0.8979616193980569, + "adjusted_hits_at_k": 0.13962235407383589, + "adjusted_inverse_harmonic_mean_rank": 0.0879454954124831, + "arithmetic_mean_rank": 3137.2890625, + "count": 59071.0, + "geometric_mean_rank": 556.73388671875, + "harmonic_mean_rank": 11.290094833975683, + "hits_at_1": 0.059436948756581065, + "hits_at_10": 0.14020416109427639, + "hits_at_3": 0.09844085930490427, + "hits_at_5": 0.113930693572142, + "inverse_arithmetic_mean_rank": 0.000318746519042179, + "inverse_geometric_mean_rank": 0.001796190277673304, + "inverse_harmonic_mean_rank": 0.08857321525685193, + "inverse_median_rank": 0.0006406150059774518, + "median_absolute_deviation": 2303.9638475577053, + "median_rank": 1561.0, + "standard_deviation": 3659.013427734375, + "variance": 13388379.0, + "z_arithmetic_mean_rank": 242.51963953978392, + "z_geometric_mean_rank": 218.60706771794617, + "z_hits_at_k": 1304.519655818148, + "z_inverse_harmonic_mean_rank": 2029.6439096084882 + } + } + }, + "times": { + "evaluation": 86.14422821998596, + "training": 413.02298378944397 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json new file mode 100644 index 0000000..68c2561 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json new file mode 100644 index 0000000..c617246 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.01334708773344851, + 0.013160222068615453, + 0.012974961877335761, + 0.012791537809202847, + 0.012609953025432497 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5243715695086415, + "adjusted_arithmetic_mean_rank_index": 0.4756950948921632, + "adjusted_geometric_mean_rank_index": 0.8507377858132705, + "adjusted_hits_at_k": 0.10921825533888653, + "adjusted_inverse_harmonic_mean_rank": 0.07060651340848918, + "arithmetic_mean_rank": 3741.741926803014, + "count": 40876.0, + "geometric_mean_rank": 783.6524314926153, + "harmonic_mean_rank": 14.031377373872546, + "hits_at_1": 0.04961346511400333, + "hits_at_10": 0.1098444074762697, + "hits_at_3": 0.07478716117036892, + "hits_at_5": 0.08706820628241511, + "inverse_arithmetic_mean_rank": 0.0002672552034753533, + "inverse_geometric_mean_rank": 0.001276075923219315, + "inverse_harmonic_mean_rank": 0.07126884078123887, + "inverse_median_rank": 0.00044853106077595876, + "median_absolute_deviation": 3269.8791929141053, + "median_rank": 2229.5, + "standard_deviation": 4000.811539735375, + "variance": 16006492.976479743, + "z_arithmetic_mean_rank": 166.3665110929076, + "z_geometric_mean_rank": 172.29643521192267, + "z_hits_at_k": 832.5744171518359, + "z_inverse_harmonic_mean_rank": 1329.5633720102164 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5243763624717652, + "adjusted_arithmetic_mean_rank_index": 0.47569030125725453, + "adjusted_geometric_mean_rank_index": 0.8507338518544061, + "adjusted_hits_at_k": 0.10921825533888653, + "adjusted_inverse_harmonic_mean_rank": 0.07060218592179501, + "arithmetic_mean_rank": 3741.776127801155, + "count": 40876.0, + "geometric_mean_rank": 783.6730591007979, + "harmonic_mean_rank": 14.032228812004098, + "hits_at_1": 0.04961346511400333, + "hits_at_10": 0.1098444074762697, + "hits_at_3": 0.07478716117036892, + "hits_at_5": 0.08706820628241511, + "inverse_arithmetic_mean_rank": 0.0002672527606796314, + "inverse_geometric_mean_rank": 0.001276042334730021, + "inverse_harmonic_mean_rank": 0.0712645163785053, + "inverse_median_rank": 0.00044853106077595876, + "median_absolute_deviation": 3269.8791929141053, + "median_rank": 2229.5, + "standard_deviation": 4000.8300621961807, + "variance": 16006641.186572695, + "z_arithmetic_mean_rank": 166.36483459818695, + "z_geometric_mean_rank": 172.2956384833655, + "z_hits_at_k": 832.5744171518359, + "z_inverse_harmonic_mean_rank": 1329.4818828171708 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5243739326030625, + "adjusted_arithmetic_mean_rank_index": 0.47569273146652935, + "adjusted_geometric_mean_rank_index": 0.8507357655093295, + "adjusted_hits_at_k": 0.10921825533888653, + "adjusted_inverse_harmonic_mean_rank": 0.07060393544377082, + "arithmetic_mean_rank": 3741.7587890625, + "count": 40876.0, + "geometric_mean_rank": 783.6630249023438, + "harmonic_mean_rank": 14.03188457904077, + "hits_at_1": 0.04961346511400333, + "hits_at_10": 0.1098444074762697, + "hits_at_3": 0.07478716117036892, + "hits_at_5": 0.08706820628241511, + "inverse_arithmetic_mean_rank": 0.00026725398492999375, + "inverse_geometric_mean_rank": 0.0012760587269440293, + "inverse_harmonic_mean_rank": 0.07126626465369348, + "inverse_median_rank": 0.00044853106373921037, + "median_absolute_deviation": 3269.8791929141053, + "median_rank": 2229.5, + "standard_deviation": 4000.82080078125, + "variance": 16006567.0, + "z_arithmetic_mean_rank": 166.36568452378552, + "z_geometric_mean_rank": 172.29602604804998, + "z_hits_at_k": 832.5744171518359, + "z_inverse_harmonic_mean_rank": 1329.514827374624 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.630345844032263, + "adjusted_arithmetic_mean_rank_index": 0.36970669621218955, + "adjusted_geometric_mean_rank_index": 0.6374260958695077, + "adjusted_hits_at_k": 0.02195491082542058, + "adjusted_inverse_harmonic_mean_rank": 0.010283827488961611, + "arithmetic_mean_rank": 4435.515706037772, + "count": 20438.0, + "geometric_mean_rank": 1873.5241837633828, + "harmonic_mean_rank": 90.91202964026317, + "hits_at_1": 0.0034249926607300128, + "hits_at_10": 0.02265388002739994, + "hits_at_3": 0.008709267051570603, + "hits_at_5": 0.012868186711028476, + "inverse_arithmetic_mean_rank": 0.0002254529272974429, + "inverse_geometric_mean_rank": 0.0005337534517388942, + "inverse_harmonic_mean_rank": 0.010999644425022489, + "inverse_median_rank": 0.00029304029304029304, + "median_absolute_deviation": 4254.327066001825, + "median_rank": 3412.5, + "standard_deviation": 3946.1367222079466, + "variance": 15571995.030358076, + "z_arithmetic_mean_rank": 91.33094104479204, + "z_geometric_mean_rank": 91.28864498343353, + "z_hits_at_k": 117.36730933110842, + "z_inverse_harmonic_mean_rank": 135.80611068364766 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6303518100379835, + "adjusted_arithmetic_mean_rank_index": 0.36970072935849974, + "adjusted_geometric_mean_rank_index": 0.6374135161138859, + "adjusted_hits_at_k": 0.02195491082542058, + "adjusted_inverse_harmonic_mean_rank": 0.010275601268548235, + "arithmetic_mean_rank": 4435.5576866621, + "count": 20438.0, + "geometric_mean_rank": 1873.5891523018224, + "harmonic_mean_rank": 90.98002096591132, + "hits_at_1": 0.0034249926607300128, + "hits_at_10": 0.02265388002739994, + "hits_at_3": 0.008709267051570603, + "hits_at_5": 0.012868186711028476, + "inverse_arithmetic_mean_rank": 0.00022545079348354325, + "inverse_geometric_mean_rank": 0.0005337349433153138, + "inverse_harmonic_mean_rank": 0.01099142415426221, + "inverse_median_rank": 0.00029304029304029304, + "median_absolute_deviation": 4254.327066001825, + "median_rank": 3412.5, + "standard_deviation": 3946.154266638542, + "variance": 15572133.496109571, + "z_arithmetic_mean_rank": 91.32946701587088, + "z_geometric_mean_rank": 91.28684338031061, + "z_hits_at_k": 117.36730933110842, + "z_inverse_harmonic_mean_rank": 135.697476908804 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6303488163402171, + "adjusted_arithmetic_mean_rank_index": 0.36970372348177094, + "adjusted_geometric_mean_rank_index": 0.637419598572049, + "adjusted_hits_at_k": 0.02195491082542058, + "adjusted_inverse_harmonic_mean_rank": 0.010278897792388431, + "arithmetic_mean_rank": 4435.53662109375, + "count": 20438.0, + "geometric_mean_rank": 1873.5577392578125, + "harmonic_mean_rank": 90.95276234198829, + "hits_at_1": 0.0034249926607300128, + "hits_at_10": 0.02265388002739994, + "hits_at_3": 0.008709267051570603, + "hits_at_5": 0.012868186711028476, + "inverse_arithmetic_mean_rank": 0.00022545186220668256, + "inverse_geometric_mean_rank": 0.0005337438778951764, + "inverse_harmonic_mean_rank": 0.010994718293875837, + "inverse_median_rank": 0.00029304029885679483, + "median_absolute_deviation": 4254.327066001825, + "median_rank": 3412.5, + "standard_deviation": 3946.1455078125, + "variance": 15572064.0, + "z_arithmetic_mean_rank": 91.33020667273607, + "z_geometric_mean_rank": 91.2877144763757, + "z_hits_at_k": 117.36730933110842, + "z_inverse_harmonic_mean_rank": 135.7410101246221 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.42129850782168216, + "adjusted_arithmetic_mean_rank_index": 0.5787814929419499, + "adjusted_geometric_mean_rank_index": 0.9386177352290703, + "adjusted_hits_at_k": 0.196479550332964, + "adjusted_inverse_harmonic_mean_rank": 0.13092791837765114, + "arithmetic_mean_rank": 3047.968147568255, + "count": 20438.0, + "geometric_mean_rank": 327.78393719514855, + "harmonic_mean_rank": 7.602363709859949, + "hits_at_1": 0.09580193756727665, + "hits_at_10": 0.19703493492513946, + "hits_at_3": 0.14086505528916723, + "hits_at_5": 0.16126822585380174, + "inverse_arithmetic_mean_rank": 0.0003280874181043607, + "inverse_geometric_mean_rank": 0.0030507901288788376, + "inverse_harmonic_mean_rank": 0.13153803713745524, + "inverse_median_rank": 0.00102880658436214, + "median_absolute_deviation": 1439.6067541689395, + "median_rank": 972.0, + "standard_deviation": 3934.2530179303467, + "variance": 15478346.809094042, + "z_arithmetic_mean_rank": 143.30325123133036, + "z_geometric_mean_rank": 134.4177059185862, + "z_hits_at_k": 1068.041113623353, + "z_inverse_harmonic_mean_rank": 1758.0301037163845 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4213021598560998, + "adjusted_arithmetic_mean_rank_index": 0.5787778404026682, + "adjusted_geometric_mean_rank_index": 0.9386166289573937, + "adjusted_hits_at_k": 0.196479550332964, + "adjusted_inverse_harmonic_mean_rank": 0.13092748954188677, + "arithmetic_mean_rank": 3047.9945689402093, + "count": 20438.0, + "geometric_mean_rank": 327.78982671088136, + "harmonic_mean_rank": 7.6023884775042605, + "hits_at_1": 0.09580193756727665, + "hits_at_10": 0.19703493492513946, + "hits_at_3": 0.14086505528916723, + "hits_at_5": 0.16126822585380174, + "inverse_arithmetic_mean_rank": 0.00032808457409676456, + "inverse_geometric_mean_rank": 0.0030507353142537413, + "inverse_harmonic_mean_rank": 0.13153760860274843, + "inverse_median_rank": 0.00102880658436214, + "median_absolute_deviation": 1439.6067541689395, + "median_rank": 972.0, + "standard_deviation": 3934.2703483879673, + "variance": 15478483.174204776, + "z_arithmetic_mean_rank": 143.3023468818294, + "z_geometric_mean_rank": 134.41754749147012, + "z_hits_at_k": 1068.041113623353, + "z_inverse_harmonic_mean_rank": 1758.0243455389639 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4213003458723233, + "adjusted_arithmetic_mean_rank_index": 0.5787796546372133, + "adjusted_geometric_mean_rank_index": 0.9386171795012745, + "adjusted_hits_at_k": 0.196479550332964, + "adjusted_inverse_harmonic_mean_rank": 0.1309276965775306, + "arithmetic_mean_rank": 3047.9814453125, + "count": 20438.0, + "geometric_mean_rank": 327.7868957519531, + "harmonic_mean_rank": 7.602376520027175, + "hits_at_1": 0.09580193756727665, + "hits_at_10": 0.19703493492513946, + "hits_at_3": 0.14086505528916723, + "hits_at_5": 0.16126822585380174, + "inverse_arithmetic_mean_rank": 0.00032808599644340575, + "inverse_geometric_mean_rank": 0.0030507626943290234, + "inverse_harmonic_mean_rank": 0.13153781549304605, + "inverse_median_rank": 0.001028806553222239, + "median_absolute_deviation": 1439.6067541689395, + "median_rank": 972.0, + "standard_deviation": 3934.26171875, + "variance": 15478415.0, + "z_arithmetic_mean_rank": 143.3027960767535, + "z_geometric_mean_rank": 134.4176263338387, + "z_hits_at_k": 1068.041113623353, + "z_inverse_harmonic_mean_rank": 1758.0271255029234 + } + } + }, + "times": { + "evaluation": 29.337369203567505, + "training": 459.2732901573181 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json new file mode 100644 index 0000000..0b5d5b4 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with RotatE as described by Sun et al., 2019;" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json new file mode 100644 index 0000000..8b73ac6 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.013424746205139526, + 0.013122108396254868, + 0.012823446353122617, + 0.012530585870327324, + 0.012243712673720453 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7855334638470861, + "adjusted_arithmetic_mean_rank_index": 0.2144770172444035, + "adjusted_geometric_mean_rank_index": 0.5966660934317238, + "adjusted_hits_at_k": 0.033463829540325396, + "adjusted_inverse_harmonic_mean_rank": 0.01997912249413563, + "arithmetic_mean_rank": 16074.5543, + "count": 10000.0, + "geometric_mean_rank": 6074.265576986406, + "harmonic_mean_rank": 49.38943424664326, + "hits_at_1": 0.0132, + "hits_at_10": 0.0337, + "hits_at_3": 0.0201, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 6.221012298922653e-05, + "inverse_geometric_mean_rank": 0.00016462895593315907, + "inverse_harmonic_mean_rank": 0.020247245493968474, + "inverse_median_rank": 7.060401736858827e-05, + "median_absolute_deviation": 16906.854398728632, + "median_rank": 14163.5, + "standard_deviation": 12884.519360893968, + "variance": 166010839.16125152, + "z_arithmetic_mean_rank": 37.14755487583042, + "z_geometric_mean_rank": 59.71486914649745, + "z_hits_at_k": 214.0517790829075, + "z_inverse_harmonic_mean_rank": 315.3461920068905 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7855372071461044, + "adjusted_arithmetic_mean_rank_index": 0.21447327376244818, + "adjusted_geometric_mean_rank_index": 0.5966631514048026, + "adjusted_hits_at_k": 0.033463829540325396, + "adjusted_inverse_harmonic_mean_rank": 0.019979117339909674, + "arithmetic_mean_rank": 16074.6309, + "count": 10000.0, + "geometric_mean_rank": 6074.309877032933, + "harmonic_mean_rank": 49.38944681599364, + "hits_at_1": 0.0132, + "hits_at_10": 0.0337, + "hits_at_3": 0.0201, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 6.220982654102497e-05, + "inverse_geometric_mean_rank": 0.00016462775529134868, + "inverse_harmonic_mean_rank": 0.020247240341152658, + "inverse_median_rank": 7.060152499293985e-05, + "median_absolute_deviation": 16906.854398728632, + "median_rank": 14164.0, + "standard_deviation": 12884.537494161952, + "variance": 166011306.43846518, + "z_arithmetic_mean_rank": 37.14690650239095, + "z_geometric_mean_rank": 59.71457470584704, + "z_hits_at_k": 214.0517790829075, + "z_inverse_harmonic_mean_rank": 315.3461106536915 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7855352962493756, + "adjusted_arithmetic_mean_rank_index": 0.21447518475256344, + "adjusted_geometric_mean_rank_index": 0.5966644040168032, + "adjusted_hits_at_k": 0.033463829540325396, + "adjusted_inverse_harmonic_mean_rank": 0.019979119736229026, + "arithmetic_mean_rank": 16074.591796875, + "count": 10000.0, + "geometric_mean_rank": 6074.291015625, + "harmonic_mean_rank": 49.389440972209925, + "hits_at_1": 0.0132, + "hits_at_10": 0.0337, + "hits_at_3": 0.0201, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 6.220997602213174e-05, + "inverse_geometric_mean_rank": 0.00016462826170027256, + "inverse_harmonic_mean_rank": 0.020247242736816405, + "inverse_median_rank": 7.060277130221948e-05, + "median_absolute_deviation": 16906.854398728632, + "median_rank": 14163.75, + "standard_deviation": 12884.5283203125, + "variance": 166011072.0, + "z_arithmetic_mean_rank": 37.1472374870861, + "z_geometric_mean_rank": 59.71470006836142, + "z_hits_at_k": 214.0517790829075, + "z_inverse_harmonic_mean_rank": 315.34614847668314 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7912971127949484, + "adjusted_arithmetic_mean_rank_index": 0.2087130868604813, + "adjusted_geometric_mean_rank_index": 0.576690351513085, + "adjusted_hits_at_k": 0.03196345740463281, + "adjusted_inverse_harmonic_mean_rank": 0.016330660837363887, + "arithmetic_mean_rank": 16192.1218, + "count": 5000.0, + "geometric_mean_rank": 6375.223395588098, + "harmonic_mean_rank": 60.24173415886561, + "hits_at_1": 0.0086, + "hits_at_10": 0.0322, + "hits_at_3": 0.0176, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 6.175842871932942e-05, + "inverse_geometric_mean_rank": 0.00015685724843650793, + "inverse_harmonic_mean_rank": 0.016599787737897196, + "inverse_median_rank": 6.924248719013987e-05, + "median_absolute_deviation": 16699.29008813785, + "median_rank": 14442.0, + "standard_deviation": 12794.627688235589, + "variance": 163702497.68056476, + "z_arithmetic_mean_rank": 25.561369516008, + "z_geometric_mean_rank": 40.81416845600836, + "z_hits_at_k": 144.5695676042832, + "z_inverse_harmonic_mean_rank": 182.2614735761567 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7913008757284563, + "adjusted_arithmetic_mean_rank_index": 0.2087093237430726, + "adjusted_geometric_mean_rank_index": 0.5766867904434522, + "adjusted_hits_at_k": 0.03196345740463281, + "adjusted_inverse_harmonic_mean_rank": 0.01633065154077127, + "arithmetic_mean_rank": 16192.1988, + "count": 5000.0, + "geometric_mean_rank": 6375.277018399314, + "harmonic_mean_rank": 60.24176788760712, + "hits_at_1": 0.0086, + "hits_at_10": 0.0322, + "hits_at_3": 0.0176, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 6.175813503475513e-05, + "inverse_geometric_mean_rank": 0.00015685592910142705, + "inverse_harmonic_mean_rank": 0.01659977844384808, + "inverse_median_rank": 6.924009001211702e-05, + "median_absolute_deviation": 16699.29008813785, + "median_rank": 14442.5, + "standard_deviation": 12794.6465912302, + "variance": 163702981.39447856, + "z_arithmetic_mean_rank": 25.560908642010773, + "z_geometric_mean_rank": 40.8139164280431, + "z_hits_at_k": 144.5695676042832, + "z_inverse_harmonic_mean_rank": 182.2613698197511 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7912989872367453, + "adjusted_arithmetic_mean_rank_index": 0.20871121232707734, + "adjusted_geometric_mean_rank_index": 0.5766885847246386, + "adjusted_hits_at_k": 0.03196345740463281, + "adjusted_inverse_harmonic_mean_rank": 0.016330654897157886, + "arithmetic_mean_rank": 16192.16015625, + "count": 5000.0, + "geometric_mean_rank": 6375.25, + "harmonic_mean_rank": 60.24175571037813, + "hits_at_1": 0.0086, + "hits_at_10": 0.0322, + "hits_at_3": 0.0176, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 6.175828457344323e-05, + "inverse_geometric_mean_rank": 0.00015685659309383482, + "inverse_harmonic_mean_rank": 0.016599781799316406, + "inverse_median_rank": 6.924128683749586e-05, + "median_absolute_deviation": 16699.29008813785, + "median_rank": 14442.25, + "standard_deviation": 12794.63671875, + "variance": 163702736.0, + "z_arithmetic_mean_rank": 25.56113993940727, + "z_geometric_mean_rank": 40.81404341489906, + "z_hits_at_k": 144.5695676042832, + "z_inverse_harmonic_mean_rank": 182.2614072793479 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7797700821308655, + "adjusted_arithmetic_mean_rank_index": 0.22024068037057487, + "adjusted_geometric_mean_rank_index": 0.615737701058511, + "adjusted_hits_at_k": 0.03496420165889074, + "adjusted_inverse_harmonic_mean_rank": 0.023627584108453507, + "arithmetic_mean_rank": 15956.9868, + "count": 5000.0, + "geometric_mean_rank": 5787.515199121641, + "harmonic_mean_rank": 41.85027909891856, + "hits_at_1": 0.0178, + "hits_at_10": 0.0352, + "hits_at_3": 0.0226, + "hits_at_5": 0.0264, + "inverse_arithmetic_mean_rank": 6.266847322327796e-05, + "inverse_geometric_mean_rank": 0.00017278572333628912, + "inverse_harmonic_mean_rank": 0.02389470325003975, + "inverse_median_rank": 7.156915369475756e-05, + "median_absolute_deviation": 17118.12521486568, + "median_rank": 13972.5, + "standard_deviation": 12972.722783125591, + "variance": 168291536.40782577, + "z_arithmetic_mean_rank": 26.973173800370787, + "z_geometric_mean_rank": 43.57766952935073, + "z_hits_at_k": 158.14551992692725, + "z_inverse_harmonic_mean_rank": 263.7063375478915 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7797738057963045, + "adjusted_arithmetic_mean_rank_index": 0.22023695652316255, + "adjusted_geometric_mean_rank_index": 0.6157353278228198, + "adjusted_hits_at_k": 0.03496420165889074, + "adjusted_inverse_harmonic_mean_rank": 0.023627583096594164, + "arithmetic_mean_rank": 15957.063, + "count": 5000.0, + "geometric_mean_rank": 5787.55093711737, + "harmonic_mean_rank": 41.85028087065065, + "hits_at_1": 0.0178, + "hits_at_10": 0.0352, + "hits_at_3": 0.0226, + "hits_at_5": 0.0264, + "inverse_arithmetic_mean_rank": 6.266817396158678e-05, + "inverse_geometric_mean_rank": 0.00017278465638836764, + "inverse_harmonic_mean_rank": 0.023894702238457233, + "inverse_median_rank": 7.156915369475756e-05, + "median_absolute_deviation": 17118.12521486568, + "median_rank": 13972.5, + "standard_deviation": 12972.74015234372, + "variance": 168291987.06023097, + "z_arithmetic_mean_rank": 26.972717735745093, + "z_geometric_mean_rank": 43.5775015680898, + "z_hits_at_k": 158.14551992692725, + "z_inverse_harmonic_mean_rank": 263.7063262545777 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7797719202172769, + "adjusted_arithmetic_mean_rank_index": 0.22023884219433731, + "adjusted_geometric_mean_rank_index": 0.6157367973009195, + "adjusted_hits_at_k": 0.03496420165889074, + "adjusted_inverse_harmonic_mean_rank": 0.02362758453284627, + "arithmetic_mean_rank": 15957.0244140625, + "count": 5000.0, + "geometric_mean_rank": 5787.52880859375, + "harmonic_mean_rank": 41.85027835582098, + "hits_at_1": 0.0178, + "hits_at_10": 0.0352, + "hits_at_3": 0.0226, + "hits_at_5": 0.0264, + "inverse_arithmetic_mean_rank": 6.266832497203723e-05, + "inverse_geometric_mean_rank": 0.00017278532322961837, + "inverse_harmonic_mean_rank": 0.023894703674316405, + "inverse_median_rank": 7.156915671657771e-05, + "median_absolute_deviation": 17118.12521486568, + "median_rank": 13972.5, + "standard_deviation": 12972.732421875, + "variance": 168291776.0, + "z_arithmetic_mean_rank": 26.972948676442513, + "z_geometric_mean_rank": 43.577605567618974, + "z_hits_at_k": 158.14551992692725, + "z_inverse_harmonic_mean_rank": 263.7063422845188 + } + } + }, + "times": { + "evaluation": 11.969523429870605, + "training": 490.47644114494324 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json new file mode 100644 index 0000000..8dcbe52 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18RR Dataset with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json new file mode 100644 index 0000000..6337d50 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 256, + "slice_size": null + }, + "losses": [ + 0.019261958489676396, + 0.019163661997862045, + 0.019064278557845992, + 0.018965127136297243, + 0.018866502912631956 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8828598120598528, + "adjusted_arithmetic_mean_rank_index": 0.11714596649184639, + "adjusted_geometric_mean_rank_index": 0.2923689706295878, + "adjusted_hits_at_k": 0.007450130412883449, + "adjusted_inverse_harmonic_mean_rank": 0.003932262355982899, + "arithmetic_mean_rank": 17897.81788645691, + "count": 5848.0, + "geometric_mean_rank": 10557.359337940927, + "harmonic_mean_rank": 237.69292302092035, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007694938440492476, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.5872732997059354e-05, + "inverse_geometric_mean_rank": 9.472065579942994e-05, + "inverse_harmonic_mean_rank": 0.004207108849900364, + "inverse_median_rank": 5.940182363598562e-05, + "median_absolute_deviation": 15698.533590646566, + "median_rank": 16834.5, + "standard_deviation": 12192.032467473826, + "variance": 148645655.6879359, + "z_arithmetic_mean_rank": 15.516025046938196, + "z_geometric_mean_rank": 22.377492200661884, + "z_hits_at_k": 36.27248694079143, + "z_inverse_harmonic_mean_rank": 47.241813608270604 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8828636752828475, + "adjusted_arithmetic_mean_rank_index": 0.11714210307827799, + "adjusted_geometric_mean_rank_index": 0.2923638535782205, + "adjusted_hits_at_k": 0.007450130412883449, + "adjusted_inverse_harmonic_mean_rank": 0.003932259472704943, + "arithmetic_mean_rank": 17897.89620383037, + "count": 5848.0, + "geometric_mean_rank": 10557.435673532134, + "harmonic_mean_rank": 237.69308587530654, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007694938440492476, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.587248850990586e-05, + "inverse_geometric_mean_rank": 9.471997092125652e-05, + "inverse_harmonic_mean_rank": 0.004207105967417995, + "inverse_median_rank": 5.940182363598562e-05, + "median_absolute_deviation": 15698.533590646566, + "median_rank": 16834.5, + "standard_deviation": 12192.040678461748, + "variance": 148645855.90526602, + "z_arithmetic_mean_rank": 15.515513336433017, + "z_geometric_mean_rank": 22.3771005490553, + "z_hits_at_k": 36.27248694079143, + "z_inverse_harmonic_mean_rank": 47.24177896885251 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8828617622546758, + "adjusted_arithmetic_mean_rank_index": 0.11714401620081982, + "adjusted_geometric_mean_rank_index": 0.2923660223318012, + "adjusted_hits_at_k": 0.007450130412883449, + "adjusted_inverse_harmonic_mean_rank": 0.00393226100790816, + "arithmetic_mean_rank": 17897.857421875, + "count": 5848.0, + "geometric_mean_rank": 10557.4033203125, + "harmonic_mean_rank": 237.69299916335532, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007694938440492476, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 5.587260966422036e-05, + "inverse_geometric_mean_rank": 9.472026431467384e-05, + "inverse_harmonic_mean_rank": 0.0042071075021976, + "inverse_median_rank": 5.9401823818916455e-05, + "median_absolute_deviation": 15698.533590646566, + "median_rank": 16834.5, + "standard_deviation": 12192.037109375, + "variance": 148645760.0, + "z_arithmetic_mean_rank": 15.515766730196077, + "z_geometric_mean_rank": 22.377266542273503, + "z_hits_at_k": 36.27248694079143, + "z_inverse_harmonic_mean_rank": 47.24179741263312 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9376485514225431, + "adjusted_arithmetic_mean_rank_index": 0.062354525070938194, + "adjusted_geometric_mean_rank_index": 0.16130982596183663, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.0026470319205615617, + "arithmetic_mean_rank": 19004.308139534885, + "count": 2924.0, + "geometric_mean_rank": 12510.769545520707, + "harmonic_mean_rank": 342.19751112357585, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.004103967168262654, + "inverse_arithmetic_mean_rank": 5.26196477481697e-05, + "inverse_geometric_mean_rank": 7.993113424090167e-05, + "inverse_harmonic_mean_rank": 0.0029222889340036013, + "inverse_median_rank": 5.450779461462989e-05, + "median_absolute_deviation": 15325.659132692408, + "median_rank": 18346.0, + "standard_deviation": 11991.670556534038, + "variance": 143800162.73644537, + "z_arithmetic_mean_rank": 5.8399060402006455, + "z_geometric_mean_rank": 8.731342208322275, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 22.484326194166155 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9376525842399235, + "adjusted_arithmetic_mean_rank_index": 0.06235049205457388, + "adjusted_geometric_mean_rank_index": 0.16130319319545638, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.002647027911227845, + "arithmetic_mean_rank": 19004.389876880985, + "count": 2924.0, + "geometric_mean_rank": 12510.868478811492, + "harmonic_mean_rank": 342.197980484163, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.004103967168262654, + "inverse_arithmetic_mean_rank": 5.2619421432545395e-05, + "inverse_geometric_mean_rank": 7.993050216246843e-05, + "inverse_harmonic_mean_rank": 0.002922284925776411, + "inverse_median_rank": 5.450779461462989e-05, + "median_absolute_deviation": 15325.659132692408, + "median_rank": 18346.0, + "standard_deviation": 11991.672594145404, + "variance": 143800211.60517797, + "z_arithmetic_mean_rank": 5.839528322038262, + "z_geometric_mean_rank": 8.730983191426013, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 22.484292138223573 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9376505974920567, + "adjusted_arithmetic_mean_rank_index": 0.06235247890046913, + "adjusted_geometric_mean_rank_index": 0.16130733900159622, + "adjusted_hits_at_k": 0.005568626686370119, + "adjusted_inverse_harmonic_mean_rank": 0.0026470297253797593, + "arithmetic_mean_rank": 19004.349609375, + "count": 2924.0, + "geometric_mean_rank": 12510.806640625, + "harmonic_mean_rank": 342.1977681067189, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.005813953488372093, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.004103967168262654, + "inverse_arithmetic_mean_rank": 5.261953265289776e-05, + "inverse_geometric_mean_rank": 7.993089820956811e-05, + "inverse_harmonic_mean_rank": 0.002922286739427642, + "inverse_median_rank": 5.450779281090945e-05, + "median_absolute_deviation": 15325.659132692408, + "median_rank": 18346.0, + "standard_deviation": 11991.6708984375, + "variance": 143800176.0, + "z_arithmetic_mean_rank": 5.839714403053733, + "z_geometric_mean_rank": 8.731207594694197, + "z_hits_at_k": 19.168941041029797, + "z_inverse_harmonic_mean_rank": 22.484307547929475 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8280953773701203, + "adjusted_arithmetic_mean_rank_index": 0.17191310084239653, + "adjusted_geometric_mean_rank_index": 0.4030514945528445, + "adjusted_hits_at_k": 0.009331633932319346, + "adjusted_inverse_harmonic_mean_rank": 0.005217492647335803, + "arithmetic_mean_rank": 16791.327633378933, + "count": 2924.0, + "geometric_mean_rank": 8908.951266735969, + "harmonic_mean_rank": 182.08539160737905, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.009575923392612859, + "hits_at_3": 0.005471956224350205, + "hits_at_5": 0.006155950752393981, + "inverse_arithmetic_mean_rank": 5.955455231616901e-05, + "inverse_geometric_mean_rank": 0.00011224665732921632, + "inverse_harmonic_mean_rank": 0.005491928765797128, + "inverse_median_rank": 6.643414715163594e-05, + "median_absolute_deviation": 15321.21132603689, + "median_rank": 15052.5, + "standard_deviation": 12289.935202396848, + "variance": 151042507.27911326, + "z_arithmetic_mean_rank": 16.10079719936988, + "z_geometric_mean_rank": 21.816275403698548, + "z_hits_at_k": 32.1295473736814, + "z_inverse_harmonic_mean_rank": 44.32812013266561 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8280990710739625, + "adjusted_arithmetic_mean_rank_index": 0.17190940695638335, + "adjusted_geometric_mean_rank_index": 0.4030475821520433, + "adjusted_hits_at_k": 0.009331633932319346, + "adjusted_inverse_harmonic_mean_rank": 0.005217490890113478, + "arithmetic_mean_rank": 16791.402530779753, + "count": 2924.0, + "geometric_mean_rank": 8909.009649452995, + "harmonic_mean_rank": 182.085449852189, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.009575923392612859, + "hits_at_3": 0.005471956224350205, + "hits_at_5": 0.006155950752393981, + "inverse_arithmetic_mean_rank": 5.9554286675394374e-05, + "inverse_geometric_mean_rank": 0.00011224592175196477, + "inverse_harmonic_mean_rank": 0.0054919270090595775, + "inverse_median_rank": 6.643414715163594e-05, + "median_absolute_deviation": 15321.952627146144, + "median_rank": 15052.5, + "standard_deviation": 12289.948889574442, + "variance": 151042843.70835203, + "z_arithmetic_mean_rank": 16.10045124255052, + "z_geometric_mean_rank": 21.816063634198812, + "z_hits_at_k": 32.1295473736814, + "z_inverse_harmonic_mean_rank": 44.32810520320282 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8280972317327835, + "adjusted_arithmetic_mean_rank_index": 0.17191124638827748, + "adjusted_geometric_mean_rank_index": 0.40305006117741937, + "adjusted_hits_at_k": 0.009331633932319346, + "adjusted_inverse_harmonic_mean_rank": 0.005217491820124018, + "arithmetic_mean_rank": 16791.365234375, + "count": 2924.0, + "geometric_mean_rank": 8908.97265625, + "harmonic_mean_rank": 182.085419026098, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.009575923392612859, + "hits_at_3": 0.005471956224350205, + "hits_at_5": 0.006155950752393981, + "inverse_arithmetic_mean_rank": 5.955441883997992e-05, + "inverse_geometric_mean_rank": 0.00011224638728890568, + "inverse_harmonic_mean_rank": 0.00549192793881355, + "inverse_median_rank": 6.643414963036776e-05, + "median_absolute_deviation": 15321.581976591517, + "median_rank": 15052.5, + "standard_deviation": 12289.9423828125, + "variance": 151042672.0, + "z_arithmetic_mean_rank": 16.100623517494924, + "z_geometric_mean_rank": 21.81619781829457, + "z_hits_at_k": 32.1295473736814, + "z_inverse_harmonic_mean_rank": 44.328113104626304 + } + } + }, + "times": { + "evaluation": 7.80864691734314, + "training": 302.2507004737854 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json new file mode 100644 index 0000000..0fb86ad --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15k Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json new file mode 100644 index 0000000..b27d575 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.004399372436803763, + 0.0007778990072353506, + 0.0003317295822744947, + 0.00021465736021517208, + 0.0001767241696370522 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.2642043057767773, + "adjusted_arithmetic_mean_rank_index": -0.2642400774509712, + "adjusted_geometric_mean_rank_index": -0.36675010197140323, + "adjusted_hits_at_k": -7.738405941025406e-05, + "adjusted_inverse_harmonic_mean_rank": -0.00017050125511989926, + "arithmetic_mean_rank": 9338.490612991147, + "count": 118142.0, + "geometric_mean_rank": 7423.285855813262, + "harmonic_mean_rank": 1923.9636122992003, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0006009717120075841, + "hits_at_3": 0.00011850146433952363, + "hits_at_5": 0.00022853853836908128, + "inverse_arithmetic_mean_rank": 0.00010708368637313402, + "inverse_geometric_mean_rank": 0.00013471123427328188, + "inverse_harmonic_mean_rank": 0.0005197603497318574, + "inverse_median_rank": 9.800078400627205e-05, + "median_absolute_deviation": 5381.846053175335, + "median_rank": 10204.0, + "standard_deviation": 4446.329053094438, + "variance": 19769842.04839168, + "z_arithmetic_mean_rank": -157.17456742126456, + "z_geometric_mean_rank": -126.27729170514175, + "z_hits_at_k": -1.0209271367266144, + "z_inverse_harmonic_mean_rank": -5.556272891232151 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.2642043527575457, + "adjusted_arithmetic_mean_rank_index": -0.2642401244381005, + "adjusted_geometric_mean_rank_index": -0.36675019622610616, + "adjusted_hits_at_k": -7.738405941025406e-05, + "adjusted_inverse_harmonic_mean_rank": -0.00017050128074743386, + "arithmetic_mean_rank": 9338.49096003115, + "count": 118142.0, + "geometric_mean_rank": 7423.286367673732, + "harmonic_mean_rank": 1923.9637070975393, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0006009717120075841, + "hits_at_3": 0.00011850146433952363, + "hits_at_5": 0.00022853853836908128, + "inverse_arithmetic_mean_rank": 0.00010708368239365565, + "inverse_geometric_mean_rank": 0.00013471122498449087, + "inverse_harmonic_mean_rank": 0.0005197603241220095, + "inverse_median_rank": 9.800078400627205e-05, + "median_absolute_deviation": 5381.846053175335, + "median_rank": 10204.0, + "standard_deviation": 4446.328912173295, + "variance": 19769840.79522816, + "z_arithmetic_mean_rank": -157.17459537002162, + "z_geometric_mean_rank": -126.27732415838051, + "z_hits_at_k": -1.0209271367266144, + "z_inverse_harmonic_mean_rank": -5.55627372637894 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.2642042545213674, + "adjusted_arithmetic_mean_rank_index": -0.26424002618862175, + "adjusted_geometric_mean_rank_index": -0.3667504227168079, + "adjusted_hits_at_k": -7.738405941025406e-05, + "adjusted_inverse_harmonic_mean_rank": -0.0001705012723187267, + "arithmetic_mean_rank": 9338.490234375, + "count": 118142.0, + "geometric_mean_rank": 7423.28759765625, + "harmonic_mean_rank": 1923.9636759190637, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0006009717120075841, + "hits_at_3": 0.00011850146433952363, + "hits_at_5": 0.00022853853836908128, + "inverse_arithmetic_mean_rank": 0.00010708368790801615, + "inverse_geometric_mean_rank": 0.00013471119746100157, + "inverse_harmonic_mean_rank": 0.0005197603325448996, + "inverse_median_rank": 9.800078260013834e-05, + "median_absolute_deviation": 5381.846053175335, + "median_rank": 10204.0, + "standard_deviation": 4446.3291015625, + "variance": 19769842.0, + "z_arithmetic_mean_rank": -157.17453692953256, + "z_geometric_mean_rank": -126.27740214235996, + "z_hits_at_k": -1.0209271367266144, + "z_inverse_harmonic_mean_rank": -5.556273451705319 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.22332789159549, + "adjusted_arithmetic_mean_rank_index": -0.22335821137582568, + "adjusted_geometric_mean_rank_index": -0.31018246346676426, + "adjusted_hits_at_k": -2.0174999586579823e-05, + "adjusted_inverse_harmonic_mean_rank": -0.0001228583296856978, + "arithmetic_mean_rank": 9011.949518376192, + "count": 59071.0, + "geometric_mean_rank": 7095.621736831286, + "harmonic_mean_rank": 1756.6468268960064, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006602224441773459, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002877892705388431, + "inverse_arithmetic_mean_rank": 0.00011096378180558028, + "inverse_geometric_mean_rank": 0.00014093197708233148, + "inverse_harmonic_mean_rank": 0.0005692663913366122, + "inverse_median_rank": 0.00010339123242349049, + "median_absolute_deviation": 5564.206126051524, + "median_rank": 9672.0, + "standard_deviation": 4425.760932990495, + "variance": 19587359.835984897, + "z_arithmetic_mean_rank": -93.9307057119106, + "z_geometric_mean_rank": -75.51395097640778, + "z_hits_at_k": -0.1879214636632005, + "z_inverse_harmonic_mean_rank": -2.826717203602767 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.2233279329594688, + "adjusted_arithmetic_mean_rank_index": -0.22335825274542032, + "adjusted_geometric_mean_rank_index": -0.31018253160783194, + "adjusted_hits_at_k": -2.0174999586579823e-05, + "adjusted_inverse_harmonic_mean_rank": -0.00012285834022563564, + "arithmetic_mean_rank": 9011.949823094243, + "count": 59071.0, + "geometric_mean_rank": 7095.62210581432, + "harmonic_mean_rank": 1756.646859397724, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006602224441773459, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002877892705388431, + "inverse_arithmetic_mean_rank": 0.00011096377805359897, + "inverse_geometric_mean_rank": 0.00014093196975365648, + "inverse_harmonic_mean_rank": 0.0005692663808039685, + "inverse_median_rank": 0.00010339123242349049, + "median_absolute_deviation": 5564.206126051524, + "median_rank": 9672.0, + "standard_deviation": 4425.760798550912, + "variance": 19587358.645990007, + "z_arithmetic_mean_rank": -93.93072310941398, + "z_geometric_mean_rank": -75.51396756535735, + "z_hits_at_k": -0.1879214636632005, + "z_inverse_harmonic_mean_rank": -2.826717446105034 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.223327850922706, + "adjusted_arithmetic_mean_rank_index": -0.22335817069751962, + "adjusted_geometric_mean_rank_index": -0.3101824348794342, + "adjusted_hits_at_k": -2.0174999586579823e-05, + "adjusted_inverse_harmonic_mean_rank": -0.00012285830888574488, + "arithmetic_mean_rank": 9011.94921875, + "count": 59071.0, + "geometric_mean_rank": 7095.62158203125, + "harmonic_mean_rank": 1756.6467627557643, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006602224441773459, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002877892705388431, + "inverse_arithmetic_mean_rank": 0.00011096378875663504, + "inverse_geometric_mean_rank": 0.0001409319811500609, + "inverse_harmonic_mean_rank": 0.0005692664121221707, + "inverse_median_rank": 0.00010339123400626704, + "median_absolute_deviation": 5564.206126051524, + "median_rank": 9672.0, + "standard_deviation": 4425.7607421875, + "variance": 19587360.0, + "z_arithmetic_mean_rank": -93.93068860512074, + "z_geometric_mean_rank": -75.5139440168195, + "z_hits_at_k": -0.1879214636632005, + "z_inverse_harmonic_mean_rank": -2.8267167250386773 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.3048588325485093, + "adjusted_arithmetic_mean_rank_index": -0.30489999655764954, + "adjusted_geometric_mean_rank_index": -0.42574765680192117, + "adjusted_hits_at_k": -0.00013459288103333754, + "adjusted_inverse_harmonic_mean_rank": -0.00021814399979022973, + "arithmetic_mean_rank": 9665.031707606102, + "count": 59071.0, + "geometric_mean_rank": 7766.080963854441, + "harmonic_mean_rank": 2126.5089606148067, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.0005417209798378223, + "hits_at_3": 8.464390309965973e-05, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00010346577541106552, + "inverse_geometric_mean_rank": 0.00012876507528755953, + "inverse_harmonic_mean_rank": 0.0004702543081271026, + "inverse_median_rank": 9.285051067780872e-05, + "median_absolute_deviation": 5008.230294111923, + "median_rank": 10770.0, + "standard_deviation": 4442.866877124927, + "variance": 19739066.087853797, + "z_arithmetic_mean_rank": -128.2600694817313, + "z_geometric_mean_rank": -103.64746647372786, + "z_hits_at_k": -1.2575282805239896, + "z_inverse_harmonic_mean_rank": -5.034420904848641 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.304858885115578, + "adjusted_arithmetic_mean_rank_index": -0.3049000491318161, + "adjusted_geometric_mean_rank_index": -0.42574777929683894, + "adjusted_hits_at_k": -0.00013459288103333754, + "adjusted_inverse_harmonic_mean_rank": -0.00021814404050530368, + "arithmetic_mean_rank": 9665.032096968056, + "count": 59071.0, + "geometric_mean_rank": 7766.081631001217, + "harmonic_mean_rank": 2126.5091446033134, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.0005417209798378223, + "hits_at_3": 8.464390309965973e-05, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00010346577124288107, + "inverse_geometric_mean_rank": 0.0001287650642259703, + "inverse_harmonic_mean_rank": 0.0004702542674400507, + "inverse_median_rank": 9.285051067780872e-05, + "median_absolute_deviation": 5008.230294111923, + "median_rank": 10770.0, + "standard_deviation": 4442.866722763831, + "variance": 19739064.71624222, + "z_arithmetic_mean_rank": -128.26009159772457, + "z_geometric_mean_rank": -103.6474962948852, + "z_hits_at_k": -1.2575282805239896, + "z_inverse_harmonic_mean_rank": -5.034421844488602 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.3048587707679191, + "adjusted_arithmetic_mean_rank_index": -0.30489993476871713, + "adjusted_geometric_mean_rank_index": -0.42574839070692905, + "adjusted_hits_at_k": -0.00013459288103333754, + "adjusted_inverse_harmonic_mean_rank": -0.00021814402267636868, + "arithmetic_mean_rank": 9665.03125, + "count": 59071.0, + "geometric_mean_rank": 7766.0849609375, + "harmonic_mean_rank": 2126.5090640356275, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.0005417209798378223, + "hits_at_3": 8.464390309965973e-05, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00010346578346798196, + "inverse_geometric_mean_rank": 0.00012876500841230154, + "inverse_harmonic_mean_rank": 0.000470254285256715, + "inverse_median_rank": 9.285051055485383e-05, + "median_absolute_deviation": 5008.230294111923, + "median_rank": 10770.0, + "standard_deviation": 4442.86669921875, + "variance": 19739064.0, + "z_arithmetic_mean_rank": -128.26004348942945, + "z_geometric_mean_rank": -103.64764514152202, + "z_hits_at_k": -1.2575282805239896, + "z_inverse_harmonic_mean_rank": -5.034421433024784 + } + } + }, + "times": { + "evaluation": 29.497436046600342, + "training": 13.04035758972168 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json new file mode 100644 index 0000000..9ec4235 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json new file mode 100644 index 0000000..e726c90 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.006185159085706167, + 0.002151926759158413, + 0.001296131423698241, + 0.0009291403910987688, + 0.000743997096846818 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6804716586083929, + "adjusted_arithmetic_mean_rank_index": 0.3195439569083198, + "adjusted_geometric_mean_rank_index": 0.6640993333234186, + "adjusted_hits_at_k": 0.007357450518285132, + "adjusted_inverse_harmonic_mean_rank": 0.004073613974139381, + "arithmetic_mean_rank": 13924.6501, + "count": 10000.0, + "geometric_mean_rank": 5058.878653374096, + "harmonic_mean_rank": 230.09195259808598, + "hits_at_1": 0.0006, + "hits_at_10": 0.0076, + "hits_at_3": 0.0024, + "hits_at_5": 0.003, + "inverse_arithmetic_mean_rank": 7.181508998922709e-05, + "inverse_geometric_mean_rank": 0.00019767226464960467, + "inverse_harmonic_mean_rank": 0.004346088547245952, + "inverse_median_rank": 9.72951936174353e-05, + "median_absolute_deviation": 14072.860258055174, + "median_rank": 10278.0, + "standard_deviation": 12819.282193674886, + "variance": 164333995.96107, + "z_arithmetic_mean_rank": 55.34521519835033, + "z_geometric_mean_rank": 66.46364729994838, + "z_hits_at_k": 47.062018740431306, + "z_inverse_harmonic_mean_rank": 64.29705082532885 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6804717367974064, + "adjusted_arithmetic_mean_rank_index": 0.3195438787154853, + "adjusted_geometric_mean_rank_index": 0.664099301991363, + "adjusted_hits_at_k": 0.007357450518285132, + "adjusted_inverse_harmonic_mean_rank": 0.0040736139668042775, + "arithmetic_mean_rank": 13924.6517, + "count": 10000.0, + "geometric_mean_rank": 5058.879125161595, + "harmonic_mean_rank": 230.0919529863171, + "hits_at_1": 0.0006, + "hits_at_10": 0.0076, + "hits_at_3": 0.0024, + "hits_at_5": 0.003, + "inverse_arithmetic_mean_rank": 7.18150817373766e-05, + "inverse_geometric_mean_rank": 0.0001976722462148287, + "inverse_harmonic_mean_rank": 0.0043460885399128546, + "inverse_median_rank": 9.72951936174353e-05, + "median_absolute_deviation": 14072.860258055174, + "median_rank": 10278.0, + "standard_deviation": 12819.283148545675, + "variance": 164334020.4425871, + "z_arithmetic_mean_rank": 55.34520165530199, + "z_geometric_mean_rank": 66.4636441642086, + "z_hits_at_k": 47.062018740431306, + "z_inverse_harmonic_mean_rank": 64.29705070955313 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6804717205334807, + "adjusted_arithmetic_mean_rank_index": 0.31954389498020574, + "adjusted_geometric_mean_rank_index": 0.6640993489570108, + "adjusted_hits_at_k": 0.007357450518285132, + "adjusted_inverse_harmonic_mean_rank": 0.004073613836279541, + "arithmetic_mean_rank": 13924.6513671875, + "count": 10000.0, + "geometric_mean_rank": 5058.87841796875, + "harmonic_mean_rank": 230.09195989470737, + "hits_at_1": 0.0006, + "hits_at_10": 0.0076, + "hits_at_3": 0.0024, + "hits_at_5": 0.003, + "inverse_arithmetic_mean_rank": 7.181508408393711e-05, + "inverse_geometric_mean_rank": 0.00019767227058764547, + "inverse_harmonic_mean_rank": 0.004346088409423828, + "inverse_median_rank": 9.72951966105029e-05, + "median_absolute_deviation": 14072.860258055174, + "median_rank": 10278.0, + "standard_deviation": 12819.2822265625, + "variance": 164334000.0, + "z_arithmetic_mean_rank": 55.34520447236186, + "z_geometric_mean_rank": 66.4636488645721, + "z_hits_at_k": 47.062018740431306, + "z_inverse_harmonic_mean_rank": 64.29704864937864 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.692792955100394, + "adjusted_arithmetic_mean_rank_index": 0.307222058616213, + "adjusted_geometric_mean_rank_index": 0.6511900481613886, + "adjusted_hits_at_k": 0.008157640382758721, + "adjusted_inverse_harmonic_mean_rank": 0.004470857752830758, + "arithmetic_mean_rank": 14176.455, + "count": 5000.0, + "geometric_mean_rank": 5253.4022629084075, + "harmonic_mean_rank": 210.82682457987022, + "hits_at_1": 0.0008, + "hits_at_10": 0.0084, + "hits_at_3": 0.0032, + "hits_at_5": 0.004, + "inverse_arithmetic_mean_rank": 7.053949665131374e-05, + "inverse_geometric_mean_rank": 0.0001903528323084051, + "inverse_harmonic_mean_rank": 0.004743229434834831, + "inverse_median_rank": 9.173891105912573e-05, + "median_absolute_deviation": 14703.707502029307, + "median_rank": 10900.5, + "standard_deviation": 12791.397666837465, + "variance": 163619854.27117497, + "z_arithmetic_mean_rank": 37.625894388727076, + "z_geometric_mean_rank": 46.08674352328238, + "z_hits_at_k": 36.89671389039824, + "z_inverse_harmonic_mean_rank": 49.897865756659414 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6927929844219537, + "adjusted_arithmetic_mean_rank_index": 0.3072220292932203, + "adjusted_geometric_mean_rank_index": 0.6511900380555931, + "adjusted_hits_at_k": 0.008157640382758721, + "adjusted_inverse_harmonic_mean_rank": 0.004470857751334049, + "arithmetic_mean_rank": 14176.4556, + "count": 5000.0, + "geometric_mean_rank": 5253.402415082114, + "harmonic_mean_rank": 210.82682464637767, + "hits_at_1": 0.0008, + "hits_at_10": 0.0084, + "hits_at_3": 0.0032, + "hits_at_5": 0.004, + "inverse_arithmetic_mean_rank": 7.053949366582152e-05, + "inverse_geometric_mean_rank": 0.00019035282679451262, + "inverse_harmonic_mean_rank": 0.004743229433338532, + "inverse_median_rank": 9.173891105912573e-05, + "median_absolute_deviation": 14703.707502029307, + "median_rank": 10900.5, + "standard_deviation": 12791.398034977592, + "variance": 163619863.68922862, + "z_arithmetic_mean_rank": 37.62589079750113, + "z_geometric_mean_rank": 46.086742808063825, + "z_hits_at_k": 36.89671389039824, + "z_inverse_harmonic_mean_rank": 49.89786573995511 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6927929111944124, + "adjusted_arithmetic_mean_rank_index": 0.30722210252434023, + "adjusted_geometric_mean_rank_index": 0.651189977939602, + "adjusted_hits_at_k": 0.008157640382758721, + "adjusted_inverse_harmonic_mean_rank": 0.0044708576117805775, + "arithmetic_mean_rank": 14176.4541015625, + "count": 5000.0, + "geometric_mean_rank": 5253.4033203125, + "harmonic_mean_rank": 210.8268308475465, + "hits_at_1": 0.0008, + "hits_at_10": 0.0084, + "hits_at_3": 0.0032, + "hits_at_5": 0.004, + "inverse_arithmetic_mean_rank": 7.053949957480654e-05, + "inverse_geometric_mean_rank": 0.000190352788195014, + "inverse_harmonic_mean_rank": 0.004743229293823242, + "inverse_median_rank": 9.173891157843173e-05, + "median_absolute_deviation": 14703.707502029307, + "median_rank": 10900.5, + "standard_deviation": 12791.3984375, + "variance": 163619872.0, + "z_arithmetic_mean_rank": 37.62589976621385, + "z_geometric_mean_rank": 46.08673855346831, + "z_hits_at_k": 36.89671389039824, + "z_inverse_harmonic_mean_rank": 49.897864182441815 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.668150933393428, + "adjusted_arithmetic_mean_rank_index": 0.33186528386756864, + "adjusted_geometric_mean_rank_index": 0.6765632325264472, + "adjusted_hits_at_k": 0.006557260662945978, + "adjusted_inverse_harmonic_mean_rank": 0.003676370200070375, + "arithmetic_mean_rank": 13672.8452, + "count": 5000.0, + "geometric_mean_rank": 4871.557887401455, + "harmonic_mean_rank": 253.23202184119089, + "hits_at_1": 0.0004, + "hits_at_10": 0.0068, + "hits_at_3": 0.0016, + "hits_at_5": 0.002, + "inverse_arithmetic_mean_rank": 7.313766705996204e-05, + "inverse_geometric_mean_rank": 0.00020527314323537914, + "inverse_harmonic_mean_rank": 0.003948947659657075, + "inverse_median_rank": 0.00010430791697089809, + "median_absolute_deviation": 13273.737662280655, + "median_rank": 9587.0, + "standard_deviation": 12842.16984141064, + "variance": 164921326.23563695, + "z_arithmetic_mean_rank": 40.643989861490056, + "z_geometric_mean_rank": 47.88248130991923, + "z_hits_at_k": 29.658946798068516, + "z_inverse_harmonic_mean_rank": 41.031792183268756 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6681510604476293, + "adjusted_arithmetic_mean_rank_index": 0.33186515680715833, + "adjusted_geometric_mean_rank_index": 0.6765631815579028, + "adjusted_hits_at_k": 0.006557260662945978, + "adjusted_inverse_harmonic_mean_rank": 0.003676370186896874, + "arithmetic_mean_rank": 13672.8478, + "count": 5000.0, + "geometric_mean_rank": 4871.558654924727, + "harmonic_mean_rank": 253.23202268572967, + "hits_at_1": 0.0004, + "hits_at_10": 0.0068, + "hits_at_3": 0.0016, + "hits_at_5": 0.002, + "inverse_arithmetic_mean_rank": 7.313765315225699e-05, + "inverse_geometric_mean_rank": 0.00020527311089420754, + "inverse_harmonic_mean_rank": 0.003948947646487178, + "inverse_median_rank": 0.00010430791697089809, + "median_absolute_deviation": 13273.737662280655, + "median_rank": 9587.0, + "standard_deviation": 12842.171420279172, + "variance": 164921366.78783515, + "z_arithmetic_mean_rank": 40.64397430022986, + "z_geometric_mean_rank": 47.88247770271704, + "z_hits_at_k": 29.658946798068516, + "z_inverse_harmonic_mean_rank": 41.03179203623995 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6681510057013178, + "adjusted_arithmetic_mean_rank_index": 0.33186521155614523, + "adjusted_geometric_mean_rank_index": 0.6765632504704139, + "adjusted_hits_at_k": 0.006557260662945978, + "adjusted_inverse_harmonic_mean_rank": 0.003676370446974989, + "arithmetic_mean_rank": 13672.8466796875, + "count": 5000.0, + "geometric_mean_rank": 4871.5576171875, + "harmonic_mean_rank": 253.2320060124054, + "hits_at_1": 0.0004, + "hits_at_10": 0.0068, + "hits_at_3": 0.0016, + "hits_at_5": 0.002, + "inverse_arithmetic_mean_rank": 7.313765672734007e-05, + "inverse_geometric_mean_rank": 0.0002052731579169631, + "inverse_harmonic_mean_rank": 0.0039489479064941406, + "inverse_median_rank": 0.00010430791735416278, + "median_absolute_deviation": 13273.737662280655, + "median_rank": 9587.0, + "standard_deviation": 12842.1708984375, + "variance": 164921344.0, + "z_arithmetic_mean_rank": 40.643981005412286, + "z_geometric_mean_rank": 47.88248257986948, + "z_hits_at_k": 29.658946798068516, + "z_inverse_harmonic_mean_rank": 41.031794938959365 + } + } + }, + "times": { + "evaluation": 9.06259036064148, + "training": 4.838503360748291 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json new file mode 100644 index 0000000..1f8d7d2 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json new file mode 100644 index 0000000..b69cd51 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 0.002535040261322091, + 0.0013169600454131234, + 0.0010097262843328746, + 0.0008632518252629737, + 0.0007637009042292636 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.2325918387095582, + "adjusted_arithmetic_mean_rank_index": 0.7675120637406266, + "adjusted_geometric_mean_rank_index": 0.9623243784288315, + "adjusted_hits_at_k": 0.21144731304754785, + "adjusted_inverse_harmonic_mean_rank": 0.11782603426599923, + "arithmetic_mean_rank": 1718.1215825024124, + "count": 118142.0, + "geometric_mean_rank": 205.60158202534976, + "harmonic_mean_rank": 8.443459883456963, + "hits_at_1": 0.06818066394677591, + "hits_at_10": 0.21198219092278783, + "hits_at_3": 0.1368861200927697, + "hits_at_5": 0.1675610705760864, + "inverse_arithmetic_mean_rank": 0.0005820309867381554, + "inverse_geometric_mean_rank": 0.004863775804393881, + "inverse_harmonic_mean_rank": 0.11843486127757559, + "inverse_median_rank": 0.0021551724137931034, + "median_absolute_deviation": 684.9622249495881, + "median_rank": 464.0, + "standard_deviation": 2626.3693262666297, + "variance": 6897815.837954232, + "z_arithmetic_mean_rank": 456.52944766267717, + "z_geometric_mean_rank": 331.34201080413635, + "z_hits_at_k": 2789.6223269150987, + "z_inverse_harmonic_mean_rank": 3839.6995940773863 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23259198767297054, + "adjusted_arithmetic_mean_rank_index": 0.7675119147570456, + "adjusted_geometric_mean_rank_index": 0.9623243588106034, + "adjusted_hits_at_k": 0.21144731304754785, + "adjusted_inverse_harmonic_mean_rank": 0.11782603255173432, + "arithmetic_mean_rank": 1718.1226828731526, + "count": 118142.0, + "geometric_mean_rank": 205.60168856428186, + "harmonic_mean_rank": 8.44346000558602, + "hits_at_1": 0.06818066394677591, + "hits_at_10": 0.21198219092278783, + "hits_at_3": 0.1368861200927697, + "hits_at_5": 0.1675610705760864, + "inverse_arithmetic_mean_rank": 0.0005820306139767256, + "inverse_geometric_mean_rank": 0.004863773284076642, + "inverse_harmonic_mean_rank": 0.11843485956449376, + "inverse_median_rank": 0.0021551724137931034, + "median_absolute_deviation": 684.9622249495881, + "median_rank": 464.0, + "standard_deviation": 2626.370702761352, + "variance": 6897823.068323158, + "z_arithmetic_mean_rank": 456.5293590446669, + "z_geometric_mean_rank": 331.3420040493006, + "z_hits_at_k": 2789.6223269150987, + "z_inverse_harmonic_mean_rank": 3839.699538213142 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2325918882218207, + "adjusted_arithmetic_mean_rank_index": 0.7675120142216605, + "adjusted_geometric_mean_rank_index": 0.9623243483069595, + "adjusted_hits_at_k": 0.21144731304754785, + "adjusted_inverse_harmonic_mean_rank": 0.11782603811895852, + "arithmetic_mean_rank": 1718.1219482421875, + "count": 118142.0, + "geometric_mean_rank": 205.60174560546875, + "harmonic_mean_rank": 8.443459608961314, + "hits_at_1": 0.06818066394677591, + "hits_at_10": 0.21198219092278783, + "hits_at_3": 0.1368861200927697, + "hits_at_5": 0.1675610705760864, + "inverse_arithmetic_mean_rank": 0.0005820308579131961, + "inverse_geometric_mean_rank": 0.004863772075623274, + "inverse_harmonic_mean_rank": 0.11843486512787578, + "inverse_median_rank": 0.0021551724057644606, + "median_absolute_deviation": 684.9622249495881, + "median_rank": 464.0, + "standard_deviation": 2626.369873046875, + "variance": 6897819.0, + "z_arithmetic_mean_rank": 456.5294182079398, + "z_geometric_mean_rank": 331.34200043274615, + "z_hits_at_k": 2789.6223269150987, + "z_inverse_harmonic_mean_rank": 3839.6997196371262 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.2635138727776829, + "adjusted_arithmetic_mean_rank_index": 0.7365861151702535, + "adjusted_geometric_mean_rank_index": 0.9456343343807989, + "adjusted_hits_at_k": 0.16618117235305171, + "adjusted_inverse_harmonic_mean_rank": 0.08985695538975279, + "arithmetic_mean_rank": 1941.2405579726092, + "count": 59071.0, + "geometric_mean_rank": 295.38940284600267, + "harmonic_mean_rank": 11.05133438110848, + "hits_at_1": 0.049127321359042506, + "hits_at_10": 0.16674848910632967, + "hits_at_3": 0.10329941934282473, + "hits_at_5": 0.1294035990587598, + "inverse_arithmetic_mean_rank": 0.0005151345081334892, + "inverse_geometric_mean_rank": 0.0033853617982407334, + "inverse_harmonic_mean_rank": 0.09048681050764633, + "inverse_median_rank": 0.001440922190201729, + "median_absolute_deviation": 1022.9955307688654, + "median_rank": 694.0, + "standard_deviation": 2756.3004548828058, + "variance": 7597192.197587161, + "z_arithmetic_mean_rank": 309.76274921507064, + "z_geometric_mean_rank": 230.21477091238273, + "z_hits_at_k": 1547.9063088866371, + "z_inverse_harmonic_mean_rank": 2067.4235301210465 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2635140313396026, + "adjusted_arithmetic_mean_rank_index": 0.7365859565868069, + "adjusted_geometric_mean_rank_index": 0.9456343123996525, + "adjusted_hits_at_k": 0.16618117235305171, + "adjusted_inverse_harmonic_mean_rank": 0.08985695495606277, + "arithmetic_mean_rank": 1941.241726058472, + "count": 59071.0, + "geometric_mean_rank": 295.38952187363765, + "harmonic_mean_rank": 11.05133443403925, + "hits_at_1": 0.049127321359042506, + "hits_at_10": 0.16674848910632967, + "hits_at_3": 0.10329941934282473, + "hits_at_5": 0.1294035990587598, + "inverse_arithmetic_mean_rank": 0.000515134198166251, + "inverse_geometric_mean_rank": 0.0033853604341043016, + "inverse_harmonic_mean_rank": 0.09048681007425645, + "inverse_median_rank": 0.001440922190201729, + "median_absolute_deviation": 1022.9955307688654, + "median_rank": 694.0, + "standard_deviation": 2756.30194280584, + "variance": 7597200.399915249, + "z_arithmetic_mean_rank": 309.76268252464115, + "z_geometric_mean_rank": 230.21476556107046, + "z_hits_at_k": 1547.9063088866371, + "z_inverse_harmonic_mean_rank": 2067.4235201427323 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.26351392827356934, + "adjusted_arithmetic_mean_rank_index": 0.7365860596668328, + "adjusted_geometric_mean_rank_index": 0.9456343115698167, + "adjusted_hits_at_k": 0.16618117235305171, + "adjusted_inverse_harmonic_mean_rank": 0.08985696537040451, + "arithmetic_mean_rank": 1941.240966796875, + "count": 59071.0, + "geometric_mean_rank": 295.3895263671875, + "harmonic_mean_rank": 11.05133316299531, + "hits_at_1": 0.049127321359042506, + "hits_at_10": 0.16674848910632967, + "hits_at_3": 0.10329941934282473, + "hits_at_5": 0.1294035990587598, + "inverse_arithmetic_mean_rank": 0.0005151344230398536, + "inverse_geometric_mean_rank": 0.0033853603526949883, + "inverse_harmonic_mean_rank": 0.09048682048139105, + "inverse_median_rank": 0.0014409221475943923, + "median_absolute_deviation": 1022.9955307688654, + "median_rank": 694.0, + "standard_deviation": 2756.301025390625, + "variance": 7597195.5, + "z_arithmetic_mean_rank": 309.7627258737507, + "z_geometric_mean_rank": 230.21476535904688, + "z_hits_at_k": 1547.9063088866371, + "z_inverse_harmonic_mean_rank": 2067.423759755293 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.20183765718366314, + "adjusted_arithmetic_mean_rank_index": 0.7982701158524677, + "adjusted_geometric_mean_rank_index": 0.9739078873536554, + "adjusted_hits_at_k": 0.2567132652680245, + "adjusted_inverse_harmonic_mean_rank": 0.14579500702367879, + "arithmetic_mean_rank": 1495.0026070322156, + "count": 59071.0, + "geometric_mean_rank": 143.10604958758364, + "harmonic_mean_rank": 6.831398460466994, + "hits_at_1": 0.08723400653450931, + "hits_at_10": 0.257215892739246, + "hits_at_3": 0.1704728208427147, + "hits_at_5": 0.205718542093413, + "inverse_arithmetic_mean_rank": 0.0006688951546279485, + "inverse_geometric_mean_rank": 0.0069878247836614405, + "inverse_harmonic_mean_rank": 0.14638291204750484, + "inverse_median_rank": 0.0037313432835820895, + "median_absolute_deviation": 395.85479234099574, + "median_rank": 268.0, + "standard_deviation": 2469.59011252709, + "variance": 6098875.323891563, + "z_arithmetic_mean_rank": 335.8024981317717, + "z_geometric_mean_rank": 237.09604384258736, + "z_hits_at_k": 2398.523522059363, + "z_inverse_harmonic_mean_rank": 3364.7197809171057 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.20183779660067122, + "adjusted_arithmetic_mean_rank_index": 0.7982699764166346, + "adjusted_geometric_mean_rank_index": 0.973907870710314, + "adjusted_hits_at_k": 0.2567132652680245, + "adjusted_inverse_harmonic_mean_rank": 0.14579500402884382, + "arithmetic_mean_rank": 1495.0036396878334, + "count": 59071.0, + "geometric_mean_rank": 143.1061402325816, + "harmonic_mean_rank": 6.831398600133778, + "hits_at_1": 0.08723400653450931, + "hits_at_10": 0.257215892739246, + "hits_at_3": 0.1704728208427147, + "hits_at_5": 0.205718542093413, + "inverse_arithmetic_mean_rank": 0.0006688946925967395, + "inverse_geometric_mean_rank": 0.006987820357496621, + "inverse_harmonic_mean_rank": 0.14638290905473106, + "inverse_median_rank": 0.0037313432835820895, + "median_absolute_deviation": 395.85479234099574, + "median_rank": 268.0, + "standard_deviation": 2469.591367385949, + "variance": 6098881.521867202, + "z_arithmetic_mean_rank": 335.8024394763114, + "z_geometric_mean_rank": 237.0960397907972, + "z_hits_at_k": 2398.523522059363, + "z_inverse_harmonic_mean_rank": 3364.719711801018 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2018377172252792, + "adjusted_arithmetic_mean_rank_index": 0.7982700558027443, + "adjusted_geometric_mean_rank_index": 0.9739078763312524, + "adjusted_hits_at_k": 0.2567132652680245, + "adjusted_inverse_harmonic_mean_rank": 0.14579501302066805, + "arithmetic_mean_rank": 1495.0030517578125, + "count": 59071.0, + "geometric_mean_rank": 143.10610961914062, + "harmonic_mean_rank": 6.8313981807921, + "hits_at_1": 0.08723400653450931, + "hits_at_10": 0.257215892739246, + "hits_at_3": 0.1704728208427147, + "hits_at_5": 0.205718542093413, + "inverse_arithmetic_mean_rank": 0.0006688949652016163, + "inverse_geometric_mean_rank": 0.006987821776419878, + "inverse_harmonic_mean_rank": 0.14638291804036668, + "inverse_median_rank": 0.0037313431967049837, + "median_absolute_deviation": 395.85479234099574, + "median_rank": 268.0, + "standard_deviation": 2469.5908203125, + "variance": 6098878.5, + "z_arithmetic_mean_rank": 335.8024728710903, + "z_geometric_mean_rank": 237.09604115920408, + "z_hits_at_k": 2398.523522059363, + "z_inverse_harmonic_mean_rank": 3364.719919318199 + } + } + }, + "times": { + "evaluation": 34.54206871986389, + "training": 83.16322159767151 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json new file mode 100644 index 0000000..e1f0574 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json new file mode 100644 index 0000000..9ce9b00 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0044969535855334955, + 0.0036679787342402833, + 0.0031641818598498335, + 0.0027769141778763067, + 0.002474804447551882 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5926951523251482, + "adjusted_arithmetic_mean_rank_index": 0.40732475287521164, + "adjusted_geometric_mean_rank_index": 0.7386937354137952, + "adjusted_hits_at_k": 0.03866510045659395, + "adjusted_inverse_harmonic_mean_rank": 0.015044241141624008, + "arithmetic_mean_rank": 12128.459, + "count": 10000.0, + "geometric_mean_rank": 3935.6613709344965, + "harmonic_mean_rank": 65.30094412759377, + "hits_at_1": 0.003, + "hits_at_10": 0.0389, + "hits_at_3": 0.0171, + "hits_at_5": 0.0254, + "inverse_arithmetic_mean_rank": 8.245070540288754e-05, + "inverse_geometric_mean_rank": 0.0002540869007138581, + "inverse_harmonic_mean_rank": 0.015313714271053502, + "inverse_median_rank": 0.00011371389583807141, + "median_absolute_deviation": 11788.170239338042, + "median_rank": 8794.0, + "standard_deviation": 11405.352572828207, + "variance": 130082067.310519, + "z_arithmetic_mean_rank": 70.5489045125688, + "z_geometric_mean_rank": 73.92912088546521, + "z_hits_at_k": 247.3217696492246, + "z_inverse_harmonic_mean_rank": 237.45508127481824 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5926953868921885, + "adjusted_arithmetic_mean_rank_index": 0.407324518296708, + "adjusted_geometric_mean_rank_index": 0.7386935859087462, + "adjusted_hits_at_k": 0.03866510045659395, + "adjusted_inverse_harmonic_mean_rank": 0.015044240820136027, + "arithmetic_mean_rank": 12128.4638, + "count": 10000.0, + "geometric_mean_rank": 3935.6636221309673, + "harmonic_mean_rank": 65.30094549811206, + "hits_at_1": 0.003, + "hits_at_10": 0.0389, + "hits_at_3": 0.0171, + "hits_at_5": 0.0254, + "inverse_arithmetic_mean_rank": 8.245067277193011e-05, + "inverse_geometric_mean_rank": 0.0002540867553763524, + "inverse_harmonic_mean_rank": 0.015313713949653477, + "inverse_median_rank": 0.00011371389583807141, + "median_absolute_deviation": 11788.170239338042, + "median_rank": 8794.0, + "standard_deviation": 11405.35493635729, + "variance": 130082121.22428958, + "z_arithmetic_mean_rank": 70.54886388342378, + "z_geometric_mean_rank": 73.92910592286799, + "z_hits_at_k": 247.3217696492246, + "z_inverse_harmonic_mean_rank": 237.45507620052078 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5926952470071567, + "adjusted_arithmetic_mean_rank_index": 0.4073246581885761, + "adjusted_geometric_mean_rank_index": 0.7386937998689465, + "adjusted_hits_at_k": 0.03866510045659395, + "adjusted_inverse_harmonic_mean_rank": 0.015044241470269793, + "arithmetic_mean_rank": 12128.4609375, + "count": 10000.0, + "geometric_mean_rank": 3935.660400390625, + "harmonic_mean_rank": 65.30094272656147, + "hits_at_1": 0.003, + "hits_at_10": 0.0389, + "hits_at_3": 0.0171, + "hits_at_5": 0.0254, + "inverse_arithmetic_mean_rank": 8.245069329859689e-05, + "inverse_geometric_mean_rank": 0.00025408697547391057, + "inverse_harmonic_mean_rank": 0.015313714599609373, + "inverse_median_rank": 0.00011371389700798318, + "median_absolute_deviation": 11788.170239338042, + "median_rank": 8794.0, + "standard_deviation": 11405.353515625, + "variance": 130082088.0, + "z_arithmetic_mean_rank": 70.5488881127837, + "z_geometric_mean_rank": 73.92912733619363, + "z_hits_at_k": 247.3217696492246, + "z_inverse_harmonic_mean_rank": 237.45508646209296 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6001467270400189, + "adjusted_arithmetic_mean_rank_index": 0.39987281445105116, + "adjusted_geometric_mean_rank_index": 0.7263469317031201, + "adjusted_hits_at_k": 0.03576438617283119, + "adjusted_inverse_harmonic_mean_rank": 0.014352630823535656, + "arithmetic_mean_rank": 12280.6576, + "count": 5000.0, + "geometric_mean_rank": 4121.685168522348, + "harmonic_mean_rank": 68.38869911196008, + "hits_at_1": 0.003, + "hits_at_10": 0.036, + "hits_at_3": 0.0158, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 8.142886420023631e-05, + "inverse_geometric_mean_rank": 0.00024261921013207972, + "inverse_harmonic_mean_rank": 0.014622298902964747, + "inverse_median_rank": 0.00011096316023080337, + "median_absolute_deviation": 11939.395665625612, + "median_rank": 9012.0, + "standard_deviation": 11401.945645159087, + "variance": 130004364.49516225, + "z_arithmetic_mean_rank": 48.97295576114048, + "z_geometric_mean_rank": 51.40582974945663, + "z_hits_at_k": 161.76103164263458, + "z_inverse_harmonic_mean_rank": 160.18529008985547 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6001469225170842, + "adjusted_arithmetic_mean_rank_index": 0.3998726189644326, + "adjusted_geometric_mean_rank_index": 0.7263468231408228, + "adjusted_hits_at_k": 0.03576438617283119, + "adjusted_inverse_harmonic_mean_rank": 0.014352630727401386, + "arithmetic_mean_rank": 12280.6616, + "count": 5000.0, + "geometric_mean_rank": 4121.686803260271, + "harmonic_mean_rank": 68.38869956145841, + "hits_at_1": 0.003, + "hits_at_10": 0.036, + "hits_at_3": 0.0158, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 8.142883767760525e-05, + "inverse_geometric_mean_rank": 0.0002426191139047722, + "inverse_harmonic_mean_rank": 0.014622298806856778, + "inverse_median_rank": 0.00011096316023080337, + "median_absolute_deviation": 11939.395665625612, + "median_rank": 9012.0, + "standard_deviation": 11401.947519572499, + "variance": 130004407.23908545, + "z_arithmetic_mean_rank": 48.97293181963414, + "z_geometric_mean_rank": 51.405822066165456, + "z_hits_at_k": 161.76103164263458, + "z_inverse_harmonic_mean_rank": 160.18528901693048 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6001468519620811, + "adjusted_arithmetic_mean_rank_index": 0.399872689522884, + "adjusted_geometric_mean_rank_index": 0.7263468741503063, + "adjusted_hits_at_k": 0.03576438617283119, + "adjusted_inverse_harmonic_mean_rank": 0.014352631114986586, + "arithmetic_mean_rank": 12280.66015625, + "count": 5000.0, + "geometric_mean_rank": 4121.68603515625, + "harmonic_mean_rank": 68.38869774921292, + "hits_at_1": 0.003, + "hits_at_10": 0.036, + "hits_at_3": 0.0158, + "hits_at_5": 0.0242, + "inverse_arithmetic_mean_rank": 8.142885053530335e-05, + "inverse_geometric_mean_rank": 0.00024261916405521333, + "inverse_harmonic_mean_rank": 0.014622299194335936, + "inverse_median_rank": 0.00011096316302428022, + "median_absolute_deviation": 11939.395665625612, + "median_rank": 9012.0, + "standard_deviation": 11401.947265625, + "variance": 130004392.0, + "z_arithmetic_mean_rank": 48.97294046102158, + "z_geometric_mean_rank": 51.40582567626513, + "z_hits_at_k": 161.76103164263458, + "z_inverse_harmonic_mean_rank": 160.1852933426495 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5852439231026143, + "adjusted_arithmetic_mean_rank_index": 0.41477634577328704, + "adjusted_geometric_mean_rank_index": 0.7505085129139782, + "adjusted_hits_at_k": 0.041565814707244096, + "adjusted_inverse_harmonic_mean_rank": 0.015735851451664715, + "arithmetic_mean_rank": 11976.2604, + "count": 5000.0, + "geometric_mean_rank": 3758.0333755135143, + "harmonic_mean_rank": 62.47996876916217, + "hits_at_1": 0.003, + "hits_at_10": 0.0418, + "hits_at_3": 0.0184, + "hits_at_5": 0.0266, + "inverse_arithmetic_mean_rank": 8.34985184523877e-05, + "inverse_geometric_mean_rank": 0.0002660966255690466, + "inverse_harmonic_mean_rank": 0.016005129639142256, + "inverse_median_rank": 0.00011619125079881485, + "median_absolute_deviation": 11648.064329689261, + "median_rank": 8606.5, + "standard_deviation": 11406.727896210721, + "variance": 130113441.29819185, + "z_arithmetic_mean_rank": 50.79821967495293, + "z_geometric_mean_rank": 53.11581847027738, + "z_hits_at_k": 188.00507565405638, + "z_inverse_harmonic_mean_rank": 175.62708635249376 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5852441967578171, + "adjusted_arithmetic_mean_rank_index": 0.4147760721047108, + "adjusted_geometric_mean_rank_index": 0.7505083263993314, + "adjusted_hits_at_k": 0.041565814707244096, + "adjusted_inverse_harmonic_mean_rank": 0.015735850904823025, + "arithmetic_mean_rank": 11976.266, + "count": 5000.0, + "geometric_mean_rank": 3758.036184193527, + "harmonic_mean_rank": 62.47997090330955, + "hits_at_1": 0.003, + "hits_at_10": 0.0418, + "hits_at_3": 0.0184, + "hits_at_5": 0.0266, + "inverse_arithmetic_mean_rank": 8.349847940919148e-05, + "inverse_geometric_mean_rank": 0.00026609642669382644, + "inverse_harmonic_mean_rank": 0.016005129092450172, + "inverse_median_rank": 0.00011619125079881485, + "median_absolute_deviation": 11648.064329689261, + "median_rank": 8606.5, + "standard_deviation": 11406.730770419892, + "variance": 130113506.86884399, + "z_arithmetic_mean_rank": 50.79818615839251, + "z_geometric_mean_rank": 53.115805270056526, + "z_hits_at_k": 188.00507565405638, + "z_inverse_harmonic_mean_rank": 175.6270802492198 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5852440829893316, + "adjusted_arithmetic_mean_rank_index": 0.4147761858787561, + "adjusted_geometric_mean_rank_index": 0.7505084432990601, + "adjusted_hits_at_k": 0.041565814707244096, + "adjusted_inverse_harmonic_mean_rank": 0.015735851817505357, + "arithmetic_mean_rank": 11976.263671875, + "count": 5000.0, + "geometric_mean_rank": 3758.034423828125, + "harmonic_mean_rank": 62.47996734140379, + "hits_at_1": 0.003, + "hits_at_10": 0.0418, + "hits_at_3": 0.0184, + "hits_at_5": 0.0266, + "inverse_arithmetic_mean_rank": 8.349849667865783e-05, + "inverse_geometric_mean_rank": 0.0002660965546965599, + "inverse_harmonic_mean_rank": 0.016005130004882812, + "inverse_median_rank": 0.00011619125143624842, + "median_absolute_deviation": 11648.064329689261, + "median_rank": 8606.5, + "standard_deviation": 11406.7294921875, + "variance": 130113472.0, + "z_arithmetic_mean_rank": 50.79820009245363, + "z_geometric_mean_rank": 53.1158135434134, + "z_hits_at_k": 188.00507565405638, + "z_inverse_harmonic_mean_rank": 175.62709043562373 + } + } + }, + "times": { + "evaluation": 10.18949007987976, + "training": 24.329554319381714 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json new file mode 100644 index 0000000..a7f48a3 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Batch_size is not mentioned in the paper. We found out that this value works.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json new file mode 100644 index 0000000..062df03 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 0.038581071503320664, + 0.03347079680852511, + 0.031049264601289154, + 0.029245134060277567, + 0.02778917855002847 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.684332083807414, + "adjusted_arithmetic_mean_rank_index": 0.3157106557301511, + "adjusted_geometric_mean_rank_index": 0.6675365146551828, + "adjusted_hits_at_k": 0.05741789660218553, + "adjusted_inverse_harmonic_mean_rank": 0.035582955378695054, + "arithmetic_mean_rank": 5055.060097171201, + "count": 118142.0, + "geometric_mean_rank": 1806.4793054632946, + "harmonic_mean_rank": 27.587316513736916, + "hits_at_1": 0.02301467725279748, + "hits_at_10": 0.05805725313605661, + "hits_at_3": 0.03980802762776997, + "hits_at_5": 0.04622403548272418, + "inverse_arithmetic_mean_rank": 0.0001978215848629767, + "inverse_geometric_mean_rank": 0.000553562942556675, + "inverse_harmonic_mean_rank": 0.03624854195231555, + "inverse_median_rank": 0.00024192572880125802, + "median_absolute_deviation": 5192.814270315871, + "median_rank": 4133.5, + "standard_deviation": 4352.196277998581, + "variance": 18941612.442224696, + "z_arithmetic_mean_rank": 187.7901574331674, + "z_geometric_mean_rank": 229.84234423340098, + "z_hits_at_k": 757.5137466511159, + "z_inverse_harmonic_mean_rank": 1159.5727563502958 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6843332216587098, + "adjusted_arithmetic_mean_rank_index": 0.31570951772479705, + "adjusted_geometric_mean_rank_index": 0.6675355590092635, + "adjusted_hits_at_k": 0.05741789660218553, + "adjusted_inverse_harmonic_mean_rank": 0.03558294665031708, + "arithmetic_mean_rank": 5055.068502310779, + "count": 118142.0, + "geometric_mean_rank": 1806.4844952029403, + "harmonic_mean_rank": 27.58732315197363, + "hits_at_1": 0.02301467725279748, + "hits_at_10": 0.05805725313605661, + "hits_at_3": 0.03980802762776997, + "hits_at_5": 0.04622403548272418, + "inverse_arithmetic_mean_rank": 0.00019782125594200727, + "inverse_geometric_mean_rank": 0.0005535613522593008, + "inverse_harmonic_mean_rank": 0.03624853322996141, + "inverse_median_rank": 0.00024192572880125802, + "median_absolute_deviation": 5192.814270315871, + "median_rank": 4133.5, + "standard_deviation": 4352.199232126487, + "variance": 18941638.156122383, + "z_arithmetic_mean_rank": 187.7894805279039, + "z_geometric_mean_rank": 229.84201519087895, + "z_hits_at_k": 757.5137466511159, + "z_inverse_harmonic_mean_rank": 1159.572471911041 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6843326073961279, + "adjusted_arithmetic_mean_rank_index": 0.31571013207054643, + "adjusted_geometric_mean_rank_index": 0.6675360531857619, + "adjusted_hits_at_k": 0.05741789660218553, + "adjusted_inverse_harmonic_mean_rank": 0.03558295000894267, + "arithmetic_mean_rank": 5055.06396484375, + "count": 118142.0, + "geometric_mean_rank": 1806.4818115234375, + "harmonic_mean_rank": 27.587320597621027, + "hits_at_1": 0.02301467725279748, + "hits_at_10": 0.05805725313605661, + "hits_at_3": 0.03980802762776997, + "hits_at_5": 0.04622403548272418, + "inverse_arithmetic_mean_rank": 0.00019782142771873623, + "inverse_geometric_mean_rank": 0.0005535621894523501, + "inverse_harmonic_mean_rank": 0.036248536586269066, + "inverse_median_rank": 0.00024192572163883597, + "median_absolute_deviation": 5192.814270315871, + "median_rank": 4133.5, + "standard_deviation": 4352.197265625, + "variance": 18941622.0, + "z_arithmetic_mean_rank": 187.78984595137916, + "z_geometric_mean_rank": 229.84218534289664, + "z_hits_at_k": 757.5137466511159, + "z_inverse_harmonic_mean_rank": 1159.5725813614984 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.723549425577274, + "adjusted_arithmetic_mean_rank_index": 0.2764881063253928, + "adjusted_geometric_mean_rank_index": 0.5866126073988215, + "adjusted_hits_at_k": 0.04221200917635138, + "adjusted_inverse_harmonic_mean_rank": 0.025949543973016322, + "arithmetic_mean_rank": 5330.20700512942, + "count": 59071.0, + "geometric_mean_rank": 2239.4875870800633, + "harmonic_mean_rank": 37.560624411329115, + "hits_at_1": 0.016810279155592422, + "hits_at_10": 0.04286367252966769, + "hits_at_3": 0.02864349680892485, + "hits_at_5": 0.03390834758172369, + "inverse_arithmetic_mean_rank": 0.00018760997444145595, + "inverse_geometric_mean_rank": 0.00044653071790580517, + "inverse_harmonic_mean_rank": 0.026623625556618218, + "inverse_median_rank": 0.00022040996253030638, + "median_absolute_deviation": 5239.516240198797, + "median_rank": 4537.0, + "standard_deviation": 4326.8477842276, + "variance": 18721611.747875288, + "z_arithmetic_mean_rank": 116.27386693384307, + "z_geometric_mean_rank": 142.81089647095337, + "z_hits_at_k": 393.18675148132706, + "z_inverse_harmonic_mean_rank": 597.0455773070048 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.723550659602649, + "adjusted_arithmetic_mean_rank_index": 0.27648687213248224, + "adjusted_geometric_mean_rank_index": 0.5866114883180238, + "adjusted_hits_at_k": 0.04221200917635138, + "adjusted_inverse_harmonic_mean_rank": 0.025949539196434005, + "arithmetic_mean_rank": 5330.216095884613, + "count": 59071.0, + "geometric_mean_rank": 2239.4936468886576, + "harmonic_mean_rank": 37.560631145471554, + "hits_at_1": 0.016810279155592422, + "hits_at_10": 0.04286367252966769, + "hits_at_3": 0.02864349680892485, + "hits_at_5": 0.03390834758172369, + "inverse_arithmetic_mean_rank": 0.00018760965447012294, + "inverse_geometric_mean_rank": 0.0004465295096457658, + "inverse_harmonic_mean_rank": 0.026623620783341486, + "inverse_median_rank": 0.00022040996253030638, + "median_absolute_deviation": 5239.516240198797, + "median_rank": 4537.0, + "standard_deviation": 4326.850308659758, + "variance": 18721633.59354905, + "z_arithmetic_mean_rank": 116.27334790832634, + "z_geometric_mean_rank": 142.81062403062407, + "z_hits_at_k": 393.18675148132706, + "z_inverse_harmonic_mean_rank": 597.0454674076806 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7235500256590511, + "adjusted_arithmetic_mean_rank_index": 0.27648750616214646, + "adjusted_geometric_mean_rank_index": 0.5866121185159, + "adjusted_hits_at_k": 0.04221200917635138, + "adjusted_inverse_harmonic_mean_rank": 0.02594954222435405, + "arithmetic_mean_rank": 5330.21142578125, + "count": 59071.0, + "geometric_mean_rank": 2239.490234375, + "harmonic_mean_rank": 37.56062687663563, + "hits_at_1": 0.016810279155592422, + "hits_at_10": 0.04286367252966769, + "hits_at_3": 0.02864349680892485, + "hits_at_5": 0.03390834758172369, + "inverse_arithmetic_mean_rank": 0.000187609824934043, + "inverse_geometric_mean_rank": 0.0004465302044991404, + "inverse_harmonic_mean_rank": 0.026623623809166087, + "inverse_median_rank": 0.0002204099582741037, + "median_absolute_deviation": 5239.516240198797, + "median_rank": 4537.0, + "standard_deviation": 4326.84912109375, + "variance": 18721622.0, + "z_arithmetic_mean_rank": 116.27361454214935, + "z_geometric_mean_rank": 142.81077745235848, + "z_hits_at_k": 393.18675148132706, + "z_inverse_harmonic_mean_rank": 597.0455370738863 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6453276235867126, + "adjusted_arithmetic_mean_rank_index": 0.3547202665689454, + "adjusted_geometric_mean_rank_index": 0.7326280379766128, + "adjusted_hits_at_k": 0.07262372071547131, + "adjusted_inverse_harmonic_mean_rank": 0.04521633023387397, + "arithmetic_mean_rank": 4779.913189212981, + "count": 59071.0, + "geometric_mean_rank": 1457.193824111371, + "harmonic_mean_rank": 21.79909769203868, + "hits_at_1": 0.029219075350002538, + "hits_at_10": 0.07325083374244554, + "hits_at_3": 0.05097255844661509, + "hits_at_5": 0.05853972338372467, + "inverse_arithmetic_mean_rank": 0.0002092088204147179, + "inverse_geometric_mean_rank": 0.000686250506592575, + "inverse_harmonic_mean_rank": 0.04587345834801288, + "inverse_median_rank": 0.00027307482250136535, + "median_absolute_deviation": 4966.717431993767, + "median_rank": 3662.0, + "standard_deviation": 4360.068978199332, + "variance": 19010201.494656168, + "z_arithmetic_mean_rank": 149.217601017942, + "z_geometric_mean_rank": 178.3569182136984, + "z_hits_at_k": 678.5379875623604, + "z_inverse_harmonic_mean_rank": 1043.5218864092278 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6453286657859862, + "adjusted_arithmetic_mean_rank_index": 0.3547192242289472, + "adjusted_geometric_mean_rank_index": 0.7326272246599947, + "adjusted_hits_at_k": 0.07262372071547131, + "adjusted_inverse_harmonic_mean_rank": 0.04521631755371532, + "arithmetic_mean_rank": 4779.920908736944, + "count": 59071.0, + "geometric_mean_rank": 1457.1982536956366, + "harmonic_mean_rank": 21.799103713512977, + "hits_at_1": 0.029219075350002538, + "hits_at_10": 0.07325083374244554, + "hits_at_3": 0.05097255844661509, + "hits_at_5": 0.05853972338372467, + "inverse_arithmetic_mean_rank": 0.00020920848254458714, + "inverse_geometric_mean_rank": 0.000686248420531575, + "inverse_harmonic_mean_rank": 0.04587344567658133, + "inverse_median_rank": 0.00027307482250136535, + "median_absolute_deviation": 4966.717431993767, + "median_rank": 3662.0, + "standard_deviation": 4360.072284056398, + "variance": 19010230.322196774, + "z_arithmetic_mean_rank": 149.2171625443372, + "z_geometric_mean_rank": 178.35672021329714, + "z_hits_at_k": 678.5379875623604, + "z_inverse_harmonic_mean_rank": 1043.5215937710814 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6453282029415623, + "adjusted_arithmetic_mean_rank_index": 0.3547196871358673, + "adjusted_geometric_mean_rank_index": 0.732627899173446, + "adjusted_hits_at_k": 0.07262372071547131, + "adjusted_inverse_harmonic_mean_rank": 0.045216325378894785, + "arithmetic_mean_rank": 4779.91748046875, + "count": 59071.0, + "geometric_mean_rank": 1457.194580078125, + "harmonic_mean_rank": 21.799099997540385, + "hits_at_1": 0.029219075350002538, + "hits_at_10": 0.07325083374244554, + "hits_at_3": 0.05097255844661509, + "hits_at_5": 0.05853972338372467, + "inverse_arithmetic_mean_rank": 0.00020920863607898355, + "inverse_geometric_mean_rank": 0.0006862501613795757, + "inverse_harmonic_mean_rank": 0.04587345349637513, + "inverse_median_rank": 0.0002730748092290014, + "median_absolute_deviation": 4966.717431993767, + "median_rank": 3662.0, + "standard_deviation": 4360.07080078125, + "variance": 19010216.0, + "z_arithmetic_mean_rank": 149.21735727203293, + "z_geometric_mean_rank": 178.35688442232845, + "z_hits_at_k": 678.5379875623604, + "z_inverse_harmonic_mean_rank": 1043.5217743639334 + } + } + }, + "times": { + "evaluation": 17.733282566070557, + "training": 112.34571981430054 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..6c90c7d --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "TransE was used to initialize the emebddigns. Could not find batch size and number of epochs.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..44a0485 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 4096, + "slice_size": null + }, + "losses": [ + 0.006557168293018543, + 0.006131826215394638, + 0.0058330251878667225, + 0.00561165085609545, + 0.005457009655934738 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9426456616120584, + "adjusted_arithmetic_mean_rank_index": 0.057362377210596005, + "adjusted_geometric_mean_rank_index": 0.12297663463498498, + "adjusted_hits_at_k": 0.0006920234342332076, + "adjusted_inverse_harmonic_mean_rank": 0.0004381064572794429, + "arithmetic_mean_rank": 6726.40737841276, + "count": 40876.0, + "geometric_mean_rank": 4599.648580410729, + "harmonic_mean_rank": 869.233415396429, + "hits_at_1": 9.785693316371466e-05, + "hits_at_10": 0.0013944612975829338, + "hits_at_3": 0.00041589196594578727, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014866777222107106, + "inverse_geometric_mean_rank": 0.00021740791334774195, + "inverse_harmonic_mean_rank": 0.0011504389756391644, + "inverse_median_rank": 0.0001520565650421957, + "median_absolute_deviation": 5317.352856670342, + "median_rank": 6576.5, + "standard_deviation": 4185.0150321375795, + "variance": 17514350.819217503, + "z_arithmetic_mean_rank": 20.061545025360317, + "z_geometric_mean_rank": 24.905953532687732, + "z_hits_at_k": 5.275317808587865, + "z_inverse_harmonic_mean_rank": 8.249809692059822 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9426473106930761, + "adjusted_arithmetic_mean_rank_index": 0.057360727898442154, + "adjusted_geometric_mean_rank_index": 0.12297434805562524, + "adjusted_hits_at_k": 0.0006920234342332076, + "adjusted_inverse_harmonic_mean_rank": 0.000438104068321181, + "arithmetic_mean_rank": 6726.419145708974, + "count": 40876.0, + "geometric_mean_rank": 4599.660570028499, + "harmonic_mean_rank": 869.2352191312187, + "hits_at_1": 9.785693316371466e-05, + "hits_at_10": 0.0013944612975829338, + "hits_at_3": 0.00041589196594578727, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014866751213949195, + "inverse_geometric_mean_rank": 0.00021740734664553826, + "inverse_harmonic_mean_rank": 0.0011504365883833812, + "inverse_median_rank": 0.0001520565650421957, + "median_absolute_deviation": 5317.352856670342, + "median_rank": 6576.5, + "standard_deviation": 4185.016936089053, + "variance": 17514366.755352203, + "z_arithmetic_mean_rank": 20.060968205645963, + "z_geometric_mean_rank": 24.905490441146412, + "z_hits_at_k": 5.275317808587865, + "z_inverse_harmonic_mean_rank": 8.249764706530177 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.942646461470457, + "adjusted_arithmetic_mean_rank_index": 0.057361577240088724, + "adjusted_geometric_mean_rank_index": 0.12297601003791969, + "adjusted_hits_at_k": 0.0006920234342332076, + "adjusted_inverse_harmonic_mean_rank": 0.0004381052745533685, + "arithmetic_mean_rank": 6726.4130859375, + "count": 40876.0, + "geometric_mean_rank": 4599.65185546875, + "harmonic_mean_rank": 869.2343083889797, + "hits_at_1": 9.785693316371466e-05, + "hits_at_10": 0.0013944612975829338, + "hits_at_3": 0.00041589196594578727, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014866764831822366, + "inverse_geometric_mean_rank": 0.00021740775264333934, + "inverse_harmonic_mean_rank": 0.0011504377937559536, + "inverse_median_rank": 0.00015205657109618187, + "median_absolute_deviation": 5317.352856670342, + "median_rank": 6576.5, + "standard_deviation": 4185.01611328125, + "variance": 17514358.0, + "z_arithmetic_mean_rank": 20.061265248873838, + "z_geometric_mean_rank": 24.90582703560534, + "z_hits_at_k": 5.275317808587865, + "z_inverse_harmonic_mean_rank": 8.249787420612163 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9866627195535737, + "adjusted_arithmetic_mean_rank_index": 0.013339176120974239, + "adjusted_geometric_mean_rank_index": 0.02607247210901087, + "adjusted_hits_at_k": 0.0003620255024674139, + "adjusted_inverse_harmonic_mean_rank": 0.00015334822445050558, + "arithmetic_mean_rank": 6942.788677952833, + "count": 20438.0, + "geometric_mean_rank": 5030.878952767664, + "harmonic_mean_rank": 1140.91162398533, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.001076426264800861, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.00014403434216218468, + "inverse_geometric_mean_rank": 0.000198772423146827, + "inverse_harmonic_mean_rank": 0.0008764920778937197, + "inverse_median_rank": 0.00014646649578908824, + "median_absolute_deviation": 5178.729549240067, + "median_rank": 6827.5, + "standard_deviation": 4102.466072979059, + "variance": 16830227.879944224, + "z_arithmetic_mean_rank": 3.2952595134808575, + "z_geometric_mean_rank": 3.7339554587159967, + "z_hits_at_k": 1.9353282494158774, + "z_inverse_harmonic_mean_rank": 2.0250851120577327 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9866642910189499, + "adjusted_arithmetic_mean_rank_index": 0.01333760443224008, + "adjusted_geometric_mean_rank_index": 0.026069825878376696, + "adjusted_hits_at_k": 0.0003620255024674139, + "adjusted_inverse_harmonic_mean_rank": 0.00015334516391599952, + "arithmetic_mean_rank": 6942.7997357862805, + "count": 20438.0, + "geometric_mean_rank": 5030.892619307924, + "harmonic_mean_rank": 1140.9156049524101, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.001076426264800861, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.00014403411275793466, + "inverse_geometric_mean_rank": 0.00019877188317678012, + "inverse_harmonic_mean_rank": 0.0008764890195727597, + "inverse_median_rank": 0.00014646649578908824, + "median_absolute_deviation": 5178.729549240067, + "median_rank": 6827.5, + "standard_deviation": 4102.46654893988, + "variance": 16830231.785170693, + "z_arithmetic_mean_rank": 3.2948712494526546, + "z_geometric_mean_rank": 3.7335764801795506, + "z_hits_at_k": 1.9353282494158774, + "z_inverse_harmonic_mean_rank": 2.025044695268523 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9866635375067887, + "adjusted_arithmetic_mean_rank_index": 0.013338358051500632, + "adjusted_geometric_mean_rank_index": 0.026071819300254817, + "adjusted_hits_at_k": 0.0003620255024674139, + "adjusted_inverse_harmonic_mean_rank": 0.00015334667558629324, + "arithmetic_mean_rank": 6942.79443359375, + "count": 20438.0, + "geometric_mean_rank": 5030.88232421875, + "harmonic_mean_rank": 1140.9136386552518, + "hits_at_1": 4.892846658185733e-05, + "hits_at_10": 0.001076426264800861, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.00014403421664610505, + "inverse_geometric_mean_rank": 0.0001987722935155034, + "inverse_harmonic_mean_rank": 0.0008764905301497307, + "inverse_median_rank": 0.00014646649651695043, + "median_absolute_deviation": 5178.729549240067, + "median_rank": 6827.5, + "standard_deviation": 4102.46630859375, + "variance": 16830230.0, + "z_arithmetic_mean_rank": 3.2950574206985834, + "z_geometric_mean_rank": 3.733861967051382, + "z_hits_at_k": 1.9353282494158774, + "z_inverse_harmonic_mean_rank": 2.0250646580755074 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8998336400258861, + "adjusted_arithmetic_mean_rank_index": 0.1001802071568092, + "adjusted_geometric_mean_rank_index": 0.21025959095733382, + "adjusted_hits_at_k": 0.0010220136154691184, + "adjusted_inverse_harmonic_mean_rank": 0.0007228586432761326, + "arithmetic_mean_rank": 6510.026078872688, + "count": 20438.0, + "geometric_mean_rank": 4205.381855120024, + "harmonic_mean_rank": 702.0569486720699, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0017124963303650064, + "hits_at_3": 0.0004403561992367159, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00015360921567508765, + "inverse_geometric_mean_rank": 0.00023779053471268175, + "inverse_harmonic_mean_rank": 0.0014243858733846092, + "inverse_median_rank": 0.00015867978419549348, + "median_absolute_deviation": 5441.150141915559, + "median_rank": 6302.0, + "standard_deviation": 4254.977323665718, + "variance": 18104832.02490948, + "z_arithmetic_mean_rank": 24.804091992690616, + "z_geometric_mean_rank": 30.11088625655215, + "z_hits_at_k": 5.55555302398681, + "z_inverse_harmonic_mean_rank": 9.706159475823041 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8998353645976944, + "adjusted_arithmetic_mean_rank_index": 0.10017848234659277, + "adjusted_geometric_mean_rank_index": 0.210257618708523, + "adjusted_hits_at_k": 0.0010220136154691184, + "adjusted_inverse_harmonic_mean_rank": 0.0007228569258798545, + "arithmetic_mean_rank": 6510.038555631667, + "count": 20438.0, + "geometric_mean_rank": 4205.392354883008, + "harmonic_mean_rank": 702.0577945559098, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0017124963303650064, + "hits_at_3": 0.0004403561992367159, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00015360892127665293, + "inverse_geometric_mean_rank": 0.00023778994101201277, + "inverse_harmonic_mean_rank": 0.001424384157194003, + "inverse_median_rank": 0.00015867978419549348, + "median_absolute_deviation": 5441.150141915559, + "median_rank": 6302.0, + "standard_deviation": 4254.9806822134115, + "variance": 18104860.606009305, + "z_arithmetic_mean_rank": 24.803664938759596, + "z_geometric_mean_rank": 30.11060381445597, + "z_hits_at_k": 5.55555302398681, + "z_inverse_harmonic_mean_rank": 9.706136415543812 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8998344897764146, + "adjusted_arithmetic_mean_rank_index": 0.10017935728880956, + "adjusted_geometric_mean_rank_index": 0.2102583105153928, + "adjusted_hits_at_k": 0.0010220136154691184, + "adjusted_inverse_harmonic_mean_rank": 0.0007228577332912151, + "arithmetic_mean_rank": 6510.0322265625, + "count": 20438.0, + "geometric_mean_rank": 4205.388671875, + "harmonic_mean_rank": 702.0573968744504, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0017124963303650064, + "hits_at_3": 0.0004403561992367159, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00015360907127615064, + "inverse_geometric_mean_rank": 0.00023779015464242548, + "inverse_harmonic_mean_rank": 0.0014243849640385327, + "inverse_median_rank": 0.00015867978800088167, + "median_absolute_deviation": 5441.150141915559, + "median_rank": 6302.0, + "standard_deviation": 4254.97900390625, + "variance": 18104848.0, + "z_arithmetic_mean_rank": 24.80388156984722, + "z_geometric_mean_rank": 30.110702886835384, + "z_hits_at_k": 5.55555302398681, + "z_inverse_harmonic_mean_rank": 9.706147257031981 + } + } + }, + "times": { + "evaluation": 9.86320161819458, + "training": 11.522887229919434 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json new file mode 100644 index 0000000..679e4fa --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Num_epochs varies from the value reported in the paper, and batch_size is not mentioned in the paper. We found out that these values work.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WordNet Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json new file mode 100644 index 0000000..6c48057 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 8192, + "slice_size": null + }, + "losses": [ + 0.06621763066039518, + 0.06459694919663586, + 0.0637450087589807, + 0.0629228597891125, + 0.06228165925657403 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9437711675374072, + "adjusted_arithmetic_mean_rank_index": 0.056231580394964786, + "adjusted_geometric_mean_rank_index": 0.13584888007932028, + "adjusted_hits_at_k": 0.00015569078806712837, + "adjusted_inverse_harmonic_mean_rank": 0.0001548875701843438, + "arithmetic_mean_rank": 19312.6093, + "count": 10000.0, + "geometric_mean_rank": 13013.095349439951, + "harmonic_mean_rank": 2334.080357303059, + "hits_at_1": 0.0, + "hits_at_10": 0.0004, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.177964222576594e-05, + "inverse_geometric_mean_rank": 7.684566762534613e-05, + "inverse_harmonic_mean_rank": 0.00042843426400085985, + "inverse_median_rank": 5.2801098262843864e-05, + "median_absolute_deviation": 15893.495782380052, + "median_rank": 18939.0, + "standard_deviation": 12153.88657263402, + "variance": 147716958.82045352, + "z_arithmetic_mean_rank": 9.739345246937546, + "z_geometric_mean_rank": 13.59587579541774, + "z_hits_at_k": 0.9958779562999378, + "z_inverse_harmonic_mean_rank": 2.444712247055369 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9437718125967681, + "adjusted_arithmetic_mean_rank_index": 0.05623093530407952, + "adjusted_geometric_mean_rank_index": 0.13584804290931352, + "adjusted_hits_at_k": 0.00015569078806712837, + "adjusted_inverse_harmonic_mean_rank": 0.00015488743965322147, + "arithmetic_mean_rank": 19312.6225, + "count": 10000.0, + "geometric_mean_rank": 13013.107955262823, + "harmonic_mean_rank": 2334.081068233282, + "hits_at_1": 0.0, + "hits_at_10": 0.0004, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.177960683485632e-05, + "inverse_geometric_mean_rank": 7.684559318479912e-05, + "inverse_harmonic_mean_rank": 0.0004284341335054494, + "inverse_median_rank": 5.2801098262843864e-05, + "median_absolute_deviation": 15893.495782380052, + "median_rank": 18939.0, + "standard_deviation": 12153.887721967558, + "variance": 147716986.75819376, + "z_arithmetic_mean_rank": 9.739233516788687, + "z_geometric_mean_rank": 13.595792010704713, + "z_hits_at_k": 0.9958779562999378, + "z_inverse_harmonic_mean_rank": 2.444710186780081 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9437714575392363, + "adjusted_arithmetic_mean_rank_index": 0.05623129037896313, + "adjusted_geometric_mean_rank_index": 0.13584762434833497, + "adjusted_hits_at_k": 0.00015569078806712837, + "adjusted_inverse_harmonic_mean_rank": 0.0001548874873742069, + "arithmetic_mean_rank": 19312.615234375, + "count": 10000.0, + "geometric_mean_rank": 13013.1142578125, + "harmonic_mean_rank": 2334.0808083236393, + "hits_at_1": 0.0, + "hits_at_10": 0.0004, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.177962520974688e-05, + "inverse_geometric_mean_rank": 7.684555748710409e-05, + "inverse_harmonic_mean_rank": 0.0004284341812133789, + "inverse_median_rank": 5.2801096899202093e-05, + "median_absolute_deviation": 15893.495782380052, + "median_rank": 18939.0, + "standard_deviation": 12153.88671875, + "variance": 147716960.0, + "z_arithmetic_mean_rank": 9.739295015982833, + "z_geometric_mean_rank": 13.59575012075264, + "z_hits_at_k": 0.9958779562999378, + "z_inverse_harmonic_mean_rank": 2.444710939997898 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9432172063057032, + "adjusted_arithmetic_mean_rank_index": 0.05678556876338092, + "adjusted_geometric_mean_rank_index": 0.12458016502255864, + "adjusted_hits_at_k": 0.0005557828463619552, + "adjusted_inverse_harmonic_mean_rank": 0.00017251546772340296, + "arithmetic_mean_rank": 19300.826, + "count": 5000.0, + "geometric_mean_rank": 13183.127109601111, + "harmonic_mean_rank": 2241.835019542241, + "hits_at_1": 0.0, + "hits_at_10": 0.0008, + "hits_at_3": 0.0002, + "hits_at_5": 0.0004, + "inverse_arithmetic_mean_rank": 5.181125408829653e-05, + "inverse_geometric_mean_rank": 7.585453676402104e-05, + "inverse_harmonic_mean_rank": 0.00044606315419418754, + "inverse_median_rank": 5.366102331571463e-05, + "median_absolute_deviation": 15631.816490813813, + "median_rank": 18635.5, + "standard_deviation": 12029.896846778196, + "variance": 144718418.144124, + "z_arithmetic_mean_rank": 6.954604180176574, + "z_geometric_mean_rank": 8.81692545777343, + "z_hits_at_k": 2.5137858136954785, + "z_inverse_harmonic_mean_rank": 1.9253919774028487 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9432178709277252, + "adjusted_arithmetic_mean_rank_index": 0.0567849041088776, + "adjusted_geometric_mean_rank_index": 0.124579314102237, + "adjusted_hits_at_k": 0.0005557828463619552, + "adjusted_inverse_harmonic_mean_rank": 0.00017251532995541503, + "arithmetic_mean_rank": 19300.8396, + "count": 5000.0, + "geometric_mean_rank": 13183.139922813003, + "harmonic_mean_rank": 2241.8357117507135, + "hits_at_1": 0.0, + "hits_at_10": 0.0008, + "hits_at_3": 0.0002, + "hits_at_5": 0.0004, + "inverse_arithmetic_mean_rank": 5.1811217580399975e-05, + "inverse_geometric_mean_rank": 7.585446303801509e-05, + "inverse_harmonic_mean_rank": 0.0004460630164638922, + "inverse_median_rank": 5.366102331571463e-05, + "median_absolute_deviation": 15631.816490813813, + "median_rank": 18635.5, + "standard_deviation": 12029.898754049089, + "variance": 144718464.03267184, + "z_arithmetic_mean_rank": 6.954522779054997, + "z_geometric_mean_rank": 8.816865235497723, + "z_hits_at_k": 2.5137858136954785, + "z_inverse_harmonic_mean_rank": 1.9253904398167867 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9432175010484657, + "adjusted_arithmetic_mean_rank_index": 0.056785274006213804, + "adjusted_geometric_mean_rank_index": 0.12457991600156137, + "adjusted_hits_at_k": 0.0005557828463619552, + "adjusted_inverse_harmonic_mean_rank": 0.00017251540288220268, + "arithmetic_mean_rank": 19300.83203125, + "count": 5000.0, + "geometric_mean_rank": 13183.130859375, + "harmonic_mean_rank": 2241.835345333606, + "hits_at_1": 0.0, + "hits_at_10": 0.0008, + "hits_at_3": 0.0002, + "hits_at_5": 0.0004, + "inverse_arithmetic_mean_rank": 5.181123924558051e-05, + "inverse_geometric_mean_rank": 7.585451385239139e-05, + "inverse_harmonic_mean_rank": 0.0004460630893707275, + "inverse_median_rank": 5.36610241397284e-05, + "median_absolute_deviation": 15631.816490813813, + "median_rank": 18635.5, + "standard_deviation": 12029.8984375, + "variance": 144718448.0, + "z_arithmetic_mean_rank": 6.95456808087404, + "z_geometric_mean_rank": 8.816907833783532, + "z_hits_at_k": 2.5137858136954785, + "z_inverse_harmonic_mean_rank": 1.9253912537301927 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9443251030846932, + "adjusted_arithmetic_mean_rank_index": 0.05567761771348001, + "adjusted_geometric_mean_rank_index": 0.14705770204849056, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00013725967285040471, + "arithmetic_mean_rank": 19324.3926, + "count": 5000.0, + "geometric_mean_rank": 12845.25660457957, + "harmonic_mean_rank": 2434.2427430574785, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.1748068914724906e-05, + "inverse_geometric_mean_rank": 7.784974880482197e-05, + "inverse_harmonic_mean_rank": 0.0004108053738075322, + "inverse_median_rank": 5.215667866270276e-05, + "median_absolute_deviation": 16158.881579492556, + "median_rank": 19173.0, + "standard_deviation": 12276.612798506974, + "variance": 150715221.80446523, + "z_arithmetic_mean_rank": 6.818913094753356, + "z_geometric_mean_rank": 10.407730321853201, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5319486518070051 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9443257285822998, + "adjusted_arithmetic_mean_rank_index": 0.05567699218530575, + "adjusted_geometric_mean_rank_index": 0.14705687849801652, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00013725954955614802, + "arithmetic_mean_rank": 19324.4054, + "count": 5000.0, + "geometric_mean_rank": 12845.269006231641, + "harmonic_mean_rank": 2434.243473442592, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.174803463810587e-05, + "inverse_geometric_mean_rank": 7.7849673643648e-05, + "inverse_harmonic_mean_rank": 0.0004108052505470068, + "inverse_median_rank": 5.215667866270276e-05, + "median_absolute_deviation": 16158.881579492556, + "median_rank": 19173.0, + "standard_deviation": 12276.613206021066, + "variance": 150715231.81025082, + "z_arithmetic_mean_rank": 6.818836485472403, + "z_geometric_mean_rank": 10.407672036628277, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5319472757256967 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9443254837891691, + "adjusted_arithmetic_mean_rank_index": 0.05567723699039928, + "adjusted_geometric_mean_rank_index": 0.1470571030338207, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.0001372596197680961, + "arithmetic_mean_rank": 19324.400390625, + "count": 5000.0, + "geometric_mean_rank": 12845.265625, + "harmonic_mean_rank": 2434.243057512688, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.1748047553701326e-05, + "inverse_geometric_mean_rank": 7.78496905695647e-05, + "inverse_harmonic_mean_rank": 0.00041080532073974605, + "inverse_median_rank": 5.2156679885229096e-05, + "median_absolute_deviation": 16158.881579492556, + "median_rank": 19173.0, + "standard_deviation": 12276.61328125, + "variance": 150715232.0, + "z_arithmetic_mean_rank": 6.818866467083083, + "z_geometric_mean_rank": 10.407687927724524, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.531948059357895 + } + } + }, + "times": { + "evaluation": 5.315532684326172, + "training": 33.14157509803772 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json new file mode 100644 index 0000000..e77bdff --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json new file mode 100644 index 0000000..a954644 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 13.087838403357605, + 0.20433549977189364, + 0.002744033488619886, + 0.0026934515536661723, + 0.002643932435918485 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9959494722590975, + "adjusted_arithmetic_mean_rank_index": 0.00405107615800071, + "adjusted_geometric_mean_rank_index": 0.021381014132858422, + "adjusted_hits_at_k": 0.0013794792690803878, + "adjusted_inverse_harmonic_mean_rank": 0.000676489179756041, + "arithmetic_mean_rank": 7356.93175162093, + "count": 118142.0, + "geometric_mean_rank": 5315.497395357781, + "harmonic_mean_rank": 731.9753418257839, + "hits_at_1": 0.0004147551251883327, + "hits_at_10": 0.0020568468453217316, + "hits_at_3": 0.000931082934096257, + "hits_at_5": 0.001278122936804862, + "inverse_arithmetic_mean_rank": 0.00013592623035814803, + "inverse_geometric_mean_rank": 0.00018812914871773557, + "inverse_harmonic_mean_rank": 0.001366166239296635, + "inverse_median_rank": 0.00013635124079629124, + "median_absolute_deviation": 5423.358915293492, + "median_rank": 7334.0, + "standard_deviation": 4278.634830765392, + "variance": 18306716.015038792, + "z_arithmetic_mean_rank": 2.409650151735491, + "z_geometric_mean_rank": 7.361788160640344, + "z_hits_at_k": 18.199456465440285, + "z_inverse_harmonic_mean_rank": 22.045342059488902 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9959510214785858, + "adjusted_arithmetic_mean_rank_index": 0.004049526728757358, + "adjusted_geometric_mean_rank_index": 0.021378717325856278, + "adjusted_hits_at_k": 0.0013794792690803878, + "adjusted_inverse_harmonic_mean_rank": 0.0006764877426722778, + "arithmetic_mean_rank": 7356.94319547663, + "count": 118142.0, + "geometric_mean_rank": 5315.509868419316, + "harmonic_mean_rank": 731.9761112672941, + "hits_at_1": 0.0004147551251883327, + "hits_at_10": 0.0020568468453217316, + "hits_at_3": 0.000931082934096257, + "hits_at_5": 0.001278122936804862, + "inverse_arithmetic_mean_rank": 0.0001359260189224845, + "inverse_geometric_mean_rank": 0.00018812870726498567, + "inverse_harmonic_mean_rank": 0.0013661648032046666, + "inverse_median_rank": 0.00013635124079629124, + "median_absolute_deviation": 5423.358915293492, + "median_rank": 7334.0, + "standard_deviation": 4278.635477992422, + "variance": 18306721.553535447, + "z_arithmetic_mean_rank": 2.40872852442817, + "z_geometric_mean_rank": 7.360997337226195, + "z_hits_at_k": 18.199456465440285, + "z_inverse_harmonic_mean_rank": 22.045295227988753 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.995950250449693, + "adjusted_arithmetic_mean_rank_index": 0.0040502978620429175, + "adjusted_geometric_mean_rank_index": 0.02138035468481103, + "adjusted_hits_at_k": 0.0013794792690803878, + "adjusted_inverse_harmonic_mean_rank": 0.0006764883999218058, + "arithmetic_mean_rank": 7356.9375, + "count": 118142.0, + "geometric_mean_rank": 5315.5009765625, + "harmonic_mean_rank": 731.97575936341, + "hits_at_1": 0.0004147551251883327, + "hits_at_10": 0.0020568468453217316, + "hits_at_3": 0.000931082934096257, + "hits_at_5": 0.001278122936804862, + "inverse_arithmetic_mean_rank": 0.0001359261223115027, + "inverse_geometric_mean_rank": 0.0001881290227174759, + "inverse_harmonic_mean_rank": 0.0013661654600005979, + "inverse_median_rank": 0.0001363512419629842, + "median_absolute_deviation": 5423.358915293492, + "median_rank": 7334.0, + "standard_deviation": 4278.63525390625, + "variance": 18306720.0, + "z_arithmetic_mean_rank": 2.409187207841228, + "z_geometric_mean_rank": 7.3615611032707875, + "z_hits_at_k": 18.199456465440285, + "z_inverse_harmonic_mean_rank": 22.045316646351523 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9842617964840297, + "adjusted_arithmetic_mean_rank_index": 0.01574034018985404, + "adjusted_geometric_mean_rank_index": 0.037777252629037306, + "adjusted_hits_at_k": 0.001351989829234509, + "adjusted_inverse_harmonic_mean_rank": 0.0007421923664650952, + "arithmetic_mean_rank": 7250.809602004368, + "count": 59071.0, + "geometric_mean_rank": 5211.424203899233, + "harmonic_mean_rank": 697.4870193828475, + "hits_at_1": 0.00045707707673816255, + "hits_at_10": 0.0020314536743918335, + "hits_at_3": 0.0010157268371959168, + "hits_at_5": 0.00130351610773476, + "inverse_arithmetic_mean_rank": 0.00013791563354850281, + "inverse_geometric_mean_rank": 0.00019188612572582198, + "inverse_harmonic_mean_rank": 0.0014337184380647297, + "inverse_median_rank": 0.00013886960144424384, + "median_absolute_deviation": 5328.472373309133, + "median_rank": 7201.0, + "standard_deviation": 4237.865694484515, + "variance": 17959505.644488726, + "z_arithmetic_mean_rank": 6.619417540422489, + "z_geometric_mean_rank": 9.196875836142056, + "z_hits_at_k": 12.593205094116266, + "z_inverse_harmonic_mean_rank": 17.076318195409776 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9842633384412486, + "adjusted_arithmetic_mean_rank_index": 0.015738798023293477, + "adjusted_geometric_mean_rank_index": 0.037774950811552466, + "adjusted_hits_at_k": 0.001351989829234509, + "adjusted_inverse_harmonic_mean_rank": 0.0007421905012426657, + "arithmetic_mean_rank": 7250.820961216164, + "count": 59071.0, + "geometric_mean_rank": 5211.436668212272, + "harmonic_mean_rank": 697.487926164658, + "hits_at_1": 0.00045707707673816255, + "hits_at_10": 0.0020314536743918335, + "hits_at_3": 0.0010157268371959168, + "hits_at_5": 0.00130351610773476, + "inverse_arithmetic_mean_rank": 0.00013791541748843186, + "inverse_geometric_mean_rank": 0.00019188566678735817, + "inverse_harmonic_mean_rank": 0.0014337165741331085, + "inverse_median_rank": 0.00013886960144424384, + "median_absolute_deviation": 5328.472373309133, + "median_rank": 7201.0, + "standard_deviation": 4237.866652494595, + "variance": 17959513.76432574, + "z_arithmetic_mean_rank": 6.618769000158636, + "z_geometric_mean_rank": 9.196315458450925, + "z_hits_at_k": 12.593205094116266, + "z_inverse_harmonic_mean_rank": 17.07627528048213 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9842626538451784, + "adjusted_arithmetic_mean_rank_index": 0.015739482712306962, + "adjusted_geometric_mean_rank_index": 0.03777578909713941, + "adjusted_hits_at_k": 0.001351989829234509, + "adjusted_inverse_harmonic_mean_rank": 0.0007421915149824226, + "arithmetic_mean_rank": 7250.81591796875, + "count": 59071.0, + "geometric_mean_rank": 5211.43212890625, + "harmonic_mean_rank": 697.4874333326487, + "hits_at_1": 0.00045707707673816255, + "hits_at_10": 0.0020314536743918335, + "hits_at_3": 0.0010157268371959168, + "hits_at_5": 0.00130351610773476, + "inverse_arithmetic_mean_rank": 0.00013791551464237273, + "inverse_geometric_mean_rank": 0.00019188583246432245, + "inverse_harmonic_mean_rank": 0.0014337175871713171, + "inverse_median_rank": 0.00013886959641240537, + "median_absolute_deviation": 5328.472373309133, + "median_rank": 7201.0, + "standard_deviation": 4237.8662109375, + "variance": 17959510.0, + "z_arithmetic_mean_rank": 6.619056938183462, + "z_geometric_mean_rank": 9.196519539158771, + "z_hits_at_k": 12.593205094116266, + "z_inverse_harmonic_mean_rank": 17.076298604546665 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0075737044040576, + "adjusted_arithmetic_mean_rank_index": -0.007574727054556751, + "adjusted_geometric_mean_rank_index": 0.004713849867451159, + "adjusted_hits_at_k": 0.001406968594468856, + "adjusted_inverse_harmonic_mean_rank": 0.0006107862423339953, + "arithmetic_mean_rank": 7463.053901237494, + "count": 59071.0, + "geometric_mean_rank": 5421.648949420608, + "harmonic_mean_rank": 770.051738846899, + "hits_at_1": 0.0003724331736385028, + "hits_at_10": 0.0020822400162516293, + "hits_at_3": 0.0008464390309965973, + "hits_at_5": 0.001252729765874964, + "inverse_arithmetic_mean_rank": 0.0001339934044740296, + "inverse_geometric_mean_rank": 0.00018444573031731727, + "inverse_harmonic_mean_rank": 0.0012986140405285406, + "inverse_median_rank": 0.00013381506757660912, + "median_absolute_deviation": 5521.210661714862, + "median_rank": 7473.0, + "standard_deviation": 4316.410842854324, + "variance": 18631402.56431038, + "z_arithmetic_mean_rank": -3.186405474881326, + "z_geometric_mean_rank": 1.1475778863209314, + "z_hits_at_k": 13.14558975014015, + "z_inverse_harmonic_mean_rank": 14.095987190833268 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.007575260846394, + "adjusted_arithmetic_mean_rank_index": -0.007576283707054099, + "adjusted_geometric_mean_rank_index": 0.004711558929027637, + "adjusted_hits_at_k": 0.001406968594468856, + "adjusted_inverse_harmonic_mean_rank": 0.0006107852333872736, + "arithmetic_mean_rank": 7463.065429737096, + "count": 59071.0, + "geometric_mean_rank": 5421.66142660916, + "harmonic_mean_rank": 770.0523367204994, + "hits_at_1": 0.0003724331736385028, + "hits_at_10": 0.0020822400162516293, + "hits_at_3": 0.0008464390309965973, + "hits_at_5": 0.001252729765874964, + "inverse_arithmetic_mean_rank": 0.00013399319748898776, + "inverse_geometric_mean_rank": 0.0001844453058415019, + "inverse_harmonic_mean_rank": 0.0012986130322762246, + "inverse_median_rank": 0.00013381506757660912, + "median_absolute_deviation": 5521.210661714862, + "median_rank": 7473.0, + "standard_deviation": 4316.411181240301, + "variance": 18631405.48553629, + "z_arithmetic_mean_rank": -3.187060300593714, + "z_geometric_mean_rank": 1.1470201616695874, + "z_hits_at_k": 13.14558975014015, + "z_inverse_harmonic_mean_rank": 14.095963905927553 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0075744038538572, + "adjusted_arithmetic_mean_rank_index": -0.0075754265988008385, + "adjusted_geometric_mean_rank_index": 0.004712240446819949, + "adjusted_hits_at_k": 0.001406968594468856, + "adjusted_inverse_harmonic_mean_rank": 0.0006107857926385235, + "arithmetic_mean_rank": 7463.05908203125, + "count": 59071.0, + "geometric_mean_rank": 5421.65771484375, + "harmonic_mean_rank": 770.0520053237399, + "hits_at_1": 0.0003724331736385028, + "hits_at_10": 0.0020822400162516293, + "hits_at_3": 0.0008464390309965973, + "hits_at_5": 0.001252729765874964, + "inverse_arithmetic_mean_rank": 0.00013399330782704055, + "inverse_geometric_mean_rank": 0.00018444543820805848, + "inverse_harmonic_mean_rank": 0.001298613591142571, + "inverse_median_rank": 0.0001338150614174083, + "median_absolute_deviation": 5521.210661714862, + "median_rank": 7473.0, + "standard_deviation": 4316.4111328125, + "variance": 18631404.0, + "z_arithmetic_mean_rank": -3.186699747083196, + "z_geometric_mean_rank": 1.1471860758946093, + "z_hits_at_k": 13.14558975014015, + "z_inverse_harmonic_mean_rank": 14.09597681256806 + } + } + }, + "times": { + "evaluation": 29.929590225219727, + "training": 10.419500827789307 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json new file mode 100644 index 0000000..0c7f574 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json new file mode 100644 index 0000000..2257533 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 20.458278117090398, + 14.580624948079425, + 10.229629155634465, + 7.0087562531152345, + 4.624470838201348 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9981434654045136, + "adjusted_arithmetic_mean_rank_index": 0.0018566253253005094, + "adjusted_geometric_mean_rank_index": -0.0037453053362628275, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": -2.435375634154849e-05, + "arithmetic_mean_rank": 20425.2423, + "count": 10000.0, + "geometric_mean_rank": 15115.057389391592, + "harmonic_mean_rank": 4012.1652711899733, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.895902752644457e-05, + "inverse_geometric_mean_rank": 6.61591930641192e-05, + "inverse_harmonic_mean_rank": 0.0002492419759426831, + "inverse_median_rank": 4.8811441401864594e-05, + "median_absolute_deviation": 15004.675752385945, + "median_rank": 20487.0, + "standard_deviation": 11726.714861801267, + "variance": 137515841.4499907, + "z_arithmetic_mean_rank": 0.3215686792065924, + "z_geometric_mean_rank": -0.3748334630216448, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": -0.3843944760649638 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9981445844847684, + "adjusted_arithmetic_mean_rank_index": 0.0018555061903556824, + "adjusted_geometric_mean_rank_index": -0.0037469601889252857, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": -2.4354352510563197e-05, + "arithmetic_mean_rank": 20425.2652, + "count": 10000.0, + "geometric_mean_rank": 15115.082307603394, + "harmonic_mean_rank": 4012.1748654002586, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.895897263551809e-05, + "inverse_geometric_mean_rank": 6.615908399631846e-05, + "inverse_harmonic_mean_rank": 0.00024924137993677375, + "inverse_median_rank": 4.8811441401864594e-05, + "median_absolute_deviation": 15004.675752385945, + "median_rank": 20487.0, + "standard_deviation": 11726.718338958643, + "variance": 137515923.00126895, + "z_arithmetic_mean_rank": 0.3213748443271811, + "z_geometric_mean_rank": -0.3749990821364122, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": -0.38440388586905394 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9981439371347137, + "adjusted_arithmetic_mean_rank_index": 0.0018561535720467148, + "adjusted_geometric_mean_rank_index": -0.00374499619021873, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": -2.4354046496305264e-05, + "arithmetic_mean_rank": 20425.251953125, + "count": 10000.0, + "geometric_mean_rank": 15115.052734375, + "harmonic_mean_rank": 4012.1699406750786, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.895900565315969e-05, + "inverse_geometric_mean_rank": 6.61592130200006e-05, + "inverse_harmonic_mean_rank": 0.00024924168586730954, + "inverse_median_rank": 4.881144195678644e-05, + "median_absolute_deviation": 15004.675752385945, + "median_rank": 20487.0, + "standard_deviation": 11726.716796875, + "variance": 137515888.0, + "z_arithmetic_mean_rank": 0.32148697124500814, + "z_geometric_mean_rank": -0.3748025234127681, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": -0.384399055805523 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9927426109595996, + "adjusted_arithmetic_mean_rank_index": 0.0072577437210114, + "adjusted_geometric_mean_rank_index": 0.004254858044036025, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": -2.3555047261868654e-05, + "arithmetic_mean_rank": 20314.2524, + "count": 5000.0, + "geometric_mean_rank": 14994.993173994697, + "harmonic_mean_rank": 3999.2596074854223, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.922652236023216e-05, + "inverse_geometric_mean_rank": 6.668892665681675e-05, + "inverse_harmonic_mean_rank": 0.0002500462830990761, + "inverse_median_rank": 4.904845987835982e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20388.0, + "standard_deviation": 11707.495051388842, + "variance": 137065440.37829423, + "z_arithmetic_mean_rank": 0.8888655325637131, + "z_geometric_mean_rank": 0.30112952732788756, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": -0.2628906243819342 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9927437349527253, + "adjusted_arithmetic_mean_rank_index": 0.007256619672954323, + "adjusted_geometric_mean_rank_index": 0.004252941731053506, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": -2.3556101099611148e-05, + "arithmetic_mean_rank": 20314.2754, + "count": 5000.0, + "geometric_mean_rank": 14995.022029956497, + "harmonic_mean_rank": 3999.2764581073648, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.922646662553369e-05, + "inverse_geometric_mean_rank": 6.668879832268584e-05, + "inverse_harmonic_mean_rank": 0.0002500452295496582, + "inverse_median_rank": 4.904725703215048e-05, + "median_absolute_deviation": 14936.476050334686, + "median_rank": 20388.5, + "standard_deviation": 11707.499174407609, + "variance": 137065536.91875485, + "z_arithmetic_mean_rank": 0.8887278689022224, + "z_geometric_mean_rank": 0.30099390390246583, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": -0.2629023859402553 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9927430663600753, + "adjusted_arithmetic_mean_rank_index": 0.007257288298279541, + "adjusted_geometric_mean_rank_index": 0.004253626494188811, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": -2.355555413243441e-05, + "arithmetic_mean_rank": 20314.26171875, + "count": 5000.0, + "geometric_mean_rank": 14995.01171875, + "harmonic_mean_rank": 3999.267712210899, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.922649895888753e-05, + "inverse_geometric_mean_rank": 6.668884452665225e-05, + "inverse_harmonic_mean_rank": 0.0002500457763671875, + "inverse_median_rank": 4.90478596475441e-05, + "median_absolute_deviation": 14936.846700889313, + "median_rank": 20388.25, + "standard_deviation": 11707.4970703125, + "variance": 137065488.0, + "z_arithmetic_mean_rank": 0.8888097563356467, + "z_geometric_mean_rank": 0.30104236671770934, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": -0.26289628140813504 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0035440694387567, + "adjusted_arithmetic_mean_rank_index": -0.003544242635263428, + "adjusted_geometric_mean_rank_index": -0.011708739851778116, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -2.5152465411934434e-05, + "arithmetic_mean_rank": 20536.2322, + "count": 5000.0, + "geometric_mean_rank": 15236.08295339673, + "harmonic_mean_rank": 4025.1544980492276, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.869442409206885e-05, + "inverse_geometric_mean_rank": 6.56336673316064e-05, + "inverse_harmonic_mean_rank": 0.00024843766878629013, + "inverse_median_rank": 4.8432013560963794e-05, + "median_absolute_deviation": 15072.875454437202, + "median_rank": 20647.5, + "standard_deviation": 11744.854405478305, + "variance": 137941605.00588316, + "z_arithmetic_mean_rank": -0.4340681859082025, + "z_geometric_mean_rank": -0.8286638855940964, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.28072546493266515 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0035451836063685, + "adjusted_arithmetic_mean_rank_index": -0.0035453568573240446, + "adjusted_geometric_mean_rank_index": -0.011710128780810125, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -2.515260391222656e-05, + "arithmetic_mean_rank": 20536.255, + "count": 5000.0, + "geometric_mean_rank": 15236.103868950964, + "harmonic_mean_rank": 4025.15674140012, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.869437002997869e-05, + "inverse_geometric_mean_rank": 6.563357723215968e-05, + "inverse_harmonic_mean_rank": 0.00024843753032388936, + "inverse_median_rank": 4.8432013560963794e-05, + "median_absolute_deviation": 15072.875454437202, + "median_rank": 20647.5, + "standard_deviation": 11744.857241038522, + "variance": 137941671.612375, + "z_arithmetic_mean_rank": -0.4342046461899116, + "z_geometric_mean_rank": -0.8287621844155808, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.2807270107278095 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0035446529412848, + "adjusted_arithmetic_mean_rank_index": -0.0035448261663069136, + "adjusted_geometric_mean_rank_index": -0.01170900286578247, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -2.5152538850884614e-05, + "arithmetic_mean_rank": 20536.244140625, + "count": 5000.0, + "geometric_mean_rank": 15236.0869140625, + "harmonic_mean_rank": 4025.155687572287, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.869439726462588e-05, + "inverse_geometric_mean_rank": 6.56336487736553e-05, + "inverse_harmonic_mean_rank": 0.00024843759536743163, + "inverse_median_rank": 4.8432015319122e-05, + "median_absolute_deviation": 15072.875454437202, + "median_rank": 20647.5, + "standard_deviation": 11744.8564453125, + "variance": 137941648.0, + "z_arithmetic_mean_rank": -0.43413965174379143, + "z_geometric_mean_rank": -0.828682499912091, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.28072628458127946 + } + } + }, + "times": { + "evaluation": 8.578476905822754, + "training": 2.8984744548797607 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json new file mode 100644 index 0000000..c233fc8 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json new file mode 100644 index 0000000..f92ac60 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 0.000208604416764187, + 0.00020803913257138926, + 0.0002074576098380109, + 0.00020695912158022746, + 0.00020641966698798603 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9207919775259267, + "adjusted_arithmetic_mean_rank_index": 0.07921874676399432, + "adjusted_geometric_mean_rank_index": 0.21897247094458427, + "adjusted_hits_at_k": 0.005318092337383576, + "adjusted_inverse_harmonic_mean_rank": 0.0051853177799535705, + "arithmetic_mean_rank": 6801.7544311083275, + "count": 118142.0, + "geometric_mean_rank": 4242.455386428855, + "harmonic_mean_rank": 170.30311795666364, + "hits_at_1": 0.003927477103824212, + "hits_at_10": 0.005992788339455909, + "hits_at_3": 0.00552724687240778, + "hits_at_5": 0.00565421272705727, + "inverse_arithmetic_mean_rank": 0.00014702089146682892, + "inverse_geometric_mean_rank": 0.00023571255532795686, + "inverse_harmonic_mean_rank": 0.005871883098784287, + "inverse_median_rank": 0.000152114390021296, + "median_absolute_deviation": 5675.401292439445, + "median_rank": 6574.0, + "standard_deviation": 4419.625361302917, + "variance": 19533088.334271934, + "z_arithmetic_mean_rank": 47.12068045997205, + "z_geometric_mean_rank": 75.39534533250384, + "z_hits_at_k": 70.1615400410659, + "z_inverse_harmonic_mean_rank": 168.97846642196097 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9207996732049827, + "adjusted_arithmetic_mean_rank_index": 0.07921105004298967, + "adjusted_geometric_mean_rank_index": 0.21896172620653143, + "adjusted_hits_at_k": 0.005318092337383576, + "adjusted_inverse_harmonic_mean_rank": 0.005185300049122959, + "arithmetic_mean_rank": 6801.811277953649, + "count": 118142.0, + "geometric_mean_rank": 4242.513736903575, + "harmonic_mean_rank": 170.3036318532829, + "hits_at_1": 0.003927477103824212, + "hits_at_10": 0.005992788339455909, + "hits_at_3": 0.00552724687240778, + "hits_at_5": 0.00565421272705727, + "inverse_arithmetic_mean_rank": 0.00014701966272443446, + "inverse_geometric_mean_rank": 0.00023570931339632063, + "inverse_harmonic_mean_rank": 0.0058718653801905, + "inverse_median_rank": 0.000152114390021296, + "median_absolute_deviation": 5675.401292439445, + "median_rank": 6574.0, + "standard_deviation": 4419.628569110977, + "variance": 19533116.688901942, + "z_arithmetic_mean_rank": 47.11610231722352, + "z_geometric_mean_rank": 75.39164576592134, + "z_hits_at_k": 70.1615400410659, + "z_inverse_harmonic_mean_rank": 168.97788861194192 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9207958725565982, + "adjusted_arithmetic_mean_rank_index": 0.07921485120595906, + "adjusted_geometric_mean_rank_index": 0.21896713294853576, + "adjusted_hits_at_k": 0.005318092337383576, + "adjusted_inverse_harmonic_mean_rank": 0.005185308909615302, + "arithmetic_mean_rank": 6801.783203125, + "count": 118142.0, + "geometric_mean_rank": 4242.484375, + "harmonic_mean_rank": 170.3033750472689, + "hits_at_1": 0.003927477103824212, + "hits_at_10": 0.005992788339455909, + "hits_at_3": 0.00552724687240778, + "hits_at_5": 0.00565421272705727, + "inverse_arithmetic_mean_rank": 0.00014702026965096593, + "inverse_geometric_mean_rank": 0.00023571094789076596, + "inverse_harmonic_mean_rank": 0.005871874234567829, + "inverse_median_rank": 0.00015211438585538417, + "median_absolute_deviation": 5675.401292439445, + "median_rank": 6574.0, + "standard_deviation": 4419.62646484375, + "variance": 19533100.0, + "z_arithmetic_mean_rank": 47.11836331468908, + "z_geometric_mean_rank": 75.39350738431921, + "z_hits_at_k": 70.1615400410659, + "z_inverse_harmonic_mean_rank": 168.9781773565226 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9450959015811221, + "adjusted_arithmetic_mean_rank_index": 0.0549115523924576, + "adjusted_geometric_mean_rank_index": 0.16122934046823933, + "adjusted_hits_at_k": 1.3705613470731061e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0007384753351438997, + "arithmetic_mean_rank": 6962.284285012951, + "count": 59071.0, + "geometric_mean_rank": 4542.932684386978, + "harmonic_mean_rank": 699.2987534521512, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006940800054172098, + "hits_at_3": 0.00027086048991891115, + "hits_at_5": 0.00038936195425843476, + "inverse_arithmetic_mean_rank": 0.00014363102094992088, + "inverse_geometric_mean_rank": 0.00022012212583223426, + "inverse_harmonic_mean_rank": 0.001430003979076768, + "inverse_median_rank": 0.00014749262536873156, + "median_absolute_deviation": 5669.470883565422, + "median_rank": 6780.0, + "standard_deviation": 4399.418868449341, + "variance": 19354886.380068082, + "z_arithmetic_mean_rank": 23.092416599277726, + "z_geometric_mean_rank": 39.251298658222595, + "z_hits_at_k": 0.12766190813382305, + "z_inverse_harmonic_mean_rank": 16.990796957990767 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9451039192990608, + "adjusted_arithmetic_mean_rank_index": 0.054903533586005304, + "adjusted_geometric_mean_rank_index": 0.16121720085611924, + "adjusted_hits_at_k": 1.3705613470731061e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0007384551039659525, + "arithmetic_mean_rank": 6962.3433495285335, + "count": 59071.0, + "geometric_mean_rank": 4542.998420235553, + "harmonic_mean_rank": 699.3086401705666, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006940800054172098, + "hits_at_3": 0.00027086048991891115, + "hits_at_5": 0.00038936195425843476, + "inverse_arithmetic_mean_rank": 0.0001436298024669692, + "inverse_geometric_mean_rank": 0.00022011894072993104, + "inverse_harmonic_mean_rank": 0.001429983761899599, + "inverse_median_rank": 0.00014749262536873156, + "median_absolute_deviation": 5669.470883565422, + "median_rank": 6780.0, + "standard_deviation": 4399.420293881387, + "variance": 19354898.922215387, + "z_arithmetic_mean_rank": 23.089044383211046, + "z_geometric_mean_rank": 39.24834326846823, + "z_hits_at_k": 0.12766190813382305, + "z_inverse_harmonic_mean_rank": 16.990331480241736 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9450998641906208, + "adjusted_arithmetic_mean_rank_index": 0.05490758924498118, + "adjusted_geometric_mean_rank_index": 0.1612233113418693, + "adjusted_hits_at_k": 1.3705613470731061e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0007384651003795702, + "arithmetic_mean_rank": 6962.3134765625, + "count": 59071.0, + "geometric_mean_rank": 4542.96533203125, + "harmonic_mean_rank": 699.3037550158945, + "hits_at_1": 8.464390309965973e-05, + "hits_at_10": 0.0006940800054172098, + "hits_at_3": 0.00027086048991891115, + "hits_at_5": 0.00038936195425843476, + "inverse_arithmetic_mean_rank": 0.0001436304155504331, + "inverse_geometric_mean_rank": 0.00022012054978404194, + "inverse_harmonic_mean_rank": 0.0014299937513953017, + "inverse_median_rank": 0.00014749262481927872, + "median_absolute_deviation": 5669.470883565422, + "median_rank": 6780.0, + "standard_deviation": 4399.41943359375, + "variance": 19354892.0, + "z_arithmetic_mean_rank": 23.090749943563594, + "z_geometric_mean_rank": 39.249830866819906, + "z_hits_at_k": 0.12766190813382305, + "z_inverse_harmonic_mean_rank": 16.990561477136694 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.896619981254972, + "adjusted_arithmetic_mean_rank_index": 0.10339397778305548, + "adjusted_geometric_mean_rank_index": 0.2727473513664458, + "adjusted_hits_at_k": 0.010622456975492961, + "adjusted_inverse_harmonic_mean_rank": 0.009632143352825863, + "arithmetic_mean_rank": 6641.224577203704, + "count": 59071.0, + "geometric_mean_rank": 3961.852168247115, + "harmonic_mean_rank": 96.95782962758972, + "hits_at_1": 0.007770310304548764, + "hits_at_10": 0.011291496673494609, + "hits_at_3": 0.01078363325489665, + "hits_at_5": 0.010919063499856106, + "inverse_arithmetic_mean_rank": 0.00015057464001933378, + "inverse_geometric_mean_rank": 0.00025240719681937065, + "inverse_harmonic_mean_rank": 0.010313762218491803, + "inverse_median_rank": 0.00015705983979896342, + "median_absolute_deviation": 5651.6796569433545, + "median_rank": 6367.0, + "standard_deviation": 4433.931733855012, + "variance": 19659750.620486513, + "z_arithmetic_mean_rank": 43.493994503669626, + "z_geometric_mean_rank": 66.39982981680666, + "z_hits_at_k": 99.24774588949502, + "z_inverse_harmonic_mean_rank": 222.29474063277573 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8966273566432531, + "adjusted_arithmetic_mean_rank_index": 0.10338660139890177, + "adjusted_geometric_mean_rank_index": 0.2727378670781261, + "adjusted_hits_at_k": 0.010622456975492961, + "adjusted_inverse_harmonic_mean_rank": 0.009632128122333099, + "arithmetic_mean_rank": 6641.279206378765, + "count": 59071.0, + "geometric_mean_rank": 3961.9038227361375, + "harmonic_mean_rank": 96.95797270839012, + "hits_at_1": 0.007770310304548764, + "hits_at_10": 0.011291496673494609, + "hits_at_3": 0.01078363325489665, + "hits_at_5": 0.010919063499856106, + "inverse_arithmetic_mean_rank": 0.000150573401437411, + "inverse_geometric_mean_rank": 0.00025240390598613475, + "inverse_harmonic_mean_rank": 0.010313746998481399, + "inverse_median_rank": 0.00015705983979896342, + "median_absolute_deviation": 5651.6796569433545, + "median_rank": 6367.0, + "standard_deviation": 4433.936553851471, + "variance": 19659793.363580257, + "z_arithmetic_mean_rank": 43.4908915336639, + "z_geometric_mean_rank": 66.39752088464945, + "z_hits_at_k": 99.24774588949502, + "z_inverse_harmonic_mean_rank": 222.2943891369233 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.896623611307688, + "adjusted_arithmetic_mean_rank_index": 0.10339034724018625, + "adjusted_geometric_mean_rank_index": 0.27274262130459126, + "adjusted_hits_at_k": 0.010622456975492961, + "adjusted_inverse_harmonic_mean_rank": 0.009632136363889678, + "arithmetic_mean_rank": 6641.25146484375, + "count": 59071.0, + "geometric_mean_rank": 3961.8779296875, + "harmonic_mean_rank": 96.9578952841512, + "hits_at_1": 0.007770310304548764, + "hits_at_10": 0.011291496673494609, + "hits_at_3": 0.01078363325489665, + "hits_at_5": 0.010919063499856106, + "inverse_arithmetic_mean_rank": 0.00015057403652463108, + "inverse_geometric_mean_rank": 0.0002524055598769337, + "inverse_harmonic_mean_rank": 0.010313755234365741, + "inverse_median_rank": 0.00015705983969382942, + "median_absolute_deviation": 5651.6796569433545, + "median_rank": 6367.0, + "standard_deviation": 4433.93408203125, + "variance": 19659772.0, + "z_arithmetic_mean_rank": 43.492467269540725, + "z_geometric_mean_rank": 66.39867829214256, + "z_hits_at_k": 99.24774588949502, + "z_inverse_harmonic_mean_rank": 222.29457933910518 + } + } + }, + "times": { + "evaluation": 33.42562985420227, + "training": 12.187381505966187 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json new file mode 100644 index 0000000..1d65f30 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json new file mode 100644 index 0000000..a2e5cc1 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0027987702639484797, + 0.0027985652882265874, + 0.0027976621540910128, + 0.0027974926507223015, + 0.0027966113623240974 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9920241586458991, + "adjusted_arithmetic_mean_rank_index": 0.007976231137628131, + "adjusted_geometric_mean_rank_index": 0.01955723705510304, + "adjusted_hits_at_k": -0.00024440697472276075, + "adjusted_inverse_harmonic_mean_rank": -6.828706250853293e-05, + "arithmetic_mean_rank": 20300.0215, + "count": 10000.0, + "geometric_mean_rank": 14764.17558586092, + "harmonic_mean_rank": 4870.429778354284, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.9261031570828635e-05, + "inverse_geometric_mean_rank": 6.773151634403897e-05, + "inverse_harmonic_mean_rank": 0.00020532068944804693, + "inverse_median_rank": 4.95221116228396e-05, + "median_absolute_deviation": 15644.418609671111, + "median_rank": 20193.0, + "standard_deviation": 11944.86457925069, + "variance": 142679789.81663775, + "z_arithmetic_mean_rank": 1.3814882717699675, + "z_geometric_mean_rank": 1.9573055423605155, + "z_hits_at_k": -1.5633520872629163, + "z_inverse_harmonic_mean_rank": -1.0778283746807773 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9920259472195815, + "adjusted_arithmetic_mean_rank_index": 0.007974442476537336, + "adjusted_geometric_mean_rank_index": 0.019554508271745097, + "adjusted_hits_at_k": -0.00024440697472276075, + "adjusted_inverse_harmonic_mean_rank": -6.828763431190998e-05, + "arithmetic_mean_rank": 20300.0581, + "count": 10000.0, + "geometric_mean_rank": 14764.216674957981, + "harmonic_mean_rank": 4870.443338478342, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.9260942755626894e-05, + "inverse_geometric_mean_rank": 6.773132784593504e-05, + "inverse_harmonic_mean_rank": 0.00020532011780110903, + "inverse_median_rank": 4.952088543343155e-05, + "median_absolute_deviation": 15644.418609671111, + "median_rank": 20193.5, + "standard_deviation": 11944.86448975979, + "variance": 142679787.67872438, + "z_arithmetic_mean_rank": 1.381178474539098, + "z_geometric_mean_rank": 1.957032443314091, + "z_hits_at_k": -1.5633520872629163, + "z_inverse_harmonic_mean_rank": -1.0778373999028603 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.992025016892492, + "adjusted_arithmetic_mean_rank_index": 0.007975372849092333, + "adjusted_geometric_mean_rank_index": 0.019554954159064608, + "adjusted_hits_at_k": -0.00024440697472276075, + "adjusted_inverse_harmonic_mean_rank": -6.828734607407729e-05, + "arithmetic_mean_rank": 20300.0390625, + "count": 10000.0, + "geometric_mean_rank": 14764.2099609375, + "harmonic_mean_rank": 4870.436503006005, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.926099063595757e-05, + "inverse_geometric_mean_rank": 6.773135828552768e-05, + "inverse_harmonic_mean_rank": 0.000205320405960083, + "inverse_median_rank": 4.952149902237579e-05, + "median_absolute_deviation": 15644.418609671111, + "median_rank": 20193.25, + "standard_deviation": 11944.8642578125, + "variance": 142679792.0, + "z_arithmetic_mean_rank": 1.38133961565336, + "z_geometric_mean_rank": 1.9570770681104932, + "z_hits_at_k": -1.5633520872629163, + "z_inverse_harmonic_mean_rank": -1.0778328504186194 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.991158386631214, + "adjusted_arithmetic_mean_rank_index": 0.008842045473061222, + "adjusted_geometric_mean_rank_index": 0.023575799917052054, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -7.339889583391683e-05, + "arithmetic_mean_rank": 20281.8348, + "count": 5000.0, + "geometric_mean_rank": 14704.057212218282, + "harmonic_mean_rank": 4994.604040839103, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.930520388618884e-05, + "inverse_geometric_mean_rank": 6.800844049824926e-05, + "inverse_harmonic_mean_rank": 0.00020021607154908683, + "inverse_median_rank": 4.940955580809328e-05, + "median_absolute_deviation": 15743.011657201734, + "median_rank": 20239.0, + "standard_deviation": 11999.952493518838, + "variance": 143998859.84670898, + "z_arithmetic_mean_rank": 1.0828970766234107, + "z_geometric_mean_rank": 1.6685326306831314, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.8191824597167967 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9911600286385629, + "adjusted_arithmetic_mean_rank_index": 0.008840403385464768, + "adjusted_geometric_mean_rank_index": 0.023573195103802513, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -7.339957108391955e-05, + "arithmetic_mean_rank": 20281.8684, + "count": 5000.0, + "geometric_mean_rank": 14704.096435660544, + "harmonic_mean_rank": 4994.620881120737, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.930512220461898e-05, + "inverse_geometric_mean_rank": 6.800825908450849e-05, + "inverse_harmonic_mean_rank": 0.00020021539648382906, + "inverse_median_rank": 4.940955580809328e-05, + "median_absolute_deviation": 15743.011657201734, + "median_rank": 20239.0, + "standard_deviation": 11999.952940452784, + "variance": 143998870.57308143, + "z_arithmetic_mean_rank": 1.082695967970082, + "z_geometric_mean_rank": 1.6683482799540457, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.8191899959740097 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.991159205802291, + "adjusted_arithmetic_mean_rank_index": 0.008841226261950053, + "adjusted_geometric_mean_rank_index": 0.023574021988738925, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -7.339924634853156e-05, + "arithmetic_mean_rank": 20281.8515625, + "count": 5000.0, + "geometric_mean_rank": 14704.083984375, + "harmonic_mean_rank": 4994.6127824240475, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.9305162974633276e-05, + "inverse_geometric_mean_rank": 6.800831761211157e-05, + "inverse_harmonic_mean_rank": 0.0002002157211303711, + "inverse_median_rank": 4.940955477650277e-05, + "median_absolute_deviation": 15743.011657201734, + "median_rank": 20239.0, + "standard_deviation": 11999.953125, + "variance": 143998864.0, + "z_arithmetic_mean_rank": 1.0827967467483717, + "z_geometric_mean_rank": 1.6684068011708462, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.8191863717024064 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9928898905190562, + "adjusted_arithmetic_mean_rank_index": 0.007110456947646893, + "adjusted_geometric_mean_rank_index": 0.015620408929683083, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": -6.31752292426308e-05, + "arithmetic_mean_rank": 20318.2082, + "count": 5000.0, + "geometric_mean_rank": 14824.539756891101, + "harmonic_mean_rank": 4752.280096951103, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.921693833219014e-05, + "inverse_geometric_mean_rank": 6.745571979967579e-05, + "inverse_harmonic_mean_rank": 0.000210425307347007, + "inverse_median_rank": 4.966969651815427e-05, + "median_absolute_deviation": 15533.964744392444, + "median_rank": 20133.0, + "standard_deviation": 11889.493608831823, + "variance": 141360058.27445275, + "z_arithmetic_mean_rank": 0.8708272728100468, + "z_geometric_mean_rank": 1.105504855526732, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": -0.7050957157046835 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9928918256522765, + "adjusted_arithmetic_mean_rank_index": 0.007108521719857874, + "adjusted_geometric_mean_rank_index": 0.015617555484825907, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": -6.317569759938329e-05, + "arithmetic_mean_rank": 20318.2478, + "count": 5000.0, + "geometric_mean_rank": 14824.582726243238, + "harmonic_mean_rank": 4752.290671527522, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.921684240901915e-05, + "inverse_geometric_mean_rank": 6.745552427791094e-05, + "inverse_harmonic_mean_rank": 0.00021042483911838908, + "inverse_median_rank": 4.966969651815427e-05, + "median_absolute_deviation": 15533.964744392444, + "median_rank": 20133.0, + "standard_deviation": 11889.49296875166, + "variance": 141360043.05399516, + "z_arithmetic_mean_rank": 0.8705902628470881, + "z_geometric_mean_rank": 1.105302908371641, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": -0.7051009430120248 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9928908832827135, + "adjusted_arithmetic_mean_rank_index": 0.007109464135473975, + "adjusted_geometric_mean_rank_index": 0.015618509538453562, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": -6.317546970748797e-05, + "arithmetic_mean_rank": 20318.228515625, + "count": 5000.0, + "geometric_mean_rank": 14824.568359375, + "harmonic_mean_rank": 4752.285526169837, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.9216887418879196e-05, + "inverse_geometric_mean_rank": 6.745559221599251e-05, + "inverse_harmonic_mean_rank": 0.000210425066947937, + "inverse_median_rank": 4.966969572706148e-05, + "median_absolute_deviation": 15533.964744392444, + "median_rank": 20133.0, + "standard_deviation": 11889.4931640625, + "variance": 141360048.0, + "z_arithmetic_mean_rank": 0.8707056817613505, + "z_geometric_mean_rank": 1.105370429710078, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": -0.7050983995214675 + } + } + }, + "times": { + "evaluation": 9.90921139717102, + "training": 4.695069313049316 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json new file mode 100644 index 0000000..f185e87 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json new file mode 100644 index 0000000..ac1aceb --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0002195790323791253, + 1.653514352525725e-05, + 1.4086949304634584e-05, + 1.1515389150776652e-05, + 9.777598791220405e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07133568713438639, + "adjusted_arithmetic_mean_rank_index": 0.9287900484289984, + "adjusted_geometric_mean_rank_index": 0.9924661457284999, + "adjusted_hits_at_k": 0.35636286355421287, + "adjusted_inverse_harmonic_mean_rank": 0.236786103943646, + "arithmetic_mean_rank": 526.9461918707996, + "count": 118142.0, + "geometric_mean_rank": 41.91341929915113, + "harmonic_mean_rank": 4.213847157632036, + "hits_at_1": 0.173883970137631, + "hits_at_10": 0.3567994447359957, + "hits_at_3": 0.2580200098186928, + "hits_at_5": 0.2985813681840497, + "inverse_arithmetic_mean_rank": 0.001897726969901297, + "inverse_geometric_mean_rank": 0.023858707228409135, + "inverse_harmonic_mean_rank": 0.2373128313846932, + "inverse_median_rank": 0.022222222222222223, + "median_absolute_deviation": 65.23449761424648, + "median_rank": 45.0, + "standard_deviation": 1306.1796683025648, + "variance": 1706105.325886998, + "z_arithmetic_mean_rank": 552.460381843816, + "z_geometric_mean_rank": 341.72025125001227, + "z_hits_at_k": 4701.491763249247, + "z_inverse_harmonic_mean_rank": 7716.354987752866 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07133572953166528, + "adjusted_arithmetic_mean_rank_index": 0.9287900060259792, + "adjusted_geometric_mean_rank_index": 0.9924661433888894, + "adjusted_hits_at_k": 0.35636286355421287, + "adjusted_inverse_harmonic_mean_rank": 0.23678610315360502, + "arithmetic_mean_rank": 526.946505053241, + "count": 118142.0, + "geometric_mean_rank": 41.913432004661814, + "harmonic_mean_rank": 4.213847171650723, + "hits_at_1": 0.173883970137631, + "hits_at_10": 0.3567994447359957, + "hits_at_3": 0.2580200098186928, + "hits_at_5": 0.2985813681840497, + "inverse_arithmetic_mean_rank": 0.0018977258420168537, + "inverse_geometric_mean_rank": 0.023858699995952973, + "inverse_harmonic_mean_rank": 0.23731283059519745, + "inverse_median_rank": 0.022222222222222223, + "median_absolute_deviation": 65.23449761424648, + "median_rank": 45.0, + "standard_deviation": 1306.1802348103465, + "variance": 1706106.805809212, + "z_arithmetic_mean_rank": 552.4603566217669, + "z_geometric_mean_rank": 341.720250444451, + "z_hits_at_k": 4701.491763249247, + "z_inverse_harmonic_mean_rank": 7716.354962007112 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07133570029177673, + "adjusted_arithmetic_mean_rank_index": 0.9287900352698266, + "adjusted_geometric_mean_rank_index": 0.9924661467040232, + "adjusted_hits_at_k": 0.35636286355421287, + "adjusted_inverse_harmonic_mean_rank": 0.23678609380201712, + "arithmetic_mean_rank": 526.9462890625, + "count": 118142.0, + "geometric_mean_rank": 41.913414001464844, + "harmonic_mean_rank": 4.2138473375876755, + "hits_at_1": 0.173883970137631, + "hits_at_10": 0.3567994447359957, + "hits_at_3": 0.2580200098186928, + "hits_at_5": 0.2985813681840497, + "inverse_arithmetic_mean_rank": 0.0018977266736328602, + "inverse_geometric_mean_rank": 0.02385871112346649, + "inverse_harmonic_mean_rank": 0.2373128212500635, + "inverse_median_rank": 0.02222222276031971, + "median_absolute_deviation": 65.23449761424648, + "median_rank": 45.0, + "standard_deviation": 1306.179931640625, + "variance": 1706105.875, + "z_arithmetic_mean_rank": 552.4603740165131, + "z_geometric_mean_rank": 341.7202515858989, + "z_hits_at_k": 4701.491763249247, + "z_inverse_harmonic_mean_rank": 7716.354657258773 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07689682971067989, + "adjusted_arithmetic_mean_rank_index": 0.9232284940236298, + "adjusted_geometric_mean_rank_index": 0.9906866025503254, + "adjusted_hits_at_k": 0.32003303624630025, + "adjusted_inverse_harmonic_mean_rank": 0.20532087082516273, + "arithmetic_mean_rank": 566.4796431413046, + "count": 59071.0, + "geometric_mean_rank": 51.43193130167176, + "harmonic_mean_rank": 4.857414947867291, + "hits_at_1": 0.1445717864942188, + "hits_at_10": 0.3204956746965516, + "hits_at_3": 0.22456027492339728, + "hits_at_5": 0.263428755226761, + "inverse_arithmetic_mean_rank": 0.0017652885008447809, + "inverse_geometric_mean_rank": 0.019443174205038955, + "inverse_harmonic_mean_rank": 0.20587082032986756, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 1355.3119844999508, + "variance": 1836870.575329195, + "z_arithmetic_mean_rank": 388.25303731980824, + "z_geometric_mean_rank": 241.1827499912326, + "z_hits_at_k": 2980.9704002169224, + "z_inverse_harmonic_mean_rank": 4724.010486752963 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07689688026665428, + "adjusted_arithmetic_mean_rank_index": 0.9232284434607918, + "adjusted_geometric_mean_rank_index": 0.9906865989916975, + "adjusted_hits_at_k": 0.32003303624630025, + "adjusted_inverse_harmonic_mean_rank": 0.20532086964081991, + "arithmetic_mean_rank": 566.4800155744782, + "count": 59071.0, + "geometric_mean_rank": 51.43195057159756, + "harmonic_mean_rank": 4.857414975791905, + "hits_at_1": 0.1445717864942188, + "hits_at_10": 0.3204956746965516, + "hits_at_3": 0.22456027492339728, + "hits_at_5": 0.263428755226761, + "inverse_arithmetic_mean_rank": 0.0017652873402530906, + "inverse_geometric_mean_rank": 0.0194431669202963, + "inverse_harmonic_mean_rank": 0.20587081914634436, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 1355.3126588901207, + "variance": 1836872.4033478086, + "z_arithmetic_mean_rank": 388.253016056193, + "z_geometric_mean_rank": 241.1827491248843, + "z_hits_at_k": 2980.9704002169224, + "z_inverse_harmonic_mean_rank": 4724.010459503673 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.0768968589307786, + "adjusted_arithmetic_mean_rank_index": 0.9232284647995641, + "adjusted_geometric_mean_rank_index": 0.9906865998727367, + "adjusted_hits_at_k": 0.32003303624630025, + "adjusted_inverse_harmonic_mean_rank": 0.20532086895293028, + "arithmetic_mean_rank": 566.4798583984375, + "count": 59071.0, + "geometric_mean_rank": 51.43194580078125, + "harmonic_mean_rank": 4.857414992011072, + "hits_at_1": 0.1445717864942188, + "hits_at_10": 0.3204956746965516, + "hits_at_3": 0.22456027492339728, + "hits_at_5": 0.263428755226761, + "inverse_arithmetic_mean_rank": 0.0017652878304943442, + "inverse_geometric_mean_rank": 0.019443169236183167, + "inverse_harmonic_mean_rank": 0.20587081845893077, + "inverse_median_rank": 0.016949152573943138, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 1355.312255859375, + "variance": 1836871.25, + "z_arithmetic_mean_rank": 388.2530250299663, + "z_geometric_mean_rank": 241.1827493393734, + "z_hits_at_k": 2980.9704002169224, + "z_inverse_harmonic_mean_rank": 4724.010443676749 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.06580473183222485, + "adjusted_arithmetic_mean_rank_index": 0.9343214092483405, + "adjusted_geometric_mean_rank_index": 0.9939121303205123, + "adjusted_hits_at_k": 0.39269253959611367, + "adjusted_inverse_harmonic_mean_rank": 0.268251217678668, + "arithmetic_mean_rank": 487.41274060029457, + "count": 59071.0, + "geometric_mean_rank": 34.156499141407, + "harmonic_mean_rank": 3.7208631886327486, + "hits_at_1": 0.20319615378104317, + "hits_at_10": 0.39310321477543975, + "hits_at_3": 0.29147974471398824, + "hits_at_5": 0.3337339811413384, + "inverse_arithmetic_mean_rank": 0.0020516492834561652, + "inverse_geometric_mean_rank": 0.02927700511284914, + "inverse_harmonic_mean_rank": 0.26875484243951886, + "inverse_median_rank": 0.030303030303030304, + "median_absolute_deviation": 47.443270992179265, + "median_rank": 33.0, + "standard_deviation": 1253.8796947498934, + "variance": 1572214.288906086, + "z_arithmetic_mean_rank": 393.0342086632433, + "z_geometric_mean_rank": 241.96603917694628, + "z_hits_at_k": 3669.005153181792, + "z_inverse_harmonic_mean_rank": 6190.816796846298 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0658047661150957, + "adjusted_arithmetic_mean_rank_index": 0.9343213749608406, + "adjusted_geometric_mean_rank_index": 0.9939121288680024, + "adjusted_hits_at_k": 0.39269253959611367, + "adjusted_inverse_harmonic_mean_rank": 0.2682512172829272, + "arithmetic_mean_rank": 487.41299453200384, + "count": 59071.0, + "geometric_mean_rank": 34.15650705224349, + "harmonic_mean_rank": 3.7208631941079378, + "hits_at_1": 0.20319615378104317, + "hits_at_10": 0.39310321477543975, + "hits_at_3": 0.29147974471398824, + "hits_at_5": 0.3337339811413384, + "inverse_arithmetic_mean_rank": 0.002051648214590921, + "inverse_geometric_mean_rank": 0.029276998332132365, + "inverse_harmonic_mean_rank": 0.2687548420440505, + "inverse_median_rank": 0.030303030303030304, + "median_absolute_deviation": 47.443270992179265, + "median_rank": 33.0, + "standard_deviation": 1253.880142343099, + "variance": 1572215.4113623502, + "z_arithmetic_mean_rank": 393.0341942397695, + "z_geometric_mean_rank": 241.96603882333548, + "z_hits_at_k": 3669.005153181792, + "z_inverse_harmonic_mean_rank": 6190.816787713223 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.06580474137447191, + "adjusted_arithmetic_mean_rank_index": 0.934321399704805, + "adjusted_geometric_mean_rank_index": 0.9939121298378724, + "adjusted_hits_at_k": 0.39269253959611367, + "adjusted_inverse_harmonic_mean_rank": 0.2682512158110723, + "arithmetic_mean_rank": 487.4128112792969, + "count": 59071.0, + "geometric_mean_rank": 34.15650177001953, + "harmonic_mean_rank": 3.7208632144714846, + "hits_at_1": 0.20319615378104317, + "hits_at_10": 0.39310321477543975, + "hits_at_3": 0.29147974471398824, + "hits_at_5": 0.3337339811413384, + "inverse_arithmetic_mean_rank": 0.0020516489166766405, + "inverse_geometric_mean_rank": 0.029277002438902855, + "inverse_harmonic_mean_rank": 0.26875484057320853, + "inverse_median_rank": 0.03030303120613098, + "median_absolute_deviation": 47.443270992179265, + "median_rank": 33.0, + "standard_deviation": 1253.8798828125, + "variance": 1572214.75, + "z_arithmetic_mean_rank": 393.0342046486335, + "z_geometric_mean_rank": 241.9660390594485, + "z_hits_at_k": 3669.005153181792, + "z_inverse_harmonic_mean_rank": 6190.816753745123 + } + } + }, + "times": { + "evaluation": 22.204118728637695, + "training": 399.59123039245605 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..9849e52 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..e5662dd --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 2048, + "slice_size": null + }, + "losses": [ + 0.0027874300545268718, + 7.365971419707463e-05, + 2.8435559064788497e-05, + 2.112122028404341e-05, + 1.900511581936777e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5633845099742385, + "adjusted_arithmetic_mean_rank_index": 0.43667668634658385, + "adjusted_geometric_mean_rank_index": 0.765892175485255, + "adjusted_hits_at_k": 0.06116118490603417, + "adjusted_inverse_harmonic_mean_rank": 0.02850689581330692, + "arithmetic_mean_rank": 4020.1253547313827, + "count": 40876.0, + "geometric_mean_rank": 1228.5381219971325, + "harmonic_mean_rank": 34.24748400705478, + "hits_at_1": 0.012501223211664546, + "hits_at_10": 0.06182111752617673, + "hits_at_3": 0.029381544182405323, + "hits_at_5": 0.04305705059203445, + "inverse_arithmetic_mean_rank": 0.0002487484622396354, + "inverse_geometric_mean_rank": 0.0008139755552512957, + "inverse_harmonic_mean_rank": 0.029199225256781078, + "inverse_median_rank": 0.0003372112628561794, + "median_absolute_deviation": 4152.0275129249385, + "median_rank": 2965.5, + "standard_deviation": 3853.139281834975, + "variance": 14846682.325219747, + "z_arithmetic_mean_rank": 152.72046645669522, + "z_geometric_mean_rank": 155.11300167144324, + "z_hits_at_k": 466.233760257905, + "z_inverse_harmonic_mean_rank": 536.8020979000396 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5633849385295965, + "adjusted_arithmetic_mean_rank_index": 0.43667625773115926, + "adjusted_geometric_mean_rank_index": 0.7658919832564306, + "adjusted_hits_at_k": 0.06116118490603417, + "adjusted_inverse_harmonic_mean_rank": 0.028506895380300473, + "arithmetic_mean_rank": 4020.128412760544, + "count": 40876.0, + "geometric_mean_rank": 1228.539129943837, + "harmonic_mean_rank": 34.24748451456186, + "hits_at_1": 0.012501223211664546, + "hits_at_10": 0.06182111752617673, + "hits_at_3": 0.029381544182405323, + "hits_at_5": 0.04305705059203445, + "inverse_arithmetic_mean_rank": 0.00024874827302178625, + "inverse_geometric_mean_rank": 0.0008139748874305007, + "inverse_harmonic_mean_rank": 0.02919922482408321, + "inverse_median_rank": 0.0003372112628561794, + "median_absolute_deviation": 4152.0275129249385, + "median_rank": 2965.5, + "standard_deviation": 3853.140645811768, + "variance": 14846692.836406728, + "z_arithmetic_mean_rank": 152.72031655552206, + "z_geometric_mean_rank": 155.11296274012767, + "z_hits_at_k": 466.233760257905, + "z_inverse_harmonic_mean_rank": 536.8020897462662 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5633846997606484, + "adjusted_arithmetic_mean_rank_index": 0.43667649653357343, + "adjusted_geometric_mean_rank_index": 0.7658920426797261, + "adjusted_hits_at_k": 0.06116118490603417, + "adjusted_inverse_harmonic_mean_rank": 0.028506894681160907, + "arithmetic_mean_rank": 4020.126708984375, + "count": 40876.0, + "geometric_mean_rank": 1228.538818359375, + "harmonic_mean_rank": 34.24748533399144, + "hits_at_1": 0.012501223211664546, + "hits_at_10": 0.06182111752617673, + "hits_at_3": 0.029381544182405323, + "hits_at_5": 0.04305705059203445, + "inverse_arithmetic_mean_rank": 0.00024874837254174054, + "inverse_geometric_mean_rank": 0.0008139751153066754, + "inverse_harmonic_mean_rank": 0.029199224125441885, + "inverse_median_rank": 0.00033721126965247095, + "median_absolute_deviation": 4152.0275129249385, + "median_rank": 2965.5, + "standard_deviation": 3853.139892578125, + "variance": 14846687.0, + "z_arithmetic_mean_rank": 152.72040007272648, + "z_geometric_mean_rank": 155.11297477488404, + "z_hits_at_k": 466.233760257905, + "z_inverse_harmonic_mean_rank": 536.8020765810452 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6717353285723936, + "adjusted_arithmetic_mean_rank_index": 0.32831132883914504, + "adjusted_geometric_mean_rank_index": 0.5146882360640462, + "adjusted_hits_at_k": 0.00848995966031373, + "adjusted_inverse_harmonic_mean_rank": 0.006057639803220773, + "arithmetic_mean_rank": 4726.758538017419, + "count": 20438.0, + "geometric_mean_rank": 2507.4076710492445, + "harmonic_mean_rank": 147.56851324594282, + "hits_at_1": 0.003571778060475585, + "hits_at_10": 0.009198551717389177, + "hits_at_3": 0.005431059790586163, + "hits_at_5": 0.00694784225462374, + "inverse_arithmetic_mean_rank": 0.00021156147325000397, + "inverse_geometric_mean_rank": 0.0003988182741666185, + "inverse_harmonic_mean_rank": 0.0067765133496558655, + "inverse_median_rank": 0.0002486634340420241, + "median_absolute_deviation": 4289.909519245959, + "median_rank": 4021.5, + "standard_deviation": 3749.7685571621982, + "variance": 14060764.232282275, + "z_arithmetic_mean_rank": 81.1047863772954, + "z_geometric_mean_rank": 73.71080657610705, + "z_hits_at_k": 45.38591523254731, + "z_inverse_harmonic_mean_rank": 79.9959453307532 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.671735815309457, + "adjusted_arithmetic_mean_rank_index": 0.32831084203290006, + "adjusted_geometric_mean_rank_index": 0.514687799316216, + "adjusted_hits_at_k": 0.00848995966031373, + "adjusted_inverse_harmonic_mean_rank": 0.006057639359175807, + "arithmetic_mean_rank": 4726.761963010079, + "count": 20438.0, + "geometric_mean_rank": 2507.4099266469725, + "harmonic_mean_rank": 147.5685229086799, + "hits_at_1": 0.003571778060475585, + "hits_at_10": 0.009198551717389177, + "hits_at_3": 0.005431059790586163, + "hits_at_5": 0.00694784225462374, + "inverse_arithmetic_mean_rank": 0.00021156131995341346, + "inverse_geometric_mean_rank": 0.00039881791540055336, + "inverse_harmonic_mean_rank": 0.006776512905932058, + "inverse_median_rank": 0.0002486634340420241, + "median_absolute_deviation": 4289.909519245959, + "median_rank": 4021.5, + "standard_deviation": 3749.7695546569394, + "variance": 14060771.7130321, + "z_arithmetic_mean_rank": 81.10466611852564, + "z_geometric_mean_rank": 73.71074402749493, + "z_hits_at_k": 45.38591523254731, + "z_inverse_harmonic_mean_rank": 79.99593946678672 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6717355030317774, + "adjusted_arithmetic_mean_rank_index": 0.3283111543549647, + "adjusted_geometric_mean_rank_index": 0.5146876603134156, + "adjusted_hits_at_k": 0.00848995966031373, + "adjusted_inverse_harmonic_mean_rank": 0.006057639189669265, + "arithmetic_mean_rank": 4726.759765625, + "count": 20438.0, + "geometric_mean_rank": 2507.41064453125, + "harmonic_mean_rank": 147.56852659726425, + "hits_at_1": 0.003571778060475585, + "hits_at_10": 0.009198551717389177, + "hits_at_3": 0.005431059790586163, + "hits_at_5": 0.00694784225462374, + "inverse_arithmetic_mean_rank": 0.00021156141883693635, + "inverse_geometric_mean_rank": 0.00039881779230199754, + "inverse_harmonic_mean_rank": 0.006776512736548112, + "inverse_median_rank": 0.0002486634475644678, + "median_absolute_deviation": 4289.909519245959, + "median_rank": 4021.5, + "standard_deviation": 3749.76904296875, + "variance": 14060768.0, + "z_arithmetic_mean_rank": 81.1047432733848, + "z_geometric_mean_rank": 73.71072412028157, + "z_hits_at_k": 45.38591523254731, + "z_inverse_harmonic_mean_rank": 79.99593722831818 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4579999658534782, + "adjusted_arithmetic_mean_rank_index": 0.5420749612327768, + "adjusted_geometric_mean_rank_index": 0.8871211856726299, + "adjusted_hits_at_k": 0.11383117308368258, + "adjusted_inverse_harmonic_mean_rank": 0.05095567511413758, + "arithmetic_mean_rank": 3313.4921714453467, + "count": 20438.0, + "geometric_mean_rank": 601.9387810872638, + "harmonic_mean_rank": 19.37160933780674, + "hits_at_1": 0.021430668362853508, + "hits_at_10": 0.11444368333496428, + "hits_at_3": 0.053332028574224484, + "hits_at_5": 0.07916625892944515, + "inverse_arithmetic_mean_rank": 0.0003017963973531284, + "inverse_geometric_mean_rank": 0.00166129850978156, + "inverse_harmonic_mean_rank": 0.0516219371639063, + "inverse_median_rank": 0.0006253908692933083, + "median_absolute_deviation": 2357.337527423907, + "median_rank": 1599.0, + "standard_deviation": 3825.433244315643, + "variance": 14633939.506715307, + "z_arithmetic_mean_rank": 134.21490718526735, + "z_geometric_mean_rank": 127.04298051729151, + "z_hits_at_k": 618.7736721675088, + "z_inverse_harmonic_mean_rank": 684.2055683452835 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.45800033781994665, + "adjusted_arithmetic_mean_rank_index": 0.5420745892148869, + "adjusted_geometric_mean_rank_index": 0.8871211018546956, + "adjusted_hits_at_k": 0.11383117308368258, + "adjusted_inverse_harmonic_mean_rank": 0.05095567469216939, + "arithmetic_mean_rank": 3313.4948625110087, + "count": 20438.0, + "geometric_mean_rank": 601.9392273131349, + "harmonic_mean_rank": 19.37160949604304, + "hits_at_1": 0.021430668362853508, + "hits_at_10": 0.11444368333496428, + "hits_at_3": 0.053332028574224484, + "hits_at_5": 0.07916625892944515, + "inverse_arithmetic_mean_rank": 0.00030179615224819973, + "inverse_geometric_mean_rank": 0.001661297278238007, + "inverse_harmonic_mean_rank": 0.05162193674223435, + "inverse_median_rank": 0.0006253908692933083, + "median_absolute_deviation": 2357.337527423907, + "median_rank": 1599.0, + "standard_deviation": 3825.4348786908963, + "variance": 14633952.011104833, + "z_arithmetic_mean_rank": 134.21481507559596, + "z_geometric_mean_rank": 127.04296851388051, + "z_hits_at_k": 618.7736721675088, + "z_inverse_harmonic_mean_rank": 684.2055626793203 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4580001368015266, + "adjusted_arithmetic_mean_rank_index": 0.5420747902610961, + "adjusted_geometric_mean_rank_index": 0.8871211511562782, + "adjusted_hits_at_k": 0.11383117308368258, + "adjusted_inverse_harmonic_mean_rank": 0.050955671222067116, + "arithmetic_mean_rank": 3313.493408203125, + "count": 20438.0, + "geometric_mean_rank": 601.93896484375, + "harmonic_mean_rank": 19.37161079731694, + "hits_at_1": 0.021430668362853508, + "hits_at_10": 0.11444368333496428, + "hits_at_3": 0.053332028574224484, + "hits_at_5": 0.07916625892944515, + "inverse_arithmetic_mean_rank": 0.00030179627356119454, + "inverse_geometric_mean_rank": 0.0016612979816272855, + "inverse_harmonic_mean_rank": 0.0516219332745682, + "inverse_median_rank": 0.0006253908504731953, + "median_absolute_deviation": 2357.337527423907, + "median_rank": 1599.0, + "standard_deviation": 3825.43408203125, + "variance": 14633945.0, + "z_arithmetic_mean_rank": 134.21486485357917, + "z_geometric_mean_rank": 127.04297557426877, + "z_hits_at_k": 618.7736721675088, + "z_inverse_harmonic_mean_rank": 684.2055160846418 + } + } + }, + "times": { + "evaluation": 9.888858556747437, + "training": 214.02531933784485 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json new file mode 100644 index 0000000..1769644 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json new file mode 100644 index 0000000..deec3f2 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.00017830922128783815, + 3.5568977629189845e-06, + 3.2398627037952276e-06, + 2.946085544536108e-06, + 2.427437591210355e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.13629356123736652, + "adjusted_arithmetic_mean_rank_index": 0.8637486485468131, + "adjusted_geometric_mean_rank_index": 0.9934277872821621, + "adjusted_hits_at_k": 0.3122319457641806, + "adjusted_inverse_harmonic_mean_rank": 0.21687724348833318, + "arithmetic_mean_rank": 2789.0069, + "count": 10000.0, + "geometric_mean_rank": 99.96215669911965, + "harmonic_mean_rank": 4.606352677882064, + "hits_at_1": 0.1676, + "hits_at_10": 0.3124, + "hits_at_3": 0.234, + "hits_at_5": 0.2644, + "inverse_arithmetic_mean_rank": 0.00035855056507748336, + "inverse_geometric_mean_rank": 0.010003785762745621, + "inverse_harmonic_mean_rank": 0.21709149731448396, + "inverse_median_rank": 0.009433962264150943, + "median_absolute_deviation": 155.6732329430882, + "median_rank": 106.0, + "standard_deviation": 6483.423392163463, + "variance": 42034778.88205239, + "z_arithmetic_mean_rank": 149.6018115743092, + "z_geometric_mean_rank": 99.42312958133098, + "z_hits_at_k": 1997.195311935326, + "z_inverse_harmonic_mean_rank": 3423.143978774425 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1362936785208867, + "adjusted_arithmetic_mean_rank_index": 0.8637485312575612, + "adjusted_geometric_mean_rank_index": 0.9934277849391755, + "adjusted_hits_at_k": 0.3122319457641806, + "adjusted_inverse_harmonic_mean_rank": 0.21687724338746267, + "arithmetic_mean_rank": 2789.0093, + "count": 10000.0, + "geometric_mean_rank": 99.96219197901941, + "harmonic_mean_rank": 4.606352680021797, + "hits_at_1": 0.1676, + "hits_at_10": 0.3124, + "hits_at_3": 0.234, + "hits_at_5": 0.2644, + "inverse_arithmetic_mean_rank": 0.0003585502565373303, + "inverse_geometric_mean_rank": 0.01000378223208516, + "inverse_harmonic_mean_rank": 0.21709149721364104, + "inverse_median_rank": 0.009433962264150943, + "median_absolute_deviation": 155.6732329430882, + "median_rank": 106.0, + "standard_deviation": 6483.426716005472, + "variance": 42034821.981813505, + "z_arithmetic_mean_rank": 149.60179125973667, + "z_geometric_mean_rank": 99.42312934684281, + "z_hits_at_k": 1997.195311935326, + "z_inverse_harmonic_mean_rank": 3423.143977182306 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1362936177602349, + "adjusted_arithmetic_mean_rank_index": 0.8637485920211825, + "adjusted_geometric_mean_rank_index": 0.993427787182279, + "adjusted_hits_at_k": 0.3122319457641806, + "adjusted_inverse_harmonic_mean_rank": 0.21687725008190317, + "arithmetic_mean_rank": 2789.008056640625, + "count": 10000.0, + "geometric_mean_rank": 99.962158203125, + "harmonic_mean_rank": 4.606352538014779, + "hits_at_1": 0.1676, + "hits_at_10": 0.3124, + "hits_at_3": 0.234, + "hits_at_5": 0.2644, + "inverse_arithmetic_mean_rank": 0.0003585504018701613, + "inverse_geometric_mean_rank": 0.010003785602748394, + "inverse_harmonic_mean_rank": 0.21709150390625, + "inverse_median_rank": 0.009433962404727936, + "median_absolute_deviation": 155.6732329430882, + "median_rank": 106.0, + "standard_deviation": 6483.42529296875, + "variance": 42034804.0, + "z_arithmetic_mean_rank": 149.60180178403425, + "z_geometric_mean_rank": 99.4231295713346, + "z_hits_at_k": 1997.195311935326, + "z_inverse_harmonic_mean_rank": 3423.1440828459217 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1465853386942658, + "adjusted_arithmetic_mean_rank_index": 0.8534563690923419, + "adjusted_geometric_mean_rank_index": 0.9921161069642516, + "adjusted_hits_at_k": 0.2958279334706153, + "adjusted_inverse_harmonic_mean_rank": 0.20868076827049847, + "arithmetic_mean_rank": 2999.5404, + "count": 5000.0, + "geometric_mean_rank": 119.71615876559561, + "harmonic_mean_rank": 4.7870419943520535, + "hits_at_1": 0.1632, + "hits_at_10": 0.296, + "hits_at_3": 0.2246, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.00033338440782461207, + "inverse_geometric_mean_rank": 0.008353091264463316, + "inverse_harmonic_mean_rank": 0.2088972691653511, + "inverse_median_rank": 0.006779661016949152, + "median_absolute_deviation": 217.20122501107068, + "median_rank": 147.5, + "standard_deviation": 6644.4210276869, + "variance": 44148330.79316784, + "z_arithmetic_mean_rank": 104.52393735493405, + "z_geometric_mean_rank": 70.21514025909521, + "z_hits_at_k": 1338.019097424571, + "z_inverse_harmonic_mean_rank": 2329.021753055102 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1465854853020648, + "adjusted_arithmetic_mean_rank_index": 0.8534562224773778, + "adjusted_geometric_mean_rank_index": 0.9921161046258482, + "adjusted_hits_at_k": 0.2958279334706153, + "adjusted_inverse_harmonic_mean_rank": 0.2086807682341606, + "arithmetic_mean_rank": 2999.5434, + "count": 5000.0, + "geometric_mean_rank": 119.71619397742072, + "harmonic_mean_rank": 4.787041995184536, + "hits_at_1": 0.1632, + "hits_at_10": 0.296, + "hits_at_3": 0.2246, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.0003333840743894554, + "inverse_geometric_mean_rank": 0.008353088807589447, + "inverse_harmonic_mean_rank": 0.20889726912902315, + "inverse_median_rank": 0.006779661016949152, + "median_absolute_deviation": 217.20122501107068, + "median_rank": 147.5, + "standard_deviation": 6644.425975185249, + "variance": 44148396.539716445, + "z_arithmetic_mean_rank": 104.52391939880428, + "z_geometric_mean_rank": 70.21514009359913, + "z_hits_at_k": 1338.019097424571, + "z_inverse_harmonic_mean_rank": 2329.0217526495467 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.14658541650330079, + "adjusted_arithmetic_mean_rank_index": 0.8534562912795043, + "adjusted_geometric_mean_rank_index": 0.9921161051208643, + "adjusted_hits_at_k": 0.2958279334706153, + "adjusted_inverse_harmonic_mean_rank": 0.20868076472917851, + "arithmetic_mean_rank": 2999.5419921875, + "count": 5000.0, + "geometric_mean_rank": 119.7161865234375, + "harmonic_mean_rank": 4.78704207548193, + "hits_at_1": 0.1632, + "hits_at_10": 0.296, + "hits_at_3": 0.2246, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.0003333842323627323, + "inverse_geometric_mean_rank": 0.008353088982403278, + "inverse_harmonic_mean_rank": 0.20889726562500002, + "inverse_median_rank": 0.006779660936444998, + "median_absolute_deviation": 217.20122501107068, + "median_rank": 147.5, + "standard_deviation": 6644.42333984375, + "variance": 44148360.0, + "z_arithmetic_mean_rank": 104.52392782509227, + "z_geometric_mean_rank": 70.21514012863295, + "z_hits_at_k": 1338.019097424571, + "z_inverse_harmonic_mean_rank": 2329.0217135315215 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1260022609588329, + "adjusted_arithmetic_mean_rank_index": 0.8740404507763229, + "adjusted_geometric_mean_rank_index": 0.9945236073848335, + "adjusted_hits_at_k": 0.3286359578704887, + "adjusted_inverse_harmonic_mean_rank": 0.22507371861079298, + "arithmetic_mean_rank": 2578.4734, + "count": 5000.0, + "geometric_mean_rank": 83.46770289802355, + "harmonic_mean_rank": 4.438807642792696, + "hits_at_1": 0.172, + "hits_at_10": 0.3288, + "hits_at_3": 0.2434, + "hits_at_5": 0.2766, + "inverse_arithmetic_mean_rank": 0.00038782637819726975, + "inverse_geometric_mean_rank": 0.01198068193180957, + "inverse_harmonic_mean_rank": 0.22528572546361694, + "inverse_median_rank": 0.012578616352201259, + "median_absolute_deviation": 116.38427415268976, + "median_rank": 79.5, + "standard_deviation": 6311.305590897373, + "variance": 39832578.26169244, + "z_arithmetic_mean_rank": 107.04491535204133, + "z_geometric_mean_rank": 70.38552459472623, + "z_hits_at_k": 1486.443332273157, + "z_inverse_harmonic_mean_rank": 2512.037021673384 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12600234891943382, + "adjusted_arithmetic_mean_rank_index": 0.8740403628114234, + "adjusted_geometric_mean_rank_index": 0.994523605102655, + "adjusted_hits_at_k": 0.3286359578704887, + "adjusted_inverse_harmonic_mean_rank": 0.22507371844538973, + "arithmetic_mean_rank": 2578.4752, + "count": 5000.0, + "geometric_mean_rank": 83.4677372648095, + "harmonic_mean_rank": 4.438807646050747, + "hits_at_1": 0.172, + "hits_at_10": 0.3288, + "hits_at_3": 0.2434, + "hits_at_5": 0.2766, + "inverse_arithmetic_mean_rank": 0.00038782610746071944, + "inverse_geometric_mean_rank": 0.011980676998915196, + "inverse_harmonic_mean_rank": 0.22528572529825894, + "inverse_median_rank": 0.012578616352201259, + "median_absolute_deviation": 116.38427415268976, + "median_rank": 79.5, + "standard_deviation": 6311.3071712114415, + "variance": 39832598.20938496, + "z_arithmetic_mean_rank": 107.04490457886119, + "z_geometric_mean_rank": 70.38552443320938, + "z_hits_at_k": 1486.443332273157, + "z_inverse_harmonic_mean_rank": 2512.037019827326 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1260023081269416, + "adjusted_arithmetic_mean_rank_index": 0.8740404036059091, + "adjusted_geometric_mean_rank_index": 0.9945236067536847, + "adjusted_hits_at_k": 0.3286359578704887, + "adjusted_inverse_harmonic_mean_rank": 0.22507373533925265, + "arithmetic_mean_rank": 2578.474365234375, + "count": 5000.0, + "geometric_mean_rank": 83.46771240234375, + "harmonic_mean_rank": 4.43880731328183, + "hits_at_1": 0.172, + "hits_at_10": 0.3288, + "hits_at_3": 0.2434, + "hits_at_5": 0.2766, + "inverse_arithmetic_mean_rank": 0.0003878262359648943, + "inverse_geometric_mean_rank": 0.011980680748820305, + "inverse_harmonic_mean_rank": 0.2252857421875, + "inverse_median_rank": 0.012578615918755531, + "median_absolute_deviation": 116.38427415268976, + "median_rank": 79.5, + "standard_deviation": 6311.30615234375, + "variance": 39832588.0, + "z_arithmetic_mean_rank": 107.044909575017, + "z_geometric_mean_rank": 70.38552455005788, + "z_hits_at_k": 1486.443332273157, + "z_inverse_harmonic_mean_rank": 2512.0372083789234 + } + } + }, + "times": { + "evaluation": 6.108567476272583, + "training": 554.8396935462952 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json new file mode 100644 index 0000000..d66b7f2 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.10.0+cu113" + }, + "title": "Learn WN18RR Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json new file mode 100644 index 0000000..ac24ad2 --- /dev/null +++ b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json @@ -0,0 +1,268 @@ +{ + "evaluator_metadata": { + "batch_size": 1024, + "slice_size": null + }, + "losses": [ + 0.0001694744666249087, + 3.830565566938213e-06, + 3.4759809618280278e-06, + 3.3146416847170494e-06, + 3.0465402742851043e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5378326165937043, + "adjusted_arithmetic_mean_rank_index": 0.4621901822268002, + "adjusted_geometric_mean_rank_index": 0.8390557160671878, + "adjusted_hits_at_k": 0.05773613104335256, + "adjusted_inverse_harmonic_mean_rank": 0.030193480446228313, + "arithmetic_mean_rank": 10903.237517099864, + "count": 5848.0, + "geometric_mean_rank": 2401.9485509616025, + "harmonic_mean_rank": 32.828776225964894, + "hits_at_1": 0.01573187414500684, + "hits_at_10": 0.05796853625170999, + "hits_at_3": 0.03317373461012312, + "hits_at_5": 0.04155266757865937, + "inverse_arithmetic_mean_rank": 9.17158778235979e-05, + "inverse_geometric_mean_rank": 0.0004163286510028108, + "inverse_harmonic_mean_rank": 0.03046108064208258, + "inverse_median_rank": 0.00018400956849756188, + "median_absolute_deviation": 7957.1261067195655, + "median_rank": 5434.5, + "standard_deviation": 12133.028938908961, + "variance": 147210391.23240232, + "z_arithmetic_mean_rank": 61.21725449573289, + "z_geometric_mean_rank": 64.22009388267874, + "z_hits_at_k": 281.10018794574233, + "z_inverse_harmonic_mean_rank": 362.7415076350221 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5378330383429395, + "adjusted_arithmetic_mean_rank_index": 0.4621897604567601, + "adjusted_geometric_mean_rank_index": 0.839055594586586, + "adjusted_hits_at_k": 0.05773613104335256, + "adjusted_inverse_harmonic_mean_rank": 0.03019348032372203, + "arithmetic_mean_rank": 10903.246067031463, + "count": 5848.0, + "geometric_mean_rank": 2401.950363195425, + "harmonic_mean_rank": 32.82877635795698, + "hits_at_1": 0.01573187414500684, + "hits_at_10": 0.05796853625170999, + "hits_at_3": 0.03317373461012312, + "hits_at_5": 0.04155266757865937, + "inverse_arithmetic_mean_rank": 9.171580590332048e-05, + "inverse_geometric_mean_rank": 0.0004163283368893827, + "inverse_harmonic_mean_rank": 0.0304610805196101, + "inverse_median_rank": 0.00018400956849756188, + "median_absolute_deviation": 7957.1261067195655, + "median_rank": 5434.5, + "standard_deviation": 12133.034735989933, + "variance": 147210531.9047383, + "z_arithmetic_mean_rank": 61.21719863214062, + "z_geometric_mean_rank": 64.22008458473165, + "z_hits_at_k": 281.10018794574233, + "z_inverse_harmonic_mean_rank": 362.7415061632436 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5378328469742241, + "adjusted_arithmetic_mean_rank_index": 0.46218995183491574, + "adjusted_geometric_mean_rank_index": 0.8390555403778629, + "adjusted_hits_at_k": 0.05773613104335256, + "adjusted_inverse_harmonic_mean_rank": 0.030193482837332662, + "arithmetic_mean_rank": 10903.2421875, + "count": 5848.0, + "geometric_mean_rank": 2401.951171875, + "harmonic_mean_rank": 32.82877364971474, + "hits_at_1": 0.01573187414500684, + "hits_at_10": 0.05796853625170999, + "hits_at_3": 0.03317373461012312, + "hits_at_5": 0.04155266757865937, + "inverse_arithmetic_mean_rank": 9.171583951683715e-05, + "inverse_geometric_mean_rank": 0.00041632819920778275, + "inverse_harmonic_mean_rank": 0.030461083032527148, + "inverse_median_rank": 0.0001840095646912232, + "median_absolute_deviation": 7957.1261067195655, + "median_rank": 5434.5, + "standard_deviation": 12133.0322265625, + "variance": 147210464.0, + "z_arithmetic_mean_rank": 61.21722398024562, + "z_geometric_mean_rank": 64.2200804356755, + "z_hits_at_k": 281.10018794574233, + "z_inverse_harmonic_mean_rank": 362.7415363615148 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6233709073515975, + "adjusted_arithmetic_mean_rank_index": 0.37664767596883575, + "adjusted_geometric_mean_rank_index": 0.7944891931289548, + "adjusted_hits_at_k": 0.047644670345667155, + "adjusted_inverse_harmonic_mean_rank": 0.027085656497426752, + "arithmetic_mean_rank": 12634.512995896033, + "count": 2924.0, + "geometric_mean_rank": 3066.366583099857, + "harmonic_mean_rank": 36.5574991137905, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.047879616963064295, + "hits_at_3": 0.029069767441860465, + "hits_at_5": 0.03488372093023256, + "inverse_arithmetic_mean_rank": 7.914828219535031e-05, + "inverse_geometric_mean_rank": 0.000326118868341266, + "inverse_harmonic_mean_rank": 0.02735416875446965, + "inverse_median_rank": 0.00013421917992081068, + "median_absolute_deviation": 10865.250358318304, + "median_rank": 7450.5, + "standard_deviation": 13004.701758352729, + "variance": 169122267.82370254, + "z_arithmetic_mean_rank": 35.27549982003005, + "z_geometric_mean_rank": 43.003933484274725, + "z_hits_at_k": 164.00773982763116, + "z_inverse_harmonic_mean_rank": 230.07003849884842 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6233713798155583, + "adjusted_arithmetic_mean_rank_index": 0.3766472034815629, + "adjusted_geometric_mean_rank_index": 0.7944890418337018, + "adjusted_hits_at_k": 0.047644670345667155, + "adjusted_inverse_harmonic_mean_rank": 0.027085656396564593, + "arithmetic_mean_rank": 12634.522571819425, + "count": 2924.0, + "geometric_mean_rank": 3066.368839795842, + "harmonic_mean_rank": 36.557499248550606, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.047879616963064295, + "hits_at_3": 0.029069767441860465, + "hits_at_5": 0.03488372093023256, + "inverse_arithmetic_mean_rank": 7.914822220749697e-05, + "inverse_geometric_mean_rank": 0.00032611862833388945, + "inverse_harmonic_mean_rank": 0.027354168653635328, + "inverse_median_rank": 0.00013421917992081068, + "median_absolute_deviation": 10865.250358318304, + "median_rank": 7450.5, + "standard_deviation": 13004.707744755677, + "variance": 169122423.5265083, + "z_arithmetic_mean_rank": 35.27545556852986, + "z_geometric_mean_rank": 43.00392529499908, + "z_hits_at_k": 164.00773982763116, + "z_inverse_harmonic_mean_rank": 230.07003764210856 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6233711816153448, + "adjusted_arithmetic_mean_rank_index": 0.37664740169155586, + "adjusted_geometric_mean_rank_index": 0.7944889725616432, + "adjusted_hits_at_k": 0.047644670345667155, + "adjusted_inverse_harmonic_mean_rank": 0.02708565640653095, + "arithmetic_mean_rank": 12634.5185546875, + "count": 2924.0, + "geometric_mean_rank": 3066.369873046875, + "harmonic_mean_rank": 36.55749923523474, + "hits_at_1": 0.016073871409028728, + "hits_at_10": 0.047879616963064295, + "hits_at_3": 0.029069767441860465, + "hits_at_5": 0.03488372093023256, + "inverse_arithmetic_mean_rank": 7.914824527688324e-05, + "inverse_geometric_mean_rank": 0.00032611851929686964, + "inverse_harmonic_mean_rank": 0.027354168663598934, + "inverse_median_rank": 0.00013421918265521526, + "median_absolute_deviation": 10865.250358318304, + "median_rank": 7450.5, + "standard_deviation": 13004.7041015625, + "variance": 169122336.0, + "z_arithmetic_mean_rank": 35.275474132182346, + "z_geometric_mean_rank": 43.00392154545658, + "z_hits_at_k": 164.00773982763116, + "z_inverse_harmonic_mean_rank": 230.07003772676444 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.45233227122762737, + "adjusted_arithmetic_mean_rank_index": 0.547694739351391, + "adjusted_geometric_mean_rank_index": 0.8739823201514549, + "adjusted_hits_at_k": 0.06782759063037647, + "adjusted_inverse_harmonic_mean_rank": 0.03330130404665707, + "arithmetic_mean_rank": 9171.962038303693, + "count": 2924.0, + "geometric_mean_rank": 1881.4961241960061, + "harmonic_mean_rank": 29.790283083367658, + "hits_at_1": 0.015389876880984952, + "hits_at_10": 0.06805745554035568, + "hits_at_3": 0.037277701778385776, + "hits_at_5": 0.048221614227086186, + "inverse_arithmetic_mean_rank": 0.00010902792617586377, + "inverse_geometric_mean_rank": 0.0005314919266322253, + "inverse_harmonic_mean_rank": 0.03356799252969551, + "inverse_median_rank": 0.00024372410431391665, + "median_absolute_deviation": 6016.399802695732, + "median_rank": 4103.0, + "standard_deviation": 10922.631783326779, + "variance": 119303885.07414031, + "z_arithmetic_mean_rank": 51.29522928879513, + "z_geometric_mean_rank": 47.30670708848514, + "z_hits_at_k": 233.53571327456675, + "z_inverse_harmonic_mean_rank": 282.929810568764 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4523326422846345, + "adjusted_arithmetic_mean_rank_index": 0.5476943682760836, + "adjusted_geometric_mean_rank_index": 0.8739822226856442, + "adjusted_hits_at_k": 0.06782759063037647, + "adjusted_inverse_harmonic_mean_rank": 0.03330130390250666, + "arithmetic_mean_rank": 9171.969562243503, + "count": 2924.0, + "geometric_mean_rank": 1881.497578627477, + "harmonic_mean_rank": 29.790283211260224, + "hits_at_1": 0.015389876880984952, + "hits_at_10": 0.06805745554035568, + "hits_at_3": 0.037277701778385776, + "hits_at_5": 0.048221614227086186, + "inverse_arithmetic_mean_rank": 0.00010902783673819735, + "inverse_geometric_mean_rank": 0.0005314915157794061, + "inverse_harmonic_mean_rank": 0.03356799238558487, + "inverse_median_rank": 0.00024369440721335446, + "median_absolute_deviation": 6017.1411038049855, + "median_rank": 4103.5, + "standard_deviation": 10922.637209525301, + "variance": 119304003.61090665, + "z_arithmetic_mean_rank": 51.29519453514204, + "z_geometric_mean_rank": 47.30670181287892, + "z_hits_at_k": 233.53571327456675, + "z_inverse_harmonic_mean_rank": 282.92980934405375 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.45233245774438646, + "adjusted_arithmetic_mean_rank_index": 0.5476945528254331, + "adjusted_geometric_mean_rank_index": 0.8739823140114357, + "adjusted_hits_at_k": 0.06782759063037647, + "adjusted_inverse_harmonic_mean_rank": 0.033301303699857135, + "arithmetic_mean_rank": 9171.9658203125, + "count": 2924.0, + "geometric_mean_rank": 1881.4962158203125, + "harmonic_mean_rank": 29.79028339105416, + "hits_at_1": 0.015389876880984952, + "hits_at_10": 0.06805745554035568, + "hits_at_3": 0.037277701778385776, + "hits_at_5": 0.048221614227086186, + "inverse_arithmetic_mean_rank": 0.00010902788199018687, + "inverse_geometric_mean_rank": 0.0005314918817020953, + "inverse_harmonic_mean_rank": 0.033567992182991245, + "inverse_median_rank": 0.0002437092480249703, + "median_absolute_deviation": 6016.770453250359, + "median_rank": 4103.25, + "standard_deviation": 10922.634765625, + "variance": 119303952.0, + "z_arithmetic_mean_rank": 51.29521181940736, + "z_geometric_mean_rank": 47.30670675613963, + "z_hits_at_k": 233.53571327456675, + "z_inverse_harmonic_mean_rank": 282.9298076223316 + } + } + }, + "times": { + "evaluation": 4.860883474349976, + "training": 332.31198930740356 + } +} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/summary.tsv b/nonasv/results/27c9fe15/summary.tsv new file mode 100644 index 0000000..bab9195 --- /dev/null +++ b/nonasv/results/27c9fe15/summary.tsv @@ -0,0 +1,39 @@ +batch_size.evaluation evaluation path training +1024 31.021438598632812 complex/trouillon2016_complex_fb15k.json 1.827806282043457 +256 4.746052265167236 complex/trouillon2016_complex_wn18.json 0.36449394226074217 +2048 15.175896167755127 conve/dettmers2018_conve_fb15k.json 32.02347240447998 +4096 6.952138900756836 conve/dettmers2018_conve_fb15k237.json 17.444180917739867 +8192 3.979680061340332 conve/dettmers2018_conve_wn18.json 32.08411064147949 +4096 2.390557289123535 conve/dettmers2018_conve_wn18rr.json 19.340176582336426 +32 113.86807107925415 convkb/nguyen2018_convkb_fb15k237.json 5.623247241973877 +4 229.65697264671326 convkb/nguyen2018_convkb_wn18rr.json 2.274687576293945 +4096 9.86320161819458 convkb/nguyen2018_transe_fb15k237.json 2.304577445983887 +2048 19.562544584274292 distmult/yang2014_distmult_fb15k.json 3.5463189125061034 +1024 4.716632843017578 distmult/yang2014_distmult_wn18.json 1.373633337020874 +512 34.93160533905029 hole/nickel2016_hole_fb15k.json 0.8114243030548096 +256 7.370163202285767 hole/nickel2016_hole_wn18.json 0.4553260326385498 +2048 28.49927306175232 kg2e/he2015_kg2e_fb15k.json 2.6026907920837403 +1024 7.71472430229187 kg2e/he2015_kg2e_wn18.json 38.04337100982666 +1024 15.51103925704956 mure/balazevic2019_mure_fb15k237.json 12.894137287139893 +256 144.05696630477905 quate/zhang2019_quate_fb15k.json 1.5077387809753418 +512 31.55549693107605 quate/zhang2019_quate_fb15k237.json 0.6327661991119384 +64 49.746681928634644 quate/zhang2019_quate_wn18.json 1.7024828910827636 +128 13.313127517700195 quate/zhang2019_quate_wn18rr.json 0.5839082717895507 +256 86.14422821998596 rotate/sun2019_rotate_fb15k.json 82.6045967578888 +256 29.337369203567505 rotate/sun2019_rotate_fb15k237.json 91.85465803146363 +256 11.969523429870605 rotate/sun2019_rotate_wn18.json 98.09528822898865 +256 7.80864691734314 rotate/sun2019_rotate_wn18rr.json 60.45014009475708 +1024 29.497436046600342 simple/kazemi2018_simple_fb15k.json 2.608071517944336 +1024 9.06259036064148 simple/kazemi2018_simple_wn18.json 0.9677006721496582 +2048 34.54206871986389 transd/ji2015_transd_fb15k.json 16.6326443195343 +1024 10.18949007987976 transd/ji2015_transd_wn18.json 4.865910863876342 +2048 17.733282566070557 transe/bordes2013_transe_fb15k.json 22.46914396286011 +8192 5.315532684326172 transe/bordes2013_transe_wn18.json 6.628315019607544 +2048 29.929590225219727 transh/wang2014_transh_fb15k.json 2.0839001655578615 +2048 8.578476905822754 transh/wang2014_transh_wn18.json 0.5796948909759522 +2048 33.42562985420227 transr/li2015_transr_fb15k.json 2.437476301193237 +1024 9.90921139717102 transr/li2015_transr_wn18.json 0.9390138626098633 +1024 22.204118728637695 tucker/balazevic2019_tucker_fb15k.json 79.91824607849121 +2048 9.888858556747437 tucker/balazevic2019_tucker_fb15k237.json 42.80506386756897 +1024 6.108567476272583 tucker/balazevic2019_tucker_wn18.json 110.96793870925903 +1024 4.860883474349976 tucker/balazevic2019_tucker_wn18rr.json 66.46239786148071 From 3a9cc416d360a84caa2f493d3003f7d3bc074368 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 11:49:15 +0200 Subject: [PATCH 2/7] remove results --- .../complex/fb15k/trouillon2016/metadata.json | 20 -- .../complex/fb15k/trouillon2016/results.json | 268 ------------------ .../complex/wn18/trouillon2016/metadata.json | 20 -- .../complex/wn18/trouillon2016/results.json | 268 ------------------ .../conve/fb15k/dettmers2018/metadata.json | 20 -- .../conve/fb15k/dettmers2018/results.json | 268 ------------------ .../conve/fb15k237/dettmers2018/metadata.json | 20 -- .../conve/fb15k237/dettmers2018/results.json | 268 ------------------ .../conve/wn18/dettmers2018/metadata.json | 20 -- .../cuda/conve/wn18/dettmers2018/results.json | 268 ------------------ .../conve/wn18rr/dettmers2018/metadata.json | 20 -- .../conve/wn18rr/dettmers2018/results.json | 268 ------------------ .../convkb/fb15k237/nguyen2018/metadata.json | 20 -- .../convkb/fb15k237/nguyen2018/results.json | 268 ------------------ .../convkb/wn18rr/nguyen2018/metadata.json | 20 -- .../convkb/wn18rr/nguyen2018/results.json | 268 ------------------ .../distmult/fb15k/yang2014/metadata.json | 20 -- .../cuda/distmult/fb15k/yang2014/results.json | 268 ------------------ .../cuda/distmult/wn18/yang2014/metadata.json | 20 -- .../cuda/distmult/wn18/yang2014/results.json | 268 ------------------ .../cuda/hole/fb15k/nickel2016/metadata.json | 20 -- .../cuda/hole/fb15k/nickel2016/results.json | 268 ------------------ .../cuda/hole/wn18/nickel2016/metadata.json | 20 -- .../cuda/hole/wn18/nickel2016/results.json | 268 ------------------ .../cuda/kg2e/fb15k/he2015/metadata.json | 19 -- .../cuda/kg2e/fb15k/he2015/results.json | 268 ------------------ .../cuda/kg2e/wn18/he2015/metadata.json | 19 -- .../cuda/kg2e/wn18/he2015/results.json | 268 ------------------ .../mure/fb15k237/balazevic2019/metadata.json | 20 -- .../mure/fb15k237/balazevic2019/results.json | 268 ------------------ .../cuda/quate/fb15k/zhang2019/metadata.json | 19 -- .../cuda/quate/fb15k/zhang2019/results.json | 268 ------------------ .../quate/fb15k237/zhang2019/metadata.json | 19 -- .../quate/fb15k237/zhang2019/results.json | 268 ------------------ .../cuda/quate/wn18/zhang2019/metadata.json | 19 -- .../cuda/quate/wn18/zhang2019/results.json | 268 ------------------ .../cuda/quate/wn18rr/zhang2019/metadata.json | 19 -- .../cuda/quate/wn18rr/zhang2019/results.json | 268 ------------------ .../cuda/rotate/fb15k/sun2019/metadata.json | 20 -- .../cuda/rotate/fb15k/sun2019/results.json | 268 ------------------ .../rotate/fb15k237/sun2019/metadata.json | 20 -- .../cuda/rotate/fb15k237/sun2019/results.json | 268 ------------------ .../cuda/rotate/wn18/sun2019/metadata.json | 20 -- .../cuda/rotate/wn18/sun2019/results.json | 268 ------------------ .../cuda/rotate/wn18rr/sun2019/metadata.json | 20 -- .../cuda/rotate/wn18rr/sun2019/results.json | 268 ------------------ .../simple/fb15k/kazemi2018/metadata.json | 20 -- .../cuda/simple/fb15k/kazemi2018/results.json | 268 ------------------ .../cuda/simple/wn18/kazemi2018/metadata.json | 20 -- .../cuda/simple/wn18/kazemi2018/results.json | 268 ------------------ .../cuda/transd/fb15k/ji2015/metadata.json | 20 -- .../cuda/transd/fb15k/ji2015/results.json | 268 ------------------ .../cuda/transd/wn18/ji2015/metadata.json | 20 -- .../cuda/transd/wn18/ji2015/results.json | 268 ------------------ .../transe/fb15k/bordes2013/metadata.json | 20 -- .../cuda/transe/fb15k/bordes2013/results.json | 268 ------------------ .../transe/fb15k237/nguyen2018/metadata.json | 20 -- .../transe/fb15k237/nguyen2018/results.json | 268 ------------------ .../cuda/transe/wn18/bordes2013/metadata.json | 20 -- .../cuda/transe/wn18/bordes2013/results.json | 268 ------------------ .../cuda/transh/fb15k/wang2014/metadata.json | 19 -- .../cuda/transh/fb15k/wang2014/results.json | 268 ------------------ .../cuda/transh/wn18/wang2014/metadata.json | 19 -- .../cuda/transh/wn18/wang2014/results.json | 268 ------------------ .../cuda/transr/fb15k/li2015/metadata.json | 19 -- .../cuda/transr/fb15k/li2015/results.json | 268 ------------------ .../cuda/transr/wn18/li2015/metadata.json | 19 -- .../cuda/transr/wn18/li2015/results.json | 268 ------------------ .../tucker/fb15k/balazevic2019/metadata.json | 20 -- .../tucker/fb15k/balazevic2019/results.json | 268 ------------------ .../fb15k237/balazevic2019/metadata.json | 20 -- .../fb15k237/balazevic2019/results.json | 268 ------------------ .../tucker/wn18/balazevic2019/metadata.json | 20 -- .../tucker/wn18/balazevic2019/results.json | 268 ------------------ .../tucker/wn18rr/balazevic2019/metadata.json | 20 -- .../tucker/wn18rr/balazevic2019/results.json | 268 ------------------ nonasv/results/27c9fe15/summary.tsv | 39 --- 77 files changed, 10973 deletions(-) delete mode 100644 nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json delete mode 100644 nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json delete mode 100644 nonasv/results/27c9fe15/summary.tsv diff --git a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json deleted file mode 100644 index 1307ab5..0000000 --- a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with ComplEx as described by Trouillon et al., 2016" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json b/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json deleted file mode 100644 index fb86d08..0000000 --- a/nonasv/results/27c9fe15/cuda/complex/fb15k/trouillon2016/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0003799774137384096, - 0.0001661697653289569, - 0.00013809030711660472, - 0.00012140296024768122, - 0.00010931310573704068 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.10869346503978508, - "adjusted_arithmetic_mean_rank_index": 0.8914272125051246, - "adjusted_geometric_mean_rank_index": 0.9861409524067312, - "adjusted_hits_at_k": 0.25224795638091657, - "adjusted_inverse_harmonic_mean_rank": 0.11382815690449287, - "arithmetic_mean_rank": 802.9025833319226, - "count": 118142.0, - "geometric_mean_rank": 76.26307316764641, - "harmonic_mean_rank": 8.738223046910447, - "hits_at_1": 0.0471635828071304, - "hits_at_10": 0.2527551590458939, - "hits_at_3": 0.12581469756733424, - "hits_at_5": 0.17230112914966736, - "inverse_arithmetic_mean_rank": 0.0012454811091155709, - "inverse_geometric_mean_rank": 0.013112505941135305, - "inverse_harmonic_mean_rank": 0.11443974302688092, - "inverse_median_rank": 0.012987012987012988, - "median_absolute_deviation": 111.19516638792014, - "median_rank": 77.0, - "standard_deviation": 1986.0993868995192, - "variance": 3944590.774642646, - "z_arithmetic_mean_rank": 530.2363209420166, - "z_geometric_mean_rank": 339.5423969620626, - "z_hits_at_k": 3327.9048141920634, - "z_inverse_harmonic_mean_rank": 3709.417282720864 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.1086934822278711, - "adjusted_arithmetic_mean_rank_index": 0.8914271953147114, - "adjusted_geometric_mean_rank_index": 0.9861409491774481, - "adjusted_hits_at_k": 0.25224795638091657, - "adjusted_inverse_harmonic_mean_rank": 0.11382815277237188, - "arithmetic_mean_rank": 802.9027102977773, - "count": 118142.0, - "geometric_mean_rank": 76.26309070462126, - "harmonic_mean_rank": 8.738223362207178, - "hits_at_1": 0.0471635828071304, - "hits_at_10": 0.2527551590458939, - "hits_at_3": 0.12581469756733424, - "hits_at_5": 0.17230112914966736, - "inverse_arithmetic_mean_rank": 0.0012454809121632234, - "inverse_geometric_mean_rank": 0.013112502925866912, - "inverse_harmonic_mean_rank": 0.11443973889761169, - "inverse_median_rank": 0.012987012987012988, - "median_absolute_deviation": 111.19516638792014, - "median_rank": 77.0, - "standard_deviation": 1986.0996845871812, - "variance": 3944591.9571173005, - "z_arithmetic_mean_rank": 530.2363107168617, - "z_geometric_mean_rank": 339.5423958501744, - "z_hits_at_k": 3327.9048141920634, - "z_inverse_harmonic_mean_rank": 3709.417148063839 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.10869349044494561, - "adjusted_arithmetic_mean_rank_index": 0.8914271870965242, - "adjusted_geometric_mean_rank_index": 0.9861409447167004, - "adjusted_hits_at_k": 0.25224795638091657, - "adjusted_inverse_harmonic_mean_rank": 0.11382816117618733, - "arithmetic_mean_rank": 802.9027709960938, - "count": 118142.0, - "geometric_mean_rank": 76.26311492919922, - "harmonic_mean_rank": 8.738222720963748, - "hits_at_1": 0.0471635828071304, - "hits_at_10": 0.2527551590458939, - "hits_at_3": 0.12581469756733424, - "hits_at_5": 0.17230112914966736, - "inverse_arithmetic_mean_rank": 0.001245480845682323, - "inverse_geometric_mean_rank": 0.013112498447299004, - "inverse_harmonic_mean_rank": 0.1144397472956273, - "inverse_median_rank": 0.012987012974917889, - "median_absolute_deviation": 111.19516638792014, - "median_rank": 77.0, - "standard_deviation": 1986.099609375, - "variance": 3944591.5, - "z_arithmetic_mean_rank": 530.2363058285418, - "z_geometric_mean_rank": 339.5423943142753, - "z_hits_at_k": 3327.9048141920634, - "z_inverse_harmonic_mean_rank": 3709.4174219262914 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.12324713991639374, - "adjusted_arithmetic_mean_rank_index": 0.8768718911367203, - "adjusted_geometric_mean_rank_index": 0.9840322542018902, - "adjusted_hits_at_k": 0.23782172866273546, - "adjusted_inverse_harmonic_mean_rank": 0.10889755510344874, - "arithmetic_mean_rank": 907.9307443584838, - "count": 59071.0, - "geometric_mean_rank": 87.46514480716856, - "harmonic_mean_rank": 9.131233166899994, - "hits_at_1": 0.045250630597078095, - "hits_at_10": 0.23834030234802187, - "hits_at_3": 0.12136242826429212, - "hits_at_5": 0.16500482470247668, - "inverse_arithmetic_mean_rank": 0.0011014055931177543, - "inverse_geometric_mean_rank": 0.011433125757748028, - "inverse_harmonic_mean_rank": 0.10951423337046323, - "inverse_median_rank": 0.01098901098901099, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 91.0, - "standard_deviation": 2127.478091029021, - "variance": 4526163.027808487, - "z_arithmetic_mean_rank": 368.75830553111393, - "z_geometric_mean_rank": 239.5627482369506, - "z_hits_at_k": 2215.207348551444, - "z_inverse_harmonic_mean_rank": 2505.5085253778548 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.12324716749237978, - "adjusted_arithmetic_mean_rank_index": 0.8768718635569903, - "adjusted_geometric_mean_rank_index": 0.9840322468993766, - "adjusted_hits_at_k": 0.23782172866273546, - "adjusted_inverse_harmonic_mean_rank": 0.1088975468405954, - "arithmetic_mean_rank": 907.9309475038513, - "count": 59071.0, - "geometric_mean_rank": 87.46518435018889, - "harmonic_mean_rank": 9.131233855375177, - "hits_at_1": 0.045250630597078095, - "hits_at_10": 0.23834030234802187, - "hits_at_3": 0.12136242826429212, - "hits_at_5": 0.16500482470247668, - "inverse_arithmetic_mean_rank": 0.001101405346683326, - "inverse_geometric_mean_rank": 0.011433120588830501, - "inverse_harmonic_mean_rank": 0.1095142251133281, - "inverse_median_rank": 0.01098901098901099, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 91.0, - "standard_deviation": 2127.478475678952, - "variance": 4526164.664477237, - "z_arithmetic_mean_rank": 368.7582939327784, - "z_geometric_mean_rank": 239.56274645915295, - "z_hits_at_k": 2215.207348551444, - "z_inverse_harmonic_mean_rank": 2505.508335266612 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.12324715387227646, - "adjusted_arithmetic_mean_rank_index": 0.8768718771789428, - "adjusted_geometric_mean_rank_index": 0.9840322534412952, - "adjusted_hits_at_k": 0.23782172866273546, - "adjusted_inverse_harmonic_mean_rank": 0.10889755434025136, - "arithmetic_mean_rank": 907.9308471679688, - "count": 59071.0, - "geometric_mean_rank": 87.46514892578125, - "harmonic_mean_rank": 9.131233230490912, - "hits_at_1": 0.045250630597078095, - "hits_at_10": 0.23834030234802187, - "hits_at_3": 0.12136242826429212, - "hits_at_5": 0.16500482470247668, - "inverse_arithmetic_mean_rank": 0.001101405476219952, - "inverse_geometric_mean_rank": 0.011433125473558903, - "inverse_harmonic_mean_rank": 0.10951423260779401, - "inverse_median_rank": 0.010989011265337467, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 91.0, - "standard_deviation": 2127.478271484375, - "variance": 4526164.0, - "z_arithmetic_mean_rank": 368.7582996613325, - "z_geometric_mean_rank": 239.56274805178367, - "z_hits_at_k": 2215.207348551444, - "z_inverse_harmonic_mean_rank": 2505.508507818254 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0942187911539081, - "adjusted_arithmetic_mean_rank_index": 0.9059035132767959, - "adjusted_geometric_mean_rank_index": 0.9879743242702755, - "adjusted_hits_at_k": 0.2666741240328079, - "adjusted_inverse_harmonic_mean_rank": 0.11875873999814945, - "arithmetic_mean_rank": 697.8744223053613, - "count": 59071.0, - "geometric_mean_rank": 66.49570342330368, - "harmonic_mean_rank": 8.377647410115342, - "hits_at_1": 0.04907653501718271, - "hits_at_10": 0.267170015743766, - "hits_at_3": 0.13026696687037634, - "hits_at_5": 0.179597433596858, - "inverse_arithmetic_mean_rank": 0.00143292255460029, - "inverse_geometric_mean_rank": 0.015038565629332768, - "inverse_harmonic_mean_rank": 0.1193652526832986, - "inverse_median_rank": 0.015384615384615385, - "median_absolute_deviation": 93.40393976585293, - "median_rank": 65.0, - "standard_deviation": 1827.8284088665332, - "variance": 3340956.692259562, - "z_arithmetic_mean_rank": 381.07985853865824, - "z_geometric_mean_rank": 240.52049145945, - "z_hits_at_k": 2491.5898231805772, - "z_inverse_harmonic_mean_rank": 2740.7652003039234 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.09421879801048227, - "adjusted_arithmetic_mean_rank_index": 0.905903506419296, - "adjusted_geometric_mean_rank_index": 0.9879743241749565, - "adjusted_hits_at_k": 0.2666741240328079, - "adjusted_inverse_harmonic_mean_rank": 0.11875873999674515, - "arithmetic_mean_rank": 697.8744730917032, - "count": 59071.0, - "geometric_mean_rank": 66.49570394244209, - "harmonic_mean_rank": 8.377647410213836, - "hits_at_1": 0.04907653501718271, - "hits_at_10": 0.267170015743766, - "hits_at_3": 0.13026696687037634, - "hits_at_5": 0.179597433596858, - "inverse_arithmetic_mean_rank": 0.0014329224503223754, - "inverse_geometric_mean_rank": 0.015038565511925226, - "inverse_harmonic_mean_rank": 0.11936525268189527, - "inverse_median_rank": 0.015384615384615385, - "median_absolute_deviation": 93.40393976585293, - "median_rank": 65.0, - "standard_deviation": 1827.828599332042, - "variance": 3340957.388536134, - "z_arithmetic_mean_rank": 381.0798556539635, - "z_geometric_mean_rank": 240.52049143624475, - "z_hits_at_k": 2491.5898231805772, - "z_inverse_harmonic_mean_rank": 2740.7652002715145 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.09421880322545154, - "adjusted_arithmetic_mean_rank_index": 0.9059035012036225, - "adjusted_geometric_mean_rank_index": 0.9879743240448803, - "adjusted_hits_at_k": 0.2666741240328079, - "adjusted_inverse_harmonic_mean_rank": 0.11875874103301752, - "arithmetic_mean_rank": 697.87451171875, - "count": 59071.0, - "geometric_mean_rank": 66.4957046508789, - "harmonic_mean_rank": 8.377647337533142, - "hits_at_1": 0.04907653501718271, - "hits_at_10": 0.267170015743766, - "hits_at_3": 0.13026696687037634, - "hits_at_5": 0.179597433596858, - "inverse_arithmetic_mean_rank": 0.001432922319509089, - "inverse_geometric_mean_rank": 0.0150385657325387, - "inverse_harmonic_mean_rank": 0.11936525371745442, - "inverse_median_rank": 0.015384615398943424, - "median_absolute_deviation": 93.40393976585293, - "median_rank": 65.0, - "standard_deviation": 1827.828369140625, - "variance": 3340956.75, - "z_arithmetic_mean_rank": 381.079853459924, - "z_geometric_mean_rank": 240.52049140457794, - "z_hits_at_k": 2491.5898231805772, - "z_inverse_harmonic_mean_rank": 2740.7652241870533 - } - } - }, - "times": { - "evaluation": 31.021438598632812, - "training": 9.139031410217285 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json deleted file mode 100644 index 11968d1..0000000 --- a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with ComplEx as described by Trouillon et al., 2016" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json b/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json deleted file mode 100644 index 017cd20..0000000 --- a/nonasv/results/27c9fe15/cuda/complex/wn18/trouillon2016/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.0004529083935333639, - 0.00038640364365897227, - 0.00035145446217553473, - 0.0003258956506732659, - 0.0003048123034097779 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0818915906396609, - "adjusted_arithmetic_mean_rank_index": 0.9181532777971985, - "adjusted_geometric_mean_rank_index": 0.9959612239406452, - "adjusted_hits_at_k": 0.3173331922397517, - "adjusted_inverse_harmonic_mean_rank": 0.1765106114247664, - "arithmetic_mean_rank": 1675.7667, - "count": 10000.0, - "geometric_mean_rank": 61.814524182048764, - "harmonic_mean_rank": 5.658159708335188, - "hits_at_1": 0.1071, - "hits_at_10": 0.3175, - "hits_at_3": 0.1959, - "hits_at_5": 0.2414, - "inverse_arithmetic_mean_rank": 0.0005967417779575165, - "inverse_geometric_mean_rank": 0.016177427768511478, - "inverse_harmonic_mean_rank": 0.17673590912021675, - "inverse_median_rank": 0.0196078431372549, - "median_absolute_deviation": 74.13011092528009, - "median_rank": 51.0, - "standard_deviation": 5371.088487231532, - "variance": 28848591.537671108, - "z_arithmetic_mean_rank": 159.02472772888677, - "z_geometric_mean_rank": 99.6766781576916, - "z_hits_at_k": 2029.8254949907523, - "z_inverse_harmonic_mean_rank": 2786.0057005980207 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.08189159552647424, - "adjusted_arithmetic_mean_rank_index": 0.9181532729101463, - "adjusted_geometric_mean_rank_index": 0.9959612237297443, - "adjusted_hits_at_k": 0.3173331922397517, - "adjusted_inverse_harmonic_mean_rank": 0.17651061139836607, - "arithmetic_mean_rank": 1675.7668, - "count": 10000.0, - "geometric_mean_rank": 61.814527357724415, - "harmonic_mean_rank": 5.658159709180158, - "hits_at_1": 0.1071, - "hits_at_10": 0.3175, - "hits_at_3": 0.1959, - "hits_at_5": 0.2414, - "inverse_arithmetic_mean_rank": 0.0005967417423474435, - "inverse_geometric_mean_rank": 0.016177426937408087, - "inverse_harmonic_mean_rank": 0.17673590909382364, - "inverse_median_rank": 0.0196078431372549, - "median_absolute_deviation": 74.13011092528009, - "median_rank": 51.0, - "standard_deviation": 5371.088492272098, - "variance": 28848591.591817763, - "z_arithmetic_mean_rank": 159.02472688244623, - "z_geometric_mean_rank": 99.67667813658444, - "z_hits_at_k": 2029.8254949907523, - "z_inverse_harmonic_mean_rank": 2786.005700181324 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.08189159775990064, - "adjusted_arithmetic_mean_rank_index": 0.9181532706766107, - "adjusted_geometric_mean_rank_index": 0.9959612240995342, - "adjusted_hits_at_k": 0.3173331922397517, - "adjusted_inverse_harmonic_mean_rank": 0.17651061539157242, - "arithmetic_mean_rank": 1675.766845703125, - "count": 10000.0, - "geometric_mean_rank": 61.81452178955078, - "harmonic_mean_rank": 5.658159581373549, - "hits_at_1": 0.1071, - "hits_at_10": 0.3175, - "hits_at_3": 0.1959, - "hits_at_5": 0.2414, - "inverse_arithmetic_mean_rank": 0.0005967417382635176, - "inverse_geometric_mean_rank": 0.016177428886294365, - "inverse_harmonic_mean_rank": 0.1767359130859375, - "inverse_median_rank": 0.019607843831181526, - "median_absolute_deviation": 74.13011092528009, - "median_rank": 51.0, - "standard_deviation": 5371.08837890625, - "variance": 28848590.0, - "z_arithmetic_mean_rank": 159.02472649559647, - "z_geometric_mean_rank": 99.67667817359336, - "z_hits_at_k": 2029.8254949907523, - "z_inverse_harmonic_mean_rank": 2786.005763209238 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0814552638078817, - "adjusted_arithmetic_mean_rank_index": 0.9185896269932536, - "adjusted_geometric_mean_rank_index": 0.9960286009146595, - "adjusted_hits_at_k": 0.31803335943219535, - "adjusted_inverse_harmonic_mean_rank": 0.1759327156692569, - "arithmetic_mean_rank": 1666.7994, - "count": 5000.0, - "geometric_mean_rank": 60.801578002011404, - "harmonic_mean_rank": 5.676716353502426, - "hits_at_1": 0.1036, - "hits_at_10": 0.3182, - "hits_at_3": 0.1976, - "hits_at_5": 0.2428, - "inverse_arithmetic_mean_rank": 0.0005999522198052147, - "inverse_geometric_mean_rank": 0.016446941557452976, - "inverse_harmonic_mean_rank": 0.17615817626382177, - "inverse_median_rank": 0.02040816326530612, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 49.0, - "standard_deviation": 5358.16252366048, - "variance": 28709905.62995964, - "z_arithmetic_mean_rank": 112.50089413340184, - "z_geometric_mean_rank": 70.49203961549347, - "z_hits_at_k": 1438.4534399644135, - "z_inverse_harmonic_mean_rank": 1963.5308287567036 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.0814552638078817, - "adjusted_arithmetic_mean_rank_index": 0.9185896269932536, - "adjusted_geometric_mean_rank_index": 0.9960286009146595, - "adjusted_hits_at_k": 0.31803335943219535, - "adjusted_inverse_harmonic_mean_rank": 0.1759327156692569, - "arithmetic_mean_rank": 1666.7994, - "count": 5000.0, - "geometric_mean_rank": 60.801578002011404, - "harmonic_mean_rank": 5.676716353502426, - "hits_at_1": 0.1036, - "hits_at_10": 0.3182, - "hits_at_3": 0.1976, - "hits_at_5": 0.2428, - "inverse_arithmetic_mean_rank": 0.0005999522198052147, - "inverse_geometric_mean_rank": 0.016446941557452976, - "inverse_harmonic_mean_rank": 0.17615817626382177, - "inverse_median_rank": 0.02040816326530612, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 49.0, - "standard_deviation": 5358.16252366048, - "variance": 28709905.62995964, - "z_arithmetic_mean_rank": 112.50089413340184, - "z_geometric_mean_rank": 70.49203961549347, - "z_hits_at_k": 1438.4534399644135, - "z_inverse_harmonic_mean_rank": 1963.5308287567036 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.08145526568820308, - "adjusted_arithmetic_mean_rank_index": 0.9185896251128404, - "adjusted_geometric_mean_rank_index": 0.9960286018934754, - "adjusted_hits_at_k": 0.31803335943219535, - "adjusted_inverse_harmonic_mean_rank": 0.1759327181170423, - "arithmetic_mean_rank": 1666.7994384765625, - "count": 5000.0, - "geometric_mean_rank": 60.80156326293945, - "harmonic_mean_rank": 5.676716274643858, - "hits_at_1": 0.1036, - "hits_at_10": 0.3182, - "hits_at_3": 0.1976, - "hits_at_5": 0.2428, - "inverse_arithmetic_mean_rank": 0.0005999521818011999, - "inverse_geometric_mean_rank": 0.016446946188807487, - "inverse_harmonic_mean_rank": 0.1761581787109375, - "inverse_median_rank": 0.020408162847161293, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 49.0, - "standard_deviation": 5358.16259765625, - "variance": 28709906.0, - "z_arithmetic_mean_rank": 112.50089390310512, - "z_geometric_mean_rank": 70.4920396847673, - "z_hits_at_k": 1438.4534399644135, - "z_inverse_harmonic_mean_rank": 1963.5308560756825 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.08232789724114213, - "adjusted_arithmetic_mean_rank_index": 0.9177169488334667, - "adjusted_geometric_mean_rank_index": 0.9958931326163891, - "adjusted_hits_at_k": 0.31663302505530083, - "adjusted_inverse_harmonic_mean_rank": 0.17708850717355146, - "arithmetic_mean_rank": 1684.734, - "count": 5000.0, - "geometric_mean_rank": 62.84434591034264, - "harmonic_mean_rank": 5.6397239876890195, - "hits_at_1": 0.1106, - "hits_at_10": 0.3168, - "hits_at_3": 0.1942, - "hits_at_5": 0.24, - "inverse_arithmetic_mean_rank": 0.0005935655124191712, - "inverse_geometric_mean_rank": 0.01591233046528414, - "inverse_harmonic_mean_rank": 0.17731364197661176, - "inverse_median_rank": 0.018518518518518517, - "median_absolute_deviation": 78.5779175807969, - "median_rank": 54.0, - "standard_deviation": 5383.968482489845, - "variance": 28987116.620444, - "z_arithmetic_mean_rank": 112.39403510187431, - "z_geometric_mean_rank": 70.48245014898455, - "z_hits_at_k": 1432.1532309511042, - "z_inverse_harmonic_mean_rank": 1976.4763690695154 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.08232790701454225, - "adjusted_arithmetic_mean_rank_index": 0.9177169390595888, - "adjusted_geometric_mean_rank_index": 0.9958931321875913, - "adjusted_hits_at_k": 0.31663302505530083, - "adjusted_inverse_harmonic_mean_rank": 0.17708850712075072, - "arithmetic_mean_rank": 1684.7342, - "count": 5000.0, - "geometric_mean_rank": 62.844352367506836, - "harmonic_mean_rank": 5.639723989367966, - "hits_at_1": 0.1106, - "hits_at_10": 0.3168, - "hits_at_3": 0.1942, - "hits_at_5": 0.24, - "inverse_arithmetic_mean_rank": 0.000593565441955176, - "inverse_geometric_mean_rank": 0.015912328830315736, - "inverse_harmonic_mean_rank": 0.17731364192382545, - "inverse_median_rank": 0.018518518518518517, - "median_absolute_deviation": 78.5779175807969, - "median_rank": 54.0, - "standard_deviation": 5383.968492213747, - "variance": 28987116.72515036, - "z_arithmetic_mean_rank": 112.39403390485428, - "z_geometric_mean_rank": 70.4824501186372, - "z_hits_at_k": 1432.1532309511042, - "z_inverse_harmonic_mean_rank": 1976.4763684802087 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.08232789767063727, - "adjusted_arithmetic_mean_rank_index": 0.9177169484039505, - "adjusted_geometric_mean_rank_index": 0.9958931331773225, - "adjusted_hits_at_k": 0.31663302505530083, - "adjusted_inverse_harmonic_mean_rank": 0.17708850044900623, - "arithmetic_mean_rank": 1684.7340087890625, - "count": 5000.0, - "geometric_mean_rank": 62.844337463378906, - "harmonic_mean_rank": 5.639724201514671, - "hits_at_1": 0.1106, - "hits_at_10": 0.3168, - "hits_at_3": 0.1942, - "hits_at_5": 0.24, - "inverse_arithmetic_mean_rank": 0.0005935655208304524, - "inverse_geometric_mean_rank": 0.015912333503365517, - "inverse_harmonic_mean_rank": 0.17731363525390625, - "inverse_median_rank": 0.018518518656492233, - "median_absolute_deviation": 78.5779175807969, - "median_rank": 54.0, - "standard_deviation": 5383.96826171875, - "variance": 28987116.0, - "z_arithmetic_mean_rank": 112.39403504927088, - "z_geometric_mean_rank": 70.48245018868356, - "z_hits_at_k": 1432.1532309511042, - "z_inverse_harmonic_mean_rank": 1976.4762940171875 - } - } - }, - "times": { - "evaluation": 4.746052265167236, - "training": 1.822469711303711 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json deleted file mode 100644 index 91b1a21..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with ConvE as described by Dettmers et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json deleted file mode 100644 index a840c4c..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/fb15k/dettmers2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 9.350439590000361e-05, - 1.3484684583750924e-05, - 1.1116209103459396e-05, - 9.792114026757163e-06, - 8.904646023875852e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.061309328820577665, - "adjusted_arithmetic_mean_rank_index": 0.9388177642514287, - "adjusted_geometric_mean_rank_index": 0.9926356631379782, - "adjusted_hits_at_k": 0.34970533694843586, - "adjusted_inverse_harmonic_mean_rank": 0.23031375805864562, - "arithmetic_mean_rank": 452.8829628751841, - "count": 118142.0, - "geometric_mean_rank": 40.99283647360712, - "harmonic_mean_rank": 4.331911916722617, - "hits_at_1": 0.16881380034196136, - "hits_at_10": 0.3501464339523624, - "hits_at_3": 0.24841292681688137, - "hits_at_5": 0.2896345076264157, - "inverse_arithmetic_mean_rank": 0.002208075997496958, - "inverse_geometric_mean_rank": 0.02439450611435101, - "inverse_harmonic_mean_rank": 0.23084495234994698, - "inverse_median_rank": 0.022727272727272728, - "median_absolute_deviation": 63.75189539574088, - "median_rank": 44.0, - "standard_deviation": 1078.3816682039305, - "variance": 1162907.0223182922, - "z_arithmetic_mean_rank": 558.4250406185861, - "z_geometric_mean_rank": 341.77861851221815, - "z_hits_at_k": 4613.659080043995, - "z_inverse_harmonic_mean_rank": 7505.435015590688 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.061309349446280914, - "adjusted_arithmetic_mean_rank_index": 0.9388177436229329, - "adjusted_geometric_mean_rank_index": 0.9926356616147918, - "adjusted_hits_at_k": 0.34970533694843586, - "adjusted_inverse_harmonic_mean_rank": 0.2303137574763396, - "arithmetic_mean_rank": 452.8831152342097, - "count": 118142.0, - "geometric_mean_rank": 40.99284474543766, - "harmonic_mean_rank": 4.331911927642317, - "hits_at_1": 0.16881380034196136, - "hits_at_10": 0.3501464339523624, - "hits_at_3": 0.24841292681688137, - "hits_at_5": 0.2896345076264157, - "inverse_arithmetic_mean_rank": 0.002208075254655602, - "inverse_geometric_mean_rank": 0.024394501191852415, - "inverse_harmonic_mean_rank": 0.23084495176804282, - "inverse_median_rank": 0.022727272727272728, - "median_absolute_deviation": 63.75189539574088, - "median_rank": 44.0, - "standard_deviation": 1078.3819740001402, - "variance": 1162907.681848439, - "z_arithmetic_mean_rank": 558.4250283484, - "z_geometric_mean_rank": 341.77861798776337, - "z_hits_at_k": 4613.659080043995, - "z_inverse_harmonic_mean_rank": 7505.434996614574 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.061309345645473065, - "adjusted_arithmetic_mean_rank_index": 0.9388177474242554, - "adjusted_geometric_mean_rank_index": 0.9926356611181276, - "adjusted_hits_at_k": 0.34970533694843586, - "adjusted_inverse_harmonic_mean_rank": 0.23031375818123168, - "arithmetic_mean_rank": 452.8830871582031, - "count": 118142.0, - "geometric_mean_rank": 40.99284744262695, - "harmonic_mean_rank": 4.331911914423821, - "hits_at_1": 0.16881380034196136, - "hits_at_10": 0.3501464339523624, - "hits_at_3": 0.24841292681688137, - "hits_at_5": 0.2896345076264157, - "inverse_arithmetic_mean_rank": 0.002208075486123562, - "inverse_geometric_mean_rank": 0.024394499137997627, - "inverse_harmonic_mean_rank": 0.23084495247244843, - "inverse_median_rank": 0.022727273404598236, - "median_absolute_deviation": 63.75189539574088, - "median_rank": 44.0, - "standard_deviation": 1078.3818359375, - "variance": 1162907.5, - "z_arithmetic_mean_rank": 558.4250306094924, - "z_geometric_mean_rank": 341.7786178167548, - "z_hits_at_k": 4613.659080043995, - "z_inverse_harmonic_mean_rank": 7505.435019585506 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0681778728311803, - "adjusted_arithmetic_mean_rank_index": 0.9319486346193927, - "adjusted_geometric_mean_rank_index": 0.9906846783036631, - "adjusted_hits_at_k": 0.31298586873037954, - "adjusted_inverse_harmonic_mean_rank": 0.19818517693798296, - "arithmetic_mean_rank": 502.2492762946285, - "count": 59071.0, - "geometric_mean_rank": 51.44235107340421, - "harmonic_mean_rank": 5.031698071957418, - "hits_at_1": 0.13917150547646054, - "hits_at_10": 0.3134533019586599, - "hits_at_3": 0.21396285825531988, - "hits_at_5": 0.2538809229571194, - "inverse_arithmetic_mean_rank": 0.0019910431875134887, - "inverse_geometric_mean_rank": 0.019439235943417094, - "inverse_harmonic_mean_rank": 0.19874006462613178, - "inverse_median_rank": 0.016666666666666666, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1139.641318600422, - "variance": 1298782.3350613085, - "z_arithmetic_mean_rank": 391.9201913278106, - "z_geometric_mean_rank": 241.18228153319507, - "z_hits_at_k": 2915.329058883141, - "z_inverse_harmonic_mean_rank": 4559.832862637971 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.068177891215171, - "adjusted_arithmetic_mean_rank_index": 0.9319486162329061, - "adjusted_geometric_mean_rank_index": 0.9906846771163226, - "adjusted_hits_at_k": 0.31298586873037954, - "adjusted_inverse_harmonic_mean_rank": 0.19818517668041305, - "arithmetic_mean_rank": 502.2494117248735, - "count": 59071.0, - "geometric_mean_rank": 51.44235750283864, - "harmonic_mean_rank": 5.031698078474057, - "hits_at_1": 0.13917150547646054, - "hits_at_10": 0.3134533019586599, - "hits_at_3": 0.21396285825531988, - "hits_at_5": 0.2538809229571194, - "inverse_arithmetic_mean_rank": 0.001991042650633882, - "inverse_geometric_mean_rank": 0.019439233513837677, - "inverse_harmonic_mean_rank": 0.19874006436874012, - "inverse_median_rank": 0.016666666666666666, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1139.6416993059368, - "variance": 1298783.2027969232, - "z_arithmetic_mean_rank": 391.9201835955869, - "z_geometric_mean_rank": 241.18228124413687, - "z_hits_at_k": 2915.329058883141, - "z_inverse_harmonic_mean_rank": 4559.832856711818 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.06817788407579185, - "adjusted_arithmetic_mean_rank_index": 0.9319486233732546, - "adjusted_geometric_mean_rank_index": 0.9906846794870225, - "adjusted_hits_at_k": 0.31298586873037954, - "adjusted_inverse_harmonic_mean_rank": 0.19818519443912916, - "arithmetic_mean_rank": 502.2493591308594, - "count": 59071.0, - "geometric_mean_rank": 51.442344665527344, - "harmonic_mean_rank": 5.03169762917033, - "hits_at_1": 0.13917150547646054, - "hits_at_10": 0.3134533019586599, - "hits_at_3": 0.21396285825531988, - "hits_at_5": 0.2538809229571194, - "inverse_arithmetic_mean_rank": 0.0019910428673028946, - "inverse_geometric_mean_rank": 0.01943923905491829, - "inverse_harmonic_mean_rank": 0.1987400821151665, - "inverse_median_rank": 0.01666666753590107, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1139.6414794921875, - "variance": 1298782.75, - "z_arithmetic_mean_rank": 391.9201865983776, - "z_geometric_mean_rank": 241.18228182128405, - "z_hits_at_k": 2915.329058883141, - "z_inverse_harmonic_mean_rank": 4559.83326530331 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.054478068986399376, - "adjusted_arithmetic_mean_rank_index": 0.9456496014933604, - "adjusted_geometric_mean_rank_index": 0.9941858125013604, - "adjusted_hits_at_k": 0.38642465227813766, - "adjusted_inverse_harmonic_mean_rank": 0.26244221727901273, - "arithmetic_mean_rank": 403.5166494557397, - "count": 59071.0, - "geometric_mean_rank": 32.66593783309161, - "harmonic_mean_rank": 3.803006686444386, - "hits_at_1": 0.1984560952074622, - "hits_at_10": 0.3868395659460649, - "hits_at_3": 0.2828629953784429, - "hits_at_5": 0.3253880922957119, - "inverse_arithmetic_mean_rank": 0.0024782124885027487, - "inverse_geometric_mean_rank": 0.0306129279100926, - "inverse_harmonic_mean_rank": 0.2629498400737623, - "inverse_median_rank": 0.03125, - "median_absolute_deviation": 45.96066877367366, - "median_rank": 32.0, - "standard_deviation": 1011.0181223766575, - "variance": 1022157.643774022, - "z_arithmetic_mean_rank": 397.799557108152, - "z_geometric_mean_rank": 242.03266658924255, - "z_hits_at_k": 3610.4430249252455, - "z_inverse_harmonic_mean_rank": 6056.754191061033 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.054478091841646606, - "adjusted_arithmetic_mean_rank_index": 0.9456495786350271, - "adjusted_geometric_mean_rank_index": 0.9941858108304271, - "adjusted_hits_at_k": 0.38642465227813766, - "adjusted_inverse_harmonic_mean_rank": 0.2624422163719718, - "arithmetic_mean_rank": 403.5168187435459, - "count": 59071.0, - "geometric_mean_rank": 32.6659469335329, - "harmonic_mean_rank": 3.8030066995537632, - "hits_at_1": 0.1984560952074622, - "hits_at_10": 0.3868395659460649, - "hits_at_3": 0.2828629953784429, - "hits_at_5": 0.3253880922957119, - "inverse_arithmetic_mean_rank": 0.002478211448815836, - "inverse_geometric_mean_rank": 0.03061291938160409, - "inverse_harmonic_mean_rank": 0.26294983916734566, - "inverse_median_rank": 0.03125, - "median_absolute_deviation": 45.96066877367366, - "median_rank": 32.0, - "standard_deviation": 1011.0183472329007, - "variance": 1022158.0984415463, - "z_arithmetic_mean_rank": 397.79954749250277, - "z_geometric_mean_rank": 242.03266618245698, - "z_hits_at_k": 3610.4430249252455, - "z_inverse_harmonic_mean_rank": 6056.754170127952 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.054478079000914746, - "adjusted_arithmetic_mean_rank_index": 0.9456495914774928, - "adjusted_geometric_mean_rank_index": 0.9941858129267364, - "adjusted_hits_at_k": 0.38642465227813766, - "adjusted_inverse_harmonic_mean_rank": 0.2624422165665027, - "arithmetic_mean_rank": 403.5167236328125, - "count": 59071.0, - "geometric_mean_rank": 32.66593551635742, - "harmonic_mean_rank": 3.8030066967422265, - "hits_at_1": 0.1984560952074622, - "hits_at_10": 0.3868395659460649, - "hits_at_3": 0.2828629953784429, - "hits_at_5": 0.3253880922957119, - "inverse_arithmetic_mean_rank": 0.0024782121181488037, - "inverse_geometric_mean_rank": 0.03061293065547943, - "inverse_harmonic_mean_rank": 0.26294983936174265, - "inverse_median_rank": 0.03125, - "median_absolute_deviation": 45.96066877367366, - "median_rank": 32.0, - "standard_deviation": 1011.0182495117188, - "variance": 1022157.875, - "z_arithmetic_mean_rank": 397.7995528948496, - "z_geometric_mean_rank": 242.03266669279952, - "z_hits_at_k": 3610.4430249252455, - "z_inverse_harmonic_mean_rank": 6056.75417461742 - } - } - }, - "times": { - "evaluation": 15.175896167755127, - "training": 160.1173620223999 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json deleted file mode 100644 index 991276c..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K-237 Dataset with ConvE as described by Dettmers et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json deleted file mode 100644 index 375b5c8..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/fb15k237/dettmers2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 4096, - "slice_size": null - }, - "losses": [ - 0.0001553554180276956, - 1.53616951009097e-05, - 1.370961714377088e-05, - 1.2225620274735016e-05, - 1.108662612753889e-05 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.18175210325712907, - "adjusted_arithmetic_mean_rank_index": 0.8183625829185686, - "adjusted_geometric_mean_rank_index": 0.9847747167425137, - "adjusted_hits_at_k": 0.3200769143496645, - "adjusted_inverse_harmonic_mean_rank": 0.2096363216603547, - "arithmetic_mean_rank": 1296.9228398082005, - "count": 40876.0, - "geometric_mean_rank": 80.83336590952712, - "harmonic_mean_rank": 4.757383657725782, - "hits_at_1": 0.1545894901653782, - "hits_at_10": 0.32055484881103824, - "hits_at_3": 0.22717487033956354, - "hits_at_5": 0.2661463939720129, - "inverse_arithmetic_mean_rank": 0.0007710558942333748, - "inverse_geometric_mean_rank": 0.012371129034998391, - "inverse_harmonic_mean_rank": 0.21019957017257668, - "inverse_median_rank": 0.011235955056179775, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 89.0, - "standard_deviation": 2518.7222644431486, - "variance": 6343961.845401622, - "z_arithmetic_mean_rank": 286.20881146568547, - "z_geometric_mean_rank": 199.44238519906037, - "z_hits_at_k": 2439.957034486236, - "z_inverse_harmonic_mean_rank": 3947.578789367722 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.18175216154065776, - "adjusted_arithmetic_mean_rank_index": 0.8183625246268708, - "adjusted_geometric_mean_rank_index": 0.9847747097533328, - "adjusted_hits_at_k": 0.3200769143496645, - "adjusted_inverse_harmonic_mean_rank": 0.2096363193214703, - "arithmetic_mean_rank": 1296.9232557001665, - "count": 40876.0, - "geometric_mean_rank": 80.83340255711062, - "harmonic_mean_rank": 4.757383710623325, - "hits_at_1": 0.1545894901653782, - "hits_at_10": 0.32055484881103824, - "hits_at_3": 0.22717487033956354, - "hits_at_5": 0.2661463939720129, - "inverse_arithmetic_mean_rank": 0.0007710556469743714, - "inverse_geometric_mean_rank": 0.012371123426277615, - "inverse_harmonic_mean_rank": 0.2101995678353591, - "inverse_median_rank": 0.011235955056179775, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 89.0, - "standard_deviation": 2518.7227934087005, - "variance": 6343964.510036528, - "z_arithmetic_mean_rank": 286.20879107912594, - "z_geometric_mean_rank": 199.44238378357022, - "z_hits_at_k": 2439.957034486236, - "z_inverse_harmonic_mean_rank": 3947.578745325114 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.1817521391185075, - "adjusted_arithmetic_mean_rank_index": 0.8183625470521637, - "adjusted_geometric_mean_rank_index": 0.9847747108300027, - "adjusted_hits_at_k": 0.3200769143496645, - "adjusted_inverse_harmonic_mean_rank": 0.20963631081804915, - "arithmetic_mean_rank": 1296.923095703125, - "count": 40876.0, - "geometric_mean_rank": 80.8333969116211, - "harmonic_mean_rank": 4.757383902941559, - "hits_at_1": 0.1545894901653782, - "hits_at_10": 0.32055484881103824, - "hits_at_3": 0.22717487033956354, - "hits_at_5": 0.2661463939720129, - "inverse_arithmetic_mean_rank": 0.0007710557547397912, - "inverse_geometric_mean_rank": 0.0123711246997118, - "inverse_harmonic_mean_rank": 0.21019955933799786, - "inverse_median_rank": 0.01123595517128706, - "median_absolute_deviation": 130.46899522849296, - "median_rank": 89.0, - "standard_deviation": 2518.72265625, - "variance": 6343963.5, - "z_arithmetic_mean_rank": 286.2087989220022, - "z_geometric_mean_rank": 199.44238400162374, - "z_hits_at_k": 2439.957034486236, - "z_inverse_harmonic_mean_rank": 3947.5785852005483 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.23954731895542877, - "adjusted_arithmetic_mean_rank_index": 0.7605607668569763, - "adjusted_geometric_mean_rank_index": 0.9634020053121783, - "adjusted_hits_at_k": 0.19567726258469004, - "adjusted_inverse_harmonic_mean_rank": 0.11268113063070488, - "arithmetic_mean_rank": 1685.6078383403465, - "count": 20438.0, - "geometric_mean_rank": 190.0114797272476, - "harmonic_mean_rank": 8.824342686842055, - "hits_at_1": 0.07123984734318427, - "hits_at_10": 0.19625207945982973, - "hits_at_3": 0.12036402779136902, - "hits_at_5": 0.15104217633819356, - "inverse_arithmetic_mean_rank": 0.0005932578012834838, - "inverse_geometric_mean_rank": 0.005262839915964299, - "inverse_harmonic_mean_rank": 0.11332288822952177, - "inverse_median_rank": 0.003125, - "median_absolute_deviation": 471.46750548478144, - "median_rank": 320.0, - "standard_deviation": 2761.296311585376, - "variance": 7624757.320375001, - "z_arithmetic_mean_rank": 187.88604932091593, - "z_geometric_mean_rank": 137.97311438795538, - "z_hits_at_k": 1046.058168464544, - "z_inverse_harmonic_mean_rank": 1488.0438353149812 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.23954740934945482, - "adjusted_arithmetic_mean_rank_index": 0.7605606764501023, - "adjusted_geometric_mean_rank_index": 0.9634019815009212, - "adjusted_hits_at_k": 0.19567726258469004, - "adjusted_inverse_harmonic_mean_rank": 0.11268112698425874, - "arithmetic_mean_rank": 1685.6084744104119, - "count": 20438.0, - "geometric_mean_rank": 190.0116027012244, - "harmonic_mean_rank": 8.8243429705819, - "hits_at_1": 0.07123984734318427, - "hits_at_10": 0.19625207945982973, - "hits_at_3": 0.12036402779136902, - "hits_at_5": 0.15104217633819356, - "inverse_arithmetic_mean_rank": 0.0005932575774156438, - "inverse_geometric_mean_rank": 0.0052628365098967515, - "inverse_harmonic_mean_rank": 0.11332288458571294, - "inverse_median_rank": 0.003125, - "median_absolute_deviation": 471.46750548478144, - "median_rank": 320.0, - "standard_deviation": 2761.296903529542, - "variance": 7624760.589441836, - "z_arithmetic_mean_rank": 187.8860269871444, - "z_geometric_mean_rank": 137.97311097783862, - "z_hits_at_k": 1046.058168464544, - "z_inverse_harmonic_mean_rank": 1488.0437871607633 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.2395473638570587, - "adjusted_arithmetic_mean_rank_index": 0.7605607219489643, - "adjusted_geometric_mean_rank_index": 0.9634019826668618, - "adjusted_hits_at_k": 0.19567726258469004, - "adjusted_inverse_harmonic_mean_rank": 0.11268113416020661, - "arithmetic_mean_rank": 1685.608154296875, - "count": 20438.0, - "geometric_mean_rank": 190.0115966796875, - "harmonic_mean_rank": 8.824342412201986, - "hits_at_1": 0.07123984734318427, - "hits_at_10": 0.19625207945982973, - "hits_at_3": 0.12036402779136902, - "hits_at_5": 0.15104217633819356, - "inverse_arithmetic_mean_rank": 0.000593257718719542, - "inverse_geometric_mean_rank": 0.005262836813926697, - "inverse_harmonic_mean_rank": 0.11332289175647078, - "inverse_median_rank": 0.0031250000465661287, - "median_absolute_deviation": 471.46750548478144, - "median_rank": 320.0, - "standard_deviation": 2761.296630859375, - "variance": 7624758.5, - "z_arithmetic_mean_rank": 187.88603822701086, - "z_geometric_mean_rank": 137.9731111448182, - "z_hits_at_k": 1046.058168464544, - "z_inverse_harmonic_mean_rank": 1488.0438819248552 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.12553912270443726, - "adjusted_arithmetic_mean_rank_index": 0.8745817643831122, - "adjusted_geometric_mean_rank_index": 0.9937285636670846, - "adjusted_hits_at_k": 0.4444736443896051, - "adjusted_inverse_harmonic_mean_rank": 0.30658945384952835, - "arithmetic_mean_rank": 908.2378412760544, - "count": 20438.0, - "geometric_mean_rank": 34.38756991757967, - "harmonic_mean_rank": 3.2565201415296796, - "hits_at_1": 0.23793913298757216, - "hits_at_10": 0.4448576181622468, - "hits_at_3": 0.3339857128877581, - "hits_at_5": 0.3812506116058323, - "inverse_arithmetic_mean_rank": 0.0011010331815673104, - "inverse_geometric_mean_rank": 0.02908027529705664, - "inverse_harmonic_mean_rank": 0.30707625211563155, - "inverse_median_rank": 0.05, - "median_absolute_deviation": 28.169442151606436, - "median_rank": 20.0, - "standard_deviation": 2181.974865634427, - "variance": 4761014.314260376, - "z_arithmetic_mean_rank": 216.54184149302714, - "z_geometric_mean_rank": 142.31002549861495, - "z_hits_at_k": 2416.1095916884306, - "z_inverse_harmonic_mean_rank": 4116.719306532869 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.12553914975654407, - "adjusted_arithmetic_mean_rank_index": 0.8745817373272656, - "adjusted_geometric_mean_rank_index": 0.9937285619905694, - "adjusted_hits_at_k": 0.4444736443896051, - "adjusted_inverse_harmonic_mean_rank": 0.30658945281817807, - "arithmetic_mean_rank": 908.2380369899207, - "count": 20438.0, - "geometric_mean_rank": 34.38757884293027, - "harmonic_mean_rank": 3.256520152459392, - "hits_at_1": 0.23793913298757216, - "hits_at_10": 0.4448576181622468, - "hits_at_3": 0.3339857128877581, - "hits_at_5": 0.3812506116058323, - "inverse_arithmetic_mean_rank": 0.0011010329443085169, - "inverse_geometric_mean_rank": 0.029080267749225088, - "inverse_harmonic_mean_rank": 0.3070762510850053, - "inverse_median_rank": 0.05, - "median_absolute_deviation": 28.169442151606436, - "median_rank": 20.0, - "standard_deviation": 2181.9752592876844, - "variance": 4761016.032143558, - "z_arithmetic_mean_rank": 216.54183479414195, - "z_geometric_mean_rank": 142.31002525852432, - "z_hits_at_k": 2416.1095916884306, - "z_inverse_harmonic_mean_rank": 4116.719292684448 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.12553914133660884, - "adjusted_arithmetic_mean_rank_index": 0.8745817457483649, - "adjusted_geometric_mean_rank_index": 0.9937285608929696, - "adjusted_hits_at_k": 0.4444736443896051, - "adjusted_inverse_harmonic_mean_rank": 0.3065894525433575, - "arithmetic_mean_rank": 908.2379760742188, - "count": 20438.0, - "geometric_mean_rank": 34.3875846862793, - "harmonic_mean_rank": 3.2565201553717964, - "hits_at_1": 0.23793913298757216, - "hits_at_10": 0.4448576181622468, - "hits_at_3": 0.3339857128877581, - "hits_at_5": 0.3812506116058323, - "inverse_arithmetic_mean_rank": 0.0011010330636054277, - "inverse_geometric_mean_rank": 0.029080262407660484, - "inverse_harmonic_mean_rank": 0.3070762508103777, - "inverse_median_rank": 0.05000000074505806, - "median_absolute_deviation": 28.169442151606436, - "median_rank": 20.0, - "standard_deviation": 2181.97509765625, - "variance": 4761015.5, - "z_arithmetic_mean_rank": 216.5418368791618, - "z_geometric_mean_rank": 142.31002510133908, - "z_hits_at_k": 2416.1095916884306, - "z_inverse_harmonic_mean_rank": 4116.719288994305 - } - } - }, - "times": { - "evaluation": 6.952138900756836, - "training": 87.22090458869934 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json deleted file mode 100644 index a3fef32..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with ConvE as described by Dettmers et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json deleted file mode 100644 index 59250d4..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/wn18/dettmers2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 8192, - "slice_size": null - }, - "losses": [ - 0.00012069320004326263, - 3.8810777744959124e-06, - 3.485230710315089e-06, - 3.3256080524316043e-06, - 3.18438816935551e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6801818998982223, - "adjusted_arithmetic_mean_rank_index": 0.3198337297791498, - "adjusted_geometric_mean_rank_index": 0.7519856699805167, - "adjusted_hits_at_k": 0.04796737344145887, - "adjusted_inverse_harmonic_mean_rank": 0.02738208123845674, - "arithmetic_mean_rank": 13918.7207, - "count": 10000.0, - "geometric_mean_rank": 3735.5159149214624, - "harmonic_mean_rank": 36.1687474865135, - "hits_at_1": 0.0169, - "hits_at_10": 0.0482, - "hits_at_3": 0.0272, - "hits_at_5": 0.0338, - "inverse_arithmetic_mean_rank": 7.18456833464587e-05, - "inverse_geometric_mean_rank": 0.0002677006396908965, - "inverse_harmonic_mean_rank": 0.027648178869696196, - "inverse_median_rank": 9.786651008025054e-05, - "median_absolute_deviation": 14536.173451338174, - "median_rank": 10218.0, - "standard_deviation": 13115.026157339204, - "variance": 172003911.1076915, - "z_arithmetic_mean_rank": 55.39540404263296, - "z_geometric_mean_rank": 75.25938942609984, - "z_hits_at_k": 306.823868162061, - "z_inverse_harmonic_mean_rank": 432.1929078869801 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6801822859564761, - "adjusted_arithmetic_mean_rank_index": 0.3198333437020291, - "adjusted_geometric_mean_rank_index": 0.7519855443955777, - "adjusted_hits_at_k": 0.04796737344145887, - "adjusted_inverse_harmonic_mean_rank": 0.027382081193512736, - "arithmetic_mean_rank": 13918.7286, - "count": 10000.0, - "geometric_mean_rank": 3735.5178059370073, - "harmonic_mean_rank": 36.168747545292184, - "hits_at_1": 0.0169, - "hits_at_10": 0.0482, - "hits_at_3": 0.0272, - "hits_at_5": 0.0338, - "inverse_arithmetic_mean_rank": 7.184564256824434e-05, - "inverse_geometric_mean_rank": 0.00026770050417392207, - "inverse_harmonic_mean_rank": 0.02764817882476449, - "inverse_median_rank": 9.786651008025054e-05, - "median_absolute_deviation": 14536.173451338174, - "median_rank": 10218.0, - "standard_deviation": 13115.03010968492, - "variance": 172004014.77794203, - "z_arithmetic_mean_rank": 55.39533717383175, - "z_geometric_mean_rank": 75.25937685744833, - "z_hits_at_k": 306.823868162061, - "z_inverse_harmonic_mean_rank": 432.1929071775936 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6801820909420813, - "adjusted_arithmetic_mean_rank_index": 0.3198335387259543, - "adjusted_geometric_mean_rank_index": 0.7519856568071709, - "adjusted_hits_at_k": 0.04796737344145887, - "adjusted_inverse_harmonic_mean_rank": 0.027382083521728977, - "arithmetic_mean_rank": 13918.724609375, - "count": 10000.0, - "geometric_mean_rank": 3735.51611328125, - "harmonic_mean_rank": 36.16874450040376, - "hits_at_1": 0.0169, - "hits_at_10": 0.0482, - "hits_at_3": 0.0272, - "hits_at_5": 0.0338, - "inverse_arithmetic_mean_rank": 7.184566493378952e-05, - "inverse_geometric_mean_rank": 0.0002677006123121828, - "inverse_harmonic_mean_rank": 0.027648181152343753, - "inverse_median_rank": 9.78665120783262e-05, - "median_absolute_deviation": 14536.173451338174, - "median_rank": 10218.0, - "standard_deviation": 13115.02734375, - "variance": 172003952.0, - "z_arithmetic_mean_rank": 55.395370952098816, - "z_geometric_mean_rank": 75.25938810769976, - "z_hits_at_k": 306.823868162061, - "z_inverse_harmonic_mean_rank": 432.19294392566024 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7005482046785227, - "adjusted_arithmetic_mean_rank_index": 0.2994664300262051, - "adjusted_geometric_mean_rank_index": 0.7317236995864804, - "adjusted_hits_at_k": 0.0445665370044485, - "adjusted_inverse_harmonic_mean_rank": 0.025789613592086494, - "arithmetic_mean_rank": 14335.1488, - "count": 5000.0, - "geometric_mean_rank": 4040.7214584881763, - "harmonic_mean_rank": 38.37865153940608, - "hits_at_1": 0.0154, - "hits_at_10": 0.0448, - "hits_at_3": 0.0266, - "hits_at_5": 0.0326, - "inverse_arithmetic_mean_rank": 6.975860620295759e-05, - "inverse_geometric_mean_rank": 0.0002474805576858908, - "inverse_harmonic_mean_rank": 0.026056152571520892, - "inverse_median_rank": 9.11410864017499e-05, - "median_absolute_deviation": 15384.963221432632, - "median_rank": 10972.0, - "standard_deviation": 13176.422049504126, - "variance": 173618098.02665854, - "z_arithmetic_mean_rank": 36.676052233641556, - "z_geometric_mean_rank": 51.78636032286495, - "z_hits_at_k": 201.57284309986937, - "z_inverse_harmonic_mean_rank": 287.82993064793305 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7005485956326534, - "adjusted_arithmetic_mean_rank_index": 0.29946603905296787, - "adjusted_geometric_mean_rank_index": 0.7317235641683568, - "adjusted_hits_at_k": 0.0445665370044485, - "adjusted_inverse_harmonic_mean_rank": 0.0257896135490425, - "arithmetic_mean_rank": 14335.1568, - "count": 5000.0, - "geometric_mean_rank": 4040.723497622826, - "harmonic_mean_rank": 38.378651602789134, - "hits_at_1": 0.0154, - "hits_at_10": 0.0448, - "hits_at_3": 0.0266, - "hits_at_5": 0.0326, - "inverse_arithmetic_mean_rank": 6.975856727287419e-05, - "inverse_geometric_mean_rank": 0.0002474804327958357, - "inverse_harmonic_mean_rank": 0.026056152528488676, - "inverse_median_rank": 9.11410864017499e-05, - "median_absolute_deviation": 15384.963221432632, - "median_rank": 10972.0, - "standard_deviation": 13176.425496128066, - "variance": 173618188.85501376, - "z_arithmetic_mean_rank": 36.67600435062886, - "z_geometric_mean_rank": 51.78635073890348, - "z_hits_at_k": 201.57284309986937, - "z_inverse_harmonic_mean_rank": 287.8299301675323 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7005483778589853, - "adjusted_arithmetic_mean_rank_index": 0.2994662568372789, - "adjusted_geometric_mean_rank_index": 0.7317235227638998, - "adjusted_hits_at_k": 0.0445665370044485, - "adjusted_inverse_harmonic_mean_rank": 0.02578961336425327, - "arithmetic_mean_rank": 14335.15234375, - "count": 5000.0, - "geometric_mean_rank": 4040.72412109375, - "harmonic_mean_rank": 38.37865187489459, - "hits_at_1": 0.0154, - "hits_at_10": 0.0448, - "hits_at_3": 0.0266, - "hits_at_5": 0.0326, - "inverse_arithmetic_mean_rank": 6.975858559599146e-05, - "inverse_geometric_mean_rank": 0.000247480405960232, - "inverse_harmonic_mean_rank": 0.026056152343750003, - "inverse_median_rank": 9.114108979701996e-05, - "median_absolute_deviation": 15384.963221432632, - "median_rank": 10972.0, - "standard_deviation": 13176.423828125, - "variance": 173618144.0, - "z_arithmetic_mean_rank": 36.676031022963286, - "z_geometric_mean_rank": 51.78634780858139, - "z_hits_at_k": 201.57284309986937, - "z_inverse_harmonic_mean_rank": 287.8299281051567 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6598165394018486, - "adjusted_arithmetic_mean_rank_index": 0.34020008515588696, - "adjusted_geometric_mean_rank_index": 0.7707405910086426, - "adjusted_hits_at_k": 0.05136820983964757, - "adjusted_inverse_harmonic_mean_rank": 0.02897454886629688, - "arithmetic_mean_rank": 13502.2926, - "count": 5000.0, - "geometric_mean_rank": 3453.363290191359, - "harmonic_mean_rank": 34.19948643516285, - "hits_at_1": 0.0184, - "hits_at_10": 0.0516, - "hits_at_3": 0.0278, - "hits_at_5": 0.035, - "inverse_arithmetic_mean_rank": 7.406149678610875e-05, - "inverse_geometric_mean_rank": 0.000289572777599251, - "inverse_harmonic_mean_rank": 0.029240205167871503, - "inverse_median_rank": 0.00010305559849538827, - "median_absolute_deviation": 13846.022118623816, - "median_rank": 9703.5, - "standard_deviation": 13040.049825970193, - "variance": 170042899.46378523, - "z_arithmetic_mean_rank": 41.664764240515254, - "z_geometric_mean_rank": 54.54770547603589, - "z_hits_at_k": 232.341991733733, - "z_inverse_harmonic_mean_rank": 323.3835557863843 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6598169205644525, - "adjusted_arithmetic_mean_rank_index": 0.3401997039746557, - "adjusted_geometric_mean_rank_index": 0.7707404745556267, - "adjusted_hits_at_k": 0.05136820983964757, - "adjusted_inverse_harmonic_mean_rank": 0.028974548819452866, - "arithmetic_mean_rank": 13502.3004, - "count": 5000.0, - "geometric_mean_rank": 3453.36504382982, - "harmonic_mean_rank": 34.199486489936845, - "hits_at_1": 0.0184, - "hits_at_10": 0.0516, - "hits_at_3": 0.0278, - "hits_at_5": 0.035, - "inverse_arithmetic_mean_rank": 7.406145400231208e-05, - "inverse_geometric_mean_rank": 0.0002895726305525433, - "inverse_harmonic_mean_rank": 0.029240205121040305, - "inverse_median_rank": 0.00010305559849538827, - "median_absolute_deviation": 13846.022118623816, - "median_rank": 9703.5, - "standard_deviation": 13040.054287055704, - "variance": 170043015.80935985, - "z_arithmetic_mean_rank": 41.66471755673467, - "z_geometric_mean_rank": 54.54769723429426, - "z_hits_at_k": 232.341991733733, - "z_inverse_harmonic_mean_rank": 323.3835552635605 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6598167005865956, - "adjusted_arithmetic_mean_rank_index": 0.34019992396326293, - "adjusted_geometric_mean_rank_index": 0.7707405105397687, - "adjusted_hits_at_k": 0.05136820983964757, - "adjusted_inverse_harmonic_mean_rank": 0.028974547555488634, - "arithmetic_mean_rank": 13502.2958984375, - "count": 5000.0, - "geometric_mean_rank": 3453.364501953125, - "harmonic_mean_rank": 34.199487967871185, - "hits_at_1": 0.0184, - "hits_at_10": 0.0516, - "hits_at_3": 0.0278, - "hits_at_5": 0.035, - "inverse_arithmetic_mean_rank": 7.406147778965533e-05, - "inverse_geometric_mean_rank": 0.0002895726647693664, - "inverse_harmonic_mean_rank": 0.029240203857421872, - "inverse_median_rank": 0.00010305560135748237, - "median_absolute_deviation": 13846.022118623816, - "median_rank": 9703.5, - "standard_deviation": 13040.0517578125, - "variance": 170042960.0, - "z_arithmetic_mean_rank": 41.66474449903673, - "z_geometric_mean_rank": 54.54769978100375, - "z_hits_at_k": 232.341991733733, - "z_inverse_harmonic_mean_rank": 323.38354115651606 - } - } - }, - "times": { - "evaluation": 3.979680061340332, - "training": 160.42055320739746 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json deleted file mode 100644 index feb9a6f..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18RR Dataset with ConvE as described by Dettmers et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json b/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json deleted file mode 100644 index d67452d..0000000 --- a/nonasv/results/27c9fe15/cuda/conve/wn18rr/dettmers2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 4096, - "slice_size": null - }, - "losses": [ - 0.00019061330150767425, - 4.929431331911384e-06, - 3.852578089436065e-06, - 3.58928681582103e-06, - 3.4748540160042394e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7533475301717399, - "adjusted_arithmetic_mean_rank_index": 0.24666463724982302, - "adjusted_geometric_mean_rank_index": 0.6438260194484903, - "adjusted_hits_at_k": 0.04046100837778324, - "adjusted_inverse_harmonic_mean_rank": 0.022035505980316127, - "arithmetic_mean_rank": 15272.273939808481, - "count": 5848.0, - "geometric_mean_rank": 5314.362994937839, - "harmonic_mean_rank": 44.83227908893811, - "hits_at_1": 0.013508891928864569, - "hits_at_10": 0.040697674418604654, - "hits_at_3": 0.022058823529411766, - "hits_at_5": 0.028214774281805747, - "inverse_arithmetic_mean_rank": 6.547813403172497e-05, - "inverse_geometric_mean_rank": 0.0001881693066417449, - "inverse_harmonic_mean_rank": 0.022305357218538983, - "inverse_median_rank": 7.543184732594101e-05, - "median_absolute_deviation": 16475.417153143502, - "median_rank": 13257.0, - "standard_deviation": 12660.061963371547, - "variance": 160277168.91640705, - "z_arithmetic_mean_rank": 32.67081918717673, - "z_geometric_mean_rank": 49.27749924271126, - "z_hits_at_k": 196.99271243043611, - "z_inverse_harmonic_mean_rank": 264.73240390539024 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7533477747862963, - "adjusted_arithmetic_mean_rank_index": 0.24666439262319972, - "adjusted_geometric_mean_rank_index": 0.6438258452392241, - "adjusted_hits_at_k": 0.04046100837778324, - "adjusted_inverse_harmonic_mean_rank": 0.022035505777574325, - "arithmetic_mean_rank": 15272.278898768809, - "count": 5848.0, - "geometric_mean_rank": 5314.365593771889, - "harmonic_mean_rank": 44.83227949632316, - "hits_at_1": 0.013508891928864569, - "hits_at_10": 0.040697674418604654, - "hits_at_3": 0.022058823529411766, - "hits_at_5": 0.028214774281805747, - "inverse_arithmetic_mean_rank": 6.54781127707546e-05, - "inverse_geometric_mean_rank": 0.0001881692146230848, - "inverse_harmonic_mean_rank": 0.022305357015853126, - "inverse_median_rank": 7.543184732594101e-05, - "median_absolute_deviation": 16475.417153143502, - "median_rank": 13257.0, - "standard_deviation": 12660.064268633454, - "variance": 160277227.28592953, - "z_arithmetic_mean_rank": 32.67078678629322, - "z_geometric_mean_rank": 49.27748590898953, - "z_hits_at_k": 196.99271243043611, - "z_inverse_harmonic_mean_rank": 264.7324014696702 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7533476499089838, - "adjusted_arithmetic_mean_rank_index": 0.2466645175066725, - "adjusted_geometric_mean_rank_index": 0.6438258365996382, - "adjusted_hits_at_k": 0.04046100837778324, - "adjusted_inverse_harmonic_mean_rank": 0.022035505581874164, - "arithmetic_mean_rank": 15272.2763671875, - "count": 5848.0, - "geometric_mean_rank": 5314.36572265625, - "harmonic_mean_rank": 44.8322798895589, - "hits_at_1": 0.013508891928864569, - "hits_at_10": 0.040697674418604654, - "hits_at_3": 0.022058823529411766, - "hits_at_5": 0.028214774281805747, - "inverse_arithmetic_mean_rank": 6.547812517965212e-05, - "inverse_geometric_mean_rank": 0.00018816921510733664, - "inverse_harmonic_mean_rank": 0.022305356820206963, - "inverse_median_rank": 7.543184619862586e-05, - "median_absolute_deviation": 16475.417153143502, - "median_rank": 13257.0, - "standard_deviation": 12660.0625, - "variance": 160277184.0, - "z_arithmetic_mean_rank": 32.67080332715375, - "z_geometric_mean_rank": 49.2774852477283, - "z_hits_at_k": 196.99271243043611, - "z_inverse_harmonic_mean_rank": 264.7323991185477 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8759271251000791, - "adjusted_arithmetic_mean_rank_index": 0.12407899680084766, - "adjusted_geometric_mean_rank_index": 0.40379021336988474, - "adjusted_hits_at_k": 0.026093526032368673, - "adjusted_inverse_harmonic_mean_rank": 0.015624904836206868, - "arithmetic_mean_rank": 17753.335157318743, - "count": 2924.0, - "geometric_mean_rank": 8893.970565775853, - "harmonic_mean_rank": 62.906612149750195, - "hits_at_1": 0.01094391244870041, - "hits_at_10": 0.026333789329685362, - "hits_at_3": 0.01744186046511628, - "hits_at_5": 0.02051983584131327, - "inverse_arithmetic_mean_rank": 5.6327444456978774e-05, - "inverse_geometric_mean_rank": 0.00011243572177403158, - "inverse_harmonic_mean_rank": 0.015896580118151715, - "inverse_median_rank": 5.901619994688542e-05, - "median_absolute_deviation": 16516.93001526166, - "median_rank": 16944.5, - "standard_deviation": 12453.794421336042, - "variance": 155096995.48890075, - "z_arithmetic_mean_rank": 11.62080349509435, - "z_geometric_mean_rank": 21.856266425692148, - "z_hits_at_k": 89.82201361985973, - "z_inverse_harmonic_mean_rank": 132.72052156271272 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8759273444583465, - "adjusted_arithmetic_mean_rank_index": 0.12407877743175677, - "adjusted_geometric_mean_rank_index": 0.40379006269602624, - "adjusted_hits_at_k": 0.026093526032368673, - "adjusted_inverse_harmonic_mean_rank": 0.015624904816403959, - "arithmetic_mean_rank": 17753.339603283173, - "count": 2924.0, - "geometric_mean_rank": 8893.972813203216, - "harmonic_mean_rank": 62.906612228093465, - "hits_at_1": 0.01094391244870041, - "hits_at_10": 0.026333789329685362, - "hits_at_3": 0.01744186046511628, - "hits_at_5": 0.02051983584131327, - "inverse_arithmetic_mean_rank": 5.6327430350910836e-05, - "inverse_geometric_mean_rank": 0.00011243569336253055, - "inverse_harmonic_mean_rank": 0.01589658009835427, - "inverse_median_rank": 5.901619994688542e-05, - "median_absolute_deviation": 16516.93001526166, - "median_rank": 16944.5, - "standard_deviation": 12453.796314877382, - "variance": 155097042.65245345, - "z_arithmetic_mean_rank": 11.620782949754982, - "z_geometric_mean_rank": 21.856258270051214, - "z_hits_at_k": 89.82201361985973, - "z_inverse_harmonic_mean_rank": 132.72052139450352 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8759272599579563, - "adjusted_arithmetic_mean_rank_index": 0.12407886193631645, - "adjusted_geometric_mean_rank_index": 0.40379000774711216, - "adjusted_hits_at_k": 0.026093526032368673, - "adjusted_inverse_harmonic_mean_rank": 0.015624902722154796, - "arithmetic_mean_rank": 17753.337890625, - "count": 2924.0, - "geometric_mean_rank": 8893.9736328125, - "harmonic_mean_rank": 62.90662051325779, - "hits_at_1": 0.01094391244870041, - "hits_at_10": 0.026333789329685362, - "hits_at_3": 0.01744186046511628, - "hits_at_5": 0.02051983584131327, - "inverse_arithmetic_mean_rank": 5.632743705064058e-05, - "inverse_geometric_mean_rank": 0.00011243568587815389, - "inverse_harmonic_mean_rank": 0.015896578004683092, - "inverse_median_rank": 5.901620170334354e-05, - "median_absolute_deviation": 16516.93001526166, - "median_rank": 16944.5, - "standard_deviation": 12453.7958984375, - "variance": 155097024.0, - "z_arithmetic_mean_rank": 11.620790864155552, - "z_geometric_mean_rank": 21.856255295788664, - "z_hits_at_k": 89.82201361985973, - "z_inverse_harmonic_mean_rank": 132.72050360560533 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6308223124201837, - "adjusted_arithmetic_mean_rank_index": 0.36919589515820417, - "adjusted_geometric_mean_rank_index": 0.7872698764693477, - "adjusted_hits_at_k": 0.054828489141919325, - "adjusted_inverse_harmonic_mean_rank": 0.0284461064058264, - "arithmetic_mean_rank": 12791.212722298222, - "count": 2924.0, - "geometric_mean_rank": 3175.460704878214, - "harmonic_mean_rank": 34.826054266273786, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.05506155950752394, - "hits_at_3": 0.02667578659370725, - "hits_at_5": 0.035909712722298225, - "inverse_arithmetic_mean_rank": 7.817867013162518e-05, - "inverse_geometric_mean_rank": 0.00031491493453651546, - "inverse_harmonic_mean_rank": 0.02871413431892625, - "inverse_median_rank": 0.00010870155986738409, - "median_absolute_deviation": 13220.363982414452, - "median_rank": 9199.5, - "standard_deviation": 12375.217687533675, - "variance": 153146012.81384635, - "z_arithmetic_mean_rank": 34.57763373270559, - "z_geometric_mean_rank": 42.6131565673655, - "z_hits_at_k": 188.77878751881445, - "z_inverse_harmonic_mean_rank": 241.679769823527 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6308225822798252, - "adjusted_arithmetic_mean_rank_index": 0.3691956252852534, - "adjusted_geometric_mean_rank_index": 0.7872697221168297, - "adjusted_hits_at_k": 0.054828489141919325, - "adjusted_inverse_harmonic_mean_rank": 0.028446106020145728, - "arithmetic_mean_rank": 12791.218194254447, - "count": 2924.0, - "geometric_mean_rank": 3175.463008200353, - "harmonic_mean_rank": 34.82605473391911, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.05506155950752394, - "hits_at_3": 0.02667578659370725, - "hits_at_5": 0.035909712722298225, - "inverse_arithmetic_mean_rank": 7.817863668756581e-05, - "inverse_geometric_mean_rank": 0.0003149147061129631, - "inverse_harmonic_mean_rank": 0.02871413393335198, - "inverse_median_rank": 0.00010870155986738409, - "median_absolute_deviation": 13220.363982414452, - "median_rank": 9199.5, - "standard_deviation": 12375.220704312107, - "variance": 153146087.48043504, - "z_arithmetic_mean_rank": 34.57760845732153, - "z_geometric_mean_rank": 42.613148212609126, - "z_hits_at_k": 188.77878751881445, - "z_inverse_harmonic_mean_rank": 241.67976654676164 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6308224652044876, - "adjusted_arithmetic_mean_rank_index": 0.3691957423663652, - "adjusted_geometric_mean_rank_index": 0.7872697463565012, - "adjusted_hits_at_k": 0.054828489141919325, - "adjusted_inverse_harmonic_mean_rank": 0.02844610641801832, - "arithmetic_mean_rank": 12791.2158203125, - "count": 2924.0, - "geometric_mean_rank": 3175.462646484375, - "harmonic_mean_rank": 34.826054251490845, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.05506155950752394, - "hits_at_3": 0.02667578659370725, - "hits_at_5": 0.035909712722298225, - "inverse_arithmetic_mean_rank": 7.817865116521716e-05, - "inverse_geometric_mean_rank": 0.0003149147378280759, - "inverse_harmonic_mean_rank": 0.028714134331114805, - "inverse_median_rank": 0.00010870156256714836, - "median_absolute_deviation": 13220.363982414452, - "median_rank": 9199.5, - "standard_deviation": 12375.21875, - "variance": 153146048.0, - "z_arithmetic_mean_rank": 34.57761942274084, - "z_geometric_mean_rank": 42.6131495246483, - "z_hits_at_k": 188.77878751881445, - "z_inverse_harmonic_mean_rank": 241.67976992711024 - } - } - }, - "times": { - "evaluation": 2.390557289123535, - "training": 96.70088291168213 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json deleted file mode 100644 index 68117f3..0000000 --- a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json deleted file mode 100644 index b66b703..0000000 --- a/nonasv/results/27c9fe15/cuda/convkb/fb15k237/nguyen2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 32, - "slice_size": null - }, - "losses": [ - 0.0027138416286498156, - 0.002708068411666065, - 0.00270294341827581, - 0.0026974449028830463, - 0.0026920965323165968 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9022145800402283, - "adjusted_arithmetic_mean_rank_index": 0.09779912563002002, - "adjusted_geometric_mean_rank_index": 0.3132169673715255, - "adjusted_hits_at_k": 0.025222428209433437, - "adjusted_inverse_harmonic_mean_rank": 0.017172625662514082, - "arithmetic_mean_rank": 6437.904565025932, - "count": 40876.0, - "geometric_mean_rank": 3602.1284793223763, - "harmonic_mean_rank": 55.950214649740836, - "hits_at_1": 0.012550151678246404, - "hits_at_10": 0.025907623055093452, - "hits_at_3": 0.02030531363147079, - "hits_at_5": 0.022971915060182015, - "inverse_arithmetic_mean_rank": 0.0001553300441004552, - "inverse_geometric_mean_rank": 0.0002776136403074989, - "inverse_harmonic_mean_rank": 0.017873032413909282, - "inverse_median_rank": 0.00016079755587715066, - "median_absolute_deviation": 5565.68872827003, - "median_rank": 6219.0, - "standard_deviation": 4305.144839101484, - "variance": 18534272.085642144, - "z_arithmetic_mean_rank": 34.203630631700776, - "z_geometric_mean_rank": 63.43454801929772, - "z_hits_at_k": 192.27141470503148, - "z_inverse_harmonic_mean_rank": 323.370932509128 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9022146588944141, - "adjusted_arithmetic_mean_rank_index": 0.09779904676478202, - "adjusted_geometric_mean_rank_index": 0.3132168544257061, - "adjusted_hits_at_k": 0.025222428209433437, - "adjusted_inverse_harmonic_mean_rank": 0.01717262556907924, - "arithmetic_mean_rank": 6437.905127703298, - "count": 40876.0, - "geometric_mean_rank": 3602.129071550762, - "harmonic_mean_rank": 55.950214942023315, - "hits_at_1": 0.012550151678246404, - "hits_at_10": 0.025907623055093452, - "hits_at_3": 0.02030531363147079, - "hits_at_5": 0.022971915060182015, - "inverse_arithmetic_mean_rank": 0.0001553300305245018, - "inverse_geometric_mean_rank": 0.0002776135946648595, - "inverse_harmonic_mean_rank": 0.017873032320541025, - "inverse_median_rank": 0.00016079755587715066, - "median_absolute_deviation": 5565.68872827003, - "median_rank": 6219.0, - "standard_deviation": 4305.144722351568, - "variance": 18534271.080391556, - "z_arithmetic_mean_rank": 34.20360304988492, - "z_geometric_mean_rank": 63.43452514484409, - "z_hits_at_k": 192.27141470503148, - "z_inverse_harmonic_mean_rank": 323.3709307496934 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9022145424612803, - "adjusted_arithmetic_mean_rank_index": 0.0977991632142351, - "adjusted_geometric_mean_rank_index": 0.3132167462681771, - "adjusted_hits_at_k": 0.025222428209433437, - "adjusted_inverse_harmonic_mean_rank": 0.01717262544251896, - "arithmetic_mean_rank": 6437.904296875, - "count": 40876.0, - "geometric_mean_rank": 3602.129638671875, - "harmonic_mean_rank": 55.950215337928626, - "hits_at_1": 0.012550151678246404, - "hits_at_10": 0.025907623055093452, - "hits_at_3": 0.02030531363147079, - "hits_at_5": 0.022971915060182015, - "inverse_arithmetic_mean_rank": 0.00015533005353063345, - "inverse_geometric_mean_rank": 0.000277613551588729, - "inverse_harmonic_mean_rank": 0.01787303219407094, - "inverse_median_rank": 0.00016079755732789636, - "median_absolute_deviation": 5565.68872827003, - "median_rank": 6219.0, - "standard_deviation": 4305.14501953125, - "variance": 18534272.0, - "z_arithmetic_mean_rank": 34.20364377615988, - "z_geometric_mean_rank": 63.43450324014324, - "z_hits_at_k": 192.27141470503148, - "z_inverse_harmonic_mean_rank": 323.3709283664868 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9861542601103468, - "adjusted_arithmetic_mean_rank_index": 0.013847707833329004, - "adjusted_geometric_mean_rank_index": 0.023761321767014065, - "adjusted_hits_at_k": 0.00011720820855638033, - "adjusted_inverse_harmonic_mean_rank": 0.00011582849847661622, - "arithmetic_mean_rank": 6939.210832762501, - "count": 20438.0, - "geometric_mean_rank": 5042.814960457132, - "harmonic_mean_rank": 1191.8958402433716, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0008317839318915745, - "hits_at_3": 0.0002446423329092866, - "hits_at_5": 0.0004403561992367159, - "inverse_arithmetic_mean_rank": 0.00014410860602168788, - "inverse_geometric_mean_rank": 0.00019830194203860891, - "inverse_harmonic_mean_rank": 0.0008389994882403578, - "inverse_median_rank": 0.00014624159110851126, - "median_absolute_deviation": 5238.774939089544, - "median_rank": 6838.0, - "standard_deviation": 4115.020282699011, - "variance": 16933391.92702425, - "z_arithmetic_mean_rank": 3.4208852603670437, - "z_geometric_mean_rank": 3.4029652710831897, - "z_hits_at_k": 0.6265756294420397, - "z_inverse_harmonic_mean_rank": 1.529607327750341 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.986154364411146, - "adjusted_arithmetic_mean_rank_index": 0.013847603517705087, - "adjusted_geometric_mean_rank_index": 0.023761114601180355, - "adjusted_hits_at_k": 0.00011720820855638033, - "adjusted_inverse_harmonic_mean_rank": 0.00011582837630638547, - "arithmetic_mean_rank": 6939.2115666895, - "count": 20438.0, - "geometric_mean_rank": 5042.816030371511, - "harmonic_mean_rank": 1191.896013674818, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0008317839318915745, - "hits_at_3": 0.0002446423329092866, - "hits_at_5": 0.0004403561992367159, - "inverse_arithmetic_mean_rank": 0.00014410859078001445, - "inverse_geometric_mean_rank": 0.00019830189996566832, - "inverse_harmonic_mean_rank": 0.0008389993661584872, - "inverse_median_rank": 0.00014624159110851126, - "median_absolute_deviation": 5238.774939089544, - "median_rank": 6838.0, - "standard_deviation": 4115.0200287993475, - "variance": 16933389.837419786, - "z_arithmetic_mean_rank": 3.4208594906306473, - "z_geometric_mean_rank": 3.4029356019366617, - "z_hits_at_k": 0.6265756294420397, - "z_inverse_harmonic_mean_rank": 1.5296057143953998 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9861543443862074, - "adjusted_arithmetic_mean_rank_index": 0.013847623545489851, - "adjusted_geometric_mean_rank_index": 0.023761420000934508, - "adjusted_hits_at_k": 0.00011720820855638033, - "adjusted_inverse_harmonic_mean_rank": 0.00011582844626088513, - "arithmetic_mean_rank": 6939.21142578125, - "count": 20438.0, - "geometric_mean_rank": 5042.814453125, - "harmonic_mean_rank": 1191.8959143682134, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0008317839318915745, - "hits_at_3": 0.0002446423329092866, - "hits_at_5": 0.0004403561992367159, - "inverse_arithmetic_mean_rank": 0.00014410859148483723, - "inverse_geometric_mean_rank": 0.00019830196106340736, - "inverse_harmonic_mean_rank": 0.0008389994360623919, - "inverse_median_rank": 0.00014624159666709602, - "median_absolute_deviation": 5238.774939089544, - "median_rank": 6838.0, - "standard_deviation": 4115.02001953125, - "variance": 16933390.0, - "z_arithmetic_mean_rank": 3.4208644382187003, - "z_geometric_mean_rank": 3.4029793396027515, - "z_hits_at_k": 0.6265756294420397, - "z_inverse_harmonic_mean_rank": 1.5296066382001356 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8205728810454175, - "adjusted_arithmetic_mean_rank_index": 0.17945192329106108, - "adjusted_geometric_mean_rank_index": 0.5168758436756876, - "adjusted_hits_at_k": 0.050327058574015544, - "adjusted_inverse_harmonic_mean_rank": 0.034229060625992506, - "arithmetic_mean_rank": 5936.598297289363, - "count": 20438.0, - "geometric_mean_rank": 2573.033054611455, - "harmonic_mean_rank": 28.647495579245472, - "hits_at_1": 0.024953517956747235, - "hits_at_10": 0.05098346217829533, - "hits_at_3": 0.040365984930032295, - "hits_at_5": 0.04550347392112731, - "inverse_arithmetic_mean_rank": 0.00016844663390086503, - "inverse_geometric_mean_rank": 0.0003886463868809515, - "inverse_harmonic_mean_rank": 0.03490706533957822, - "inverse_median_rank": 0.00017921146953405018, - "median_absolute_deviation": 5862.20917197115, - "median_rank": 5580.0, - "standard_deviation": 4430.861800611265, - "variance": 19632536.296116095, - "z_arithmetic_mean_rank": 44.43135166020866, - "z_geometric_mean_rank": 74.02083142469466, - "z_hits_at_k": 273.57232645173275, - "z_inverse_harmonic_mean_rank": 459.60952979376 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8205729351496311, - "adjusted_arithmetic_mean_rank_index": 0.17945186917936795, - "adjusted_geometric_mean_rank_index": 0.5168757872948017, - "adjusted_hits_at_k": 0.050327058574015544, - "adjusted_inverse_harmonic_mean_rank": 0.03422906056129243, - "arithmetic_mean_rank": 5936.598688717096, - "count": 20438.0, - "geometric_mean_rank": 2573.0333547692935, - "harmonic_mean_rank": 28.647495632306196, - "hits_at_1": 0.024953517956747235, - "hits_at_10": 0.05098346217829533, - "hits_at_3": 0.040365984930032295, - "hits_at_5": 0.04550347392112731, - "inverse_arithmetic_mean_rank": 0.0001684466227943902, - "inverse_geometric_mean_rank": 0.00038864634154331173, - "inverse_harmonic_mean_rank": 0.034907065274923564, - "inverse_median_rank": 0.00017921146953405018, - "median_absolute_deviation": 5862.20917197115, - "median_rank": 5580.0, - "standard_deviation": 4430.861770787402, - "variance": 19632536.031825267, - "z_arithmetic_mean_rank": 44.43133826243826, - "z_geometric_mean_rank": 74.02082335049249, - "z_hits_at_k": 273.57232645173275, - "z_inverse_harmonic_mean_rank": 459.60952892500194 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8205729274223448, - "adjusted_arithmetic_mean_rank_index": 0.17945187690772257, - "adjusted_geometric_mean_rank_index": 0.5168757240617707, - "adjusted_hits_at_k": 0.050327058574015544, - "adjusted_inverse_harmonic_mean_rank": 0.034229060705167576, - "arithmetic_mean_rank": 5936.5986328125, - "count": 20438.0, - "geometric_mean_rank": 2573.03369140625, - "harmonic_mean_rank": 28.64749551431377, - "hits_at_1": 0.024953517956747235, - "hits_at_10": 0.05098346217829533, - "hits_at_3": 0.040365984930032295, - "hits_at_5": 0.04550347392112731, - "inverse_arithmetic_mean_rank": 0.0001684466260485351, - "inverse_geometric_mean_rank": 0.0003886462945956737, - "inverse_harmonic_mean_rank": 0.03490706541869771, - "inverse_median_rank": 0.0001792114635463804, - "median_absolute_deviation": 5862.20917197115, - "median_rank": 5580.0, - "standard_deviation": 4430.86181640625, - "variance": 19632536.0, - "z_arithmetic_mean_rank": 44.431340175938175, - "z_geometric_mean_rank": 74.02081429500736, - "z_hits_at_k": 273.57232645173275, - "z_inverse_harmonic_mean_rank": 459.60953085688055 - } - } - }, - "times": { - "evaluation": 113.86807107925415, - "training": 28.116236209869385 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json deleted file mode 100644 index eab649f..0000000 --- a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18RR Dataset with ConvKB as described by Nguyen et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json deleted file mode 100644 index 840c2db..0000000 --- a/nonasv/results/27c9fe15/cuda/convkb/wn18rr/nguyen2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 4, - "slice_size": null - }, - "losses": [ - 0.002721442517725209, - 0.002717859683797624, - 0.0027141772589411987, - 0.0027092453467933092, - 0.00270177292591784 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9368993007191023, - "adjusted_arithmetic_mean_rank_index": 0.06310381205252158, - "adjusted_geometric_mean_rank_index": 0.17085137097247383, - "adjusted_hits_at_k": 0.005397640591231649, - "adjusted_inverse_harmonic_mean_rank": 0.003779893635427336, - "arithmetic_mean_rank": 18993.336012311902, - "count": 5848.0, - "geometric_mean_rank": 12370.145090151156, - "harmonic_mean_rank": 246.62237264877194, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.005642954856361149, - "hits_at_3": 0.00393296853625171, - "hits_at_5": 0.004445964432284542, - "inverse_arithmetic_mean_rank": 5.265004522384997e-05, - "inverse_geometric_mean_rank": 8.083979554905775e-05, - "inverse_harmonic_mean_rank": 0.004054782172678848, - "inverse_median_rank": 5.4449919686368463e-05, - "median_absolute_deviation": 15544.342959921983, - "median_rank": 18365.5, - "standard_deviation": 12003.289451468483, - "variance": 144078957.65573457, - "z_arithmetic_mean_rank": 8.358122415015941, - "z_geometric_mean_rank": 13.07671335017524, - "z_hits_at_k": 26.279519552834255, - "z_inverse_harmonic_mean_rank": 45.411270769422465 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9368993260240565, - "adjusted_arithmetic_mean_rank_index": 0.06310378674631922, - "adjusted_geometric_mean_rank_index": 0.17085134895810905, - "adjusted_hits_at_k": 0.005397640591231649, - "adjusted_inverse_harmonic_mean_rank": 0.0037798936340233966, - "arithmetic_mean_rank": 18993.336525307797, - "count": 5848.0, - "geometric_mean_rank": 12370.14541855895, - "harmonic_mean_rank": 246.62237273413962, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.005642954856361149, - "hits_at_3": 0.00393296853625171, - "hits_at_5": 0.004445964432284542, - "inverse_arithmetic_mean_rank": 5.2650043801811407e-05, - "inverse_geometric_mean_rank": 8.083979340288905e-05, - "inverse_harmonic_mean_rank": 0.004054782171275296, - "inverse_median_rank": 5.4449919686368463e-05, - "median_absolute_deviation": 15544.342959921983, - "median_rank": 18365.5, - "standard_deviation": 12003.289482275293, - "variance": 144078958.39530066, - "z_arithmetic_mean_rank": 8.358119063200405, - "z_geometric_mean_rank": 13.076711665228046, - "z_hits_at_k": 26.279519552834255, - "z_inverse_harmonic_mean_rank": 45.41127075255568 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9368992970287965, - "adjusted_arithmetic_mean_rank_index": 0.0631038157430095, - "adjusted_geometric_mean_rank_index": 0.170851277512596, - "adjusted_hits_at_k": 0.005397640591231649, - "adjusted_inverse_harmonic_mean_rank": 0.0037798933020507103, - "arithmetic_mean_rank": 18993.3359375, - "count": 5848.0, - "geometric_mean_rank": 12370.146484375, - "harmonic_mean_rank": 246.62239292001001, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.005642954856361149, - "hits_at_3": 0.00393296853625171, - "hits_at_5": 0.004445964432284542, - "inverse_arithmetic_mean_rank": 5.265004438115284e-05, - "inverse_geometric_mean_rank": 8.083978900685906e-05, - "inverse_harmonic_mean_rank": 0.004054781839394211, - "inverse_median_rank": 5.4449919844046235e-05, - "median_absolute_deviation": 15544.342959921983, - "median_rank": 18365.5, - "standard_deviation": 12003.2900390625, - "variance": 144078976.0, - "z_arithmetic_mean_rank": 8.358122903822377, - "z_geometric_mean_rank": 13.07670619689327, - "z_hits_at_k": 26.279519552834255, - "z_inverse_harmonic_mean_rank": 45.41126676426843 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9748546833736785, - "adjusted_arithmetic_mean_rank_index": 0.025146557325686825, - "adjusted_geometric_mean_rank_index": 0.08085733565234698, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.004117111631119762, - "arithmetic_mean_rank": 19758.40389876881, - "count": 2924.0, - "geometric_mean_rank": 13710.786124097143, - "harmonic_mean_rank": 227.68862534181577, - "hits_at_1": 0.0030779753761969904, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.005129958960328317, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.0611375550547997e-05, - "inverse_geometric_mean_rank": 7.293527817799361e-05, - "inverse_harmonic_mean_rank": 0.004391962920847529, - "inverse_median_rank": 5.133074968559916e-05, - "median_absolute_deviation": 15015.795269024737, - "median_rank": 19481.5, - "standard_deviation": 11717.673398597328, - "variance": 137303869.87619546, - "z_arithmetic_mean_rank": 2.355138329567282, - "z_geometric_mean_rank": 4.376627793280527, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 34.97142598577112 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9748547339948171, - "adjusted_arithmetic_mean_rank_index": 0.02514650670205054, - "adjusted_geometric_mean_rank_index": 0.08085728684521754, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.004117111628311883, - "arithmetic_mean_rank": 19758.4049247606, - "count": 2924.0, - "geometric_mean_rank": 13710.786852096542, - "harmonic_mean_rank": 227.688625487342, - "hits_at_1": 0.0030779753761969904, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.005129958960328317, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.061137292245853e-05, - "inverse_geometric_mean_rank": 7.293527430536112e-05, - "inverse_harmonic_mean_rank": 0.004391962918040425, - "inverse_median_rank": 5.133074968559916e-05, - "median_absolute_deviation": 15015.795269024737, - "median_rank": 19481.5, - "standard_deviation": 11717.6733947239, - "variance": 137303869.78542036, - "z_arithmetic_mean_rank": 2.3551335883351356, - "z_geometric_mean_rank": 4.37662515145913, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 34.97142596192053 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.974854703015735, - "adjusted_arithmetic_mean_rank_index": 0.025146537682661285, - "adjusted_geometric_mean_rank_index": 0.0808574005395446, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.004117111497430378, - "arithmetic_mean_rank": 19758.404296875, - "count": 2924.0, - "geometric_mean_rank": 13710.78515625, - "harmonic_mean_rank": 227.68863227064298, - "hits_at_1": 0.0030779753761969904, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.005129958960328317, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.061137562734075e-05, - "inverse_geometric_mean_rank": 7.293528324225917e-05, - "inverse_harmonic_mean_rank": 0.004391962787195041, - "inverse_median_rank": 5.1330749556655064e-05, - "median_absolute_deviation": 15015.795269024737, - "median_rank": 19481.5, - "standard_deviation": 11717.673828125, - "variance": 137303888.0, - "z_arithmetic_mean_rank": 2.355136489870432, - "z_geometric_mean_rank": 4.3766313054802595, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 34.97142485019141 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8989607553411842, - "adjusted_arithmetic_mean_rank_index": 0.10104422784179035, - "adjusted_geometric_mean_rank_index": 0.2521623042350343, - "adjusted_hits_at_k": 0.005226654514911831, - "adjusted_inverse_harmonic_mean_rank": 0.0034426756775355026, - "arithmetic_mean_rank": 18228.268125854993, - "count": 2924.0, - "geometric_mean_rank": 11160.591972363447, - "harmonic_mean_rank": 268.99064364646364, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.005471956224350205, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.003761969904240766, - "inverse_arithmetic_mean_rank": 5.485984697479839e-05, - "inverse_geometric_mean_rank": 8.960098196191226e-05, - "inverse_harmonic_mean_rank": 0.003717601424510167, - "inverse_median_rank": 5.9044076403034865e-05, - "median_absolute_deviation": 15788.231024866154, - "median_rank": 16936.5, - "standard_deviation": 12234.51624272486, - "variance": 149683387.6934984, - "z_arithmetic_mean_rank": 9.463459228386938, - "z_geometric_mean_rank": 13.648981209524969, - "z_hits_at_k": 17.99578134554909, - "z_inverse_harmonic_mean_rank": 29.24917222250908 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8989607553411842, - "adjusted_arithmetic_mean_rank_index": 0.10104422784179035, - "adjusted_geometric_mean_rank_index": 0.2521623042350343, - "adjusted_hits_at_k": 0.005226654514911831, - "adjusted_inverse_harmonic_mean_rank": 0.0034426756775355026, - "arithmetic_mean_rank": 18228.268125854993, - "count": 2924.0, - "geometric_mean_rank": 11160.591972363447, - "harmonic_mean_rank": 268.99064364646364, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.005471956224350205, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.003761969904240766, - "inverse_arithmetic_mean_rank": 5.485984697479839e-05, - "inverse_geometric_mean_rank": 8.960098196191226e-05, - "inverse_harmonic_mean_rank": 0.003717601424510167, - "inverse_median_rank": 5.9044076403034865e-05, - "median_absolute_deviation": 15788.231024866154, - "median_rank": 16936.5, - "standard_deviation": 12234.51624272486, - "variance": 149683387.6934984, - "z_arithmetic_mean_rank": 9.463459228386938, - "z_geometric_mean_rank": 13.648981209524969, - "z_hits_at_k": 17.99578134554909, - "z_inverse_harmonic_mean_rank": 29.24917222250908 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8989607283288666, - "adjusted_arithmetic_mean_rank_index": 0.10104425485544011, - "adjusted_geometric_mean_rank_index": 0.2521627740916954, - "adjusted_hits_at_k": 0.005226654514911831, - "adjusted_inverse_harmonic_mean_rank": 0.0034426757969596754, - "arithmetic_mean_rank": 18228.267578125, - "count": 2924.0, - "geometric_mean_rank": 11160.5849609375, - "harmonic_mean_rank": 268.99063500779636, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.005471956224350205, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.003761969904240766, - "inverse_arithmetic_mean_rank": 5.485984729602933e-05, - "inverse_geometric_mean_rank": 8.960103878052905e-05, - "inverse_harmonic_mean_rank": 0.0037176015439013936, - "inverse_median_rank": 5.904407589696348e-05, - "median_absolute_deviation": 15788.231024866154, - "median_rank": 16936.5, - "standard_deviation": 12234.5166015625, - "variance": 149683392.0, - "z_arithmetic_mean_rank": 9.463461758393635, - "z_geometric_mean_rank": 13.649006641814537, - "z_hits_at_k": 17.99578134554909, - "z_inverse_harmonic_mean_rank": 29.249173237143776 - } - } - }, - "times": { - "evaluation": 229.65697264671326, - "training": 11.373437881469727 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json deleted file mode 100644 index bd530c9..0000000 --- a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "For each positive triple, DistMult creates two negatives where head and tail are corrupted", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with DistMult as described by Yang et al., 2014" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json b/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json deleted file mode 100644 index fa61601..0000000 --- a/nonasv/results/27c9fe15/cuda/distmult/fb15k/yang2014/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 1.9813735472315636e-05, - 1.0421016274152843e-05, - 6.530047764306711e-06, - 4.6684656685330165e-06, - 3.782523409532725e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.04563974028451357, - "adjusted_arithmetic_mean_rank_index": 0.9544894743555506, - "adjusted_geometric_mean_rank_index": 0.9927695844264502, - "adjusted_hits_at_k": 0.3147999080140758, - "adjusted_inverse_harmonic_mean_rank": 0.18212225677789062, - "arithmetic_mean_rank": 337.134025156168, - "count": 118142.0, - "geometric_mean_rank": 40.26556227491919, - "harmonic_mean_rank": 5.473851925109878, - "hits_at_1": 0.11285571600277632, - "hits_at_10": 0.31526468148499265, - "hits_at_3": 0.19793130300824432, - "hits_at_5": 0.24297032384757325, - "inverse_arithmetic_mean_rank": 0.0029661793986435446, - "inverse_geometric_mean_rank": 0.02483511823757357, - "inverse_harmonic_mean_rank": 0.18268671014149268, - "inverse_median_rank": 0.023809523809523808, - "median_absolute_deviation": 60.78669095872968, - "median_rank": 42.0, - "standard_deviation": 889.735761711197, - "variance": 791629.7256678039, - "z_arithmetic_mean_rank": 567.7468447904905, - "z_geometric_mean_rank": 341.824729522192, - "z_hits_at_k": 4153.1520985060315, - "z_inverse_harmonic_mean_rank": 5934.9765930662215 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.04563978153592007, - "adjusted_arithmetic_mean_rank_index": 0.9544894330985589, - "adjusted_geometric_mean_rank_index": 0.9927695715296923, - "adjusted_hits_at_k": 0.3147999080140758, - "adjusted_inverse_harmonic_mean_rank": 0.18212213620472156, - "arithmetic_mean_rank": 337.13432987421913, - "count": 118142.0, - "geometric_mean_rank": 40.265632312173224, - "harmonic_mean_rank": 5.473855535359432, - "hits_at_1": 0.11285571600277632, - "hits_at_10": 0.31526468148499265, - "hits_at_3": 0.19793130300824432, - "hits_at_5": 0.24297032384757325, - "inverse_arithmetic_mean_rank": 0.0029661767176694474, - "inverse_geometric_mean_rank": 0.0248350750398542, - "inverse_harmonic_mean_rank": 0.18268658965153647, - "inverse_median_rank": 0.023809523809523808, - "median_absolute_deviation": 60.78669095872968, - "median_rank": 42.0, - "standard_deviation": 889.7358922933918, - "variance": 791629.958035118, - "z_arithmetic_mean_rank": 567.7468202501184, - "z_geometric_mean_rank": 341.82472508165426, - "z_hits_at_k": 4153.1520985060315, - "z_inverse_harmonic_mean_rank": 5934.972663843354 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.045639757899224476, - "adjusted_arithmetic_mean_rank_index": 0.9544894567384548, - "adjusted_geometric_mean_rank_index": 0.9927695770908282, - "adjusted_hits_at_k": 0.3147999080140758, - "adjusted_inverse_harmonic_mean_rank": 0.18212220161411308, - "arithmetic_mean_rank": 337.1341552734375, - "count": 118142.0, - "geometric_mean_rank": 40.265602111816406, - "harmonic_mean_rank": 5.47385357684495, - "hits_at_1": 0.11285571600277632, - "hits_at_10": 0.31526468148499265, - "hits_at_3": 0.19793130300824432, - "hits_at_5": 0.24297032384757325, - "inverse_arithmetic_mean_rank": 0.00296617834828794, - "inverse_geometric_mean_rank": 0.024835092946887016, - "inverse_harmonic_mean_rank": 0.18268665501578607, - "inverse_median_rank": 0.02380952425301075, - "median_absolute_deviation": 60.78669095872968, - "median_rank": 42.0, - "standard_deviation": 889.73583984375, - "variance": 791629.875, - "z_arithmetic_mean_rank": 567.7468343115373, - "z_geometric_mean_rank": 341.82472699643273, - "z_hits_at_k": 4153.1520985060315, - "z_inverse_harmonic_mean_rank": 5934.9747953961605 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0492053071915437, - "adjusted_arithmetic_mean_rank_index": 0.950923776041295, - "adjusted_geometric_mean_rank_index": 0.9914887878798211, - "adjusted_hits_at_k": 0.28574585583230167, - "adjusted_inverse_harmonic_mean_rank": 0.1620356021690724, - "arithmetic_mean_rank": 362.48314739889287, - "count": 59071.0, - "geometric_mean_rank": 47.08810772419198, - "harmonic_mean_rank": 6.149475038945622, - "hits_at_1": 0.09717120075840938, - "hits_at_10": 0.28623182272180936, - "hits_at_3": 0.1740786511147602, - "hits_at_5": 0.21628210120025054, - "inverse_arithmetic_mean_rank": 0.0027587489437117326, - "inverse_geometric_mean_rank": 0.021236784579607138, - "inverse_harmonic_mean_rank": 0.16261550679803366, - "inverse_median_rank": 0.02, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 50.0, - "standard_deviation": 902.9855077973994, - "variance": 815382.8272921271, - "z_arithmetic_mean_rank": 399.89996701532084, - "z_geometric_mean_rank": 241.37804208791823, - "z_hits_at_k": 2661.6000279583345, - "z_inverse_harmonic_mean_rank": 3728.105578345422 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.049205341661526236, - "adjusted_arithmetic_mean_rank_index": 0.9509237415666327, - "adjusted_geometric_mean_rank_index": 0.9914887809523989, - "adjusted_hits_at_k": 0.28574585583230167, - "adjusted_inverse_harmonic_mean_rank": 0.16203559831201408, - "arithmetic_mean_rank": 362.48340133060213, - "count": 59071.0, - "geometric_mean_rank": 47.088145236097084, - "harmonic_mean_rank": 6.149475184703368, - "hits_at_1": 0.09717120075840938, - "hits_at_10": 0.28623182272180936, - "hits_at_3": 0.1740786511147602, - "hits_at_5": 0.21628210120025054, - "inverse_arithmetic_mean_rank": 0.00275874701111611, - "inverse_geometric_mean_rank": 0.021236767661713177, - "inverse_harmonic_mean_rank": 0.1626155029436446, - "inverse_median_rank": 0.02, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 50.0, - "standard_deviation": 902.9855805051617, - "variance": 815382.9586002439, - "z_arithmetic_mean_rank": 399.89995251740135, - "z_geometric_mean_rank": 241.37804040143664, - "z_hits_at_k": 2661.6000279583345, - "z_inverse_harmonic_mean_rank": 3728.1054896024525 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.04920532469835637, - "adjusted_arithmetic_mean_rank_index": 0.9509237585321055, - "adjusted_geometric_mean_rank_index": 0.9914887835904787, - "adjusted_hits_at_k": 0.28574585583230167, - "adjusted_inverse_harmonic_mean_rank": 0.16203561446886694, - "arithmetic_mean_rank": 362.4832763671875, - "count": 59071.0, - "geometric_mean_rank": 47.088130950927734, - "harmonic_mean_rank": 6.149474574137982, - "hits_at_1": 0.09717120075840938, - "hits_at_10": 0.28623182272180936, - "hits_at_3": 0.1740786511147602, - "hits_at_5": 0.21628210120025054, - "inverse_arithmetic_mean_rank": 0.002758747898042202, - "inverse_geometric_mean_rank": 0.02123677358031273, - "inverse_harmonic_mean_rank": 0.16261551908931626, - "inverse_median_rank": 0.019999999552965164, - "median_absolute_deviation": 71.1649064882689, - "median_rank": 50.0, - "standard_deviation": 902.9854736328125, - "variance": 815382.8125, - "z_arithmetic_mean_rank": 399.8999596520342, - "z_geometric_mean_rank": 241.3780410436774, - "z_hits_at_k": 2661.6000279583345, - "z_inverse_harmonic_mean_rank": 3728.10586133837 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.0420935281672918, - "adjusted_arithmetic_mean_rank_index": 0.9580358145531809, - "adjusted_geometric_mean_rank_index": 0.9938616322089793, - "adjusted_hits_at_k": 0.3438538392238848, - "adjusted_inverse_harmonic_mean_rank": 0.2022088351751535, - "arithmetic_mean_rank": 311.78490291344315, - "count": 59071.0, - "geometric_mean_rank": 34.43152812195992, - "harmonic_mean_rank": 4.931989991474332, - "hits_at_1": 0.12854023124714326, - "hits_at_10": 0.34429754024817594, - "hits_at_3": 0.22178395490172842, - "hits_at_5": 0.26965854649489596, - "inverse_arithmetic_mean_rank": 0.003207339388968481, - "inverse_geometric_mean_rank": 0.029043148955163994, - "inverse_harmonic_mean_rank": 0.20275791348495165, - "inverse_median_rank": 0.029411764705882353, - "median_absolute_deviation": 48.92587321068486, - "median_rank": 34.0, - "standard_deviation": 875.5520932797359, - "variance": 766591.4680465274, - "z_arithmetic_mean_rank": 403.00997549321005, - "z_geometric_mean_rank": 241.95374550665005, - "z_hits_at_k": 3212.6953808476746, - "z_inverse_harmonic_mean_rank": 4666.662332815998 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.042093576163310976, - "adjusted_arithmetic_mean_rank_index": 0.9580357665506809, - "adjusted_geometric_mean_rank_index": 0.9938616152525771, - "adjusted_hits_at_k": 0.3438538392238848, - "adjusted_inverse_harmonic_mean_rank": 0.20220859788631668, - "arithmetic_mean_rank": 311.7852584178362, - "count": 59071.0, - "geometric_mean_rank": 34.43162047198766, - "harmonic_mean_rank": 4.9319957594468615, - "hits_at_1": 0.12854023124714326, - "hits_at_10": 0.34429754024817594, - "hits_at_3": 0.22178395490172842, - "hits_at_5": 0.26965854649489596, - "inverse_arithmetic_mean_rank": 0.0032073357318897324, - "inverse_geometric_mean_rank": 0.02904307105770884, - "inverse_harmonic_mean_rank": 0.2027576763594284, - "inverse_median_rank": 0.029411764705882353, - "median_absolute_deviation": 48.92587321068486, - "median_rank": 34.0, - "standard_deviation": 875.5522866297537, - "variance": 766591.8066225903, - "z_arithmetic_mean_rank": 403.00995530034663, - "z_geometric_mean_rank": 241.95374137864584, - "z_hits_at_k": 3212.6953808476746, - "z_inverse_harmonic_mean_rank": 4666.656856562322 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.04209355000943946, - "adjusted_arithmetic_mean_rank_index": 0.9580357927080839, - "adjusted_geometric_mean_rank_index": 0.9938616248602049, - "adjusted_hits_at_k": 0.3438538392238848, - "adjusted_inverse_harmonic_mean_rank": 0.20220871254805994, - "arithmetic_mean_rank": 311.7850646972656, - "count": 59071.0, - "geometric_mean_rank": 34.43156814575195, - "harmonic_mean_rank": 4.9319929722690325, - "hits_at_1": 0.12854023124714326, - "hits_at_10": 0.34429754024817594, - "hits_at_3": 0.22178395490172842, - "hits_at_5": 0.26965854649489596, - "inverse_arithmetic_mean_rank": 0.003207337809726596, - "inverse_geometric_mean_rank": 0.02904311567544937, - "inverse_harmonic_mean_rank": 0.20275779094225593, - "inverse_median_rank": 0.029411764815449715, - "median_absolute_deviation": 48.92587321068486, - "median_rank": 34.0, - "standard_deviation": 875.55224609375, - "variance": 766591.75, - "z_arithmetic_mean_rank": 403.0099663037916, - "z_geometric_mean_rank": 241.95374371760477, - "z_hits_at_k": 3212.6953808476746, - "z_inverse_harmonic_mean_rank": 4666.659502775273 - } - } - }, - "times": { - "evaluation": 19.562544584274292, - "training": 17.731594562530518 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json deleted file mode 100644 index e4b25e5..0000000 --- a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "For each positive triple, DistMult create two negatives where head and tail are corrupted", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with DistMult as described by Yang et al., 2014" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json b/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json deleted file mode 100644 index 7db2211..0000000 --- a/nonasv/results/27c9fe15/cuda/distmult/wn18/yang2014/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 2.0877770193848152e-05, - 1.682386126120239e-05, - 1.3304294049895757e-05, - 1.077110180917705e-05, - 8.764388467831282e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.043555229029547775, - "adjusted_arithmetic_mean_rank_index": 0.9564915129253, - "adjusted_geometric_mean_rank_index": 0.9997352943268814, - "adjusted_hits_at_k": 0.7667430042934947, - "adjusted_inverse_harmonic_mean_rank": 0.6225622001033762, - "arithmetic_mean_rank": 891.2808, - "count": 10000.0, - "geometric_mean_rank": 4.985848515101419, - "harmonic_mean_rank": 1.6059988219783015, - "hits_at_1": 0.5354, - "hits_at_10": 0.7668, - "hits_at_3": 0.6879, - "hits_at_5": 0.7253, - "inverse_arithmetic_mean_rank": 0.0011219808616992535, - "inverse_geometric_mean_rank": 0.20056766605947685, - "inverse_harmonic_mean_rank": 0.6226654629597922, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4327.1181617043185, - "variance": 18723951.585351363, - "z_arithmetic_mean_rank": 165.66493427204637, - "z_geometric_mean_rank": 100.05439045229771, - "z_hits_at_k": 4904.480641422729, - "z_inverse_harmonic_mean_rank": 9826.388478656007 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.04355523391636111, - "adjusted_arithmetic_mean_rank_index": 0.9564915080382479, - "adjusted_geometric_mean_rank_index": 0.999735294324712, - "adjusted_hits_at_k": 0.7667430042934947, - "adjusted_inverse_harmonic_mean_rank": 0.6225622001029469, - "arithmetic_mean_rank": 891.2809, - "count": 10000.0, - "geometric_mean_rank": 4.985848547766591, - "harmonic_mean_rank": 1.6059988219794084, - "hits_at_1": 0.5354, - "hits_at_10": 0.7668, - "hits_at_3": 0.6879, - "hits_at_5": 0.7253, - "inverse_arithmetic_mean_rank": 0.0011219807358151621, - "inverse_geometric_mean_rank": 0.2005676647454423, - "inverse_harmonic_mean_rank": 0.6226654629593631, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4327.118493847283, - "variance": 18723954.459795184, - "z_arithmetic_mean_rank": 165.66493342560585, - "z_geometric_mean_rank": 100.0543904520806, - "z_hits_at_k": 4904.480641422729, - "z_inverse_harmonic_mean_rank": 9826.388478649233 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.043555230141488696, - "adjusted_arithmetic_mean_rank_index": 0.9564915118133047, - "adjusted_geometric_mean_rank_index": 0.9997352943327442, - "adjusted_hits_at_k": 0.7667430042934947, - "adjusted_inverse_harmonic_mean_rank": 0.622562215663466, - "arithmetic_mean_rank": 891.2808227539062, - "count": 10000.0, - "geometric_mean_rank": 4.985848426818848, - "harmonic_mean_rank": 1.6059987818561974, - "hits_at_1": 0.5354, - "hits_at_10": 0.7668, - "hits_at_3": 0.6879, - "hits_at_5": 0.7253, - "inverse_arithmetic_mean_rank": 0.0011219808366149664, - "inverse_geometric_mean_rank": 0.20056766271591187, - "inverse_harmonic_mean_rank": 0.622665478515625, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4327.1181640625, - "variance": 18723950.0, - "z_arithmetic_mean_rank": 165.6649340794481, - "z_geometric_mean_rank": 100.05439045288448, - "z_hits_at_k": 4904.480641422729, - "z_inverse_harmonic_mean_rank": 9826.388724253135 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.043580116319214164, - "adjusted_arithmetic_mean_rank_index": 0.9564666255030494, - "adjusted_geometric_mean_rank_index": 0.9997343914817656, - "adjusted_hits_at_k": 0.766142856314531, - "adjusted_inverse_harmonic_mean_rank": 0.6197832789019377, - "arithmetic_mean_rank": 891.7694, - "count": 5000.0, - "geometric_mean_rank": 4.999549826111161, - "harmonic_mean_rank": 1.6131964522263165, - "hits_at_1": 0.5318, - "hits_at_10": 0.7662, - "hits_at_3": 0.6866, - "hits_at_5": 0.7258, - "inverse_arithmetic_mean_rank": 0.0011213661289566563, - "inverse_geometric_mean_rank": 0.2000180085769518, - "inverse_harmonic_mean_rank": 0.6198873042523337, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4331.006425096093, - "variance": 18757616.654223643, - "z_arithmetic_mean_rank": 117.1397405499343, - "z_geometric_mean_rank": 70.75430993104793, - "z_hits_at_k": 3465.2365686963662, - "z_inverse_harmonic_mean_rank": 6917.210199606586 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.043580116319214164, - "adjusted_arithmetic_mean_rank_index": 0.9564666255030494, - "adjusted_geometric_mean_rank_index": 0.9997343914817656, - "adjusted_hits_at_k": 0.766142856314531, - "adjusted_inverse_harmonic_mean_rank": 0.6197832789019377, - "arithmetic_mean_rank": 891.7694, - "count": 5000.0, - "geometric_mean_rank": 4.999549826111161, - "harmonic_mean_rank": 1.6131964522263165, - "hits_at_1": 0.5318, - "hits_at_10": 0.7662, - "hits_at_3": 0.6866, - "hits_at_5": 0.7258, - "inverse_arithmetic_mean_rank": 0.0011213661289566563, - "inverse_geometric_mean_rank": 0.2000180085769518, - "inverse_harmonic_mean_rank": 0.6198873042523337, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4331.006425096093, - "variance": 18757616.654223643, - "z_arithmetic_mean_rank": 117.1397405499343, - "z_geometric_mean_rank": 70.75430993104793, - "z_hits_at_k": 3465.2365686963662, - "z_inverse_harmonic_mean_rank": 6917.210199606586 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.04358011676781876, - "adjusted_arithmetic_mean_rank_index": 0.9564666250544229, - "adjusted_geometric_mean_rank_index": 0.9997343915108016, - "adjusted_hits_at_k": 0.766142856314531, - "adjusted_inverse_harmonic_mean_rank": 0.6197832793372231, - "arithmetic_mean_rank": 891.7694091796875, - "count": 5000.0, - "geometric_mean_rank": 4.999549388885498, - "harmonic_mean_rank": 1.6131964510938386, - "hits_at_1": 0.5318, - "hits_at_10": 0.7662, - "hits_at_3": 0.6866, - "hits_at_5": 0.7258, - "inverse_arithmetic_mean_rank": 0.0011213661637157202, - "inverse_geometric_mean_rank": 0.2000180333852768, - "inverse_harmonic_mean_rank": 0.6198873046875, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4331.00634765625, - "variance": 18757616.0, - "z_arithmetic_mean_rank": 117.13974049499042, - "z_geometric_mean_rank": 70.7543099331029, - "z_hits_at_k": 3465.2365686963662, - "z_inverse_harmonic_mean_rank": 6917.210204464671 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.04353034289378082, - "adjusted_arithmetic_mean_rank_index": 0.9565163991935886, - "adjusted_geometric_mean_rank_index": 0.9997362210601108, - "adjusted_hits_at_k": 0.7673431522656075, - "adjusted_inverse_harmonic_mean_rank": 0.625341121272479, - "arithmetic_mean_rank": 890.7922, - "count": 5000.0, - "geometric_mean_rank": 4.972184752657029, - "harmonic_mean_rank": 1.5988651340536348, - "hits_at_1": 0.539, - "hits_at_10": 0.7674, - "hits_at_3": 0.6892, - "hits_at_5": 0.7248, - "inverse_arithmetic_mean_rank": 0.0011225962688043295, - "inverse_geometric_mean_rank": 0.20111883402273448, - "inverse_harmonic_mean_rank": 0.6254436216672509, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4323.226346031302, - "variance": 18690286.03901916, - "z_arithmetic_mean_rank": 117.14585622847781, - "z_geometric_mean_rank": 70.75443745443081, - "z_hits_at_k": 3470.7465355941936, - "z_inverse_harmonic_mean_rank": 6979.402382059741 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.04353035266718092, - "adjusted_arithmetic_mean_rank_index": 0.9565163894197108, - "adjusted_geometric_mean_rank_index": 0.9997362210557844, - "adjusted_hits_at_k": 0.7673431522656075, - "adjusted_inverse_harmonic_mean_rank": 0.6253411212716203, - "arithmetic_mean_rank": 890.7924, - "count": 5000.0, - "geometric_mean_rank": 4.972184817808335, - "harmonic_mean_rank": 1.5988651340558293, - "hits_at_1": 0.539, - "hits_at_10": 0.7674, - "hits_at_3": 0.6892, - "hits_at_5": 0.7248, - "inverse_arithmetic_mean_rank": 0.0011225960167599094, - "inverse_geometric_mean_rank": 0.2011188313874433, - "inverse_harmonic_mean_rank": 0.6254436216663924, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4323.227010937807, - "variance": 18690291.788102243, - "z_arithmetic_mean_rank": 117.1458550314578, - "z_geometric_mean_rank": 70.75443745412461, - "z_hits_at_k": 3470.7465355941936, - "z_inverse_harmonic_mean_rank": 6979.402382050157 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.043530347651632335, - "adjusted_arithmetic_mean_rank_index": 0.9565163944355045, - "adjusted_geometric_mean_rank_index": 0.9997362210347283, - "adjusted_hits_at_k": 0.7673431522656075, - "adjusted_inverse_harmonic_mean_rank": 0.6253411031158858, - "arithmetic_mean_rank": 890.7922973632812, - "count": 5000.0, - "geometric_mean_rank": 4.972185134887695, - "harmonic_mean_rank": 1.5988651804559029, - "hits_at_1": 0.539, - "hits_at_10": 0.7674, - "hits_at_3": 0.6892, - "hits_at_5": 0.7248, - "inverse_arithmetic_mean_rank": 0.0011225960915908217, - "inverse_geometric_mean_rank": 0.2011188119649887, - "inverse_harmonic_mean_rank": 0.625443603515625, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 4323.2265625, - "variance": 18690288.0, - "z_arithmetic_mean_rank": 117.14585564574882, - "z_geometric_mean_rank": 70.7544374526344, - "z_hits_at_k": 3470.7465355941936, - "z_inverse_harmonic_mean_rank": 6979.402179414871 - } - } - }, - "times": { - "evaluation": 4.716632843017578, - "training": 6.86816668510437 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json deleted file mode 100644 index 69aa27e..0000000 --- a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with HolE as described by Nickel et al., 2016" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json b/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json deleted file mode 100644 index 9ec0e43..0000000 --- a/nonasv/results/27c9fe15/cuda/hole/fb15k/nickel2016/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 512, - "slice_size": null - }, - "losses": [ - 1.8794194563786133e-05, - 1.6593579347000567e-06, - 1.1399794914663178e-06, - 9.579132348384458e-07, - 8.295052289118864e-07 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.03301642030053835, - "adjusted_arithmetic_mean_rank_index": 0.9671145034611406, - "adjusted_geometric_mean_rank_index": 0.9959516506556431, - "adjusted_hits_at_k": 0.4080815117156307, - "adjusted_inverse_harmonic_mean_rank": 0.2193426474691437, - "arithmetic_mean_rank": 243.88742360887744, - "count": 118142.0, - "geometric_mean_rank": 22.985003721360656, - "harmonic_mean_rank": 4.547906003106001, - "hits_at_1": 0.12495979414602766, - "hits_at_10": 0.4084830119686479, - "hits_at_3": 0.2488022887711398, - "hits_at_5": 0.3122682873152647, - "inverse_arithmetic_mean_rank": 0.004100252424674842, - "inverse_geometric_mean_rank": 0.04350662771790939, - "inverse_harmonic_mean_rank": 0.21988141340587253, - "inverse_median_rank": 0.05263157894736842, - "median_absolute_deviation": 26.686839933100835, - "median_rank": 19.0, - "standard_deviation": 880.602477092814, - "variance": 775460.7226619999, - "z_arithmetic_mean_rank": 575.2564304199459, - "z_geometric_mean_rank": 342.92036031626384, - "z_hits_at_k": 5383.815381126172, - "z_inverse_harmonic_mean_rank": 7147.909879999795 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.03301642946751757, - "adjusted_arithmetic_mean_rank_index": 0.9671144942929202, - "adjusted_geometric_mean_rank_index": 0.9959516461528362, - "adjusted_hits_at_k": 0.4080815117156307, - "adjusted_inverse_harmonic_mean_rank": 0.2193426163386077, - "arithmetic_mean_rank": 243.88749132399994, - "count": 118142.0, - "geometric_mean_rank": 22.985028174346066, - "harmonic_mean_rank": 4.547906646548572, - "hits_at_1": 0.12495979414602766, - "hits_at_10": 0.4084830119686479, - "hits_at_3": 0.2488022887711398, - "hits_at_5": 0.3122682873152647, - "inverse_arithmetic_mean_rank": 0.004100251286243782, - "inverse_geometric_mean_rank": 0.04350658143269604, - "inverse_harmonic_mean_rank": 0.21988138229682105, - "inverse_median_rank": 0.05263157894736842, - "median_absolute_deviation": 26.686839933100835, - "median_rank": 19.0, - "standard_deviation": 880.6024965562463, - "variance": 775460.7569410936, - "z_arithmetic_mean_rank": 575.2564249665298, - "z_geometric_mean_rank": 342.9203587658832, - "z_hits_at_k": 5383.815381126172, - "z_inverse_harmonic_mean_rank": 7147.908865521905 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.033016424031897265, - "adjusted_arithmetic_mean_rank_index": 0.9671144997292764, - "adjusted_geometric_mean_rank_index": 0.9959516491211775, - "adjusted_hits_at_k": 0.4080815117156307, - "adjusted_inverse_harmonic_mean_rank": 0.2193426353823576, - "arithmetic_mean_rank": 243.887451171875, - "count": 118142.0, - "geometric_mean_rank": 22.98501205444336, - "harmonic_mean_rank": 4.547906252929906, - "hits_at_1": 0.12495979414602766, - "hits_at_10": 0.4084830119686479, - "hits_at_3": 0.2488022887711398, - "hits_at_5": 0.3122682873152647, - "inverse_arithmetic_mean_rank": 0.004100251942873001, - "inverse_geometric_mean_rank": 0.04350661113858223, - "inverse_harmonic_mean_rank": 0.21988140132742803, - "inverse_median_rank": 0.05263157933950424, - "median_absolute_deviation": 26.686839933100835, - "median_rank": 19.0, - "standard_deviation": 880.6024780273438, - "variance": 775460.75, - "z_arithmetic_mean_rank": 575.2564282001686, - "z_geometric_mean_rank": 342.92035978792546, - "z_hits_at_k": 5383.815381126172, - "z_inverse_harmonic_mean_rank": 7147.909486117169 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.03559929270353428, - "adjusted_arithmetic_mean_rank_index": 0.9645316377297036, - "adjusted_geometric_mean_rank_index": 0.9953978865707077, - "adjusted_hits_at_k": 0.38825165063719563, - "adjusted_inverse_harmonic_mean_rank": 0.20451332513669285, - "arithmetic_mean_rank": 262.2510538165936, - "count": 59071.0, - "geometric_mean_rank": 25.920386954674456, - "harmonic_mean_rank": 4.876530312323604, - "hits_at_1": 0.11244096087758798, - "hits_at_10": 0.3886678742530176, - "hits_at_3": 0.23251680181476528, - "hits_at_5": 0.29401906180697807, - "inverse_arithmetic_mean_rank": 0.003813140063487998, - "inverse_geometric_mean_rank": 0.03857967096512273, - "inverse_harmonic_mean_rank": 0.2050638334950722, - "inverse_median_rank": 0.045454545454545456, - "median_absolute_deviation": 31.134646588617642, - "median_rank": 22.0, - "standard_deviation": 897.8766380331401, - "variance": 806182.4571256945, - "z_arithmetic_mean_rank": 405.62259545037574, - "z_geometric_mean_rank": 242.3297125453849, - "z_hits_at_k": 3616.39751933023, - "z_inverse_harmonic_mean_rank": 4705.430523179204 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.03559929729953194, - "adjusted_arithmetic_mean_rank_index": 0.964531633133082, - "adjusted_geometric_mean_rank_index": 0.9953978825519514, - "adjusted_hits_at_k": 0.38825165063719563, - "adjusted_inverse_harmonic_mean_rank": 0.20451329120962963, - "arithmetic_mean_rank": 262.2510876741548, - "count": 59071.0, - "geometric_mean_rank": 25.920408716190114, - "harmonic_mean_rank": 4.876531118569549, - "hits_at_1": 0.11244096087758798, - "hits_at_10": 0.3886678742530176, - "hits_at_3": 0.23251680181476528, - "hits_at_5": 0.29401906180697807, - "inverse_arithmetic_mean_rank": 0.003813139571197864, - "inverse_geometric_mean_rank": 0.038579638575505615, - "inverse_harmonic_mean_rank": 0.20506379959148785, - "inverse_median_rank": 0.045454545454545456, - "median_absolute_deviation": 31.134646588617642, - "median_rank": 22.0, - "standard_deviation": 897.8766322354212, - "variance": 806182.4467144217, - "z_arithmetic_mean_rank": 405.6225935173198, - "z_geometric_mean_rank": 242.32971156701828, - "z_hits_at_k": 3616.39751933023, - "z_inverse_harmonic_mean_rank": 4705.429742587335 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.035599294644504544, - "adjusted_arithmetic_mean_rank_index": 0.9645316357884699, - "adjusted_geometric_mean_rank_index": 0.9953978847516574, - "adjusted_hits_at_k": 0.38825165063719563, - "adjusted_inverse_harmonic_mean_rank": 0.20451331644494522, - "arithmetic_mean_rank": 262.2510681152344, - "count": 59071.0, - "geometric_mean_rank": 25.92039680480957, - "harmonic_mean_rank": 4.876530518875093, - "hits_at_1": 0.11244096087758798, - "hits_at_10": 0.3886678742530176, - "hits_at_3": 0.23251680181476528, - "hits_at_5": 0.29401906180697807, - "inverse_arithmetic_mean_rank": 0.0038131398614495993, - "inverse_geometric_mean_rank": 0.038579657673835754, - "inverse_harmonic_mean_rank": 0.2050638248093396, - "inverse_median_rank": 0.04545454680919647, - "median_absolute_deviation": 31.134646588617642, - "median_rank": 22.0, - "standard_deviation": 897.8766479492188, - "variance": 806182.5, - "z_arithmetic_mean_rank": 405.62259463401233, - "z_geometric_mean_rank": 242.32971210253692, - "z_hits_at_k": 3616.39751933023, - "z_inverse_harmonic_mean_rank": 4705.4303231999875 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.030447568375566038, - "adjusted_arithmetic_mean_rank_index": 0.9696833468576376, - "adjusted_geometric_mean_rank_index": 0.9964412605584122, - "adjusted_hits_at_k": 0.4279112902287407, - "adjusted_inverse_harmonic_mean_rank": 0.23417191353708747, - "arithmetic_mean_rank": 225.52379340116133, - "count": 59071.0, - "geometric_mean_rank": 20.3820412478714, - "harmonic_mean_rank": 4.260776690468067, - "hits_at_1": 0.13747862741446734, - "hits_at_10": 0.42829814968427826, - "hits_at_3": 0.26508777572751435, - "hits_at_5": 0.3305175128235513, - "inverse_arithmetic_mean_rank": 0.0044341219386160365, - "inverse_geometric_mean_rank": 0.049062799345695324, - "inverse_harmonic_mean_rank": 0.2346989933166728, - "inverse_median_rank": 0.058823529411764705, - "median_absolute_deviation": 23.721635496089633, - "median_rank": 17.0, - "standard_deviation": 862.5917588114866, - "variance": 744064.5423694939, - "z_arithmetic_mean_rank": 407.9096584040882, - "z_geometric_mean_rank": 242.5817511775936, - "z_hits_at_k": 3998.0610035746577, - "z_inverse_harmonic_mean_rank": 5404.319981173845 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.030447582088714377, - "adjusted_arithmetic_mean_rank_index": 0.9696833331426375, - "adjusted_geometric_mean_rank_index": 0.9964412557375856, - "adjusted_hits_at_k": 0.4279112902287407, - "adjusted_inverse_harmonic_mean_rank": 0.23417188520306811, - "arithmetic_mean_rank": 225.52389497384505, - "count": 59071.0, - "geometric_mean_rank": 20.382067503646027, - "harmonic_mean_rank": 4.260777204496071, - "hits_at_1": 0.13747862741446734, - "hits_at_10": 0.42829814968427826, - "hits_at_3": 0.26508777572751435, - "hits_at_5": 0.3305175128235513, - "inverse_arithmetic_mean_rank": 0.004434119941551977, - "inverse_geometric_mean_rank": 0.049062736143971455, - "inverse_harmonic_mean_rank": 0.23469896500215426, - "inverse_median_rank": 0.058823529411764705, - "median_absolute_deviation": 23.721635496089633, - "median_rank": 17.0, - "standard_deviation": 862.5918060275918, - "variance": 744064.6238259425, - "z_arithmetic_mean_rank": 407.90965263469866, - "z_geometric_mean_rank": 242.58175000397242, - "z_hits_at_k": 3998.0610035746577, - "z_inverse_harmonic_mean_rank": 5404.319327269179 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.030447575947657034, - "adjusted_arithmetic_mean_rank_index": 0.9696833392845241, - "adjusted_geometric_mean_rank_index": 0.9964412569308693, - "adjusted_hits_at_k": 0.4279112902287407, - "adjusted_inverse_harmonic_mean_rank": 0.23417189805527577, - "arithmetic_mean_rank": 225.5238494873047, - "count": 59071.0, - "geometric_mean_rank": 20.382061004638672, - "harmonic_mean_rank": 4.260776971334831, - "hits_at_1": 0.13747862741446734, - "hits_at_10": 0.42829814968427826, - "hits_at_3": 0.26508777572751435, - "hits_at_5": 0.3305175128235513, - "inverse_arithmetic_mean_rank": 0.004434120841324329, - "inverse_geometric_mean_rank": 0.04906275123357773, - "inverse_harmonic_mean_rank": 0.2346989778455164, - "inverse_median_rank": 0.05882352963089943, - "median_absolute_deviation": 23.721635496089633, - "median_rank": 17.0, - "standard_deviation": 862.591796875, - "variance": 744064.625, - "z_arithmetic_mean_rank": 407.9096552183615, - "z_geometric_mean_rank": 242.58175029447509, - "z_hits_at_k": 3998.0610035746577, - "z_inverse_harmonic_mean_rank": 5404.3196238779465 - } - } - }, - "times": { - "evaluation": 34.93160533905029, - "training": 4.057121515274048 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json deleted file mode 100644 index 83e9544..0000000 --- a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with HolE as described by Nickel et al., 2016" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json b/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json deleted file mode 100644 index d1cbe0f..0000000 --- a/nonasv/results/27c9fe15/cuda/hole/wn18/nickel2016/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 8.82791924415056e-05, - 2.338981202127039e-06, - 2.670299490484847e-07, - 2.541239686084076e-07, - 2.686744262802388e-07 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.04818624212508355, - "adjusted_arithmetic_mean_rank_index": 0.9518602735097387, - "adjusted_geometric_mean_rank_index": 0.9998959797743731, - "adjusted_hits_at_k": 0.8902731885548729, - "adjusted_inverse_harmonic_mean_rank": 0.771540162192838, - "arithmetic_mean_rank": 986.0463, - "count": 10000.0, - "geometric_mean_rank": 2.566301382855976, - "harmonic_mean_rank": 1.2960038160389173, - "hits_at_1": 0.7075, - "hits_at_10": 0.8903, - "hits_at_3": 0.8156, - "hits_at_5": 0.8519, - "inverse_arithmetic_mean_rank": 0.0010141511610560275, - "inverse_geometric_mean_rank": 0.3896658462176113, - "inverse_harmonic_mean_rank": 0.7716026663072505, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5035.254535130901, - "variance": 25353788.233556304, - "z_arithmetic_mean_rank": 164.86280067963153, - "z_geometric_mean_rank": 100.07047199367629, - "z_hits_at_k": 5694.642917372759, - "z_inverse_harmonic_mean_rank": 12177.824736762357 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.048186247011896895, - "adjusted_arithmetic_mean_rank_index": 0.9518602686226866, - "adjusted_geometric_mean_rank_index": 0.9998959797654242, - "adjusted_hits_at_k": 0.8902731885548729, - "adjusted_inverse_harmonic_mean_rank": 0.7715401621652603, - "arithmetic_mean_rank": 986.0464, - "count": 10000.0, - "geometric_mean_rank": 2.5663015176053334, - "harmonic_mean_rank": 1.296003816085225, - "hits_at_1": 0.7075, - "hits_at_10": 0.8903, - "hits_at_3": 0.8156, - "hits_at_5": 0.8519, - "inverse_arithmetic_mean_rank": 0.0010141510582057803, - "inverse_geometric_mean_rank": 0.38966582575734116, - "inverse_harmonic_mean_rank": 0.7716026662796803, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5035.254553371363, - "variance": 25353788.41724704, - "z_arithmetic_mean_rank": 164.862799833191, - "z_geometric_mean_rank": 100.07047199278067, - "z_hits_at_k": 5694.642917372759, - "z_inverse_harmonic_mean_rank": 12177.824736327077 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.048186246362866995, - "adjusted_arithmetic_mean_rank_index": 0.9518602692717482, - "adjusted_geometric_mean_rank_index": 0.9998959797609986, - "adjusted_hits_at_k": 0.8902731885548729, - "adjusted_inverse_harmonic_mean_rank": 0.7715401325962403, - "arithmetic_mean_rank": 986.04638671875, - "count": 10000.0, - "geometric_mean_rank": 2.5663015842437744, - "harmonic_mean_rank": 1.2960038657365307, - "hits_at_1": 0.7075, - "hits_at_10": 0.8903, - "hits_at_3": 0.8156, - "hits_at_5": 0.8519, - "inverse_arithmetic_mean_rank": 0.0010141510283574462, - "inverse_geometric_mean_rank": 0.389665812253952, - "inverse_harmonic_mean_rank": 0.7716026367187501, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5035.25439453125, - "variance": 25353788.0, - "z_arithmetic_mean_rank": 164.86279994560888, - "z_geometric_mean_rank": 100.07047199233776, - "z_hits_at_k": 5694.642917372759, - "z_inverse_harmonic_mean_rank": 12177.824269615996 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.04813570932708946, - "adjusted_arithmetic_mean_rank_index": 0.9519108098558059, - "adjusted_geometric_mean_rank_index": 0.9998956496843767, - "adjusted_hits_at_k": 0.8889728701920999, - "adjusted_inverse_harmonic_mean_rank": 0.7717651857990683, - "arithmetic_mean_rank": 984.9894, - "count": 5000.0, - "geometric_mean_rank": 2.5713136366255833, - "harmonic_mean_rank": 1.2956260718711086, - "hits_at_1": 0.7092, - "hits_at_10": 0.889, - "hits_at_3": 0.8126, - "hits_at_5": 0.8516, - "inverse_arithmetic_mean_rank": 0.0010152393518143444, - "inverse_geometric_mean_rank": 0.3889062717811164, - "inverse_harmonic_mean_rank": 0.7718276296769999, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5033.722594947763, - "variance": 25338363.16288764, - "z_arithmetic_mean_rank": 116.5817837444569, - "z_geometric_mean_rank": 70.76572267521647, - "z_hits_at_k": 4020.7923013041427, - "z_inverse_harmonic_mean_rank": 8613.433431712892 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.04813571910094273, - "adjusted_arithmetic_mean_rank_index": 0.9519108000814749, - "adjusted_geometric_mean_rank_index": 0.9998956496664444, - "adjusted_hits_at_k": 0.8889728701920999, - "adjusted_inverse_harmonic_mean_rank": 0.7717651857439131, - "arithmetic_mean_rank": 984.9896, - "count": 5000.0, - "geometric_mean_rank": 2.571313906650664, - "harmonic_mean_rank": 1.2956260719636692, - "hits_at_1": 0.7092, - "hits_at_10": 0.889, - "hits_at_3": 0.8126, - "hits_at_5": 0.8516, - "inverse_arithmetic_mean_rank": 0.0010152391456721979, - "inverse_geometric_mean_rank": 0.38890623094034354, - "inverse_harmonic_mean_rank": 0.7718276296218598, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5033.722631481778, - "variance": 25338363.53069184, - "z_arithmetic_mean_rank": 116.58178254738158, - "z_geometric_mean_rank": 70.76572267394735, - "z_hits_at_k": 4020.7923013041427, - "z_inverse_harmonic_mean_rank": 8613.433431097323 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.048135714309463874, - "adjusted_arithmetic_mean_rank_index": 0.9519108048731879, - "adjusted_geometric_mean_rank_index": 0.9998956496696731, - "adjusted_hits_at_k": 0.8889728701920999, - "adjusted_inverse_harmonic_mean_rank": 0.7717651928427455, - "arithmetic_mean_rank": 984.989501953125, - "count": 5000.0, - "geometric_mean_rank": 2.5713138580322266, - "harmonic_mean_rank": 1.2956260600504965, - "hits_at_1": 0.7092, - "hits_at_10": 0.889, - "hits_at_3": 0.8126, - "hits_at_5": 0.8516, - "inverse_arithmetic_mean_rank": 0.0010152392787858844, - "inverse_geometric_mean_rank": 0.38890624046325684, - "inverse_harmonic_mean_rank": 0.77182763671875, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5033.72265625, - "variance": 25338364.0, - "z_arithmetic_mean_rank": 116.58178313422906, - "z_geometric_mean_rank": 70.76572267417586, - "z_hits_at_k": 4020.7923013041427, - "z_inverse_harmonic_mean_rank": 8613.433510325203 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.048236772580123974, - "adjusted_arithmetic_mean_rank_index": 0.9518097395069041, - "adjusted_geometric_mean_rank_index": 0.9998963195837571, - "adjusted_hits_at_k": 0.8915735069028025, - "adjusted_inverse_harmonic_mean_rank": 0.7713151385892261, - "arithmetic_mean_rank": 987.1032, - "count": 5000.0, - "geometric_mean_rank": 2.5612988994572374, - "harmonic_mean_rank": 1.2963817805361464, - "hits_at_1": 0.7058, - "hits_at_10": 0.8916, - "hits_at_3": 0.8186, - "hits_at_5": 0.8522, - "inverse_arithmetic_mean_rank": 0.0010130653005683702, - "inverse_geometric_mean_rank": 0.39042690418205744, - "inverse_harmonic_mean_rank": 0.7713777029375009, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5036.785787598055, - "variance": 25369211.07014976, - "z_arithmetic_mean_rank": 116.5694252551693, - "z_geometric_mean_rank": 70.76576812420075, - "z_hits_at_k": 4032.649084277442, - "z_inverse_harmonic_mean_rank": 8608.611416172516 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.048236772580123974, - "adjusted_arithmetic_mean_rank_index": 0.9518097395069041, - "adjusted_geometric_mean_rank_index": 0.9998963195837571, - "adjusted_hits_at_k": 0.8915735069028025, - "adjusted_inverse_harmonic_mean_rank": 0.7713151385892261, - "arithmetic_mean_rank": 987.1032, - "count": 5000.0, - "geometric_mean_rank": 2.5612988994572374, - "harmonic_mean_rank": 1.2963817805361464, - "hits_at_1": 0.7058, - "hits_at_10": 0.8916, - "hits_at_3": 0.8186, - "hits_at_5": 0.8522, - "inverse_arithmetic_mean_rank": 0.0010130653005683702, - "inverse_geometric_mean_rank": 0.39042690418205744, - "inverse_harmonic_mean_rank": 0.7713777029375009, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5036.785787598055, - "variance": 25369211.07014976, - "z_arithmetic_mean_rank": 116.5694252551693, - "z_geometric_mean_rank": 70.76576812420075, - "z_hits_at_k": 4032.649084277442, - "z_inverse_harmonic_mean_rank": 8608.611416172516 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.04823677309074595, - "adjusted_arithmetic_mean_rank_index": 0.9518097389962572, - "adjusted_geometric_mean_rank_index": 0.9998963195713949, - "adjusted_hits_at_k": 0.8915735069028025, - "adjusted_inverse_harmonic_mean_rank": 0.7713151700353283, - "arithmetic_mean_rank": 987.1032104492188, - "count": 5000.0, - "geometric_mean_rank": 2.5612990856170654, - "harmonic_mean_rank": 1.2963817277021077, - "hits_at_1": 0.7058, - "hits_at_10": 0.8916, - "hits_at_3": 0.8186, - "hits_at_5": 0.8522, - "inverse_arithmetic_mean_rank": 0.0010130653390660882, - "inverse_geometric_mean_rank": 0.3904268741607666, - "inverse_harmonic_mean_rank": 0.771377734375, - "inverse_median_rank": 1.0, - "median_absolute_deviation": 0.0, - "median_rank": 1.0, - "standard_deviation": 5036.78564453125, - "variance": 25369212.0, - "z_arithmetic_mean_rank": 116.56942519262968, - "z_geometric_mean_rank": 70.76576812332583, - "z_hits_at_k": 4032.649084277442, - "z_inverse_harmonic_mean_rank": 8608.611767140961 - } - } - }, - "times": { - "evaluation": 7.370163202285767, - "training": 2.276630163192749 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json deleted file mode 100644 index e88fb54..0000000 --- a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with KG2E as described by He et al., 2015" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json b/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json deleted file mode 100644 index 681a02a..0000000 --- a/nonasv/results/27c9fe15/cuda/kg2e/fb15k/he2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 0.007800171632286395, - 0.006663566764337815, - 0.006155564227366854, - 0.005809586482315281, - 0.005533811495539998 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9175757107818011, - "adjusted_arithmetic_mean_rank_index": 0.08243544897129595, - "adjusted_geometric_mean_rank_index": 0.12309208616904688, - "adjusted_hits_at_k": -0.00029760758581000226, - "adjusted_inverse_harmonic_mean_rank": -3.625504840630545e-05, - "arithmetic_mean_rank": 6777.996343383386, - "count": 118142.0, - "geometric_mean_rank": 4763.144298573742, - "harmonic_mean_rank": 1529.2532991391572, - "hits_at_1": 1.6928780619931947e-05, - "hits_at_10": 0.0003808975639484688, - "hits_at_3": 6.771512247972779e-05, - "hits_at_5": 0.0001269658546494896, - "inverse_arithmetic_mean_rank": 0.00014753622594916124, - "inverse_geometric_mean_rank": 0.0002099453506582692, - "inverse_harmonic_mean_rank": 0.0006539139072401656, - "inverse_median_rank": 0.00015420200462606013, - "median_absolute_deviation": 5329.954975527639, - "median_rank": 6485.0, - "standard_deviation": 4252.917905988223, - "variance": 18087310.715075247, - "z_arithmetic_mean_rank": 49.034030557477536, - "z_geometric_mean_rank": 42.38236114512432, - "z_hits_at_k": -3.926333960309983, - "z_inverse_harmonic_mean_rank": -1.1814748371711765 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9175761049619076, - "adjusted_arithmetic_mean_rank_index": 0.08243505473781987, - "adjusted_geometric_mean_rank_index": 0.12309148448714802, - "adjusted_hits_at_k": -0.00029760758581000226, - "adjusted_inverse_harmonic_mean_rank": -3.6256189824248686e-05, - "arithmetic_mean_rank": 6777.999255133653, - "count": 118142.0, - "geometric_mean_rank": 4763.147566073076, - "harmonic_mean_rank": 1529.2559666394513, - "hits_at_1": 1.6928780619931947e-05, - "hits_at_10": 0.0003808975639484688, - "hits_at_3": 6.771512247972779e-05, - "hits_at_5": 0.0001269658546494896, - "inverse_arithmetic_mean_rank": 0.00014753616256930104, - "inverse_geometric_mean_rank": 0.00020994520663663563, - "inverse_harmonic_mean_rank": 0.000653912766609965, - "inverse_median_rank": 0.00015420200462606013, - "median_absolute_deviation": 5329.954975527639, - "median_rank": 6485.0, - "standard_deviation": 4252.918495996746, - "variance": 18087315.733591225, - "z_arithmetic_mean_rank": 49.03379606058866, - "z_geometric_mean_rank": 42.3821539774637, - "z_hits_at_k": -3.926333960309983, - "z_inverse_harmonic_mean_rank": -1.181512033551762 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9175758752917029, - "adjusted_arithmetic_mean_rank_index": 0.0824352844391204, - "adjusted_geometric_mean_rank_index": 0.12309132402137501, - "adjusted_hits_at_k": -0.00029760758581000226, - "adjusted_inverse_harmonic_mean_rank": -3.6255670257842945e-05, - "arithmetic_mean_rank": 6777.99755859375, - "count": 118142.0, - "geometric_mean_rank": 4763.1484375, - "harmonic_mean_rank": 1529.254752408621, - "hits_at_1": 1.6928780619931947e-05, - "hits_at_10": 0.0003808975639484688, - "hits_at_3": 6.771512247972779e-05, - "hits_at_5": 0.0001269658546494896, - "inverse_arithmetic_mean_rank": 0.00014753619325347245, - "inverse_geometric_mean_rank": 0.0002099451667163521, - "inverse_harmonic_mean_rank": 0.0006539132858177951, - "inverse_median_rank": 0.00015420200361404568, - "median_absolute_deviation": 5329.954975527639, - "median_rank": 6485.0, - "standard_deviation": 4252.91845703125, - "variance": 18087316.0, - "z_arithmetic_mean_rank": 49.03393269089429, - "z_geometric_mean_rank": 42.38209872680903, - "z_hits_at_k": -3.926333960309983, - "z_inverse_harmonic_mean_rank": -1.1814951019888085 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9418988796405415, - "adjusted_arithmetic_mean_rank_index": 0.058109008371985404, - "adjusted_geometric_mean_rank_index": 0.07097860130274114, - "adjusted_hits_at_k": -0.0002573392909877556, - "adjusted_inverse_harmonic_mean_rank": -8.08950121906492e-05, - "arithmetic_mean_rank": 6938.732626838889, - "count": 59071.0, - "geometric_mean_rank": 5031.639313961612, - "harmonic_mean_rank": 1636.1238648057067, - "hits_at_1": 3.3857561239863894e-05, - "hits_at_10": 0.00042321951549829863, - "hits_at_3": 0.00010157268371959168, - "hits_at_5": 0.0001862165868192514, - "inverse_arithmetic_mean_rank": 0.00014411853774737172, - "inverse_geometric_mean_rank": 0.00019874238545382933, - "inverse_harmonic_mean_rank": 0.0006112006685500869, - "inverse_median_rank": 0.00014949917775452235, - "median_absolute_deviation": 5303.268135594538, - "median_rank": 6689.0, - "standard_deviation": 4216.167552085728, - "variance": 17776068.827260558, - "z_arithmetic_mean_rank": 24.437069633476995, - "z_geometric_mean_rank": 17.279747408168674, - "z_hits_at_k": -2.3970050662423676, - "z_inverse_harmonic_mean_rank": -1.861227669543867 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9418992771943402, - "adjusted_arithmetic_mean_rank_index": 0.058108610764213586, - "adjusted_geometric_mean_rank_index": 0.07097804978650235, - "adjusted_hits_at_k": -0.0002573392909877556, - "adjusted_inverse_harmonic_mean_rank": -8.089554686966131e-05, - "arithmetic_mean_rank": 6938.735555517936, - "count": 59071.0, - "geometric_mean_rank": 5031.642300415183, - "harmonic_mean_rank": 1636.1252950993955, - "hits_at_1": 3.3857561239863894e-05, - "hits_at_10": 0.00042321951549829863, - "hits_at_3": 0.00010157268371959168, - "hits_at_5": 0.0001862165868192514, - "inverse_arithmetic_mean_rank": 0.00014411847691828572, - "inverse_geometric_mean_rank": 0.00019874226749335612, - "inverse_harmonic_mean_rank": 0.0006112001342410939, - "inverse_median_rank": 0.00014949917775452235, - "median_absolute_deviation": 5303.268135594538, - "median_rank": 6689.0, - "standard_deviation": 4216.168227987367, - "variance": 17776074.526690133, - "z_arithmetic_mean_rank": 24.436902424139248, - "z_geometric_mean_rank": 17.27961314148652, - "z_hits_at_k": -2.3970050662423676, - "z_inverse_harmonic_mean_rank": -1.8612399714070431 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9418990506630458, - "adjusted_arithmetic_mean_rank_index": 0.0581088373262626, - "adjusted_geometric_mean_rank_index": 0.07097745746800044, - "adjusted_hits_at_k": -0.0002573392909877556, - "adjusted_inverse_harmonic_mean_rank": -8.089523831127923e-05, - "arithmetic_mean_rank": 6938.73388671875, - "count": 59071.0, - "geometric_mean_rank": 5031.6455078125, - "harmonic_mean_rank": 1636.1244696896456, - "hits_at_1": 3.3857561239863894e-05, - "hits_at_10": 0.00042321951549829863, - "hits_at_3": 0.00010157268371959168, - "hits_at_5": 0.0001862165868192514, - "inverse_arithmetic_mean_rank": 0.00014411851589102298, - "inverse_geometric_mean_rank": 0.00019874214194715023, - "inverse_harmonic_mean_rank": 0.0006112004425859413, - "inverse_median_rank": 0.000149499173858203, - "median_absolute_deviation": 5303.268135594538, - "median_rank": 6689.0, - "standard_deviation": 4216.16796875, - "variance": 17776072.0, - "z_arithmetic_mean_rank": 24.436997702181714, - "z_geometric_mean_rank": 17.27946894148951, - "z_hits_at_k": -2.3970050662423676, - "z_inverse_harmonic_mean_rank": -1.8612328721139875 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8933845741729127, - "adjusted_arithmetic_mean_rank_index": 0.10662982173068158, - "adjusted_geometric_mean_rank_index": 0.17228976563296583, - "adjusted_hits_at_k": -0.00033787571296769276, - "adjusted_inverse_harmonic_mean_rank": 8.384746007820249e-06, - "arithmetic_mean_rank": 6617.260059927883, - "count": 59071.0, - "geometric_mean_rank": 4508.976536947512, - "harmonic_mean_rank": 1435.4881314087258, - "hits_at_1": 0.0, - "hits_at_10": 0.00033857561239863894, - "hits_at_3": 3.3857561239863894e-05, - "hits_at_5": 6.771512247972779e-05, - "inverse_arithmetic_mean_rank": 0.0001511199485804852, - "inverse_geometric_mean_rank": 0.00022177981894689128, - "inverse_harmonic_mean_rank": 0.0006966271459302441, - "inverse_median_rank": 0.0001589067217543302, - "median_absolute_deviation": 5362.572224334762, - "median_rank": 6293.0, - "standard_deviation": 4283.325845327644, - "variance": 18346880.297251772, - "z_arithmetic_mean_rank": 44.85519350085002, - "z_geometric_mean_rank": 41.943619470153436, - "z_hits_at_k": -3.156840548303876, - "z_inverse_harmonic_mean_rank": 0.19350676903425246 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8933849649976404, - "adjusted_arithmetic_mean_rank_index": 0.10662943085318222, - "adjusted_geometric_mean_rank_index": 0.17228912115264017, - "adjusted_hits_at_k": -0.00033787571296769276, - "adjusted_inverse_harmonic_mean_rank": 8.382997853248193e-06, - "arithmetic_mean_rank": 6617.26295474937, - "count": 59071.0, - "geometric_mean_rank": 4508.980046994949, - "harmonic_mean_rank": 1435.4917312315515, - "hits_at_1": 0.0, - "hits_at_10": 0.00033857561239863894, - "hits_at_3": 3.3857561239863894e-05, - "hits_at_5": 6.771512247972779e-05, - "inverse_arithmetic_mean_rank": 0.00015111988247078437, - "inverse_geometric_mean_rank": 0.00022177964630082123, - "inverse_harmonic_mean_rank": 0.0006966253989788363, - "inverse_median_rank": 0.0001589067217543302, - "median_absolute_deviation": 5362.572224334762, - "median_rank": 6293.0, - "standard_deviation": 4283.326350392866, - "variance": 18346884.623969864, - "z_arithmetic_mean_rank": 44.855029073248204, - "z_geometric_mean_rank": 41.94346257263001, - "z_hits_at_k": -3.156840548303876, - "z_inverse_harmonic_mean_rank": 0.19346642437233008 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8933847322055644, - "adjusted_arithmetic_mean_rank_index": 0.10662966367669136, - "adjusted_geometric_mean_rank_index": 0.1722885954471729, - "adjusted_hits_at_k": -0.00033787571296769276, - "adjusted_inverse_harmonic_mean_rank": 8.383922294825192e-06, - "arithmetic_mean_rank": 6617.26123046875, - "count": 59071.0, - "geometric_mean_rank": 4508.98291015625, - "harmonic_mean_rank": 1435.4898276070808, - "hits_at_1": 0.0, - "hits_at_10": 0.00033857561239863894, - "hits_at_3": 3.3857561239863894e-05, - "hits_at_5": 6.771512247972779e-05, - "inverse_arithmetic_mean_rank": 0.0001511199225205928, - "inverse_geometric_mean_rank": 0.00022177951177582145, - "inverse_harmonic_mean_rank": 0.0006966263227841681, - "inverse_median_rank": 0.00015890672511886805, - "median_absolute_deviation": 5362.572224334762, - "median_rank": 6293.0, - "standard_deviation": 4283.326171875, - "variance": 18346882.0, - "z_arithmetic_mean_rank": 44.855127013424635, - "z_geometric_mean_rank": 41.943334590622555, - "z_hits_at_k": -3.156840548303876, - "z_inverse_harmonic_mean_rank": 0.19348775903202942 - } - } - }, - "times": { - "evaluation": 28.49927306175232, - "training": 13.013453960418701 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json deleted file mode 100644 index edae5ec..0000000 --- a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with KG2E as described by He et al., 2015" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json b/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json deleted file mode 100644 index cab0e2e..0000000 --- a/nonasv/results/27c9fe15/cuda/kg2e/wn18/he2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.18607304607422293, - 0.14845974718735003, - 0.13420732638760424, - 0.12613810872870132, - 0.11995924184578269 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6805475312723067, - "adjusted_arithmetic_mean_rank_index": 0.31946808053646936, - "adjusted_geometric_mean_rank_index": 0.6303612792466775, - "adjusted_hits_at_k": 0.020160578927561585, - "adjusted_inverse_harmonic_mean_rank": 0.011663063971064072, - "arithmetic_mean_rank": 13926.2027, - "count": 10000.0, - "geometric_mean_rank": 5566.894863075257, - "harmonic_mean_rank": 83.79797808279572, - "hits_at_1": 0.0065, - "hits_at_10": 0.0204, - "hits_at_3": 0.0118, - "hits_at_5": 0.0149, - "inverse_arithmetic_mean_rank": 7.180708349161111e-05, - "inverse_geometric_mean_rank": 0.0001796333547868697, - "inverse_harmonic_mean_rank": 0.01193346215360901, - "inverse_median_rank": 8.941744534358654e-05, - "median_absolute_deviation": 13907.550110691798, - "median_rank": 11183.5, - "standard_deviation": 11967.17692409587, - "variance": 143213323.53261268, - "z_arithmetic_mean_rank": 55.33207336281332, - "z_geometric_mean_rank": 63.08711307648896, - "z_hits_at_k": 128.95738013444262, - "z_inverse_harmonic_mean_rank": 184.08730470957238 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.680547995519574, - "adjusted_arithmetic_mean_rank_index": 0.3194676162665141, - "adjusted_geometric_mean_rank_index": 0.6303610204221396, - "adjusted_hits_at_k": 0.020160578927561585, - "adjusted_inverse_harmonic_mean_rank": 0.011663063881889167, - "arithmetic_mean_rank": 13926.2122, - "count": 10000.0, - "geometric_mean_rank": 5566.898760367626, - "harmonic_mean_rank": 83.79797870881961, - "hits_at_1": 0.0065, - "hits_at_10": 0.0204, - "hits_at_3": 0.0118, - "hits_at_5": 0.0149, - "inverse_arithmetic_mean_rank": 7.180703450720074e-05, - "inverse_geometric_mean_rank": 0.00017963322902857356, - "inverse_harmonic_mean_rank": 0.011933462064458502, - "inverse_median_rank": 8.941744534358654e-05, - "median_absolute_deviation": 13907.550110691798, - "median_rank": 11183.5, - "standard_deviation": 11967.181705713803, - "variance": 143213437.97757113, - "z_arithmetic_mean_rank": 55.33199295096378, - "z_geometric_mean_rank": 63.08708717310082, - "z_hits_at_k": 128.95738013444262, - "z_inverse_harmonic_mean_rank": 184.08730330205475 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.680547790655196, - "adjusted_arithmetic_mean_rank_index": 0.3194678211409039, - "adjusted_geometric_mean_rank_index": 0.6303612364287918, - "adjusted_hits_at_k": 0.020160578927561585, - "adjusted_inverse_harmonic_mean_rank": 0.011663063578822358, - "arithmetic_mean_rank": 13926.2080078125, - "count": 10000.0, - "geometric_mean_rank": 5566.8955078125, - "harmonic_mean_rank": 83.79798083640323, - "hits_at_1": 0.0065, - "hits_at_10": 0.0204, - "hits_at_3": 0.0118, - "hits_at_5": 0.0149, - "inverse_arithmetic_mean_rank": 7.180705870268866e-05, - "inverse_geometric_mean_rank": 0.00017963333812076598, - "inverse_harmonic_mean_rank": 0.011933461761474608, - "inverse_median_rank": 8.941744454205036e-05, - "median_absolute_deviation": 13907.550110691798, - "median_rank": 11183.5, - "standard_deviation": 11967.1796875, - "variance": 143213392.0, - "z_arithmetic_mean_rank": 55.332028435337506, - "z_geometric_mean_rank": 63.08710879123718, - "z_hits_at_k": 128.95738013444262, - "z_inverse_harmonic_mean_rank": 184.08729851851314 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6931484301437246, - "adjusted_arithmetic_mean_rank_index": 0.3068665662002288, - "adjusted_geometric_mean_rank_index": 0.6209297388690878, - "adjusted_hits_at_k": 0.020160573334964144, - "adjusted_inverse_harmonic_mean_rank": 0.011987723703128655, - "arithmetic_mean_rank": 14183.729, - "count": 5000.0, - "geometric_mean_rank": 5709.0639095025035, - "harmonic_mean_rank": 81.57911845258644, - "hits_at_1": 0.0062, - "hits_at_10": 0.0204, - "hits_at_3": 0.0128, - "hits_at_5": 0.0162, - "inverse_arithmetic_mean_rank": 7.05033210941918e-05, - "inverse_geometric_mean_rank": 0.00017516006404054101, - "inverse_harmonic_mean_rank": 0.012258038809051328, - "inverse_median_rank": 8.567879021548216e-05, - "median_absolute_deviation": 14252.996427603604, - "median_rank": 11671.5, - "standard_deviation": 12023.017353042413, - "variance": 144552946.27155897, - "z_arithmetic_mean_rank": 37.582356759430326, - "z_geometric_mean_rank": 43.94512738951765, - "z_hits_at_k": 91.18554769571841, - "z_inverse_harmonic_mean_rank": 133.79129042696715 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6931488504194151, - "adjusted_arithmetic_mean_rank_index": 0.30686614590399874, - "adjusted_geometric_mean_rank_index": 0.6209294933217391, - "adjusted_hits_at_k": 0.020160573334964144, - "adjusted_inverse_harmonic_mean_rank": 0.011987723620118795, - "arithmetic_mean_rank": 14183.7376, - "count": 5000.0, - "geometric_mean_rank": 5709.067606969973, - "harmonic_mean_rank": 81.57911900487858, - "hits_at_1": 0.0062, - "hits_at_10": 0.0204, - "hits_at_3": 0.0128, - "hits_at_5": 0.0162, - "inverse_arithmetic_mean_rank": 7.050327834604047e-05, - "inverse_geometric_mean_rank": 0.00017515995059843746, - "inverse_harmonic_mean_rank": 0.012258038726064182, - "inverse_median_rank": 8.567879021548216e-05, - "median_absolute_deviation": 14252.996427603604, - "median_rank": 11671.5, - "standard_deviation": 12023.021595936118, - "variance": 144553048.29634625, - "z_arithmetic_mean_rank": 37.58230528519168, - "z_geometric_mean_rank": 43.945110011368655, - "z_hits_at_k": 91.18554769571841, - "z_inverse_harmonic_mean_rank": 133.79128950051972 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6931486928160312, - "adjusted_arithmetic_mean_rank_index": 0.30686630351508504, - "adjusted_geometric_mean_rank_index": 0.6209295406345121, - "adjusted_hits_at_k": 0.020160573334964144, - "adjusted_inverse_harmonic_mean_rank": 0.01198772322402437, - "arithmetic_mean_rank": 14183.734375, - "count": 5000.0, - "geometric_mean_rank": 5709.06689453125, - "harmonic_mean_rank": 81.57912164022632, - "hits_at_1": 0.0062, - "hits_at_10": 0.0204, - "hits_at_3": 0.0128, - "hits_at_5": 0.0162, - "inverse_arithmetic_mean_rank": 7.050329440971836e-05, - "inverse_geometric_mean_rank": 0.00017515997751615942, - "inverse_harmonic_mean_rank": 0.012258038330078125, - "inverse_median_rank": 8.567878830945119e-05, - "median_absolute_deviation": 14252.996427603604, - "median_rank": 11671.5, - "standard_deviation": 12023.01953125, - "variance": 144552992.0, - "z_arithmetic_mean_rank": 37.58232458803117, - "z_geometric_mean_rank": 43.94511335984065, - "z_hits_at_k": 91.18554769571841, - "z_inverse_harmonic_mean_rank": 133.79128507983188 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6679472166416922, - "adjusted_arithmetic_mean_rank_index": 0.3320690105748175, - "adjusted_geometric_mean_rank_index": 0.6395941772795828, - "adjusted_hits_at_k": 0.020160584520158967, - "adjusted_inverse_harmonic_mean_rank": 0.01133840424277726, - "arithmetic_mean_rank": 13668.6764, - "count": 5000.0, - "geometric_mean_rank": 5428.2661584767875, - "harmonic_mean_rank": 86.14091336829217, - "hits_at_1": 0.0068, - "hits_at_10": 0.0204, - "hits_at_3": 0.0108, - "hits_at_5": 0.0136, - "inverse_arithmetic_mean_rank": 7.315997326558993e-05, - "inverse_geometric_mean_rank": 0.00018422088578659665, - "inverse_harmonic_mean_rank": 0.011608885498166688, - "inverse_median_rank": 9.337940050424876e-05, - "median_absolute_deviation": 13553.949481578213, - "median_rank": 10709.0, - "standard_deviation": 11905.505499695637, - "variance": 141741061.20328304, - "z_arithmetic_mean_rank": 40.668940546682066, - "z_geometric_mean_rank": 45.26606644756686, - "z_hits_at_k": 91.18772829639525, - "z_inverse_harmonic_mean_rank": 126.54738811957063 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6679477248584975, - "adjusted_arithmetic_mean_rank_index": 0.332068502333176, - "adjusted_geometric_mean_rank_index": 0.6395939060174631, - "adjusted_hits_at_k": 0.020160584520158967, - "adjusted_inverse_harmonic_mean_rank": 0.011338404147437312, - "arithmetic_mean_rank": 13668.6868, - "count": 5000.0, - "geometric_mean_rank": 5428.270243348978, - "harmonic_mean_rank": 86.14091407554555, - "hits_at_1": 0.0068, - "hits_at_10": 0.0204, - "hits_at_3": 0.0108, - "hits_at_5": 0.0136, - "inverse_arithmetic_mean_rank": 7.315991760086273e-05, - "inverse_geometric_mean_rank": 0.000184220747157026, - "inverse_harmonic_mean_rank": 0.011608885402852823, - "inverse_median_rank": 9.337940050424876e-05, - "median_absolute_deviation": 13553.949481578213, - "median_rank": 10709.0, - "standard_deviation": 11905.51086663255, - "variance": 141741188.99550575, - "z_arithmetic_mean_rank": 40.6688783016413, - "z_geometric_mean_rank": 45.26604724950413, - "z_hits_at_k": 91.18772829639525, - "z_inverse_harmonic_mean_rank": 126.54738705548604 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6679474727353167, - "adjusted_arithmetic_mean_rank_index": 0.3320687544686779, - "adjusted_geometric_mean_rank_index": 0.6395942775560269, - "adjusted_hits_at_k": 0.020160584520158967, - "adjusted_inverse_harmonic_mean_rank": 0.01133840393739812, - "arithmetic_mean_rank": 13668.681640625, - "count": 5000.0, - "geometric_mean_rank": 5428.2646484375, - "harmonic_mean_rank": 86.140915633664, - "hits_at_1": 0.0068, - "hits_at_10": 0.0204, - "hits_at_3": 0.0108, - "hits_at_5": 0.0136, - "inverse_arithmetic_mean_rank": 7.315994298551232e-05, - "inverse_geometric_mean_rank": 0.00018422093125991523, - "inverse_harmonic_mean_rank": 0.011608885192871093, - "inverse_median_rank": 9.337939991382882e-05, - "median_absolute_deviation": 13553.949481578213, - "median_rank": 10709.0, - "standard_deviation": 11905.5087890625, - "variance": 141741136.0, - "z_arithmetic_mean_rank": 40.668909181016986, - "z_geometric_mean_rank": 45.266073544442264, - "z_hits_at_k": 91.18772829639525, - "z_inverse_harmonic_mean_rank": 126.54738471124864 - } - } - }, - "times": { - "evaluation": 7.71472430229187, - "training": 190.2168550491333 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json deleted file mode 100644 index 5922760..0000000 --- a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "The number of epochs has been inferred from the convergence experiment plot, Figure 2b. The publication favours SGD optimizer over others to ensure comaparabiltiy to RSGD, which is used to optimize the hyperbolic models studied in the same paper. The evaluation uses the non-deterministic variant of rank computation. While it is not guaranteed to yield the same results as average ranking definition due to the unknown properties of the employed sort algorithm (e.g. stability), it is closest to the average rank definition.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K-237 Dataset with MuRE as described by Bala\u017eevi\u0107 et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json deleted file mode 100644 index c01d201..0000000 --- a/nonasv/results/27c9fe15/cuda/mure/fb15k237/balazevic2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0009751350673539609, - 0.0007105607929926378, - 0.0006821246302445668, - 0.000649823348379108, - 0.0006163581072503264 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.12330314877380849, - "adjusted_arithmetic_mean_rank_index": 0.876819729646672, - "adjusted_geometric_mean_rank_index": 0.9872007499946741, - "adjusted_hits_at_k": 0.30210753600136414, - "adjusted_inverse_harmonic_mean_rank": 0.18525654778353684, - "arithmetic_mean_rank": 879.8504501418926, - "count": 40876.0, - "geometric_mean_rank": 68.11252538045748, - "harmonic_mean_rank": 5.3810548114598244, - "hits_at_1": 0.12488991095019082, - "hits_at_10": 0.3025981015754966, - "hits_at_3": 0.20099814071826988, - "hits_at_5": 0.24297876504550348, - "inverse_arithmetic_mean_rank": 0.0011365567862569497, - "inverse_geometric_mean_rank": 0.014681587482100834, - "inverse_harmonic_mean_rank": 0.18583717041319459, - "inverse_median_rank": 0.014084507042253521, - "median_absolute_deviation": 103.78215529539213, - "median_rank": 71.0, - "standard_deviation": 1980.2107549290874, - "variance": 3921234.633936826, - "z_arithmetic_mean_rank": 306.65323406753174, - "z_geometric_mean_rank": 199.93371976539004, - "z_hits_at_k": 2302.975861709799, - "z_inverse_harmonic_mean_rank": 3488.4928948840657 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.12330350876030921, - "adjusted_arithmetic_mean_rank_index": 0.8768193696097153, - "adjusted_geometric_mean_rank_index": 0.9872007362227379, - "adjusted_hits_at_k": 0.30210753600136414, - "adjusted_inverse_harmonic_mean_rank": 0.1852565465306213, - "arithmetic_mean_rank": 879.8530188863881, - "count": 40876.0, - "geometric_mean_rank": 68.11259759323725, - "harmonic_mean_rank": 5.381054847713081, - "hits_at_1": 0.12488991095019082, - "hits_at_10": 0.3025981015754966, - "hits_at_3": 0.20099814071826988, - "hits_at_5": 0.24297876504550348, - "inverse_arithmetic_mean_rank": 0.0011365534680618354, - "inverse_geometric_mean_rank": 0.014681571916724076, - "inverse_harmonic_mean_rank": 0.1858371691611719, - "inverse_median_rank": 0.014084507042253521, - "median_absolute_deviation": 103.78215529539213, - "median_rank": 71.0, - "standard_deviation": 1980.216275380783, - "variance": 3921256.4972829404, - "z_arithmetic_mean_rank": 306.6531081505463, - "z_geometric_mean_rank": 199.93371697621626, - "z_hits_at_k": 2302.975861709799, - "z_inverse_harmonic_mean_rank": 3488.4928712909077 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.12330333032140904, - "adjusted_arithmetic_mean_rank_index": 0.8768195480736256, - "adjusted_geometric_mean_rank_index": 0.9872007411578119, - "adjusted_hits_at_k": 0.30210753600136414, - "adjusted_inverse_harmonic_mean_rank": 0.185256550124665, - "arithmetic_mean_rank": 879.8517456054688, - "count": 40876.0, - "geometric_mean_rank": 68.1125717163086, - "harmonic_mean_rank": 5.381054743719011, - "hits_at_1": 0.12488991095019082, - "hits_at_10": 0.3025981015754966, - "hits_at_3": 0.20099814071826988, - "hits_at_5": 0.24297876504550348, - "inverse_arithmetic_mean_rank": 0.001136555103585124, - "inverse_geometric_mean_rank": 0.014681577682495117, - "inverse_harmonic_mean_rank": 0.18583717275265435, - "inverse_median_rank": 0.014084506779909134, - "median_absolute_deviation": 103.78215529539213, - "median_rank": 71.0, - "standard_deviation": 1980.21337890625, - "variance": 3921245.25, - "z_arithmetic_mean_rank": 306.6531705653545, - "z_geometric_mean_rank": 199.93371797569654, - "z_hits_at_k": 2302.975861709799, - "z_inverse_harmonic_mean_rank": 3488.4929389689255 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.17396412366101632, - "adjusted_arithmetic_mean_rank_index": 0.8261532837214485, - "adjusted_geometric_mean_rank_index": 0.9696588363086467, - "adjusted_hits_at_k": 0.18368121518304942, - "adjusted_inverse_harmonic_mean_rank": 0.1040972149746863, - "arithmetic_mean_rank": 1224.122614737254, - "count": 20438.0, - "geometric_mean_rank": 157.6978818065583, - "harmonic_mean_rank": 9.54697859231374, - "hits_at_1": 0.06306879342401409, - "hits_at_10": 0.1842646051472747, - "hits_at_3": 0.10822976807906841, - "hits_at_5": 0.13822291809374695, - "inverse_arithmetic_mean_rank": 0.0008169116295712258, - "inverse_geometric_mean_rank": 0.006341239264244906, - "inverse_harmonic_mean_rank": 0.10474518093138897, - "inverse_median_rank": 0.00423728813559322, - "median_absolute_deviation": 346.9289191303109, - "median_rank": 236.0, - "standard_deviation": 2284.762164276684, - "variance": 5220138.147310277, - "z_arithmetic_mean_rank": 204.08977609163793, - "z_geometric_mean_rank": 138.86918316715838, - "z_hits_at_k": 981.929290085826, - "z_inverse_harmonic_mean_rank": 1374.6864106662656 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.17396472165226567, - "adjusted_arithmetic_mean_rank_index": 0.8261526856452046, - "adjusted_geometric_mean_rank_index": 0.9696587831390359, - "adjusted_hits_at_k": 0.18368121518304942, - "adjusted_inverse_harmonic_mean_rank": 0.1040972127486539, - "arithmetic_mean_rank": 1224.12682258538, - "count": 20438.0, - "geometric_mean_rank": 157.69815640266404, - "harmonic_mean_rank": 9.546978795058282, - "hits_at_1": 0.06306879342401409, - "hits_at_10": 0.1842646051472747, - "hits_at_3": 0.10822976807906841, - "hits_at_5": 0.13822291809374695, - "inverse_arithmetic_mean_rank": 0.0008169088214961095, - "inverse_geometric_mean_rank": 0.006341228222393516, - "inverse_harmonic_mean_rank": 0.10474517870696656, - "inverse_median_rank": 0.00423728813559322, - "median_absolute_deviation": 346.9289191303109, - "median_rank": 236.0, - "standard_deviation": 2284.7695923634533, - "variance": 5220172.090188661, - "z_arithmetic_mean_rank": 204.08962834514932, - "z_geometric_mean_rank": 138.8691755525004, - "z_hits_at_k": 981.929290085826, - "z_inverse_harmonic_mean_rank": 1374.6863812697395 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.17396442794298037, - "adjusted_arithmetic_mean_rank_index": 0.8261529793962359, - "adjusted_geometric_mean_rank_index": 0.9696588108466823, - "adjusted_hits_at_k": 0.18368121518304942, - "adjusted_inverse_harmonic_mean_rank": 0.10409721372836692, - "arithmetic_mean_rank": 1224.124755859375, - "count": 20438.0, - "geometric_mean_rank": 157.69801330566406, - "harmonic_mean_rank": 9.546978705827115, - "hits_at_1": 0.06306879342401409, - "hits_at_10": 0.1842646051472747, - "hits_at_3": 0.10822976807906841, - "hits_at_5": 0.13822291809374695, - "inverse_arithmetic_mean_rank": 0.0008169101784005761, - "inverse_geometric_mean_rank": 0.006341233849525452, - "inverse_harmonic_mean_rank": 0.104745179685971, - "inverse_median_rank": 0.0042372881434857845, - "median_absolute_deviation": 346.9289191303109, - "median_rank": 236.0, - "standard_deviation": 2284.765869140625, - "variance": 5220155.0, - "z_arithmetic_mean_rank": 204.08970091229077, - "z_geometric_mean_rank": 138.86917952063646, - "z_hits_at_k": 981.929290085826, - "z_inverse_harmonic_mean_rank": 1374.6863942076282 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.07402909794266327, - "adjusted_arithmetic_mean_rank_index": 0.9260989099858921, - "adjusted_geometric_mean_rank_index": 0.9946618446851297, - "adjusted_hits_at_k": 0.4205310753878879, - "adjusted_inverse_harmonic_mean_rank": 0.26641415717647093, - "arithmetic_mean_rank": 535.578285546531, - "count": 20438.0, - "geometric_mean_rank": 29.41901349946047, - "harmonic_mean_rank": 3.7463123189439558, - "hits_at_1": 0.18671102847636756, - "hits_at_10": 0.42093159800371854, - "hits_at_3": 0.2937665133574714, - "hits_at_5": 0.34773461199726, - "inverse_arithmetic_mean_rank": 0.0018671406720299532, - "inverse_geometric_mean_rank": 0.03399162245934384, - "inverse_harmonic_mean_rank": 0.26692915989500016, - "inverse_median_rank": 0.047619047619047616, - "median_absolute_deviation": 29.65204437011204, - "median_rank": 21.0, - "standard_deviation": 1544.4366202382741, - "variance": 2385284.473933023, - "z_arithmetic_mean_rank": 229.29721558336044, - "z_geometric_mean_rank": 142.44367894315846, - "z_hits_at_k": 2285.960433588959, - "z_inverse_harmonic_mean_rank": 3577.2668975115416 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.07402922644017056, - "adjusted_arithmetic_mean_rank_index": 0.926098781470621, - "adjusted_geometric_mean_rank_index": 0.99466184259013, - "adjusted_hits_at_k": 0.4205310753878879, - "adjusted_inverse_harmonic_mean_rank": 0.2664141568966516, - "arithmetic_mean_rank": 535.579215187396, - "count": 20438.0, - "geometric_mean_rank": 29.419024652718747, - "harmonic_mean_rank": 3.7463123228684223, - "hits_at_1": 0.18671102847636756, - "hits_at_10": 0.42093159800371854, - "hits_at_3": 0.2937665133574714, - "hits_at_5": 0.34773461199726, - "inverse_arithmetic_mean_rank": 0.0018671374311083112, - "inverse_geometric_mean_rank": 0.03399160957253508, - "inverse_harmonic_mean_rank": 0.2669291596153773, - "inverse_median_rank": 0.047619047619047616, - "median_absolute_deviation": 29.65204437011204, - "median_rank": 21.0, - "standard_deviation": 1544.439056923411, - "variance": 2385292.0005504754, - "z_arithmetic_mean_rank": 229.2971837636558, - "z_geometric_mean_rank": 142.44367864313745, - "z_hits_at_k": 2285.960433588959, - "z_inverse_harmonic_mean_rank": 3577.266893754277 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.07402916011594705, - "adjusted_arithmetic_mean_rank_index": 0.9260988478040133, - "adjusted_geometric_mean_rank_index": 0.9946618435206058, - "adjusted_hits_at_k": 0.4205310753878879, - "adjusted_inverse_harmonic_mean_rank": 0.26641415115115213, - "arithmetic_mean_rank": 535.5787353515625, - "count": 20438.0, - "geometric_mean_rank": 29.41901969909668, - "harmonic_mean_rank": 3.746312403449072, - "hits_at_1": 0.18671102847636756, - "hits_at_10": 0.42093159800371854, - "hits_at_3": 0.2937665133574714, - "hits_at_5": 0.34773461199726, - "inverse_arithmetic_mean_rank": 0.0018671391298994422, - "inverse_geometric_mean_rank": 0.03399161621928215, - "inverse_harmonic_mean_rank": 0.26692915387391136, - "inverse_median_rank": 0.0476190485060215, - "median_absolute_deviation": 29.65204437011204, - "median_rank": 21.0, - "standard_deviation": 1544.437744140625, - "variance": 2385288.0, - "z_arithmetic_mean_rank": 229.29720018745456, - "z_geometric_mean_rank": 142.44367877638916, - "z_hits_at_k": 2285.960433588959, - "z_inverse_harmonic_mean_rank": 3577.2668166067815 - } - } - }, - "times": { - "evaluation": 15.51103925704956, - "training": 64.47068643569946 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json deleted file mode 100644 index 0621d95..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Data Set with QuatE^1 as described by Zhang et al., 2019." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json deleted file mode 100644 index cbefaad..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/fb15k/zhang2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.00029434056921607014, - 0.00026399555195481854, - 0.00024915793555821663, - 0.0002383932134910319, - 0.00023161682828993547 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.094754230917993, - "adjusted_arithmetic_mean_rank_index": 0.9053683339153923, - "adjusted_geometric_mean_rank_index": 0.9882856801373995, - "adjusted_hits_at_k": 0.28365521929977294, - "adjusted_inverse_harmonic_mean_rank": 0.1615919542527177, - "arithmetic_mean_rank": 699.9355182746186, - "count": 118142.0, - "geometric_mean_rank": 64.6158947427477, - "harmonic_mean_rank": 6.166346705456983, - "hits_at_1": 0.09879636369792284, - "hits_at_10": 0.28414111831524774, - "hits_at_3": 0.1777352677286655, - "hits_at_5": 0.2195747490308273, - "inverse_arithmetic_mean_rank": 0.0014287030360526033, - "inverse_geometric_mean_rank": 0.015476068295289483, - "inverse_harmonic_mean_rank": 0.16217057648007982, - "inverse_median_rank": 0.013888888888888888, - "median_absolute_deviation": 105.26475751389773, - "median_rank": 72.0, - "standard_deviation": 1722.2560768708345, - "variance": 2966165.9943185174, - "z_arithmetic_mean_rank": 538.5287410327303, - "z_geometric_mean_rank": 340.28085731372414, - "z_hits_at_k": 3742.2605258015683, - "z_inverse_harmonic_mean_rank": 5265.937744705862 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.09475424810607905, - "adjusted_arithmetic_mean_rank_index": 0.9053683167249791, - "adjusted_geometric_mean_rank_index": 0.9882856765779632, - "adjusted_hits_at_k": 0.28365521929977294, - "adjusted_inverse_harmonic_mean_rank": 0.16159195298409093, - "arithmetic_mean_rank": 699.9356452404734, - "count": 118142.0, - "geometric_mean_rank": 64.61591407265558, - "harmonic_mean_rank": 6.166346753661744, - "hits_at_1": 0.09879636369792284, - "hits_at_10": 0.28414111831524774, - "hits_at_3": 0.1777352677286655, - "hits_at_5": 0.2195747490308273, - "inverse_arithmetic_mean_rank": 0.0014287027768909168, - "inverse_geometric_mean_rank": 0.015476063665609954, - "inverse_harmonic_mean_rank": 0.16217057521232858, - "inverse_median_rank": 0.013888888888888888, - "median_absolute_deviation": 105.26475751389773, - "median_rank": 72.0, - "standard_deviation": 1722.2561085189475, - "variance": 2966166.1033308282, - "z_arithmetic_mean_rank": 538.5287308075752, - "z_geometric_mean_rank": 340.28085608815945, - "z_hits_at_k": 3742.2605258015683, - "z_inverse_harmonic_mean_rank": 5265.937703364017 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.09475423478978842, - "adjusted_arithmetic_mean_rank_index": 0.9053683300430727, - "adjusted_geometric_mean_rank_index": 0.9882856809181254, - "adjusted_hits_at_k": 0.28365521929977294, - "adjusted_inverse_harmonic_mean_rank": 0.1615919542822823, - "arithmetic_mean_rank": 699.935546875, - "count": 118142.0, - "geometric_mean_rank": 64.61589050292969, - "harmonic_mean_rank": 6.1663467043336, - "hits_at_1": 0.09879636369792284, - "hits_at_10": 0.28414111831524774, - "hits_at_3": 0.1777352677286655, - "hits_at_5": 0.2195747490308273, - "inverse_arithmetic_mean_rank": 0.0014287029625847936, - "inverse_geometric_mean_rank": 0.015476069413125515, - "inverse_harmonic_mean_rank": 0.16217057650962402, - "inverse_median_rank": 0.013888888992369175, - "median_absolute_deviation": 105.26475751389773, - "median_rank": 72.0, - "standard_deviation": 1722.256103515625, - "variance": 2966166.0, - "z_arithmetic_mean_rank": 538.5287387294077, - "z_geometric_mean_rank": 340.2808575825392, - "z_hits_at_k": 3742.2605258015683, - "z_inverse_harmonic_mean_rank": 5265.937745669309 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.09877014319497167, - "adjusted_arithmetic_mean_rank_index": 0.9013522109413388, - "adjusted_geometric_mean_rank_index": 0.9865650476892843, - "adjusted_hits_at_k": 0.26121629197880863, - "adjusted_inverse_harmonic_mean_rank": 0.14501653728384634, - "arithmetic_mean_rank": 727.6148533121159, - "count": 59071.0, - "geometric_mean_rank": 73.75009958894451, - "harmonic_mean_rank": 6.867744152780259, - "hits_at_1": 0.08549034213065633, - "hits_at_10": 0.2617189483841479, - "hits_at_3": 0.15826717001574375, - "hits_at_5": 0.19796516056948418, - "inverse_arithmetic_mean_rank": 0.0013743534720985725, - "inverse_geometric_mean_rank": 0.013559303723976323, - "inverse_harmonic_mean_rank": 0.1456082197813341, - "inverse_median_rank": 0.011627906976744186, - "median_absolute_deviation": 126.02118857297617, - "median_rank": 86.0, - "standard_deviation": 1733.651730374744, - "variance": 3005548.3222313435, - "z_arithmetic_mean_rank": 379.0532201489248, - "z_geometric_mean_rank": 240.17935705842558, - "z_hits_at_k": 2433.117666777289, - "z_inverse_harmonic_mean_rank": 3336.531937198153 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.09877016157896236, - "adjusted_arithmetic_mean_rank_index": 0.9013521925548522, - "adjusted_geometric_mean_rank_index": 0.9865650431569083, - "adjusted_hits_at_k": 0.26121629197880863, - "adjusted_inverse_harmonic_mean_rank": 0.14501653567957498, - "arithmetic_mean_rank": 727.6149887423609, - "count": 59071.0, - "geometric_mean_rank": 73.7501241317043, - "harmonic_mean_rank": 6.8677442283948125, - "hits_at_1": 0.08549034213065633, - "hits_at_10": 0.2617189483841479, - "hits_at_3": 0.15826717001574375, - "hits_at_5": 0.19796516056948418, - "inverse_arithmetic_mean_rank": 0.001374353216291545, - "inverse_geometric_mean_rank": 0.013559299211675658, - "inverse_harmonic_mean_rank": 0.14560821817817296, - "inverse_median_rank": 0.011627906976744186, - "median_absolute_deviation": 126.02118857297617, - "median_rank": 86.0, - "standard_deviation": 1733.6517922746639, - "variance": 3005548.536857154, - "z_arithmetic_mean_rank": 379.0532124167011, - "z_geometric_mean_rank": 240.1793559550182, - "z_hits_at_k": 2433.117666777289, - "z_inverse_harmonic_mean_rank": 3336.5319002871724 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.09877015349627322, - "adjusted_arithmetic_mean_rank_index": 0.9013522006386387, - "adjusted_geometric_mean_rank_index": 0.9865650505822927, - "adjusted_hits_at_k": 0.26121629197880863, - "adjusted_inverse_harmonic_mean_rank": 0.14501652890756012, - "arithmetic_mean_rank": 727.6149291992188, - "count": 59071.0, - "geometric_mean_rank": 73.75008392333984, - "harmonic_mean_rank": 6.867744547582034, - "hits_at_1": 0.08549034213065633, - "hits_at_10": 0.2617189483841479, - "hits_at_3": 0.15826717001574375, - "hits_at_5": 0.19796516056948418, - "inverse_arithmetic_mean_rank": 0.0013743533054366708, - "inverse_geometric_mean_rank": 0.013559306971728802, - "inverse_harmonic_mean_rank": 0.1456082114108446, - "inverse_median_rank": 0.011627906933426857, - "median_absolute_deviation": 126.02118857297617, - "median_rank": 86.0, - "standard_deviation": 1733.6517333984375, - "variance": 3005548.25, - "z_arithmetic_mean_rank": 379.0532158162438, - "z_geometric_mean_rank": 240.17935776272876, - "z_hits_at_k": 2433.117666777289, - "z_inverse_harmonic_mean_rank": 3336.531744477054 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.09076011801823436, - "adjusted_arithmetic_mean_rank_index": 0.9093626534248616, - "adjusted_geometric_mean_rank_index": 0.9897888902290393, - "adjusted_hits_at_k": 0.3060940531920783, - "adjusted_inverse_harmonic_mean_rank": 0.17816730833215674, - "arithmetic_mean_rank": 672.2561832371214, - "count": 59071.0, - "geometric_mean_rank": 56.61299275088353, - "harmonic_mean_rank": 5.594939792094619, - "hits_at_1": 0.11210238526518934, - "hits_at_10": 0.3065632882463476, - "hits_at_3": 0.19720336544158723, - "hits_at_5": 0.24118433749217044, - "inverse_arithmetic_mean_rank": 0.0014875281550921417, - "inverse_geometric_mean_rank": 0.017663789731101852, - "inverse_harmonic_mean_rank": 0.1787329331788256, - "inverse_median_rank": 0.016666666666666666, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1710.3366262901154, - "variance": 2925251.375229454, - "z_arithmetic_mean_rank": 382.5349899284484, - "z_geometric_mean_rank": 240.96224413000627, - "z_hits_at_k": 2859.8981271075604, - "z_inverse_harmonic_mean_rank": 4111.821652168115 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.09076013401690743, - "adjusted_arithmetic_mean_rank_index": 0.9093626374240282, - "adjusted_geometric_mean_rank_index": 0.9897888874690414, - "adjusted_hits_at_k": 0.3060940531920783, - "adjusted_inverse_harmonic_mean_rank": 0.17816730739917327, - "arithmetic_mean_rank": 672.2563017385858, - "count": 59071.0, - "geometric_mean_rank": 56.613007782720985, - "harmonic_mean_rank": 5.594939821280028, - "hits_at_1": 0.11210238526518934, - "hits_at_10": 0.3065632882463476, - "hits_at_3": 0.19720336544158723, - "hits_at_5": 0.24118433749217044, - "inverse_arithmetic_mean_rank": 0.0014875278928792562, - "inverse_geometric_mean_rank": 0.017663785041027495, - "inverse_harmonic_mean_rank": 0.17873293224648426, - "inverse_median_rank": 0.016666666666666666, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1710.3366270097567, - "variance": 2925251.377691112, - "z_arithmetic_mean_rank": 382.53498319749394, - "z_geometric_mean_rank": 240.96224345808997, - "z_hits_at_k": 2859.8981271075604, - "z_inverse_harmonic_mean_rank": 4111.821630636322 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.0907601237356731, - "adjusted_arithmetic_mean_rank_index": 0.9093626477066508, - "adjusted_geometric_mean_rank_index": 0.9897888890885413, - "adjusted_hits_at_k": 0.3060940531920783, - "adjusted_inverse_harmonic_mean_rank": 0.17816731676754022, - "arithmetic_mean_rank": 672.2562255859375, - "count": 59071.0, - "geometric_mean_rank": 56.612998962402344, - "harmonic_mean_rank": 5.594939528220595, - "hits_at_1": 0.11210238526518934, - "hits_at_10": 0.3065632882463476, - "hits_at_3": 0.19720336544158723, - "hits_at_5": 0.24118433749217044, - "inverse_arithmetic_mean_rank": 0.001487528090365231, - "inverse_geometric_mean_rank": 0.01766378805041313, - "inverse_harmonic_mean_rank": 0.17873294160840345, - "inverse_median_rank": 0.01666666753590107, - "median_absolute_deviation": 87.47353089183052, - "median_rank": 60.0, - "standard_deviation": 1710.336669921875, - "variance": 2925251.5, - "z_arithmetic_mean_rank": 382.53498752301016, - "z_geometric_mean_rank": 240.96224385235422, - "z_hits_at_k": 2859.8981271075604, - "z_inverse_harmonic_mean_rank": 4111.821846843517 - } - } - }, - "times": { - "evaluation": 144.05696630477905, - "training": 7.538693904876709 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json deleted file mode 100644 index 7a7a296..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K-237 Data Set with QuatE^1 as described by Zhang et al., 2019." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json deleted file mode 100644 index 5574951..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/fb15k237/zhang2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 512, - "slice_size": null - }, - "losses": [ - 0.0007930459803240636, - 0.0007622064488433046, - 0.0007567433983401318, - 0.0007538720628881989, - 0.0007519833328350528 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9998827472540538, - "adjusted_arithmetic_mean_rank_index": 0.00011726918016985444, - "adjusted_geometric_mean_rank_index": -0.04594329569284983, - "adjusted_hits_at_k": -0.0004586043346634101, - "adjusted_inverse_harmonic_mean_rank": -0.00020229649213419107, - "arithmetic_mean_rank": 7134.832273216557, - "count": 40876.0, - "geometric_mean_rank": 5485.375721194342, - "harmonic_mean_rank": 1958.8929986398573, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0002446423329092866, - "hits_at_3": 9.785693316371466e-05, - "hits_at_5": 0.00017124963303650063, - "inverse_arithmetic_mean_rank": 0.00014015746435328263, - "inverse_geometric_mean_rank": 0.00018230291794529399, - "inverse_harmonic_mean_rank": 0.0005104924060141838, - "inverse_median_rank": 0.00014019346698443852, - "median_absolute_deviation": 4894.069923286992, - "median_rank": 7133.0, - "standard_deviation": 3938.004768056505, - "variance": 15507881.553235766, - "z_arithmetic_mean_rank": 0.04101296097667273, - "z_geometric_mean_rank": -9.304707280866893, - "z_hits_at_k": -3.4959561975327444, - "z_inverse_harmonic_mean_rank": -3.8093653580044204 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9998827781100396, - "adjusted_arithmetic_mean_rank_index": 0.0001172383198593252, - "adjusted_geometric_mean_rank_index": -0.04594334860250249, - "adjusted_hits_at_k": -0.0004586043346634101, - "adjusted_inverse_harmonic_mean_rank": -0.00020229651184680824, - "arithmetic_mean_rank": 7134.832493394657, - "count": 40876.0, - "geometric_mean_rank": 5485.375998624691, - "harmonic_mean_rank": 1958.8930742284265, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0002446423329092866, - "hits_at_3": 9.785693316371466e-05, - "hits_at_5": 0.00017124963303650063, - "inverse_arithmetic_mean_rank": 0.00014015746002807888, - "inverse_geometric_mean_rank": 0.00018230290872507605, - "inverse_harmonic_mean_rank": 0.0005104923863156147, - "inverse_median_rank": 0.00014019346698443852, - "median_absolute_deviation": 4894.069923286992, - "median_rank": 7133.0, - "standard_deviation": 3938.004705572819, - "variance": 15507881.061113667, - "z_arithmetic_mean_rank": 0.04100216809217727, - "z_geometric_mean_rank": -9.304717996441866, - "z_hits_at_k": -3.4959561975327444, - "z_inverse_harmonic_mean_rank": -3.8093657292049308 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9998827133446111, - "adjusted_arithmetic_mean_rank_index": 0.00011730309436530639, - "adjusted_geometric_mean_rank_index": -0.04594287878634984, - "adjusted_hits_at_k": -0.0004586043346634101, - "adjusted_inverse_harmonic_mean_rank": -0.00020229652761597373, - "arithmetic_mean_rank": 7134.83203125, - "count": 40876.0, - "geometric_mean_rank": 5485.37353515625, - "harmonic_mean_rank": 1958.8931346957265, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0002446423329092866, - "hits_at_3": 9.785693316371466e-05, - "hits_at_5": 0.00017124963303650063, - "inverse_arithmetic_mean_rank": 0.0001401574700139463, - "inverse_geometric_mean_rank": 0.00018230298883281648, - "inverse_harmonic_mean_rank": 0.000510492370557687, - "inverse_median_rank": 0.00014019347145222127, - "median_absolute_deviation": 4894.069923286992, - "median_rank": 7133.0, - "standard_deviation": 3938.0048828125, - "variance": 15507882.0, - "z_arithmetic_mean_rank": 0.04102482190694445, - "z_geometric_mean_rank": -9.304622846503037, - "z_hits_at_k": -3.4959561975327444, - "z_inverse_harmonic_mean_rank": -3.809366026147858 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9989789751387469, - "adjusted_arithmetic_mean_rank_index": 0.0010211699831054188, - "adjusted_geometric_mean_rank_index": -0.029203457964425805, - "adjusted_hits_at_k": -0.00042138983804789365, - "adjusted_inverse_harmonic_mean_rank": -0.00014306071821701925, - "arithmetic_mean_rank": 7029.453713670614, - "count": 20438.0, - "geometric_mean_rank": 5316.3532096388135, - "harmonic_mean_rank": 1723.253977787341, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.00029357079949114397, - "hits_at_3": 0.00014678539974557198, - "hits_at_5": 0.0002446423329092866, - "inverse_arithmetic_mean_rank": 0.00014225856527872685, - "inverse_geometric_mean_rank": 0.00018809886412117052, - "inverse_harmonic_mean_rank": 0.000580297514405857, - "inverse_median_rank": 0.00014240956992309884, - "median_absolute_deviation": 4910.378547690553, - "median_rank": 7022.0, - "standard_deviation": 3964.421908029607, - "variance": 15716641.06486511, - "z_arithmetic_mean_rank": 0.25226596239463106, - "z_geometric_mean_rank": -4.182357960676967, - "z_hits_at_k": -2.2526801345004044, - "z_inverse_harmonic_mean_rank": -1.8892304206305193 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9989790029522932, - "adjusted_arithmetic_mean_rank_index": 0.0010211421656057817, - "adjusted_geometric_mean_rank_index": -0.02920352481459032, - "adjusted_hits_at_k": -0.00042138983804789365, - "adjusted_inverse_harmonic_mean_rank": -0.00014306075211986604, - "arithmetic_mean_rank": 7029.45390938448, - "count": 20438.0, - "geometric_mean_rank": 5316.353554888564, - "harmonic_mean_rank": 1723.25407839257, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.00029357079949114397, - "hits_at_3": 0.00014678539974557198, - "hits_at_5": 0.0002446423329092866, - "inverse_arithmetic_mean_rank": 0.00014225856131796773, - "inverse_geometric_mean_rank": 0.00018809885190582682, - "inverse_harmonic_mean_rank": 0.0005802974805275305, - "inverse_median_rank": 0.00014240956992309884, - "median_absolute_deviation": 4910.378547690553, - "median_rank": 7022.0, - "standard_deviation": 3964.4218002748275, - "variance": 15716640.210494306, - "z_arithmetic_mean_rank": 0.2522590904649462, - "z_geometric_mean_rank": -4.182367534588337, - "z_hits_at_k": -2.2526801345004044, - "z_inverse_harmonic_mean_rank": -1.8892308683445416 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9989789608720815, - "adjusted_arithmetic_mean_rank_index": 0.0010211842517985348, - "adjusted_geometric_mean_rank_index": -0.029203233576663656, - "adjusted_hits_at_k": -0.00042138983804789365, - "adjusted_inverse_harmonic_mean_rank": -0.000143060756398357, - "arithmetic_mean_rank": 7029.45361328125, - "count": 20438.0, - "geometric_mean_rank": 5316.35205078125, - "harmonic_mean_rank": 1723.2540910888076, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.00029357079949114397, - "hits_at_3": 0.00014678539974557198, - "hits_at_5": 0.0002446423329092866, - "inverse_arithmetic_mean_rank": 0.00014225856284610927, - "inverse_geometric_mean_rank": 0.00018809890025295317, - "inverse_harmonic_mean_rank": 0.000580297476252134, - "inverse_median_rank": 0.00014240956807043403, - "median_absolute_deviation": 4910.378547690553, - "median_rank": 7022.0, - "standard_deviation": 3964.421875, - "variance": 15716641.0, - "z_arithmetic_mean_rank": 0.25226948727851095, - "z_geometric_mean_rank": -4.182325825100975, - "z_hits_at_k": -2.2526801345004044, - "z_inverse_harmonic_mean_rank": -1.889230924845414 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.000761777182881, - "adjusted_arithmetic_mean_rank_index": -0.0007618824923663059, - "adjusted_geometric_mean_rank_index": -0.06292935156569657, - "adjusted_hits_at_k": -0.0004958179572368788, - "adjusted_inverse_harmonic_mean_rank": -0.00026153100818154026, - "arithmetic_mean_rank": 7240.210832762501, - "count": 20438.0, - "geometric_mean_rank": 5659.771955729911, - "harmonic_mean_rank": 2269.1827184376725, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0001957138663274293, - "hits_at_3": 4.892846658185733e-05, - "hits_at_5": 9.785693316371466e-05, - "inverse_arithmetic_mean_rank": 0.0001381175249034081, - "inverse_geometric_mean_rank": 0.00017668556398064899, - "inverse_harmonic_mean_rank": 0.0004406872976225105, - "inverse_median_rank": 0.00013803575125957622, - "median_absolute_deviation": 4854.039663387341, - "median_rank": 7244.5, - "standard_deviation": 3908.5691448383595, - "variance": 15276912.759982465, - "z_arithmetic_mean_rank": -0.18863809493518602, - "z_geometric_mean_rank": -9.011995783715662, - "z_hits_at_k": -2.695211599906065, - "z_inverse_harmonic_mean_rank": -3.5116985829733194 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.0007618109980143, - "adjusted_arithmetic_mean_rank_index": -0.0007619163121743711, - "adjusted_geometric_mean_rank_index": -0.06292939006290288, - "adjusted_hits_at_k": -0.0004958179572368788, - "adjusted_inverse_harmonic_mean_rank": -0.00026153101370422917, - "arithmetic_mean_rank": 7240.211077404834, - "count": 20438.0, - "geometric_mean_rank": 5659.7721606794785, - "harmonic_mean_rank": 2269.1827468550846, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0001957138663274293, - "hits_at_3": 4.892846658185733e-05, - "hits_at_5": 9.785693316371466e-05, - "inverse_arithmetic_mean_rank": 0.00013811752023650087, - "inverse_geometric_mean_rank": 0.00017668555758257695, - "inverse_harmonic_mean_rank": 0.0004406872921036987, - "inverse_median_rank": 0.00013803575125957622, - "median_absolute_deviation": 4854.039663387341, - "median_rank": 7244.5, - "standard_deviation": 3908.5691269052727, - "variance": 15276912.619797047, - "z_arithmetic_mean_rank": -0.1886464685416911, - "z_geometric_mean_rank": -9.012001296829226, - "z_hits_at_k": -2.695211599906065, - "z_inverse_harmonic_mean_rank": -3.511698657129034 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.0007617241684128, - "adjusted_arithmetic_mean_rank_index": -0.0007618294705693529, - "adjusted_geometric_mean_rank_index": -0.06292953818011693, - "adjusted_hits_at_k": -0.0004958179572368788, - "adjusted_inverse_harmonic_mean_rank": -0.0002615309942692217, - "arithmetic_mean_rank": 7240.21044921875, - "count": 20438.0, - "geometric_mean_rank": 5659.77294921875, - "harmonic_mean_rank": 2269.182646850792, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.0001957138663274293, - "hits_at_3": 4.892846658185733e-05, - "hits_at_5": 9.785693316371466e-05, - "inverse_arithmetic_mean_rank": 0.00013811753888148814, - "inverse_geometric_mean_rank": 0.00017668552754912525, - "inverse_harmonic_mean_rank": 0.00044068731152506214, - "inverse_median_rank": 0.00013803575711790472, - "median_absolute_deviation": 4854.039663387341, - "median_rank": 7244.5, - "standard_deviation": 3908.569091796875, - "variance": 15276912.0, - "z_arithmetic_mean_rank": -0.1886249670173754, - "z_geometric_mean_rank": -9.012022508420841, - "z_hits_at_k": -2.695211599906065, - "z_inverse_harmonic_mean_rank": -3.5116983961661417 - } - } - }, - "times": { - "evaluation": 31.55549693107605, - "training": 3.1638309955596924 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json deleted file mode 100644 index 8801287..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Data Set with QuatE^1 as described by Zhang et al., 2019." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json deleted file mode 100644 index d28be8e..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/wn18/zhang2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 64, - "slice_size": null - }, - "losses": [ - 0.0003714370767319881, - 0.00035896927278014634, - 0.00035071555033651267, - 0.0003477621583505325, - 0.0003448637009772764 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7635239113975782, - "adjusted_arithmetic_mean_rank_index": 0.23648764531221989, - "adjusted_geometric_mean_rank_index": 0.5844441843224377, - "adjusted_hits_at_k": 0.013458941400830942, - "adjusted_inverse_harmonic_mean_rank": 0.007385369297529207, - "arithmetic_mean_rank": 15624.1677, - "count": 10000.0, - "geometric_mean_rank": 6258.298951492494, - "harmonic_mean_rank": 130.6005122071634, - "hits_at_1": 0.0036, - "hits_at_10": 0.0137, - "hits_at_3": 0.0074, - "hits_at_5": 0.0098, - "inverse_arithmetic_mean_rank": 6.400340928240292e-05, - "inverse_geometric_mean_rank": 0.00015978782856985725, - "inverse_harmonic_mean_rank": 0.007656937810578895, - "inverse_median_rank": 9.570293808019906e-05, - "median_absolute_deviation": 13722.966134487851, - "median_rank": 10449.0, - "standard_deviation": 14467.476928558646, - "variance": 209307888.6783767, - "z_arithmetic_mean_rank": 40.9598095616972, - "z_geometric_mean_rank": 58.49168969786516, - "z_hits_at_k": 86.09027690476483, - "z_inverse_harmonic_mean_rank": 116.56908781774791 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7635239358316449, - "adjusted_arithmetic_mean_rank_index": 0.23648762087695907, - "adjusted_geometric_mean_rank_index": 0.5844441164415516, - "adjusted_hits_at_k": 0.013458941400830942, - "adjusted_inverse_harmonic_mean_rank": 0.007385369198125054, - "arithmetic_mean_rank": 15624.1682, - "count": 10000.0, - "geometric_mean_rank": 6258.2999736199245, - "harmonic_mean_rank": 130.6005139021859, - "hits_at_1": 0.0036, - "hits_at_10": 0.0137, - "hits_at_3": 0.0074, - "hits_at_5": 0.0098, - "inverse_arithmetic_mean_rank": 6.400340723418479e-05, - "inverse_geometric_mean_rank": 0.0001597878024727505, - "inverse_harmonic_mean_rank": 0.007656937711201937, - "inverse_median_rank": 9.570293808019906e-05, - "median_absolute_deviation": 13722.966134487851, - "median_rank": 10449.0, - "standard_deviation": 14467.47674622319, - "variance": 209307883.40250874, - "z_arithmetic_mean_rank": 40.95980532949457, - "z_geometric_mean_rank": 58.49168290428617, - "z_hits_at_k": 86.09027690476483, - "z_inverse_harmonic_mean_rank": 116.56908624877403 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7635239245308891, - "adjusted_arithmetic_mean_rank_index": 0.23648763217826718, - "adjusted_geometric_mean_rank_index": 0.5844440952331575, - "adjusted_hits_at_k": 0.013458941400830942, - "adjusted_inverse_harmonic_mean_rank": 0.007385369658435771, - "arithmetic_mean_rank": 15624.16796875, - "count": 10000.0, - "geometric_mean_rank": 6258.30029296875, - "harmonic_mean_rank": 130.6005060530473, - "hits_at_1": 0.0036, - "hits_at_10": 0.0137, - "hits_at_3": 0.0074, - "hits_at_5": 0.0098, - "inverse_arithmetic_mean_rank": 6.400340498657897e-05, - "inverse_geometric_mean_rank": 0.00015978779993019998, - "inverse_harmonic_mean_rank": 0.0076569381713867185, - "inverse_median_rank": 9.570294059813023e-05, - "median_absolute_deviation": 13722.966134487851, - "median_rank": 10449.0, - "standard_deviation": 14467.4765625, - "variance": 209307888.0, - "z_arithmetic_mean_rank": 40.95980728688829, - "z_geometric_mean_rank": 58.49168078173138, - "z_hits_at_k": 86.09027690476483, - "z_inverse_harmonic_mean_rank": 116.56909351421984 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7621333909495468, - "adjusted_arithmetic_mean_rank_index": 0.23787823398522556, - "adjusted_geometric_mean_rank_index": 0.5880241496733576, - "adjusted_hits_at_k": 0.01435915574139819, - "adjusted_inverse_harmonic_mean_rank": 0.007780013358221885, - "arithmetic_mean_rank": 15595.3516, - "count": 5000.0, - "geometric_mean_rank": 6204.558347786061, - "harmonic_mean_rank": 124.20077311450835, - "hits_at_1": 0.0036, - "hits_at_10": 0.0146, - "hits_at_3": 0.0088, - "hits_at_5": 0.0114, - "inverse_arithmetic_mean_rank": 6.412167071629215e-05, - "inverse_geometric_mean_rank": 0.0001611718262520369, - "inverse_harmonic_mean_rank": 0.00805147967217594, - "inverse_median_rank": 9.573042312847023e-05, - "median_absolute_deviation": 13729.637844471126, - "median_rank": 10446.0, - "standard_deviation": 14485.4557773229, - "variance": 209828429.07677743, - "z_arithmetic_mean_rank": 29.133263899145902, - "z_geometric_mean_rank": 41.61629657579756, - "z_hits_at_k": 64.94594468981366, - "z_inverse_harmonic_mean_rank": 86.83033180551921 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7621334398188131, - "adjusted_arithmetic_mean_rank_index": 0.23787818511357084, - "adjusted_geometric_mean_rank_index": 0.5880240150809598, - "adjusted_hits_at_k": 0.01435915574139819, - "adjusted_inverse_harmonic_mean_rank": 0.007780013159413577, - "arithmetic_mean_rank": 15595.3526, - "count": 5000.0, - "geometric_mean_rank": 6204.560374486879, - "harmonic_mean_rank": 124.20077618045295, - "hits_at_1": 0.0036, - "hits_at_10": 0.0146, - "hits_at_3": 0.0088, - "hits_at_5": 0.0114, - "inverse_arithmetic_mean_rank": 6.412166660470376e-05, - "inverse_geometric_mean_rank": 0.00016117177360574892, - "inverse_harmonic_mean_rank": 0.008051479473422024, - "inverse_median_rank": 9.573042312847023e-05, - "median_absolute_deviation": 13729.637844471126, - "median_rank": 10446.0, - "standard_deviation": 14485.455415093902, - "variance": 209828418.58267325, - "z_arithmetic_mean_rank": 29.133257913769313, - "z_geometric_mean_rank": 41.616287050275275, - "z_hits_at_k": 64.94594468981366, - "z_inverse_harmonic_mean_rank": 86.83032958668093 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7621333891169493, - "adjusted_arithmetic_mean_rank_index": 0.23787823581791256, - "adjusted_geometric_mean_rank_index": 0.588024003632733, - "adjusted_hits_at_k": 0.01435915574139819, - "adjusted_inverse_harmonic_mean_rank": 0.007780013788702784, - "arithmetic_mean_rank": 15595.3515625, - "count": 5000.0, - "geometric_mean_rank": 6204.560546875, - "harmonic_mean_rank": 124.20076647579945, - "hits_at_1": 0.0036, - "hits_at_10": 0.0146, - "hits_at_3": 0.0088, - "hits_at_5": 0.0114, - "inverse_arithmetic_mean_rank": 6.412166840163991e-05, - "inverse_geometric_mean_rank": 0.00016117177437990904, - "inverse_harmonic_mean_rank": 0.008051480102539062, - "inverse_median_rank": 9.5730421890039e-05, - "median_absolute_deviation": 13729.637844471126, - "median_rank": 10446.0, - "standard_deviation": 14485.455078125, - "variance": 209828416.0, - "z_arithmetic_mean_rank": 29.133264123597513, - "z_geometric_mean_rank": 41.61628624004869, - "z_hits_at_k": 64.94594468981366, - "z_inverse_harmonic_mean_rank": 86.83033660998389 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7649143673741152, - "adjusted_arithmetic_mean_rank_index": 0.23509712111702274, - "adjusted_geometric_mean_rank_index": 0.5808749569746401, - "adjusted_hits_at_k": 0.012558727070539942, - "adjusted_inverse_harmonic_mean_rank": 0.006990725241428644, - "arithmetic_mean_rank": 15652.9838, - "count": 5000.0, - "geometric_mean_rank": 6312.505027892547, - "harmonic_mean_rank": 137.69560445684527, - "hits_at_1": 0.0036, - "hits_at_10": 0.0128, - "hits_at_3": 0.006, - "hits_at_5": 0.0082, - "inverse_arithmetic_mean_rank": 6.388558327135048e-05, - "inverse_geometric_mean_rank": 0.00015841571540638498, - "inverse_harmonic_mean_rank": 0.007262395948981848, - "inverse_median_rank": 9.563886763580719e-05, - "median_absolute_deviation": 13704.433606756531, - "median_rank": 10456.0, - "standard_deviation": 14449.418242432375, - "variance": 208785687.54473752, - "z_arithmetic_mean_rank": 28.79266217842425, - "z_geometric_mean_rank": 41.11032485001479, - "z_hits_at_k": 56.803997459095015, - "z_inverse_harmonic_mean_rank": 78.02315047356781 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7649143673741152, - "adjusted_arithmetic_mean_rank_index": 0.23509712111702274, - "adjusted_geometric_mean_rank_index": 0.5808749569746401, - "adjusted_hits_at_k": 0.012558727070539942, - "adjusted_inverse_harmonic_mean_rank": 0.006990725241428644, - "arithmetic_mean_rank": 15652.9838, - "count": 5000.0, - "geometric_mean_rank": 6312.505027892547, - "harmonic_mean_rank": 137.69560445684527, - "hits_at_1": 0.0036, - "hits_at_10": 0.0128, - "hits_at_3": 0.006, - "hits_at_5": 0.0082, - "inverse_arithmetic_mean_rank": 6.388558327135048e-05, - "inverse_geometric_mean_rank": 0.00015841571540638498, - "inverse_harmonic_mean_rank": 0.007262395948981848, - "inverse_median_rank": 9.563886763580719e-05, - "median_absolute_deviation": 13704.433606756531, - "median_rank": 10456.0, - "standard_deviation": 14449.418242432375, - "variance": 208785687.54473752, - "z_arithmetic_mean_rank": 28.79266217842425, - "z_geometric_mean_rank": 41.11032485001479, - "z_hits_at_k": 56.803997459095015, - "z_inverse_harmonic_mean_rank": 78.02315047356781 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7649143000292802, - "adjusted_arithmetic_mean_rank_index": 0.23509718846514893, - "adjusted_geometric_mean_rank_index": 0.5808749017587895, - "adjusted_hits_at_k": 0.012558727070539942, - "adjusted_inverse_harmonic_mean_rank": 0.006990725532760876, - "arithmetic_mean_rank": 15652.982421875, - "count": 5000.0, - "geometric_mean_rank": 6312.505859375, - "harmonic_mean_rank": 137.69559893467442, - "hits_at_1": 0.0036, - "hits_at_10": 0.0128, - "hits_at_3": 0.006, - "hits_at_5": 0.0082, - "inverse_arithmetic_mean_rank": 6.38855854049325e-05, - "inverse_geometric_mean_rank": 0.00015841569984331727, - "inverse_harmonic_mean_rank": 0.007262396240234376, - "inverse_median_rank": 9.563886851537973e-05, - "median_absolute_deviation": 13704.433606756531, - "median_rank": 10456.0, - "standard_deviation": 14449.4189453125, - "variance": 208785696.0, - "z_arithmetic_mean_rank": 28.7926704266403, - "z_geometric_mean_rank": 41.110320942217555, - "z_hits_at_k": 56.803997459095015, - "z_inverse_harmonic_mean_rank": 78.02315372511293 - } - } - }, - "times": { - "evaluation": 49.746681928634644, - "training": 8.512414455413818 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json deleted file mode 100644 index 8eb29b5..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18RR Data Set with QuatE^1 as described by Zhang et al., 2019." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json b/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json deleted file mode 100644 index 175356f..0000000 --- a/nonasv/results/27c9fe15/cuda/quate/wn18rr/zhang2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 128, - "slice_size": null - }, - "losses": [ - 0.0003800833376947446, - 0.00037256451326309726, - 0.0003562385337013802, - 0.000347856839113493, - 0.0003418508737124453 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.0166994688460609, - "adjusted_arithmetic_mean_rank_index": -0.01670029263459827, - "adjusted_geometric_mean_rank_index": 0.22735517405740358, - "adjusted_hits_at_k": 0.01155511005618705, - "adjusted_inverse_harmonic_mean_rank": 0.006916144054771029, - "arithmetic_mean_rank": 20611.088748290014, - "count": 5848.0, - "geometric_mean_rank": 11527.227772272285, - "harmonic_mean_rank": 139.07882417125074, - "hits_at_1": 0.004274965800273598, - "hits_at_10": 0.01179890560875513, - "hits_at_3": 0.0070109439124487, - "hits_at_5": 0.008891928864569083, - "inverse_arithmetic_mean_rank": 4.851757285664807e-05, - "inverse_geometric_mean_rank": 8.675112696266926e-05, - "inverse_harmonic_mean_rank": 0.007190167201648746, - "inverse_median_rank": 4.828585224529213e-05, - "median_absolute_deviation": 17863.132829664744, - "median_rank": 20710.0, - "standard_deviation": 13176.818342055025, - "variance": 173628541.6195177, - "z_arithmetic_mean_rank": -2.21195971632244, - "z_geometric_mean_rank": 17.401431565374203, - "z_hits_at_k": 56.25842171670579, - "z_inverse_harmonic_mean_rank": 83.08987517740859 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.0166994857160303, - "adjusted_arithmetic_mean_rank_index": -0.016700309505399913, - "adjusted_geometric_mean_rank_index": 0.2273551670159204, - "adjusted_hits_at_k": 0.01155511005618705, - "adjusted_inverse_harmonic_mean_rank": 0.0069161440545278995, - "arithmetic_mean_rank": 20611.089090287278, - "count": 5848.0, - "geometric_mean_rank": 11527.22787731633, - "harmonic_mean_rank": 139.07882417595226, - "hits_at_1": 0.004274965800273598, - "hits_at_10": 0.01179890560875513, - "hits_at_3": 0.0070109439124487, - "hits_at_5": 0.008891928864569083, - "inverse_arithmetic_mean_rank": 4.851757205160195e-05, - "inverse_geometric_mean_rank": 8.675112617213319e-05, - "inverse_harmonic_mean_rank": 0.007190167201405685, - "inverse_median_rank": 4.828585224529213e-05, - "median_absolute_deviation": 17863.132829664744, - "median_rank": 20710.0, - "standard_deviation": 13176.818782175542, - "variance": 173628553.2182941, - "z_arithmetic_mean_rank": -2.21196195086614, - "z_geometric_mean_rank": 17.401431026429407, - "z_hits_at_k": 56.25842171670579, - "z_inverse_harmonic_mean_rank": 83.08987517448767 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.0166995228826816, - "adjusted_arithmetic_mean_rank_index": -0.016700346673884647, - "adjusted_geometric_mean_rank_index": 0.22735558246544507, - "adjusted_hits_at_k": 0.01155511005618705, - "adjusted_inverse_harmonic_mean_rank": 0.00691614434941122, - "arithmetic_mean_rank": 20611.08984375, - "count": 5848.0, - "geometric_mean_rank": 11527.2216796875, - "harmonic_mean_rank": 139.0788184736221, - "hits_at_1": 0.004274965800273598, - "hits_at_10": 0.01179890560875513, - "hits_at_3": 0.0070109439124487, - "hits_at_5": 0.008891928864569083, - "inverse_arithmetic_mean_rank": 4.851756966672838e-05, - "inverse_geometric_mean_rank": 8.675116987433285e-05, - "inverse_harmonic_mean_rank": 0.007190167496207637, - "inverse_median_rank": 4.828585224458948e-05, - "median_absolute_deviation": 17863.132829664744, - "median_rank": 20710.0, - "standard_deviation": 13176.8193359375, - "variance": 173628560.0, - "z_arithmetic_mean_rank": -2.211966873845199, - "z_geometric_mean_rank": 17.4014628243267, - "z_hits_at_k": 56.25842171670579, - "z_inverse_harmonic_mean_rank": 83.0898787171869 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.0044911327331056, - "adjusted_arithmetic_mean_rank_index": -0.004491354330853348, - "adjusted_geometric_mean_rank_index": 0.2916570535589309, - "adjusted_hits_at_k": 0.014462749736302826, - "adjusted_inverse_harmonic_mean_rank": 0.009295211355255004, - "arithmetic_mean_rank": 20359.076949384405, - "count": 2924.0, - "geometric_mean_rank": 10566.530983280216, - "harmonic_mean_rank": 104.50813006584458, - "hits_at_1": 0.006155950752393981, - "hits_at_10": 0.014705882352941176, - "hits_at_3": 0.009575923392612859, - "hits_at_5": 0.012311901504787962, - "inverse_arithmetic_mean_rank": 4.9118140399299235e-05, - "inverse_geometric_mean_rank": 9.463843919847812e-05, - "inverse_harmonic_mean_rank": 0.009568633553867601, - "inverse_median_rank": 4.81278275098662e-05, - "median_absolute_deviation": 19138.912038688817, - "median_rank": 20778.0, - "standard_deviation": 13625.52007285251, - "variance": 185654797.2557067, - "z_arithmetic_mean_rank": -0.4206444882002103, - "z_geometric_mean_rank": 15.786747812228626, - "z_hits_at_k": 49.785272491856105, - "z_inverse_harmonic_mean_rank": 78.95506001715788 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.0044911496068185, - "adjusted_arithmetic_mean_rank_index": -0.004491371205398886, - "adjusted_geometric_mean_rank_index": 0.29165704689618044, - "adjusted_hits_at_k": 0.014462749736302826, - "adjusted_inverse_harmonic_mean_rank": 0.009295211354996282, - "arithmetic_mean_rank": 20359.07729138167, - "count": 2924.0, - "geometric_mean_rank": 10566.531082660744, - "harmonic_mean_rank": 104.50813006866956, - "hits_at_1": 0.006155950752393981, - "hits_at_10": 0.014705882352941176, - "hits_at_3": 0.009575923392612859, - "hits_at_5": 0.012311901504787962, - "inverse_arithmetic_mean_rank": 4.911813957419948e-05, - "inverse_geometric_mean_rank": 9.463843830838298e-05, - "inverse_harmonic_mean_rank": 0.00956863355360895, - "inverse_median_rank": 4.81278275098662e-05, - "median_absolute_deviation": 19138.912038688817, - "median_rank": 20778.0, - "standard_deviation": 13625.520474534556, - "variance": 185654808.20196038, - "z_arithmetic_mean_rank": -0.420646068610921, - "z_geometric_mean_rank": 15.78674745158876, - "z_hits_at_k": 49.785272491856105, - "z_inverse_harmonic_mean_rank": 78.95506001496025 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.004491094371774, - "adjusted_arithmetic_mean_rank_index": -0.004491315967628973, - "adjusted_geometric_mean_rank_index": 0.29165651190528796, - "adjusted_hits_at_k": 0.014462749736302826, - "adjusted_inverse_harmonic_mean_rank": 0.009295210999538096, - "arithmetic_mean_rank": 20359.076171875, - "count": 2924.0, - "geometric_mean_rank": 10566.5390625, - "harmonic_mean_rank": 104.5081339498945, - "hits_at_1": 0.006155950752393981, - "hits_at_10": 0.014705882352941176, - "hits_at_3": 0.009575923392612859, - "hits_at_5": 0.012311901504787962, - "inverse_arithmetic_mean_rank": 4.9118141760118306e-05, - "inverse_geometric_mean_rank": 9.463836613576859e-05, - "inverse_harmonic_mean_rank": 0.009568633198248867, - "inverse_median_rank": 4.812782572116703e-05, - "median_absolute_deviation": 19138.912038688817, - "median_rank": 20778.0, - "standard_deviation": 13625.5205078125, - "variance": 185654800.0, - "z_arithmetic_mean_rank": -0.4206408952352092, - "z_geometric_mean_rank": 15.786718493721295, - "z_hits_at_k": 49.785272491856105, - "z_inverse_harmonic_mean_rank": 78.95505699563974 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.028902389254621, - "adjusted_arithmetic_mean_rank_index": -0.028903814699699826, - "adjusted_geometric_mean_rank_index": 0.15736049054991996, - "adjusted_hits_at_k": 0.00864747069608476, - "adjusted_inverse_harmonic_mean_rank": 0.004537077020969589, - "arithmetic_mean_rank": 20863.100547195623, - "count": 2924.0, - "geometric_mean_rank": 12575.27000338155, - "harmonic_mean_rank": 207.82671892798226, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.008891928864569083, - "hits_at_3": 0.004445964432284542, - "hits_at_5": 0.005471956224350205, - "inverse_arithmetic_mean_rank": 4.7931514193580305e-05, - "inverse_geometric_mean_rank": 7.952115538919603e-05, - "inverse_harmonic_mean_rank": 0.004811700849429894, - "inverse_median_rank": 4.8497781226508885e-05, - "median_absolute_deviation": 16431.680387697586, - "median_rank": 20619.5, - "standard_deviation": 12707.291847193621, - "variance": 161475266.08975345, - "z_arithmetic_mean_rank": -2.7070331259667455, - "z_geometric_mean_rank": 8.517571193494355, - "z_hits_at_k": 29.773919702326012, - "z_inverse_harmonic_mean_rank": 38.54726950873501 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.0289024061208485, - "adjusted_arithmetic_mean_rank_index": -0.02890383156675913, - "adjusted_geometric_mean_rank_index": 0.1573604831171146, - "adjusted_hits_at_k": 0.00864747069608476, - "adjusted_inverse_harmonic_mean_rank": 0.004537077020742053, - "arithmetic_mean_rank": 20863.100889192887, - "count": 2924.0, - "geometric_mean_rank": 12575.27011429743, - "harmonic_mean_rank": 207.82671893780727, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.008891928864569083, - "hits_at_3": 0.004445964432284542, - "hits_at_5": 0.005471956224350205, - "inverse_arithmetic_mean_rank": 4.793151340786552e-05, - "inverse_geometric_mean_rank": 7.952115468780682e-05, - "inverse_harmonic_mean_rank": 0.0048117008492024205, - "inverse_median_rank": 4.8497781226508885e-05, - "median_absolute_deviation": 16431.680387697586, - "median_rank": 20619.5, - "standard_deviation": 12707.292329251442, - "variance": 161475278.34105256, - "z_arithmetic_mean_rank": -2.7070347056782538, - "z_geometric_mean_rank": 8.517570791173235, - "z_hits_at_k": 29.773919702326012, - "z_inverse_harmonic_mean_rank": 38.547269506801854 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.0289023430042623, - "adjusted_arithmetic_mean_rank_index": -0.02890376844706033, - "adjusted_geometric_mean_rank_index": 0.15736052218886976, - "adjusted_hits_at_k": 0.00864747069608476, - "adjusted_inverse_harmonic_mean_rank": 0.004537077313478786, - "arithmetic_mean_rank": 20863.099609375, - "count": 2924.0, - "geometric_mean_rank": 12575.26953125, - "harmonic_mean_rank": 207.82670629742728, - "hits_at_1": 0.0023939808481532147, - "hits_at_10": 0.008891928864569083, - "hits_at_3": 0.004445964432284542, - "hits_at_5": 0.005471956224350205, - "inverse_arithmetic_mean_rank": 4.793151674675755e-05, - "inverse_geometric_mean_rank": 7.952115993248299e-05, - "inverse_harmonic_mean_rank": 0.004811701141858395, - "inverse_median_rank": 4.8497782699996606e-05, - "median_absolute_deviation": 16431.680387697586, - "median_rank": 20619.5, - "standard_deviation": 12707.2919921875, - "variance": 161475264.0, - "z_arithmetic_mean_rank": -2.7070287941016034, - "z_geometric_mean_rank": 8.51757290603991, - "z_hits_at_k": 29.773919702326012, - "z_inverse_harmonic_mean_rank": 38.547271993910115 - } - } - }, - "times": { - "evaluation": 13.313127517700195, - "training": 2.919541358947754 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json deleted file mode 100644 index fafc4ec..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with RotatE as described by Sun et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json deleted file mode 100644 index c271294..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/fb15k/sun2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.0029586357451772804, - 0.0027738664607501995, - 0.002592642678937381, - 0.0024149229981592704, - 0.0022405725565874963 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.46055182855559323, - "adjusted_arithmetic_mean_rank_index": 0.5395212094828172, - "adjusted_geometric_mean_rank_index": 0.8576850966163081, - "adjusted_hits_at_k": 0.10618046742846822, - "adjusted_inverse_harmonic_mean_rank": 0.06260713816785825, - "arithmetic_mean_rank": 3402.0283811007093, - "count": 118142.0, - "geometric_mean_rank": 773.8566421415263, - "harmonic_mean_rank": 15.80925834723783, - "hits_at_1": 0.04050210763318718, - "hits_at_10": 0.10678674815053071, - "hits_at_3": 0.06778283760220752, - "hits_at_5": 0.08225694503224933, - "inverse_arithmetic_mean_rank": 0.0002939422861829433, - "inverse_geometric_mean_rank": 0.0012922290066964567, - "inverse_harmonic_mean_rank": 0.06325407416564348, - "inverse_median_rank": 0.0005046681806712087, - "median_absolute_deviation": 2890.333024976671, - "median_rank": 1981.5, - "standard_deviation": 3716.0480418830743, - "variance": 13809013.04958303, - "z_arithmetic_mean_rank": 320.9165450339127, - "z_geometric_mean_rank": 295.3132134234969, - "z_hits_at_k": 1400.8378652422487, - "z_inverse_harmonic_mean_rank": 2040.2333364354065 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.46055483303303296, - "adjusted_arithmetic_mean_rank_index": 0.5395182045985893, - "adjusted_geometric_mean_rank_index": 0.8576832347776194, - "adjusted_hits_at_k": 0.10618046742846822, - "adjusted_inverse_harmonic_mean_rank": 0.06260632151148991, - "arithmetic_mean_rank": 3402.050574732102, - "count": 118142.0, - "geometric_mean_rank": 773.8667530600802, - "harmonic_mean_rank": 15.809462318094637, - "hits_at_1": 0.04050210763318718, - "hits_at_10": 0.10678674815053071, - "hits_at_3": 0.06778283760220752, - "hits_at_5": 0.08224848064193936, - "inverse_arithmetic_mean_rank": 0.00029394036861981276, - "inverse_geometric_mean_rank": 0.0012922121231410022, - "inverse_harmonic_mean_rank": 0.06325325807288558, - "inverse_median_rank": 0.0005045408678102926, - "median_absolute_deviation": 2891.074326085924, - "median_rank": 1982.0, - "standard_deviation": 3716.0616388245403, - "variance": 13809114.103543328, - "z_arithmetic_mean_rank": 320.9147576768122, - "z_geometric_mean_rank": 295.3125723658777, - "z_hits_at_k": 1400.8378652422487, - "z_inverse_harmonic_mean_rank": 2040.2067233431 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.4605533406584393, - "adjusted_arithmetic_mean_rank_index": 0.5395196971752416, - "adjusted_geometric_mean_rank_index": 0.8576842786891514, - "adjusted_hits_at_k": 0.10618046742846822, - "adjusted_inverse_harmonic_mean_rank": 0.0626066837363943, - "arithmetic_mean_rank": 3402.03955078125, - "count": 118142.0, - "geometric_mean_rank": 773.861083984375, - "harmonic_mean_rank": 15.809371846927815, - "hits_at_1": 0.04050210763318718, - "hits_at_10": 0.10678674815053071, - "hits_at_3": 0.06778283760220752, - "hits_at_5": 0.08224848064193936, - "inverse_arithmetic_mean_rank": 0.00029394132434390485, - "inverse_geometric_mean_rank": 0.0012922215973958373, - "inverse_harmonic_mean_rank": 0.06325362004780265, - "inverse_median_rank": 0.0005046045407652855, - "median_absolute_deviation": 2890.7036755312974, - "median_rank": 1981.75, - "standard_deviation": 3716.054931640625, - "variance": 13809065.0, - "z_arithmetic_mean_rank": 320.9156454872152, - "z_geometric_mean_rank": 295.31293179950933, - "z_hits_at_k": 1400.8378652422487, - "z_inverse_harmonic_mean_rank": 2040.2185274815236 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.49774717752314473, - "adjusted_arithmetic_mean_rank_index": 0.5023210100871964, - "adjusted_geometric_mean_rank_index": 0.8015392564962838, - "adjusted_hits_at_k": 0.07273844154098844, - "adjusted_inverse_harmonic_mean_rank": 0.03726819296769626, - "arithmetic_mean_rank": 3666.7785207631496, - "count": 59071.0, - "geometric_mean_rank": 1075.6624565891095, - "harmonic_mean_rank": 26.36126849123481, - "hits_at_1": 0.0215672665097933, - "hits_at_10": 0.07336933520678506, - "hits_at_3": 0.037124815899510756, - "hits_at_5": 0.05058319649235665, - "inverse_arithmetic_mean_rank": 0.0002727189532548791, - "inverse_geometric_mean_rank": 0.0009296596658871663, - "inverse_harmonic_mean_rank": 0.03793444159686407, - "inverse_median_rank": 0.00041999160016799666, - "median_absolute_deviation": 3337.33759385611, - "median_rank": 2381.0, - "standard_deviation": 3753.600259522372, - "variance": 14089514.90828642, - "z_arithmetic_mean_rank": 211.24527583191812, - "z_geometric_mean_rank": 195.13481015090386, - "z_hits_at_k": 677.5273694704455, - "z_inverse_harmonic_mean_rank": 857.4643858375553 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.4977502821195721, - "adjusted_arithmetic_mean_rank_index": 0.5023179050692781, - "adjusted_geometric_mean_rank_index": 0.8015365635075814, - "adjusted_hits_at_k": 0.07273844154098844, - "adjusted_inverse_harmonic_mean_rank": 0.03726745035530307, - "arithmetic_mean_rank": 3666.801391545767, - "count": 59071.0, - "geometric_mean_rank": 1075.6770390894235, - "harmonic_mean_rank": 26.36178419778149, - "hits_at_1": 0.0215672665097933, - "hits_at_10": 0.07336933520678506, - "hits_at_3": 0.037124815899510756, - "hits_at_5": 0.05056626771173672, - "inverse_arithmetic_mean_rank": 0.00027271725223668104, - "inverse_geometric_mean_rank": 0.0009296470628828471, - "inverse_harmonic_mean_rank": 0.03793369949838814, - "inverse_median_rank": 0.00041999160016799666, - "median_absolute_deviation": 3337.33759385611, - "median_rank": 2381.0, - "standard_deviation": 3753.612967564641, - "variance": 14089610.310269432, - "z_arithmetic_mean_rank": 211.24397005263864, - "z_geometric_mean_rank": 195.13415454254167, - "z_hits_at_k": 677.5273694704455, - "z_inverse_harmonic_mean_rank": 857.4472998554097 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.497748741075718, - "adjusted_arithmetic_mean_rank_index": 0.5023194463223497, - "adjusted_geometric_mean_rank_index": 0.8015379454914331, - "adjusted_hits_at_k": 0.07273844154098844, - "adjusted_inverse_harmonic_mean_rank": 0.03726779246443373, - "arithmetic_mean_rank": 3666.7900390625, - "count": 59071.0, - "geometric_mean_rank": 1075.6695556640625, - "harmonic_mean_rank": 26.361546617879256, - "hits_at_1": 0.0215672665097933, - "hits_at_10": 0.07336933520678506, - "hits_at_3": 0.037124815899510756, - "hits_at_5": 0.05056626771173672, - "inverse_arithmetic_mean_rank": 0.0002727180835790932, - "inverse_geometric_mean_rank": 0.0009296535281464458, - "inverse_harmonic_mean_rank": 0.03793404137076569, - "inverse_median_rank": 0.0004199915856588632, - "median_absolute_deviation": 3337.33759385611, - "median_rank": 2381.0, - "standard_deviation": 3753.6064453125, - "variance": 14089562.0, - "z_arithmetic_mean_rank": 211.24461820874544, - "z_geometric_mean_rank": 195.13449098664515, - "z_hits_at_k": 677.5273694704455, - "z_inverse_harmonic_mean_rank": 857.4551710821112 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.42355838525406514, - "adjusted_arithmetic_mean_rank_index": 0.5765194496169137, - "adjusted_geometric_mean_rank_index": 0.8979622403295612, - "adjusted_hits_at_k": 0.13962235407383589, - "adjusted_inverse_harmonic_mean_rank": 0.08794598722854557, - "arithmetic_mean_rank": 3137.278241438269, - "count": 59071.0, - "geometric_mean_rank": 556.7305049257791, - "harmonic_mean_rank": 11.290032187524611, - "hits_at_1": 0.059436948756581065, - "hits_at_10": 0.14020416109427639, - "hits_at_3": 0.09844085930490427, - "hits_at_5": 0.113930693572142, - "inverse_arithmetic_mean_rank": 0.00031874762869026085, - "inverse_geometric_mean_rank": 0.0017962011981601685, - "inverse_harmonic_mean_rank": 0.0885737067344229, - "inverse_median_rank": 0.0006406149903907751, - "median_absolute_deviation": 2303.9638475577053, - "median_rank": 1561.0, - "standard_deviation": 3659.00613800757, - "variance": 13388325.917977072, - "z_arithmetic_mean_rank": 242.52025418259967, - "z_geometric_mean_rank": 218.6072188825533, - "z_hits_at_k": 1304.519655818148, - "z_inverse_harmonic_mean_rank": 2029.655259950779 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.42356129015598803, - "adjusted_arithmetic_mean_rank_index": 0.5765165443227522, - "adjusted_geometric_mean_rank_index": 0.897960954948751, - "adjusted_hits_at_k": 0.13962235407383589, - "adjusted_inverse_harmonic_mean_rank": 0.08794509652848301, - "arithmetic_mean_rank": 3137.299757918437, - "count": 59071.0, - "geometric_mean_rank": 556.7375055236851, - "harmonic_mean_rank": 11.290145643455114, - "hits_at_1": 0.059436948756581065, - "hits_at_10": 0.14020416109427639, - "hits_at_3": 0.09844085930490427, - "hits_at_5": 0.113930693572142, - "inverse_arithmetic_mean_rank": 0.0003187454426297756, - "inverse_geometric_mean_rank": 0.001796178612143919, - "inverse_harmonic_mean_rank": 0.08857281664738302, - "inverse_median_rank": 0.0006406149903907751, - "median_absolute_deviation": 2303.9638475577053, - "median_rank": 1561.0, - "standard_deviation": 3659.020621260588, - "variance": 13388431.90681022, - "z_arithmetic_mean_rank": 242.51903203358285, - "z_geometric_mean_rank": 218.60690595901207, - "z_hits_at_k": 1304.519655818148, - "z_inverse_harmonic_mean_rank": 2029.6347039922414 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.42355984618646597, - "adjusted_arithmetic_mean_rank_index": 0.5765179884872483, - "adjusted_geometric_mean_rank_index": 0.8979616193980569, - "adjusted_hits_at_k": 0.13962235407383589, - "adjusted_inverse_harmonic_mean_rank": 0.0879454954124831, - "arithmetic_mean_rank": 3137.2890625, - "count": 59071.0, - "geometric_mean_rank": 556.73388671875, - "harmonic_mean_rank": 11.290094833975683, - "hits_at_1": 0.059436948756581065, - "hits_at_10": 0.14020416109427639, - "hits_at_3": 0.09844085930490427, - "hits_at_5": 0.113930693572142, - "inverse_arithmetic_mean_rank": 0.000318746519042179, - "inverse_geometric_mean_rank": 0.001796190277673304, - "inverse_harmonic_mean_rank": 0.08857321525685193, - "inverse_median_rank": 0.0006406150059774518, - "median_absolute_deviation": 2303.9638475577053, - "median_rank": 1561.0, - "standard_deviation": 3659.013427734375, - "variance": 13388379.0, - "z_arithmetic_mean_rank": 242.51963953978392, - "z_geometric_mean_rank": 218.60706771794617, - "z_hits_at_k": 1304.519655818148, - "z_inverse_harmonic_mean_rank": 2029.6439096084882 - } - } - }, - "times": { - "evaluation": 86.14422821998596, - "training": 413.02298378944397 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json deleted file mode 100644 index 68c2561..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K-237 Dataset with RotatE as described by Sun et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json deleted file mode 100644 index c617246..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/fb15k237/sun2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.01334708773344851, - 0.013160222068615453, - 0.012974961877335761, - 0.012791537809202847, - 0.012609953025432497 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.5243715695086415, - "adjusted_arithmetic_mean_rank_index": 0.4756950948921632, - "adjusted_geometric_mean_rank_index": 0.8507377858132705, - "adjusted_hits_at_k": 0.10921825533888653, - "adjusted_inverse_harmonic_mean_rank": 0.07060651340848918, - "arithmetic_mean_rank": 3741.741926803014, - "count": 40876.0, - "geometric_mean_rank": 783.6524314926153, - "harmonic_mean_rank": 14.031377373872546, - "hits_at_1": 0.04961346511400333, - "hits_at_10": 0.1098444074762697, - "hits_at_3": 0.07478716117036892, - "hits_at_5": 0.08706820628241511, - "inverse_arithmetic_mean_rank": 0.0002672552034753533, - "inverse_geometric_mean_rank": 0.001276075923219315, - "inverse_harmonic_mean_rank": 0.07126884078123887, - "inverse_median_rank": 0.00044853106077595876, - "median_absolute_deviation": 3269.8791929141053, - "median_rank": 2229.5, - "standard_deviation": 4000.811539735375, - "variance": 16006492.976479743, - "z_arithmetic_mean_rank": 166.3665110929076, - "z_geometric_mean_rank": 172.29643521192267, - "z_hits_at_k": 832.5744171518359, - "z_inverse_harmonic_mean_rank": 1329.5633720102164 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.5243763624717652, - "adjusted_arithmetic_mean_rank_index": 0.47569030125725453, - "adjusted_geometric_mean_rank_index": 0.8507338518544061, - "adjusted_hits_at_k": 0.10921825533888653, - "adjusted_inverse_harmonic_mean_rank": 0.07060218592179501, - "arithmetic_mean_rank": 3741.776127801155, - "count": 40876.0, - "geometric_mean_rank": 783.6730591007979, - "harmonic_mean_rank": 14.032228812004098, - "hits_at_1": 0.04961346511400333, - "hits_at_10": 0.1098444074762697, - "hits_at_3": 0.07478716117036892, - "hits_at_5": 0.08706820628241511, - "inverse_arithmetic_mean_rank": 0.0002672527606796314, - "inverse_geometric_mean_rank": 0.001276042334730021, - "inverse_harmonic_mean_rank": 0.0712645163785053, - "inverse_median_rank": 0.00044853106077595876, - "median_absolute_deviation": 3269.8791929141053, - "median_rank": 2229.5, - "standard_deviation": 4000.8300621961807, - "variance": 16006641.186572695, - "z_arithmetic_mean_rank": 166.36483459818695, - "z_geometric_mean_rank": 172.2956384833655, - "z_hits_at_k": 832.5744171518359, - "z_inverse_harmonic_mean_rank": 1329.4818828171708 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.5243739326030625, - "adjusted_arithmetic_mean_rank_index": 0.47569273146652935, - "adjusted_geometric_mean_rank_index": 0.8507357655093295, - "adjusted_hits_at_k": 0.10921825533888653, - "adjusted_inverse_harmonic_mean_rank": 0.07060393544377082, - "arithmetic_mean_rank": 3741.7587890625, - "count": 40876.0, - "geometric_mean_rank": 783.6630249023438, - "harmonic_mean_rank": 14.03188457904077, - "hits_at_1": 0.04961346511400333, - "hits_at_10": 0.1098444074762697, - "hits_at_3": 0.07478716117036892, - "hits_at_5": 0.08706820628241511, - "inverse_arithmetic_mean_rank": 0.00026725398492999375, - "inverse_geometric_mean_rank": 0.0012760587269440293, - "inverse_harmonic_mean_rank": 0.07126626465369348, - "inverse_median_rank": 0.00044853106373921037, - "median_absolute_deviation": 3269.8791929141053, - "median_rank": 2229.5, - "standard_deviation": 4000.82080078125, - "variance": 16006567.0, - "z_arithmetic_mean_rank": 166.36568452378552, - "z_geometric_mean_rank": 172.29602604804998, - "z_hits_at_k": 832.5744171518359, - "z_inverse_harmonic_mean_rank": 1329.514827374624 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.630345844032263, - "adjusted_arithmetic_mean_rank_index": 0.36970669621218955, - "adjusted_geometric_mean_rank_index": 0.6374260958695077, - "adjusted_hits_at_k": 0.02195491082542058, - "adjusted_inverse_harmonic_mean_rank": 0.010283827488961611, - "arithmetic_mean_rank": 4435.515706037772, - "count": 20438.0, - "geometric_mean_rank": 1873.5241837633828, - "harmonic_mean_rank": 90.91202964026317, - "hits_at_1": 0.0034249926607300128, - "hits_at_10": 0.02265388002739994, - "hits_at_3": 0.008709267051570603, - "hits_at_5": 0.012868186711028476, - "inverse_arithmetic_mean_rank": 0.0002254529272974429, - "inverse_geometric_mean_rank": 0.0005337534517388942, - "inverse_harmonic_mean_rank": 0.010999644425022489, - "inverse_median_rank": 0.00029304029304029304, - "median_absolute_deviation": 4254.327066001825, - "median_rank": 3412.5, - "standard_deviation": 3946.1367222079466, - "variance": 15571995.030358076, - "z_arithmetic_mean_rank": 91.33094104479204, - "z_geometric_mean_rank": 91.28864498343353, - "z_hits_at_k": 117.36730933110842, - "z_inverse_harmonic_mean_rank": 135.80611068364766 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6303518100379835, - "adjusted_arithmetic_mean_rank_index": 0.36970072935849974, - "adjusted_geometric_mean_rank_index": 0.6374135161138859, - "adjusted_hits_at_k": 0.02195491082542058, - "adjusted_inverse_harmonic_mean_rank": 0.010275601268548235, - "arithmetic_mean_rank": 4435.5576866621, - "count": 20438.0, - "geometric_mean_rank": 1873.5891523018224, - "harmonic_mean_rank": 90.98002096591132, - "hits_at_1": 0.0034249926607300128, - "hits_at_10": 0.02265388002739994, - "hits_at_3": 0.008709267051570603, - "hits_at_5": 0.012868186711028476, - "inverse_arithmetic_mean_rank": 0.00022545079348354325, - "inverse_geometric_mean_rank": 0.0005337349433153138, - "inverse_harmonic_mean_rank": 0.01099142415426221, - "inverse_median_rank": 0.00029304029304029304, - "median_absolute_deviation": 4254.327066001825, - "median_rank": 3412.5, - "standard_deviation": 3946.154266638542, - "variance": 15572133.496109571, - "z_arithmetic_mean_rank": 91.32946701587088, - "z_geometric_mean_rank": 91.28684338031061, - "z_hits_at_k": 117.36730933110842, - "z_inverse_harmonic_mean_rank": 135.697476908804 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6303488163402171, - "adjusted_arithmetic_mean_rank_index": 0.36970372348177094, - "adjusted_geometric_mean_rank_index": 0.637419598572049, - "adjusted_hits_at_k": 0.02195491082542058, - "adjusted_inverse_harmonic_mean_rank": 0.010278897792388431, - "arithmetic_mean_rank": 4435.53662109375, - "count": 20438.0, - "geometric_mean_rank": 1873.5577392578125, - "harmonic_mean_rank": 90.95276234198829, - "hits_at_1": 0.0034249926607300128, - "hits_at_10": 0.02265388002739994, - "hits_at_3": 0.008709267051570603, - "hits_at_5": 0.012868186711028476, - "inverse_arithmetic_mean_rank": 0.00022545186220668256, - "inverse_geometric_mean_rank": 0.0005337438778951764, - "inverse_harmonic_mean_rank": 0.010994718293875837, - "inverse_median_rank": 0.00029304029885679483, - "median_absolute_deviation": 4254.327066001825, - "median_rank": 3412.5, - "standard_deviation": 3946.1455078125, - "variance": 15572064.0, - "z_arithmetic_mean_rank": 91.33020667273607, - "z_geometric_mean_rank": 91.2877144763757, - "z_hits_at_k": 117.36730933110842, - "z_inverse_harmonic_mean_rank": 135.7410101246221 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.42129850782168216, - "adjusted_arithmetic_mean_rank_index": 0.5787814929419499, - "adjusted_geometric_mean_rank_index": 0.9386177352290703, - "adjusted_hits_at_k": 0.196479550332964, - "adjusted_inverse_harmonic_mean_rank": 0.13092791837765114, - "arithmetic_mean_rank": 3047.968147568255, - "count": 20438.0, - "geometric_mean_rank": 327.78393719514855, - "harmonic_mean_rank": 7.602363709859949, - "hits_at_1": 0.09580193756727665, - "hits_at_10": 0.19703493492513946, - "hits_at_3": 0.14086505528916723, - "hits_at_5": 0.16126822585380174, - "inverse_arithmetic_mean_rank": 0.0003280874181043607, - "inverse_geometric_mean_rank": 0.0030507901288788376, - "inverse_harmonic_mean_rank": 0.13153803713745524, - "inverse_median_rank": 0.00102880658436214, - "median_absolute_deviation": 1439.6067541689395, - "median_rank": 972.0, - "standard_deviation": 3934.2530179303467, - "variance": 15478346.809094042, - "z_arithmetic_mean_rank": 143.30325123133036, - "z_geometric_mean_rank": 134.4177059185862, - "z_hits_at_k": 1068.041113623353, - "z_inverse_harmonic_mean_rank": 1758.0301037163845 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.4213021598560998, - "adjusted_arithmetic_mean_rank_index": 0.5787778404026682, - "adjusted_geometric_mean_rank_index": 0.9386166289573937, - "adjusted_hits_at_k": 0.196479550332964, - "adjusted_inverse_harmonic_mean_rank": 0.13092748954188677, - "arithmetic_mean_rank": 3047.9945689402093, - "count": 20438.0, - "geometric_mean_rank": 327.78982671088136, - "harmonic_mean_rank": 7.6023884775042605, - "hits_at_1": 0.09580193756727665, - "hits_at_10": 0.19703493492513946, - "hits_at_3": 0.14086505528916723, - "hits_at_5": 0.16126822585380174, - "inverse_arithmetic_mean_rank": 0.00032808457409676456, - "inverse_geometric_mean_rank": 0.0030507353142537413, - "inverse_harmonic_mean_rank": 0.13153760860274843, - "inverse_median_rank": 0.00102880658436214, - "median_absolute_deviation": 1439.6067541689395, - "median_rank": 972.0, - "standard_deviation": 3934.2703483879673, - "variance": 15478483.174204776, - "z_arithmetic_mean_rank": 143.3023468818294, - "z_geometric_mean_rank": 134.41754749147012, - "z_hits_at_k": 1068.041113623353, - "z_inverse_harmonic_mean_rank": 1758.0243455389639 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.4213003458723233, - "adjusted_arithmetic_mean_rank_index": 0.5787796546372133, - "adjusted_geometric_mean_rank_index": 0.9386171795012745, - "adjusted_hits_at_k": 0.196479550332964, - "adjusted_inverse_harmonic_mean_rank": 0.1309276965775306, - "arithmetic_mean_rank": 3047.9814453125, - "count": 20438.0, - "geometric_mean_rank": 327.7868957519531, - "harmonic_mean_rank": 7.602376520027175, - "hits_at_1": 0.09580193756727665, - "hits_at_10": 0.19703493492513946, - "hits_at_3": 0.14086505528916723, - "hits_at_5": 0.16126822585380174, - "inverse_arithmetic_mean_rank": 0.00032808599644340575, - "inverse_geometric_mean_rank": 0.0030507626943290234, - "inverse_harmonic_mean_rank": 0.13153781549304605, - "inverse_median_rank": 0.001028806553222239, - "median_absolute_deviation": 1439.6067541689395, - "median_rank": 972.0, - "standard_deviation": 3934.26171875, - "variance": 15478415.0, - "z_arithmetic_mean_rank": 143.3027960767535, - "z_geometric_mean_rank": 134.4176263338387, - "z_hits_at_k": 1068.041113623353, - "z_inverse_harmonic_mean_rank": 1758.0271255029234 - } - } - }, - "times": { - "evaluation": 29.337369203567505, - "training": 459.2732901573181 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json deleted file mode 100644 index 0b5d5b4..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with RotatE as described by Sun et al., 2019;" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json deleted file mode 100644 index 8b73ac6..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/wn18/sun2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.013424746205139526, - 0.013122108396254868, - 0.012823446353122617, - 0.012530585870327324, - 0.012243712673720453 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7855334638470861, - "adjusted_arithmetic_mean_rank_index": 0.2144770172444035, - "adjusted_geometric_mean_rank_index": 0.5966660934317238, - "adjusted_hits_at_k": 0.033463829540325396, - "adjusted_inverse_harmonic_mean_rank": 0.01997912249413563, - "arithmetic_mean_rank": 16074.5543, - "count": 10000.0, - "geometric_mean_rank": 6074.265576986406, - "harmonic_mean_rank": 49.38943424664326, - "hits_at_1": 0.0132, - "hits_at_10": 0.0337, - "hits_at_3": 0.0201, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 6.221012298922653e-05, - "inverse_geometric_mean_rank": 0.00016462895593315907, - "inverse_harmonic_mean_rank": 0.020247245493968474, - "inverse_median_rank": 7.060401736858827e-05, - "median_absolute_deviation": 16906.854398728632, - "median_rank": 14163.5, - "standard_deviation": 12884.519360893968, - "variance": 166010839.16125152, - "z_arithmetic_mean_rank": 37.14755487583042, - "z_geometric_mean_rank": 59.71486914649745, - "z_hits_at_k": 214.0517790829075, - "z_inverse_harmonic_mean_rank": 315.3461920068905 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7855372071461044, - "adjusted_arithmetic_mean_rank_index": 0.21447327376244818, - "adjusted_geometric_mean_rank_index": 0.5966631514048026, - "adjusted_hits_at_k": 0.033463829540325396, - "adjusted_inverse_harmonic_mean_rank": 0.019979117339909674, - "arithmetic_mean_rank": 16074.6309, - "count": 10000.0, - "geometric_mean_rank": 6074.309877032933, - "harmonic_mean_rank": 49.38944681599364, - "hits_at_1": 0.0132, - "hits_at_10": 0.0337, - "hits_at_3": 0.0201, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 6.220982654102497e-05, - "inverse_geometric_mean_rank": 0.00016462775529134868, - "inverse_harmonic_mean_rank": 0.020247240341152658, - "inverse_median_rank": 7.060152499293985e-05, - "median_absolute_deviation": 16906.854398728632, - "median_rank": 14164.0, - "standard_deviation": 12884.537494161952, - "variance": 166011306.43846518, - "z_arithmetic_mean_rank": 37.14690650239095, - "z_geometric_mean_rank": 59.71457470584704, - "z_hits_at_k": 214.0517790829075, - "z_inverse_harmonic_mean_rank": 315.3461106536915 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7855352962493756, - "adjusted_arithmetic_mean_rank_index": 0.21447518475256344, - "adjusted_geometric_mean_rank_index": 0.5966644040168032, - "adjusted_hits_at_k": 0.033463829540325396, - "adjusted_inverse_harmonic_mean_rank": 0.019979119736229026, - "arithmetic_mean_rank": 16074.591796875, - "count": 10000.0, - "geometric_mean_rank": 6074.291015625, - "harmonic_mean_rank": 49.389440972209925, - "hits_at_1": 0.0132, - "hits_at_10": 0.0337, - "hits_at_3": 0.0201, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 6.220997602213174e-05, - "inverse_geometric_mean_rank": 0.00016462826170027256, - "inverse_harmonic_mean_rank": 0.020247242736816405, - "inverse_median_rank": 7.060277130221948e-05, - "median_absolute_deviation": 16906.854398728632, - "median_rank": 14163.75, - "standard_deviation": 12884.5283203125, - "variance": 166011072.0, - "z_arithmetic_mean_rank": 37.1472374870861, - "z_geometric_mean_rank": 59.71470006836142, - "z_hits_at_k": 214.0517790829075, - "z_inverse_harmonic_mean_rank": 315.34614847668314 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7912971127949484, - "adjusted_arithmetic_mean_rank_index": 0.2087130868604813, - "adjusted_geometric_mean_rank_index": 0.576690351513085, - "adjusted_hits_at_k": 0.03196345740463281, - "adjusted_inverse_harmonic_mean_rank": 0.016330660837363887, - "arithmetic_mean_rank": 16192.1218, - "count": 5000.0, - "geometric_mean_rank": 6375.223395588098, - "harmonic_mean_rank": 60.24173415886561, - "hits_at_1": 0.0086, - "hits_at_10": 0.0322, - "hits_at_3": 0.0176, - "hits_at_5": 0.022, - "inverse_arithmetic_mean_rank": 6.175842871932942e-05, - "inverse_geometric_mean_rank": 0.00015685724843650793, - "inverse_harmonic_mean_rank": 0.016599787737897196, - "inverse_median_rank": 6.924248719013987e-05, - "median_absolute_deviation": 16699.29008813785, - "median_rank": 14442.0, - "standard_deviation": 12794.627688235589, - "variance": 163702497.68056476, - "z_arithmetic_mean_rank": 25.561369516008, - "z_geometric_mean_rank": 40.81416845600836, - "z_hits_at_k": 144.5695676042832, - "z_inverse_harmonic_mean_rank": 182.2614735761567 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7913008757284563, - "adjusted_arithmetic_mean_rank_index": 0.2087093237430726, - "adjusted_geometric_mean_rank_index": 0.5766867904434522, - "adjusted_hits_at_k": 0.03196345740463281, - "adjusted_inverse_harmonic_mean_rank": 0.01633065154077127, - "arithmetic_mean_rank": 16192.1988, - "count": 5000.0, - "geometric_mean_rank": 6375.277018399314, - "harmonic_mean_rank": 60.24176788760712, - "hits_at_1": 0.0086, - "hits_at_10": 0.0322, - "hits_at_3": 0.0176, - "hits_at_5": 0.022, - "inverse_arithmetic_mean_rank": 6.175813503475513e-05, - "inverse_geometric_mean_rank": 0.00015685592910142705, - "inverse_harmonic_mean_rank": 0.01659977844384808, - "inverse_median_rank": 6.924009001211702e-05, - "median_absolute_deviation": 16699.29008813785, - "median_rank": 14442.5, - "standard_deviation": 12794.6465912302, - "variance": 163702981.39447856, - "z_arithmetic_mean_rank": 25.560908642010773, - "z_geometric_mean_rank": 40.8139164280431, - "z_hits_at_k": 144.5695676042832, - "z_inverse_harmonic_mean_rank": 182.2613698197511 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7912989872367453, - "adjusted_arithmetic_mean_rank_index": 0.20871121232707734, - "adjusted_geometric_mean_rank_index": 0.5766885847246386, - "adjusted_hits_at_k": 0.03196345740463281, - "adjusted_inverse_harmonic_mean_rank": 0.016330654897157886, - "arithmetic_mean_rank": 16192.16015625, - "count": 5000.0, - "geometric_mean_rank": 6375.25, - "harmonic_mean_rank": 60.24175571037813, - "hits_at_1": 0.0086, - "hits_at_10": 0.0322, - "hits_at_3": 0.0176, - "hits_at_5": 0.022, - "inverse_arithmetic_mean_rank": 6.175828457344323e-05, - "inverse_geometric_mean_rank": 0.00015685659309383482, - "inverse_harmonic_mean_rank": 0.016599781799316406, - "inverse_median_rank": 6.924128683749586e-05, - "median_absolute_deviation": 16699.29008813785, - "median_rank": 14442.25, - "standard_deviation": 12794.63671875, - "variance": 163702736.0, - "z_arithmetic_mean_rank": 25.56113993940727, - "z_geometric_mean_rank": 40.81404341489906, - "z_hits_at_k": 144.5695676042832, - "z_inverse_harmonic_mean_rank": 182.2614072793479 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.7797700821308655, - "adjusted_arithmetic_mean_rank_index": 0.22024068037057487, - "adjusted_geometric_mean_rank_index": 0.615737701058511, - "adjusted_hits_at_k": 0.03496420165889074, - "adjusted_inverse_harmonic_mean_rank": 0.023627584108453507, - "arithmetic_mean_rank": 15956.9868, - "count": 5000.0, - "geometric_mean_rank": 5787.515199121641, - "harmonic_mean_rank": 41.85027909891856, - "hits_at_1": 0.0178, - "hits_at_10": 0.0352, - "hits_at_3": 0.0226, - "hits_at_5": 0.0264, - "inverse_arithmetic_mean_rank": 6.266847322327796e-05, - "inverse_geometric_mean_rank": 0.00017278572333628912, - "inverse_harmonic_mean_rank": 0.02389470325003975, - "inverse_median_rank": 7.156915369475756e-05, - "median_absolute_deviation": 17118.12521486568, - "median_rank": 13972.5, - "standard_deviation": 12972.722783125591, - "variance": 168291536.40782577, - "z_arithmetic_mean_rank": 26.973173800370787, - "z_geometric_mean_rank": 43.57766952935073, - "z_hits_at_k": 158.14551992692725, - "z_inverse_harmonic_mean_rank": 263.7063375478915 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.7797738057963045, - "adjusted_arithmetic_mean_rank_index": 0.22023695652316255, - "adjusted_geometric_mean_rank_index": 0.6157353278228198, - "adjusted_hits_at_k": 0.03496420165889074, - "adjusted_inverse_harmonic_mean_rank": 0.023627583096594164, - "arithmetic_mean_rank": 15957.063, - "count": 5000.0, - "geometric_mean_rank": 5787.55093711737, - "harmonic_mean_rank": 41.85028087065065, - "hits_at_1": 0.0178, - "hits_at_10": 0.0352, - "hits_at_3": 0.0226, - "hits_at_5": 0.0264, - "inverse_arithmetic_mean_rank": 6.266817396158678e-05, - "inverse_geometric_mean_rank": 0.00017278465638836764, - "inverse_harmonic_mean_rank": 0.023894702238457233, - "inverse_median_rank": 7.156915369475756e-05, - "median_absolute_deviation": 17118.12521486568, - "median_rank": 13972.5, - "standard_deviation": 12972.74015234372, - "variance": 168291987.06023097, - "z_arithmetic_mean_rank": 26.972717735745093, - "z_geometric_mean_rank": 43.5775015680898, - "z_hits_at_k": 158.14551992692725, - "z_inverse_harmonic_mean_rank": 263.7063262545777 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7797719202172769, - "adjusted_arithmetic_mean_rank_index": 0.22023884219433731, - "adjusted_geometric_mean_rank_index": 0.6157367973009195, - "adjusted_hits_at_k": 0.03496420165889074, - "adjusted_inverse_harmonic_mean_rank": 0.02362758453284627, - "arithmetic_mean_rank": 15957.0244140625, - "count": 5000.0, - "geometric_mean_rank": 5787.52880859375, - "harmonic_mean_rank": 41.85027835582098, - "hits_at_1": 0.0178, - "hits_at_10": 0.0352, - "hits_at_3": 0.0226, - "hits_at_5": 0.0264, - "inverse_arithmetic_mean_rank": 6.266832497203723e-05, - "inverse_geometric_mean_rank": 0.00017278532322961837, - "inverse_harmonic_mean_rank": 0.023894703674316405, - "inverse_median_rank": 7.156915671657771e-05, - "median_absolute_deviation": 17118.12521486568, - "median_rank": 13972.5, - "standard_deviation": 12972.732421875, - "variance": 168291776.0, - "z_arithmetic_mean_rank": 26.972948676442513, - "z_geometric_mean_rank": 43.577605567618974, - "z_hits_at_k": 158.14551992692725, - "z_inverse_harmonic_mean_rank": 263.7063422845188 - } - } - }, - "times": { - "evaluation": 11.969523429870605, - "training": 490.47644114494324 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json deleted file mode 100644 index 8dcbe52..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Could not find number of epochs. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18RR Dataset with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json b/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json deleted file mode 100644 index 6337d50..0000000 --- a/nonasv/results/27c9fe15/cuda/rotate/wn18rr/sun2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 256, - "slice_size": null - }, - "losses": [ - 0.019261958489676396, - 0.019163661997862045, - 0.019064278557845992, - 0.018965127136297243, - 0.018866502912631956 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8828598120598528, - "adjusted_arithmetic_mean_rank_index": 0.11714596649184639, - "adjusted_geometric_mean_rank_index": 0.2923689706295878, - "adjusted_hits_at_k": 0.007450130412883449, - "adjusted_inverse_harmonic_mean_rank": 0.003932262355982899, - "arithmetic_mean_rank": 17897.81788645691, - "count": 5848.0, - "geometric_mean_rank": 10557.359337940927, - "harmonic_mean_rank": 237.69292302092035, - "hits_at_1": 0.002051983584131327, - "hits_at_10": 0.007694938440492476, - "hits_at_3": 0.004103967168262654, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.5872732997059354e-05, - "inverse_geometric_mean_rank": 9.472065579942994e-05, - "inverse_harmonic_mean_rank": 0.004207108849900364, - "inverse_median_rank": 5.940182363598562e-05, - "median_absolute_deviation": 15698.533590646566, - "median_rank": 16834.5, - "standard_deviation": 12192.032467473826, - "variance": 148645655.6879359, - "z_arithmetic_mean_rank": 15.516025046938196, - "z_geometric_mean_rank": 22.377492200661884, - "z_hits_at_k": 36.27248694079143, - "z_inverse_harmonic_mean_rank": 47.241813608270604 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8828636752828475, - "adjusted_arithmetic_mean_rank_index": 0.11714210307827799, - "adjusted_geometric_mean_rank_index": 0.2923638535782205, - "adjusted_hits_at_k": 0.007450130412883449, - "adjusted_inverse_harmonic_mean_rank": 0.003932259472704943, - "arithmetic_mean_rank": 17897.89620383037, - "count": 5848.0, - "geometric_mean_rank": 10557.435673532134, - "harmonic_mean_rank": 237.69308587530654, - "hits_at_1": 0.002051983584131327, - "hits_at_10": 0.007694938440492476, - "hits_at_3": 0.004103967168262654, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.587248850990586e-05, - "inverse_geometric_mean_rank": 9.471997092125652e-05, - "inverse_harmonic_mean_rank": 0.004207105967417995, - "inverse_median_rank": 5.940182363598562e-05, - "median_absolute_deviation": 15698.533590646566, - "median_rank": 16834.5, - "standard_deviation": 12192.040678461748, - "variance": 148645855.90526602, - "z_arithmetic_mean_rank": 15.515513336433017, - "z_geometric_mean_rank": 22.3771005490553, - "z_hits_at_k": 36.27248694079143, - "z_inverse_harmonic_mean_rank": 47.24177896885251 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8828617622546758, - "adjusted_arithmetic_mean_rank_index": 0.11714401620081982, - "adjusted_geometric_mean_rank_index": 0.2923660223318012, - "adjusted_hits_at_k": 0.007450130412883449, - "adjusted_inverse_harmonic_mean_rank": 0.00393226100790816, - "arithmetic_mean_rank": 17897.857421875, - "count": 5848.0, - "geometric_mean_rank": 10557.4033203125, - "harmonic_mean_rank": 237.69299916335532, - "hits_at_1": 0.002051983584131327, - "hits_at_10": 0.007694938440492476, - "hits_at_3": 0.004103967168262654, - "hits_at_5": 0.005129958960328317, - "inverse_arithmetic_mean_rank": 5.587260966422036e-05, - "inverse_geometric_mean_rank": 9.472026431467384e-05, - "inverse_harmonic_mean_rank": 0.0042071075021976, - "inverse_median_rank": 5.9401823818916455e-05, - "median_absolute_deviation": 15698.533590646566, - "median_rank": 16834.5, - "standard_deviation": 12192.037109375, - "variance": 148645760.0, - "z_arithmetic_mean_rank": 15.515766730196077, - "z_geometric_mean_rank": 22.377266542273503, - "z_hits_at_k": 36.27248694079143, - "z_inverse_harmonic_mean_rank": 47.24179741263312 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9376485514225431, - "adjusted_arithmetic_mean_rank_index": 0.062354525070938194, - "adjusted_geometric_mean_rank_index": 0.16130982596183663, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.0026470319205615617, - "arithmetic_mean_rank": 19004.308139534885, - "count": 2924.0, - "geometric_mean_rank": 12510.769545520707, - "harmonic_mean_rank": 342.19751112357585, - "hits_at_1": 0.0013679890560875513, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.004103967168262654, - "inverse_arithmetic_mean_rank": 5.26196477481697e-05, - "inverse_geometric_mean_rank": 7.993113424090167e-05, - "inverse_harmonic_mean_rank": 0.0029222889340036013, - "inverse_median_rank": 5.450779461462989e-05, - "median_absolute_deviation": 15325.659132692408, - "median_rank": 18346.0, - "standard_deviation": 11991.670556534038, - "variance": 143800162.73644537, - "z_arithmetic_mean_rank": 5.8399060402006455, - "z_geometric_mean_rank": 8.731342208322275, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 22.484326194166155 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9376525842399235, - "adjusted_arithmetic_mean_rank_index": 0.06235049205457388, - "adjusted_geometric_mean_rank_index": 0.16130319319545638, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.002647027911227845, - "arithmetic_mean_rank": 19004.389876880985, - "count": 2924.0, - "geometric_mean_rank": 12510.868478811492, - "harmonic_mean_rank": 342.197980484163, - "hits_at_1": 0.0013679890560875513, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.004103967168262654, - "inverse_arithmetic_mean_rank": 5.2619421432545395e-05, - "inverse_geometric_mean_rank": 7.993050216246843e-05, - "inverse_harmonic_mean_rank": 0.002922284925776411, - "inverse_median_rank": 5.450779461462989e-05, - "median_absolute_deviation": 15325.659132692408, - "median_rank": 18346.0, - "standard_deviation": 11991.672594145404, - "variance": 143800211.60517797, - "z_arithmetic_mean_rank": 5.839528322038262, - "z_geometric_mean_rank": 8.730983191426013, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 22.484292138223573 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9376505974920567, - "adjusted_arithmetic_mean_rank_index": 0.06235247890046913, - "adjusted_geometric_mean_rank_index": 0.16130733900159622, - "adjusted_hits_at_k": 0.005568626686370119, - "adjusted_inverse_harmonic_mean_rank": 0.0026470297253797593, - "arithmetic_mean_rank": 19004.349609375, - "count": 2924.0, - "geometric_mean_rank": 12510.806640625, - "harmonic_mean_rank": 342.1977681067189, - "hits_at_1": 0.0013679890560875513, - "hits_at_10": 0.005813953488372093, - "hits_at_3": 0.0027359781121751026, - "hits_at_5": 0.004103967168262654, - "inverse_arithmetic_mean_rank": 5.261953265289776e-05, - "inverse_geometric_mean_rank": 7.993089820956811e-05, - "inverse_harmonic_mean_rank": 0.002922286739427642, - "inverse_median_rank": 5.450779281090945e-05, - "median_absolute_deviation": 15325.659132692408, - "median_rank": 18346.0, - "standard_deviation": 11991.6708984375, - "variance": 143800176.0, - "z_arithmetic_mean_rank": 5.839714403053733, - "z_geometric_mean_rank": 8.731207594694197, - "z_hits_at_k": 19.168941041029797, - "z_inverse_harmonic_mean_rank": 22.484307547929475 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8280953773701203, - "adjusted_arithmetic_mean_rank_index": 0.17191310084239653, - "adjusted_geometric_mean_rank_index": 0.4030514945528445, - "adjusted_hits_at_k": 0.009331633932319346, - "adjusted_inverse_harmonic_mean_rank": 0.005217492647335803, - "arithmetic_mean_rank": 16791.327633378933, - "count": 2924.0, - "geometric_mean_rank": 8908.951266735969, - "harmonic_mean_rank": 182.08539160737905, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.009575923392612859, - "hits_at_3": 0.005471956224350205, - "hits_at_5": 0.006155950752393981, - "inverse_arithmetic_mean_rank": 5.955455231616901e-05, - "inverse_geometric_mean_rank": 0.00011224665732921632, - "inverse_harmonic_mean_rank": 0.005491928765797128, - "inverse_median_rank": 6.643414715163594e-05, - "median_absolute_deviation": 15321.21132603689, - "median_rank": 15052.5, - "standard_deviation": 12289.935202396848, - "variance": 151042507.27911326, - "z_arithmetic_mean_rank": 16.10079719936988, - "z_geometric_mean_rank": 21.816275403698548, - "z_hits_at_k": 32.1295473736814, - "z_inverse_harmonic_mean_rank": 44.32812013266561 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8280990710739625, - "adjusted_arithmetic_mean_rank_index": 0.17190940695638335, - "adjusted_geometric_mean_rank_index": 0.4030475821520433, - "adjusted_hits_at_k": 0.009331633932319346, - "adjusted_inverse_harmonic_mean_rank": 0.005217490890113478, - "arithmetic_mean_rank": 16791.402530779753, - "count": 2924.0, - "geometric_mean_rank": 8909.009649452995, - "harmonic_mean_rank": 182.085449852189, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.009575923392612859, - "hits_at_3": 0.005471956224350205, - "hits_at_5": 0.006155950752393981, - "inverse_arithmetic_mean_rank": 5.9554286675394374e-05, - "inverse_geometric_mean_rank": 0.00011224592175196477, - "inverse_harmonic_mean_rank": 0.0054919270090595775, - "inverse_median_rank": 6.643414715163594e-05, - "median_absolute_deviation": 15321.952627146144, - "median_rank": 15052.5, - "standard_deviation": 12289.948889574442, - "variance": 151042843.70835203, - "z_arithmetic_mean_rank": 16.10045124255052, - "z_geometric_mean_rank": 21.816063634198812, - "z_hits_at_k": 32.1295473736814, - "z_inverse_harmonic_mean_rank": 44.32810520320282 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8280972317327835, - "adjusted_arithmetic_mean_rank_index": 0.17191124638827748, - "adjusted_geometric_mean_rank_index": 0.40305006117741937, - "adjusted_hits_at_k": 0.009331633932319346, - "adjusted_inverse_harmonic_mean_rank": 0.005217491820124018, - "arithmetic_mean_rank": 16791.365234375, - "count": 2924.0, - "geometric_mean_rank": 8908.97265625, - "harmonic_mean_rank": 182.085419026098, - "hits_at_1": 0.0027359781121751026, - "hits_at_10": 0.009575923392612859, - "hits_at_3": 0.005471956224350205, - "hits_at_5": 0.006155950752393981, - "inverse_arithmetic_mean_rank": 5.955441883997992e-05, - "inverse_geometric_mean_rank": 0.00011224638728890568, - "inverse_harmonic_mean_rank": 0.00549192793881355, - "inverse_median_rank": 6.643414963036776e-05, - "median_absolute_deviation": 15321.581976591517, - "median_rank": 15052.5, - "standard_deviation": 12289.9423828125, - "variance": 151042672.0, - "z_arithmetic_mean_rank": 16.100623517494924, - "z_geometric_mean_rank": 21.81619781829457, - "z_hits_at_k": 32.1295473736814, - "z_inverse_harmonic_mean_rank": 44.328113104626304 - } - } - }, - "times": { - "evaluation": 7.80864691734314, - "training": 302.2507004737854 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json deleted file mode 100644 index 0fb86ad..0000000 --- a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15k Dataset with SimplE as described by Kazemi et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json b/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json deleted file mode 100644 index b27d575..0000000 --- a/nonasv/results/27c9fe15/cuda/simple/fb15k/kazemi2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.004399372436803763, - 0.0007778990072353506, - 0.0003317295822744947, - 0.00021465736021517208, - 0.0001767241696370522 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.2642043057767773, - "adjusted_arithmetic_mean_rank_index": -0.2642400774509712, - "adjusted_geometric_mean_rank_index": -0.36675010197140323, - "adjusted_hits_at_k": -7.738405941025406e-05, - "adjusted_inverse_harmonic_mean_rank": -0.00017050125511989926, - "arithmetic_mean_rank": 9338.490612991147, - "count": 118142.0, - "geometric_mean_rank": 7423.285855813262, - "harmonic_mean_rank": 1923.9636122992003, - "hits_at_1": 6.771512247972779e-05, - "hits_at_10": 0.0006009717120075841, - "hits_at_3": 0.00011850146433952363, - "hits_at_5": 0.00022853853836908128, - "inverse_arithmetic_mean_rank": 0.00010708368637313402, - "inverse_geometric_mean_rank": 0.00013471123427328188, - "inverse_harmonic_mean_rank": 0.0005197603497318574, - "inverse_median_rank": 9.800078400627205e-05, - "median_absolute_deviation": 5381.846053175335, - "median_rank": 10204.0, - "standard_deviation": 4446.329053094438, - "variance": 19769842.04839168, - "z_arithmetic_mean_rank": -157.17456742126456, - "z_geometric_mean_rank": -126.27729170514175, - "z_hits_at_k": -1.0209271367266144, - "z_inverse_harmonic_mean_rank": -5.556272891232151 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.2642043527575457, - "adjusted_arithmetic_mean_rank_index": -0.2642401244381005, - "adjusted_geometric_mean_rank_index": -0.36675019622610616, - "adjusted_hits_at_k": -7.738405941025406e-05, - "adjusted_inverse_harmonic_mean_rank": -0.00017050128074743386, - "arithmetic_mean_rank": 9338.49096003115, - "count": 118142.0, - "geometric_mean_rank": 7423.286367673732, - "harmonic_mean_rank": 1923.9637070975393, - "hits_at_1": 6.771512247972779e-05, - "hits_at_10": 0.0006009717120075841, - "hits_at_3": 0.00011850146433952363, - "hits_at_5": 0.00022853853836908128, - "inverse_arithmetic_mean_rank": 0.00010708368239365565, - "inverse_geometric_mean_rank": 0.00013471122498449087, - "inverse_harmonic_mean_rank": 0.0005197603241220095, - "inverse_median_rank": 9.800078400627205e-05, - "median_absolute_deviation": 5381.846053175335, - "median_rank": 10204.0, - "standard_deviation": 4446.328912173295, - "variance": 19769840.79522816, - "z_arithmetic_mean_rank": -157.17459537002162, - "z_geometric_mean_rank": -126.27732415838051, - "z_hits_at_k": -1.0209271367266144, - "z_inverse_harmonic_mean_rank": -5.55627372637894 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.2642042545213674, - "adjusted_arithmetic_mean_rank_index": -0.26424002618862175, - "adjusted_geometric_mean_rank_index": -0.3667504227168079, - "adjusted_hits_at_k": -7.738405941025406e-05, - "adjusted_inverse_harmonic_mean_rank": -0.0001705012723187267, - "arithmetic_mean_rank": 9338.490234375, - "count": 118142.0, - "geometric_mean_rank": 7423.28759765625, - "harmonic_mean_rank": 1923.9636759190637, - "hits_at_1": 6.771512247972779e-05, - "hits_at_10": 0.0006009717120075841, - "hits_at_3": 0.00011850146433952363, - "hits_at_5": 0.00022853853836908128, - "inverse_arithmetic_mean_rank": 0.00010708368790801615, - "inverse_geometric_mean_rank": 0.00013471119746100157, - "inverse_harmonic_mean_rank": 0.0005197603325448996, - "inverse_median_rank": 9.800078260013834e-05, - "median_absolute_deviation": 5381.846053175335, - "median_rank": 10204.0, - "standard_deviation": 4446.3291015625, - "variance": 19769842.0, - "z_arithmetic_mean_rank": -157.17453692953256, - "z_geometric_mean_rank": -126.27740214235996, - "z_hits_at_k": -1.0209271367266144, - "z_inverse_harmonic_mean_rank": -5.556273451705319 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.22332789159549, - "adjusted_arithmetic_mean_rank_index": -0.22335821137582568, - "adjusted_geometric_mean_rank_index": -0.31018246346676426, - "adjusted_hits_at_k": -2.0174999586579823e-05, - "adjusted_inverse_harmonic_mean_rank": -0.0001228583296856978, - "arithmetic_mean_rank": 9011.949518376192, - "count": 59071.0, - "geometric_mean_rank": 7095.621736831286, - "harmonic_mean_rank": 1756.6468268960064, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006602224441773459, - "hits_at_3": 0.0001523590255793875, - "hits_at_5": 0.0002877892705388431, - "inverse_arithmetic_mean_rank": 0.00011096378180558028, - "inverse_geometric_mean_rank": 0.00014093197708233148, - "inverse_harmonic_mean_rank": 0.0005692663913366122, - "inverse_median_rank": 0.00010339123242349049, - "median_absolute_deviation": 5564.206126051524, - "median_rank": 9672.0, - "standard_deviation": 4425.760932990495, - "variance": 19587359.835984897, - "z_arithmetic_mean_rank": -93.9307057119106, - "z_geometric_mean_rank": -75.51395097640778, - "z_hits_at_k": -0.1879214636632005, - "z_inverse_harmonic_mean_rank": -2.826717203602767 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.2233279329594688, - "adjusted_arithmetic_mean_rank_index": -0.22335825274542032, - "adjusted_geometric_mean_rank_index": -0.31018253160783194, - "adjusted_hits_at_k": -2.0174999586579823e-05, - "adjusted_inverse_harmonic_mean_rank": -0.00012285834022563564, - "arithmetic_mean_rank": 9011.949823094243, - "count": 59071.0, - "geometric_mean_rank": 7095.62210581432, - "harmonic_mean_rank": 1756.646859397724, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006602224441773459, - "hits_at_3": 0.0001523590255793875, - "hits_at_5": 0.0002877892705388431, - "inverse_arithmetic_mean_rank": 0.00011096377805359897, - "inverse_geometric_mean_rank": 0.00014093196975365648, - "inverse_harmonic_mean_rank": 0.0005692663808039685, - "inverse_median_rank": 0.00010339123242349049, - "median_absolute_deviation": 5564.206126051524, - "median_rank": 9672.0, - "standard_deviation": 4425.760798550912, - "variance": 19587358.645990007, - "z_arithmetic_mean_rank": -93.93072310941398, - "z_geometric_mean_rank": -75.51396756535735, - "z_hits_at_k": -0.1879214636632005, - "z_inverse_harmonic_mean_rank": -2.826717446105034 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.223327850922706, - "adjusted_arithmetic_mean_rank_index": -0.22335817069751962, - "adjusted_geometric_mean_rank_index": -0.3101824348794342, - "adjusted_hits_at_k": -2.0174999586579823e-05, - "adjusted_inverse_harmonic_mean_rank": -0.00012285830888574488, - "arithmetic_mean_rank": 9011.94921875, - "count": 59071.0, - "geometric_mean_rank": 7095.62158203125, - "harmonic_mean_rank": 1756.6467627557643, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006602224441773459, - "hits_at_3": 0.0001523590255793875, - "hits_at_5": 0.0002877892705388431, - "inverse_arithmetic_mean_rank": 0.00011096378875663504, - "inverse_geometric_mean_rank": 0.0001409319811500609, - "inverse_harmonic_mean_rank": 0.0005692664121221707, - "inverse_median_rank": 0.00010339123400626704, - "median_absolute_deviation": 5564.206126051524, - "median_rank": 9672.0, - "standard_deviation": 4425.7607421875, - "variance": 19587360.0, - "z_arithmetic_mean_rank": -93.93068860512074, - "z_geometric_mean_rank": -75.5139440168195, - "z_hits_at_k": -0.1879214636632005, - "z_inverse_harmonic_mean_rank": -2.8267167250386773 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.3048588325485093, - "adjusted_arithmetic_mean_rank_index": -0.30489999655764954, - "adjusted_geometric_mean_rank_index": -0.42574765680192117, - "adjusted_hits_at_k": -0.00013459288103333754, - "adjusted_inverse_harmonic_mean_rank": -0.00021814399979022973, - "arithmetic_mean_rank": 9665.031707606102, - "count": 59071.0, - "geometric_mean_rank": 7766.080963854441, - "harmonic_mean_rank": 2126.5089606148067, - "hits_at_1": 5.078634185979584e-05, - "hits_at_10": 0.0005417209798378223, - "hits_at_3": 8.464390309965973e-05, - "hits_at_5": 0.00016928780619931947, - "inverse_arithmetic_mean_rank": 0.00010346577541106552, - "inverse_geometric_mean_rank": 0.00012876507528755953, - "inverse_harmonic_mean_rank": 0.0004702543081271026, - "inverse_median_rank": 9.285051067780872e-05, - "median_absolute_deviation": 5008.230294111923, - "median_rank": 10770.0, - "standard_deviation": 4442.866877124927, - "variance": 19739066.087853797, - "z_arithmetic_mean_rank": -128.2600694817313, - "z_geometric_mean_rank": -103.64746647372786, - "z_hits_at_k": -1.2575282805239896, - "z_inverse_harmonic_mean_rank": -5.034420904848641 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.304858885115578, - "adjusted_arithmetic_mean_rank_index": -0.3049000491318161, - "adjusted_geometric_mean_rank_index": -0.42574777929683894, - "adjusted_hits_at_k": -0.00013459288103333754, - "adjusted_inverse_harmonic_mean_rank": -0.00021814404050530368, - "arithmetic_mean_rank": 9665.032096968056, - "count": 59071.0, - "geometric_mean_rank": 7766.081631001217, - "harmonic_mean_rank": 2126.5091446033134, - "hits_at_1": 5.078634185979584e-05, - "hits_at_10": 0.0005417209798378223, - "hits_at_3": 8.464390309965973e-05, - "hits_at_5": 0.00016928780619931947, - "inverse_arithmetic_mean_rank": 0.00010346577124288107, - "inverse_geometric_mean_rank": 0.0001287650642259703, - "inverse_harmonic_mean_rank": 0.0004702542674400507, - "inverse_median_rank": 9.285051067780872e-05, - "median_absolute_deviation": 5008.230294111923, - "median_rank": 10770.0, - "standard_deviation": 4442.866722763831, - "variance": 19739064.71624222, - "z_arithmetic_mean_rank": -128.26009159772457, - "z_geometric_mean_rank": -103.6474962948852, - "z_hits_at_k": -1.2575282805239896, - "z_inverse_harmonic_mean_rank": -5.034421844488602 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.3048587707679191, - "adjusted_arithmetic_mean_rank_index": -0.30489993476871713, - "adjusted_geometric_mean_rank_index": -0.42574839070692905, - "adjusted_hits_at_k": -0.00013459288103333754, - "adjusted_inverse_harmonic_mean_rank": -0.00021814402267636868, - "arithmetic_mean_rank": 9665.03125, - "count": 59071.0, - "geometric_mean_rank": 7766.0849609375, - "harmonic_mean_rank": 2126.5090640356275, - "hits_at_1": 5.078634185979584e-05, - "hits_at_10": 0.0005417209798378223, - "hits_at_3": 8.464390309965973e-05, - "hits_at_5": 0.00016928780619931947, - "inverse_arithmetic_mean_rank": 0.00010346578346798196, - "inverse_geometric_mean_rank": 0.00012876500841230154, - "inverse_harmonic_mean_rank": 0.000470254285256715, - "inverse_median_rank": 9.285051055485383e-05, - "median_absolute_deviation": 5008.230294111923, - "median_rank": 10770.0, - "standard_deviation": 4442.86669921875, - "variance": 19739064.0, - "z_arithmetic_mean_rank": -128.26004348942945, - "z_geometric_mean_rank": -103.64764514152202, - "z_hits_at_k": -1.2575282805239896, - "z_inverse_harmonic_mean_rank": -5.034421433024784 - } - } - }, - "times": { - "evaluation": 29.497436046600342, - "training": 13.04035758972168 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json deleted file mode 100644 index 9ec4235..0000000 --- a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with SimplE as described by Kazemi et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json b/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json deleted file mode 100644 index e726c90..0000000 --- a/nonasv/results/27c9fe15/cuda/simple/wn18/kazemi2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.006185159085706167, - 0.002151926759158413, - 0.001296131423698241, - 0.0009291403910987688, - 0.000743997096846818 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6804716586083929, - "adjusted_arithmetic_mean_rank_index": 0.3195439569083198, - "adjusted_geometric_mean_rank_index": 0.6640993333234186, - "adjusted_hits_at_k": 0.007357450518285132, - "adjusted_inverse_harmonic_mean_rank": 0.004073613974139381, - "arithmetic_mean_rank": 13924.6501, - "count": 10000.0, - "geometric_mean_rank": 5058.878653374096, - "harmonic_mean_rank": 230.09195259808598, - "hits_at_1": 0.0006, - "hits_at_10": 0.0076, - "hits_at_3": 0.0024, - "hits_at_5": 0.003, - "inverse_arithmetic_mean_rank": 7.181508998922709e-05, - "inverse_geometric_mean_rank": 0.00019767226464960467, - "inverse_harmonic_mean_rank": 0.004346088547245952, - "inverse_median_rank": 9.72951936174353e-05, - "median_absolute_deviation": 14072.860258055174, - "median_rank": 10278.0, - "standard_deviation": 12819.282193674886, - "variance": 164333995.96107, - "z_arithmetic_mean_rank": 55.34521519835033, - "z_geometric_mean_rank": 66.46364729994838, - "z_hits_at_k": 47.062018740431306, - "z_inverse_harmonic_mean_rank": 64.29705082532885 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6804717367974064, - "adjusted_arithmetic_mean_rank_index": 0.3195438787154853, - "adjusted_geometric_mean_rank_index": 0.664099301991363, - "adjusted_hits_at_k": 0.007357450518285132, - "adjusted_inverse_harmonic_mean_rank": 0.0040736139668042775, - "arithmetic_mean_rank": 13924.6517, - "count": 10000.0, - "geometric_mean_rank": 5058.879125161595, - "harmonic_mean_rank": 230.0919529863171, - "hits_at_1": 0.0006, - "hits_at_10": 0.0076, - "hits_at_3": 0.0024, - "hits_at_5": 0.003, - "inverse_arithmetic_mean_rank": 7.18150817373766e-05, - "inverse_geometric_mean_rank": 0.0001976722462148287, - "inverse_harmonic_mean_rank": 0.0043460885399128546, - "inverse_median_rank": 9.72951936174353e-05, - "median_absolute_deviation": 14072.860258055174, - "median_rank": 10278.0, - "standard_deviation": 12819.283148545675, - "variance": 164334020.4425871, - "z_arithmetic_mean_rank": 55.34520165530199, - "z_geometric_mean_rank": 66.4636441642086, - "z_hits_at_k": 47.062018740431306, - "z_inverse_harmonic_mean_rank": 64.29705070955313 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6804717205334807, - "adjusted_arithmetic_mean_rank_index": 0.31954389498020574, - "adjusted_geometric_mean_rank_index": 0.6640993489570108, - "adjusted_hits_at_k": 0.007357450518285132, - "adjusted_inverse_harmonic_mean_rank": 0.004073613836279541, - "arithmetic_mean_rank": 13924.6513671875, - "count": 10000.0, - "geometric_mean_rank": 5058.87841796875, - "harmonic_mean_rank": 230.09195989470737, - "hits_at_1": 0.0006, - "hits_at_10": 0.0076, - "hits_at_3": 0.0024, - "hits_at_5": 0.003, - "inverse_arithmetic_mean_rank": 7.181508408393711e-05, - "inverse_geometric_mean_rank": 0.00019767227058764547, - "inverse_harmonic_mean_rank": 0.004346088409423828, - "inverse_median_rank": 9.72951966105029e-05, - "median_absolute_deviation": 14072.860258055174, - "median_rank": 10278.0, - "standard_deviation": 12819.2822265625, - "variance": 164334000.0, - "z_arithmetic_mean_rank": 55.34520447236186, - "z_geometric_mean_rank": 66.4636488645721, - "z_hits_at_k": 47.062018740431306, - "z_inverse_harmonic_mean_rank": 64.29704864937864 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.692792955100394, - "adjusted_arithmetic_mean_rank_index": 0.307222058616213, - "adjusted_geometric_mean_rank_index": 0.6511900481613886, - "adjusted_hits_at_k": 0.008157640382758721, - "adjusted_inverse_harmonic_mean_rank": 0.004470857752830758, - "arithmetic_mean_rank": 14176.455, - "count": 5000.0, - "geometric_mean_rank": 5253.4022629084075, - "harmonic_mean_rank": 210.82682457987022, - "hits_at_1": 0.0008, - "hits_at_10": 0.0084, - "hits_at_3": 0.0032, - "hits_at_5": 0.004, - "inverse_arithmetic_mean_rank": 7.053949665131374e-05, - "inverse_geometric_mean_rank": 0.0001903528323084051, - "inverse_harmonic_mean_rank": 0.004743229434834831, - "inverse_median_rank": 9.173891105912573e-05, - "median_absolute_deviation": 14703.707502029307, - "median_rank": 10900.5, - "standard_deviation": 12791.397666837465, - "variance": 163619854.27117497, - "z_arithmetic_mean_rank": 37.625894388727076, - "z_geometric_mean_rank": 46.08674352328238, - "z_hits_at_k": 36.89671389039824, - "z_inverse_harmonic_mean_rank": 49.897865756659414 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6927929844219537, - "adjusted_arithmetic_mean_rank_index": 0.3072220292932203, - "adjusted_geometric_mean_rank_index": 0.6511900380555931, - "adjusted_hits_at_k": 0.008157640382758721, - "adjusted_inverse_harmonic_mean_rank": 0.004470857751334049, - "arithmetic_mean_rank": 14176.4556, - "count": 5000.0, - "geometric_mean_rank": 5253.402415082114, - "harmonic_mean_rank": 210.82682464637767, - "hits_at_1": 0.0008, - "hits_at_10": 0.0084, - "hits_at_3": 0.0032, - "hits_at_5": 0.004, - "inverse_arithmetic_mean_rank": 7.053949366582152e-05, - "inverse_geometric_mean_rank": 0.00019035282679451262, - "inverse_harmonic_mean_rank": 0.004743229433338532, - "inverse_median_rank": 9.173891105912573e-05, - "median_absolute_deviation": 14703.707502029307, - "median_rank": 10900.5, - "standard_deviation": 12791.398034977592, - "variance": 163619863.68922862, - "z_arithmetic_mean_rank": 37.62589079750113, - "z_geometric_mean_rank": 46.086742808063825, - "z_hits_at_k": 36.89671389039824, - "z_inverse_harmonic_mean_rank": 49.89786573995511 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6927929111944124, - "adjusted_arithmetic_mean_rank_index": 0.30722210252434023, - "adjusted_geometric_mean_rank_index": 0.651189977939602, - "adjusted_hits_at_k": 0.008157640382758721, - "adjusted_inverse_harmonic_mean_rank": 0.0044708576117805775, - "arithmetic_mean_rank": 14176.4541015625, - "count": 5000.0, - "geometric_mean_rank": 5253.4033203125, - "harmonic_mean_rank": 210.8268308475465, - "hits_at_1": 0.0008, - "hits_at_10": 0.0084, - "hits_at_3": 0.0032, - "hits_at_5": 0.004, - "inverse_arithmetic_mean_rank": 7.053949957480654e-05, - "inverse_geometric_mean_rank": 0.000190352788195014, - "inverse_harmonic_mean_rank": 0.004743229293823242, - "inverse_median_rank": 9.173891157843173e-05, - "median_absolute_deviation": 14703.707502029307, - "median_rank": 10900.5, - "standard_deviation": 12791.3984375, - "variance": 163619872.0, - "z_arithmetic_mean_rank": 37.62589976621385, - "z_geometric_mean_rank": 46.08673855346831, - "z_hits_at_k": 36.89671389039824, - "z_inverse_harmonic_mean_rank": 49.897864182441815 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.668150933393428, - "adjusted_arithmetic_mean_rank_index": 0.33186528386756864, - "adjusted_geometric_mean_rank_index": 0.6765632325264472, - "adjusted_hits_at_k": 0.006557260662945978, - "adjusted_inverse_harmonic_mean_rank": 0.003676370200070375, - "arithmetic_mean_rank": 13672.8452, - "count": 5000.0, - "geometric_mean_rank": 4871.557887401455, - "harmonic_mean_rank": 253.23202184119089, - "hits_at_1": 0.0004, - "hits_at_10": 0.0068, - "hits_at_3": 0.0016, - "hits_at_5": 0.002, - "inverse_arithmetic_mean_rank": 7.313766705996204e-05, - "inverse_geometric_mean_rank": 0.00020527314323537914, - "inverse_harmonic_mean_rank": 0.003948947659657075, - "inverse_median_rank": 0.00010430791697089809, - "median_absolute_deviation": 13273.737662280655, - "median_rank": 9587.0, - "standard_deviation": 12842.16984141064, - "variance": 164921326.23563695, - "z_arithmetic_mean_rank": 40.643989861490056, - "z_geometric_mean_rank": 47.88248130991923, - "z_hits_at_k": 29.658946798068516, - "z_inverse_harmonic_mean_rank": 41.031792183268756 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6681510604476293, - "adjusted_arithmetic_mean_rank_index": 0.33186515680715833, - "adjusted_geometric_mean_rank_index": 0.6765631815579028, - "adjusted_hits_at_k": 0.006557260662945978, - "adjusted_inverse_harmonic_mean_rank": 0.003676370186896874, - "arithmetic_mean_rank": 13672.8478, - "count": 5000.0, - "geometric_mean_rank": 4871.558654924727, - "harmonic_mean_rank": 253.23202268572967, - "hits_at_1": 0.0004, - "hits_at_10": 0.0068, - "hits_at_3": 0.0016, - "hits_at_5": 0.002, - "inverse_arithmetic_mean_rank": 7.313765315225699e-05, - "inverse_geometric_mean_rank": 0.00020527311089420754, - "inverse_harmonic_mean_rank": 0.003948947646487178, - "inverse_median_rank": 0.00010430791697089809, - "median_absolute_deviation": 13273.737662280655, - "median_rank": 9587.0, - "standard_deviation": 12842.171420279172, - "variance": 164921366.78783515, - "z_arithmetic_mean_rank": 40.64397430022986, - "z_geometric_mean_rank": 47.88247770271704, - "z_hits_at_k": 29.658946798068516, - "z_inverse_harmonic_mean_rank": 41.03179203623995 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6681510057013178, - "adjusted_arithmetic_mean_rank_index": 0.33186521155614523, - "adjusted_geometric_mean_rank_index": 0.6765632504704139, - "adjusted_hits_at_k": 0.006557260662945978, - "adjusted_inverse_harmonic_mean_rank": 0.003676370446974989, - "arithmetic_mean_rank": 13672.8466796875, - "count": 5000.0, - "geometric_mean_rank": 4871.5576171875, - "harmonic_mean_rank": 253.2320060124054, - "hits_at_1": 0.0004, - "hits_at_10": 0.0068, - "hits_at_3": 0.0016, - "hits_at_5": 0.002, - "inverse_arithmetic_mean_rank": 7.313765672734007e-05, - "inverse_geometric_mean_rank": 0.0002052731579169631, - "inverse_harmonic_mean_rank": 0.0039489479064941406, - "inverse_median_rank": 0.00010430791735416278, - "median_absolute_deviation": 13273.737662280655, - "median_rank": 9587.0, - "standard_deviation": 12842.1708984375, - "variance": 164921344.0, - "z_arithmetic_mean_rank": 40.643981005412286, - "z_geometric_mean_rank": 47.88248257986948, - "z_hits_at_k": 29.658946798068516, - "z_inverse_harmonic_mean_rank": 41.031794938959365 - } - } - }, - "times": { - "evaluation": 9.06259036064148, - "training": 4.838503360748291 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json deleted file mode 100644 index 1f8d7d2..0000000 --- a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with TransD as described by Ji et al., 2015" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json b/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json deleted file mode 100644 index b69cd51..0000000 --- a/nonasv/results/27c9fe15/cuda/transd/fb15k/ji2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 0.002535040261322091, - 0.0013169600454131234, - 0.0010097262843328746, - 0.0008632518252629737, - 0.0007637009042292636 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.2325918387095582, - "adjusted_arithmetic_mean_rank_index": 0.7675120637406266, - "adjusted_geometric_mean_rank_index": 0.9623243784288315, - "adjusted_hits_at_k": 0.21144731304754785, - "adjusted_inverse_harmonic_mean_rank": 0.11782603426599923, - "arithmetic_mean_rank": 1718.1215825024124, - "count": 118142.0, - "geometric_mean_rank": 205.60158202534976, - "harmonic_mean_rank": 8.443459883456963, - "hits_at_1": 0.06818066394677591, - "hits_at_10": 0.21198219092278783, - "hits_at_3": 0.1368861200927697, - "hits_at_5": 0.1675610705760864, - "inverse_arithmetic_mean_rank": 0.0005820309867381554, - "inverse_geometric_mean_rank": 0.004863775804393881, - "inverse_harmonic_mean_rank": 0.11843486127757559, - "inverse_median_rank": 0.0021551724137931034, - "median_absolute_deviation": 684.9622249495881, - "median_rank": 464.0, - "standard_deviation": 2626.3693262666297, - "variance": 6897815.837954232, - "z_arithmetic_mean_rank": 456.52944766267717, - "z_geometric_mean_rank": 331.34201080413635, - "z_hits_at_k": 2789.6223269150987, - "z_inverse_harmonic_mean_rank": 3839.6995940773863 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.23259198767297054, - "adjusted_arithmetic_mean_rank_index": 0.7675119147570456, - "adjusted_geometric_mean_rank_index": 0.9623243588106034, - "adjusted_hits_at_k": 0.21144731304754785, - "adjusted_inverse_harmonic_mean_rank": 0.11782603255173432, - "arithmetic_mean_rank": 1718.1226828731526, - "count": 118142.0, - "geometric_mean_rank": 205.60168856428186, - "harmonic_mean_rank": 8.44346000558602, - "hits_at_1": 0.06818066394677591, - "hits_at_10": 0.21198219092278783, - "hits_at_3": 0.1368861200927697, - "hits_at_5": 0.1675610705760864, - "inverse_arithmetic_mean_rank": 0.0005820306139767256, - "inverse_geometric_mean_rank": 0.004863773284076642, - "inverse_harmonic_mean_rank": 0.11843485956449376, - "inverse_median_rank": 0.0021551724137931034, - "median_absolute_deviation": 684.9622249495881, - "median_rank": 464.0, - "standard_deviation": 2626.370702761352, - "variance": 6897823.068323158, - "z_arithmetic_mean_rank": 456.5293590446669, - "z_geometric_mean_rank": 331.3420040493006, - "z_hits_at_k": 2789.6223269150987, - "z_inverse_harmonic_mean_rank": 3839.699538213142 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.2325918882218207, - "adjusted_arithmetic_mean_rank_index": 0.7675120142216605, - "adjusted_geometric_mean_rank_index": 0.9623243483069595, - "adjusted_hits_at_k": 0.21144731304754785, - "adjusted_inverse_harmonic_mean_rank": 0.11782603811895852, - "arithmetic_mean_rank": 1718.1219482421875, - "count": 118142.0, - "geometric_mean_rank": 205.60174560546875, - "harmonic_mean_rank": 8.443459608961314, - "hits_at_1": 0.06818066394677591, - "hits_at_10": 0.21198219092278783, - "hits_at_3": 0.1368861200927697, - "hits_at_5": 0.1675610705760864, - "inverse_arithmetic_mean_rank": 0.0005820308579131961, - "inverse_geometric_mean_rank": 0.004863772075623274, - "inverse_harmonic_mean_rank": 0.11843486512787578, - "inverse_median_rank": 0.0021551724057644606, - "median_absolute_deviation": 684.9622249495881, - "median_rank": 464.0, - "standard_deviation": 2626.369873046875, - "variance": 6897819.0, - "z_arithmetic_mean_rank": 456.5294182079398, - "z_geometric_mean_rank": 331.34200043274615, - "z_hits_at_k": 2789.6223269150987, - "z_inverse_harmonic_mean_rank": 3839.6997196371262 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.2635138727776829, - "adjusted_arithmetic_mean_rank_index": 0.7365861151702535, - "adjusted_geometric_mean_rank_index": 0.9456343343807989, - "adjusted_hits_at_k": 0.16618117235305171, - "adjusted_inverse_harmonic_mean_rank": 0.08985695538975279, - "arithmetic_mean_rank": 1941.2405579726092, - "count": 59071.0, - "geometric_mean_rank": 295.38940284600267, - "harmonic_mean_rank": 11.05133438110848, - "hits_at_1": 0.049127321359042506, - "hits_at_10": 0.16674848910632967, - "hits_at_3": 0.10329941934282473, - "hits_at_5": 0.1294035990587598, - "inverse_arithmetic_mean_rank": 0.0005151345081334892, - "inverse_geometric_mean_rank": 0.0033853617982407334, - "inverse_harmonic_mean_rank": 0.09048681050764633, - "inverse_median_rank": 0.001440922190201729, - "median_absolute_deviation": 1022.9955307688654, - "median_rank": 694.0, - "standard_deviation": 2756.3004548828058, - "variance": 7597192.197587161, - "z_arithmetic_mean_rank": 309.76274921507064, - "z_geometric_mean_rank": 230.21477091238273, - "z_hits_at_k": 1547.9063088866371, - "z_inverse_harmonic_mean_rank": 2067.4235301210465 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.2635140313396026, - "adjusted_arithmetic_mean_rank_index": 0.7365859565868069, - "adjusted_geometric_mean_rank_index": 0.9456343123996525, - "adjusted_hits_at_k": 0.16618117235305171, - "adjusted_inverse_harmonic_mean_rank": 0.08985695495606277, - "arithmetic_mean_rank": 1941.241726058472, - "count": 59071.0, - "geometric_mean_rank": 295.38952187363765, - "harmonic_mean_rank": 11.05133443403925, - "hits_at_1": 0.049127321359042506, - "hits_at_10": 0.16674848910632967, - "hits_at_3": 0.10329941934282473, - "hits_at_5": 0.1294035990587598, - "inverse_arithmetic_mean_rank": 0.000515134198166251, - "inverse_geometric_mean_rank": 0.0033853604341043016, - "inverse_harmonic_mean_rank": 0.09048681007425645, - "inverse_median_rank": 0.001440922190201729, - "median_absolute_deviation": 1022.9955307688654, - "median_rank": 694.0, - "standard_deviation": 2756.30194280584, - "variance": 7597200.399915249, - "z_arithmetic_mean_rank": 309.76268252464115, - "z_geometric_mean_rank": 230.21476556107046, - "z_hits_at_k": 1547.9063088866371, - "z_inverse_harmonic_mean_rank": 2067.4235201427323 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.26351392827356934, - "adjusted_arithmetic_mean_rank_index": 0.7365860596668328, - "adjusted_geometric_mean_rank_index": 0.9456343115698167, - "adjusted_hits_at_k": 0.16618117235305171, - "adjusted_inverse_harmonic_mean_rank": 0.08985696537040451, - "arithmetic_mean_rank": 1941.240966796875, - "count": 59071.0, - "geometric_mean_rank": 295.3895263671875, - "harmonic_mean_rank": 11.05133316299531, - "hits_at_1": 0.049127321359042506, - "hits_at_10": 0.16674848910632967, - "hits_at_3": 0.10329941934282473, - "hits_at_5": 0.1294035990587598, - "inverse_arithmetic_mean_rank": 0.0005151344230398536, - "inverse_geometric_mean_rank": 0.0033853603526949883, - "inverse_harmonic_mean_rank": 0.09048682048139105, - "inverse_median_rank": 0.0014409221475943923, - "median_absolute_deviation": 1022.9955307688654, - "median_rank": 694.0, - "standard_deviation": 2756.301025390625, - "variance": 7597195.5, - "z_arithmetic_mean_rank": 309.7627258737507, - "z_geometric_mean_rank": 230.21476535904688, - "z_hits_at_k": 1547.9063088866371, - "z_inverse_harmonic_mean_rank": 2067.423759755293 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.20183765718366314, - "adjusted_arithmetic_mean_rank_index": 0.7982701158524677, - "adjusted_geometric_mean_rank_index": 0.9739078873536554, - "adjusted_hits_at_k": 0.2567132652680245, - "adjusted_inverse_harmonic_mean_rank": 0.14579500702367879, - "arithmetic_mean_rank": 1495.0026070322156, - "count": 59071.0, - "geometric_mean_rank": 143.10604958758364, - "harmonic_mean_rank": 6.831398460466994, - "hits_at_1": 0.08723400653450931, - "hits_at_10": 0.257215892739246, - "hits_at_3": 0.1704728208427147, - "hits_at_5": 0.205718542093413, - "inverse_arithmetic_mean_rank": 0.0006688951546279485, - "inverse_geometric_mean_rank": 0.0069878247836614405, - "inverse_harmonic_mean_rank": 0.14638291204750484, - "inverse_median_rank": 0.0037313432835820895, - "median_absolute_deviation": 395.85479234099574, - "median_rank": 268.0, - "standard_deviation": 2469.59011252709, - "variance": 6098875.323891563, - "z_arithmetic_mean_rank": 335.8024981317717, - "z_geometric_mean_rank": 237.09604384258736, - "z_hits_at_k": 2398.523522059363, - "z_inverse_harmonic_mean_rank": 3364.7197809171057 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.20183779660067122, - "adjusted_arithmetic_mean_rank_index": 0.7982699764166346, - "adjusted_geometric_mean_rank_index": 0.973907870710314, - "adjusted_hits_at_k": 0.2567132652680245, - "adjusted_inverse_harmonic_mean_rank": 0.14579500402884382, - "arithmetic_mean_rank": 1495.0036396878334, - "count": 59071.0, - "geometric_mean_rank": 143.1061402325816, - "harmonic_mean_rank": 6.831398600133778, - "hits_at_1": 0.08723400653450931, - "hits_at_10": 0.257215892739246, - "hits_at_3": 0.1704728208427147, - "hits_at_5": 0.205718542093413, - "inverse_arithmetic_mean_rank": 0.0006688946925967395, - "inverse_geometric_mean_rank": 0.006987820357496621, - "inverse_harmonic_mean_rank": 0.14638290905473106, - "inverse_median_rank": 0.0037313432835820895, - "median_absolute_deviation": 395.85479234099574, - "median_rank": 268.0, - "standard_deviation": 2469.591367385949, - "variance": 6098881.521867202, - "z_arithmetic_mean_rank": 335.8024394763114, - "z_geometric_mean_rank": 237.0960397907972, - "z_hits_at_k": 2398.523522059363, - "z_inverse_harmonic_mean_rank": 3364.719711801018 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.2018377172252792, - "adjusted_arithmetic_mean_rank_index": 0.7982700558027443, - "adjusted_geometric_mean_rank_index": 0.9739078763312524, - "adjusted_hits_at_k": 0.2567132652680245, - "adjusted_inverse_harmonic_mean_rank": 0.14579501302066805, - "arithmetic_mean_rank": 1495.0030517578125, - "count": 59071.0, - "geometric_mean_rank": 143.10610961914062, - "harmonic_mean_rank": 6.8313981807921, - "hits_at_1": 0.08723400653450931, - "hits_at_10": 0.257215892739246, - "hits_at_3": 0.1704728208427147, - "hits_at_5": 0.205718542093413, - "inverse_arithmetic_mean_rank": 0.0006688949652016163, - "inverse_geometric_mean_rank": 0.006987821776419878, - "inverse_harmonic_mean_rank": 0.14638291804036668, - "inverse_median_rank": 0.0037313431967049837, - "median_absolute_deviation": 395.85479234099574, - "median_rank": 268.0, - "standard_deviation": 2469.5908203125, - "variance": 6098878.5, - "z_arithmetic_mean_rank": 335.8024728710903, - "z_geometric_mean_rank": 237.09604115920408, - "z_hits_at_k": 2398.523522059363, - "z_inverse_harmonic_mean_rank": 3364.719919318199 - } - } - }, - "times": { - "evaluation": 34.54206871986389, - "training": 83.16322159767151 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json deleted file mode 100644 index e1f0574..0000000 --- a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with TransD as described by Ji et al., 2015" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json b/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json deleted file mode 100644 index 9ce9b00..0000000 --- a/nonasv/results/27c9fe15/cuda/transd/wn18/ji2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0044969535855334955, - 0.0036679787342402833, - 0.0031641818598498335, - 0.0027769141778763067, - 0.002474804447551882 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.5926951523251482, - "adjusted_arithmetic_mean_rank_index": 0.40732475287521164, - "adjusted_geometric_mean_rank_index": 0.7386937354137952, - "adjusted_hits_at_k": 0.03866510045659395, - "adjusted_inverse_harmonic_mean_rank": 0.015044241141624008, - "arithmetic_mean_rank": 12128.459, - "count": 10000.0, - "geometric_mean_rank": 3935.6613709344965, - "harmonic_mean_rank": 65.30094412759377, - "hits_at_1": 0.003, - "hits_at_10": 0.0389, - "hits_at_3": 0.0171, - "hits_at_5": 0.0254, - "inverse_arithmetic_mean_rank": 8.245070540288754e-05, - "inverse_geometric_mean_rank": 0.0002540869007138581, - "inverse_harmonic_mean_rank": 0.015313714271053502, - "inverse_median_rank": 0.00011371389583807141, - "median_absolute_deviation": 11788.170239338042, - "median_rank": 8794.0, - "standard_deviation": 11405.352572828207, - "variance": 130082067.310519, - "z_arithmetic_mean_rank": 70.5489045125688, - "z_geometric_mean_rank": 73.92912088546521, - "z_hits_at_k": 247.3217696492246, - "z_inverse_harmonic_mean_rank": 237.45508127481824 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.5926953868921885, - "adjusted_arithmetic_mean_rank_index": 0.407324518296708, - "adjusted_geometric_mean_rank_index": 0.7386935859087462, - "adjusted_hits_at_k": 0.03866510045659395, - "adjusted_inverse_harmonic_mean_rank": 0.015044240820136027, - "arithmetic_mean_rank": 12128.4638, - "count": 10000.0, - "geometric_mean_rank": 3935.6636221309673, - "harmonic_mean_rank": 65.30094549811206, - "hits_at_1": 0.003, - "hits_at_10": 0.0389, - "hits_at_3": 0.0171, - "hits_at_5": 0.0254, - "inverse_arithmetic_mean_rank": 8.245067277193011e-05, - "inverse_geometric_mean_rank": 0.0002540867553763524, - "inverse_harmonic_mean_rank": 0.015313713949653477, - "inverse_median_rank": 0.00011371389583807141, - "median_absolute_deviation": 11788.170239338042, - "median_rank": 8794.0, - "standard_deviation": 11405.35493635729, - "variance": 130082121.22428958, - "z_arithmetic_mean_rank": 70.54886388342378, - "z_geometric_mean_rank": 73.92910592286799, - "z_hits_at_k": 247.3217696492246, - "z_inverse_harmonic_mean_rank": 237.45507620052078 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.5926952470071567, - "adjusted_arithmetic_mean_rank_index": 0.4073246581885761, - "adjusted_geometric_mean_rank_index": 0.7386937998689465, - "adjusted_hits_at_k": 0.03866510045659395, - "adjusted_inverse_harmonic_mean_rank": 0.015044241470269793, - "arithmetic_mean_rank": 12128.4609375, - "count": 10000.0, - "geometric_mean_rank": 3935.660400390625, - "harmonic_mean_rank": 65.30094272656147, - "hits_at_1": 0.003, - "hits_at_10": 0.0389, - "hits_at_3": 0.0171, - "hits_at_5": 0.0254, - "inverse_arithmetic_mean_rank": 8.245069329859689e-05, - "inverse_geometric_mean_rank": 0.00025408697547391057, - "inverse_harmonic_mean_rank": 0.015313714599609373, - "inverse_median_rank": 0.00011371389700798318, - "median_absolute_deviation": 11788.170239338042, - "median_rank": 8794.0, - "standard_deviation": 11405.353515625, - "variance": 130082088.0, - "z_arithmetic_mean_rank": 70.5488881127837, - "z_geometric_mean_rank": 73.92912733619363, - "z_hits_at_k": 247.3217696492246, - "z_inverse_harmonic_mean_rank": 237.45508646209296 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6001467270400189, - "adjusted_arithmetic_mean_rank_index": 0.39987281445105116, - "adjusted_geometric_mean_rank_index": 0.7263469317031201, - "adjusted_hits_at_k": 0.03576438617283119, - "adjusted_inverse_harmonic_mean_rank": 0.014352630823535656, - "arithmetic_mean_rank": 12280.6576, - "count": 5000.0, - "geometric_mean_rank": 4121.685168522348, - "harmonic_mean_rank": 68.38869911196008, - "hits_at_1": 0.003, - "hits_at_10": 0.036, - "hits_at_3": 0.0158, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 8.142886420023631e-05, - "inverse_geometric_mean_rank": 0.00024261921013207972, - "inverse_harmonic_mean_rank": 0.014622298902964747, - "inverse_median_rank": 0.00011096316023080337, - "median_absolute_deviation": 11939.395665625612, - "median_rank": 9012.0, - "standard_deviation": 11401.945645159087, - "variance": 130004364.49516225, - "z_arithmetic_mean_rank": 48.97295576114048, - "z_geometric_mean_rank": 51.40582974945663, - "z_hits_at_k": 161.76103164263458, - "z_inverse_harmonic_mean_rank": 160.18529008985547 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6001469225170842, - "adjusted_arithmetic_mean_rank_index": 0.3998726189644326, - "adjusted_geometric_mean_rank_index": 0.7263468231408228, - "adjusted_hits_at_k": 0.03576438617283119, - "adjusted_inverse_harmonic_mean_rank": 0.014352630727401386, - "arithmetic_mean_rank": 12280.6616, - "count": 5000.0, - "geometric_mean_rank": 4121.686803260271, - "harmonic_mean_rank": 68.38869956145841, - "hits_at_1": 0.003, - "hits_at_10": 0.036, - "hits_at_3": 0.0158, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 8.142883767760525e-05, - "inverse_geometric_mean_rank": 0.0002426191139047722, - "inverse_harmonic_mean_rank": 0.014622298806856778, - "inverse_median_rank": 0.00011096316023080337, - "median_absolute_deviation": 11939.395665625612, - "median_rank": 9012.0, - "standard_deviation": 11401.947519572499, - "variance": 130004407.23908545, - "z_arithmetic_mean_rank": 48.97293181963414, - "z_geometric_mean_rank": 51.405822066165456, - "z_hits_at_k": 161.76103164263458, - "z_inverse_harmonic_mean_rank": 160.18528901693048 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6001468519620811, - "adjusted_arithmetic_mean_rank_index": 0.399872689522884, - "adjusted_geometric_mean_rank_index": 0.7263468741503063, - "adjusted_hits_at_k": 0.03576438617283119, - "adjusted_inverse_harmonic_mean_rank": 0.014352631114986586, - "arithmetic_mean_rank": 12280.66015625, - "count": 5000.0, - "geometric_mean_rank": 4121.68603515625, - "harmonic_mean_rank": 68.38869774921292, - "hits_at_1": 0.003, - "hits_at_10": 0.036, - "hits_at_3": 0.0158, - "hits_at_5": 0.0242, - "inverse_arithmetic_mean_rank": 8.142885053530335e-05, - "inverse_geometric_mean_rank": 0.00024261916405521333, - "inverse_harmonic_mean_rank": 0.014622299194335936, - "inverse_median_rank": 0.00011096316302428022, - "median_absolute_deviation": 11939.395665625612, - "median_rank": 9012.0, - "standard_deviation": 11401.947265625, - "variance": 130004392.0, - "z_arithmetic_mean_rank": 48.97294046102158, - "z_geometric_mean_rank": 51.40582567626513, - "z_hits_at_k": 161.76103164263458, - "z_inverse_harmonic_mean_rank": 160.1852933426495 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.5852439231026143, - "adjusted_arithmetic_mean_rank_index": 0.41477634577328704, - "adjusted_geometric_mean_rank_index": 0.7505085129139782, - "adjusted_hits_at_k": 0.041565814707244096, - "adjusted_inverse_harmonic_mean_rank": 0.015735851451664715, - "arithmetic_mean_rank": 11976.2604, - "count": 5000.0, - "geometric_mean_rank": 3758.0333755135143, - "harmonic_mean_rank": 62.47996876916217, - "hits_at_1": 0.003, - "hits_at_10": 0.0418, - "hits_at_3": 0.0184, - "hits_at_5": 0.0266, - "inverse_arithmetic_mean_rank": 8.34985184523877e-05, - "inverse_geometric_mean_rank": 0.0002660966255690466, - "inverse_harmonic_mean_rank": 0.016005129639142256, - "inverse_median_rank": 0.00011619125079881485, - "median_absolute_deviation": 11648.064329689261, - "median_rank": 8606.5, - "standard_deviation": 11406.727896210721, - "variance": 130113441.29819185, - "z_arithmetic_mean_rank": 50.79821967495293, - "z_geometric_mean_rank": 53.11581847027738, - "z_hits_at_k": 188.00507565405638, - "z_inverse_harmonic_mean_rank": 175.62708635249376 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.5852441967578171, - "adjusted_arithmetic_mean_rank_index": 0.4147760721047108, - "adjusted_geometric_mean_rank_index": 0.7505083263993314, - "adjusted_hits_at_k": 0.041565814707244096, - "adjusted_inverse_harmonic_mean_rank": 0.015735850904823025, - "arithmetic_mean_rank": 11976.266, - "count": 5000.0, - "geometric_mean_rank": 3758.036184193527, - "harmonic_mean_rank": 62.47997090330955, - "hits_at_1": 0.003, - "hits_at_10": 0.0418, - "hits_at_3": 0.0184, - "hits_at_5": 0.0266, - "inverse_arithmetic_mean_rank": 8.349847940919148e-05, - "inverse_geometric_mean_rank": 0.00026609642669382644, - "inverse_harmonic_mean_rank": 0.016005129092450172, - "inverse_median_rank": 0.00011619125079881485, - "median_absolute_deviation": 11648.064329689261, - "median_rank": 8606.5, - "standard_deviation": 11406.730770419892, - "variance": 130113506.86884399, - "z_arithmetic_mean_rank": 50.79818615839251, - "z_geometric_mean_rank": 53.115805270056526, - "z_hits_at_k": 188.00507565405638, - "z_inverse_harmonic_mean_rank": 175.6270802492198 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.5852440829893316, - "adjusted_arithmetic_mean_rank_index": 0.4147761858787561, - "adjusted_geometric_mean_rank_index": 0.7505084432990601, - "adjusted_hits_at_k": 0.041565814707244096, - "adjusted_inverse_harmonic_mean_rank": 0.015735851817505357, - "arithmetic_mean_rank": 11976.263671875, - "count": 5000.0, - "geometric_mean_rank": 3758.034423828125, - "harmonic_mean_rank": 62.47996734140379, - "hits_at_1": 0.003, - "hits_at_10": 0.0418, - "hits_at_3": 0.0184, - "hits_at_5": 0.0266, - "inverse_arithmetic_mean_rank": 8.349849667865783e-05, - "inverse_geometric_mean_rank": 0.0002660965546965599, - "inverse_harmonic_mean_rank": 0.016005130004882812, - "inverse_median_rank": 0.00011619125143624842, - "median_absolute_deviation": 11648.064329689261, - "median_rank": 8606.5, - "standard_deviation": 11406.7294921875, - "variance": 130113472.0, - "z_arithmetic_mean_rank": 50.79820009245363, - "z_geometric_mean_rank": 53.1158135434134, - "z_hits_at_k": 188.00507565405638, - "z_inverse_harmonic_mean_rank": 175.62709043562373 - } - } - }, - "times": { - "evaluation": 10.18949007987976, - "training": 24.329554319381714 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json deleted file mode 100644 index a7f48a3..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Batch_size is not mentioned in the paper. We found out that this value works.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with TransE as described by Bordes et al., 2013" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json b/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json deleted file mode 100644 index 062df03..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/fb15k/bordes2013/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 0.038581071503320664, - 0.03347079680852511, - 0.031049264601289154, - 0.029245134060277567, - 0.02778917855002847 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.684332083807414, - "adjusted_arithmetic_mean_rank_index": 0.3157106557301511, - "adjusted_geometric_mean_rank_index": 0.6675365146551828, - "adjusted_hits_at_k": 0.05741789660218553, - "adjusted_inverse_harmonic_mean_rank": 0.035582955378695054, - "arithmetic_mean_rank": 5055.060097171201, - "count": 118142.0, - "geometric_mean_rank": 1806.4793054632946, - "harmonic_mean_rank": 27.587316513736916, - "hits_at_1": 0.02301467725279748, - "hits_at_10": 0.05805725313605661, - "hits_at_3": 0.03980802762776997, - "hits_at_5": 0.04622403548272418, - "inverse_arithmetic_mean_rank": 0.0001978215848629767, - "inverse_geometric_mean_rank": 0.000553562942556675, - "inverse_harmonic_mean_rank": 0.03624854195231555, - "inverse_median_rank": 0.00024192572880125802, - "median_absolute_deviation": 5192.814270315871, - "median_rank": 4133.5, - "standard_deviation": 4352.196277998581, - "variance": 18941612.442224696, - "z_arithmetic_mean_rank": 187.7901574331674, - "z_geometric_mean_rank": 229.84234423340098, - "z_hits_at_k": 757.5137466511159, - "z_inverse_harmonic_mean_rank": 1159.5727563502958 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6843332216587098, - "adjusted_arithmetic_mean_rank_index": 0.31570951772479705, - "adjusted_geometric_mean_rank_index": 0.6675355590092635, - "adjusted_hits_at_k": 0.05741789660218553, - "adjusted_inverse_harmonic_mean_rank": 0.03558294665031708, - "arithmetic_mean_rank": 5055.068502310779, - "count": 118142.0, - "geometric_mean_rank": 1806.4844952029403, - "harmonic_mean_rank": 27.58732315197363, - "hits_at_1": 0.02301467725279748, - "hits_at_10": 0.05805725313605661, - "hits_at_3": 0.03980802762776997, - "hits_at_5": 0.04622403548272418, - "inverse_arithmetic_mean_rank": 0.00019782125594200727, - "inverse_geometric_mean_rank": 0.0005535613522593008, - "inverse_harmonic_mean_rank": 0.03624853322996141, - "inverse_median_rank": 0.00024192572880125802, - "median_absolute_deviation": 5192.814270315871, - "median_rank": 4133.5, - "standard_deviation": 4352.199232126487, - "variance": 18941638.156122383, - "z_arithmetic_mean_rank": 187.7894805279039, - "z_geometric_mean_rank": 229.84201519087895, - "z_hits_at_k": 757.5137466511159, - "z_inverse_harmonic_mean_rank": 1159.572471911041 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6843326073961279, - "adjusted_arithmetic_mean_rank_index": 0.31571013207054643, - "adjusted_geometric_mean_rank_index": 0.6675360531857619, - "adjusted_hits_at_k": 0.05741789660218553, - "adjusted_inverse_harmonic_mean_rank": 0.03558295000894267, - "arithmetic_mean_rank": 5055.06396484375, - "count": 118142.0, - "geometric_mean_rank": 1806.4818115234375, - "harmonic_mean_rank": 27.587320597621027, - "hits_at_1": 0.02301467725279748, - "hits_at_10": 0.05805725313605661, - "hits_at_3": 0.03980802762776997, - "hits_at_5": 0.04622403548272418, - "inverse_arithmetic_mean_rank": 0.00019782142771873623, - "inverse_geometric_mean_rank": 0.0005535621894523501, - "inverse_harmonic_mean_rank": 0.036248536586269066, - "inverse_median_rank": 0.00024192572163883597, - "median_absolute_deviation": 5192.814270315871, - "median_rank": 4133.5, - "standard_deviation": 4352.197265625, - "variance": 18941622.0, - "z_arithmetic_mean_rank": 187.78984595137916, - "z_geometric_mean_rank": 229.84218534289664, - "z_hits_at_k": 757.5137466511159, - "z_inverse_harmonic_mean_rank": 1159.5725813614984 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.723549425577274, - "adjusted_arithmetic_mean_rank_index": 0.2764881063253928, - "adjusted_geometric_mean_rank_index": 0.5866126073988215, - "adjusted_hits_at_k": 0.04221200917635138, - "adjusted_inverse_harmonic_mean_rank": 0.025949543973016322, - "arithmetic_mean_rank": 5330.20700512942, - "count": 59071.0, - "geometric_mean_rank": 2239.4875870800633, - "harmonic_mean_rank": 37.560624411329115, - "hits_at_1": 0.016810279155592422, - "hits_at_10": 0.04286367252966769, - "hits_at_3": 0.02864349680892485, - "hits_at_5": 0.03390834758172369, - "inverse_arithmetic_mean_rank": 0.00018760997444145595, - "inverse_geometric_mean_rank": 0.00044653071790580517, - "inverse_harmonic_mean_rank": 0.026623625556618218, - "inverse_median_rank": 0.00022040996253030638, - "median_absolute_deviation": 5239.516240198797, - "median_rank": 4537.0, - "standard_deviation": 4326.8477842276, - "variance": 18721611.747875288, - "z_arithmetic_mean_rank": 116.27386693384307, - "z_geometric_mean_rank": 142.81089647095337, - "z_hits_at_k": 393.18675148132706, - "z_inverse_harmonic_mean_rank": 597.0455773070048 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.723550659602649, - "adjusted_arithmetic_mean_rank_index": 0.27648687213248224, - "adjusted_geometric_mean_rank_index": 0.5866114883180238, - "adjusted_hits_at_k": 0.04221200917635138, - "adjusted_inverse_harmonic_mean_rank": 0.025949539196434005, - "arithmetic_mean_rank": 5330.216095884613, - "count": 59071.0, - "geometric_mean_rank": 2239.4936468886576, - "harmonic_mean_rank": 37.560631145471554, - "hits_at_1": 0.016810279155592422, - "hits_at_10": 0.04286367252966769, - "hits_at_3": 0.02864349680892485, - "hits_at_5": 0.03390834758172369, - "inverse_arithmetic_mean_rank": 0.00018760965447012294, - "inverse_geometric_mean_rank": 0.0004465295096457658, - "inverse_harmonic_mean_rank": 0.026623620783341486, - "inverse_median_rank": 0.00022040996253030638, - "median_absolute_deviation": 5239.516240198797, - "median_rank": 4537.0, - "standard_deviation": 4326.850308659758, - "variance": 18721633.59354905, - "z_arithmetic_mean_rank": 116.27334790832634, - "z_geometric_mean_rank": 142.81062403062407, - "z_hits_at_k": 393.18675148132706, - "z_inverse_harmonic_mean_rank": 597.0454674076806 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.7235500256590511, - "adjusted_arithmetic_mean_rank_index": 0.27648750616214646, - "adjusted_geometric_mean_rank_index": 0.5866121185159, - "adjusted_hits_at_k": 0.04221200917635138, - "adjusted_inverse_harmonic_mean_rank": 0.02594954222435405, - "arithmetic_mean_rank": 5330.21142578125, - "count": 59071.0, - "geometric_mean_rank": 2239.490234375, - "harmonic_mean_rank": 37.56062687663563, - "hits_at_1": 0.016810279155592422, - "hits_at_10": 0.04286367252966769, - "hits_at_3": 0.02864349680892485, - "hits_at_5": 0.03390834758172369, - "inverse_arithmetic_mean_rank": 0.000187609824934043, - "inverse_geometric_mean_rank": 0.0004465302044991404, - "inverse_harmonic_mean_rank": 0.026623623809166087, - "inverse_median_rank": 0.0002204099582741037, - "median_absolute_deviation": 5239.516240198797, - "median_rank": 4537.0, - "standard_deviation": 4326.84912109375, - "variance": 18721622.0, - "z_arithmetic_mean_rank": 116.27361454214935, - "z_geometric_mean_rank": 142.81077745235848, - "z_hits_at_k": 393.18675148132706, - "z_inverse_harmonic_mean_rank": 597.0455370738863 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6453276235867126, - "adjusted_arithmetic_mean_rank_index": 0.3547202665689454, - "adjusted_geometric_mean_rank_index": 0.7326280379766128, - "adjusted_hits_at_k": 0.07262372071547131, - "adjusted_inverse_harmonic_mean_rank": 0.04521633023387397, - "arithmetic_mean_rank": 4779.913189212981, - "count": 59071.0, - "geometric_mean_rank": 1457.193824111371, - "harmonic_mean_rank": 21.79909769203868, - "hits_at_1": 0.029219075350002538, - "hits_at_10": 0.07325083374244554, - "hits_at_3": 0.05097255844661509, - "hits_at_5": 0.05853972338372467, - "inverse_arithmetic_mean_rank": 0.0002092088204147179, - "inverse_geometric_mean_rank": 0.000686250506592575, - "inverse_harmonic_mean_rank": 0.04587345834801288, - "inverse_median_rank": 0.00027307482250136535, - "median_absolute_deviation": 4966.717431993767, - "median_rank": 3662.0, - "standard_deviation": 4360.068978199332, - "variance": 19010201.494656168, - "z_arithmetic_mean_rank": 149.217601017942, - "z_geometric_mean_rank": 178.3569182136984, - "z_hits_at_k": 678.5379875623604, - "z_inverse_harmonic_mean_rank": 1043.5218864092278 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6453286657859862, - "adjusted_arithmetic_mean_rank_index": 0.3547192242289472, - "adjusted_geometric_mean_rank_index": 0.7326272246599947, - "adjusted_hits_at_k": 0.07262372071547131, - "adjusted_inverse_harmonic_mean_rank": 0.04521631755371532, - "arithmetic_mean_rank": 4779.920908736944, - "count": 59071.0, - "geometric_mean_rank": 1457.1982536956366, - "harmonic_mean_rank": 21.799103713512977, - "hits_at_1": 0.029219075350002538, - "hits_at_10": 0.07325083374244554, - "hits_at_3": 0.05097255844661509, - "hits_at_5": 0.05853972338372467, - "inverse_arithmetic_mean_rank": 0.00020920848254458714, - "inverse_geometric_mean_rank": 0.000686248420531575, - "inverse_harmonic_mean_rank": 0.04587344567658133, - "inverse_median_rank": 0.00027307482250136535, - "median_absolute_deviation": 4966.717431993767, - "median_rank": 3662.0, - "standard_deviation": 4360.072284056398, - "variance": 19010230.322196774, - "z_arithmetic_mean_rank": 149.2171625443372, - "z_geometric_mean_rank": 178.35672021329714, - "z_hits_at_k": 678.5379875623604, - "z_inverse_harmonic_mean_rank": 1043.5215937710814 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6453282029415623, - "adjusted_arithmetic_mean_rank_index": 0.3547196871358673, - "adjusted_geometric_mean_rank_index": 0.732627899173446, - "adjusted_hits_at_k": 0.07262372071547131, - "adjusted_inverse_harmonic_mean_rank": 0.045216325378894785, - "arithmetic_mean_rank": 4779.91748046875, - "count": 59071.0, - "geometric_mean_rank": 1457.194580078125, - "harmonic_mean_rank": 21.799099997540385, - "hits_at_1": 0.029219075350002538, - "hits_at_10": 0.07325083374244554, - "hits_at_3": 0.05097255844661509, - "hits_at_5": 0.05853972338372467, - "inverse_arithmetic_mean_rank": 0.00020920863607898355, - "inverse_geometric_mean_rank": 0.0006862501613795757, - "inverse_harmonic_mean_rank": 0.04587345349637513, - "inverse_median_rank": 0.0002730748092290014, - "median_absolute_deviation": 4966.717431993767, - "median_rank": 3662.0, - "standard_deviation": 4360.07080078125, - "variance": 19010216.0, - "z_arithmetic_mean_rank": 149.21735727203293, - "z_geometric_mean_rank": 178.35688442232845, - "z_hits_at_k": 678.5379875623604, - "z_inverse_harmonic_mean_rank": 1043.5217743639334 - } - } - }, - "times": { - "evaluation": 17.733282566070557, - "training": 112.34571981430054 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json deleted file mode 100644 index 6c90c7d..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "TransE was used to initialize the emebddigns. Could not find batch size and number of epochs.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json b/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json deleted file mode 100644 index 44a0485..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/fb15k237/nguyen2018/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 4096, - "slice_size": null - }, - "losses": [ - 0.006557168293018543, - 0.006131826215394638, - 0.0058330251878667225, - 0.00561165085609545, - 0.005457009655934738 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9426456616120584, - "adjusted_arithmetic_mean_rank_index": 0.057362377210596005, - "adjusted_geometric_mean_rank_index": 0.12297663463498498, - "adjusted_hits_at_k": 0.0006920234342332076, - "adjusted_inverse_harmonic_mean_rank": 0.0004381064572794429, - "arithmetic_mean_rank": 6726.40737841276, - "count": 40876.0, - "geometric_mean_rank": 4599.648580410729, - "harmonic_mean_rank": 869.233415396429, - "hits_at_1": 9.785693316371466e-05, - "hits_at_10": 0.0013944612975829338, - "hits_at_3": 0.00041589196594578727, - "hits_at_5": 0.0006605342988550739, - "inverse_arithmetic_mean_rank": 0.00014866777222107106, - "inverse_geometric_mean_rank": 0.00021740791334774195, - "inverse_harmonic_mean_rank": 0.0011504389756391644, - "inverse_median_rank": 0.0001520565650421957, - "median_absolute_deviation": 5317.352856670342, - "median_rank": 6576.5, - "standard_deviation": 4185.0150321375795, - "variance": 17514350.819217503, - "z_arithmetic_mean_rank": 20.061545025360317, - "z_geometric_mean_rank": 24.905953532687732, - "z_hits_at_k": 5.275317808587865, - "z_inverse_harmonic_mean_rank": 8.249809692059822 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9426473106930761, - "adjusted_arithmetic_mean_rank_index": 0.057360727898442154, - "adjusted_geometric_mean_rank_index": 0.12297434805562524, - "adjusted_hits_at_k": 0.0006920234342332076, - "adjusted_inverse_harmonic_mean_rank": 0.000438104068321181, - "arithmetic_mean_rank": 6726.419145708974, - "count": 40876.0, - "geometric_mean_rank": 4599.660570028499, - "harmonic_mean_rank": 869.2352191312187, - "hits_at_1": 9.785693316371466e-05, - "hits_at_10": 0.0013944612975829338, - "hits_at_3": 0.00041589196594578727, - "hits_at_5": 0.0006605342988550739, - "inverse_arithmetic_mean_rank": 0.00014866751213949195, - "inverse_geometric_mean_rank": 0.00021740734664553826, - "inverse_harmonic_mean_rank": 0.0011504365883833812, - "inverse_median_rank": 0.0001520565650421957, - "median_absolute_deviation": 5317.352856670342, - "median_rank": 6576.5, - "standard_deviation": 4185.016936089053, - "variance": 17514366.755352203, - "z_arithmetic_mean_rank": 20.060968205645963, - "z_geometric_mean_rank": 24.905490441146412, - "z_hits_at_k": 5.275317808587865, - "z_inverse_harmonic_mean_rank": 8.249764706530177 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.942646461470457, - "adjusted_arithmetic_mean_rank_index": 0.057361577240088724, - "adjusted_geometric_mean_rank_index": 0.12297601003791969, - "adjusted_hits_at_k": 0.0006920234342332076, - "adjusted_inverse_harmonic_mean_rank": 0.0004381052745533685, - "arithmetic_mean_rank": 6726.4130859375, - "count": 40876.0, - "geometric_mean_rank": 4599.65185546875, - "harmonic_mean_rank": 869.2343083889797, - "hits_at_1": 9.785693316371466e-05, - "hits_at_10": 0.0013944612975829338, - "hits_at_3": 0.00041589196594578727, - "hits_at_5": 0.0006605342988550739, - "inverse_arithmetic_mean_rank": 0.00014866764831822366, - "inverse_geometric_mean_rank": 0.00021740775264333934, - "inverse_harmonic_mean_rank": 0.0011504377937559536, - "inverse_median_rank": 0.00015205657109618187, - "median_absolute_deviation": 5317.352856670342, - "median_rank": 6576.5, - "standard_deviation": 4185.01611328125, - "variance": 17514358.0, - "z_arithmetic_mean_rank": 20.061265248873838, - "z_geometric_mean_rank": 24.90582703560534, - "z_hits_at_k": 5.275317808587865, - "z_inverse_harmonic_mean_rank": 8.249787420612163 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9866627195535737, - "adjusted_arithmetic_mean_rank_index": 0.013339176120974239, - "adjusted_geometric_mean_rank_index": 0.02607247210901087, - "adjusted_hits_at_k": 0.0003620255024674139, - "adjusted_inverse_harmonic_mean_rank": 0.00015334822445050558, - "arithmetic_mean_rank": 6942.788677952833, - "count": 20438.0, - "geometric_mean_rank": 5030.878952767664, - "harmonic_mean_rank": 1140.91162398533, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.001076426264800861, - "hits_at_3": 0.0003914277326548586, - "hits_at_5": 0.0005871415989822879, - "inverse_arithmetic_mean_rank": 0.00014403434216218468, - "inverse_geometric_mean_rank": 0.000198772423146827, - "inverse_harmonic_mean_rank": 0.0008764920778937197, - "inverse_median_rank": 0.00014646649578908824, - "median_absolute_deviation": 5178.729549240067, - "median_rank": 6827.5, - "standard_deviation": 4102.466072979059, - "variance": 16830227.879944224, - "z_arithmetic_mean_rank": 3.2952595134808575, - "z_geometric_mean_rank": 3.7339554587159967, - "z_hits_at_k": 1.9353282494158774, - "z_inverse_harmonic_mean_rank": 2.0250851120577327 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9866642910189499, - "adjusted_arithmetic_mean_rank_index": 0.01333760443224008, - "adjusted_geometric_mean_rank_index": 0.026069825878376696, - "adjusted_hits_at_k": 0.0003620255024674139, - "adjusted_inverse_harmonic_mean_rank": 0.00015334516391599952, - "arithmetic_mean_rank": 6942.7997357862805, - "count": 20438.0, - "geometric_mean_rank": 5030.892619307924, - "harmonic_mean_rank": 1140.9156049524101, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.001076426264800861, - "hits_at_3": 0.0003914277326548586, - "hits_at_5": 0.0005871415989822879, - "inverse_arithmetic_mean_rank": 0.00014403411275793466, - "inverse_geometric_mean_rank": 0.00019877188317678012, - "inverse_harmonic_mean_rank": 0.0008764890195727597, - "inverse_median_rank": 0.00014646649578908824, - "median_absolute_deviation": 5178.729549240067, - "median_rank": 6827.5, - "standard_deviation": 4102.46654893988, - "variance": 16830231.785170693, - "z_arithmetic_mean_rank": 3.2948712494526546, - "z_geometric_mean_rank": 3.7335764801795506, - "z_hits_at_k": 1.9353282494158774, - "z_inverse_harmonic_mean_rank": 2.025044695268523 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9866635375067887, - "adjusted_arithmetic_mean_rank_index": 0.013338358051500632, - "adjusted_geometric_mean_rank_index": 0.026071819300254817, - "adjusted_hits_at_k": 0.0003620255024674139, - "adjusted_inverse_harmonic_mean_rank": 0.00015334667558629324, - "arithmetic_mean_rank": 6942.79443359375, - "count": 20438.0, - "geometric_mean_rank": 5030.88232421875, - "harmonic_mean_rank": 1140.9136386552518, - "hits_at_1": 4.892846658185733e-05, - "hits_at_10": 0.001076426264800861, - "hits_at_3": 0.0003914277326548586, - "hits_at_5": 0.0005871415989822879, - "inverse_arithmetic_mean_rank": 0.00014403421664610505, - "inverse_geometric_mean_rank": 0.0001987722935155034, - "inverse_harmonic_mean_rank": 0.0008764905301497307, - "inverse_median_rank": 0.00014646649651695043, - "median_absolute_deviation": 5178.729549240067, - "median_rank": 6827.5, - "standard_deviation": 4102.46630859375, - "variance": 16830230.0, - "z_arithmetic_mean_rank": 3.2950574206985834, - "z_geometric_mean_rank": 3.733861967051382, - "z_hits_at_k": 1.9353282494158774, - "z_inverse_harmonic_mean_rank": 2.0250646580755074 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.8998336400258861, - "adjusted_arithmetic_mean_rank_index": 0.1001802071568092, - "adjusted_geometric_mean_rank_index": 0.21025959095733382, - "adjusted_hits_at_k": 0.0010220136154691184, - "adjusted_inverse_harmonic_mean_rank": 0.0007228586432761326, - "arithmetic_mean_rank": 6510.026078872688, - "count": 20438.0, - "geometric_mean_rank": 4205.381855120024, - "harmonic_mean_rank": 702.0569486720699, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0017124963303650064, - "hits_at_3": 0.0004403561992367159, - "hits_at_5": 0.0007339269987278598, - "inverse_arithmetic_mean_rank": 0.00015360921567508765, - "inverse_geometric_mean_rank": 0.00023779053471268175, - "inverse_harmonic_mean_rank": 0.0014243858733846092, - "inverse_median_rank": 0.00015867978419549348, - "median_absolute_deviation": 5441.150141915559, - "median_rank": 6302.0, - "standard_deviation": 4254.977323665718, - "variance": 18104832.02490948, - "z_arithmetic_mean_rank": 24.804091992690616, - "z_geometric_mean_rank": 30.11088625655215, - "z_hits_at_k": 5.55555302398681, - "z_inverse_harmonic_mean_rank": 9.706159475823041 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8998353645976944, - "adjusted_arithmetic_mean_rank_index": 0.10017848234659277, - "adjusted_geometric_mean_rank_index": 0.210257618708523, - "adjusted_hits_at_k": 0.0010220136154691184, - "adjusted_inverse_harmonic_mean_rank": 0.0007228569258798545, - "arithmetic_mean_rank": 6510.038555631667, - "count": 20438.0, - "geometric_mean_rank": 4205.392354883008, - "harmonic_mean_rank": 702.0577945559098, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0017124963303650064, - "hits_at_3": 0.0004403561992367159, - "hits_at_5": 0.0007339269987278598, - "inverse_arithmetic_mean_rank": 0.00015360892127665293, - "inverse_geometric_mean_rank": 0.00023778994101201277, - "inverse_harmonic_mean_rank": 0.001424384157194003, - "inverse_median_rank": 0.00015867978419549348, - "median_absolute_deviation": 5441.150141915559, - "median_rank": 6302.0, - "standard_deviation": 4254.9806822134115, - "variance": 18104860.606009305, - "z_arithmetic_mean_rank": 24.803664938759596, - "z_geometric_mean_rank": 30.11060381445597, - "z_hits_at_k": 5.55555302398681, - "z_inverse_harmonic_mean_rank": 9.706136415543812 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.8998344897764146, - "adjusted_arithmetic_mean_rank_index": 0.10017935728880956, - "adjusted_geometric_mean_rank_index": 0.2102583105153928, - "adjusted_hits_at_k": 0.0010220136154691184, - "adjusted_inverse_harmonic_mean_rank": 0.0007228577332912151, - "arithmetic_mean_rank": 6510.0322265625, - "count": 20438.0, - "geometric_mean_rank": 4205.388671875, - "harmonic_mean_rank": 702.0573968744504, - "hits_at_1": 0.00014678539974557198, - "hits_at_10": 0.0017124963303650064, - "hits_at_3": 0.0004403561992367159, - "hits_at_5": 0.0007339269987278598, - "inverse_arithmetic_mean_rank": 0.00015360907127615064, - "inverse_geometric_mean_rank": 0.00023779015464242548, - "inverse_harmonic_mean_rank": 0.0014243849640385327, - "inverse_median_rank": 0.00015867978800088167, - "median_absolute_deviation": 5441.150141915559, - "median_rank": 6302.0, - "standard_deviation": 4254.97900390625, - "variance": 18104848.0, - "z_arithmetic_mean_rank": 24.80388156984722, - "z_geometric_mean_rank": 30.110702886835384, - "z_hits_at_k": 5.55555302398681, - "z_inverse_harmonic_mean_rank": 9.706147257031981 - } - } - }, - "times": { - "evaluation": 9.86320161819458, - "training": 11.522887229919434 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json deleted file mode 100644 index 679e4fa..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Num_epochs varies from the value reported in the paper, and batch_size is not mentioned in the paper. We found out that these values work.", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WordNet Dataset with TransE as described by Bordes et al., 2013" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json b/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json deleted file mode 100644 index 6c48057..0000000 --- a/nonasv/results/27c9fe15/cuda/transe/wn18/bordes2013/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 8192, - "slice_size": null - }, - "losses": [ - 0.06621763066039518, - 0.06459694919663586, - 0.0637450087589807, - 0.0629228597891125, - 0.06228165925657403 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9437711675374072, - "adjusted_arithmetic_mean_rank_index": 0.056231580394964786, - "adjusted_geometric_mean_rank_index": 0.13584888007932028, - "adjusted_hits_at_k": 0.00015569078806712837, - "adjusted_inverse_harmonic_mean_rank": 0.0001548875701843438, - "arithmetic_mean_rank": 19312.6093, - "count": 10000.0, - "geometric_mean_rank": 13013.095349439951, - "harmonic_mean_rank": 2334.080357303059, - "hits_at_1": 0.0, - "hits_at_10": 0.0004, - "hits_at_3": 0.0001, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 5.177964222576594e-05, - "inverse_geometric_mean_rank": 7.684566762534613e-05, - "inverse_harmonic_mean_rank": 0.00042843426400085985, - "inverse_median_rank": 5.2801098262843864e-05, - "median_absolute_deviation": 15893.495782380052, - "median_rank": 18939.0, - "standard_deviation": 12153.88657263402, - "variance": 147716958.82045352, - "z_arithmetic_mean_rank": 9.739345246937546, - "z_geometric_mean_rank": 13.59587579541774, - "z_hits_at_k": 0.9958779562999378, - "z_inverse_harmonic_mean_rank": 2.444712247055369 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9437718125967681, - "adjusted_arithmetic_mean_rank_index": 0.05623093530407952, - "adjusted_geometric_mean_rank_index": 0.13584804290931352, - "adjusted_hits_at_k": 0.00015569078806712837, - "adjusted_inverse_harmonic_mean_rank": 0.00015488743965322147, - "arithmetic_mean_rank": 19312.6225, - "count": 10000.0, - "geometric_mean_rank": 13013.107955262823, - "harmonic_mean_rank": 2334.081068233282, - "hits_at_1": 0.0, - "hits_at_10": 0.0004, - "hits_at_3": 0.0001, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 5.177960683485632e-05, - "inverse_geometric_mean_rank": 7.684559318479912e-05, - "inverse_harmonic_mean_rank": 0.0004284341335054494, - "inverse_median_rank": 5.2801098262843864e-05, - "median_absolute_deviation": 15893.495782380052, - "median_rank": 18939.0, - "standard_deviation": 12153.887721967558, - "variance": 147716986.75819376, - "z_arithmetic_mean_rank": 9.739233516788687, - "z_geometric_mean_rank": 13.595792010704713, - "z_hits_at_k": 0.9958779562999378, - "z_inverse_harmonic_mean_rank": 2.444710186780081 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9437714575392363, - "adjusted_arithmetic_mean_rank_index": 0.05623129037896313, - "adjusted_geometric_mean_rank_index": 0.13584762434833497, - "adjusted_hits_at_k": 0.00015569078806712837, - "adjusted_inverse_harmonic_mean_rank": 0.0001548874873742069, - "arithmetic_mean_rank": 19312.615234375, - "count": 10000.0, - "geometric_mean_rank": 13013.1142578125, - "harmonic_mean_rank": 2334.0808083236393, - "hits_at_1": 0.0, - "hits_at_10": 0.0004, - "hits_at_3": 0.0001, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 5.177962520974688e-05, - "inverse_geometric_mean_rank": 7.684555748710409e-05, - "inverse_harmonic_mean_rank": 0.0004284341812133789, - "inverse_median_rank": 5.2801096899202093e-05, - "median_absolute_deviation": 15893.495782380052, - "median_rank": 18939.0, - "standard_deviation": 12153.88671875, - "variance": 147716960.0, - "z_arithmetic_mean_rank": 9.739295015982833, - "z_geometric_mean_rank": 13.59575012075264, - "z_hits_at_k": 0.9958779562999378, - "z_inverse_harmonic_mean_rank": 2.444710939997898 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9432172063057032, - "adjusted_arithmetic_mean_rank_index": 0.05678556876338092, - "adjusted_geometric_mean_rank_index": 0.12458016502255864, - "adjusted_hits_at_k": 0.0005557828463619552, - "adjusted_inverse_harmonic_mean_rank": 0.00017251546772340296, - "arithmetic_mean_rank": 19300.826, - "count": 5000.0, - "geometric_mean_rank": 13183.127109601111, - "harmonic_mean_rank": 2241.835019542241, - "hits_at_1": 0.0, - "hits_at_10": 0.0008, - "hits_at_3": 0.0002, - "hits_at_5": 0.0004, - "inverse_arithmetic_mean_rank": 5.181125408829653e-05, - "inverse_geometric_mean_rank": 7.585453676402104e-05, - "inverse_harmonic_mean_rank": 0.00044606315419418754, - "inverse_median_rank": 5.366102331571463e-05, - "median_absolute_deviation": 15631.816490813813, - "median_rank": 18635.5, - "standard_deviation": 12029.896846778196, - "variance": 144718418.144124, - "z_arithmetic_mean_rank": 6.954604180176574, - "z_geometric_mean_rank": 8.81692545777343, - "z_hits_at_k": 2.5137858136954785, - "z_inverse_harmonic_mean_rank": 1.9253919774028487 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9432178709277252, - "adjusted_arithmetic_mean_rank_index": 0.0567849041088776, - "adjusted_geometric_mean_rank_index": 0.124579314102237, - "adjusted_hits_at_k": 0.0005557828463619552, - "adjusted_inverse_harmonic_mean_rank": 0.00017251532995541503, - "arithmetic_mean_rank": 19300.8396, - "count": 5000.0, - "geometric_mean_rank": 13183.139922813003, - "harmonic_mean_rank": 2241.8357117507135, - "hits_at_1": 0.0, - "hits_at_10": 0.0008, - "hits_at_3": 0.0002, - "hits_at_5": 0.0004, - "inverse_arithmetic_mean_rank": 5.1811217580399975e-05, - "inverse_geometric_mean_rank": 7.585446303801509e-05, - "inverse_harmonic_mean_rank": 0.0004460630164638922, - "inverse_median_rank": 5.366102331571463e-05, - "median_absolute_deviation": 15631.816490813813, - "median_rank": 18635.5, - "standard_deviation": 12029.898754049089, - "variance": 144718464.03267184, - "z_arithmetic_mean_rank": 6.954522779054997, - "z_geometric_mean_rank": 8.816865235497723, - "z_hits_at_k": 2.5137858136954785, - "z_inverse_harmonic_mean_rank": 1.9253904398167867 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9432175010484657, - "adjusted_arithmetic_mean_rank_index": 0.056785274006213804, - "adjusted_geometric_mean_rank_index": 0.12457991600156137, - "adjusted_hits_at_k": 0.0005557828463619552, - "adjusted_inverse_harmonic_mean_rank": 0.00017251540288220268, - "arithmetic_mean_rank": 19300.83203125, - "count": 5000.0, - "geometric_mean_rank": 13183.130859375, - "harmonic_mean_rank": 2241.835345333606, - "hits_at_1": 0.0, - "hits_at_10": 0.0008, - "hits_at_3": 0.0002, - "hits_at_5": 0.0004, - "inverse_arithmetic_mean_rank": 5.181123924558051e-05, - "inverse_geometric_mean_rank": 7.585451385239139e-05, - "inverse_harmonic_mean_rank": 0.0004460630893707275, - "inverse_median_rank": 5.36610241397284e-05, - "median_absolute_deviation": 15631.816490813813, - "median_rank": 18635.5, - "standard_deviation": 12029.8984375, - "variance": 144718448.0, - "z_arithmetic_mean_rank": 6.95456808087404, - "z_geometric_mean_rank": 8.816907833783532, - "z_hits_at_k": 2.5137858136954785, - "z_inverse_harmonic_mean_rank": 1.9253912537301927 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9443251030846932, - "adjusted_arithmetic_mean_rank_index": 0.05567761771348001, - "adjusted_geometric_mean_rank_index": 0.14705770204849056, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": 0.00013725967285040471, - "arithmetic_mean_rank": 19324.3926, - "count": 5000.0, - "geometric_mean_rank": 12845.25660457957, - "harmonic_mean_rank": 2434.2427430574785, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 5.1748068914724906e-05, - "inverse_geometric_mean_rank": 7.784974880482197e-05, - "inverse_harmonic_mean_rank": 0.0004108053738075322, - "inverse_median_rank": 5.215667866270276e-05, - "median_absolute_deviation": 16158.881579492556, - "median_rank": 19173.0, - "standard_deviation": 12276.612798506974, - "variance": 150715221.80446523, - "z_arithmetic_mean_rank": 6.818913094753356, - "z_geometric_mean_rank": 10.407730321853201, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": 1.5319486518070051 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9443257285822998, - "adjusted_arithmetic_mean_rank_index": 0.05567699218530575, - "adjusted_geometric_mean_rank_index": 0.14705687849801652, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": 0.00013725954955614802, - "arithmetic_mean_rank": 19324.4054, - "count": 5000.0, - "geometric_mean_rank": 12845.269006231641, - "harmonic_mean_rank": 2434.243473442592, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 5.174803463810587e-05, - "inverse_geometric_mean_rank": 7.7849673643648e-05, - "inverse_harmonic_mean_rank": 0.0004108052505470068, - "inverse_median_rank": 5.215667866270276e-05, - "median_absolute_deviation": 16158.881579492556, - "median_rank": 19173.0, - "standard_deviation": 12276.613206021066, - "variance": 150715231.81025082, - "z_arithmetic_mean_rank": 6.818836485472403, - "z_geometric_mean_rank": 10.407672036628277, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": 1.5319472757256967 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9443254837891691, - "adjusted_arithmetic_mean_rank_index": 0.05567723699039928, - "adjusted_geometric_mean_rank_index": 0.1470571030338207, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": 0.0001372596197680961, - "arithmetic_mean_rank": 19324.400390625, - "count": 5000.0, - "geometric_mean_rank": 12845.265625, - "harmonic_mean_rank": 2434.243057512688, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 5.1748047553701326e-05, - "inverse_geometric_mean_rank": 7.78496905695647e-05, - "inverse_harmonic_mean_rank": 0.00041080532073974605, - "inverse_median_rank": 5.2156679885229096e-05, - "median_absolute_deviation": 16158.881579492556, - "median_rank": 19173.0, - "standard_deviation": 12276.61328125, - "variance": 150715232.0, - "z_arithmetic_mean_rank": 6.818866467083083, - "z_geometric_mean_rank": 10.407687927724524, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": 1.531948059357895 - } - } - }, - "times": { - "evaluation": 5.315532684326172, - "training": 33.14157509803772 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json deleted file mode 100644 index e77bdff..0000000 --- a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with TransH as described by Wang et al., 2014" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json b/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json deleted file mode 100644 index a954644..0000000 --- a/nonasv/results/27c9fe15/cuda/transh/fb15k/wang2014/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 13.087838403357605, - 0.20433549977189364, - 0.002744033488619886, - 0.0026934515536661723, - 0.002643932435918485 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9959494722590975, - "adjusted_arithmetic_mean_rank_index": 0.00405107615800071, - "adjusted_geometric_mean_rank_index": 0.021381014132858422, - "adjusted_hits_at_k": 0.0013794792690803878, - "adjusted_inverse_harmonic_mean_rank": 0.000676489179756041, - "arithmetic_mean_rank": 7356.93175162093, - "count": 118142.0, - "geometric_mean_rank": 5315.497395357781, - "harmonic_mean_rank": 731.9753418257839, - "hits_at_1": 0.0004147551251883327, - "hits_at_10": 0.0020568468453217316, - "hits_at_3": 0.000931082934096257, - "hits_at_5": 0.001278122936804862, - "inverse_arithmetic_mean_rank": 0.00013592623035814803, - "inverse_geometric_mean_rank": 0.00018812914871773557, - "inverse_harmonic_mean_rank": 0.001366166239296635, - "inverse_median_rank": 0.00013635124079629124, - "median_absolute_deviation": 5423.358915293492, - "median_rank": 7334.0, - "standard_deviation": 4278.634830765392, - "variance": 18306716.015038792, - "z_arithmetic_mean_rank": 2.409650151735491, - "z_geometric_mean_rank": 7.361788160640344, - "z_hits_at_k": 18.199456465440285, - "z_inverse_harmonic_mean_rank": 22.045342059488902 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9959510214785858, - "adjusted_arithmetic_mean_rank_index": 0.004049526728757358, - "adjusted_geometric_mean_rank_index": 0.021378717325856278, - "adjusted_hits_at_k": 0.0013794792690803878, - "adjusted_inverse_harmonic_mean_rank": 0.0006764877426722778, - "arithmetic_mean_rank": 7356.94319547663, - "count": 118142.0, - "geometric_mean_rank": 5315.509868419316, - "harmonic_mean_rank": 731.9761112672941, - "hits_at_1": 0.0004147551251883327, - "hits_at_10": 0.0020568468453217316, - "hits_at_3": 0.000931082934096257, - "hits_at_5": 0.001278122936804862, - "inverse_arithmetic_mean_rank": 0.0001359260189224845, - "inverse_geometric_mean_rank": 0.00018812870726498567, - "inverse_harmonic_mean_rank": 0.0013661648032046666, - "inverse_median_rank": 0.00013635124079629124, - "median_absolute_deviation": 5423.358915293492, - "median_rank": 7334.0, - "standard_deviation": 4278.635477992422, - "variance": 18306721.553535447, - "z_arithmetic_mean_rank": 2.40872852442817, - "z_geometric_mean_rank": 7.360997337226195, - "z_hits_at_k": 18.199456465440285, - "z_inverse_harmonic_mean_rank": 22.045295227988753 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.995950250449693, - "adjusted_arithmetic_mean_rank_index": 0.0040502978620429175, - "adjusted_geometric_mean_rank_index": 0.02138035468481103, - "adjusted_hits_at_k": 0.0013794792690803878, - "adjusted_inverse_harmonic_mean_rank": 0.0006764883999218058, - "arithmetic_mean_rank": 7356.9375, - "count": 118142.0, - "geometric_mean_rank": 5315.5009765625, - "harmonic_mean_rank": 731.97575936341, - "hits_at_1": 0.0004147551251883327, - "hits_at_10": 0.0020568468453217316, - "hits_at_3": 0.000931082934096257, - "hits_at_5": 0.001278122936804862, - "inverse_arithmetic_mean_rank": 0.0001359261223115027, - "inverse_geometric_mean_rank": 0.0001881290227174759, - "inverse_harmonic_mean_rank": 0.0013661654600005979, - "inverse_median_rank": 0.0001363512419629842, - "median_absolute_deviation": 5423.358915293492, - "median_rank": 7334.0, - "standard_deviation": 4278.63525390625, - "variance": 18306720.0, - "z_arithmetic_mean_rank": 2.409187207841228, - "z_geometric_mean_rank": 7.3615611032707875, - "z_hits_at_k": 18.199456465440285, - "z_inverse_harmonic_mean_rank": 22.045316646351523 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9842617964840297, - "adjusted_arithmetic_mean_rank_index": 0.01574034018985404, - "adjusted_geometric_mean_rank_index": 0.037777252629037306, - "adjusted_hits_at_k": 0.001351989829234509, - "adjusted_inverse_harmonic_mean_rank": 0.0007421923664650952, - "arithmetic_mean_rank": 7250.809602004368, - "count": 59071.0, - "geometric_mean_rank": 5211.424203899233, - "harmonic_mean_rank": 697.4870193828475, - "hits_at_1": 0.00045707707673816255, - "hits_at_10": 0.0020314536743918335, - "hits_at_3": 0.0010157268371959168, - "hits_at_5": 0.00130351610773476, - "inverse_arithmetic_mean_rank": 0.00013791563354850281, - "inverse_geometric_mean_rank": 0.00019188612572582198, - "inverse_harmonic_mean_rank": 0.0014337184380647297, - "inverse_median_rank": 0.00013886960144424384, - "median_absolute_deviation": 5328.472373309133, - "median_rank": 7201.0, - "standard_deviation": 4237.865694484515, - "variance": 17959505.644488726, - "z_arithmetic_mean_rank": 6.619417540422489, - "z_geometric_mean_rank": 9.196875836142056, - "z_hits_at_k": 12.593205094116266, - "z_inverse_harmonic_mean_rank": 17.076318195409776 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9842633384412486, - "adjusted_arithmetic_mean_rank_index": 0.015738798023293477, - "adjusted_geometric_mean_rank_index": 0.037774950811552466, - "adjusted_hits_at_k": 0.001351989829234509, - "adjusted_inverse_harmonic_mean_rank": 0.0007421905012426657, - "arithmetic_mean_rank": 7250.820961216164, - "count": 59071.0, - "geometric_mean_rank": 5211.436668212272, - "harmonic_mean_rank": 697.487926164658, - "hits_at_1": 0.00045707707673816255, - "hits_at_10": 0.0020314536743918335, - "hits_at_3": 0.0010157268371959168, - "hits_at_5": 0.00130351610773476, - "inverse_arithmetic_mean_rank": 0.00013791541748843186, - "inverse_geometric_mean_rank": 0.00019188566678735817, - "inverse_harmonic_mean_rank": 0.0014337165741331085, - "inverse_median_rank": 0.00013886960144424384, - "median_absolute_deviation": 5328.472373309133, - "median_rank": 7201.0, - "standard_deviation": 4237.866652494595, - "variance": 17959513.76432574, - "z_arithmetic_mean_rank": 6.618769000158636, - "z_geometric_mean_rank": 9.196315458450925, - "z_hits_at_k": 12.593205094116266, - "z_inverse_harmonic_mean_rank": 17.07627528048213 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9842626538451784, - "adjusted_arithmetic_mean_rank_index": 0.015739482712306962, - "adjusted_geometric_mean_rank_index": 0.03777578909713941, - "adjusted_hits_at_k": 0.001351989829234509, - "adjusted_inverse_harmonic_mean_rank": 0.0007421915149824226, - "arithmetic_mean_rank": 7250.81591796875, - "count": 59071.0, - "geometric_mean_rank": 5211.43212890625, - "harmonic_mean_rank": 697.4874333326487, - "hits_at_1": 0.00045707707673816255, - "hits_at_10": 0.0020314536743918335, - "hits_at_3": 0.0010157268371959168, - "hits_at_5": 0.00130351610773476, - "inverse_arithmetic_mean_rank": 0.00013791551464237273, - "inverse_geometric_mean_rank": 0.00019188583246432245, - "inverse_harmonic_mean_rank": 0.0014337175871713171, - "inverse_median_rank": 0.00013886959641240537, - "median_absolute_deviation": 5328.472373309133, - "median_rank": 7201.0, - "standard_deviation": 4237.8662109375, - "variance": 17959510.0, - "z_arithmetic_mean_rank": 6.619056938183462, - "z_geometric_mean_rank": 9.196519539158771, - "z_hits_at_k": 12.593205094116266, - "z_inverse_harmonic_mean_rank": 17.076298604546665 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.0075737044040576, - "adjusted_arithmetic_mean_rank_index": -0.007574727054556751, - "adjusted_geometric_mean_rank_index": 0.004713849867451159, - "adjusted_hits_at_k": 0.001406968594468856, - "adjusted_inverse_harmonic_mean_rank": 0.0006107862423339953, - "arithmetic_mean_rank": 7463.053901237494, - "count": 59071.0, - "geometric_mean_rank": 5421.648949420608, - "harmonic_mean_rank": 770.051738846899, - "hits_at_1": 0.0003724331736385028, - "hits_at_10": 0.0020822400162516293, - "hits_at_3": 0.0008464390309965973, - "hits_at_5": 0.001252729765874964, - "inverse_arithmetic_mean_rank": 0.0001339934044740296, - "inverse_geometric_mean_rank": 0.00018444573031731727, - "inverse_harmonic_mean_rank": 0.0012986140405285406, - "inverse_median_rank": 0.00013381506757660912, - "median_absolute_deviation": 5521.210661714862, - "median_rank": 7473.0, - "standard_deviation": 4316.410842854324, - "variance": 18631402.56431038, - "z_arithmetic_mean_rank": -3.186405474881326, - "z_geometric_mean_rank": 1.1475778863209314, - "z_hits_at_k": 13.14558975014015, - "z_inverse_harmonic_mean_rank": 14.095987190833268 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.007575260846394, - "adjusted_arithmetic_mean_rank_index": -0.007576283707054099, - "adjusted_geometric_mean_rank_index": 0.004711558929027637, - "adjusted_hits_at_k": 0.001406968594468856, - "adjusted_inverse_harmonic_mean_rank": 0.0006107852333872736, - "arithmetic_mean_rank": 7463.065429737096, - "count": 59071.0, - "geometric_mean_rank": 5421.66142660916, - "harmonic_mean_rank": 770.0523367204994, - "hits_at_1": 0.0003724331736385028, - "hits_at_10": 0.0020822400162516293, - "hits_at_3": 0.0008464390309965973, - "hits_at_5": 0.001252729765874964, - "inverse_arithmetic_mean_rank": 0.00013399319748898776, - "inverse_geometric_mean_rank": 0.0001844453058415019, - "inverse_harmonic_mean_rank": 0.0012986130322762246, - "inverse_median_rank": 0.00013381506757660912, - "median_absolute_deviation": 5521.210661714862, - "median_rank": 7473.0, - "standard_deviation": 4316.411181240301, - "variance": 18631405.48553629, - "z_arithmetic_mean_rank": -3.187060300593714, - "z_geometric_mean_rank": 1.1470201616695874, - "z_hits_at_k": 13.14558975014015, - "z_inverse_harmonic_mean_rank": 14.095963905927553 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.0075744038538572, - "adjusted_arithmetic_mean_rank_index": -0.0075754265988008385, - "adjusted_geometric_mean_rank_index": 0.004712240446819949, - "adjusted_hits_at_k": 0.001406968594468856, - "adjusted_inverse_harmonic_mean_rank": 0.0006107857926385235, - "arithmetic_mean_rank": 7463.05908203125, - "count": 59071.0, - "geometric_mean_rank": 5421.65771484375, - "harmonic_mean_rank": 770.0520053237399, - "hits_at_1": 0.0003724331736385028, - "hits_at_10": 0.0020822400162516293, - "hits_at_3": 0.0008464390309965973, - "hits_at_5": 0.001252729765874964, - "inverse_arithmetic_mean_rank": 0.00013399330782704055, - "inverse_geometric_mean_rank": 0.00018444543820805848, - "inverse_harmonic_mean_rank": 0.001298613591142571, - "inverse_median_rank": 0.0001338150614174083, - "median_absolute_deviation": 5521.210661714862, - "median_rank": 7473.0, - "standard_deviation": 4316.4111328125, - "variance": 18631404.0, - "z_arithmetic_mean_rank": -3.186699747083196, - "z_geometric_mean_rank": 1.1471860758946093, - "z_hits_at_k": 13.14558975014015, - "z_inverse_harmonic_mean_rank": 14.09597681256806 - } - } - }, - "times": { - "evaluation": 29.929590225219727, - "training": 10.419500827789307 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json deleted file mode 100644 index 0c7f574..0000000 --- a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with TransH as described by Wang et al., 2014" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json b/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json deleted file mode 100644 index 2257533..0000000 --- a/nonasv/results/27c9fe15/cuda/transh/wn18/wang2014/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 20.458278117090398, - 14.580624948079425, - 10.229629155634465, - 7.0087562531152345, - 4.624470838201348 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9981434654045136, - "adjusted_arithmetic_mean_rank_index": 0.0018566253253005094, - "adjusted_geometric_mean_rank_index": -0.0037453053362628275, - "adjusted_hits_at_k": -4.435809332781619e-05, - "adjusted_inverse_harmonic_mean_rank": -2.435375634154849e-05, - "arithmetic_mean_rank": 20425.2423, - "count": 10000.0, - "geometric_mean_rank": 15115.057389391592, - "harmonic_mean_rank": 4012.1652711899733, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.895902752644457e-05, - "inverse_geometric_mean_rank": 6.61591930641192e-05, - "inverse_harmonic_mean_rank": 0.0002492419759426831, - "inverse_median_rank": 4.8811441401864594e-05, - "median_absolute_deviation": 15004.675752385945, - "median_rank": 20487.0, - "standard_deviation": 11726.714861801267, - "variance": 137515841.4499907, - "z_arithmetic_mean_rank": 0.3215686792065924, - "z_geometric_mean_rank": -0.3748334630216448, - "z_hits_at_k": -0.28373706548148925, - "z_inverse_harmonic_mean_rank": -0.3843944760649638 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9981445844847684, - "adjusted_arithmetic_mean_rank_index": 0.0018555061903556824, - "adjusted_geometric_mean_rank_index": -0.0037469601889252857, - "adjusted_hits_at_k": -4.435809332781619e-05, - "adjusted_inverse_harmonic_mean_rank": -2.4354352510563197e-05, - "arithmetic_mean_rank": 20425.2652, - "count": 10000.0, - "geometric_mean_rank": 15115.082307603394, - "harmonic_mean_rank": 4012.1748654002586, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.895897263551809e-05, - "inverse_geometric_mean_rank": 6.615908399631846e-05, - "inverse_harmonic_mean_rank": 0.00024924137993677375, - "inverse_median_rank": 4.8811441401864594e-05, - "median_absolute_deviation": 15004.675752385945, - "median_rank": 20487.0, - "standard_deviation": 11726.718338958643, - "variance": 137515923.00126895, - "z_arithmetic_mean_rank": 0.3213748443271811, - "z_geometric_mean_rank": -0.3749990821364122, - "z_hits_at_k": -0.28373706548148925, - "z_inverse_harmonic_mean_rank": -0.38440388586905394 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9981439371347137, - "adjusted_arithmetic_mean_rank_index": 0.0018561535720467148, - "adjusted_geometric_mean_rank_index": -0.00374499619021873, - "adjusted_hits_at_k": -4.435809332781619e-05, - "adjusted_inverse_harmonic_mean_rank": -2.4354046496305264e-05, - "arithmetic_mean_rank": 20425.251953125, - "count": 10000.0, - "geometric_mean_rank": 15115.052734375, - "harmonic_mean_rank": 4012.1699406750786, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.895900565315969e-05, - "inverse_geometric_mean_rank": 6.61592130200006e-05, - "inverse_harmonic_mean_rank": 0.00024924168586730954, - "inverse_median_rank": 4.881144195678644e-05, - "median_absolute_deviation": 15004.675752385945, - "median_rank": 20487.0, - "standard_deviation": 11726.716796875, - "variance": 137515888.0, - "z_arithmetic_mean_rank": 0.32148697124500814, - "z_geometric_mean_rank": -0.3748025234127681, - "z_hits_at_k": -0.28373706548148925, - "z_inverse_harmonic_mean_rank": -0.384399055805523 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9927426109595996, - "adjusted_arithmetic_mean_rank_index": 0.0072577437210114, - "adjusted_geometric_mean_rank_index": 0.004254858044036025, - "adjusted_hits_at_k": -4.436380124831586e-05, - "adjusted_inverse_harmonic_mean_rank": -2.3555047261868654e-05, - "arithmetic_mean_rank": 20314.2524, - "count": 5000.0, - "geometric_mean_rank": 14994.993173994697, - "harmonic_mean_rank": 3999.2596074854223, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.922652236023216e-05, - "inverse_geometric_mean_rank": 6.668892665681675e-05, - "inverse_harmonic_mean_rank": 0.0002500462830990761, - "inverse_median_rank": 4.904845987835982e-05, - "median_absolute_deviation": 14937.21735144394, - "median_rank": 20388.0, - "standard_deviation": 11707.495051388842, - "variance": 137065440.37829423, - "z_arithmetic_mean_rank": 0.8888655325637131, - "z_geometric_mean_rank": 0.30112952732788756, - "z_hits_at_k": -0.20065587657052963, - "z_inverse_harmonic_mean_rank": -0.2628906243819342 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9927437349527253, - "adjusted_arithmetic_mean_rank_index": 0.007256619672954323, - "adjusted_geometric_mean_rank_index": 0.004252941731053506, - "adjusted_hits_at_k": -4.436380124831586e-05, - "adjusted_inverse_harmonic_mean_rank": -2.3556101099611148e-05, - "arithmetic_mean_rank": 20314.2754, - "count": 5000.0, - "geometric_mean_rank": 14995.022029956497, - "harmonic_mean_rank": 3999.2764581073648, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.922646662553369e-05, - "inverse_geometric_mean_rank": 6.668879832268584e-05, - "inverse_harmonic_mean_rank": 0.0002500452295496582, - "inverse_median_rank": 4.904725703215048e-05, - "median_absolute_deviation": 14936.476050334686, - "median_rank": 20388.5, - "standard_deviation": 11707.499174407609, - "variance": 137065536.91875485, - "z_arithmetic_mean_rank": 0.8887278689022224, - "z_geometric_mean_rank": 0.30099390390246583, - "z_hits_at_k": -0.20065587657052963, - "z_inverse_harmonic_mean_rank": -0.2629023859402553 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9927430663600753, - "adjusted_arithmetic_mean_rank_index": 0.007257288298279541, - "adjusted_geometric_mean_rank_index": 0.004253626494188811, - "adjusted_hits_at_k": -4.436380124831586e-05, - "adjusted_inverse_harmonic_mean_rank": -2.355555413243441e-05, - "arithmetic_mean_rank": 20314.26171875, - "count": 5000.0, - "geometric_mean_rank": 14995.01171875, - "harmonic_mean_rank": 3999.267712210899, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.922649895888753e-05, - "inverse_geometric_mean_rank": 6.668884452665225e-05, - "inverse_harmonic_mean_rank": 0.0002500457763671875, - "inverse_median_rank": 4.90478596475441e-05, - "median_absolute_deviation": 14936.846700889313, - "median_rank": 20388.25, - "standard_deviation": 11707.4970703125, - "variance": 137065488.0, - "z_arithmetic_mean_rank": 0.8888097563356467, - "z_geometric_mean_rank": 0.30104236671770934, - "z_hits_at_k": -0.20065587657052963, - "z_inverse_harmonic_mean_rank": -0.26289628140813504 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 1.0035440694387567, - "adjusted_arithmetic_mean_rank_index": -0.003544242635263428, - "adjusted_geometric_mean_rank_index": -0.011708739851778116, - "adjusted_hits_at_k": -4.4352385407381626e-05, - "adjusted_inverse_harmonic_mean_rank": -2.5152465411934434e-05, - "arithmetic_mean_rank": 20536.2322, - "count": 5000.0, - "geometric_mean_rank": 15236.08295339673, - "harmonic_mean_rank": 4025.1544980492276, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.869442409206885e-05, - "inverse_geometric_mean_rank": 6.56336673316064e-05, - "inverse_harmonic_mean_rank": 0.00024843766878629013, - "inverse_median_rank": 4.8432013560963794e-05, - "median_absolute_deviation": 15072.875454437202, - "median_rank": 20647.5, - "standard_deviation": 11744.854405478305, - "variance": 137941605.00588316, - "z_arithmetic_mean_rank": -0.4340681859082025, - "z_geometric_mean_rank": -0.8286638855940964, - "z_hits_at_k": -0.20060892906062588, - "z_inverse_harmonic_mean_rank": -0.28072546493266515 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 1.0035451836063685, - "adjusted_arithmetic_mean_rank_index": -0.0035453568573240446, - "adjusted_geometric_mean_rank_index": -0.011710128780810125, - "adjusted_hits_at_k": -4.4352385407381626e-05, - "adjusted_inverse_harmonic_mean_rank": -2.515260391222656e-05, - "arithmetic_mean_rank": 20536.255, - "count": 5000.0, - "geometric_mean_rank": 15236.103868950964, - "harmonic_mean_rank": 4025.15674140012, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.869437002997869e-05, - "inverse_geometric_mean_rank": 6.563357723215968e-05, - "inverse_harmonic_mean_rank": 0.00024843753032388936, - "inverse_median_rank": 4.8432013560963794e-05, - "median_absolute_deviation": 15072.875454437202, - "median_rank": 20647.5, - "standard_deviation": 11744.857241038522, - "variance": 137941671.612375, - "z_arithmetic_mean_rank": -0.4342046461899116, - "z_geometric_mean_rank": -0.8287621844155808, - "z_hits_at_k": -0.20060892906062588, - "z_inverse_harmonic_mean_rank": -0.2807270107278095 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 1.0035446529412848, - "adjusted_arithmetic_mean_rank_index": -0.0035448261663069136, - "adjusted_geometric_mean_rank_index": -0.01170900286578247, - "adjusted_hits_at_k": -4.4352385407381626e-05, - "adjusted_inverse_harmonic_mean_rank": -2.5152538850884614e-05, - "arithmetic_mean_rank": 20536.244140625, - "count": 5000.0, - "geometric_mean_rank": 15236.0869140625, - "harmonic_mean_rank": 4025.155687572287, - "hits_at_1": 0.0, - "hits_at_10": 0.0002, - "hits_at_3": 0.0, - "hits_at_5": 0.0002, - "inverse_arithmetic_mean_rank": 4.869439726462588e-05, - "inverse_geometric_mean_rank": 6.56336487736553e-05, - "inverse_harmonic_mean_rank": 0.00024843759536743163, - "inverse_median_rank": 4.8432015319122e-05, - "median_absolute_deviation": 15072.875454437202, - "median_rank": 20647.5, - "standard_deviation": 11744.8564453125, - "variance": 137941648.0, - "z_arithmetic_mean_rank": -0.43413965174379143, - "z_geometric_mean_rank": -0.828682499912091, - "z_hits_at_k": -0.20060892906062588, - "z_inverse_harmonic_mean_rank": -0.28072628458127946 - } - } - }, - "times": { - "evaluation": 8.578476905822754, - "training": 2.8984744548797607 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json deleted file mode 100644 index c233fc8..0000000 --- a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json b/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json deleted file mode 100644 index f92ac60..0000000 --- a/nonasv/results/27c9fe15/cuda/transr/fb15k/li2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 0.000208604416764187, - 0.00020803913257138926, - 0.0002074576098380109, - 0.00020695912158022746, - 0.00020641966698798603 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9207919775259267, - "adjusted_arithmetic_mean_rank_index": 0.07921874676399432, - "adjusted_geometric_mean_rank_index": 0.21897247094458427, - "adjusted_hits_at_k": 0.005318092337383576, - "adjusted_inverse_harmonic_mean_rank": 0.0051853177799535705, - "arithmetic_mean_rank": 6801.7544311083275, - "count": 118142.0, - "geometric_mean_rank": 4242.455386428855, - "harmonic_mean_rank": 170.30311795666364, - "hits_at_1": 0.003927477103824212, - "hits_at_10": 0.005992788339455909, - "hits_at_3": 0.00552724687240778, - "hits_at_5": 0.00565421272705727, - "inverse_arithmetic_mean_rank": 0.00014702089146682892, - "inverse_geometric_mean_rank": 0.00023571255532795686, - "inverse_harmonic_mean_rank": 0.005871883098784287, - "inverse_median_rank": 0.000152114390021296, - "median_absolute_deviation": 5675.401292439445, - "median_rank": 6574.0, - "standard_deviation": 4419.625361302917, - "variance": 19533088.334271934, - "z_arithmetic_mean_rank": 47.12068045997205, - "z_geometric_mean_rank": 75.39534533250384, - "z_hits_at_k": 70.1615400410659, - "z_inverse_harmonic_mean_rank": 168.97846642196097 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9207996732049827, - "adjusted_arithmetic_mean_rank_index": 0.07921105004298967, - "adjusted_geometric_mean_rank_index": 0.21896172620653143, - "adjusted_hits_at_k": 0.005318092337383576, - "adjusted_inverse_harmonic_mean_rank": 0.005185300049122959, - "arithmetic_mean_rank": 6801.811277953649, - "count": 118142.0, - "geometric_mean_rank": 4242.513736903575, - "harmonic_mean_rank": 170.3036318532829, - "hits_at_1": 0.003927477103824212, - "hits_at_10": 0.005992788339455909, - "hits_at_3": 0.00552724687240778, - "hits_at_5": 0.00565421272705727, - "inverse_arithmetic_mean_rank": 0.00014701966272443446, - "inverse_geometric_mean_rank": 0.00023570931339632063, - "inverse_harmonic_mean_rank": 0.0058718653801905, - "inverse_median_rank": 0.000152114390021296, - "median_absolute_deviation": 5675.401292439445, - "median_rank": 6574.0, - "standard_deviation": 4419.628569110977, - "variance": 19533116.688901942, - "z_arithmetic_mean_rank": 47.11610231722352, - "z_geometric_mean_rank": 75.39164576592134, - "z_hits_at_k": 70.1615400410659, - "z_inverse_harmonic_mean_rank": 168.97788861194192 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9207958725565982, - "adjusted_arithmetic_mean_rank_index": 0.07921485120595906, - "adjusted_geometric_mean_rank_index": 0.21896713294853576, - "adjusted_hits_at_k": 0.005318092337383576, - "adjusted_inverse_harmonic_mean_rank": 0.005185308909615302, - "arithmetic_mean_rank": 6801.783203125, - "count": 118142.0, - "geometric_mean_rank": 4242.484375, - "harmonic_mean_rank": 170.3033750472689, - "hits_at_1": 0.003927477103824212, - "hits_at_10": 0.005992788339455909, - "hits_at_3": 0.00552724687240778, - "hits_at_5": 0.00565421272705727, - "inverse_arithmetic_mean_rank": 0.00014702026965096593, - "inverse_geometric_mean_rank": 0.00023571094789076596, - "inverse_harmonic_mean_rank": 0.005871874234567829, - "inverse_median_rank": 0.00015211438585538417, - "median_absolute_deviation": 5675.401292439445, - "median_rank": 6574.0, - "standard_deviation": 4419.62646484375, - "variance": 19533100.0, - "z_arithmetic_mean_rank": 47.11836331468908, - "z_geometric_mean_rank": 75.39350738431921, - "z_hits_at_k": 70.1615400410659, - "z_inverse_harmonic_mean_rank": 168.9781773565226 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9450959015811221, - "adjusted_arithmetic_mean_rank_index": 0.0549115523924576, - "adjusted_geometric_mean_rank_index": 0.16122934046823933, - "adjusted_hits_at_k": 1.3705613470731061e-05, - "adjusted_inverse_harmonic_mean_rank": 0.0007384753351438997, - "arithmetic_mean_rank": 6962.284285012951, - "count": 59071.0, - "geometric_mean_rank": 4542.932684386978, - "harmonic_mean_rank": 699.2987534521512, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006940800054172098, - "hits_at_3": 0.00027086048991891115, - "hits_at_5": 0.00038936195425843476, - "inverse_arithmetic_mean_rank": 0.00014363102094992088, - "inverse_geometric_mean_rank": 0.00022012212583223426, - "inverse_harmonic_mean_rank": 0.001430003979076768, - "inverse_median_rank": 0.00014749262536873156, - "median_absolute_deviation": 5669.470883565422, - "median_rank": 6780.0, - "standard_deviation": 4399.418868449341, - "variance": 19354886.380068082, - "z_arithmetic_mean_rank": 23.092416599277726, - "z_geometric_mean_rank": 39.251298658222595, - "z_hits_at_k": 0.12766190813382305, - "z_inverse_harmonic_mean_rank": 16.990796957990767 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9451039192990608, - "adjusted_arithmetic_mean_rank_index": 0.054903533586005304, - "adjusted_geometric_mean_rank_index": 0.16121720085611924, - "adjusted_hits_at_k": 1.3705613470731061e-05, - "adjusted_inverse_harmonic_mean_rank": 0.0007384551039659525, - "arithmetic_mean_rank": 6962.3433495285335, - "count": 59071.0, - "geometric_mean_rank": 4542.998420235553, - "harmonic_mean_rank": 699.3086401705666, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006940800054172098, - "hits_at_3": 0.00027086048991891115, - "hits_at_5": 0.00038936195425843476, - "inverse_arithmetic_mean_rank": 0.0001436298024669692, - "inverse_geometric_mean_rank": 0.00022011894072993104, - "inverse_harmonic_mean_rank": 0.001429983761899599, - "inverse_median_rank": 0.00014749262536873156, - "median_absolute_deviation": 5669.470883565422, - "median_rank": 6780.0, - "standard_deviation": 4399.420293881387, - "variance": 19354898.922215387, - "z_arithmetic_mean_rank": 23.089044383211046, - "z_geometric_mean_rank": 39.24834326846823, - "z_hits_at_k": 0.12766190813382305, - "z_inverse_harmonic_mean_rank": 16.990331480241736 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9450998641906208, - "adjusted_arithmetic_mean_rank_index": 0.05490758924498118, - "adjusted_geometric_mean_rank_index": 0.1612233113418693, - "adjusted_hits_at_k": 1.3705613470731061e-05, - "adjusted_inverse_harmonic_mean_rank": 0.0007384651003795702, - "arithmetic_mean_rank": 6962.3134765625, - "count": 59071.0, - "geometric_mean_rank": 4542.96533203125, - "harmonic_mean_rank": 699.3037550158945, - "hits_at_1": 8.464390309965973e-05, - "hits_at_10": 0.0006940800054172098, - "hits_at_3": 0.00027086048991891115, - "hits_at_5": 0.00038936195425843476, - "inverse_arithmetic_mean_rank": 0.0001436304155504331, - "inverse_geometric_mean_rank": 0.00022012054978404194, - "inverse_harmonic_mean_rank": 0.0014299937513953017, - "inverse_median_rank": 0.00014749262481927872, - "median_absolute_deviation": 5669.470883565422, - "median_rank": 6780.0, - "standard_deviation": 4399.41943359375, - "variance": 19354892.0, - "z_arithmetic_mean_rank": 23.090749943563594, - "z_geometric_mean_rank": 39.249830866819906, - "z_hits_at_k": 0.12766190813382305, - "z_inverse_harmonic_mean_rank": 16.990561477136694 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.896619981254972, - "adjusted_arithmetic_mean_rank_index": 0.10339397778305548, - "adjusted_geometric_mean_rank_index": 0.2727473513664458, - "adjusted_hits_at_k": 0.010622456975492961, - "adjusted_inverse_harmonic_mean_rank": 0.009632143352825863, - "arithmetic_mean_rank": 6641.224577203704, - "count": 59071.0, - "geometric_mean_rank": 3961.852168247115, - "harmonic_mean_rank": 96.95782962758972, - "hits_at_1": 0.007770310304548764, - "hits_at_10": 0.011291496673494609, - "hits_at_3": 0.01078363325489665, - "hits_at_5": 0.010919063499856106, - "inverse_arithmetic_mean_rank": 0.00015057464001933378, - "inverse_geometric_mean_rank": 0.00025240719681937065, - "inverse_harmonic_mean_rank": 0.010313762218491803, - "inverse_median_rank": 0.00015705983979896342, - "median_absolute_deviation": 5651.6796569433545, - "median_rank": 6367.0, - "standard_deviation": 4433.931733855012, - "variance": 19659750.620486513, - "z_arithmetic_mean_rank": 43.493994503669626, - "z_geometric_mean_rank": 66.39982981680666, - "z_hits_at_k": 99.24774588949502, - "z_inverse_harmonic_mean_rank": 222.29474063277573 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.8966273566432531, - "adjusted_arithmetic_mean_rank_index": 0.10338660139890177, - "adjusted_geometric_mean_rank_index": 0.2727378670781261, - "adjusted_hits_at_k": 0.010622456975492961, - "adjusted_inverse_harmonic_mean_rank": 0.009632128122333099, - "arithmetic_mean_rank": 6641.279206378765, - "count": 59071.0, - "geometric_mean_rank": 3961.9038227361375, - "harmonic_mean_rank": 96.95797270839012, - "hits_at_1": 0.007770310304548764, - "hits_at_10": 0.011291496673494609, - "hits_at_3": 0.01078363325489665, - "hits_at_5": 0.010919063499856106, - "inverse_arithmetic_mean_rank": 0.000150573401437411, - "inverse_geometric_mean_rank": 0.00025240390598613475, - "inverse_harmonic_mean_rank": 0.010313746998481399, - "inverse_median_rank": 0.00015705983979896342, - "median_absolute_deviation": 5651.6796569433545, - "median_rank": 6367.0, - "standard_deviation": 4433.936553851471, - "variance": 19659793.363580257, - "z_arithmetic_mean_rank": 43.4908915336639, - "z_geometric_mean_rank": 66.39752088464945, - "z_hits_at_k": 99.24774588949502, - "z_inverse_harmonic_mean_rank": 222.2943891369233 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.896623611307688, - "adjusted_arithmetic_mean_rank_index": 0.10339034724018625, - "adjusted_geometric_mean_rank_index": 0.27274262130459126, - "adjusted_hits_at_k": 0.010622456975492961, - "adjusted_inverse_harmonic_mean_rank": 0.009632136363889678, - "arithmetic_mean_rank": 6641.25146484375, - "count": 59071.0, - "geometric_mean_rank": 3961.8779296875, - "harmonic_mean_rank": 96.9578952841512, - "hits_at_1": 0.007770310304548764, - "hits_at_10": 0.011291496673494609, - "hits_at_3": 0.01078363325489665, - "hits_at_5": 0.010919063499856106, - "inverse_arithmetic_mean_rank": 0.00015057403652463108, - "inverse_geometric_mean_rank": 0.0002524055598769337, - "inverse_harmonic_mean_rank": 0.010313755234365741, - "inverse_median_rank": 0.00015705983969382942, - "median_absolute_deviation": 5651.6796569433545, - "median_rank": 6367.0, - "standard_deviation": 4433.93408203125, - "variance": 19659772.0, - "z_arithmetic_mean_rank": 43.492467269540725, - "z_geometric_mean_rank": 66.39867829214256, - "z_hits_at_k": 99.24774588949502, - "z_inverse_harmonic_mean_rank": 222.29457933910518 - } - } - }, - "times": { - "evaluation": 33.42562985420227, - "training": 12.187381505966187 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json deleted file mode 100644 index 1d65f30..0000000 --- a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json b/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json deleted file mode 100644 index a2e5cc1..0000000 --- a/nonasv/results/27c9fe15/cuda/transr/wn18/li2015/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0027987702639484797, - 0.0027985652882265874, - 0.0027976621540910128, - 0.0027974926507223015, - 0.0027966113623240974 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9920241586458991, - "adjusted_arithmetic_mean_rank_index": 0.007976231137628131, - "adjusted_geometric_mean_rank_index": 0.01955723705510304, - "adjusted_hits_at_k": -0.00024440697472276075, - "adjusted_inverse_harmonic_mean_rank": -6.828706250853293e-05, - "arithmetic_mean_rank": 20300.0215, - "count": 10000.0, - "geometric_mean_rank": 14764.17558586092, - "harmonic_mean_rank": 4870.429778354284, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.9261031570828635e-05, - "inverse_geometric_mean_rank": 6.773151634403897e-05, - "inverse_harmonic_mean_rank": 0.00020532068944804693, - "inverse_median_rank": 4.95221116228396e-05, - "median_absolute_deviation": 15644.418609671111, - "median_rank": 20193.0, - "standard_deviation": 11944.86457925069, - "variance": 142679789.81663775, - "z_arithmetic_mean_rank": 1.3814882717699675, - "z_geometric_mean_rank": 1.9573055423605155, - "z_hits_at_k": -1.5633520872629163, - "z_inverse_harmonic_mean_rank": -1.0778283746807773 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9920259472195815, - "adjusted_arithmetic_mean_rank_index": 0.007974442476537336, - "adjusted_geometric_mean_rank_index": 0.019554508271745097, - "adjusted_hits_at_k": -0.00024440697472276075, - "adjusted_inverse_harmonic_mean_rank": -6.828763431190998e-05, - "arithmetic_mean_rank": 20300.0581, - "count": 10000.0, - "geometric_mean_rank": 14764.216674957981, - "harmonic_mean_rank": 4870.443338478342, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.9260942755626894e-05, - "inverse_geometric_mean_rank": 6.773132784593504e-05, - "inverse_harmonic_mean_rank": 0.00020532011780110903, - "inverse_median_rank": 4.952088543343155e-05, - "median_absolute_deviation": 15644.418609671111, - "median_rank": 20193.5, - "standard_deviation": 11944.86448975979, - "variance": 142679787.67872438, - "z_arithmetic_mean_rank": 1.381178474539098, - "z_geometric_mean_rank": 1.957032443314091, - "z_hits_at_k": -1.5633520872629163, - "z_inverse_harmonic_mean_rank": -1.0778373999028603 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.992025016892492, - "adjusted_arithmetic_mean_rank_index": 0.007975372849092333, - "adjusted_geometric_mean_rank_index": 0.019554954159064608, - "adjusted_hits_at_k": -0.00024440697472276075, - "adjusted_inverse_harmonic_mean_rank": -6.828734607407729e-05, - "arithmetic_mean_rank": 20300.0390625, - "count": 10000.0, - "geometric_mean_rank": 14764.2099609375, - "harmonic_mean_rank": 4870.436503006005, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.926099063595757e-05, - "inverse_geometric_mean_rank": 6.773135828552768e-05, - "inverse_harmonic_mean_rank": 0.000205320405960083, - "inverse_median_rank": 4.952149902237579e-05, - "median_absolute_deviation": 15644.418609671111, - "median_rank": 20193.25, - "standard_deviation": 11944.8642578125, - "variance": 142679792.0, - "z_arithmetic_mean_rank": 1.38133961565336, - "z_geometric_mean_rank": 1.9570770681104932, - "z_hits_at_k": -1.5633520872629163, - "z_inverse_harmonic_mean_rank": -1.0778328504186194 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.991158386631214, - "adjusted_arithmetic_mean_rank_index": 0.008842045473061222, - "adjusted_geometric_mean_rank_index": 0.023575799917052054, - "adjusted_hits_at_k": -0.0002444126837850729, - "adjusted_inverse_harmonic_mean_rank": -7.339889583391683e-05, - "arithmetic_mean_rank": 20281.8348, - "count": 5000.0, - "geometric_mean_rank": 14704.057212218282, - "harmonic_mean_rank": 4994.604040839103, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.930520388618884e-05, - "inverse_geometric_mean_rank": 6.800844049824926e-05, - "inverse_harmonic_mean_rank": 0.00020021607154908683, - "inverse_median_rank": 4.940955580809328e-05, - "median_absolute_deviation": 15743.011657201734, - "median_rank": 20239.0, - "standard_deviation": 11999.952493518838, - "variance": 143998859.84670898, - "z_arithmetic_mean_rank": 1.0828970766234107, - "z_geometric_mean_rank": 1.6685326306831314, - "z_hits_at_k": -1.1054697733258656, - "z_inverse_harmonic_mean_rank": -0.8191824597167967 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9911600286385629, - "adjusted_arithmetic_mean_rank_index": 0.008840403385464768, - "adjusted_geometric_mean_rank_index": 0.023573195103802513, - "adjusted_hits_at_k": -0.0002444126837850729, - "adjusted_inverse_harmonic_mean_rank": -7.339957108391955e-05, - "arithmetic_mean_rank": 20281.8684, - "count": 5000.0, - "geometric_mean_rank": 14704.096435660544, - "harmonic_mean_rank": 4994.620881120737, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.930512220461898e-05, - "inverse_geometric_mean_rank": 6.800825908450849e-05, - "inverse_harmonic_mean_rank": 0.00020021539648382906, - "inverse_median_rank": 4.940955580809328e-05, - "median_absolute_deviation": 15743.011657201734, - "median_rank": 20239.0, - "standard_deviation": 11999.952940452784, - "variance": 143998870.57308143, - "z_arithmetic_mean_rank": 1.082695967970082, - "z_geometric_mean_rank": 1.6683482799540457, - "z_hits_at_k": -1.1054697733258656, - "z_inverse_harmonic_mean_rank": -0.8191899959740097 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.991159205802291, - "adjusted_arithmetic_mean_rank_index": 0.008841226261950053, - "adjusted_geometric_mean_rank_index": 0.023574021988738925, - "adjusted_hits_at_k": -0.0002444126837850729, - "adjusted_inverse_harmonic_mean_rank": -7.339924634853156e-05, - "arithmetic_mean_rank": 20281.8515625, - "count": 5000.0, - "geometric_mean_rank": 14704.083984375, - "harmonic_mean_rank": 4994.6127824240475, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.9305162974633276e-05, - "inverse_geometric_mean_rank": 6.800831761211157e-05, - "inverse_harmonic_mean_rank": 0.0002002157211303711, - "inverse_median_rank": 4.940955477650277e-05, - "median_absolute_deviation": 15743.011657201734, - "median_rank": 20239.0, - "standard_deviation": 11999.953125, - "variance": 143998864.0, - "z_arithmetic_mean_rank": 1.0827967467483717, - "z_geometric_mean_rank": 1.6684068011708462, - "z_hits_at_k": -1.1054697733258656, - "z_inverse_harmonic_mean_rank": -0.8191863717024064 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.9928898905190562, - "adjusted_arithmetic_mean_rank_index": 0.007110456947646893, - "adjusted_geometric_mean_rank_index": 0.015620408929683083, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": -6.31752292426308e-05, - "arithmetic_mean_rank": 20318.2082, - "count": 5000.0, - "geometric_mean_rank": 14824.539756891101, - "harmonic_mean_rank": 4752.280096951103, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.921693833219014e-05, - "inverse_geometric_mean_rank": 6.745571979967579e-05, - "inverse_harmonic_mean_rank": 0.000210425307347007, - "inverse_median_rank": 4.966969651815427e-05, - "median_absolute_deviation": 15533.964744392444, - "median_rank": 20133.0, - "standard_deviation": 11889.493608831823, - "variance": 141360058.27445275, - "z_arithmetic_mean_rank": 0.8708272728100468, - "z_geometric_mean_rank": 1.105504855526732, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": -0.7050957157046835 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.9928918256522765, - "adjusted_arithmetic_mean_rank_index": 0.007108521719857874, - "adjusted_geometric_mean_rank_index": 0.015617555484825907, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": -6.317569759938329e-05, - "arithmetic_mean_rank": 20318.2478, - "count": 5000.0, - "geometric_mean_rank": 14824.582726243238, - "harmonic_mean_rank": 4752.290671527522, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.921684240901915e-05, - "inverse_geometric_mean_rank": 6.745552427791094e-05, - "inverse_harmonic_mean_rank": 0.00021042483911838908, - "inverse_median_rank": 4.966969651815427e-05, - "median_absolute_deviation": 15533.964744392444, - "median_rank": 20133.0, - "standard_deviation": 11889.49296875166, - "variance": 141360043.05399516, - "z_arithmetic_mean_rank": 0.8705902628470881, - "z_geometric_mean_rank": 1.105302908371641, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": -0.7051009430120248 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.9928908832827135, - "adjusted_arithmetic_mean_rank_index": 0.007109464135473975, - "adjusted_geometric_mean_rank_index": 0.015618509538453562, - "adjusted_hits_at_k": -0.00024440126566051376, - "adjusted_inverse_harmonic_mean_rank": -6.317546970748797e-05, - "arithmetic_mean_rank": 20318.228515625, - "count": 5000.0, - "geometric_mean_rank": 14824.568359375, - "harmonic_mean_rank": 4752.285526169837, - "hits_at_1": 0.0, - "hits_at_10": 0.0, - "hits_at_3": 0.0, - "hits_at_5": 0.0, - "inverse_arithmetic_mean_rank": 4.9216887418879196e-05, - "inverse_geometric_mean_rank": 6.745559221599251e-05, - "inverse_harmonic_mean_rank": 0.000210425066947937, - "inverse_median_rank": 4.966969572706148e-05, - "median_absolute_deviation": 15533.964744392444, - "median_rank": 20133.0, - "standard_deviation": 11889.4931640625, - "variance": 141360048.0, - "z_arithmetic_mean_rank": 0.8707056817613505, - "z_geometric_mean_rank": 1.105370429710078, - "z_hits_at_k": -1.1054439510948424, - "z_inverse_harmonic_mean_rank": -0.7050983995214675 - } - } - }, - "times": { - "evaluation": 9.90921139717102, - "training": 4.695069313049316 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json deleted file mode 100644 index f185e87..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K Dataset with TuckER as described by Balazevic et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json deleted file mode 100644 index ac1aceb..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/fb15k/balazevic2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0002195790323791253, - 1.653514352525725e-05, - 1.4086949304634584e-05, - 1.1515389150776652e-05, - 9.777598791220405e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.07133568713438639, - "adjusted_arithmetic_mean_rank_index": 0.9287900484289984, - "adjusted_geometric_mean_rank_index": 0.9924661457284999, - "adjusted_hits_at_k": 0.35636286355421287, - "adjusted_inverse_harmonic_mean_rank": 0.236786103943646, - "arithmetic_mean_rank": 526.9461918707996, - "count": 118142.0, - "geometric_mean_rank": 41.91341929915113, - "harmonic_mean_rank": 4.213847157632036, - "hits_at_1": 0.173883970137631, - "hits_at_10": 0.3567994447359957, - "hits_at_3": 0.2580200098186928, - "hits_at_5": 0.2985813681840497, - "inverse_arithmetic_mean_rank": 0.001897726969901297, - "inverse_geometric_mean_rank": 0.023858707228409135, - "inverse_harmonic_mean_rank": 0.2373128313846932, - "inverse_median_rank": 0.022222222222222223, - "median_absolute_deviation": 65.23449761424648, - "median_rank": 45.0, - "standard_deviation": 1306.1796683025648, - "variance": 1706105.325886998, - "z_arithmetic_mean_rank": 552.460381843816, - "z_geometric_mean_rank": 341.72025125001227, - "z_hits_at_k": 4701.491763249247, - "z_inverse_harmonic_mean_rank": 7716.354987752866 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.07133572953166528, - "adjusted_arithmetic_mean_rank_index": 0.9287900060259792, - "adjusted_geometric_mean_rank_index": 0.9924661433888894, - "adjusted_hits_at_k": 0.35636286355421287, - "adjusted_inverse_harmonic_mean_rank": 0.23678610315360502, - "arithmetic_mean_rank": 526.946505053241, - "count": 118142.0, - "geometric_mean_rank": 41.913432004661814, - "harmonic_mean_rank": 4.213847171650723, - "hits_at_1": 0.173883970137631, - "hits_at_10": 0.3567994447359957, - "hits_at_3": 0.2580200098186928, - "hits_at_5": 0.2985813681840497, - "inverse_arithmetic_mean_rank": 0.0018977258420168537, - "inverse_geometric_mean_rank": 0.023858699995952973, - "inverse_harmonic_mean_rank": 0.23731283059519745, - "inverse_median_rank": 0.022222222222222223, - "median_absolute_deviation": 65.23449761424648, - "median_rank": 45.0, - "standard_deviation": 1306.1802348103465, - "variance": 1706106.805809212, - "z_arithmetic_mean_rank": 552.4603566217669, - "z_geometric_mean_rank": 341.720250444451, - "z_hits_at_k": 4701.491763249247, - "z_inverse_harmonic_mean_rank": 7716.354962007112 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.07133570029177673, - "adjusted_arithmetic_mean_rank_index": 0.9287900352698266, - "adjusted_geometric_mean_rank_index": 0.9924661467040232, - "adjusted_hits_at_k": 0.35636286355421287, - "adjusted_inverse_harmonic_mean_rank": 0.23678609380201712, - "arithmetic_mean_rank": 526.9462890625, - "count": 118142.0, - "geometric_mean_rank": 41.913414001464844, - "harmonic_mean_rank": 4.2138473375876755, - "hits_at_1": 0.173883970137631, - "hits_at_10": 0.3567994447359957, - "hits_at_3": 0.2580200098186928, - "hits_at_5": 0.2985813681840497, - "inverse_arithmetic_mean_rank": 0.0018977266736328602, - "inverse_geometric_mean_rank": 0.02385871112346649, - "inverse_harmonic_mean_rank": 0.2373128212500635, - "inverse_median_rank": 0.02222222276031971, - "median_absolute_deviation": 65.23449761424648, - "median_rank": 45.0, - "standard_deviation": 1306.179931640625, - "variance": 1706105.875, - "z_arithmetic_mean_rank": 552.4603740165131, - "z_geometric_mean_rank": 341.7202515858989, - "z_hits_at_k": 4701.491763249247, - "z_inverse_harmonic_mean_rank": 7716.354657258773 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.07689682971067989, - "adjusted_arithmetic_mean_rank_index": 0.9232284940236298, - "adjusted_geometric_mean_rank_index": 0.9906866025503254, - "adjusted_hits_at_k": 0.32003303624630025, - "adjusted_inverse_harmonic_mean_rank": 0.20532087082516273, - "arithmetic_mean_rank": 566.4796431413046, - "count": 59071.0, - "geometric_mean_rank": 51.43193130167176, - "harmonic_mean_rank": 4.857414947867291, - "hits_at_1": 0.1445717864942188, - "hits_at_10": 0.3204956746965516, - "hits_at_3": 0.22456027492339728, - "hits_at_5": 0.263428755226761, - "inverse_arithmetic_mean_rank": 0.0017652885008447809, - "inverse_geometric_mean_rank": 0.019443174205038955, - "inverse_harmonic_mean_rank": 0.20587082032986756, - "inverse_median_rank": 0.01694915254237288, - "median_absolute_deviation": 85.99092867332492, - "median_rank": 59.0, - "standard_deviation": 1355.3119844999508, - "variance": 1836870.575329195, - "z_arithmetic_mean_rank": 388.25303731980824, - "z_geometric_mean_rank": 241.1827499912326, - "z_hits_at_k": 2980.9704002169224, - "z_inverse_harmonic_mean_rank": 4724.010486752963 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.07689688026665428, - "adjusted_arithmetic_mean_rank_index": 0.9232284434607918, - "adjusted_geometric_mean_rank_index": 0.9906865989916975, - "adjusted_hits_at_k": 0.32003303624630025, - "adjusted_inverse_harmonic_mean_rank": 0.20532086964081991, - "arithmetic_mean_rank": 566.4800155744782, - "count": 59071.0, - "geometric_mean_rank": 51.43195057159756, - "harmonic_mean_rank": 4.857414975791905, - "hits_at_1": 0.1445717864942188, - "hits_at_10": 0.3204956746965516, - "hits_at_3": 0.22456027492339728, - "hits_at_5": 0.263428755226761, - "inverse_arithmetic_mean_rank": 0.0017652873402530906, - "inverse_geometric_mean_rank": 0.0194431669202963, - "inverse_harmonic_mean_rank": 0.20587081914634436, - "inverse_median_rank": 0.01694915254237288, - "median_absolute_deviation": 85.99092867332492, - "median_rank": 59.0, - "standard_deviation": 1355.3126588901207, - "variance": 1836872.4033478086, - "z_arithmetic_mean_rank": 388.253016056193, - "z_geometric_mean_rank": 241.1827491248843, - "z_hits_at_k": 2980.9704002169224, - "z_inverse_harmonic_mean_rank": 4724.010459503673 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.0768968589307786, - "adjusted_arithmetic_mean_rank_index": 0.9232284647995641, - "adjusted_geometric_mean_rank_index": 0.9906865998727367, - "adjusted_hits_at_k": 0.32003303624630025, - "adjusted_inverse_harmonic_mean_rank": 0.20532086895293028, - "arithmetic_mean_rank": 566.4798583984375, - "count": 59071.0, - "geometric_mean_rank": 51.43194580078125, - "harmonic_mean_rank": 4.857414992011072, - "hits_at_1": 0.1445717864942188, - "hits_at_10": 0.3204956746965516, - "hits_at_3": 0.22456027492339728, - "hits_at_5": 0.263428755226761, - "inverse_arithmetic_mean_rank": 0.0017652878304943442, - "inverse_geometric_mean_rank": 0.019443169236183167, - "inverse_harmonic_mean_rank": 0.20587081845893077, - "inverse_median_rank": 0.016949152573943138, - "median_absolute_deviation": 85.99092867332492, - "median_rank": 59.0, - "standard_deviation": 1355.312255859375, - "variance": 1836871.25, - "z_arithmetic_mean_rank": 388.2530250299663, - "z_geometric_mean_rank": 241.1827493393734, - "z_hits_at_k": 2980.9704002169224, - "z_inverse_harmonic_mean_rank": 4724.010443676749 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.06580473183222485, - "adjusted_arithmetic_mean_rank_index": 0.9343214092483405, - "adjusted_geometric_mean_rank_index": 0.9939121303205123, - "adjusted_hits_at_k": 0.39269253959611367, - "adjusted_inverse_harmonic_mean_rank": 0.268251217678668, - "arithmetic_mean_rank": 487.41274060029457, - "count": 59071.0, - "geometric_mean_rank": 34.156499141407, - "harmonic_mean_rank": 3.7208631886327486, - "hits_at_1": 0.20319615378104317, - "hits_at_10": 0.39310321477543975, - "hits_at_3": 0.29147974471398824, - "hits_at_5": 0.3337339811413384, - "inverse_arithmetic_mean_rank": 0.0020516492834561652, - "inverse_geometric_mean_rank": 0.02927700511284914, - "inverse_harmonic_mean_rank": 0.26875484243951886, - "inverse_median_rank": 0.030303030303030304, - "median_absolute_deviation": 47.443270992179265, - "median_rank": 33.0, - "standard_deviation": 1253.8796947498934, - "variance": 1572214.288906086, - "z_arithmetic_mean_rank": 393.0342086632433, - "z_geometric_mean_rank": 241.96603917694628, - "z_hits_at_k": 3669.005153181792, - "z_inverse_harmonic_mean_rank": 6190.816796846298 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.0658047661150957, - "adjusted_arithmetic_mean_rank_index": 0.9343213749608406, - "adjusted_geometric_mean_rank_index": 0.9939121288680024, - "adjusted_hits_at_k": 0.39269253959611367, - "adjusted_inverse_harmonic_mean_rank": 0.2682512172829272, - "arithmetic_mean_rank": 487.41299453200384, - "count": 59071.0, - "geometric_mean_rank": 34.15650705224349, - "harmonic_mean_rank": 3.7208631941079378, - "hits_at_1": 0.20319615378104317, - "hits_at_10": 0.39310321477543975, - "hits_at_3": 0.29147974471398824, - "hits_at_5": 0.3337339811413384, - "inverse_arithmetic_mean_rank": 0.002051648214590921, - "inverse_geometric_mean_rank": 0.029276998332132365, - "inverse_harmonic_mean_rank": 0.2687548420440505, - "inverse_median_rank": 0.030303030303030304, - "median_absolute_deviation": 47.443270992179265, - "median_rank": 33.0, - "standard_deviation": 1253.880142343099, - "variance": 1572215.4113623502, - "z_arithmetic_mean_rank": 393.0341942397695, - "z_geometric_mean_rank": 241.96603882333548, - "z_hits_at_k": 3669.005153181792, - "z_inverse_harmonic_mean_rank": 6190.816787713223 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.06580474137447191, - "adjusted_arithmetic_mean_rank_index": 0.934321399704805, - "adjusted_geometric_mean_rank_index": 0.9939121298378724, - "adjusted_hits_at_k": 0.39269253959611367, - "adjusted_inverse_harmonic_mean_rank": 0.2682512158110723, - "arithmetic_mean_rank": 487.4128112792969, - "count": 59071.0, - "geometric_mean_rank": 34.15650177001953, - "harmonic_mean_rank": 3.7208632144714846, - "hits_at_1": 0.20319615378104317, - "hits_at_10": 0.39310321477543975, - "hits_at_3": 0.29147974471398824, - "hits_at_5": 0.3337339811413384, - "inverse_arithmetic_mean_rank": 0.0020516489166766405, - "inverse_geometric_mean_rank": 0.029277002438902855, - "inverse_harmonic_mean_rank": 0.26875484057320853, - "inverse_median_rank": 0.03030303120613098, - "median_absolute_deviation": 47.443270992179265, - "median_rank": 33.0, - "standard_deviation": 1253.8798828125, - "variance": 1572214.75, - "z_arithmetic_mean_rank": 393.0342046486335, - "z_geometric_mean_rank": 241.9660390594485, - "z_hits_at_k": 3669.005153181792, - "z_inverse_harmonic_mean_rank": 6190.816753745123 - } - } - }, - "times": { - "evaluation": 22.204118728637695, - "training": 399.59123039245605 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json deleted file mode 100644 index 9849e52..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn FB15K-237 Dataset with TuckER as described by Balazevic et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json deleted file mode 100644 index e5662dd..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/fb15k237/balazevic2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 2048, - "slice_size": null - }, - "losses": [ - 0.0027874300545268718, - 7.365971419707463e-05, - 2.8435559064788497e-05, - 2.112122028404341e-05, - 1.900511581936777e-05 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.5633845099742385, - "adjusted_arithmetic_mean_rank_index": 0.43667668634658385, - "adjusted_geometric_mean_rank_index": 0.765892175485255, - "adjusted_hits_at_k": 0.06116118490603417, - "adjusted_inverse_harmonic_mean_rank": 0.02850689581330692, - "arithmetic_mean_rank": 4020.1253547313827, - "count": 40876.0, - "geometric_mean_rank": 1228.5381219971325, - "harmonic_mean_rank": 34.24748400705478, - "hits_at_1": 0.012501223211664546, - "hits_at_10": 0.06182111752617673, - "hits_at_3": 0.029381544182405323, - "hits_at_5": 0.04305705059203445, - "inverse_arithmetic_mean_rank": 0.0002487484622396354, - "inverse_geometric_mean_rank": 0.0008139755552512957, - "inverse_harmonic_mean_rank": 0.029199225256781078, - "inverse_median_rank": 0.0003372112628561794, - "median_absolute_deviation": 4152.0275129249385, - "median_rank": 2965.5, - "standard_deviation": 3853.139281834975, - "variance": 14846682.325219747, - "z_arithmetic_mean_rank": 152.72046645669522, - "z_geometric_mean_rank": 155.11300167144324, - "z_hits_at_k": 466.233760257905, - "z_inverse_harmonic_mean_rank": 536.8020979000396 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.5633849385295965, - "adjusted_arithmetic_mean_rank_index": 0.43667625773115926, - "adjusted_geometric_mean_rank_index": 0.7658919832564306, - "adjusted_hits_at_k": 0.06116118490603417, - "adjusted_inverse_harmonic_mean_rank": 0.028506895380300473, - "arithmetic_mean_rank": 4020.128412760544, - "count": 40876.0, - "geometric_mean_rank": 1228.539129943837, - "harmonic_mean_rank": 34.24748451456186, - "hits_at_1": 0.012501223211664546, - "hits_at_10": 0.06182111752617673, - "hits_at_3": 0.029381544182405323, - "hits_at_5": 0.04305705059203445, - "inverse_arithmetic_mean_rank": 0.00024874827302178625, - "inverse_geometric_mean_rank": 0.0008139748874305007, - "inverse_harmonic_mean_rank": 0.02919922482408321, - "inverse_median_rank": 0.0003372112628561794, - "median_absolute_deviation": 4152.0275129249385, - "median_rank": 2965.5, - "standard_deviation": 3853.140645811768, - "variance": 14846692.836406728, - "z_arithmetic_mean_rank": 152.72031655552206, - "z_geometric_mean_rank": 155.11296274012767, - "z_hits_at_k": 466.233760257905, - "z_inverse_harmonic_mean_rank": 536.8020897462662 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.5633846997606484, - "adjusted_arithmetic_mean_rank_index": 0.43667649653357343, - "adjusted_geometric_mean_rank_index": 0.7658920426797261, - "adjusted_hits_at_k": 0.06116118490603417, - "adjusted_inverse_harmonic_mean_rank": 0.028506894681160907, - "arithmetic_mean_rank": 4020.126708984375, - "count": 40876.0, - "geometric_mean_rank": 1228.538818359375, - "harmonic_mean_rank": 34.24748533399144, - "hits_at_1": 0.012501223211664546, - "hits_at_10": 0.06182111752617673, - "hits_at_3": 0.029381544182405323, - "hits_at_5": 0.04305705059203445, - "inverse_arithmetic_mean_rank": 0.00024874837254174054, - "inverse_geometric_mean_rank": 0.0008139751153066754, - "inverse_harmonic_mean_rank": 0.029199224125441885, - "inverse_median_rank": 0.00033721126965247095, - "median_absolute_deviation": 4152.0275129249385, - "median_rank": 2965.5, - "standard_deviation": 3853.139892578125, - "variance": 14846687.0, - "z_arithmetic_mean_rank": 152.72040007272648, - "z_geometric_mean_rank": 155.11297477488404, - "z_hits_at_k": 466.233760257905, - "z_inverse_harmonic_mean_rank": 536.8020765810452 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6717353285723936, - "adjusted_arithmetic_mean_rank_index": 0.32831132883914504, - "adjusted_geometric_mean_rank_index": 0.5146882360640462, - "adjusted_hits_at_k": 0.00848995966031373, - "adjusted_inverse_harmonic_mean_rank": 0.006057639803220773, - "arithmetic_mean_rank": 4726.758538017419, - "count": 20438.0, - "geometric_mean_rank": 2507.4076710492445, - "harmonic_mean_rank": 147.56851324594282, - "hits_at_1": 0.003571778060475585, - "hits_at_10": 0.009198551717389177, - "hits_at_3": 0.005431059790586163, - "hits_at_5": 0.00694784225462374, - "inverse_arithmetic_mean_rank": 0.00021156147325000397, - "inverse_geometric_mean_rank": 0.0003988182741666185, - "inverse_harmonic_mean_rank": 0.0067765133496558655, - "inverse_median_rank": 0.0002486634340420241, - "median_absolute_deviation": 4289.909519245959, - "median_rank": 4021.5, - "standard_deviation": 3749.7685571621982, - "variance": 14060764.232282275, - "z_arithmetic_mean_rank": 81.1047863772954, - "z_geometric_mean_rank": 73.71080657610705, - "z_hits_at_k": 45.38591523254731, - "z_inverse_harmonic_mean_rank": 79.9959453307532 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.671735815309457, - "adjusted_arithmetic_mean_rank_index": 0.32831084203290006, - "adjusted_geometric_mean_rank_index": 0.514687799316216, - "adjusted_hits_at_k": 0.00848995966031373, - "adjusted_inverse_harmonic_mean_rank": 0.006057639359175807, - "arithmetic_mean_rank": 4726.761963010079, - "count": 20438.0, - "geometric_mean_rank": 2507.4099266469725, - "harmonic_mean_rank": 147.5685229086799, - "hits_at_1": 0.003571778060475585, - "hits_at_10": 0.009198551717389177, - "hits_at_3": 0.005431059790586163, - "hits_at_5": 0.00694784225462374, - "inverse_arithmetic_mean_rank": 0.00021156131995341346, - "inverse_geometric_mean_rank": 0.00039881791540055336, - "inverse_harmonic_mean_rank": 0.006776512905932058, - "inverse_median_rank": 0.0002486634340420241, - "median_absolute_deviation": 4289.909519245959, - "median_rank": 4021.5, - "standard_deviation": 3749.7695546569394, - "variance": 14060771.7130321, - "z_arithmetic_mean_rank": 81.10466611852564, - "z_geometric_mean_rank": 73.71074402749493, - "z_hits_at_k": 45.38591523254731, - "z_inverse_harmonic_mean_rank": 79.99593946678672 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6717355030317774, - "adjusted_arithmetic_mean_rank_index": 0.3283111543549647, - "adjusted_geometric_mean_rank_index": 0.5146876603134156, - "adjusted_hits_at_k": 0.00848995966031373, - "adjusted_inverse_harmonic_mean_rank": 0.006057639189669265, - "arithmetic_mean_rank": 4726.759765625, - "count": 20438.0, - "geometric_mean_rank": 2507.41064453125, - "harmonic_mean_rank": 147.56852659726425, - "hits_at_1": 0.003571778060475585, - "hits_at_10": 0.009198551717389177, - "hits_at_3": 0.005431059790586163, - "hits_at_5": 0.00694784225462374, - "inverse_arithmetic_mean_rank": 0.00021156141883693635, - "inverse_geometric_mean_rank": 0.00039881779230199754, - "inverse_harmonic_mean_rank": 0.006776512736548112, - "inverse_median_rank": 0.0002486634475644678, - "median_absolute_deviation": 4289.909519245959, - "median_rank": 4021.5, - "standard_deviation": 3749.76904296875, - "variance": 14060768.0, - "z_arithmetic_mean_rank": 81.1047432733848, - "z_geometric_mean_rank": 73.71072412028157, - "z_hits_at_k": 45.38591523254731, - "z_inverse_harmonic_mean_rank": 79.99593722831818 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.4579999658534782, - "adjusted_arithmetic_mean_rank_index": 0.5420749612327768, - "adjusted_geometric_mean_rank_index": 0.8871211856726299, - "adjusted_hits_at_k": 0.11383117308368258, - "adjusted_inverse_harmonic_mean_rank": 0.05095567511413758, - "arithmetic_mean_rank": 3313.4921714453467, - "count": 20438.0, - "geometric_mean_rank": 601.9387810872638, - "harmonic_mean_rank": 19.37160933780674, - "hits_at_1": 0.021430668362853508, - "hits_at_10": 0.11444368333496428, - "hits_at_3": 0.053332028574224484, - "hits_at_5": 0.07916625892944515, - "inverse_arithmetic_mean_rank": 0.0003017963973531284, - "inverse_geometric_mean_rank": 0.00166129850978156, - "inverse_harmonic_mean_rank": 0.0516219371639063, - "inverse_median_rank": 0.0006253908692933083, - "median_absolute_deviation": 2357.337527423907, - "median_rank": 1599.0, - "standard_deviation": 3825.433244315643, - "variance": 14633939.506715307, - "z_arithmetic_mean_rank": 134.21490718526735, - "z_geometric_mean_rank": 127.04298051729151, - "z_hits_at_k": 618.7736721675088, - "z_inverse_harmonic_mean_rank": 684.2055683452835 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.45800033781994665, - "adjusted_arithmetic_mean_rank_index": 0.5420745892148869, - "adjusted_geometric_mean_rank_index": 0.8871211018546956, - "adjusted_hits_at_k": 0.11383117308368258, - "adjusted_inverse_harmonic_mean_rank": 0.05095567469216939, - "arithmetic_mean_rank": 3313.4948625110087, - "count": 20438.0, - "geometric_mean_rank": 601.9392273131349, - "harmonic_mean_rank": 19.37160949604304, - "hits_at_1": 0.021430668362853508, - "hits_at_10": 0.11444368333496428, - "hits_at_3": 0.053332028574224484, - "hits_at_5": 0.07916625892944515, - "inverse_arithmetic_mean_rank": 0.00030179615224819973, - "inverse_geometric_mean_rank": 0.001661297278238007, - "inverse_harmonic_mean_rank": 0.05162193674223435, - "inverse_median_rank": 0.0006253908692933083, - "median_absolute_deviation": 2357.337527423907, - "median_rank": 1599.0, - "standard_deviation": 3825.4348786908963, - "variance": 14633952.011104833, - "z_arithmetic_mean_rank": 134.21481507559596, - "z_geometric_mean_rank": 127.04296851388051, - "z_hits_at_k": 618.7736721675088, - "z_inverse_harmonic_mean_rank": 684.2055626793203 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.4580001368015266, - "adjusted_arithmetic_mean_rank_index": 0.5420747902610961, - "adjusted_geometric_mean_rank_index": 0.8871211511562782, - "adjusted_hits_at_k": 0.11383117308368258, - "adjusted_inverse_harmonic_mean_rank": 0.050955671222067116, - "arithmetic_mean_rank": 3313.493408203125, - "count": 20438.0, - "geometric_mean_rank": 601.93896484375, - "harmonic_mean_rank": 19.37161079731694, - "hits_at_1": 0.021430668362853508, - "hits_at_10": 0.11444368333496428, - "hits_at_3": 0.053332028574224484, - "hits_at_5": 0.07916625892944515, - "inverse_arithmetic_mean_rank": 0.00030179627356119454, - "inverse_geometric_mean_rank": 0.0016612979816272855, - "inverse_harmonic_mean_rank": 0.0516219332745682, - "inverse_median_rank": 0.0006253908504731953, - "median_absolute_deviation": 2357.337527423907, - "median_rank": 1599.0, - "standard_deviation": 3825.43408203125, - "variance": 14633945.0, - "z_arithmetic_mean_rank": 134.21486485357917, - "z_geometric_mean_rank": 127.04297557426877, - "z_hits_at_k": 618.7736721675088, - "z_inverse_harmonic_mean_rank": 684.2055160846418 - } - } - }, - "times": { - "evaluation": 9.888858556747437, - "training": 214.02531933784485 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json deleted file mode 100644 index 1769644..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18 Dataset with TuckER as described by Balazevic et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json deleted file mode 100644 index deec3f2..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/wn18/balazevic2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.00017830922128783815, - 3.5568977629189845e-06, - 3.2398627037952276e-06, - 2.946085544536108e-06, - 2.427437591210355e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.13629356123736652, - "adjusted_arithmetic_mean_rank_index": 0.8637486485468131, - "adjusted_geometric_mean_rank_index": 0.9934277872821621, - "adjusted_hits_at_k": 0.3122319457641806, - "adjusted_inverse_harmonic_mean_rank": 0.21687724348833318, - "arithmetic_mean_rank": 2789.0069, - "count": 10000.0, - "geometric_mean_rank": 99.96215669911965, - "harmonic_mean_rank": 4.606352677882064, - "hits_at_1": 0.1676, - "hits_at_10": 0.3124, - "hits_at_3": 0.234, - "hits_at_5": 0.2644, - "inverse_arithmetic_mean_rank": 0.00035855056507748336, - "inverse_geometric_mean_rank": 0.010003785762745621, - "inverse_harmonic_mean_rank": 0.21709149731448396, - "inverse_median_rank": 0.009433962264150943, - "median_absolute_deviation": 155.6732329430882, - "median_rank": 106.0, - "standard_deviation": 6483.423392163463, - "variance": 42034778.88205239, - "z_arithmetic_mean_rank": 149.6018115743092, - "z_geometric_mean_rank": 99.42312958133098, - "z_hits_at_k": 1997.195311935326, - "z_inverse_harmonic_mean_rank": 3423.143978774425 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.1362936785208867, - "adjusted_arithmetic_mean_rank_index": 0.8637485312575612, - "adjusted_geometric_mean_rank_index": 0.9934277849391755, - "adjusted_hits_at_k": 0.3122319457641806, - "adjusted_inverse_harmonic_mean_rank": 0.21687724338746267, - "arithmetic_mean_rank": 2789.0093, - "count": 10000.0, - "geometric_mean_rank": 99.96219197901941, - "harmonic_mean_rank": 4.606352680021797, - "hits_at_1": 0.1676, - "hits_at_10": 0.3124, - "hits_at_3": 0.234, - "hits_at_5": 0.2644, - "inverse_arithmetic_mean_rank": 0.0003585502565373303, - "inverse_geometric_mean_rank": 0.01000378223208516, - "inverse_harmonic_mean_rank": 0.21709149721364104, - "inverse_median_rank": 0.009433962264150943, - "median_absolute_deviation": 155.6732329430882, - "median_rank": 106.0, - "standard_deviation": 6483.426716005472, - "variance": 42034821.981813505, - "z_arithmetic_mean_rank": 149.60179125973667, - "z_geometric_mean_rank": 99.42312934684281, - "z_hits_at_k": 1997.195311935326, - "z_inverse_harmonic_mean_rank": 3423.143977182306 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.1362936177602349, - "adjusted_arithmetic_mean_rank_index": 0.8637485920211825, - "adjusted_geometric_mean_rank_index": 0.993427787182279, - "adjusted_hits_at_k": 0.3122319457641806, - "adjusted_inverse_harmonic_mean_rank": 0.21687725008190317, - "arithmetic_mean_rank": 2789.008056640625, - "count": 10000.0, - "geometric_mean_rank": 99.962158203125, - "harmonic_mean_rank": 4.606352538014779, - "hits_at_1": 0.1676, - "hits_at_10": 0.3124, - "hits_at_3": 0.234, - "hits_at_5": 0.2644, - "inverse_arithmetic_mean_rank": 0.0003585504018701613, - "inverse_geometric_mean_rank": 0.010003785602748394, - "inverse_harmonic_mean_rank": 0.21709150390625, - "inverse_median_rank": 0.009433962404727936, - "median_absolute_deviation": 155.6732329430882, - "median_rank": 106.0, - "standard_deviation": 6483.42529296875, - "variance": 42034804.0, - "z_arithmetic_mean_rank": 149.60180178403425, - "z_geometric_mean_rank": 99.4231295713346, - "z_hits_at_k": 1997.195311935326, - "z_inverse_harmonic_mean_rank": 3423.1440828459217 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.1465853386942658, - "adjusted_arithmetic_mean_rank_index": 0.8534563690923419, - "adjusted_geometric_mean_rank_index": 0.9921161069642516, - "adjusted_hits_at_k": 0.2958279334706153, - "adjusted_inverse_harmonic_mean_rank": 0.20868076827049847, - "arithmetic_mean_rank": 2999.5404, - "count": 5000.0, - "geometric_mean_rank": 119.71615876559561, - "harmonic_mean_rank": 4.7870419943520535, - "hits_at_1": 0.1632, - "hits_at_10": 0.296, - "hits_at_3": 0.2246, - "hits_at_5": 0.2522, - "inverse_arithmetic_mean_rank": 0.00033338440782461207, - "inverse_geometric_mean_rank": 0.008353091264463316, - "inverse_harmonic_mean_rank": 0.2088972691653511, - "inverse_median_rank": 0.006779661016949152, - "median_absolute_deviation": 217.20122501107068, - "median_rank": 147.5, - "standard_deviation": 6644.4210276869, - "variance": 44148330.79316784, - "z_arithmetic_mean_rank": 104.52393735493405, - "z_geometric_mean_rank": 70.21514025909521, - "z_hits_at_k": 1338.019097424571, - "z_inverse_harmonic_mean_rank": 2329.021753055102 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.1465854853020648, - "adjusted_arithmetic_mean_rank_index": 0.8534562224773778, - "adjusted_geometric_mean_rank_index": 0.9921161046258482, - "adjusted_hits_at_k": 0.2958279334706153, - "adjusted_inverse_harmonic_mean_rank": 0.2086807682341606, - "arithmetic_mean_rank": 2999.5434, - "count": 5000.0, - "geometric_mean_rank": 119.71619397742072, - "harmonic_mean_rank": 4.787041995184536, - "hits_at_1": 0.1632, - "hits_at_10": 0.296, - "hits_at_3": 0.2246, - "hits_at_5": 0.2522, - "inverse_arithmetic_mean_rank": 0.0003333840743894554, - "inverse_geometric_mean_rank": 0.008353088807589447, - "inverse_harmonic_mean_rank": 0.20889726912902315, - "inverse_median_rank": 0.006779661016949152, - "median_absolute_deviation": 217.20122501107068, - "median_rank": 147.5, - "standard_deviation": 6644.425975185249, - "variance": 44148396.539716445, - "z_arithmetic_mean_rank": 104.52391939880428, - "z_geometric_mean_rank": 70.21514009359913, - "z_hits_at_k": 1338.019097424571, - "z_inverse_harmonic_mean_rank": 2329.0217526495467 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.14658541650330079, - "adjusted_arithmetic_mean_rank_index": 0.8534562912795043, - "adjusted_geometric_mean_rank_index": 0.9921161051208643, - "adjusted_hits_at_k": 0.2958279334706153, - "adjusted_inverse_harmonic_mean_rank": 0.20868076472917851, - "arithmetic_mean_rank": 2999.5419921875, - "count": 5000.0, - "geometric_mean_rank": 119.7161865234375, - "harmonic_mean_rank": 4.78704207548193, - "hits_at_1": 0.1632, - "hits_at_10": 0.296, - "hits_at_3": 0.2246, - "hits_at_5": 0.2522, - "inverse_arithmetic_mean_rank": 0.0003333842323627323, - "inverse_geometric_mean_rank": 0.008353088982403278, - "inverse_harmonic_mean_rank": 0.20889726562500002, - "inverse_median_rank": 0.006779660936444998, - "median_absolute_deviation": 217.20122501107068, - "median_rank": 147.5, - "standard_deviation": 6644.42333984375, - "variance": 44148360.0, - "z_arithmetic_mean_rank": 104.52392782509227, - "z_geometric_mean_rank": 70.21514012863295, - "z_hits_at_k": 1338.019097424571, - "z_inverse_harmonic_mean_rank": 2329.0217135315215 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.1260022609588329, - "adjusted_arithmetic_mean_rank_index": 0.8740404507763229, - "adjusted_geometric_mean_rank_index": 0.9945236073848335, - "adjusted_hits_at_k": 0.3286359578704887, - "adjusted_inverse_harmonic_mean_rank": 0.22507371861079298, - "arithmetic_mean_rank": 2578.4734, - "count": 5000.0, - "geometric_mean_rank": 83.46770289802355, - "harmonic_mean_rank": 4.438807642792696, - "hits_at_1": 0.172, - "hits_at_10": 0.3288, - "hits_at_3": 0.2434, - "hits_at_5": 0.2766, - "inverse_arithmetic_mean_rank": 0.00038782637819726975, - "inverse_geometric_mean_rank": 0.01198068193180957, - "inverse_harmonic_mean_rank": 0.22528572546361694, - "inverse_median_rank": 0.012578616352201259, - "median_absolute_deviation": 116.38427415268976, - "median_rank": 79.5, - "standard_deviation": 6311.305590897373, - "variance": 39832578.26169244, - "z_arithmetic_mean_rank": 107.04491535204133, - "z_geometric_mean_rank": 70.38552459472623, - "z_hits_at_k": 1486.443332273157, - "z_inverse_harmonic_mean_rank": 2512.037021673384 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.12600234891943382, - "adjusted_arithmetic_mean_rank_index": 0.8740403628114234, - "adjusted_geometric_mean_rank_index": 0.994523605102655, - "adjusted_hits_at_k": 0.3286359578704887, - "adjusted_inverse_harmonic_mean_rank": 0.22507371844538973, - "arithmetic_mean_rank": 2578.4752, - "count": 5000.0, - "geometric_mean_rank": 83.4677372648095, - "harmonic_mean_rank": 4.438807646050747, - "hits_at_1": 0.172, - "hits_at_10": 0.3288, - "hits_at_3": 0.2434, - "hits_at_5": 0.2766, - "inverse_arithmetic_mean_rank": 0.00038782610746071944, - "inverse_geometric_mean_rank": 0.011980676998915196, - "inverse_harmonic_mean_rank": 0.22528572529825894, - "inverse_median_rank": 0.012578616352201259, - "median_absolute_deviation": 116.38427415268976, - "median_rank": 79.5, - "standard_deviation": 6311.3071712114415, - "variance": 39832598.20938496, - "z_arithmetic_mean_rank": 107.04490457886119, - "z_geometric_mean_rank": 70.38552443320938, - "z_hits_at_k": 1486.443332273157, - "z_inverse_harmonic_mean_rank": 2512.037019827326 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.1260023081269416, - "adjusted_arithmetic_mean_rank_index": 0.8740404036059091, - "adjusted_geometric_mean_rank_index": 0.9945236067536847, - "adjusted_hits_at_k": 0.3286359578704887, - "adjusted_inverse_harmonic_mean_rank": 0.22507373533925265, - "arithmetic_mean_rank": 2578.474365234375, - "count": 5000.0, - "geometric_mean_rank": 83.46771240234375, - "harmonic_mean_rank": 4.43880731328183, - "hits_at_1": 0.172, - "hits_at_10": 0.3288, - "hits_at_3": 0.2434, - "hits_at_5": 0.2766, - "inverse_arithmetic_mean_rank": 0.0003878262359648943, - "inverse_geometric_mean_rank": 0.011980680748820305, - "inverse_harmonic_mean_rank": 0.2252857421875, - "inverse_median_rank": 0.012578615918755531, - "median_absolute_deviation": 116.38427415268976, - "median_rank": 79.5, - "standard_deviation": 6311.30615234375, - "variance": 39832588.0, - "z_arithmetic_mean_rank": 107.044909575017, - "z_geometric_mean_rank": 70.38552455005788, - "z_hits_at_k": 1486.443332273157, - "z_inverse_harmonic_mean_rank": 2512.0372083789234 - } - } - }, - "times": { - "evaluation": 6.108567476272583, - "training": 554.8396935462952 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json deleted file mode 100644 index d66b7f2..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", - "system": { - "gpu": { - "compute_capability": [ - 7, - 5 - ], - "cuda": "11.3", - "cudnn": 8200, - "name": "Quadro RTX 8000", - "total_memory": 50962169856 - }, - "machine": "x86_64", - "release": "5.4.0-81-generic", - "system": "Linux", - "torch_version": "1.10.0+cu113" - }, - "title": "Learn WN18RR Dataset with TuckER as described by Balazevic et al., 2019" -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json b/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json deleted file mode 100644 index ac24ad2..0000000 --- a/nonasv/results/27c9fe15/cuda/tucker/wn18rr/balazevic2019/results.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "evaluator_metadata": { - "batch_size": 1024, - "slice_size": null - }, - "losses": [ - 0.0001694744666249087, - 3.830565566938213e-06, - 3.4759809618280278e-06, - 3.3146416847170494e-06, - 3.0465402742851043e-06 - ], - "metrics": { - "both": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.5378326165937043, - "adjusted_arithmetic_mean_rank_index": 0.4621901822268002, - "adjusted_geometric_mean_rank_index": 0.8390557160671878, - "adjusted_hits_at_k": 0.05773613104335256, - "adjusted_inverse_harmonic_mean_rank": 0.030193480446228313, - "arithmetic_mean_rank": 10903.237517099864, - "count": 5848.0, - "geometric_mean_rank": 2401.9485509616025, - "harmonic_mean_rank": 32.828776225964894, - "hits_at_1": 0.01573187414500684, - "hits_at_10": 0.05796853625170999, - "hits_at_3": 0.03317373461012312, - "hits_at_5": 0.04155266757865937, - "inverse_arithmetic_mean_rank": 9.17158778235979e-05, - "inverse_geometric_mean_rank": 0.0004163286510028108, - "inverse_harmonic_mean_rank": 0.03046108064208258, - "inverse_median_rank": 0.00018400956849756188, - "median_absolute_deviation": 7957.1261067195655, - "median_rank": 5434.5, - "standard_deviation": 12133.028938908961, - "variance": 147210391.23240232, - "z_arithmetic_mean_rank": 61.21725449573289, - "z_geometric_mean_rank": 64.22009388267874, - "z_hits_at_k": 281.10018794574233, - "z_inverse_harmonic_mean_rank": 362.7415076350221 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.5378330383429395, - "adjusted_arithmetic_mean_rank_index": 0.4621897604567601, - "adjusted_geometric_mean_rank_index": 0.839055594586586, - "adjusted_hits_at_k": 0.05773613104335256, - "adjusted_inverse_harmonic_mean_rank": 0.03019348032372203, - "arithmetic_mean_rank": 10903.246067031463, - "count": 5848.0, - "geometric_mean_rank": 2401.950363195425, - "harmonic_mean_rank": 32.82877635795698, - "hits_at_1": 0.01573187414500684, - "hits_at_10": 0.05796853625170999, - "hits_at_3": 0.03317373461012312, - "hits_at_5": 0.04155266757865937, - "inverse_arithmetic_mean_rank": 9.171580590332048e-05, - "inverse_geometric_mean_rank": 0.0004163283368893827, - "inverse_harmonic_mean_rank": 0.0304610805196101, - "inverse_median_rank": 0.00018400956849756188, - "median_absolute_deviation": 7957.1261067195655, - "median_rank": 5434.5, - "standard_deviation": 12133.034735989933, - "variance": 147210531.9047383, - "z_arithmetic_mean_rank": 61.21719863214062, - "z_geometric_mean_rank": 64.22008458473165, - "z_hits_at_k": 281.10018794574233, - "z_inverse_harmonic_mean_rank": 362.7415061632436 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.5378328469742241, - "adjusted_arithmetic_mean_rank_index": 0.46218995183491574, - "adjusted_geometric_mean_rank_index": 0.8390555403778629, - "adjusted_hits_at_k": 0.05773613104335256, - "adjusted_inverse_harmonic_mean_rank": 0.030193482837332662, - "arithmetic_mean_rank": 10903.2421875, - "count": 5848.0, - "geometric_mean_rank": 2401.951171875, - "harmonic_mean_rank": 32.82877364971474, - "hits_at_1": 0.01573187414500684, - "hits_at_10": 0.05796853625170999, - "hits_at_3": 0.03317373461012312, - "hits_at_5": 0.04155266757865937, - "inverse_arithmetic_mean_rank": 9.171583951683715e-05, - "inverse_geometric_mean_rank": 0.00041632819920778275, - "inverse_harmonic_mean_rank": 0.030461083032527148, - "inverse_median_rank": 0.0001840095646912232, - "median_absolute_deviation": 7957.1261067195655, - "median_rank": 5434.5, - "standard_deviation": 12133.0322265625, - "variance": 147210464.0, - "z_arithmetic_mean_rank": 61.21722398024562, - "z_geometric_mean_rank": 64.2200804356755, - "z_hits_at_k": 281.10018794574233, - "z_inverse_harmonic_mean_rank": 362.7415363615148 - } - }, - "head": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.6233709073515975, - "adjusted_arithmetic_mean_rank_index": 0.37664767596883575, - "adjusted_geometric_mean_rank_index": 0.7944891931289548, - "adjusted_hits_at_k": 0.047644670345667155, - "adjusted_inverse_harmonic_mean_rank": 0.027085656497426752, - "arithmetic_mean_rank": 12634.512995896033, - "count": 2924.0, - "geometric_mean_rank": 3066.366583099857, - "harmonic_mean_rank": 36.5574991137905, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.047879616963064295, - "hits_at_3": 0.029069767441860465, - "hits_at_5": 0.03488372093023256, - "inverse_arithmetic_mean_rank": 7.914828219535031e-05, - "inverse_geometric_mean_rank": 0.000326118868341266, - "inverse_harmonic_mean_rank": 0.02735416875446965, - "inverse_median_rank": 0.00013421917992081068, - "median_absolute_deviation": 10865.250358318304, - "median_rank": 7450.5, - "standard_deviation": 13004.701758352729, - "variance": 169122267.82370254, - "z_arithmetic_mean_rank": 35.27549982003005, - "z_geometric_mean_rank": 43.003933484274725, - "z_hits_at_k": 164.00773982763116, - "z_inverse_harmonic_mean_rank": 230.07003849884842 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.6233713798155583, - "adjusted_arithmetic_mean_rank_index": 0.3766472034815629, - "adjusted_geometric_mean_rank_index": 0.7944890418337018, - "adjusted_hits_at_k": 0.047644670345667155, - "adjusted_inverse_harmonic_mean_rank": 0.027085656396564593, - "arithmetic_mean_rank": 12634.522571819425, - "count": 2924.0, - "geometric_mean_rank": 3066.368839795842, - "harmonic_mean_rank": 36.557499248550606, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.047879616963064295, - "hits_at_3": 0.029069767441860465, - "hits_at_5": 0.03488372093023256, - "inverse_arithmetic_mean_rank": 7.914822220749697e-05, - "inverse_geometric_mean_rank": 0.00032611862833388945, - "inverse_harmonic_mean_rank": 0.027354168653635328, - "inverse_median_rank": 0.00013421917992081068, - "median_absolute_deviation": 10865.250358318304, - "median_rank": 7450.5, - "standard_deviation": 13004.707744755677, - "variance": 169122423.5265083, - "z_arithmetic_mean_rank": 35.27545556852986, - "z_geometric_mean_rank": 43.00392529499908, - "z_hits_at_k": 164.00773982763116, - "z_inverse_harmonic_mean_rank": 230.07003764210856 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.6233711816153448, - "adjusted_arithmetic_mean_rank_index": 0.37664740169155586, - "adjusted_geometric_mean_rank_index": 0.7944889725616432, - "adjusted_hits_at_k": 0.047644670345667155, - "adjusted_inverse_harmonic_mean_rank": 0.02708565640653095, - "arithmetic_mean_rank": 12634.5185546875, - "count": 2924.0, - "geometric_mean_rank": 3066.369873046875, - "harmonic_mean_rank": 36.55749923523474, - "hits_at_1": 0.016073871409028728, - "hits_at_10": 0.047879616963064295, - "hits_at_3": 0.029069767441860465, - "hits_at_5": 0.03488372093023256, - "inverse_arithmetic_mean_rank": 7.914824527688324e-05, - "inverse_geometric_mean_rank": 0.00032611851929686964, - "inverse_harmonic_mean_rank": 0.027354168663598934, - "inverse_median_rank": 0.00013421918265521526, - "median_absolute_deviation": 10865.250358318304, - "median_rank": 7450.5, - "standard_deviation": 13004.7041015625, - "variance": 169122336.0, - "z_arithmetic_mean_rank": 35.275474132182346, - "z_geometric_mean_rank": 43.00392154545658, - "z_hits_at_k": 164.00773982763116, - "z_inverse_harmonic_mean_rank": 230.07003772676444 - } - }, - "tail": { - "optimistic": { - "adjusted_arithmetic_mean_rank": 0.45233227122762737, - "adjusted_arithmetic_mean_rank_index": 0.547694739351391, - "adjusted_geometric_mean_rank_index": 0.8739823201514549, - "adjusted_hits_at_k": 0.06782759063037647, - "adjusted_inverse_harmonic_mean_rank": 0.03330130404665707, - "arithmetic_mean_rank": 9171.962038303693, - "count": 2924.0, - "geometric_mean_rank": 1881.4961241960061, - "harmonic_mean_rank": 29.790283083367658, - "hits_at_1": 0.015389876880984952, - "hits_at_10": 0.06805745554035568, - "hits_at_3": 0.037277701778385776, - "hits_at_5": 0.048221614227086186, - "inverse_arithmetic_mean_rank": 0.00010902792617586377, - "inverse_geometric_mean_rank": 0.0005314919266322253, - "inverse_harmonic_mean_rank": 0.03356799252969551, - "inverse_median_rank": 0.00024372410431391665, - "median_absolute_deviation": 6016.399802695732, - "median_rank": 4103.0, - "standard_deviation": 10922.631783326779, - "variance": 119303885.07414031, - "z_arithmetic_mean_rank": 51.29522928879513, - "z_geometric_mean_rank": 47.30670708848514, - "z_hits_at_k": 233.53571327456675, - "z_inverse_harmonic_mean_rank": 282.929810568764 - }, - "pessimistic": { - "adjusted_arithmetic_mean_rank": 0.4523326422846345, - "adjusted_arithmetic_mean_rank_index": 0.5476943682760836, - "adjusted_geometric_mean_rank_index": 0.8739822226856442, - "adjusted_hits_at_k": 0.06782759063037647, - "adjusted_inverse_harmonic_mean_rank": 0.03330130390250666, - "arithmetic_mean_rank": 9171.969562243503, - "count": 2924.0, - "geometric_mean_rank": 1881.497578627477, - "harmonic_mean_rank": 29.790283211260224, - "hits_at_1": 0.015389876880984952, - "hits_at_10": 0.06805745554035568, - "hits_at_3": 0.037277701778385776, - "hits_at_5": 0.048221614227086186, - "inverse_arithmetic_mean_rank": 0.00010902783673819735, - "inverse_geometric_mean_rank": 0.0005314915157794061, - "inverse_harmonic_mean_rank": 0.03356799238558487, - "inverse_median_rank": 0.00024369440721335446, - "median_absolute_deviation": 6017.1411038049855, - "median_rank": 4103.5, - "standard_deviation": 10922.637209525301, - "variance": 119304003.61090665, - "z_arithmetic_mean_rank": 51.29519453514204, - "z_geometric_mean_rank": 47.30670181287892, - "z_hits_at_k": 233.53571327456675, - "z_inverse_harmonic_mean_rank": 282.92980934405375 - }, - "realistic": { - "adjusted_arithmetic_mean_rank": 0.45233245774438646, - "adjusted_arithmetic_mean_rank_index": 0.5476945528254331, - "adjusted_geometric_mean_rank_index": 0.8739823140114357, - "adjusted_hits_at_k": 0.06782759063037647, - "adjusted_inverse_harmonic_mean_rank": 0.033301303699857135, - "arithmetic_mean_rank": 9171.9658203125, - "count": 2924.0, - "geometric_mean_rank": 1881.4962158203125, - "harmonic_mean_rank": 29.79028339105416, - "hits_at_1": 0.015389876880984952, - "hits_at_10": 0.06805745554035568, - "hits_at_3": 0.037277701778385776, - "hits_at_5": 0.048221614227086186, - "inverse_arithmetic_mean_rank": 0.00010902788199018687, - "inverse_geometric_mean_rank": 0.0005314918817020953, - "inverse_harmonic_mean_rank": 0.033567992182991245, - "inverse_median_rank": 0.0002437092480249703, - "median_absolute_deviation": 6016.770453250359, - "median_rank": 4103.25, - "standard_deviation": 10922.634765625, - "variance": 119303952.0, - "z_arithmetic_mean_rank": 51.29521181940736, - "z_geometric_mean_rank": 47.30670675613963, - "z_hits_at_k": 233.53571327456675, - "z_inverse_harmonic_mean_rank": 282.9298076223316 - } - } - }, - "times": { - "evaluation": 4.860883474349976, - "training": 332.31198930740356 - } -} \ No newline at end of file diff --git a/nonasv/results/27c9fe15/summary.tsv b/nonasv/results/27c9fe15/summary.tsv deleted file mode 100644 index bab9195..0000000 --- a/nonasv/results/27c9fe15/summary.tsv +++ /dev/null @@ -1,39 +0,0 @@ -batch_size.evaluation evaluation path training -1024 31.021438598632812 complex/trouillon2016_complex_fb15k.json 1.827806282043457 -256 4.746052265167236 complex/trouillon2016_complex_wn18.json 0.36449394226074217 -2048 15.175896167755127 conve/dettmers2018_conve_fb15k.json 32.02347240447998 -4096 6.952138900756836 conve/dettmers2018_conve_fb15k237.json 17.444180917739867 -8192 3.979680061340332 conve/dettmers2018_conve_wn18.json 32.08411064147949 -4096 2.390557289123535 conve/dettmers2018_conve_wn18rr.json 19.340176582336426 -32 113.86807107925415 convkb/nguyen2018_convkb_fb15k237.json 5.623247241973877 -4 229.65697264671326 convkb/nguyen2018_convkb_wn18rr.json 2.274687576293945 -4096 9.86320161819458 convkb/nguyen2018_transe_fb15k237.json 2.304577445983887 -2048 19.562544584274292 distmult/yang2014_distmult_fb15k.json 3.5463189125061034 -1024 4.716632843017578 distmult/yang2014_distmult_wn18.json 1.373633337020874 -512 34.93160533905029 hole/nickel2016_hole_fb15k.json 0.8114243030548096 -256 7.370163202285767 hole/nickel2016_hole_wn18.json 0.4553260326385498 -2048 28.49927306175232 kg2e/he2015_kg2e_fb15k.json 2.6026907920837403 -1024 7.71472430229187 kg2e/he2015_kg2e_wn18.json 38.04337100982666 -1024 15.51103925704956 mure/balazevic2019_mure_fb15k237.json 12.894137287139893 -256 144.05696630477905 quate/zhang2019_quate_fb15k.json 1.5077387809753418 -512 31.55549693107605 quate/zhang2019_quate_fb15k237.json 0.6327661991119384 -64 49.746681928634644 quate/zhang2019_quate_wn18.json 1.7024828910827636 -128 13.313127517700195 quate/zhang2019_quate_wn18rr.json 0.5839082717895507 -256 86.14422821998596 rotate/sun2019_rotate_fb15k.json 82.6045967578888 -256 29.337369203567505 rotate/sun2019_rotate_fb15k237.json 91.85465803146363 -256 11.969523429870605 rotate/sun2019_rotate_wn18.json 98.09528822898865 -256 7.80864691734314 rotate/sun2019_rotate_wn18rr.json 60.45014009475708 -1024 29.497436046600342 simple/kazemi2018_simple_fb15k.json 2.608071517944336 -1024 9.06259036064148 simple/kazemi2018_simple_wn18.json 0.9677006721496582 -2048 34.54206871986389 transd/ji2015_transd_fb15k.json 16.6326443195343 -1024 10.18949007987976 transd/ji2015_transd_wn18.json 4.865910863876342 -2048 17.733282566070557 transe/bordes2013_transe_fb15k.json 22.46914396286011 -8192 5.315532684326172 transe/bordes2013_transe_wn18.json 6.628315019607544 -2048 29.929590225219727 transh/wang2014_transh_fb15k.json 2.0839001655578615 -2048 8.578476905822754 transh/wang2014_transh_wn18.json 0.5796948909759522 -2048 33.42562985420227 transr/li2015_transr_fb15k.json 2.437476301193237 -1024 9.90921139717102 transr/li2015_transr_wn18.json 0.9390138626098633 -1024 22.204118728637695 tucker/balazevic2019_tucker_fb15k.json 79.91824607849121 -2048 9.888858556747437 tucker/balazevic2019_tucker_fb15k237.json 42.80506386756897 -1024 6.108567476272583 tucker/balazevic2019_tucker_wn18.json 110.96793870925903 -1024 4.860883474349976 tucker/balazevic2019_tucker_wn18rr.json 66.46239786148071 From 423b095ed03b012b77c913e9b2f859f8016091af Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 11:55:21 +0200 Subject: [PATCH 3/7] fix relative path --- nonasv/pipeline.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nonasv/pipeline.py b/nonasv/pipeline.py index ad8ed5e..d4f369f 100644 --- a/nonasv/pipeline.py +++ b/nonasv/pipeline.py @@ -61,9 +61,16 @@ def main( log_level: str, debug: bool, ): - """""" + """ + Run all packaged reproducibility configurations and save results for a given commit. + + Also reduces the number of training epochs to reduce runtime. + """ logging.basicConfig(level=log_level) + # make absolute + output_root = output_root.expanduser().resolve() + device = resolve_device(device=None) logging.info(f"Running on device: {device}") From 038b4590e2985962a3cc37092fed54b573ebbd57 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 11:58:37 +0200 Subject: [PATCH 4/7] fix logging --- nonasv/pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nonasv/pipeline.py b/nonasv/pipeline.py index d4f369f..a6ac2e9 100644 --- a/nonasv/pipeline.py +++ b/nonasv/pipeline.py @@ -70,9 +70,10 @@ def main( # make absolute output_root = output_root.expanduser().resolve() + logger.info(f"Resolved output root: {output_root}") device = resolve_device(device=None) - logging.info(f"Running on device: {device}") + logger.info(f"Running on device: {device}") system_information = _collect_system_information() From cd403c2aec75d6d9467a0bfb91d4f2d46acaed9d Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 14:06:08 +0200 Subject: [PATCH 5/7] add v1.8.0 results --- .../complex/fb15k/trouillon2016/metadata.json | 20 ++ .../complex/fb15k/trouillon2016/results.json | 246 ++++++++++++++++++ .../complex/wn18/trouillon2016/metadata.json | 20 ++ .../complex/wn18/trouillon2016/results.json | 246 ++++++++++++++++++ .../conve/fb15k/dettmers2018/metadata.json | 20 ++ .../conve/fb15k/dettmers2018/results.json | 246 ++++++++++++++++++ .../conve/fb15k237/dettmers2018/metadata.json | 20 ++ .../conve/fb15k237/dettmers2018/results.json | 246 ++++++++++++++++++ .../conve/wn18/dettmers2018/metadata.json | 20 ++ .../cuda/conve/wn18/dettmers2018/results.json | 246 ++++++++++++++++++ .../conve/wn18rr/dettmers2018/metadata.json | 20 ++ .../conve/wn18rr/dettmers2018/results.json | 246 ++++++++++++++++++ .../convkb/fb15k237/nguyen2018/metadata.json | 20 ++ .../convkb/fb15k237/nguyen2018/results.json | 246 ++++++++++++++++++ .../convkb/wn18rr/nguyen2018/metadata.json | 20 ++ .../convkb/wn18rr/nguyen2018/results.json | 246 ++++++++++++++++++ .../distmult/fb15k/yang2014/metadata.json | 20 ++ .../cuda/distmult/fb15k/yang2014/results.json | 246 ++++++++++++++++++ .../cuda/distmult/wn18/yang2014/metadata.json | 20 ++ .../cuda/distmult/wn18/yang2014/results.json | 246 ++++++++++++++++++ .../cuda/hole/fb15k/nickel2016/metadata.json | 20 ++ .../cuda/hole/fb15k/nickel2016/results.json | 246 ++++++++++++++++++ .../cuda/hole/wn18/nickel2016/metadata.json | 20 ++ .../cuda/hole/wn18/nickel2016/results.json | 246 ++++++++++++++++++ .../cuda/kg2e/fb15k/he2015/metadata.json | 19 ++ .../cuda/kg2e/fb15k/he2015/results.json | 246 ++++++++++++++++++ .../cuda/kg2e/wn18/he2015/metadata.json | 19 ++ .../v1.8.0/cuda/kg2e/wn18/he2015/results.json | 246 ++++++++++++++++++ .../mure/fb15k237/balazevic2019/metadata.json | 20 ++ .../mure/fb15k237/balazevic2019/results.json | 246 ++++++++++++++++++ .../cuda/quate/fb15k/zhang2019/metadata.json | 19 ++ .../cuda/quate/fb15k/zhang2019/results.json | 246 ++++++++++++++++++ .../quate/fb15k237/zhang2019/metadata.json | 19 ++ .../quate/fb15k237/zhang2019/results.json | 246 ++++++++++++++++++ .../cuda/quate/wn18/zhang2019/metadata.json | 19 ++ .../cuda/quate/wn18/zhang2019/results.json | 246 ++++++++++++++++++ .../cuda/quate/wn18rr/zhang2019/metadata.json | 19 ++ .../cuda/quate/wn18rr/zhang2019/results.json | 246 ++++++++++++++++++ .../cuda/rotate/fb15k/sun2019/metadata.json | 20 ++ .../cuda/rotate/fb15k/sun2019/results.json | 246 ++++++++++++++++++ .../rotate/fb15k237/sun2019/metadata.json | 20 ++ .../cuda/rotate/fb15k237/sun2019/results.json | 246 ++++++++++++++++++ .../cuda/rotate/wn18/sun2019/metadata.json | 20 ++ .../cuda/rotate/wn18/sun2019/results.json | 246 ++++++++++++++++++ .../cuda/rotate/wn18rr/sun2019/metadata.json | 20 ++ .../cuda/rotate/wn18rr/sun2019/results.json | 246 ++++++++++++++++++ .../simple/fb15k/kazemi2018/metadata.json | 20 ++ .../cuda/simple/fb15k/kazemi2018/results.json | 246 ++++++++++++++++++ .../cuda/simple/wn18/kazemi2018/metadata.json | 20 ++ .../cuda/simple/wn18/kazemi2018/results.json | 246 ++++++++++++++++++ .../cuda/transd/fb15k/ji2015/metadata.json | 20 ++ .../cuda/transd/fb15k/ji2015/results.json | 246 ++++++++++++++++++ .../cuda/transd/wn18/ji2015/metadata.json | 20 ++ .../cuda/transd/wn18/ji2015/results.json | 246 ++++++++++++++++++ .../transe/fb15k/bordes2013/metadata.json | 20 ++ .../cuda/transe/fb15k/bordes2013/results.json | 246 ++++++++++++++++++ .../transe/fb15k237/nguyen2018/metadata.json | 20 ++ .../transe/fb15k237/nguyen2018/results.json | 246 ++++++++++++++++++ .../cuda/transe/wn18/bordes2013/metadata.json | 20 ++ .../cuda/transe/wn18/bordes2013/results.json | 246 ++++++++++++++++++ .../cuda/transh/fb15k/wang2014/metadata.json | 19 ++ .../cuda/transh/fb15k/wang2014/results.json | 246 ++++++++++++++++++ .../cuda/transh/wn18/wang2014/metadata.json | 19 ++ .../cuda/transh/wn18/wang2014/results.json | 246 ++++++++++++++++++ .../cuda/transr/fb15k/li2015/metadata.json | 19 ++ .../cuda/transr/fb15k/li2015/results.json | 246 ++++++++++++++++++ .../cuda/transr/wn18/li2015/metadata.json | 19 ++ .../cuda/transr/wn18/li2015/results.json | 246 ++++++++++++++++++ .../tucker/fb15k/balazevic2019/metadata.json | 20 ++ .../tucker/fb15k/balazevic2019/results.json | 246 ++++++++++++++++++ .../fb15k237/balazevic2019/metadata.json | 20 ++ .../fb15k237/balazevic2019/results.json | 246 ++++++++++++++++++ .../tucker/wn18/balazevic2019/metadata.json | 20 ++ .../tucker/wn18/balazevic2019/results.json | 246 ++++++++++++++++++ .../tucker/wn18rr/balazevic2019/metadata.json | 20 ++ .../tucker/wn18rr/balazevic2019/results.json | 246 ++++++++++++++++++ nonasv/results/v1.8.0/summary.tsv | 39 +++ 77 files changed, 10137 insertions(+) create mode 100644 nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json create mode 100644 nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json create mode 100644 nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json create mode 100644 nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json create mode 100644 nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json create mode 100644 nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json create mode 100644 nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json create mode 100644 nonasv/results/v1.8.0/summary.tsv diff --git a/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json new file mode 100644 index 0000000..b62165b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json new file mode 100644 index 0000000..79e0b72 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00027370825300399417, + 5.7055877953340055e-05, + 3.4271220052227624e-05, + 2.3875505992467534e-05, + 1.8529287064680124e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08070188773133745, + "adjusted_arithmetic_mean_rank_index": 0.9194225797048108, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393374384795, + "arithmetic_mean_rank": 596.132933249818, + "count": 118142.0, + "geometric_mean_rank": 59.0259733036713, + "harmonic_mean_rank": 8.15661454878056, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774782002874778, + "inverse_geometric_mean_rank": 0.016941694376732996, + "inverse_harmonic_mean_rank": 0.1225998843048803, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5974290660563, + "variance": 2711283.093406946, + "z_arithmetic_mean_rank": 546.8884494603583, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225642125115 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08070189231482705, + "adjusted_arithmetic_mean_rank_index": 0.9194225751207006, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393369414213, + "arithmetic_mean_rank": 596.1329671073793, + "count": 118142.0, + "geometric_mean_rank": 59.025974985687476, + "harmonic_mean_rank": 8.156614552085223, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774781050145706, + "inverse_geometric_mean_rank": 0.01694169389395903, + "inverse_harmonic_mean_rank": 0.12259988425520878, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5974825402982, + "variance": 2711283.2695080475, + "z_arithmetic_mean_rank": 546.8884467336502, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225625927046 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08070189617277465, + "adjusted_arithmetic_mean_rank_index": 0.9194225712622306, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393371896833, + "arithmetic_mean_rank": 596.1329956054688, + "count": 118142.0, + "geometric_mean_rank": 59.02597427368164, + "harmonic_mean_rank": 8.156614672134552, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774780815467238, + "inverse_geometric_mean_rank": 0.016941694542765617, + "inverse_harmonic_mean_rank": 0.12259988428001783, + "inverse_median_rank": 0.0181818176060915, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5975341796875, + "variance": 2711283.5, + "z_arithmetic_mean_rank": 546.8884444385656, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225634017374 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08704475810034824, + "adjusted_arithmetic_mean_rank_index": 0.9130791879154981, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476825209027, + "arithmetic_mean_rank": 641.2368844272147, + "count": 59071.0, + "geometric_mean_rank": 64.55592253351908, + "harmonic_mean_rank": 8.658960015297517, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594860874125335, + "inverse_geometric_mean_rank": 0.01549044550452787, + "inverse_harmonic_mean_rank": 0.11548731005032141, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.3770042289605, + "variance": 2915136.2345698597, + "z_arithmetic_mean_rank": 383.9848643283128, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031892984183 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08704476269634591, + "adjusted_arithmetic_mean_rank_index": 0.9130791833188765, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476816142525, + "arithmetic_mean_rank": 641.2369182847759, + "count": 59071.0, + "geometric_mean_rank": 64.55592524572788, + "harmonic_mean_rank": 8.658960022090657, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594860050710555, + "inverse_geometric_mean_rank": 0.015490444853722812, + "inverse_harmonic_mean_rank": 0.11548730995971913, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.3770524809593, + "variance": 2915136.3993385686, + "z_arithmetic_mean_rank": 383.9848623952569, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031890898167 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08704475715205905, + "adjusted_arithmetic_mean_rank_index": 0.913079188863916, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476820670574, + "arithmetic_mean_rank": 641.2368774414062, + "count": 59071.0, + "geometric_mean_rank": 64.55589294433594, + "harmonic_mean_rank": 8.658960279801065, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594861470162868, + "inverse_geometric_mean_rank": 0.015490452758967876, + "inverse_harmonic_mean_rank": 0.11548731000496829, + "inverse_median_rank": 0.01666666753590107, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.376953125, + "variance": 2915136.25, + "z_arithmetic_mean_rank": 383.984864727159, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031891939978 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07439344805062242, + "adjusted_arithmetic_mean_rank_index": 0.9257315333259909, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.12911307222450374, + "arithmetic_mean_rank": 551.0289820724213, + "count": 59071.0, + "geometric_mean_rank": 53.9697271406308, + "harmonic_mean_rank": 7.70935969532766, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147865766315915, + "inverse_geometric_mean_rank": 0.018528905980833756, + "inverse_harmonic_mean_rank": 0.1297124585594392, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.201383964887, + "variance": 2503361.219420404, + "z_arithmetic_mean_rank": 389.4207678791215, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.7269258899264 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0743934526216719, + "adjusted_arithmetic_mean_rank_index": 0.9257315287543242, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.12911307221575694, + "arithmetic_mean_rank": 551.0290159299826, + "count": 59071.0, + "geometric_mean_rank": 53.969727949047844, + "harmonic_mean_rank": 7.709359695847163, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147864651234021, + "inverse_geometric_mean_rank": 0.018528905703287735, + "inverse_harmonic_mean_rank": 0.1297124585506984, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.2014431967566, + "variance": 2503361.4068538994, + "z_arithmetic_mean_rank": 389.42076595599167, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.726925688064 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07439345759056779, + "adjusted_arithmetic_mean_rank_index": 0.9257315237847574, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.1291130722201289, + "arithmetic_mean_rank": 551.029052734375, + "count": 59071.0, + "geometric_mean_rank": 53.9697265625, + "harmonic_mean_rank": 7.709359214767926, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147863447666168, + "inverse_geometric_mean_rank": 0.018528906628489494, + "inverse_harmonic_mean_rank": 0.12971245855506738, + "inverse_median_rank": 0.019999999552965164, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.201416015625, + "variance": 2503361.25, + "z_arithmetic_mean_rank": 389.42076386548007, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.7269257889625 + } + } + }, + "times": { + "evaluation": 29.81851315498352, + "training": 8.202208042144775 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json new file mode 100644 index 0000000..26a4bcb --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json new file mode 100644 index 0000000..9570d2a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00019845912073571508, + 0.00012037810856069427, + 0.00010923121339792283, + 0.00010423280733917918, + 0.00010124416824204406 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23338080058024066, + "adjusted_arithmetic_mean_rank_index": 0.7666566644999107, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.01730672159221292, + "arithmetic_mean_rank": 4775.7257, + "count": 10000.0, + "geometric_mean_rank": 1159.3200186029587, + "harmonic_mean_rank": 56.89714039894227, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.000209392260531211, + "inverse_geometric_mean_rank": 0.0008625745988627474, + "inverse_harmonic_mean_rank": 0.01757557573172149, + "inverse_median_rank": 0.0006228589224540642, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.395222348466, + "variance": 54204863.41005951, + "z_arithmetic_mean_rank": 132.78541860258312, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.16558831999345 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23338082501430735, + "adjusted_arithmetic_mean_rank_index": 0.7666566400646498, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.017306721511205696, + "arithmetic_mean_rank": 4775.7262, + "count": 10000.0, + "geometric_mean_rank": 1159.3202177617752, + "harmonic_mean_rank": 56.897140661113966, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.0002093922386086539, + "inverse_geometric_mean_rank": 0.000862574450681655, + "inverse_harmonic_mean_rank": 0.017575575650736426, + "inverse_median_rank": 0.0006228589224540642, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.395261056931, + "variance": 54204863.980033554, + "z_arithmetic_mean_rank": 132.78541437038052, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.1655870413927 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2333807950062192, + "adjusted_arithmetic_mean_rank_index": 0.7666566700742046, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.017306721551698052, + "arithmetic_mean_rank": 4775.7255859375, + "count": 10000.0, + "geometric_mean_rank": 1159.3203125, + "harmonic_mean_rank": 56.897141937862685, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.0002093922666972503, + "inverse_geometric_mean_rank": 0.0008625743794254959, + "inverse_harmonic_mean_rank": 0.017575575691217704, + "inverse_median_rank": 0.0006228589336387813, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.39501953125, + "variance": 54204860.0, + "z_arithmetic_mean_rank": 132.78541956805435, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.1655876805154 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23330198500215898, + "adjusted_arithmetic_mean_rank_index": 0.7667354847984572, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.01738759586962493, + "arithmetic_mean_rank": 4774.0022, + "count": 5000.0, + "geometric_mean_rank": 1156.8772914097908, + "harmonic_mean_rank": 56.63657920733289, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.0002094678548744699, + "inverse_geometric_mean_rank": 0.0008643959108069124, + "inverse_harmonic_mean_rank": 0.017656433598138767, + "inverse_median_rank": 0.0006309148264984228, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.204723369773, + "variance": 54260972.02659516, + "z_arithmetic_mean_rank": 93.90311524197872, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.05759979374437 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2333019947760123, + "adjusted_arithmetic_mean_rank_index": 0.7667354750241262, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.017387595855387426, + "arithmetic_mean_rank": 4774.0024, + "count": 5000.0, + "geometric_mean_rank": 1156.8773531346058, + "harmonic_mean_rank": 56.636579252990074, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.0002094678460991138, + "inverse_geometric_mean_rank": 0.0008643958646873497, + "inverse_harmonic_mean_rank": 0.01765643358390516, + "inverse_median_rank": 0.0006309148264984228, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.204695526336, + "variance": 54260971.61639424, + "z_arithmetic_mean_rank": 93.9031140449034, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.057599634844 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23330199679950533, + "adjusted_arithmetic_mean_rank_index": 0.7667354730005342, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.01738759586250523, + "arithmetic_mean_rank": 4774.00244140625, + "count": 5000.0, + "geometric_mean_rank": 1156.8773193359375, + "harmonic_mean_rank": 56.63658078767159, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.00020946784934494644, + "inverse_geometric_mean_rank": 0.000864395871758461, + "inverse_harmonic_mean_rank": 0.017656433591021013, + "inverse_median_rank": 0.0006309148157015443, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.20458984375, + "variance": 54260972.0, + "z_arithmetic_mean_rank": 93.9031137970714, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.05759971428358 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23345961250403727, + "adjusted_arithmetic_mean_rank_index": 0.7665778478560465, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.017225847315741973, + "arithmetic_mean_rank": 4777.4492, + "count": 5000.0, + "geometric_mean_rank": 1161.7679035740384, + "harmonic_mean_rank": 57.160110137198345, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.00020931672073038474, + "inverse_geometric_mean_rank": 0.0008607571244855542, + "inverse_harmonic_mean_rank": 0.017494717865304207, + "inverse_median_rank": 0.0006142506142506142, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.5833454965605, + "variance": 54148748.85261936, + "z_arithmetic_mean_rank": 93.8838251268764, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685901455532 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23345965159763768, + "adjusted_arithmetic_mean_rank_index": 0.7665778087605357, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.017225847167965026, + "arithmetic_mean_rank": 4777.45, + "count": 5000.0, + "geometric_mean_rank": 1161.7682407469797, + "harmonic_mean_rank": 57.16011061989465, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.00020931668567959896, + "inverse_geometric_mean_rank": 0.0008607568746732412, + "inverse_harmonic_mean_rank": 0.01749471771756769, + "inverse_median_rank": 0.0006142506142506142, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.583450685329, + "variance": 54148750.4007, + "z_arithmetic_mean_rank": 93.88382033879634, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685736522388 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23345963728113361, + "adjusted_arithmetic_mean_rank_index": 0.7665778230777394, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.01722584724183194, + "arithmetic_mean_rank": 4777.44970703125, + "count": 5000.0, + "geometric_mean_rank": 1161.7679443359375, + "harmonic_mean_rank": 57.16011163386548, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.0002093166986014694, + "inverse_geometric_mean_rank": 0.0008607570780441165, + "inverse_harmonic_mean_rank": 0.017494717791414392, + "inverse_median_rank": 0.0006142506026662886, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.58349609375, + "variance": 54148748.0, + "z_arithmetic_mean_rank": 93.88382209224363, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685818964897 + } + } + }, + "times": { + "evaluation": 4.551660537719727, + "training": 1.768416166305542 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json new file mode 100644 index 0000000..a2debf8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json new file mode 100644 index 0000000..bf5bc98 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 9.414209683117643e-05, + 1.3477408468953255e-05, + 1.114161235120257e-05, + 9.765913843640251e-06, + 8.849434282325717e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05739133898406139, + "adjusted_arithmetic_mean_rank_index": 0.9427362845602079, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540388113957916, + "arithmetic_mean_rank": 423.94135023954226, + "count": 118142.0, + "geometric_mean_rank": 37.30836518451274, + "harmonic_mean_rank": 4.23851726046015, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.002358816849158412, + "inverse_geometric_mean_rank": 0.0268036402842737, + "inverse_harmonic_mean_rank": 0.23593156251331063, + "inverse_median_rank": 0.02564102564102564, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.6051548955982, + "variance": 1150481.81830861, + "z_arithmetic_mean_rank": 560.7558442589916, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.311289450006 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05739136763087145, + "adjusted_arithmetic_mean_rank_index": 0.9427362559095191, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540385571304592, + "arithmetic_mean_rank": 423.9415618493, + "count": 118142.0, + "geometric_mean_rank": 37.308392746564536, + "harmonic_mean_rank": 4.2385177169333454, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.0023588156717587257, + "inverse_geometric_mean_rank": 0.026803620482741992, + "inverse_harmonic_mean_rank": 0.23593153710432535, + "inverse_median_rank": 0.02564102564102564, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.60537847533, + "variance": 1150482.2979342062, + "z_arithmetic_mean_rank": 560.7558272170666, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.310460853424 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.057391350697851125, + "adjusted_arithmetic_mean_rank_index": 0.9427362728448321, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540386809111943, + "arithmetic_mean_rank": 423.9414367675781, + "count": 118142.0, + "geometric_mean_rank": 37.30837631225586, + "harmonic_mean_rank": 4.238517393849456, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.0023588163312524557, + "inverse_geometric_mean_rank": 0.02680363319814205, + "inverse_harmonic_mean_rank": 0.23593154947385622, + "inverse_median_rank": 0.025641025975346565, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.605224609375, + "variance": 1150482.0, + "z_arithmetic_mean_rank": 560.7558372904836, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.310864228486 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.06338498032097208, + "adjusted_arithmetic_mean_rank_index": 0.9367421778295518, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237043022013573, + "arithmetic_mean_rank": 466.94124020246824, + "count": 59071.0, + "geometric_mean_rank": 46.42692536529115, + "harmonic_mean_rank": 4.927991655022916, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415970873902563, + "inverse_geometric_mean_rank": 0.021539225183057283, + "inverse_harmonic_mean_rank": 0.20292242154686638, + "inverse_median_rank": 0.019230769230769232, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.815756992061, + "variance": 1278744.2762615283, + "z_arithmetic_mean_rank": 393.9360603384787, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.126923320398 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.06338501249295578, + "adjusted_arithmetic_mean_rank_index": 0.9367421456532004, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237037988647139, + "arithmetic_mean_rank": 466.9414772053969, + "count": 59071.0, + "geometric_mean_rank": 46.42698260283127, + "harmonic_mean_rank": 4.927992876535459, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415960003915496, + "inverse_geometric_mean_rank": 0.021539198628407454, + "inverse_harmonic_mean_rank": 0.20292237124803492, + "inverse_median_rank": 0.019230769230769232, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.8160192560426, + "variance": 1278744.8694060827, + "z_arithmetic_mean_rank": 393.93604680708717, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.125765246417 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.06338499457588637, + "adjusted_arithmetic_mean_rank_index": 0.9367421635727022, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237040438327544, + "arithmetic_mean_rank": 466.94134521484375, + "count": 59071.0, + "geometric_mean_rank": 46.426937103271484, + "harmonic_mean_rank": 4.9279921737017185, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415965165942907, + "inverse_geometric_mean_rank": 0.021539220586419106, + "inverse_harmonic_mean_rank": 0.2029223957278862, + "inverse_median_rank": 0.01923076994717121, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.8157958984375, + "variance": 1278744.5, + "z_arithmetic_mean_rank": 393.9360543429259, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.12632886744 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05143023263283102, + "adjusted_arithmetic_mean_rank_index": 0.9486978493854841, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.2684372067255253, + "arithmetic_mean_rank": 380.9414602766163, + "count": 59071.0, + "geometric_mean_rank": 29.980751509802925, + "harmonic_mean_rank": 3.718291753763027, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250752524386856, + "inverse_geometric_mean_rank": 0.03335473427585783, + "inverse_harmonic_mean_rank": 0.2689407034797549, + "inverse_median_rank": 0.03571428571428571, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2182020168108, + "variance": 1018521.3792820444, + "z_arithmetic_mean_rank": 399.0818414336863, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.1091319390425 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05143025777360297, + "adjusted_arithmetic_mean_rank_index": 0.9486978242413174, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.2684372062060287, + "arithmetic_mean_rank": 380.9416464932031, + "count": 59071.0, + "geometric_mean_rank": 29.980758845331998, + "harmonic_mean_rank": 3.718291760940484, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250739692170737, + "inverse_geometric_mean_rank": 0.033354726114802793, + "inverse_harmonic_mean_rank": 0.26894070296061584, + "inverse_median_rank": 0.03571428571428571, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2183812342437, + "variance": 1018521.7410210673, + "z_arithmetic_mean_rank": 399.08183085647215, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.109119949877 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05143023769916584, + "adjusted_arithmetic_mean_rank_index": 0.9486978443184652, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.26843720646541763, + "arithmetic_mean_rank": 380.9414978027344, + "count": 59071.0, + "geometric_mean_rank": 29.980751037597656, + "harmonic_mean_rank": 3.718291663785221, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250749360769987, + "inverse_geometric_mean_rank": 0.033354733139276505, + "inverse_harmonic_mean_rank": 0.26894070321982627, + "inverse_median_rank": 0.0357142873108387, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2183227539062, + "variance": 1018521.5625, + "z_arithmetic_mean_rank": 399.08183930218024, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.1091259361665 + } + } + }, + "times": { + "evaluation": 14.673102378845215, + "training": 173.3771436214447 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json new file mode 100644 index 0000000..c8b9c51 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json new file mode 100644 index 0000000..cca2c8a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001577785732766328, + 1.5780817098527492e-05, + 1.386544718458381e-05, + 1.2271771170897758e-05, + 1.1084717919739344e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.18198389342227317, + "adjusted_arithmetic_mean_rank_index": 0.818130760265559, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.20563247146486757, + "arithmetic_mean_rank": 1298.5768176925335, + "count": 40876.0, + "geometric_mean_rank": 82.93930185831799, + "harmonic_mean_rank": 4.849694078852137, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700738118649919, + "inverse_geometric_mean_rank": 0.01205701009767675, + "inverse_harmonic_mean_rank": 0.20619857329984156, + "inverse_median_rank": 0.010526315789473684, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.477521691501, + "variance": 6332659.117178599, + "z_arithmetic_mean_rank": 286.12773531756534, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.183867427056 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.18198398599023052, + "adjusted_arithmetic_mean_rank_index": 0.8181306676846272, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.2056324713907482, + "arithmetic_mean_rank": 1298.5774782268325, + "count": 40876.0, + "geometric_mean_rank": 82.93931610724847, + "harmonic_mean_rank": 4.849694080594148, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700734201593186, + "inverse_geometric_mean_rank": 0.012057008026288814, + "inverse_harmonic_mean_rank": 0.206198573225775, + "inverse_median_rank": 0.010526315789473684, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.4785795775047, + "variance": 6332664.441472416, + "z_arithmetic_mean_rank": 286.1277029389119, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.1838660313433 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.18198393977321362, + "adjusted_arithmetic_mean_rank_index": 0.8181307139081219, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.20563247142779512, + "arithmetic_mean_rank": 1298.5771484375, + "count": 40876.0, + "geometric_mean_rank": 82.93929290771484, + "harmonic_mean_rank": 4.849694433624096, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700736168771982, + "inverse_geometric_mean_rank": 0.012057011015713215, + "inverse_harmonic_mean_rank": 0.20619857326279553, + "inverse_median_rank": 0.010526316240429878, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.47802734375, + "variance": 6332661.5, + "z_arithmetic_mean_rank": 286.12771910481655, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.1838667289594 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23351524215549604, + "adjusted_arithmetic_mean_rank_index": 0.7665937010171862, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.10774751052365336, + "arithmetic_mean_rank": 1643.162295723652, + "count": 20438.0, + "geometric_mean_rank": 190.31729087252333, + "harmonic_mean_rank": 9.225701931373973, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.0006085826108610884, + "inverse_geometric_mean_rank": 0.00525438332699792, + "inverse_harmonic_mean_rank": 0.10839283638671288, + "inverse_median_rank": 0.003115264797507788, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.7272310550084, + "variance": 7310140.939948384, + "z_arithmetic_mean_rank": 189.37640251104858, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.891462908065 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2335153603630686, + "adjusted_arithmetic_mean_rank_index": 0.7665935827928123, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.10774751039939984, + "arithmetic_mean_rank": 1643.163127507584, + "count": 20438.0, + "geometric_mean_rank": 190.31733690440817, + "harmonic_mean_rank": 9.225701941941985, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.0006085823027911052, + "inverse_geometric_mean_rank": 0.005254382056124903, + "inverse_harmonic_mean_rank": 0.10839283626254922, + "inverse_median_rank": 0.003115264797507788, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.7283336983282, + "variance": 7310146.902443139, + "z_arithmetic_mean_rank": 189.37637330534733, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.8914612671986 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23351530241195553, + "adjusted_arithmetic_mean_rank_index": 0.7665936407521623, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.1077475104615027, + "arithmetic_mean_rank": 1643.1627197265625, + "count": 20438.0, + "geometric_mean_rank": 190.3172607421875, + "harmonic_mean_rank": 9.225701024597301, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.000608582457061857, + "inverse_geometric_mean_rank": 0.005254384130239487, + "inverse_harmonic_mean_rank": 0.10839283632460717, + "inverse_median_rank": 0.0031152646988630295, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.727783203125, + "variance": 7310143.5, + "z_arithmetic_mean_rank": 189.37638762340546, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.891462087316 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1318632966013177, + "adjusted_arithmetic_mean_rank_index": 0.8682567162207461, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.30351535382197037, + "arithmetic_mean_rank": 953.991339661415, + "count": 20438.0, + "geometric_mean_rank": 36.14452350182288, + "harmonic_mean_rank": 3.289427045621327, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482275450790926, + "inverse_geometric_mean_rank": 0.027666708621834974, + "inverse_harmonic_mean_rank": 0.3040043102129702, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.2331867137273, + "variance": 5117698.991068945, + "z_arithmetic_mean_rank": 214.97579285996775, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.441934546446 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1318633642315847, + "adjusted_arithmetic_mean_rank_index": 0.8682566485811298, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.3035153537979841, + "arithmetic_mean_rank": 953.9918289460809, + "count": 20438.0, + "geometric_mean_rank": 36.14452717879933, + "harmonic_mean_rank": 3.289427045880684, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482270074626807, + "inverse_geometric_mean_rank": 0.02766670580730553, + "inverse_harmonic_mean_rank": 0.30400431018900076, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.2341702664685, + "variance": 5117703.441121218, + "z_arithmetic_mean_rank": 214.97577611275477, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.4419342243714 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.13186332942742457, + "adjusted_arithmetic_mean_rank_index": 0.8682566833901013, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.3035153538099757, + "arithmetic_mean_rank": 953.9915771484375, + "count": 20438.0, + "geometric_mean_rank": 36.1445198059082, + "harmonic_mean_rank": 3.289426970315094, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482273064553738, + "inverse_geometric_mean_rank": 0.027666712179780006, + "inverse_harmonic_mean_rank": 0.30400431020098395, + "inverse_median_rank": 0.0476190485060215, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.233642578125, + "variance": 5117701.5, + "z_arithmetic_mean_rank": 214.97578473127288, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.4419343853883 + } + } + }, + "times": { + "evaluation": 6.211552381515503, + "training": 95.12276983261108 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json new file mode 100644 index 0000000..6655aca --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json new file mode 100644 index 0000000..4179e29 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001197095220165262, + 3.842027433118703e-06, + 3.4459802265337745e-06, + 3.297556057019098e-06, + 3.1852552669447856e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6690416172263688, + "adjusted_arithmetic_mean_rank_index": 0.3309745568824283, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.02654567437482097, + "arithmetic_mean_rank": 13690.7545, + "count": 10000.0, + "geometric_mean_rank": 3642.850837532699, + "harmonic_mean_rank": 37.29673164073776, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304199341241566e-05, + "inverse_geometric_mean_rank": 0.0002745102790641023, + "inverse_harmonic_mean_rank": 0.02681200083783586, + "inverse_median_rank": 0.00010062893081761007, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.265564228193, + "variance": 168798964.49142975, + "z_arithmetic_mean_rank": 57.3250023347874, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.9912410223125 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6690419739637427, + "adjusted_arithmetic_mean_rank_index": 0.3309742001276206, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.026545674264344184, + "arithmetic_mean_rank": 13690.7618, + "count": 10000.0, + "geometric_mean_rank": 3642.8529066914157, + "harmonic_mean_rank": 37.29673179437403, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304195446596696e-05, + "inverse_geometric_mean_rank": 0.00027451012314088735, + "inverse_harmonic_mean_rank": 0.026812000727389294, + "inverse_median_rank": 0.00010062893081761007, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.268889068635, + "variance": 168799050.88586074, + "z_arithmetic_mean_rank": 57.324940544629314, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.9912392785705 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6690418268248473, + "adjusted_arithmetic_mean_rank_index": 0.33097434727370667, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.026545674319570493, + "arithmetic_mean_rank": 13690.7587890625, + "count": 10000.0, + "geometric_mean_rank": 3642.852783203125, + "harmonic_mean_rank": 37.29673348542801, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304197060875595e-05, + "inverse_geometric_mean_rank": 0.00027451012283563614, + "inverse_harmonic_mean_rank": 0.026812000782600494, + "inverse_median_rank": 0.00010062893124995753, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.267578125, + "variance": 168799008.0, + "z_arithmetic_mean_rank": 57.32496603042439, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.99124015025075 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6834586222406983, + "adjusted_arithmetic_mean_rank_index": 0.31655684766020065, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609226782556, + "arithmetic_mean_rank": 13985.4488, + "count": 5000.0, + "geometric_mean_rank": 3911.6517271442685, + "harmonic_mean_rank": 39.23983846635467, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150288948896657e-05, + "inverse_geometric_mean_rank": 0.00025564648127047285, + "inverse_harmonic_mean_rank": 0.02548430470368597, + "inverse_median_rank": 9.482268158543523e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.836726111675, + "variance": 168605984.80377856, + "z_arithmetic_mean_rank": 38.76913842625525, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.4459661806924 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6834589350040029, + "adjusted_arithmetic_mean_rank_index": 0.31655653488161084, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609157271326, + "arithmetic_mean_rank": 13985.4552, + "count": 5000.0, + "geometric_mean_rank": 3911.6534371979524, + "harmonic_mean_rank": 39.239838573356344, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150285676793701e-05, + "inverse_geometric_mean_rank": 0.0002556463695097522, + "inverse_harmonic_mean_rank": 0.025484304634193756, + "inverse_median_rank": 9.482268158543523e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.840143828995, + "variance": 168606073.56079298, + "z_arithmetic_mean_rank": 38.76910011984509, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.44596540489897 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6834588336002752, + "adjusted_arithmetic_mean_rank_index": 0.3165566362902943, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609192021116, + "arithmetic_mean_rank": 13985.453125, + "count": 5000.0, + "geometric_mean_rank": 3911.652099609375, + "harmonic_mean_rank": 39.239838303317676, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150286546675488e-05, + "inverse_geometric_mean_rank": 0.00025564644602127373, + "inverse_harmonic_mean_rank": 0.02548430466893404, + "inverse_median_rank": 9.482268069405109e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.8388671875, + "variance": 168606048.0, + "z_arithmetic_mean_rank": 38.76911253950152, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.44596579273065 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6546252806566224, + "adjusted_arithmetic_mean_rank_index": 0.34539159759475413, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.02787373950740589, + "arithmetic_mean_rank": 13396.0602, + "count": 5000.0, + "geometric_mean_rank": 3392.5214078812855, + "harmonic_mean_rank": 35.536985383870416, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464881353698307e-05, + "inverse_geometric_mean_rank": 0.00029476601022380137, + "inverse_harmonic_mean_rank": 0.028139696971985744, + "inverse_median_rank": 0.0001077179942909463, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.007916497856, + "variance": 168818254.71817595, + "z_arithmetic_mean_rank": 42.30057578570636, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747511731894 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6546256813660266, + "adjusted_arithmetic_mean_rank_index": 0.34539119686576747, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.027873739355963534, + "arithmetic_mean_rank": 13396.0684, + "count": 5000.0, + "geometric_mean_rank": 3392.5237787159485, + "harmonic_mean_rank": 35.536985575071206, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464876784295906e-05, + "inverse_geometric_mean_rank": 0.0002947658042292321, + "inverse_harmonic_mean_rank": 0.02813969682058482, + "inverse_median_rank": 0.0001077179942909463, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.011191056576, + "variance": 168818339.81092143, + "z_arithmetic_mean_rank": 42.30052670788574, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747342707806 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.654625488494084, + "adjusted_arithmetic_mean_rank_index": 0.3453913897471357, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.027873739431666374, + "arithmetic_mean_rank": 13396.064453125, + "count": 5000.0, + "geometric_mean_rank": 3392.524658203125, + "harmonic_mean_rank": 35.53698501304006, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464879308827221e-05, + "inverse_geometric_mean_rank": 0.0002947657194454223, + "inverse_harmonic_mean_rank": 0.028139696896266952, + "inverse_median_rank": 0.00010771799134090543, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.009765625, + "variance": 168818304.0, + "z_arithmetic_mean_rank": 42.300550330327596, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747427199386 + } + } + }, + "times": { + "evaluation": 3.723423957824707, + "training": 167.36736845970154 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json new file mode 100644 index 0000000..99eeec8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json new file mode 100644 index 0000000..eb2b85e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00018945981818183324, + 4.8747250159544595e-06, + 3.806483352484587e-06, + 3.5589507950760996e-06, + 3.4506096300906347e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.748950246120412, + "adjusted_arithmetic_mean_rank_index": 0.25106213822015777, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347310534693, + "arithmetic_mean_rank": 15183.129787961696, + "count": 5848.0, + "geometric_mean_rank": 5271.940706937559, + "harmonic_mean_rank": 45.24629960213599, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586257339332459e-05, + "inverse_geometric_mean_rank": 0.00018968346868622018, + "inverse_harmonic_mean_rank": 0.02210125488257148, + "inverse_median_rank": 7.812194836139214e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.650059969405, + "variance": 162171312.14987877, + "z_arithmetic_mean_rank": 33.253269759253385, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.2796617047961 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7489507100445709, + "adjusted_arithmetic_mean_rank_index": 0.25106167427311343, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347249979118, + "arithmetic_mean_rank": 15183.139192886458, + "count": 5848.0, + "geometric_mean_rank": 5271.943980987781, + "harmonic_mean_rank": 45.246299726072834, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586253259592824e-05, + "inverse_geometric_mean_rank": 0.00018968335088656127, + "inverse_harmonic_mean_rank": 0.022101254822032613, + "inverse_median_rank": 7.812194836139214e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.654306532657, + "variance": 162171420.30689073, + "z_arithmetic_mean_rank": 33.25320830930188, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.2796609772874 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7489504434858746, + "adjusted_arithmetic_mean_rank_index": 0.25106194084495925, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347280254862, + "arithmetic_mean_rank": 15183.1337890625, + "count": 5848.0, + "geometric_mean_rank": 5271.947265625, + "harmonic_mean_rank": 45.24629964608544, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586255767615512e-05, + "inverse_geometric_mean_rank": 0.00018968322547152638, + "inverse_harmonic_mean_rank": 0.022101254852300002, + "inverse_median_rank": 7.812194962752983e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.65234375, + "variance": 162171376.0, + "z_arithmetic_mean_rank": 33.25324361683795, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.27966134101723 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8731896014158192, + "adjusted_arithmetic_mean_rank_index": 0.12681665555773147, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.01581431335858354, + "arithmetic_mean_rank": 17697.850889192887, + "count": 2924.0, + "geometric_mean_rank": 8745.380971246836, + "harmonic_mean_rank": 62.166104430540265, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.6504035787229144e-05, + "inverse_geometric_mean_rank": 0.000114346076321639, + "inverse_harmonic_mean_rank": 0.016085936366132203, + "inverse_median_rank": 5.902665053271552e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.980565244461, + "variance": 157301276.4989698, + "z_arithmetic_mean_rank": 11.877203008876947, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.32938882569934 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8731901245009187, + "adjusted_arithmetic_mean_rank_index": 0.12681613244682233, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.015814313287617128, + "arithmetic_mean_rank": 17697.86149110807, + "count": 2924.0, + "geometric_mean_rank": 8745.387291427725, + "harmonic_mean_rank": 62.166104704723125, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.6504001938450566e-05, + "inverse_geometric_mean_rank": 0.0001143459936851745, + "inverse_harmonic_mean_rank": 0.016085936295185375, + "inverse_median_rank": 5.902665053271552e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.982420271721, + "variance": 157301323.0304049, + "z_arithmetic_mean_rank": 11.877154016144601, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.3293882228989 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8731898273653809, + "adjusted_arithmetic_mean_rank_index": 0.12681642959702122, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.015814313323097975, + "arithmetic_mean_rank": 17697.85546875, + "count": 2924.0, + "geometric_mean_rank": 8745.3857421875, + "harmonic_mean_rank": 62.16610392070862, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.650402090395801e-05, + "inverse_geometric_mean_rank": 0.00011434601037763059, + "inverse_harmonic_mean_rank": 0.01608593633065643, + "inverse_median_rank": 5.902664997847751e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.982421875, + "variance": 157301312.0, + "z_arithmetic_mean_rank": 11.877181846189643, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.3293885242791 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.624766004283246, + "adjusted_arithmetic_mean_rank_index": 0.37525250198790927, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380588003943, + "arithmetic_mean_rank": 12668.408686730507, + "count": 2924.0, + "geometric_mean_rank": 3178.061528576585, + "harmonic_mean_rank": 35.56621163641454, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893651244827991e-05, + "inverse_geometric_mean_rank": 0.0003146572182470891, + "inverse_harmonic_mean_rank": 0.028116573399010757, + "inverse_median_rank": 0.00011536686663590217, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.52628128213, + "variance": 154393703.36683288, + "z_arithmetic_mean_rank": 35.14487495983464, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.60145660877637 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6247664090727081, + "adjusted_arithmetic_mean_rank_index": 0.3752520971784833, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380537859197, + "arithmetic_mean_rank": 12668.416894664842, + "count": 2924.0, + "geometric_mean_rank": 3178.063179193508, + "harmonic_mean_rank": 35.56621169982792, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893646130489583e-05, + "inverse_geometric_mean_rank": 0.00031465705482097066, + "inverse_harmonic_mean_rank": 0.028116573348879844, + "inverse_median_rank": 0.00011536686663590217, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.532628786203, + "variance": 154393861.10903057, + "z_arithmetic_mean_rank": 35.144837046758546, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.6014561827437 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6247662212382751, + "adjusted_arithmetic_mean_rank_index": 0.3752522850221801, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380562929834, + "arithmetic_mean_rank": 12668.4130859375, + "count": 2924.0, + "geometric_mean_rank": 3178.061767578125, + "harmonic_mean_rank": 35.56621185760246, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893648580648005e-05, + "inverse_geometric_mean_rank": 0.00031465719803236425, + "inverse_harmonic_mean_rank": 0.028116573373943566, + "inverse_median_rank": 0.0001153668636106886, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.529296875, + "variance": 154393776.0, + "z_arithmetic_mean_rank": 35.144854639561274, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.6014563957453 + } + } + }, + "times": { + "evaluation": 2.1725149154663086, + "training": 100.6689465045929 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..4dad16b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..baea645 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0027069504231270925, + 0.0027013514650969135, + 0.002696092348934795, + 0.0026906675357372457, + 0.002684772505929381 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8925691737374529, + "adjusted_arithmetic_mean_rank_index": 0.10744588383943188, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802826417809, + "arithmetic_mean_rank": 6369.078138761131, + "count": 40876.0, + "geometric_mean_rank": 3537.461831461074, + "harmonic_mean_rank": 84.89284921971634, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.0001570085934280142, + "inverse_geometric_mean_rank": 0.00028268856249029015, + "inverse_harmonic_mean_rank": 0.0117795551591376, + "inverse_median_rank": 0.00016385384237260363, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.34208695286, + "variance": 18934177.95768727, + "z_arithmetic_mean_rank": 37.577425156574975, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.54523866731483 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8925692423063102, + "adjusted_arithmetic_mean_rank_index": 0.10744581526096397, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802744426081, + "arithmetic_mean_rank": 6369.078628045797, + "count": 40876.0, + "geometric_mean_rank": 3537.4622506253572, + "harmonic_mean_rank": 84.89284981019289, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.00015700858136631713, + "inverse_geometric_mean_rank": 0.0002826885289936928, + "inverse_harmonic_mean_rank": 0.011779555077204303, + "inverse_median_rank": 0.00016385384237260363, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.342209829061, + "variance": 18934179.027040057, + "z_arithmetic_mean_rank": 37.57740117238728, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.54523712336098 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8925692402371914, + "adjusted_arithmetic_mean_rank_index": 0.10744581733037284, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802785395542, + "arithmetic_mean_rank": 6369.07861328125, + "count": 40876.0, + "geometric_mean_rank": 3537.4609375, + "harmonic_mean_rank": 84.89284887628386, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.0001570085878483951, + "inverse_geometric_mean_rank": 0.0002826886484399438, + "inverse_harmonic_mean_rank": 0.011779555118144566, + "inverse_median_rank": 0.00016385383787564933, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.34228515625, + "variance": 18934178.0, + "z_arithmetic_mean_rank": 37.57740189612889, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.5452378948407 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.984182731634944, + "adjusted_arithmetic_mean_rank_index": 0.015819516529003597, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789023046226185e-05, + "arithmetic_mean_rank": 6925.337899990214, + "count": 20438.0, + "geometric_mean_rank": 4991.1658658403585, + "harmonic_mean_rank": 1248.3012815646675, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.0001443972863766565, + "inverse_geometric_mean_rank": 0.00020035399080684145, + "inverse_harmonic_mean_rank": 0.0008010886592590553, + "inverse_median_rank": 0.0001472211998527788, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232259464963, + "variance": 17290895.52365509, + "z_arithmetic_mean_rank": 3.90799340739639, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286023633405836 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.98418282202897, + "adjusted_arithmetic_mean_rank_index": 0.0158194261221295, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789019902602824e-05, + "arithmetic_mean_rank": 6925.33853606028, + "count": 20438.0, + "geometric_mean_rank": 4991.166433981173, + "harmonic_mean_rank": 1248.301330514943, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.00014439727311423028, + "inverse_geometric_mean_rank": 0.0002003539680006936, + "inverse_harmonic_mean_rank": 0.000801088627845558, + "inverse_median_rank": 0.0001472211998527788, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232367660528, + "variance": 17290896.423459683, + "z_arithmetic_mean_rank": 3.90797107362485, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286019482001478 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9841827996952898, + "adjusted_arithmetic_mean_rank_index": 0.015819448458984087, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789021474093581e-05, + "arithmetic_mean_rank": 6925.33837890625, + "count": 20438.0, + "geometric_mean_rank": 4991.166015625, + "harmonic_mean_rank": 1248.301271763787, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.00014439727237913758, + "inverse_geometric_mean_rank": 0.00020035398483742028, + "inverse_harmonic_mean_rank": 0.0008010886435490997, + "inverse_median_rank": 0.00014722120249643922, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232421875, + "variance": 17290896.0, + "z_arithmetic_mean_rank": 3.9079765916367535, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286021557279845 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8034636814188896, + "adjusted_arithmetic_mean_rank_index": 0.19656348812496072, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481903275268, + "arithmetic_mean_rank": 5812.8183775320485, + "count": 20438.0, + "geometric_mean_rank": 2507.1569539869356, + "harmonic_mean_rank": 43.940550500523216, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.00017203358767671844, + "inverse_geometric_mean_rank": 0.00039885815621147223, + "inverse_harmonic_mean_rank": 0.022758021659016144, + "inverse_median_rank": 0.00018950161076369148, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.506076973391, + "variance": 19958610.547794178, + "z_arithmetic_mean_rank": 48.668085046221506, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.36406123610016 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8034637287600763, + "adjusted_arithmetic_mean_rank_index": 0.19656344077722931, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481770729127, + "arithmetic_mean_rank": 5812.818720031314, + "count": 20438.0, + "geometric_mean_rank": 2507.1572627599153, + "harmonic_mean_rank": 43.94055075625994, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.0001720335775402631, + "inverse_geometric_mean_rank": 0.00039885810708945534, + "inverse_harmonic_mean_rank": 0.022758021526563053, + "inverse_median_rank": 0.00018950161076369148, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.506179077078, + "variance": 19958611.460091874, + "z_arithmetic_mean_rank": 48.66807332317242, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.3640594563414 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8034636789091726, + "adjusted_arithmetic_mean_rank_index": 0.19656349063502465, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481836952594, + "arithmetic_mean_rank": 5812.818359375, + "count": 20438.0, + "geometric_mean_rank": 2507.1572265625, + "harmonic_mean_rank": 43.94055321163615, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.00017203358584083617, + "inverse_geometric_mean_rank": 0.0003988581011071801, + "inverse_harmonic_mean_rank": 0.02275802159274003, + "inverse_median_rank": 0.00018950161756947637, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.505859375, + "variance": 19958610.0, + "z_arithmetic_mean_rank": 48.668085667700126, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.3640603455547 + } + } + }, + "times": { + "evaluation": 113.07699227333069, + "training": 29.087645292282104 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json new file mode 100644 index 0000000..407b565 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json new file mode 100644 index 0000000..cd957e6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002713467231609252, + 0.0027108328806589995, + 0.0027068490911217875, + 0.0027012186428765706, + 0.002693269666272096 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9336325239278058, + "adjusted_arithmetic_mean_rank_index": 0.06637074999464865, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.003291821314377242, + "arithmetic_mean_rank": 18927.110123119015, + "count": 5848.0, + "geometric_mean_rank": 12085.68222373154, + "harmonic_mean_rank": 280.35985102877726, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.2834267539793294e-05, + "inverse_geometric_mean_rank": 8.27425362911158e-05, + "inverse_harmonic_mean_rank": 0.0035668445261706036, + "inverse_median_rank": 5.537251861900939e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.122573466682, + "variance": 146533992.51865265, + "z_arithmetic_mean_rank": 8.79082951074308, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.54761785640589 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9336326251476224, + "adjusted_arithmetic_mean_rank_index": 0.066370648769839, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.003291821259997524, + "arithmetic_mean_rank": 18927.1121751026, + "count": 5848.0, + "geometric_mean_rank": 12085.684555207396, + "harmonic_mean_rank": 280.3598553019333, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.283426181176417e-05, + "inverse_geometric_mean_rank": 8.274252032907205e-05, + "inverse_harmonic_mean_rank": 0.0035668444718058904, + "inverse_median_rank": 5.537251861900939e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.12287157998, + "variance": 146533999.73604876, + "z_arithmetic_mean_rank": 8.790816103480935, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.547617203093296 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9336325833680886, + "adjusted_arithmetic_mean_rank_index": 0.06637069055143363, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.0032918212871810037, + "arithmetic_mean_rank": 18927.111328125, + "count": 5848.0, + "geometric_mean_rank": 12085.6796875, + "harmonic_mean_rank": 280.3598644266672, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.283426435198635e-05, + "inverse_geometric_mean_rank": 8.274255378637463e-05, + "inverse_harmonic_mean_rank": 0.0035668444989818693, + "inverse_median_rank": 5.537251854548231e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.1220703125, + "variance": 146533984.0, + "z_arithmetic_mean_rank": 8.790821637468042, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.547617529672955 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9786446205308251, + "adjusted_arithmetic_mean_rank_index": 0.02135643316860325, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0038716852646274304, + "arithmetic_mean_rank": 19835.21853625171, + "count": 2924.0, + "geometric_mean_rank": 13690.242814848845, + "harmonic_mean_rank": 241.1611840217548, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041537597240769e-05, + "inverse_geometric_mean_rank": 7.304472342268248e-05, + "inverse_harmonic_mean_rank": 0.004146604288979571, + "inverse_median_rank": 5.22875816993464e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.606562661385, + "variance": 139774025.93548405, + "z_arithmetic_mean_rank": 2.0001685990965115, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.8867339055593 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9786447892679542, + "adjusted_arithmetic_mean_rank_index": 0.02135626442314853, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0038716852510751894, + "arithmetic_mean_rank": 19835.22195622435, + "count": 2924.0, + "geometric_mean_rank": 13690.245472897186, + "harmonic_mean_rank": 241.1611848097182, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041536727983006e-05, + "inverse_geometric_mean_rank": 7.304470924058426e-05, + "inverse_harmonic_mean_rank": 0.00414660427543107, + "inverse_median_rank": 5.22875816993464e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.607099287705, + "variance": 139774038.62412804, + "z_arithmetic_mean_rank": 2.0001527949892903, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.88673379044433 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9786447274416155, + "adjusted_arithmetic_mean_rank_index": 0.021356326252537805, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.003871685257851033, + "arithmetic_mean_rank": 19835.220703125, + "count": 2924.0, + "geometric_mean_rank": 13690.232421875, + "harmonic_mean_rank": 241.16118319861056, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041537224315107e-05, + "inverse_geometric_mean_rank": 7.304477912839502e-05, + "inverse_harmonic_mean_rank": 0.004146604282205044, + "inverse_median_rank": 5.228758163866587e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.6064453125, + "variance": 139774032.0, + "z_arithmetic_mean_rank": 2.0001585857129527, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.886733847999466 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8886403950097489, + "adjusted_arithmetic_mean_rank_index": 0.11136509716598464, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.002711957429127143, + "arithmetic_mean_rank": 18019.00170998632, + "count": 2924.0, + "geometric_mean_rank": 10669.183650606674, + "harmonic_mean_rank": 334.77456423921825, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.54969701482291e-05, + "inverse_geometric_mean_rank": 9.372788328965898e-05, + "inverse_harmonic_mean_rank": 0.0029870847633616357, + "inverse_median_rank": 5.899879052479424e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.407712992797, + "variance": 151644637.3218165, + "z_arithmetic_mean_rank": 10.430076799100135, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.040947604288164 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.888640428742204, + "adjusted_arithmetic_mean_rank_index": 0.11136506343186581, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.0027119573339199534, + "arithmetic_mean_rank": 18019.00239398085, + "count": 2924.0, + "geometric_mean_rank": 10669.185695548289, + "harmonic_mean_rank": 334.77457490652637, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.549696804158507e-05, + "inverse_geometric_mean_rank": 9.372786532502189e-05, + "inverse_harmonic_mean_rank": 0.0029870846681807114, + "inverse_median_rank": 5.899879052479424e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.407582130434, + "variance": 151644634.0988315, + "z_arithmetic_mean_rank": 10.430073639677133, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.04094679540236 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8886404070005824, + "adjusted_arithmetic_mean_rank_index": 0.11136508517455967, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.0027119573815110663, + "arithmetic_mean_rank": 18019.001953125, + "count": 2924.0, + "geometric_mean_rank": 10669.185546875, + "harmonic_mean_rank": 334.77456747877113, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.549697016249411e-05, + "inverse_geometric_mean_rank": 9.372786735184491e-05, + "inverse_harmonic_mean_rank": 0.002987084715758695, + "inverse_median_rank": 5.89987903367728e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.408203125, + "variance": 151644640.0, + "z_arithmetic_mean_rank": 10.43007567602399, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.040947199739215 + } + } + }, + "times": { + "evaluation": 228.67922687530518, + "training": 11.537832736968994 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json new file mode 100644 index 0000000..03f1778 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult creates two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json new file mode 100644 index 0000000..6a1120c --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 1.9983569268922784e-05, + 1.040764747672111e-05, + 6.409794851883364e-06, + 4.78967919205319e-06, + 3.924975132708391e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.043914466668474816, + "adjusted_arithmetic_mean_rank_index": 0.956214981563262, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.17646159546294732, + "arithmetic_mean_rank": 324.3896836010902, + "count": 118142.0, + "geometric_mean_rank": 40.0771716312718, + "harmonic_mean_rank": 5.64876151716278, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.0030827120915155985, + "inverse_geometric_mean_rank": 0.02495186060534547, + "inverse_harmonic_mean_rank": 0.17702995549762082, + "inverse_median_rank": 0.024390243902439025, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2480600719877, + "variance": 728032.2520166102, + "z_arithmetic_mean_rank": 568.7732063158522, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.507692889807 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.043914504482264104, + "adjusted_arithmetic_mean_rank_index": 0.9562149437443529, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.1764615264839941, + "arithmetic_mean_rank": 324.38996292597045, + "count": 118142.0, + "geometric_mean_rank": 40.077258631969784, + "harmonic_mean_rank": 5.6487637166600075, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.003082709437061749, + "inverse_geometric_mean_rank": 0.02495180643923325, + "inverse_harmonic_mean_rank": 0.17702988656627303, + "inverse_median_rank": 0.024390243902439025, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2481634963606, + "variance": 728032.4285099121, + "z_arithmetic_mean_rank": 568.7731838205111, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.505445012585 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04391448256486668, + "adjusted_arithmetic_mean_rank_index": 0.9562149656647179, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.17646156018151202, + "arithmetic_mean_rank": 324.3898010253906, + "count": 118142.0, + "geometric_mean_rank": 40.07722091674805, + "harmonic_mean_rank": 5.648762876681754, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.0030827110167592764, + "inverse_geometric_mean_rank": 0.024951830506324768, + "inverse_harmonic_mean_rank": 0.1770299202405348, + "inverse_median_rank": 0.024390242993831635, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2481079101562, + "variance": 728032.375, + "z_arithmetic_mean_rank": 568.7731968591232, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.506543142948 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.046864358757327716, + "adjusted_arithmetic_mean_rank_index": 0.9532650422909069, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807807836110954, + "arithmetic_mean_rank": 345.2379678691744, + "count": 59071.0, + "geometric_mean_rank": 46.03301841722873, + "harmonic_mean_rank": 6.302757159725196, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.002896552792765086, + "inverse_geometric_mean_rank": 0.021723537460357607, + "inverse_harmonic_mean_rank": 0.15866072175365242, + "inverse_median_rank": 0.020833333333333332, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2229416788517, + "variance": 734831.171740544, + "z_arithmetic_mean_rank": 400.8845593870582, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.051103974374 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04686439782330793, + "adjusted_arithmetic_mean_rank_index": 0.953265003219623, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807805731536828, + "arithmetic_mean_rank": 345.2382556584449, + "count": 59071.0, + "geometric_mean_rank": 46.033083293710646, + "harmonic_mean_rank": 6.302757995183501, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.002896550378209915, + "inverse_geometric_mean_rank": 0.021723506844405247, + "inverse_harmonic_mean_rank": 0.15866070072247565, + "inverse_median_rank": 0.020833333333333332, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2230169884216, + "variance": 734831.3008547318, + "z_arithmetic_mean_rank": 400.8845429560828, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.050619755201 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.046864376441567664, + "adjusted_arithmetic_mean_rank_index": 0.9532650246042661, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807806772369895, + "arithmetic_mean_rank": 345.23809814453125, + "count": 59071.0, + "geometric_mean_rank": 46.033042907714844, + "harmonic_mean_rank": 6.30275744512894, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.0028965517412871122, + "inverse_geometric_mean_rank": 0.02172352559864521, + "inverse_harmonic_mean_rank": 0.15866071112360333, + "inverse_median_rank": 0.02083333395421505, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2229614257812, + "variance": 734831.25, + "z_arithmetic_mean_rank": 400.88455194914644, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.0508592294586 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04098058733238346, + "adjusted_arithmetic_mean_rank_index": 0.959148905664551, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.19484504281516982, + "arithmetic_mean_rank": 303.54139933300604, + "count": 59071.0, + "geometric_mean_rank": 34.891904576070026, + "harmonic_mean_rank": 5.117728501747324, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.0032944435328998746, + "inverse_geometric_mean_rank": 0.02865994310570916, + "inverse_harmonic_mean_rank": 0.19539918924158917, + "inverse_median_rank": 0.02857142857142857, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7426172750083, + "variance": 720364.0303788312, + "z_arithmetic_mean_rank": 403.4782114554786, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.717570495163 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.040980623900779035, + "adjusted_arithmetic_mean_rank_index": 0.9591488690912178, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.1948449259031866, + "arithmetic_mean_rank": 303.541670193496, + "count": 59071.0, + "geometric_mean_rank": 34.89200689003693, + "harmonic_mean_rank": 5.1177315617004036, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.0032944405931565803, + "inverse_geometric_mean_rank": 0.028659859066047018, + "inverse_harmonic_mean_rank": 0.19539907241007043, + "inverse_median_rank": 0.02857142857142857, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7427487439139, + "variance": 720364.2535453746, + "z_arithmetic_mean_rank": 403.4781960704398, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.71487235021 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04098060557075636, + "adjusted_arithmetic_mean_rank_index": 0.9591488874237154, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.1948449828898034, + "arithmetic_mean_rank": 303.5415344238281, + "count": 59071.0, + "geometric_mean_rank": 34.8919677734375, + "harmonic_mean_rank": 5.117730112424955, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.003294442081823945, + "inverse_geometric_mean_rank": 0.028659891337156296, + "inverse_harmonic_mean_rank": 0.19539912935746628, + "inverse_median_rank": 0.02857142873108387, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7426147460938, + "variance": 720364.0625, + "z_arithmetic_mean_rank": 403.4782037822386, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.716187511824 + } + } + }, + "times": { + "evaluation": 20.92709469795227, + "training": 15.750773191452026 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json new file mode 100644 index 0000000..561f095 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult create two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json new file mode 100644 index 0000000..c4a98ae --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 2.09204011979469e-05, + 1.6905181007187295e-05, + 1.3378267618860774e-05, + 1.0851657965730716e-05, + 8.871422993938076e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04896868446936024, + "adjusted_arithmetic_mean_rank_index": 0.9510777929270965, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576, + "count": 10000.0, + "geometric_mean_rank": 5.3526386004716215, + "harmonic_mean_rank": 1.6319800035619036, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.00099794662502435, + "inverse_geometric_mean_rank": 0.18682374706035448, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982390329411, + "variance": 20903022.977482237, + "z_arithmetic_mean_rank": 164.72727454841043, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04896868446936024, + "adjusted_arithmetic_mean_rank_index": 0.9510777929270965, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576, + "count": 10000.0, + "geometric_mean_rank": 5.3526386004716215, + "harmonic_mean_rank": 1.6319800035619036, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.00099794662502435, + "inverse_geometric_mean_rank": 0.18682374706035448, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982390329411, + "variance": 20903022.977482237, + "z_arithmetic_mean_rank": 164.72727454841043, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04896868530928128, + "adjusted_arithmetic_mean_rank_index": 0.9510777920871344, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576171875, + "count": 10000.0, + "geometric_mean_rank": 5.352639198303223, + "harmonic_mean_rank": 1.6319799215470276, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.000997946597635746, + "inverse_geometric_mean_rank": 0.18682372570037842, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982421875, + "variance": 20903022.0, + "z_arithmetic_mean_rank": 164.7272744029285, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04896234274102222, + "adjusted_arithmetic_mean_rank_index": 0.9510841360429304, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046, + "count": 5000.0, + "geometric_mean_rank": 5.329299858068893, + "harmonic_mean_rank": 1.629419167883981, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.000998099020605355, + "inverse_geometric_mean_rank": 0.18764190918736492, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.616893806392, + "variance": 20927119.72509884, + "z_arithmetic_mean_rank": 116.48053990240568, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04896234274102222, + "adjusted_arithmetic_mean_rank_index": 0.9510841360429304, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046, + "count": 5000.0, + "geometric_mean_rank": 5.329299858068893, + "harmonic_mean_rank": 1.629419167883981, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.000998099020605355, + "inverse_geometric_mean_rank": 0.18764190918736492, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.616893806392, + "variance": 20927119.72509884, + "z_arithmetic_mean_rank": 116.48053990240568, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.048962342841242394, + "adjusted_arithmetic_mean_rank_index": 0.9510841359427052, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046020507812, + "count": 5000.0, + "geometric_mean_rank": 5.329300403594971, + "harmonic_mean_rank": 1.6294191629877437, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.0009980989852920175, + "inverse_geometric_mean_rank": 0.18764188885688782, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.61669921875, + "variance": 20927120.0, + "z_arithmetic_mean_rank": 116.48053989013098, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04897502590366397, + "adjusted_arithmetic_mean_rank_index": 0.9510714501053755, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2106, + "count": 5000.0, + "geometric_mean_rank": 5.376079550840021, + "harmonic_mean_rank": 1.63454890125634, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977942759735329, + "inverse_geometric_mean_rank": 0.1860091523094647, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.346362779653, + "variance": 20878926.183047645, + "z_arithmetic_mean_rank": 116.47900595429857, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04897502590366397, + "adjusted_arithmetic_mean_rank_index": 0.9510714501053755, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2106, + "count": 5000.0, + "geometric_mean_rank": 5.376079550840021, + "harmonic_mean_rank": 1.63454890125634, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977942759735329, + "inverse_geometric_mean_rank": 0.1860091523094647, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.346362779653, + "variance": 20878926.183047645, + "z_arithmetic_mean_rank": 116.47900595429857, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04897502450064657, + "adjusted_arithmetic_mean_rank_index": 0.9510714515084615, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2105712890625, + "count": 5000.0, + "geometric_mean_rank": 5.376080513000488, + "harmonic_mean_rank": 1.6345490025499922, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977943263947964, + "inverse_geometric_mean_rank": 0.18600912392139435, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.34619140625, + "variance": 20878926.0, + "z_arithmetic_mean_rank": 116.4790061261364, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + } + } + }, + "times": { + "evaluation": 5.345269203186035, + "training": 5.9453136920928955 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json new file mode 100644 index 0000000..8fa85b6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json new file mode 100644 index 0000000..1584b34 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 1.8612450855828065e-05, + 1.7000744598844161e-06, + 1.2025011869850476e-06, + 1.0128473853421426e-06, + 9.114429453462934e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.03748523558448788, + "adjusted_arithmetic_mean_rank_index": 0.9626450831264751, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514178450284293, + "arithmetic_mean_rank": 276.8979025240812, + "count": 118142.0, + "geometric_mean_rank": 25.875818197160168, + "harmonic_mean_rank": 4.861676761958902, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.003611439418227562, + "inverse_geometric_mean_rank": 0.03864612096052478, + "inverse_harmonic_mean_rank": 0.20569035107901182, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3979695616902, + "variance": 935858.8315120807, + "z_arithmetic_mean_rank": 572.5979419177419, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.133990894033 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.03748524704321191, + "adjusted_arithmetic_mean_rank_index": 0.9626450716661996, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514173170088618, + "arithmetic_mean_rank": 276.8979871679843, + "count": 118142.0, + "geometric_mean_rank": 25.87585088024041, + "harmonic_mean_rank": 4.861678009119727, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.0036114383142602444, + "inverse_geometric_mean_rank": 0.038646072147665314, + "inverse_harmonic_mean_rank": 0.20569029831349603, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3980848472062, + "variance": 935859.0545660424, + "z_arithmetic_mean_rank": 572.5979351009719, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.132270190683 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.03748523777447217, + "adjusted_arithmetic_mean_rank_index": 0.9626450809361943, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514175745144342, + "arithmetic_mean_rank": 276.8979187011719, + "count": 118142.0, + "geometric_mean_rank": 25.875835418701172, + "harmonic_mean_rank": 4.8616774504878295, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.0036114391405135393, + "inverse_geometric_mean_rank": 0.038646094501018524, + "inverse_harmonic_mean_rank": 0.20569032404628168, + "inverse_median_rank": 0.04545454680919647, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3980712890625, + "variance": 935859.0, + "z_arithmetic_mean_rank": 572.597940614925, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.133109346518 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04029054472522748, + "adjusted_arithmetic_mean_rank_index": 0.95983974880713, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.1901638974649306, + "arithmetic_mean_rank": 296.8103299419343, + "count": 59071.0, + "geometric_mean_rank": 29.242573492518023, + "harmonic_mean_rank": 5.243169382262753, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.003369154975824569, + "inverse_geometric_mean_rank": 0.034196716655456434, + "inverse_harmonic_mean_rank": 0.190724336196905, + "inverse_median_rank": 0.038461538461538464, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.321567958725, + "variance": 982718.451100145, + "z_arithmetic_mean_rank": 403.64947597155924, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.279737592333 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0402905585132205, + "adjusted_arithmetic_mean_rank_index": 0.959839735017265, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.19016385703379793, + "arithmetic_mean_rank": 296.810431514618, + "count": 59071.0, + "geometric_mean_rank": 29.24260990758518, + "harmonic_mean_rank": 5.243170492978996, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.0033691538228525827, + "inverse_geometric_mean_rank": 0.03419667407116805, + "inverse_harmonic_mean_rank": 0.19072429579375227, + "inverse_median_rank": 0.038461538461538464, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.3216042325861, + "variance": 982718.5230182679, + "z_arithmetic_mean_rank": 403.6494701723915, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.278807355217 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.040290553459769014, + "adjusted_arithmetic_mean_rank_index": 0.9598397400714026, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.19016387690131012, + "arithmetic_mean_rank": 296.8103942871094, + "count": 59071.0, + "geometric_mean_rank": 29.242595672607422, + "harmonic_mean_rank": 5.243169799388333, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.0033691541757434607, + "inverse_geometric_mean_rank": 0.03419668972492218, + "inverse_harmonic_mean_rank": 0.19072431564751535, + "inverse_median_rank": 0.03846153989434242, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.3215942382812, + "variance": 982718.5, + "z_arithmetic_mean_rank": 403.6494722978504, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.279264465766 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.034695154364071244, + "adjusted_arithmetic_mean_rank_index": 0.9654351873326302, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.22011961471257302, + "arithmetic_mean_rank": 256.9854751062281, + "count": 59071.0, + "geometric_mean_rank": 22.896684094640214, + "harmonic_mean_rank": 4.531933604744529, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912705069679048, + "inverse_geometric_mean_rank": 0.04367444630264544, + "inverse_harmonic_mean_rank": 0.22065636596111857, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.4469228517189, + "variance": 888206.2023926738, + "z_arithmetic_mean_rank": 406.1226159573892, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.014994415809 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.034695163506170135, + "adjusted_arithmetic_mean_rank_index": 0.9654351781892969, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.22011954953983928, + "arithmetic_mean_rank": 256.9855428213506, + "count": 59071.0, + "geometric_mean_rank": 22.896713422380326, + "harmonic_mean_rank": 4.531934942368793, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912694816267274, + "inverse_geometric_mean_rank": 0.043674390361306305, + "inverse_harmonic_mean_rank": 0.22065630083323984, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.447120656698, + "variance": 888206.5752341007, + "z_arithmetic_mean_rank": 406.12261211112946, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.013490331473 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.03469515828527225, + "adjusted_arithmetic_mean_rank_index": 0.9654351834108997, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.2201195811734192, + "arithmetic_mean_rank": 256.9855041503906, + "count": 59071.0, + "geometric_mean_rank": 22.89670181274414, + "harmonic_mean_rank": 4.531934150162851, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912701420485973, + "inverse_geometric_mean_rank": 0.04367441311478615, + "inverse_harmonic_mean_rank": 0.220656332445048, + "inverse_median_rank": 0.05263157933950424, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.447021484375, + "variance": 888206.4375, + "z_arithmetic_mean_rank": 406.1226143076633, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.0142203848145 + } + } + }, + "times": { + "evaluation": 34.076919078826904, + "training": 4.2241315841674805 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json new file mode 100644 index 0000000..e53e97a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json new file mode 100644 index 0000000..2bd19a4 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.795345775860285e-05, + 2.2625781074151614e-06, + 2.8825618770567654e-07, + 2.830078377052869e-07, + 2.627437319533504e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05202866037834784, + "adjusted_arithmetic_mean_rank_index": 0.9480176674754901, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824278747, + "arithmetic_mean_rank": 1064.6746, + "count": 10000.0, + "geometric_mean_rank": 2.5872329490836847, + "harmonic_mean_rank": 1.2936486730420298, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.0009392541157645726, + "inverse_geometric_mean_rank": 0.386513321250863, + "inverse_harmonic_mean_rank": 0.7730074021167497, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.040841194875, + "variance": 26092081.235314842, + "z_arithmetic_mean_rank": 164.19725888705435, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.002853722233 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.052028670151974515, + "adjusted_arithmetic_mean_rank_index": 0.9480176577013858, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824037771, + "arithmetic_mean_rank": 1064.6748, + "count": 10000.0, + "geometric_mean_rank": 2.587233083873683, + "harmonic_mean_rank": 1.2936486730823467, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.000939253939324947, + "inverse_geometric_mean_rank": 0.38651330111424287, + "inverse_harmonic_mean_rank": 0.7730074020926587, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.041468042811, + "variance": 26092087.63924496, + "z_arithmetic_mean_rank": 164.19725719417332, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.002853341883 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05202866441569558, + "adjusted_arithmetic_mean_rank_index": 0.9480176634379451, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824158229, + "arithmetic_mean_rank": 1064.6746826171875, + "count": 10000.0, + "geometric_mean_rank": 2.587232828140259, + "harmonic_mean_rank": 1.2936486399760676, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.0009392540669068694, + "inverse_geometric_mean_rank": 0.386513352394104, + "inverse_harmonic_mean_rank": 0.7730074021047012, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.041015625, + "variance": 26092084.0, + "z_arithmetic_mean_rank": 164.19725818774901, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.00285353201 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05201507874896203, + "adjusted_arithmetic_mean_rank_index": 0.9480312508427307, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076970289, + "arithmetic_mean_rank": 1064.372, + "count": 5000.0, + "geometric_mean_rank": 2.602302684720317, + "harmonic_mean_rank": 1.2978888205900572, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395211448628862, + "inverse_geometric_mean_rank": 0.3842750521957346, + "inverse_harmonic_mean_rank": 0.7704820198276857, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.4860026446195, + "variance": 26096629.239216004, + "z_arithmetic_mean_rank": 116.10664898896995, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384857467 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05201509829666856, + "adjusted_arithmetic_mean_rank_index": 0.9480312312940687, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076488339, + "arithmetic_mean_rank": 1064.3724, + "count": 5000.0, + "geometric_mean_rank": 2.6023029558705306, + "harmonic_mean_rank": 1.2978888206712202, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395207917830263, + "inverse_geometric_mean_rank": 0.3842750121557145, + "inverse_harmonic_mean_rank": 0.7704820197795038, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.4872562548535, + "variance": 26096642.04731824, + "z_arithmetic_mean_rank": 116.10664659481931, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384319577 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05201508815056893, + "adjusted_arithmetic_mean_rank_index": 0.9480312414406642, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076729255, + "arithmetic_mean_rank": 1064.3721923828125, + "count": 5000.0, + "geometric_mean_rank": 2.6023025512695312, + "harmonic_mean_rank": 1.297888801348993, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395209490321577, + "inverse_geometric_mean_rank": 0.38427507877349854, + "inverse_harmonic_mean_rank": 0.7704820198035889, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.48681640625, + "variance": 26096636.0, + "z_arithmetic_mean_rank": 116.10664783748636, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384588457 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05204224137802127, + "adjusted_arithmetic_mean_rank_index": 0.9480040847380733, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9772, + "count": 5000.0, + "geometric_mean_rank": 2.57225048113251, + "harmonic_mean_rank": 1.2894361400416687, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872384122401, + "inverse_geometric_mean_rank": 0.3887646274478371, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595623018737, + "variance": 26087533.048280153, + "z_arithmetic_mean_rank": 116.10334157194073, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05204224137802127, + "adjusted_arithmetic_mean_rank_index": 0.9480040847380733, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9772, + "count": 5000.0, + "geometric_mean_rank": 2.57225048113251, + "harmonic_mean_rank": 1.2894361400416687, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872384122401, + "inverse_geometric_mean_rank": 0.3887646274478371, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595623018737, + "variance": 26087533.048280153, + "z_arithmetic_mean_rank": 116.10334157194073, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05204224005135856, + "adjusted_arithmetic_mean_rank_index": 0.9480040860648008, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9771728515625, + "count": 5000.0, + "geometric_mean_rank": 2.5722501277923584, + "harmonic_mean_rank": 1.2894360933309963, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872429892421, + "inverse_geometric_mean_rank": 0.3887646794319153, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595703125, + "variance": 26087534.0, + "z_arithmetic_mean_rank": 116.10334173442685, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + } + } + }, + "times": { + "evaluation": 7.570884943008423, + "training": 2.3045146465301514 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json new file mode 100644 index 0000000..f9417f7 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json new file mode 100644 index 0000000..b403d8e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.008025066050702042, + 0.006834184265635721, + 0.006292605368404343, + 0.00592642079037386, + 0.005657277270934352 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9204867973289346, + "adjusted_arithmetic_mean_rank_index": 0.07952396828054997, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.6141685120576216e-05, + "arithmetic_mean_rank": 6799.500110037074, + "count": 118142.0, + "geometric_mean_rank": 4816.9160901851965, + "harmonic_mean_rank": 1528.9884148398814, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00014706963509329918, + "inverse_geometric_mean_rank": 0.00020760170641908625, + "inverse_harmonic_mean_rank": 0.0006540271922889108, + "inverse_median_rank": 0.00015332720024532353, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.87870870036, + "variance": 18035978.76641244, + "z_arithmetic_mean_rank": 47.302231496041486, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.1777805690503058 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9204871410906554, + "adjusted_arithmetic_mean_rank_index": 0.07952362447228589, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.6142143183925403e-05, + "arithmetic_mean_rank": 6799.502649354167, + "count": 118142.0, + "geometric_mean_rank": 4816.919034257423, + "harmonic_mean_rank": 1528.9894849646314, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00014706958016921757, + "inverse_geometric_mean_rank": 0.00020760157953415967, + "inverse_harmonic_mean_rank": 0.0006540267345416912, + "inverse_median_rank": 0.00015332720024532353, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.879017549971, + "variance": 18035981.389706206, + "z_arithmetic_mean_rank": 47.30202699294068, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.1777954963595967 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9204869807367789, + "adjusted_arithmetic_mean_rank_index": 0.07952378484787326, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.614191435791117e-05, + "arithmetic_mean_rank": 6799.50146484375, + "count": 118142.0, + "geometric_mean_rank": 4816.91455078125, + "harmonic_mean_rank": 1528.9889966880858, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.0001470696006435901, + "inverse_geometric_mean_rank": 0.0002076017699437216, + "inverse_harmonic_mean_rank": 0.0006540269632097826, + "inverse_median_rank": 0.00015332720067817718, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.87841796875, + "variance": 18035978.0, + "z_arithmetic_mean_rank": 47.3021223871134, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.177788039406984 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9334855474142308, + "adjusted_arithmetic_mean_rank_index": 0.06652348282188125, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.608883349467907e-05, + "arithmetic_mean_rank": 6876.753720099541, + "count": 59071.0, + "geometric_mean_rank": 4918.491902159602, + "harmonic_mean_rank": 1623.3673438831504, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.0001454174514171092, + "inverse_geometric_mean_rank": 0.00020331435323923616, + "inverse_harmonic_mean_rank": 0.0006160035211796029, + "inverse_median_rank": 0.00015015015015015014, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.158683323923, + "variance": 17928099.75556737, + "z_arithmetic_mean_rank": 27.975679288367417, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.7506473935605982 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9334858415580818, + "adjusted_arithmetic_mean_rank_index": 0.06652318863809625, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.608907332910548e-05, + "arithmetic_mean_rank": 6876.75588698346, + "count": 59071.0, + "geometric_mean_rank": 4918.49423226719, + "harmonic_mean_rank": 1623.3679754868276, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.00014541740559568668, + "inverse_geometric_mean_rank": 0.00020331425692026237, + "inverse_harmonic_mean_rank": 0.0006160032815111514, + "inverse_median_rank": 0.00015015015015015014, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.158920923222, + "variance": 17928101.767633703, + "z_arithmetic_mean_rank": 27.97555557278804, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.750652911656939 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9334856389650233, + "adjusted_arithmetic_mean_rank_index": 0.06652339125865947, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.60889534720333e-05, + "arithmetic_mean_rank": 6876.75439453125, + "count": 59071.0, + "geometric_mean_rank": 4918.4921875, + "harmonic_mean_rank": 1623.3677168983902, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.00014541743439622223, + "inverse_geometric_mean_rank": 0.00020331433915998787, + "inverse_harmonic_mean_rank": 0.0006160034012852777, + "inverse_median_rank": 0.00015015015378594398, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.15869140625, + "variance": 17928100.0, + "z_arithmetic_mean_rank": 27.97564078251628, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.7506501539924884 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9075586077133286, + "adjusted_arithmetic_mean_rank_index": 0.0924538743206843, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.805311688501677e-06, + "arithmetic_mean_rank": 6722.246499974607, + "count": 59071.0, + "geometric_mean_rank": 4717.438003648486, + "harmonic_mean_rank": 1444.9804962161888, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014875979332263067, + "inverse_geometric_mean_rank": 0.00021197946835265155, + "inverse_harmonic_mean_rank": 0.0006920508633982187, + "inverse_median_rank": 0.00015683814303638644, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.159407152596, + "variance": 18131921.536722146, + "z_arithmetic_mean_rank": 38.891900551347305, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08782061726419244 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9075590008235809, + "adjusted_arithmetic_mean_rank_index": 0.09245348115735175, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.804635397057607e-06, + "arithmetic_mean_rank": 6722.249411724873, + "count": 59071.0, + "geometric_mean_rank": 4717.441535332689, + "harmonic_mean_rank": 1444.9819073210315, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014875972888715063, + "inverse_geometric_mean_rank": 0.00021197930965549883, + "inverse_harmonic_mean_rank": 0.000692050187572231, + "inverse_median_rank": 0.00015683814303638644, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.159800469012, + "variance": 18131924.8863303, + "z_arithmetic_mean_rank": 38.89173516218062, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08780500951982617 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9075588824796018, + "adjusted_arithmetic_mean_rank_index": 0.09245359951731047, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.8049731916007037e-06, + "arithmetic_mean_rank": 6722.24853515625, + "count": 59071.0, + "geometric_mean_rank": 4717.43994140625, + "harmonic_mean_rank": 1444.9812400101264, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.000148759747389704, + "inverse_geometric_mean_rank": 0.0002119793789461255, + "inverse_harmonic_mean_rank": 0.0006920505251342877, + "inverse_median_rank": 0.00015683814126532525, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.15966796875, + "variance": 18131922.0, + "z_arithmetic_mean_rank": 38.89178495180573, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08781280528735103 + } + } + }, + "times": { + "evaluation": 28.00254464149475, + "training": 13.801588296890259 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json new file mode 100644 index 0000000..b5fa871 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json new file mode 100644 index 0000000..a66458d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.1859842339811534, + 0.1488544849477889, + 0.13528452142724368, + 0.12644040748196536, + 0.1193752745758366 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6755986993844032, + "adjusted_arithmetic_mean_rank_index": 0.32441715427636864, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160908295552288, + "arithmetic_mean_rank": 13824.9336, + "count": 10000.0, + "geometric_mean_rank": 5556.814652149579, + "harmonic_mean_rank": 87.47801345454457, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233307796863487e-05, + "inverse_geometric_mean_rank": 0.00017995921451386963, + "inverse_harmonic_mean_rank": 0.011431443862401164, + "inverse_median_rank": 9.026085386767759e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.625858922253, + "variance": 143152271.94399107, + "z_arithmetic_mean_rank": 56.189256060985315, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.161387036573 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6755991538580438, + "adjusted_arithmetic_mean_rank_index": 0.3244166997805177, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160907981794924, + "arithmetic_mean_rank": 13824.9429, + "count": 10000.0, + "geometric_mean_rank": 5556.819712994077, + "harmonic_mean_rank": 87.4780158548855, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233302931037784e-05, + "inverse_geometric_mean_rank": 0.00017995905061695598, + "inverse_harmonic_mean_rank": 0.011431443548729641, + "inverse_median_rank": 9.026085386767759e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.629322851568, + "variance": 143152354.8332396, + "z_arithmetic_mean_rank": 56.189177342016805, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.16138208429393 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6755988899701234, + "adjusted_arithmetic_mean_rank_index": 0.3244169636813343, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160908138604079, + "arithmetic_mean_rank": 13824.9375, + "count": 10000.0, + "geometric_mean_rank": 5556.8173828125, + "harmonic_mean_rank": 87.4780140344442, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233305950649083e-05, + "inverse_geometric_mean_rank": 0.00017995912639889866, + "inverse_harmonic_mean_rank": 0.011431443705495894, + "inverse_median_rank": 9.026085172081366e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.626953125, + "variance": 143152304.0, + "z_arithmetic_mean_rank": 56.189223049804966, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.16138455933606 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6871711031180322, + "adjusted_arithmetic_mean_rank_index": 0.31284418534778335, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.01172720074827386, + "arithmetic_mean_rank": 14061.4164, + "count": 5000.0, + "geometric_mean_rank": 5711.947514522259, + "harmonic_mean_rank": 83.35009273134703, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111659107115269e-05, + "inverse_geometric_mean_rank": 0.00017507163668040794, + "inverse_harmonic_mean_rank": 0.01199758713194462, + "inverse_median_rank": 8.702462796971542e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.630931162526, + "variance": 144183200.57901102, + "z_arithmetic_mean_rank": 38.31444373181431, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.8836740037781 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6871715038460162, + "adjusted_arithmetic_mean_rank_index": 0.3128437846002152, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.011727200219012527, + "arithmetic_mean_rank": 14061.4246, + "count": 5000.0, + "geometric_mean_rank": 5711.953444634062, + "harmonic_mean_rank": 83.35009640724563, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111654959910676e-05, + "inverse_geometric_mean_rank": 0.0001750714549222075, + "inverse_harmonic_mean_rank": 0.011997586602828092, + "inverse_median_rank": 8.702462796971542e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.632850995855, + "variance": 144183246.68431485, + "z_arithmetic_mean_rank": 38.31439465172629, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.88366809685547 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6871713229533725, + "adjusted_arithmetic_mean_rank_index": 0.31284396550169946, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.011727200483508837, + "arithmetic_mean_rank": 14061.4208984375, + "count": 5000.0, + "geometric_mean_rank": 5711.947265625, + "harmonic_mean_rank": 83.35008958074279, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111657032510266e-05, + "inverse_geometric_mean_rank": 0.00017507164739072323, + "inverse_harmonic_mean_rank": 0.011997586867252037, + "inverse_median_rank": 8.702462946530432e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.6318359375, + "variance": 144183216.0, + "z_arithmetic_mean_rank": 38.31441680697182, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.8836710488173 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6640268322053902, + "adjusted_arithmetic_mean_rank_index": 0.3359895865979101, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615849420157, + "arithmetic_mean_rank": 13588.4508, + "count": 5000.0, + "geometric_mean_rank": 5405.8950996728145, + "harmonic_mean_rank": 92.036109949627, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359190644455216e-05, + "inverse_geometric_mean_rank": 0.00018498324173188709, + "inverse_harmonic_mean_rank": 0.010865300592857712, + "inverse_median_rank": 9.43040362127499e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.773685842129, + "variance": 142009495.07957938, + "z_arithmetic_mean_rank": 41.149098791246644, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600139198488 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6640273404221955, + "adjusted_arithmetic_mean_rank_index": 0.3359890783562687, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615751166754, + "arithmetic_mean_rank": 13588.4612, + "count": 5000.0, + "geometric_mean_rank": 5405.899334093384, + "harmonic_mean_rank": 92.03611078166908, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359185012060085e-05, + "inverse_geometric_mean_rank": 0.00018498309683521116, + "inverse_harmonic_mean_rank": 0.01086530049463119, + "inverse_median_rank": 9.43040362127499e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.778750706693, + "variance": 142009615.79329458, + "z_arithmetic_mean_rank": 41.14903654620588, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600029538334 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6640270889862069, + "adjusted_arithmetic_mean_rank_index": 0.33598932980454477, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615800288752, + "arithmetic_mean_rank": 13588.4560546875, + "count": 5000.0, + "geometric_mean_rank": 5405.8955078125, + "harmonic_mean_rank": 92.03610861232633, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359188020927832e-05, + "inverse_geometric_mean_rank": 0.00018498323333915323, + "inverse_harmonic_mean_rank": 0.010865300543739749, + "inverse_median_rank": 9.430403588339686e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.7763671875, + "variance": 142009552.0, + "z_arithmetic_mean_rank": 41.1490673414161, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600084363162 + } + } + }, + "times": { + "evaluation": 7.501351594924927, + "training": 198.48107266426086 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..2acde2b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "The number of epochs has been inferred from the convergence experiment plot, Figure 2b. The publication favours SGD optimizer over others to ensure comaparabiltiy to RSGD, which is used to optimize the hyperbolic models studied in the same paper. The evaluation uses the non-deterministic variant of rank computation. While it is not guaranteed to yield the same results as average ranking definition due to the unknown properties of the employed sort algorithm (e.g. stability), it is closest to the average rank definition.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with MuRE as described by Bala\u017eevi\u0107 et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..26d64c8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.000975197495752765, + 0.0007105042391938026, + 0.0006824577425091475, + 0.0006504202312117928, + 0.0006158363464985977 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1223330365810328, + "adjusted_arithmetic_mean_rank_index": 0.8777899778110253, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.1809697637390021, + "arithmetic_mean_rank": 872.9280506898914, + "count": 40876.0, + "geometric_mean_rank": 68.13149559739709, + "harmonic_mean_rank": 5.508020077803989, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.001145569785745436, + "inverse_geometric_mean_rank": 0.014677499609126505, + "inverse_harmonic_mean_rank": 0.18155344132272905, + "inverse_median_rank": 0.014285714285714285, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1981.9991308986464, + "variance": 3928320.5548829897, + "z_arithmetic_mean_rank": 306.9925623551912, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.770157360449 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12233337256843348, + "adjusted_arithmetic_mean_rank_index": 0.8777896417765324, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.18096974666989954, + "arithmetic_mean_rank": 872.9304481847539, + "count": 40876.0, + "geometric_mean_rank": 68.13161233034441, + "harmonic_mean_rank": 5.508020595282298, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.0011455666394493231, + "inverse_geometric_mean_rank": 0.014677474461508093, + "inverse_harmonic_mean_rank": 0.18155342426579069, + "inverse_median_rank": 0.014285714285714285, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1982.0041488880931, + "variance": 3928340.446209614, + "z_arithmetic_mean_rank": 306.9924448326714, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.7698359389187 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.12233320609058031, + "adjusted_arithmetic_mean_rank_index": 0.8777898082777192, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.1809697551097282, + "arithmetic_mean_rank": 872.9292602539062, + "count": 40876.0, + "geometric_mean_rank": 68.13153839111328, + "harmonic_mean_rank": 5.508020259533576, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.001145568210631609, + "inverse_geometric_mean_rank": 0.01467749010771513, + "inverse_harmonic_mean_rank": 0.18155343269960478, + "inverse_median_rank": 0.014285714365541935, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1982.0015869140625, + "variance": 3928330.5, + "z_arithmetic_mean_rank": 306.9925030637145, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.769994866 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.17011602215789823, + "adjusted_arithmetic_mean_rank_index": 0.8300019321687586, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.10238267347048406, + "arithmetic_mean_rank": 1197.0449163323221, + "count": 20438.0, + "geometric_mean_rank": 153.8541949078438, + "harmonic_mean_rank": 9.705733847342007, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353905407860078, + "inverse_geometric_mean_rank": 0.00649966028289956, + "inverse_harmonic_mean_rank": 0.1030318794774965, + "inverse_median_rank": 0.0043859649122807015, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.4322111858924, + "variance": 5123125.374633859, + "z_arithmetic_mean_rank": 205.04053161769326, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0445282016553 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.17011651584834828, + "adjusted_arithmetic_mean_rank_index": 0.8300014384081386, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.1023826397223952, + "arithmetic_mean_rank": 1197.0483902534495, + "count": 20438.0, + "geometric_mean_rank": 153.8546405130579, + "harmonic_mean_rank": 9.705737024156553, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353881164221534, + "inverse_geometric_mean_rank": 0.006499641458101671, + "inverse_harmonic_mean_rank": 0.10303184575381609, + "inverse_median_rank": 0.0043859649122807015, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.438376850168, + "variance": 5123153.285798122, + "z_arithmetic_mean_rank": 205.04040964094105, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0440825313349 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.17011626581502265, + "adjusted_arithmetic_mean_rank_index": 0.8300016884770023, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.1023826564070964, + "arithmetic_mean_rank": 1197.046630859375, + "count": 20438.0, + "geometric_mean_rank": 153.8544464111328, + "harmonic_mean_rank": 9.705735764932905, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353893645107746, + "inverse_geometric_mean_rank": 0.006499649491161108, + "inverse_harmonic_mean_rank": 0.10303186242645, + "inverse_median_rank": 0.004385964944958687, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.435302734375, + "variance": 5123139.0, + "z_arithmetic_mean_rank": 205.0404714170054, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0443028660675 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0758581855656241, + "adjusted_arithmetic_mean_rank_index": 0.9242695695064783, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.2595551852130943, + "arithmetic_mean_rank": 548.8111850474606, + "count": 20438.0, + "geometric_mean_rank": 30.1707775671542, + "harmonic_mean_rank": 3.8450446518083097, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.0018221202979190757, + "inverse_geometric_mean_rank": 0.03314465455105349, + "inverse_harmonic_mean_rank": 0.2600750031679616, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.5251807712627, + "variance": 2523412.249944373, + "z_arithmetic_mean_rank": 228.8442805094061, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168288279998 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07585836816734498, + "adjusted_arithmetic_mean_rank_index": 0.9242693868795142, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.25955518482262385, + "arithmetic_mean_rank": 548.8125061160583, + "count": 20438.0, + "geometric_mean_rank": 30.170793570171003, + "harmonic_mean_rank": 3.845044657577116, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.001822115911820217, + "inverse_geometric_mean_rank": 0.03314463697065865, + "inverse_harmonic_mean_rank": 0.2600750027777653, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.5284781455484, + "variance": 2523422.725879412, + "z_arithmetic_mean_rank": 228.84423529193106, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168283036969 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.075858274521055, + "adjusted_arithmetic_mean_rank_index": 0.92426948053875, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.25955518501775515, + "arithmetic_mean_rank": 548.8118286132812, + "count": 20438.0, + "geometric_mean_rank": 30.170780181884766, + "harmonic_mean_rank": 3.845044527972988, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.0018221181817352772, + "inverse_geometric_mean_rank": 0.03314465284347534, + "inverse_harmonic_mean_rank": 0.2600750029727596, + "inverse_median_rank": 0.0476190485060215, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.52685546875, + "variance": 2523417.75, + "z_arithmetic_mean_rank": 228.8442584814649, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168285657088 + } + } + }, + "times": { + "evaluation": 14.712653875350952, + "training": 66.19852590560913 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json new file mode 100644 index 0000000..1e46a74 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json new file mode 100644 index 0000000..734222d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 6.981585166240256e-05, + 4.908339620017992e-05, + 4.2677227727666245e-05, + 3.223726586237798e-05, + 2.246727524548819e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.19171643534750452, + "adjusted_arithmetic_mean_rank_index": 0.808393001386489, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.09786852485330108, + "arithmetic_mean_rank": 1416.1810109867786, + "count": 118142.0, + "geometric_mean_rank": 142.07525085299747, + "harmonic_mean_rank": 10.153199039470467, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061244235319972, + "inverse_geometric_mean_rank": 0.007038523557031624, + "inverse_harmonic_mean_rank": 0.09849112541894524, + "inverse_median_rank": 0.006172839506172839, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.2438173108694, + "variance": 8295804.447157488, + "z_arithmetic_mean_rank": 480.846136305248, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.3268537224485 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.19171644451448372, + "adjusted_arithmetic_mean_rank_index": 0.8083929922182685, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.09786852189072592, + "arithmetic_mean_rank": 1416.181078701901, + "count": 118142.0, + "geometric_mean_rank": 142.07527498310955, + "harmonic_mean_rank": 10.153199344664023, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061243897684464, + "inverse_geometric_mean_rank": 0.007038522361606436, + "inverse_harmonic_mean_rank": 0.09849112245841468, + "inverse_median_rank": 0.006172839506172839, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.2439285569008, + "variance": 8295805.087988889, + "z_arithmetic_mean_rank": 480.84613085183196, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.3267571784318 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.19171643795844864, + "adjusted_arithmetic_mean_rank_index": 0.8083929987751913, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.0978685233581836, + "arithmetic_mean_rank": 1416.1810302734375, + "count": 118142.0, + "geometric_mean_rank": 142.07528686523438, + "harmonic_mean_rank": 10.153199094165512, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061244104988873, + "inverse_geometric_mean_rank": 0.007038521580398083, + "inverse_harmonic_mean_rank": 0.0984911239248596, + "inverse_median_rank": 0.006172839552164078, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.243896484375, + "variance": 8295805.5, + "z_arithmetic_mean_rank": 480.846134752003, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.326804999753 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1892347211170444, + "adjusted_arithmetic_mean_rank_index": 0.8108753512298704, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794790708412, + "arithmetic_mean_rank": 1394.0446919808367, + "count": 59071.0, + "geometric_mean_rank": 143.2472431101072, + "harmonic_mean_rank": 10.589286510880694, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173371167742637, + "inverse_geometric_mean_rank": 0.00698093714258325, + "inverse_harmonic_mean_rank": 0.09443506878130845, + "inverse_median_rank": 0.006211180124223602, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.4004548954404, + "variance": 8153311.757817088, + "z_arithmetic_mean_rank": 341.0042259751849, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3277329424395 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.18923473260703857, + "adjusted_arithmetic_mean_rank_index": 0.8108753397383164, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794198527219, + "arithmetic_mean_rank": 1394.0447766247398, + "count": 59071.0, + "geometric_mean_rank": 143.24729012298587, + "harmonic_mean_rank": 10.58928717445167, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173370732188383, + "inverse_geometric_mean_rank": 0.006980934851482661, + "inverse_harmonic_mean_rank": 0.09443506286359465, + "inverse_median_rank": 0.006211180124223602, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.4005966875957, + "variance": 8153312.5675638765, + "z_arithmetic_mean_rank": 341.0042211425451, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3275966937404 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1892347523240481, + "adjusted_arithmetic_mean_rank_index": 0.8108753200186299, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794491851856, + "arithmetic_mean_rank": 1394.044921875, + "count": 59071.0, + "geometric_mean_rank": 143.24728393554688, + "harmonic_mean_rank": 10.5892863676429, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173370104283094, + "inverse_geometric_mean_rank": 0.006980935111641884, + "inverse_harmonic_mean_rank": 0.09443506579481109, + "inverse_median_rank": 0.006211180239915848, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.400390625, + "variance": 8153312.0, + "z_arithmetic_mean_rank": 341.00421284965967, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3276641816997 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.19418467821182647, + "adjusted_arithmetic_mean_rank_index": 0.8059241281789679, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.1019290863931256, + "arithmetic_mean_rank": 1438.3173299927207, + "count": 59071.0, + "geometric_mean_rank": 140.9128473727532, + "harmonic_mean_rank": 9.751608771153105, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952568665810771, + "inverse_geometric_mean_rank": 0.0070965850072898275, + "inverse_harmonic_mean_rank": 0.10254718205658207, + "inverse_median_rank": 0.006134969325153374, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.70602699475, + "variance": 8437317.103259625, + "z_arithmetic_mean_rank": 339.02225596678153, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632272404034 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.19418468506840061, + "adjusted_arithmetic_mean_rank_index": 0.8059241213214678, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.10192908638977595, + "arithmetic_mean_rank": 1438.3173807790624, + "count": 59071.0, + "geometric_mean_rank": 140.91284899139066, + "harmonic_mean_rank": 9.751608771471417, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952568420318689, + "inverse_geometric_mean_rank": 0.007096584925772787, + "inverse_harmonic_mean_rank": 0.10254718205323472, + "inverse_median_rank": 0.006134969325153374, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.7061084857787, + "variance": 8437317.576674597, + "z_arithmetic_mean_rank": 339.02225308208676, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632271630986 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1941846688624345, + "adjusted_arithmetic_mean_rank_index": 0.8059241375296222, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.10192908639145049, + "arithmetic_mean_rank": 1438.3172607421875, + "count": 59071.0, + "geometric_mean_rank": 140.9127655029297, + "harmonic_mean_rank": 9.751608993533074, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952569237910211, + "inverse_geometric_mean_rank": 0.0070965890772640705, + "inverse_harmonic_mean_rank": 0.1025471820549081, + "inverse_median_rank": 0.0061349691823124886, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.705810546875, + "variance": 8437316.0, + "z_arithmetic_mean_rank": 339.0222599002535, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632272017444 + } + } + }, + "times": { + "evaluation": 143.83830213546753, + "training": 7.28100323677063 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json new file mode 100644 index 0000000..45d8a38 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json new file mode 100644 index 0000000..1630779 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 6.97649384102165e-05, + 4.809209213251929e-05, + 4.411240719934941e-05, + 3.876147011645326e-05, + 3.176101996801703e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.28883692078052037, + "adjusted_arithmetic_mean_rank_index": 0.7112627563151099, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011291035, + "arithmetic_mean_rank": 2061.044647225756, + "count": 40876.0, + "geometric_mean_rank": 312.6961306089733, + "harmonic_mean_rank": 16.044739074726174, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.00048519084792560796, + "inverse_geometric_mean_rank": 0.003197992882267228, + "inverse_harmonic_mean_rank": 0.062325725294916735, + "inverse_median_rank": 0.002197802197802198, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.958078763932, + "variance": 10536243.849092837, + "z_arithmetic_mean_rank": 248.7524141179013, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389978492628 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2888369413511776, + "adjusted_arithmetic_mean_rank_index": 0.7112627357415695, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011019156, + "arithmetic_mean_rank": 2061.044794011156, + "count": 40876.0, + "geometric_mean_rank": 312.696136716117, + "harmonic_mean_rank": 16.044739075425582, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.0004851908133708361, + "inverse_geometric_mean_rank": 0.0031979928198085025, + "inverse_harmonic_mean_rank": 0.06232572529219989, + "inverse_median_rank": 0.002197802197802198, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.958340975742, + "variance": 10536245.55134999, + "z_arithmetic_mean_rank": 248.75240692264498, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389977980665 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2888369250560297, + "adjusted_arithmetic_mean_rank_index": 0.7112627520390014, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011155085, + "arithmetic_mean_rank": 2061.044677734375, + "count": 40876.0, + "geometric_mean_rank": 312.696044921875, + "harmonic_mean_rank": 16.04473833353234, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.0004851908306591213, + "inverse_geometric_mean_rank": 0.0031979938503354788, + "inverse_harmonic_mean_rank": 0.06232572529355821, + "inverse_median_rank": 0.002197802299633622, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.9580078125, + "variance": 10536244.0, + "z_arithmetic_mean_rank": 248.75241262240291, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389978236628 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.280102424914663, + "adjusted_arithmetic_mean_rank_index": 0.7199998966579886, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280967176, + "arithmetic_mean_rank": 1970.9794500440357, + "count": 20438.0, + "geometric_mean_rank": 345.32466566691244, + "harmonic_mean_rank": 20.285374139606326, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619615758337, + "inverse_geometric_mean_rank": 0.002895825579295756, + "inverse_harmonic_mean_rank": 0.04929660124175589, + "inverse_median_rank": 0.0020964360587002098, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9989276446954, + "variance": 9815682.28062281, + "z_arithmetic_mean_rank": 177.86604567255606, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138905064652 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2801024457748228, + "adjusted_arithmetic_mean_rank_index": 0.7199998757948638, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280770392, + "arithmetic_mean_rank": 1970.9795968294354, + "count": 20438.0, + "geometric_mean_rank": 345.3246714815715, + "harmonic_mean_rank": 20.285374140415495, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619237909026, + "inverse_geometric_mean_rank": 0.002895825530535155, + "inverse_harmonic_mean_rank": 0.049296601239789475, + "inverse_median_rank": 0.0020964360587002098, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9992511936284, + "variance": 9815684.307979835, + "z_arithmetic_mean_rank": 177.86604051860877, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138904804784 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2801024309038104, + "adjusted_arithmetic_mean_rank_index": 0.7199998906679899, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280868778, + "arithmetic_mean_rank": 1970.9794921875, + "count": 20438.0, + "geometric_mean_rank": 345.3247375488281, + "harmonic_mean_rank": 20.2853743489536, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619540780783, + "inverse_geometric_mean_rank": 0.0028958250768482685, + "inverse_harmonic_mean_rank": 0.04929660124077262, + "inverse_median_rank": 0.0020964359864592552, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9990234375, + "variance": 9815683.0, + "z_arithmetic_mean_rank": 177.86604419280948, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138904934711 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.297332296051859, + "adjusted_arithmetic_mean_rank_index": 0.702764842029959, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526033128848, + "arithmetic_mean_rank": 2151.1098444074764, + "count": 20438.0, + "geometric_mean_rank": 283.1505531439164, + "harmonic_mean_rank": 13.270546071704294, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.000464876306804988, + "inverse_geometric_mean_rank": 0.0035316900811128977, + "inverse_harmonic_mean_rank": 0.0753548493480776, + "inverse_median_rank": 0.002331002331002331, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.6977105125507, + "variance": 11240581.9380761, + "z_arithmetic_mean_rank": 174.00087587815491, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038917634 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.29733231634093904, + "adjusted_arithmetic_mean_rank_index": 0.7027648217380741, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526032781874, + "arithmetic_mean_rank": 2151.109991192876, + "count": 20438.0, + "geometric_mean_rank": 283.15055943636077, + "harmonic_mean_rank": 13.270546072314913, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.00046487627508320033, + "inverse_geometric_mean_rank": 0.0035316900026282805, + "inverse_harmonic_mean_rank": 0.07535484934461029, + "inverse_median_rank": 0.002331002331002331, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.697915893017, + "variance": 11240583.31523338, + "z_arithmetic_mean_rank": 174.00087085399105, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038451735 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2973322986606437, + "adjusted_arithmetic_mean_rank_index": 0.7027648394208137, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526032955348, + "arithmetic_mean_rank": 2151.10986328125, + "count": 20438.0, + "geometric_mean_rank": 283.1506042480469, + "harmonic_mean_rank": 13.270544968026048, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.00046487629879266024, + "inverse_geometric_mean_rank": 0.0035316895227879286, + "inverse_harmonic_mean_rank": 0.0753548493463438, + "inverse_median_rank": 0.0023310023825615644, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.697998046875, + "variance": 11240584.0, + "z_arithmetic_mean_rank": 174.00087523214427, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038684666 + } + } + }, + "times": { + "evaluation": 33.202207803726196, + "training": 4.496615648269653 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json new file mode 100644 index 0000000..2af423b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json new file mode 100644 index 0000000..71c3e23 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.946888238228267e-05, + 5.3340319112503506e-05, + 5.038819471869176e-05, + 5.192173830135024e-05, + 5.254753102436471e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.48480670674081655, + "adjusted_arithmetic_mean_rank_index": 0.5152184710241546, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.1720038860548289, + "arithmetic_mean_rank": 9920.7126, + "count": 10000.0, + "geometric_mean_rank": 314.4064429963176, + "harmonic_mean_rank": 5.806175348825523, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079921073411601, + "inverse_geometric_mean_rank": 0.0031805963976753245, + "inverse_harmonic_mean_rank": 0.17223041674107908, + "inverse_median_rank": 0.0024691358024691358, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.096689350818, + "variance": 236516614.98040128, + "z_arithmetic_mean_rank": 89.23616465442358, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.8725122286223 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4848067165144432, + "adjusted_arithmetic_mean_rank_index": 0.5152184612500503, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.17200388599540753, + "arithmetic_mean_rank": 9920.7128, + "count": 10000.0, + "geometric_mean_rank": 314.40646818713793, + "harmonic_mean_rank": 5.806175350828168, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079920870201988, + "inverse_geometric_mean_rank": 0.0031805961428401333, + "inverse_harmonic_mean_rank": 0.17223041668167396, + "inverse_median_rank": 0.0024691358024691358, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.0967778643, + "variance": 236516617.70291612, + "z_arithmetic_mean_rank": 89.23616296154255, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.872511290728 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48480672094311783, + "adjusted_arithmetic_mean_rank_index": 0.5152184568211593, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.17200388602510677, + "arithmetic_mean_rank": 9920.712890625, + "count": 10000.0, + "geometric_mean_rank": 314.4066162109375, + "harmonic_mean_rank": 5.806175241595575, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079921048600227, + "inverse_geometric_mean_rank": 0.0031805946491658688, + "inverse_harmonic_mean_rank": 0.1722304167113651, + "inverse_median_rank": 0.0024691359139978886, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.0966796875, + "variance": 236516624.0, + "z_arithmetic_mean_rank": 89.23616219445582, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.872511759495 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4848100783439825, + "adjusted_arithmetic_mean_rank_index": 0.5152150998399522, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.17161366984021761, + "arithmetic_mean_rank": 9920.5516, + "count": 5000.0, + "geometric_mean_rank": 315.7243009873847, + "harmonic_mean_rank": 5.819356283506532, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084659808634, + "inverse_geometric_mean_rank": 0.0031673203388926235, + "inverse_harmonic_mean_rank": 0.17184031210363296, + "inverse_median_rank": 0.0024509803921568627, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.899898307129, + "variance": 236633609.2813375, + "z_arithmetic_mean_rank": 63.09907895732225, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.327289103086 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.48481008811783566, + "adjusted_arithmetic_mean_rank_index": 0.5152150900656214, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.1716136697215684, + "arithmetic_mean_rank": 9920.5518, + "count": 5000.0, + "geometric_mean_rank": 315.7243496164708, + "harmonic_mean_rank": 5.819356287523477, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084456592426, + "inverse_geometric_mean_rank": 0.0031673198510496882, + "inverse_harmonic_mean_rank": 0.1718403119850162, + "inverse_median_rank": 0.0024509803921568627, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.899786207956, + "variance": 236633605.83251676, + "z_arithmetic_mean_rank": 63.099077760246935, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.3272877788788 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48481008605616355, + "adjusted_arithmetic_mean_rank_index": 0.5152150921273941, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.17161366978087017, + "arithmetic_mean_rank": 9920.5517578125, + "count": 5000.0, + "geometric_mean_rank": 315.7243347167969, + "harmonic_mean_rank": 5.819356435439596, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084757646546, + "inverse_geometric_mean_rank": 0.003167320042848587, + "inverse_harmonic_mean_rank": 0.17184031204430175, + "inverse_median_rank": 0.0024509804788976908, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.900390625, + "variance": 236633616.0, + "z_arithmetic_mean_rank": 63.09907801275501, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.3272884407274 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4848033352939752, + "adjusted_arithmetic_mean_rank_index": 0.5152218420520442, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226489954, + "arithmetic_mean_rank": 9920.8736, + "count": 5000.0, + "geometric_mean_rank": 313.0940858478499, + "harmonic_mean_rank": 5.793053989259962, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757492324062, + "inverse_geometric_mean_rank": 0.003193928104045876, + "inverse_harmonic_mean_rank": 0.1726205213785252, + "inverse_median_rank": 0.002509410288582183, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.29253795267, + "variance": 236399620.62762305, + "z_arithmetic_mean_rank": 63.099915365470764, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.082340135151 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4848033450673752, + "adjusted_arithmetic_mean_rank_index": 0.5152218322781665, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226470606, + "arithmetic_mean_rank": 9920.8738, + "count": 5000.0, + "geometric_mean_rank": 313.0940877952246, + "harmonic_mean_rank": 5.793053989266453, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757289121045, + "inverse_geometric_mean_rank": 0.0031939280841803627, + "inverse_harmonic_mean_rank": 0.17262052137833178, + "inverse_median_rank": 0.002509410288582183, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.292827178075, + "variance": 236399629.52147353, + "z_arithmetic_mean_rank": 63.09991416845076, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.0823401329917 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48480335598609564, + "adjusted_arithmetic_mean_rank_index": 0.5152218213589126, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226480278, + "arithmetic_mean_rank": 9920.8740234375, + "count": 5000.0, + "geometric_mean_rank": 313.0939636230469, + "harmonic_mean_rank": 5.793054034867086, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757339553908, + "inverse_geometric_mean_rank": 0.003193929325789213, + "inverse_harmonic_mean_rank": 0.17262052137842848, + "inverse_median_rank": 0.002509410260245204, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.29296875, + "variance": 236399632.0, + "z_arithmetic_mean_rank": 63.09991283115496, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.0823401340713 + } + } + }, + "times": { + "evaluation": 50.53933072090149, + "training": 13.76006269454956 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json new file mode 100644 index 0000000..669b607 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json new file mode 100644 index 0000000..d822612 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.776860787075418e-05, + 5.3628090098298616e-05, + 4.521653484294621e-05, + 4.545193286306426e-05, + 4.5890227263061786e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.135452151724743, + "adjusted_arithmetic_mean_rank_index": -0.135458833609716, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.003784262744786251, + "arithmetic_mean_rank": 23018.508207934337, + "count": 5848.0, + "geometric_mean_rank": 13912.042658983803, + "harmonic_mean_rank": 246.35699128213304, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.3443301840703396e-05, + "inverse_geometric_mean_rank": 7.188017061996592e-05, + "inverse_harmonic_mean_rank": 0.004059150076462736, + "inverse_median_rank": 4.062150908906266e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.646932078893, + "variance": 194062923.94623905, + "z_arithmetic_mean_rank": -17.94157082876319, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.46376082000466 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.1354521938996665, + "adjusted_arithmetic_mean_rank_index": -0.1354588757867199, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.0037842626820186074, + "arithmetic_mean_rank": 23018.509062927496, + "count": 5848.0, + "geometric_mean_rank": 13912.04449785986, + "harmonic_mean_rank": 246.35699509056116, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.344330022705736e-05, + "inverse_geometric_mean_rank": 7.188016111893788e-05, + "inverse_harmonic_mean_rank": 0.004059150013712412, + "inverse_median_rank": 4.062150908906266e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.646769802916, + "variance": 194062919.42502043, + "z_arithmetic_mean_rank": -17.941576415122427, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.46376006592036 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.1354522285621818, + "adjusted_arithmetic_mean_rank_index": -0.13545891045094516, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.003784262713393222, + "arithmetic_mean_rank": 23018.509765625, + "count": 5848.0, + "geometric_mean_rank": 13912.0439453125, + "harmonic_mean_rank": 246.35699470836892, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.344329863670282e-05, + "inverse_geometric_mean_rank": 7.188016752479598e-05, + "inverse_harmonic_mean_rank": 0.00405915004507837, + "inverse_median_rank": 4.062150765093975e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.6474609375, + "variance": 194062928.0, + "z_arithmetic_mean_rank": -17.94158100641141, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.4637604428519 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.1362039608020613, + "adjusted_arithmetic_mean_rank_index": -0.13621068126497105, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.004292715311898555, + "arithmetic_mean_rank": 23028.639192886458, + "count": 2924.0, + "geometric_mean_rank": 14080.544453702405, + "harmonic_mean_rank": 218.9372805893627, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.3424189837013896e-05, + "inverse_geometric_mean_rank": 7.101998103042495e-05, + "inverse_harmonic_mean_rank": 0.004567518137194704, + "inverse_median_rank": 4.088808929958703e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.226635277537, + "variance": 191690300.58219853, + "z_arithmetic_mean_rank": -12.757014496609344, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.463032644858544 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.136203994549487, + "adjusted_arithmetic_mean_rank_index": -0.1362107150140619, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.004292715310077607, + "arithmetic_mean_rank": 23028.639876880985, + "count": 2924.0, + "geometric_mean_rank": 14080.544938681376, + "harmonic_mean_rank": 218.9372806766231, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.342418854723263e-05, + "inverse_geometric_mean_rank": 7.101997858426982e-05, + "inverse_harmonic_mean_rank": 0.004567518135374258, + "inverse_median_rank": 4.088808929958703e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.226529869527, + "variance": 191690297.663403, + "z_arithmetic_mean_rank": -12.75701765743078, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.46303262939111 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.136204031460734, + "adjusted_arithmetic_mean_rank_index": -0.13621075192713006, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0042927153109880555, + "arithmetic_mean_rank": 23028.640625, + "count": 2924.0, + "geometric_mean_rank": 14080.5458984375, + "harmonic_mean_rank": 218.93728156515598, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.342418833402917e-05, + "inverse_geometric_mean_rank": 7.101997471181676e-05, + "inverse_harmonic_mean_rank": 0.0045675181362844555, + "inverse_median_rank": 4.088808782398701e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.2265625, + "variance": 191690288.0, + "z_arithmetic_mean_rank": -12.757021114579231, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.463032637124606 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.1347006761552578, + "adjusted_arithmetic_mean_rank_index": -0.13470731949585657, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758102346691497, + "arithmetic_mean_rank": 23008.377222982217, + "count": 2924.0, + "geometric_mean_rank": 13745.557324276184, + "harmonic_mean_rank": 281.6281020828012, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.346243067508199e-05, + "inverse_geometric_mean_rank": 7.275077877227202e-05, + "inverse_harmonic_mean_rank": 0.00355078201573077, + "inverse_median_rank": 4.030389134070895e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.539305947788, + "variance": 196435342.03656742, + "z_arithmetic_mean_rank": -12.61623007115594, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.831473742158146 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.1347007267539404, + "adjusted_arithmetic_mean_rank_index": -0.1347073700970347, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758101109548166, + "arithmetic_mean_rank": 23008.378248974008, + "count": 2924.0, + "geometric_mean_rank": 13745.560484575846, + "harmonic_mean_rank": 281.6281118924212, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.346242873700115e-05, + "inverse_geometric_mean_rank": 7.275076204583429e-05, + "inverse_harmonic_mean_rank": 0.003550781892050567, + "inverse_median_rank": 4.030389134070895e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.539087735984, + "variance": 196435335.9198552, + "z_arithmetic_mean_rank": -12.61623481029045, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.831472691073998 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.13470066284675, + "adjusted_arithmetic_mean_rank_index": -0.13470730618669258, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758101727935946, + "arithmetic_mean_rank": 23008.376953125, + "count": 2924.0, + "geometric_mean_rank": 13745.55859375, + "harmonic_mean_rank": 281.62810942326075, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.3462430767249316e-05, + "inverse_geometric_mean_rank": 7.27507722331211e-05, + "inverse_harmonic_mean_rank": 0.003550781953872285, + "inverse_median_rank": 4.03038902732078e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.5390625, + "variance": 196435328.0, + "z_arithmetic_mean_rank": -12.61622882466483, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.83147321645984 + } + } + }, + "times": { + "evaluation": 13.765980958938599, + "training": 4.9371349811553955 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json new file mode 100644 index 0000000..5dda922 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json new file mode 100644 index 0000000..df07c04 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002959653189315609, + 0.002774793042292991, + 0.0025934358465079417, + 0.002415573638258217, + 0.0022410840366893685 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.46136554119551004, + "adjusted_arithmetic_mean_rank_index": 0.5387073866711023, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.06391836136475008, + "arithmetic_mean_rank": 3408.039156269574, + "count": 118142.0, + "geometric_mean_rank": 767.5402133699725, + "harmonic_mean_rank": 15.488413386485405, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.0002934238587489106, + "inverse_geometric_mean_rank": 0.0013028633322147206, + "inverse_harmonic_mean_rank": 0.06456439242979928, + "inverse_median_rank": 0.0005056890012642225, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6833542856266, + "variance": 13910537.923235284, + "z_arithmetic_mean_rank": 320.43246915252945, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.963308705239 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4613685399435877, + "adjusted_arithmetic_mean_rank_index": 0.5387043875170122, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.0639183208909707, + "arithmetic_mean_rank": 3408.061307579015, + "count": 118142.0, + "geometric_mean_rank": 767.5465563380474, + "harmonic_mean_rank": 15.488423089084028, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.00029342195158759337, + "inverse_geometric_mean_rank": 0.0013028525654143826, + "inverse_harmonic_mean_rank": 0.06456435198395262, + "inverse_median_rank": 0.0005056890012642225, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6976699738793, + "variance": 13910644.709408585, + "z_arithmetic_mean_rank": 320.43068520381394, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.961989750941 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4613670488334821, + "adjusted_arithmetic_mean_rank_index": 0.5387058788290051, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.0639183407927078, + "arithmetic_mean_rank": 3408.05029296875, + "count": 118142.0, + "geometric_mean_rank": 767.54345703125, + "harmonic_mean_rank": 15.48841803698083, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.00029342289781197906, + "inverse_geometric_mean_rank": 0.0013028578832745552, + "inverse_harmonic_mean_rank": 0.06456437187195466, + "inverse_median_rank": 0.0005056890076957643, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6904296875, + "variance": 13910590.0, + "z_arithmetic_mean_rank": 320.43157226197576, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.9626383061805 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4984113221667289, + "adjusted_arithmetic_mean_rank_index": 0.5016567752769987, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.041081447926066904, + "arithmetic_mean_rank": 3671.671107650116, + "count": 59071.0, + "geometric_mean_rank": 1054.047269125659, + "harmonic_mean_rank": 23.954931595237145, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.00027235554892605945, + "inverse_geometric_mean_rank": 0.0009487240556388978, + "inverse_harmonic_mean_rank": 0.041745057631424234, + "inverse_median_rank": 0.00042283298097251583, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.175230589348, + "variance": 14176544.517043548, + "z_arithmetic_mean_rank": 210.96593958501688, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.1995310257179 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.49841446353113766, + "adjusted_arithmetic_mean_rank_index": 0.5016536334861073, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.04108138464536481, + "arithmetic_mean_rank": 3671.6942492932235, + "count": 59071.0, + "geometric_mean_rank": 1054.0570794994069, + "harmonic_mean_rank": 23.954967883080986, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.0002723538323466049, + "inverse_geometric_mean_rank": 0.0009487152256260357, + "inverse_harmonic_mean_rank": 0.041744994394514894, + "inverse_median_rank": 0.00042283298097251583, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.1876813762074, + "variance": 14176638.27598714, + "z_arithmetic_mean_rank": 210.96461834129, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.198075067056 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.49841288452990823, + "adjusted_arithmetic_mean_rank_index": 0.5016552127017074, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.04108141567445039, + "arithmetic_mean_rank": 3671.6826171875, + "count": 59071.0, + "geometric_mean_rank": 1054.052734375, + "harmonic_mean_rank": 23.954949947576385, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.00027235469315201044, + "inverse_geometric_mean_rank": 0.0009487191564403474, + "inverse_harmonic_mean_rank": 0.041745025402127114, + "inverse_median_rank": 0.0004228329926263541, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.18115234375, + "variance": 14176590.0, + "z_arithmetic_mean_rank": 210.96528246209795, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.1987889824271 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4245208539977861, + "adjusted_arithmetic_mean_rank_index": 0.5755568509144449, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.0867551881570143, + "arithmetic_mean_rank": 3144.4072048890316, + "count": 59071.0, + "geometric_mean_rank": 558.9103984195146, + "harmonic_mean_rank": 11.443778283670872, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.0003180249677729926, + "inverse_geometric_mean_rank": 0.001789195554113499, + "inverse_harmonic_mean_rank": 0.0873837272281743, + "inverse_median_rank": 0.0006397952655150352, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3674.9867643094444, + "variance": 13505527.7178496, + "z_arithmetic_mean_rank": 242.11532477015106, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1734876125245 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4245237109036899, + "adjusted_arithmetic_mean_rank_index": 0.5755539936227831, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.08675517049007114, + "arithmetic_mean_rank": 3144.428365864807, + "count": 59071.0, + "geometric_mean_rank": 558.914434146379, + "harmonic_mean_rank": 11.443780595742925, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.0003180228275688423, + "inverse_geometric_mean_rank": 0.0017891826349542821, + "inverse_harmonic_mean_rank": 0.08738370957339037, + "inverse_median_rank": 0.0006397952655150352, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3675.002923388804, + "variance": 13505646.486916255, + "z_arithmetic_mean_rank": 242.11412281399757, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1730798872231 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4245222742466193, + "adjusted_arithmetic_mean_rank_index": 0.5755554304738404, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.08675517926450192, + "arithmetic_mean_rank": 3144.417724609375, + "count": 59071.0, + "geometric_mean_rank": 558.91259765625, + "harmonic_mean_rank": 11.443779172897282, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.00031802390003576875, + "inverse_geometric_mean_rank": 0.001789188478142023, + "inverse_harmonic_mean_rank": 0.08738371834178216, + "inverse_median_rank": 0.0006397952674888074, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3674.994873046875, + "variance": 13505587.0, + "z_arithmetic_mean_rank": 242.1147272437074, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1732823873047 + } + } + }, + "times": { + "evaluation": 88.11779618263245, + "training": 402.42151856422424 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json new file mode 100644 index 0000000..f25d2fd --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json new file mode 100644 index 0000000..4c66513 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.013367026912761475, + 0.013179833175412171, + 0.012994235904604514, + 0.012810520019806017, + 0.012628654588765545 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5276142010546685, + "adjusted_arithmetic_mean_rank_index": 0.4724520088567171, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.07283633819545277, + "arithmetic_mean_rank": 3764.880272042274, + "count": 40876.0, + "geometric_mean_rank": 793.899945390293, + "harmonic_mean_rank": 13.605983362668397, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.00026561269621929996, + "inverse_geometric_mean_rank": 0.0012596045708359194, + "inverse_harmonic_mean_rank": 0.07349707649531335, + "inverse_median_rank": 0.0004452359750667854, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.336655248859, + "variance": 16058747.068501113, + "z_arithmetic_mean_rank": 165.23229525867953, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5523220323657 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5276190317306638, + "adjusted_arithmetic_mean_rank_index": 0.47244717750365106, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.072836276435286, + "arithmetic_mean_rank": 3764.914742146981, + "count": 40876.0, + "geometric_mean_rank": 793.912000370915, + "harmonic_mean_rank": 13.605994787744137, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.00026561026437207973, + "inverse_geometric_mean_rank": 0.0012595854446497859, + "inverse_harmonic_mean_rank": 0.07349701477915965, + "inverse_median_rank": 0.0004452359750667854, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.35491441866, + "variance": 16058893.410115385, + "z_arithmetic_mean_rank": 165.23060557265566, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5511590510541 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5276166099241587, + "adjusted_arithmetic_mean_rank_index": 0.4724495996495982, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.07283630710675247, + "arithmetic_mean_rank": 3764.8974609375, + "count": 40876.0, + "geometric_mean_rank": 793.9061279296875, + "harmonic_mean_rank": 13.605989531484779, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.0002656114811543375, + "inverse_geometric_mean_rank": 0.001259594806469977, + "inverse_harmonic_mean_rank": 0.07349704542876827, + "inverse_median_rank": 0.0004452359862625599, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.345703125, + "variance": 16058820.0, + "z_arithmetic_mean_rank": 165.23145267823844, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5517366133274 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6411589651715538, + "adjusted_arithmetic_mean_rank_index": 0.35889203816598525, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873180036128378, + "arithmetic_mean_rank": 4511.603728349153, + "count": 20438.0, + "geometric_mean_rank": 1934.0093597003513, + "harmonic_mean_rank": 86.29191849008801, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022165067240200888, + "inverse_geometric_mean_rank": 0.0005170605793525924, + "inverse_harmonic_mean_rank": 0.011588570720152269, + "inverse_median_rank": 0.0002860411899313501, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.2526904656297, + "variance": 15778791.436911434, + "z_arithmetic_mean_rank": 88.65932890858505, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58897920591045 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6411647017155157, + "adjusted_arithmetic_mean_rank_index": 0.35888630080666817, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873108507002505, + "arithmetic_mean_rank": 4511.644094334084, + "count": 20438.0, + "geometric_mean_rank": 1934.0403711432396, + "harmonic_mean_rank": 86.29245073513225, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022164868927844792, + "inverse_geometric_mean_rank": 0.0005170522885253349, + "inverse_harmonic_mean_rank": 0.011588499242760177, + "inverse_median_rank": 0.0002860411899313501, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.267678960659, + "variance": 15778910.5133155, + "z_arithmetic_mean_rank": 88.65791157308394, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58803460698596 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6411618493738521, + "adjusted_arithmetic_mean_rank_index": 0.35888915355374507, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873143966700935, + "arithmetic_mean_rank": 4511.6240234375, + "count": 20438.0, + "geometric_mean_rank": 1934.02490234375, + "harmonic_mean_rank": 86.29218338315411, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022164967958815396, + "inverse_geometric_mean_rank": 0.0005170564400032163, + "inverse_harmonic_mean_rank": 0.01158853467681221, + "inverse_median_rank": 0.00028604117687791586, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.260009765625, + "variance": 15778850.0, + "z_arithmetic_mean_rank": 88.65861630490991, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58850288047023 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.417177903862162, + "adjusted_arithmetic_mean_rank_index": 0.5829026665413757, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.1347981805690344, + "arithmetic_mean_rank": 3018.156815735395, + "count": 20438.0, + "geometric_mean_rank": 325.8914545213797, + "harmonic_mean_rank": 7.385219894424195, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.0003313280459074964, + "inverse_geometric_mean_rank": 0.0030685063573349888, + "inverse_harmonic_mean_rank": 0.13540558227047444, + "inverse_median_rank": 0.0010378827192527244, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.731777005309, + "variance": 15223510.859693006, + "z_arithmetic_mean_rank": 144.3236321227174, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9979156699924 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.41718185346975445, + "adjusted_arithmetic_mean_rank_index": 0.5828987163877821, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.13479812857761928, + "arithmetic_mean_rank": 3018.1853899598786, + "count": 20438.0, + "geometric_mean_rank": 325.89612592231913, + "harmonic_mean_rank": 7.385222728122889, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.000331324909108149, + "inverse_geometric_mean_rank": 0.003068462373309589, + "inverse_harmonic_mean_rank": 0.1354055303155591, + "inverse_median_rank": 0.0010378827192527244, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.7517676140965, + "variance": 15223666.856079726, + "z_arithmetic_mean_rank": 144.32265408547934, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9972175570608 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.41717988415761514, + "adjusted_arithmetic_mean_rank_index": 0.5829006859721628, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.1347981544609555, + "arithmetic_mean_rank": 3018.171142578125, + "count": 20438.0, + "geometric_mean_rank": 325.89361572265625, + "harmonic_mean_rank": 7.385221466974059, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.0003313264751341194, + "inverse_geometric_mean_rank": 0.0030684859957545996, + "inverse_harmonic_mean_rank": 0.13540555618072433, + "inverse_median_rank": 0.0010378827573731542, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.741943359375, + "variance": 15223590.0, + "z_arithmetic_mean_rank": 144.32314174420506, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9975651046645 + } + } + }, + "times": { + "evaluation": 28.418932676315308, + "training": 440.5546112060547 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json new file mode 100644 index 0000000..6ee6351 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with RotatE as described by Sun et al., 2019;" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json new file mode 100644 index 0000000..e447740 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.013387655595250347, + 0.013083627746816562, + 0.012784079545507152, + 0.0124903927645521, + 0.012203190229969347 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7891292153102102, + "adjusted_arithmetic_mean_rank_index": 0.21088109005502975, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.01896367615359624, + "arithmetic_mean_rank": 16148.135, + "count": 10000.0, + "geometric_mean_rank": 6089.0981506932985, + "harmonic_mean_rank": 51.99646411776393, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192665592652031e-05, + "inverse_geometric_mean_rank": 0.0001642279324872011, + "inverse_harmonic_mean_rank": 0.019232076968448378, + "inverse_median_rank": 6.963788300835655e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.230432867844, + "variance": 165900336.00377497, + "z_arithmetic_mean_rank": 36.524738015017164, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.3186043703253 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7891330123641752, + "adjusted_arithmetic_mean_rank_index": 0.21087729281550072, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.01896367431133423, + "arithmetic_mean_rank": 16148.2127, + "count": 10000.0, + "geometric_mean_rank": 6089.140137166568, + "harmonic_mean_rank": 51.99646909720074, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192635795539155e-05, + "inverse_geometric_mean_rank": 0.00016422680008565635, + "inverse_harmonic_mean_rank": 0.019232075126690392, + "inverse_median_rank": 6.963788300835655e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.246949242031, + "variance": 165900761.47345868, + "z_arithmetic_mean_rank": 36.52408033073195, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.31857529245616 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7891311604909889, + "adjusted_arithmetic_mean_rank_index": 0.21087914477918912, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.018963675232121602, + "arithmetic_mean_rank": 16148.1748046875, + "count": 10000.0, + "geometric_mean_rank": 6089.12109375, + "harmonic_mean_rank": 51.99646808658853, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192650471348315e-05, + "inverse_geometric_mean_rank": 0.00016422731277998537, + "inverse_harmonic_mean_rank": 0.01923207604722585, + "inverse_median_rank": 6.963788473512977e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.23828125, + "variance": 165900528.0, + "z_arithmetic_mean_rank": 36.52440109201268, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.31858982596697 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7980273197378187, + "adjusted_arithmetic_mean_rank_index": 0.2019825510012615, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.0189419066190694, + "arithmetic_mean_rank": 16329.8404, + "count": 5000.0, + "geometric_mean_rank": 6326.548079401762, + "harmonic_mean_rank": 52.0553560301053, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123758564106971e-05, + "inverse_geometric_mean_rank": 0.00015806407972395587, + "inverse_harmonic_mean_rank": 0.019210319096111216, + "inverse_median_rank": 6.911088842047065e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.251613807723, + "variance": 166210151.67372784, + "z_arithmetic_mean_rank": 24.737071831918513, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40478313251361 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.79803106312362, + "adjusted_arithmetic_mean_rank_index": 0.20197880743251473, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.018941905090615847, + "arithmetic_mean_rank": 16329.917, + "count": 5000.0, + "geometric_mean_rank": 6326.5873648947, + "harmonic_mean_rank": 52.05536017071492, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123729838920798e-05, + "inverse_geometric_mean_rank": 0.00015806309821134415, + "inverse_harmonic_mean_rank": 0.019210317568075837, + "inverse_median_rank": 6.911088842047065e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.268639789934, + "variance": 166210590.680511, + "z_arithmetic_mean_rank": 24.736613352071913, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40476607391446 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7980291537861127, + "adjusted_arithmetic_mean_rank_index": 0.20198071686333463, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.01894190585454245, + "arithmetic_mean_rank": 16329.8779296875, + "count": 5000.0, + "geometric_mean_rank": 6326.56640625, + "harmonic_mean_rank": 52.05535901882765, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123744242358953e-05, + "inverse_geometric_mean_rank": 0.00015806361625436693, + "inverse_harmonic_mean_rank": 0.019210318331793434, + "inverse_median_rank": 6.911088712513447e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.259765625, + "variance": 166210352.0, + "z_arithmetic_mean_rank": 24.736847202605617, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40477459986386 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7802315234433033, + "adjusted_arithmetic_mean_rank_index": 0.2197792165077822, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.018985445687869747, + "arithmetic_mean_rank": 15966.4296, + "count": 5000.0, + "geometric_mean_rank": 5860.560265003558, + "harmonic_mean_rank": 51.93770530750026, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263141009308681e-05, + "inverse_geometric_mean_rank": 0.00017063215030336233, + "inverse_harmonic_mean_rank": 0.01925383484078553, + "inverse_median_rank": 7.000840100812098e-05, + "median_absolute_deviation": 16935.02384088024, + "median_rank": 14284.0, + "standard_deviation": 12865.631994933006, + "variance": 165524486.62904385, + "z_arithmetic_mean_rank": 26.91665769738404, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.89565239009264 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7802353741629438, + "adjusted_arithmetic_mean_rank_index": 0.21977536559995947, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.018985443531799297, + "arithmetic_mean_rank": 15966.5084, + "count": 5000.0, + "geometric_mean_rank": 5860.6046943714355, + "harmonic_mean_rank": 51.937711121964185, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263110098636218e-05, + "inverse_geometric_mean_rank": 0.00017063085673742963, + "inverse_harmonic_mean_rank": 0.019253832685304943, + "inverse_median_rank": 7.000840100812098e-05, + "median_absolute_deviation": 16935.76514198949, + "median_rank": 14284.0, + "standard_deviation": 12865.648035024487, + "variance": 165524899.36112943, + "z_arithmetic_mean_rank": 26.91618607149813, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.89562832629335 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7802334365863733, + "adjusted_arithmetic_mean_rank_index": 0.21977730327121803, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.01898544460944743, + "arithmetic_mean_rank": 15966.46875, + "count": 5000.0, + "geometric_mean_rank": 5860.5830078125, + "harmonic_mean_rank": 51.937706135894416, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263125396799296e-05, + "inverse_geometric_mean_rank": 0.0001706314942566678, + "inverse_harmonic_mean_rank": 0.01925383376265825, + "inverse_median_rank": 7.000839832471684e-05, + "median_absolute_deviation": 16935.394491434865, + "median_rank": 14284.0, + "standard_deviation": 12865.6396484375, + "variance": 165524688.0, + "z_arithmetic_mean_rank": 26.91642338071611, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.8956403538727 + } + } + }, + "times": { + "evaluation": 11.729146718978882, + "training": 471.5053610801697 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json new file mode 100644 index 0000000..0acad86 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json new file mode 100644 index 0000000..9d80498 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.019358374847747738, + 0.01926067069163421, + 0.019161655377482285, + 0.01906288573317298, + 0.01896479291372804 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8705811229150148, + "adjusted_arithmetic_mean_rank_index": 0.12942526134716914, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.003645520836678461, + "arithmetic_mean_rank": 17648.897571819427, + "count": 5848.0, + "geometric_mean_rank": 10309.546076029572, + "harmonic_mean_rank": 255.07299036983989, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.6660762856753884e-05, + "inverse_geometric_mean_rank": 9.699748103605368e-05, + "inverse_harmonic_mean_rank": 0.003920446451621798, + "inverse_median_rank": 6.00817111271329e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.257654460096, + "variance": 144582772.14084223, + "z_arithmetic_mean_rank": 17.14242203045866, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.796929167098405 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8705851548377035, + "adjusted_arithmetic_mean_rank_index": 0.12942122922558474, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.0036455178545416692, + "arithmetic_mean_rank": 17648.979309165527, + "count": 5848.0, + "geometric_mean_rank": 10309.62711068822, + "harmonic_mean_rank": 255.07318434092105, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.666050044495642e-05, + "inverse_geometric_mean_rank": 9.699671862654254e-05, + "inverse_harmonic_mean_rank": 0.003920443470307872, + "inverse_median_rank": 6.00817111271329e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.267924256845, + "variance": 144583019.114312, + "z_arithmetic_mean_rank": 17.141887974516578, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.796893339999976 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8705830924839433, + "adjusted_arithmetic_mean_rank_index": 0.12942329168108146, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.0036455193447481517, + "arithmetic_mean_rank": 17648.9375, + "count": 5848.0, + "geometric_mean_rank": 10309.5869140625, + "harmonic_mean_rank": 255.0730929684683, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.6660635891603306e-05, + "inverse_geometric_mean_rank": 9.699709335109219e-05, + "inverse_harmonic_mean_rank": 0.00392044496010316, + "inverse_median_rank": 6.00817111262586e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.2626953125, + "variance": 144582896.0, + "z_arithmetic_mean_rank": 17.142161147482767, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.79691124319425 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9373203914542778, + "adjusted_arithmetic_mean_rank_index": 0.0626827012310005, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593373166947965, + "arithmetic_mean_rank": 18997.656976744187, + "count": 2924.0, + "geometric_mean_rank": 12721.473902043308, + "harmonic_mean_rank": 544.9904552697009, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.2638070116969746e-05, + "inverse_geometric_mean_rank": 7.860724375965439e-05, + "inverse_harmonic_mean_rank": 0.0018348945203180253, + "inverse_median_rank": 5.425935973955508e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.0, + "standard_deviation": 11759.070452955062, + "variance": 138275737.91756076, + "z_arithmetic_mean_rank": 5.870641867908702, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245268635772078 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9373248123670547, + "adjusted_arithmetic_mean_rank_index": 0.06267828010009058, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593357925949337, + "arithmetic_mean_rank": 18997.74658002736, + "count": 2924.0, + "geometric_mean_rank": 12721.566306576726, + "harmonic_mean_rank": 544.9909078250481, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.263782184837207e-05, + "inverse_geometric_mean_rank": 7.860667278706282e-05, + "inverse_harmonic_mean_rank": 0.0018348929966387952, + "inverse_median_rank": 5.4257887740430265e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.5, + "standard_deviation": 11759.076669307115, + "variance": 138275884.1146429, + "z_arithmetic_mean_rank": 5.870227800299716, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245255689816194 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.937322571986191, + "adjusted_arithmetic_mean_rank_index": 0.06268052059149731, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593365544549359, + "arithmetic_mean_rank": 18997.701171875, + "count": 2924.0, + "geometric_mean_rank": 12721.5244140625, + "harmonic_mean_rank": 544.9906554581698, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.263794810161926e-05, + "inverse_geometric_mean_rank": 7.860692858230323e-05, + "inverse_harmonic_mean_rank": 0.0018348937582885335, + "inverse_median_rank": 5.425862400443293e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.25, + "standard_deviation": 11759.0732421875, + "variance": 138275808.0, + "z_arithmetic_mean_rank": 5.870437636863841, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245262161180847 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8038714603881213, + "adjusted_arithmetic_mean_rank_index": 0.19613821253058217, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.0057317041228105246, + "arithmetic_mean_rank": 16300.138166894665, + "count": 2924.0, + "geometric_mean_rank": 8354.907702691988, + "harmonic_mean_rank": 166.50021132920975, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134917322547517e-05, + "inverse_geometric_mean_rank": 0.00011969013130782949, + "inverse_harmonic_mean_rank": 0.0060059983829255734, + "inverse_median_rank": 6.83900971139379e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.723008761996, + "variance": 147251502.49937776, + "z_arithmetic_mean_rank": 18.369638890388757, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.69689065121688 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8038751034932807, + "adjusted_arithmetic_mean_rank_index": 0.19613456924574735, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.005731699682636964, + "arithmetic_mean_rank": 16300.212038303693, + "count": 2924.0, + "geometric_mean_rank": 8354.978357223932, + "harmonic_mean_rank": 166.5003343872566, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134889519535763e-05, + "inverse_geometric_mean_rank": 0.00011968911913881547, + "inverse_harmonic_mean_rank": 0.006005993943976948, + "inverse_median_rank": 6.83900971139379e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.735588882228, + "variance": 147251807.8120849, + "z_arithmetic_mean_rank": 18.369297672703908, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.69685292724479 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8038732672486356, + "adjusted_arithmetic_mean_rank_index": 0.1961364055809548, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.005731701901189846, + "arithmetic_mean_rank": 16300.1748046875, + "count": 2924.0, + "geometric_mean_rank": 8354.947265625, + "harmonic_mean_rank": 166.50025295140748, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134903378551826e-05, + "inverse_geometric_mean_rank": 0.00011968956096097827, + "inverse_harmonic_mean_rank": 0.006005996161917786, + "inverse_median_rank": 6.839009438408539e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.7294921875, + "variance": 147251648.0, + "z_arithmetic_mean_rank": 18.369469657623156, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.696871776198755 + } + } + }, + "times": { + "evaluation": 7.746156454086304, + "training": 290.5797860622406 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json new file mode 100644 index 0000000..8ca03b4 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json new file mode 100644 index 0000000..b7ad605 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0005177027531600979, + 9.552033202193208e-05, + 7.630582763936014e-05, + 6.830938954821957e-05, + 6.340357705154206e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6174970603345358, + "adjusted_arithmetic_mean_rank_index": 0.3825547282632912, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.0015853138783117442, + "arithmetic_mean_rank": 4561.35964347988, + "count": 118142.0, + "geometric_mean_rank": 2305.144626685176, + "harmonic_mean_rank": 439.683412149987, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.00021923287751042053, + "inverse_geometric_mean_rank": 0.0004338122599439721, + "inverse_harmonic_mean_rank": 0.0022743637179991564, + "inverse_median_rank": 0.0002976633427593392, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.257051319545, + "variance": 16918883.57022996, + "z_arithmetic_mean_rank": 227.55016767242148, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.66200401260044 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.617497263153951, + "adjusted_arithmetic_mean_rank_index": 0.3825545254164153, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.001585313588680166, + "arithmetic_mean_rank": 4561.361141676965, + "count": 118142.0, + "geometric_mean_rank": 2305.145527422239, + "harmonic_mean_rank": 439.6834681033637, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.0002192328055025159, + "inverse_geometric_mean_rank": 0.00043381209043155894, + "inverse_harmonic_mean_rank": 0.0022743634285674655, + "inverse_median_rank": 0.0002976633427593392, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.25777071006, + "variance": 16918889.48830669, + "z_arithmetic_mean_rank": 227.55004701559199, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.66199457412396 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6174971561916859, + "adjusted_arithmetic_mean_rank_index": 0.38255463239316256, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.00158531373336889, + "arithmetic_mean_rank": 4561.3603515625, + "count": 118142.0, + "geometric_mean_rank": 2305.14501953125, + "harmonic_mean_rank": 439.68341606952293, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.00021923283929936588, + "inverse_geometric_mean_rank": 0.00043381217983551323, + "inverse_harmonic_mean_rank": 0.0022743635731563338, + "inverse_median_rank": 0.0002976633550133556, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.25732421875, + "variance": 16918886.0, + "z_arithmetic_mean_rank": 227.5501106472111, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.661999289221434 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6461976610370368, + "adjusted_arithmetic_mean_rank_index": 0.3538503724133438, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.0012519077863274576, + "arithmetic_mean_rank": 4760.375971288788, + "count": 59071.0, + "geometric_mean_rank": 2569.324969383426, + "harmonic_mean_rank": 514.6465541118973, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.0002100674413179318, + "inverse_geometric_mean_rank": 0.0003892072867061169, + "inverse_harmonic_mean_rank": 0.0019430811146217727, + "inverse_median_rank": 0.0002697599136768276, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.129880173063, + "variance": 16639300.579320708, + "z_arithmetic_mean_rank": 148.80767083723669, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803820514157955 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6461979000289159, + "adjusted_arithmetic_mean_rank_index": 0.3538501333890185, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.001251907536445022, + "arithmetic_mean_rank": 4760.377731881973, + "count": 59071.0, + "geometric_mean_rank": 2569.3260504361556, + "harmonic_mean_rank": 514.6466202502345, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.00021006736362592362, + "inverse_geometric_mean_rank": 0.00038920712294581887, + "inverse_harmonic_mean_rank": 0.0019430808649122655, + "inverse_median_rank": 0.0002697599136768276, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.130608036787, + "variance": 16639306.51742257, + "z_arithmetic_mean_rank": 148.80757031832837, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803814764877608 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6461977943164811, + "adjusted_arithmetic_mean_rank_index": 0.35385023911580515, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.0012519076612958744, + "arithmetic_mean_rank": 4760.376953125, + "count": 59071.0, + "geometric_mean_rank": 2569.32470703125, + "harmonic_mean_rank": 514.6466113756944, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.0002100674028042704, + "inverse_geometric_mean_rank": 0.00038920732913538814, + "inverse_harmonic_mean_rank": 0.001943080989676716, + "inverse_median_rank": 0.0002697599120438099, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.130126953125, + "variance": 16639303.0, + "z_arithmetic_mean_rank": 148.8076147805025, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803817637438655 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5889522536778217, + "adjusted_arithmetic_mean_rank_index": 0.4111032486444999, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.001918718705307047, + "arithmetic_mean_rank": 4362.3433156709725, + "count": 59071.0, + "geometric_mean_rank": 2068.1275483850927, + "harmonic_mean_rank": 383.78193993408456, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.0002292345942621414, + "inverse_geometric_mean_rank": 0.0004835291714869592, + "inverse_harmonic_mean_rank": 0.002605646321376541, + "inverse_median_rank": 0.0003324468085106383, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.54172953822, + "variance": 17119251.56367013, + "z_arithmetic_mean_rank": 172.93582102530266, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.281014237433794 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5889524205211264, + "adjusted_arithmetic_mean_rank_index": 0.4111030817786667, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.0019187183759264768, + "arithmetic_mean_rank": 4362.344551471958, + "count": 59071.0, + "geometric_mean_rank": 2068.1282944578934, + "harmonic_mean_rank": 383.7819884146966, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.0002292345293226727, + "inverse_geometric_mean_rank": 0.0004835289970548584, + "inverse_harmonic_mean_rank": 0.002605645992222666, + "inverse_median_rank": 0.0003324468085106383, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.542417043861, + "variance": 17119257.25283716, + "z_arithmetic_mean_rank": 172.93575083106327, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.28100663584755 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.588952378237803, + "adjusted_arithmetic_mean_rank_index": 0.41110312406769955, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.0019187185404529977, + "arithmetic_mean_rank": 4362.34423828125, + "count": 59071.0, + "geometric_mean_rank": 2068.12646484375, + "harmonic_mean_rank": 383.7819330855535, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.00022923454525880516, + "inverse_geometric_mean_rank": 0.00048352943849749863, + "inverse_harmonic_mean_rank": 0.002605646156635952, + "inverse_median_rank": 0.0003324467979837209, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.54248046875, + "variance": 17119256.0, + "z_arithmetic_mean_rank": 172.93576862048388, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.28101043286125 + } + } + }, + "times": { + "evaluation": 29.054805994033813, + "training": 10.112524032592773 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json new file mode 100644 index 0000000..99e152d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json new file mode 100644 index 0000000..48bb631 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.007010572051467946, + 0.0005912138870406634, + 0.0003330073621221427, + 0.00022124117972636016, + 0.00013660546551546888 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.055875071158110745, + "adjusted_arithmetic_mean_rank_index": 0.9441710687196261, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846, + "count": 10000.0, + "geometric_mean_rank": 55.061474085931685, + "harmonic_mean_rank": 5.073295495004501, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745963519186807, + "inverse_geometric_mean_rank": 0.018161518858709633, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.387928648988, + "variance": 17634858.97528284, + "z_arithmetic_mean_rank": 163.5310255525711, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.055875071158110745, + "adjusted_arithmetic_mean_rank_index": 0.9441710687196261, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846, + "count": 10000.0, + "geometric_mean_rank": 55.061474085931685, + "harmonic_mean_rank": 5.073295495004501, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745963519186807, + "inverse_geometric_mean_rank": 0.018161518858709633, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.387928648988, + "variance": 17634858.97528284, + "z_arithmetic_mean_rank": 163.5310255525711, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05587507328654702, + "adjusted_arithmetic_mean_rank_index": 0.9441710665910859, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846435546875, + "count": 10000.0, + "geometric_mean_rank": 55.06148910522461, + "harmonic_mean_rank": 5.073295558172437, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745962986722589, + "inverse_geometric_mean_rank": 0.018161514773964882, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.016949152573943138, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.38818359375, + "variance": 17634860.0, + "z_arithmetic_mean_rank": 163.5310251839066, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05311946552504412, + "adjusted_arithmetic_mean_rank_index": 0.9469268100934394, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708, + "count": 5000.0, + "geometric_mean_rank": 58.671546702675336, + "harmonic_mean_rank": 5.294065622093927, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.000919987915038748, + "inverse_geometric_mean_rank": 0.017044036781024584, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925373134328358, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.5450086197116, + "variance": 15614707.955147358, + "z_arithmetic_mean_rank": 115.97138666054661, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05311946552504412, + "adjusted_arithmetic_mean_rank_index": 0.9469268100934394, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708, + "count": 5000.0, + "geometric_mean_rank": 58.671546702675336, + "harmonic_mean_rank": 5.294065622093927, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.000919987915038748, + "inverse_geometric_mean_rank": 0.017044036781024584, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925373134328358, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.5450086197116, + "variance": 15614707.955147358, + "z_arithmetic_mean_rank": 115.97138666054661, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05311946675632055, + "adjusted_arithmetic_mean_rank_index": 0.9469268088621028, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708251953125, + "count": 5000.0, + "geometric_mean_rank": 58.67153549194336, + "harmonic_mean_rank": 5.294065567053748, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.0009199879132211208, + "inverse_geometric_mean_rank": 0.01704403944313526, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925372786819935, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.545166015625, + "variance": 15614709.0, + "z_arithmetic_mean_rank": 115.97138650974318, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0586305490274944, + "adjusted_arithmetic_mean_rank_index": 0.9414154551220333, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.7984, + "count": 5000.0, + "geometric_mean_rank": 51.673529997079186, + "harmonic_mean_rank": 4.870201125174862, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334733568572854, + "inverse_geometric_mean_rank": 0.019352267980463585, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.679208081433, + "variance": 19648644.96175744, + "z_arithmetic_mean_rank": 115.29642319772988, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0586305490274944, + "adjusted_arithmetic_mean_rank_index": 0.9414154551220333, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.7984, + "count": 5000.0, + "geometric_mean_rank": 51.673529997079186, + "harmonic_mean_rank": 4.870201125174862, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334733568572854, + "inverse_geometric_mean_rank": 0.019352267980463585, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.679208081433, + "variance": 19648644.96175744, + "z_arithmetic_mean_rank": 115.29642319772988, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.0586305460878389, + "adjusted_arithmetic_mean_rank_index": 0.9414154580618325, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.79833984375, + "count": 5000.0, + "geometric_mean_rank": 51.67353439331055, + "harmonic_mean_rank": 4.870200709104077, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334734011441469, + "inverse_geometric_mean_rank": 0.019352266564965248, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.019999999552965164, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.67919921875, + "variance": 19648646.0, + "z_arithmetic_mean_rank": 115.29642355777105, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + } + } + }, + "times": { + "evaluation": 8.916642427444458, + "training": 4.890740156173706 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json new file mode 100644 index 0000000..87c1425 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json new file mode 100644 index 0000000..a6230c9 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002552040782367046, + 0.0013616517112822108, + 0.0010789413683163901, + 0.0009065022953803493, + 0.000778262092445874 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23547394843537406, + "adjusted_arithmetic_mean_rank_index": 0.7646295637944857, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693621079345207, + "arithmetic_mean_rank": 1739.4113016539418, + "count": 118142.0, + "geometric_mean_rank": 208.93563489743488, + "harmonic_mean_rank": 8.507332970126315, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.00057490715338525, + "inverse_geometric_mean_rank": 0.0047861629754584155, + "inverse_harmonic_mean_rank": 0.11754565191130073, + "inverse_median_rank": 0.0020491803278688526, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.43720587806, + "variance": 6705185.043185574, + "z_arithmetic_mean_rank": 454.8148868492797, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7021416245047 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2354740939611692, + "adjusted_arithmetic_mean_rank_index": 0.7646294182489872, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693620537414412, + "arithmetic_mean_rank": 1739.4123766315113, + "count": 118142.0, + "geometric_mean_rank": 208.9357986842343, + "harmonic_mean_rank": 8.507333362076508, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.0005749067980857805, + "inverse_geometric_mean_rank": 0.00478615922353883, + "inverse_harmonic_mean_rank": 0.11754564649573288, + "inverse_median_rank": 0.0020491803278688526, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.438397730749, + "variance": 6705191.215642387, + "z_arithmetic_mean_rank": 454.8148002763004, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7019650207967 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23547402473044568, + "adjusted_arithmetic_mean_rank_index": 0.7646294874890841, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693620806333842, + "arithmetic_mean_rank": 1739.411865234375, + "count": 118142.0, + "geometric_mean_rank": 208.9356231689453, + "harmonic_mean_rank": 8.507333741808493, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.0005749069387093186, + "inverse_geometric_mean_rank": 0.004786163102835417, + "inverse_harmonic_mean_rank": 0.11754564918307125, + "inverse_median_rank": 0.0020491802133619785, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.437744140625, + "variance": 6705187.5, + "z_arithmetic_mean_rank": 454.8148414615074, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7020526559136 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.26508252584909736, + "adjusted_arithmetic_mean_rank_index": 0.7350172491329787, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.0895138719752979, + "arithmetic_mean_rank": 1952.7964314130454, + "count": 59071.0, + "geometric_mean_rank": 302.6048275128738, + "harmonic_mean_rank": 11.09336609853906, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120861467758823, + "inverse_geometric_mean_rank": 0.0033046399431861566, + "inverse_harmonic_mean_rank": 0.09014396452053404, + "inverse_median_rank": 0.0012755102040816326, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.174746370657, + "variance": 7204794.06905398, + "z_arithmetic_mean_rank": 309.10298079580843, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.5298871552714 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.26508269130501355, + "adjusted_arithmetic_mean_rank_index": 0.7350170836545997, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.08951386824429253, + "arithmetic_mean_rank": 1952.79765028525, + "count": 59071.0, + "geometric_mean_rank": 302.6050304340035, + "harmonic_mean_rank": 11.09336655736919, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120858271485156, + "inverse_geometric_mean_rank": 0.003304637727157991, + "inverse_harmonic_mean_rank": 0.09014396079211066, + "inverse_median_rank": 0.0012755102040816326, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.1761261754814, + "variance": 7204801.476330414, + "z_arithmetic_mean_rank": 309.102911205795, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.52980131252 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2650826192047391, + "adjusted_arithmetic_mean_rank_index": 0.7350171557646628, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.08951387009760452, + "arithmetic_mean_rank": 1952.797119140625, + "count": 59071.0, + "geometric_mean_rank": 302.6047668457031, + "harmonic_mean_rank": 11.093367293323608, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120859714224935, + "inverse_geometric_mean_rank": 0.003304640529677272, + "inverse_harmonic_mean_rank": 0.09014396264414008, + "inverse_median_rank": 0.0012755101779475808, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.175537109375, + "variance": 7204798.0, + "z_arithmetic_mean_rank": 309.1029415308459, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.5298439534126 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.2060260937923374, + "adjusted_arithmetic_mean_rank_index": 0.7940811136940227, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.14435844556744704, + "arithmetic_mean_rank": 1526.0261718948384, + "count": 59071.0, + "geometric_mean_rank": 144.2610809906429, + "harmonic_mean_rank": 6.899057304639579, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.0006552967559909661, + "inverse_geometric_mean_rank": 0.006931876519522699, + "inverse_harmonic_mean_rank": 0.14494733930206743, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.7534430493765, + "variance": 6114509.590112546, + "z_arithmetic_mean_rank": 334.04034098527376, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.566198726866 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.20602621949619718, + "adjusted_arithmetic_mean_rank_index": 0.7940809879731895, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.1443584384598429, + "arithmetic_mean_rank": 1526.0271029777725, + "count": 59071.0, + "geometric_mean_rank": 144.26121042736472, + "harmonic_mean_rank": 6.899057642707337, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.0006552963561713135, + "inverse_geometric_mean_rank": 0.00693187029997574, + "inverse_harmonic_mean_rank": 0.1449473321993551, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.754416623252, + "variance": 6114514.4049298, + "z_arithmetic_mean_rank": 334.040288099203, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.5660346945256 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.20602616959994496, + "adjusted_arithmetic_mean_rank_index": 0.794081037876179, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.14435844198491637, + "arithmetic_mean_rank": 1526.0267333984375, + "count": 59071.0, + "geometric_mean_rank": 144.26116943359375, + "harmonic_mean_rank": 6.8990578575211945, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.000655296491459012, + "inverse_geometric_mean_rank": 0.006931872107088566, + "inverse_harmonic_mean_rank": 0.14494733572200244, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.754150390625, + "variance": 6114513.0, + "z_arithmetic_mean_rank": 334.0403090915315, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.566116047685 + } + } + }, + "times": { + "evaluation": 33.93540859222412, + "training": 87.27861523628235 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json new file mode 100644 index 0000000..f9e9946 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json new file mode 100644 index 0000000..ce10c4c --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.004478471603307385, + 0.0036625902134173917, + 0.0031688402850889408, + 0.0027868693412893444, + 0.002493769001652959 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.596269284526057, + "adjusted_arithmetic_mean_rank_index": 0.40375044600459786, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145151990078, + "arithmetic_mean_rank": 12201.5973, + "count": 10000.0, + "geometric_mean_rank": 3947.2342081152287, + "harmonic_mean_rank": 63.71653149146111, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.19564828614693e-05, + "inverse_geometric_mean_rank": 0.0002533419471142787, + "inverse_harmonic_mean_rank": 0.01569451407024586, + "inverse_median_rank": 0.00011169440411035407, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.701157837608, + "variance": 131599927.45473272, + "z_arithmetic_mean_rank": 69.92983230462272, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.46718862459554 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5962695093194706, + "adjusted_arithmetic_mean_rank_index": 0.40375022120019854, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145101858308, + "arithmetic_mean_rank": 12201.6019, + "count": 10000.0, + "geometric_mean_rank": 3947.2356013178296, + "harmonic_mean_rank": 63.71653169493022, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.195645196390157e-05, + "inverse_geometric_mean_rank": 0.0002533418576955829, + "inverse_harmonic_mean_rank": 0.015694514020127805, + "inverse_median_rank": 0.00011169440411035407, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.704064428108, + "variance": 131599994.14181638, + "z_arithmetic_mean_rank": 69.92979336835873, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.4671878333264 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5962693973809026, + "adjusted_arithmetic_mean_rank_index": 0.40375033314423703, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145126921166, + "arithmetic_mean_rank": 12201.599609375, + "count": 10000.0, + "geometric_mean_rank": 3947.233642578125, + "harmonic_mean_rank": 63.71652988749175, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.195646660169587e-05, + "inverse_geometric_mean_rank": 0.0002533419756218791, + "inverse_harmonic_mean_rank": 0.015694514045183804, + "inverse_median_rank": 0.00011169440404046327, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.7021484375, + "variance": 131599960.0, + "z_arithmetic_mean_rank": 69.92981275713693, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.46718822891316 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6025079727031526, + "adjusted_arithmetic_mean_rank_index": 0.397511453389935, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001105085272, + "arithmetic_mean_rank": 12328.9752, + "count": 5000.0, + "geometric_mean_rank": 4126.3291122784585, + "harmonic_mean_rank": 69.97528513726976, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.11097421949555e-05, + "inverse_geometric_mean_rank": 0.00024234615630255055, + "inverse_harmonic_mean_rank": 0.014290759916709316, + "inverse_median_rank": 0.00011035700491088672, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.727300846267, + "variance": 131646418.17418496, + "z_arithmetic_mean_rank": 48.68375672934111, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407298857802 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6025082268233375, + "adjusted_arithmetic_mean_rank_index": 0.3975111992573307, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001033309185, + "arithmetic_mean_rank": 12328.9804, + "count": 5000.0, + "geometric_mean_rank": 4126.33083194677, + "harmonic_mean_rank": 69.97528548862813, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.110970798526048e-05, + "inverse_geometric_mean_rank": 0.000242346055303619, + "inverse_harmonic_mean_rank": 0.014290759844952868, + "inverse_median_rank": 0.00011035700491088672, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.7308259875, + "variance": 131646499.06721582, + "z_arithmetic_mean_rank": 48.68372560538286, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407218750725 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6025080870114209, + "adjusted_arithmetic_mean_rank_index": 0.3975113390760803, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001069192063, + "arithmetic_mean_rank": 12328.9775390625, + "count": 5000.0, + "geometric_mean_rank": 4126.33056640625, + "harmonic_mean_rank": 69.97528079642154, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.110972703434527e-05, + "inverse_geometric_mean_rank": 0.00024234606826212257, + "inverse_harmonic_mean_rank": 0.014290759880825928, + "inverse_median_rank": 0.0001103570029954426, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.7294921875, + "variance": 131646464.0, + "z_arithmetic_mean_rank": 48.683742729171186, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407258798497 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5900308856058019, + "adjusted_arithmetic_mean_rank_index": 0.40998914933416863, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.016829289182556138, + "arithmetic_mean_rank": 12074.2194, + "count": 5000.0, + "geometric_mean_rank": 3775.912553206831, + "harmonic_mean_rank": 58.48545518832573, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282108903868353e-05, + "inverse_geometric_mean_rank": 0.0002648366417147859, + "inverse_harmonic_mean_rank": 0.017098268223782413, + "inverse_median_rank": 0.00011267605633802817, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.259958531791, + "variance": 131520986.47646362, + "z_arithmetic_mean_rank": 50.211925256721074, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.8308926335976 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5900310810738039, + "adjusted_arithmetic_mean_rank_index": 0.40998895385661416, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.01682928915406868, + "arithmetic_mean_rank": 12074.2234, + "count": 5000.0, + "geometric_mean_rank": 3775.913645043353, + "harmonic_mean_rank": 58.4854552857418, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282106160136145e-05, + "inverse_geometric_mean_rank": 0.00026483656513509026, + "inverse_harmonic_mean_rank": 0.01709826819530275, + "inverse_median_rank": 0.00011267605633802817, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.26223330686, + "variance": 131521038.65189244, + "z_arithmetic_mean_rank": 50.21190131632078, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.83089231565046 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5900309492856118, + "adjusted_arithmetic_mean_rank_index": 0.4099890856512466, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.016829289168311508, + "arithmetic_mean_rank": 12074.220703125, + "count": 5000.0, + "geometric_mean_rank": 3775.914306640625, + "harmonic_mean_rank": 58.48545551786481, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282108319690451e-05, + "inverse_geometric_mean_rank": 0.0002648365043569356, + "inverse_harmonic_mean_rank": 0.017098268209541682, + "inverse_median_rank": 0.00011267605441389605, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.26171875, + "variance": 131521016.0, + "z_arithmetic_mean_rank": 50.21191745738754, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.83089247461402 + } + } + }, + "times": { + "evaluation": 9.962103366851807, + "training": 25.386447191238403 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json new file mode 100644 index 0000000..d1f10e5 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Batch_size is not mentioned in the paper. We found out that this value works.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json new file mode 100644 index 0000000..c28d148 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.038617746736009165, + 0.033566348636352834, + 0.0312234644151795, + 0.029337170743096344, + 0.027740906089835815 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.685622737188115, + "adjusted_arithmetic_mean_rank_index": 0.3144198276027439, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.03358499005785345, + "arithmetic_mean_rank": 5064.593963196831, + "count": 118142.0, + "geometric_mean_rank": 1813.7212167611826, + "harmonic_mean_rank": 29.19541337015125, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.00019744919479562547, + "inverse_geometric_mean_rank": 0.0005513526504286753, + "inverse_harmonic_mean_rank": 0.034251955515121356, + "inverse_median_rank": 0.00024274790629930817, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.964777631248, + "variance": 19079116.298627198, + "z_arithmetic_mean_rank": 187.02235054142858, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4633203991789 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6856238337880042, + "adjusted_arithmetic_mean_rank_index": 0.31441873085438155, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.03358498142594515, + "arithmetic_mean_rank": 5064.6020636183575, + "count": 118142.0, + "geometric_mean_rank": 1813.7260518976943, + "harmonic_mean_rank": 29.195420722673642, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.00019744887899160224, + "inverse_geometric_mean_rank": 0.0005513511806006778, + "inverse_harmonic_mean_rank": 0.03425194688917031, + "inverse_median_rank": 0.00024274790629930817, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.967432633831, + "variance": 19079139.49254978, + "z_arithmetic_mean_rank": 187.02169817653726, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4630391036653 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6856233032390106, + "adjusted_arithmetic_mean_rank_index": 0.31441926147520827, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.033584985710982486, + "arithmetic_mean_rank": 5064.59814453125, + "count": 118142.0, + "geometric_mean_rank": 1813.7232666015625, + "harmonic_mean_rank": 29.195417225218765, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.0001974490296561271, + "inverse_geometric_mean_rank": 0.0005513520445674658, + "inverse_harmonic_mean_rank": 0.03425195117125036, + "inverse_median_rank": 0.0002427479048492387, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.96630859375, + "variance": 19079128.0, + "z_arithmetic_mean_rank": 187.0220137989807, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4631787439089 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.731205300619267, + "adjusted_arithmetic_mean_rank_index": 0.26883119189486704, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.025005942011049314, + "arithmetic_mean_rank": 5386.605914915948, + "count": 59071.0, + "geometric_mean_rank": 2308.3841018965627, + "harmonic_mean_rank": 38.93978400157192, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564565810001414, + "inverse_geometric_mean_rank": 0.0004332034686854768, + "inverse_harmonic_mean_rank": 0.025680676604668173, + "inverse_median_rank": 0.000216076058772688, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.978304404924, + "variance": 18835411.68270544, + "z_arithmetic_mean_rank": 113.05384036036361, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3352390168044 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.731206507068656, + "adjusted_arithmetic_mean_rank_index": 0.2688299852816862, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.025005932334410102, + "arithmetic_mean_rank": 5386.614802525774, + "count": 59071.0, + "geometric_mean_rank": 2308.390778038668, + "harmonic_mean_rank": 38.93979866417694, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564535179517601, + "inverse_geometric_mean_rank": 0.0004332022158092545, + "inverse_harmonic_mean_rank": 0.02568066693472558, + "inverse_median_rank": 0.000216076058772688, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.980567193157, + "variance": 18835431.323614236, + "z_arithmetic_mean_rank": 113.05333293318245, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3350163772601 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7312059028722491, + "adjusted_arithmetic_mean_rank_index": 0.26883058956012096, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.02500593713381818, + "arithmetic_mean_rank": 5386.6103515625, + "count": 59071.0, + "geometric_mean_rank": 2308.387939453125, + "harmonic_mean_rank": 38.93978953297726, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564550555311143, + "inverse_geometric_mean_rank": 0.00043320274562574923, + "inverse_harmonic_mean_rank": 0.025680671730812277, + "inverse_median_rank": 0.0002160760632250458, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.9794921875, + "variance": 18835422.0, + "z_arithmetic_mean_rank": 113.05358705547087, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3351268017581 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6402876073228272, + "adjusted_arithmetic_mean_rank_index": 0.35976096336838814, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216400555455868, + "arithmetic_mean_rank": 4742.582011477713, + "count": 59071.0, + "geometric_mean_rank": 1425.059481837078, + "harmonic_mean_rank": 23.35180920857272, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085560514923302, + "inverse_geometric_mean_rank": 0.0007017250948085874, + "inverse_harmonic_mean_rank": 0.04282323442557454, + "inverse_median_rank": 0.0002795638803466592, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.1204832951535, + "variance": 19115437.520449042, + "z_arithmetic_mean_rank": 151.33803436996612, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.0790266986371 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6402885946695075, + "adjusted_arithmetic_mean_rank_index": 0.3597599758883899, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216399796737732, + "arithmetic_mean_rank": 4742.589324710941, + "count": 59071.0, + "geometric_mean_rank": 1425.062958416174, + "harmonic_mean_rank": 23.35181334306899, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085528000275454, + "inverse_geometric_mean_rank": 0.0007017233828822606, + "inverse_harmonic_mean_rank": 0.042823226843615045, + "inverse_median_rank": 0.0002795638803466592, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.123426145244, + "variance": 19115463.253448024, + "z_arithmetic_mean_rank": 151.33761897391952, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.0788515984075 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6402881373681507, + "adjusted_arithmetic_mean_rank_index": 0.35976043325149454, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216400173804585, + "arithmetic_mean_rank": 4742.5859375, + "count": 59071.0, + "geometric_mean_rank": 1425.0611572265625, + "harmonic_mean_rank": 23.351808771264732, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085543266963214, + "inverse_geometric_mean_rank": 0.0007017242605797946, + "inverse_harmonic_mean_rank": 0.04282323061168842, + "inverse_median_rank": 0.0002795638865791261, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.1220703125, + "variance": 19115450.0, + "z_arithmetic_mean_rank": 151.33781136953849, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.078938619515 + } + } + }, + "times": { + "evaluation": 16.924992561340332, + "training": 115.15393090248108 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..a0879b0 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "TransE was used to initialize the emebddigns. Could not find batch size and number of epochs.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..c4959c0 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00629558999520504, + 0.005996476387571496, + 0.005775562846809344, + 0.005573130315875932, + 0.0054310510201602494 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.948802150747234, + "adjusted_arithmetic_mean_rank_index": 0.05120502517776149, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.00032750514360668293, + "arithmetic_mean_rank": 6770.338046775614, + "count": 40876.0, + "geometric_mean_rank": 4738.385571789443, + "harmonic_mean_rank": 961.6156853712986, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770311217713152, + "inverse_geometric_mean_rank": 0.00021104234445453788, + "inverse_harmonic_mean_rank": 0.0010399164814100142, + "inverse_median_rank": 0.00015076134479119553, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.817822927988, + "variance": 17362370.76947034, + "z_arithmetic_mean_rank": 17.908112740115257, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.167120029921174 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9488038203989086, + "adjusted_arithmetic_mean_rank_index": 0.05120335529206732, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.0003275041272765357, + "arithmetic_mean_rank": 6770.3499608572265, + "count": 40876.0, + "geometric_mean_rank": 4738.39669939512, + "harmonic_mean_rank": 961.616624507759, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770285225749026, + "inverse_geometric_mean_rank": 0.00021104184884470628, + "inverse_harmonic_mean_rank": 0.0010399154658041494, + "inverse_median_rank": 0.00015076134479119553, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.819516552299, + "variance": 17362384.88352114, + "z_arithmetic_mean_rank": 17.90752872514463, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.167100891809513 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9488029500029765, + "adjusted_arithmetic_mean_rank_index": 0.05120422580999462, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.000327504635134217, + "arithmetic_mean_rank": 6770.34375, + "count": 40876.0, + "geometric_mean_rank": 4738.38818359375, + "harmonic_mean_rank": 961.6161284897561, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770298730582, + "inverse_geometric_mean_rank": 0.00021104222105350345, + "inverse_harmonic_mean_rank": 0.0010399159732999085, + "inverse_median_rank": 0.00015076134877745062, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.81884765625, + "variance": 17362378.0, + "z_arithmetic_mean_rank": 17.90783317442731, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.16711045507696 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9754310949337636, + "adjusted_arithmetic_mean_rank_index": 0.024572397131068624, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.000123487588625063, + "arithmetic_mean_rank": 6863.75579802329, + "count": 20438.0, + "geometric_mean_rank": 4989.246896709227, + "harmonic_mean_rank": 1181.1213732617991, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.000145692829032174, + "inverse_geometric_mean_rank": 0.00020043105115915853, + "inverse_harmonic_mean_rank": 0.0008466530389153723, + "inverse_median_rank": 0.0001485663348685188, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.871604112257, + "variance": 16874609.115871806, + "z_arithmetic_mean_rank": 6.070271857934642, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.63075169696039 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9754327220262328, + "adjusted_arithmetic_mean_rank_index": 0.02457076980733508, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0001234869334234348, + "arithmetic_mean_rank": 6863.76724728447, + "count": 20438.0, + "geometric_mean_rank": 4989.257177530884, + "harmonic_mean_rank": 1181.122286638947, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.0001456925860059769, + "inverse_geometric_mean_rank": 0.00020043063815260902, + "inverse_harmonic_mean_rank": 0.0008466523841876217, + "inverse_median_rank": 0.0001485663348685188, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.873714924148, + "variance": 16874626.45776472, + "z_arithmetic_mean_rank": 6.069869850047041, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.630743044502447 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9754318669566006, + "adjusted_arithmetic_mean_rank_index": 0.024571624998501296, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00012348726092708357, + "arithmetic_mean_rank": 6863.76123046875, + "count": 20438.0, + "geometric_mean_rank": 4989.2529296875, + "harmonic_mean_rank": 1181.1218161462336, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.00014569271297659725, + "inverse_geometric_mean_rank": 0.00020043080439791083, + "inverse_harmonic_mean_rank": 0.0008466527114544019, + "inverse_median_rank": 0.00014856633788440377, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.87255859375, + "variance": 16874618.0, + "z_arithmetic_mean_rank": 6.070081113231595, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.6307473694482741 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9229022158891286, + "adjusted_arithmetic_mean_rank_index": 0.07710844225101199, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315183662818469, + "arithmetic_mean_rank": 6676.920295527938, + "count": 20438.0, + "geometric_mean_rank": 4500.13765439255, + "harmonic_mean_rank": 810.9116768895075, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00014976964764275817, + "inverse_geometric_mean_rank": 0.0002222154246823778, + "inverse_harmonic_mean_rank": 0.0012331799239046562, + "inverse_median_rank": 0.00015339776039269826, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.875639960587, + "variance": 17832678.670572538, + "z_arithmetic_mean_rank": 19.091644440437506, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136944512524542 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9229039269348835, + "adjusted_arithmetic_mean_rank_index": 0.0771067309687189, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315169888308492, + "arithmetic_mean_rank": 6676.932674429983, + "count": 20438.0, + "geometric_mean_rank": 4500.149517638199, + "harmonic_mean_rank": 810.9125820357525, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00014976936997277286, + "inverse_geometric_mean_rank": 0.00022221483888047063, + "inverse_harmonic_mean_rank": 0.001233178547420677, + "inverse_median_rank": 0.00015339776039269826, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.876949482508, + "variance": 17832689.730470695, + "z_arithmetic_mean_rank": 19.091220735949094, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136926016848831 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9229030416322332, + "adjusted_arithmetic_mean_rank_index": 0.07710761639375507, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315176770387331, + "arithmetic_mean_rank": 6676.92626953125, + "count": 20438.0, + "geometric_mean_rank": 4500.14208984375, + "harmonic_mean_rank": 810.9121597190133, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.0001497695193393156, + "inverse_geometric_mean_rank": 0.00022221521066967398, + "inverse_harmonic_mean_rank": 0.0012331792351454151, + "inverse_median_rank": 0.00015339776291511953, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.8759765625, + "variance": 17832682.0, + "z_arithmetic_mean_rank": 19.091439962527602, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136935257736431 + } + } + }, + "times": { + "evaluation": 9.182297706604004, + "training": 11.888080596923828 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json new file mode 100644 index 0000000..ed2916a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Num_epochs varies from the value reported in the paper, and batch_size is not mentioned in the paper. We found out that these values work.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WordNet Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json new file mode 100644 index 0000000..fc1ce94 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.06581889594586139, + 0.06439249604551456, + 0.06347951248280359, + 0.06278877398983285, + 0.06220475568033866 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9305804659508103, + "adjusted_arithmetic_mean_rank_index": 0.06942292661802851, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213703543287622, + "arithmetic_mean_rank": 19042.6849, + "count": 10000.0, + "geometric_mean_rank": 12709.213673151666, + "harmonic_mean_rank": 2194.5404157572866, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.2513603268203e-05, + "inverse_geometric_mean_rank": 7.868307400579073e-05, + "inverse_harmonic_mean_rank": 0.0004556762740935543, + "inverse_median_rank": 5.388221348132981e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.146736336086, + "variance": 145664302.943212, + "z_arithmetic_mean_rank": 12.024094745989629, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.874811972549873 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9305812576145713, + "adjusted_arithmetic_mean_rank_index": 0.06942213491557847, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213673413773629, + "arithmetic_mean_rank": 19042.7011, + "count": 10000.0, + "geometric_mean_rank": 12709.230306238696, + "harmonic_mean_rank": 2194.5418664009517, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.251355859384886e-05, + "inverse_geometric_mean_rank": 7.868297103004899e-05, + "inverse_harmonic_mean_rank": 0.0004556759728808454, + "inverse_median_rank": 5.388221348132981e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.149000246818, + "variance": 145664357.5901588, + "z_arithmetic_mean_rank": 12.023957622625176, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.87480721697189 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9305809747902494, + "adjusted_arithmetic_mean_rank_index": 0.06942241775372227, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213688474468513, + "arithmetic_mean_rank": 19042.6953125, + "count": 10000.0, + "geometric_mean_rank": 12709.228515625, + "harmonic_mean_rank": 2194.541126665028, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.251357288216241e-05, + "inverse_geometric_mean_rank": 7.868298416724429e-05, + "inverse_harmonic_mean_rank": 0.00045567612344658983, + "inverse_median_rank": 5.388221325119957e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.1474609375, + "variance": 145664320.0, + "z_arithmetic_mean_rank": 12.024006610370321, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.874809594119726 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9428129206391556, + "adjusted_arithmetic_mean_rank_index": 0.05718987418803789, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0002227443657399582, + "arithmetic_mean_rank": 19292.5532, + "count": 5000.0, + "geometric_mean_rank": 12902.47875932424, + "harmonic_mean_rank": 2014.9983994265895, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833471165444296e-05, + "inverse_geometric_mean_rank": 7.750448721160107e-05, + "inverse_harmonic_mean_rank": 0.000496278309841125, + "inverse_median_rank": 5.309405612041732e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.543824106264, + "variance": 147053065.11796975, + "z_arithmetic_mean_rank": 7.0041200036088185, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859812309410803 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9428137709643899, + "adjusted_arithmetic_mean_rank_index": 0.0571890238212468, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0002227442098313817, + "arithmetic_mean_rank": 19292.5706, + "count": 5000.0, + "geometric_mean_rank": 12902.494552544491, + "harmonic_mean_rank": 2014.9990322764907, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833424416754504e-05, + "inverse_geometric_mean_rank": 7.750439234270328e-05, + "inverse_harmonic_mean_rank": 0.0004962781539752044, + "inverse_median_rank": 5.309405612041732e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.546118389013, + "variance": 147053120.76141563, + "z_arithmetic_mean_rank": 7.004015858056192, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859794908934996 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9428133751233326, + "adjusted_arithmetic_mean_rank_index": 0.05718941968164948, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00022274428777780813, + "arithmetic_mean_rank": 19292.5625, + "count": 5000.0, + "geometric_mean_rank": 12902.4775390625, + "harmonic_mean_rank": 2014.9986899134094, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833445468219e-05, + "inverse_geometric_mean_rank": 7.7504497312475e-05, + "inverse_harmonic_mean_rank": 0.000496278231900305, + "inverse_median_rank": 5.3094056056579575e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.5458984375, + "variance": 147053104.0, + "z_arithmetic_mean_rank": 7.004064339606558, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859803608295463 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9183485784203501, + "adjusted_arithmetic_mean_rank_index": 0.08165541183470171, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00014152970559830536, + "arithmetic_mean_rank": 18792.8166, + "count": 5000.0, + "geometric_mean_rank": 12518.843487581595, + "harmonic_mean_rank": 2409.2075769020707, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321182137221517e-05, + "inverse_geometric_mean_rank": 7.987958320526789e-05, + "inverse_harmonic_mean_rank": 0.0004150742383459837, + "inverse_median_rank": 5.4580683895969216e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.276376702497, + "variance": 144150672.43376443, + "z_arithmetic_mean_rank": 10.000448652139795, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.579606283327427 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9183493114253579, + "adjusted_arithmetic_mean_rank_index": 0.08165467879387245, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00014152925891660332, + "arithmetic_mean_rank": 18792.8316, + "count": 5000.0, + "geometric_mean_rank": 12518.860931831336, + "harmonic_mean_rank": 2409.210168861732, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321177889978006e-05, + "inverse_geometric_mean_rank": 7.98794718980646e-05, + "inverse_harmonic_mean_rank": 0.00041507379178648627, + "inverse_median_rank": 5.4580683895969216e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.278561021372, + "variance": 144150724.88484144, + "z_arithmetic_mean_rank": 10.000358875638653, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5796012979342877 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9183490461691707, + "adjusted_arithmetic_mean_rank_index": 0.08165494406302254, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.0001415294821840739, + "arithmetic_mean_rank": 18792.826171875, + "count": 5000.0, + "geometric_mean_rank": 12518.8505859375, + "harmonic_mean_rank": 2409.2088752176546, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321179560269229e-05, + "inverse_geometric_mean_rank": 7.987953722476959e-05, + "inverse_harmonic_mean_rank": 0.00041507401499287465, + "inverse_median_rank": 5.458068335428834e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.27734375, + "variance": 144150704.0, + "z_arithmetic_mean_rank": 10.000391363510005, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5796037898118622 + } + } + }, + "times": { + "evaluation": 4.560383558273315, + "training": 34.424185276031494 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json new file mode 100644 index 0000000..80f6ec5 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json new file mode 100644 index 0000000..9b4f3bc --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 13.100006539226563, + 0.20497526620080622, + 0.0027243127839496222, + 0.0026737430891539487, + 0.0026241765188342097 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9938447796648451, + "adjusted_arithmetic_mean_rank_index": 0.0061560537149736305, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251325085160447, + "arithmetic_mean_rank": 7341.384698075198, + "count": 118142.0, + "geometric_mean_rank": 5285.389111711265, + "harmonic_mean_rank": 660.1936951360921, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.0001362140850978951, + "inverse_geometric_mean_rank": 0.00018920082871178186, + "inverse_harmonic_mean_rank": 0.0015147069827648996, + "inverse_median_rank": 0.0001367053998632946, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.364307754461, + "variance": 18786713.952335812, + "z_arithmetic_mean_rank": 3.6617272028031493, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.889311668222454 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9938462624237342, + "adjusted_arithmetic_mean_rank_index": 0.006154570755327993, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251309310513212, + "arithmetic_mean_rank": 7341.395650996259, + "count": 118142.0, + "geometric_mean_rank": 5285.401584977365, + "harmonic_mean_rank": 660.1943822093159, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.0001362138818746672, + "inverse_geometric_mean_rank": 0.00018920038220790797, + "inverse_harmonic_mean_rank": 0.0015147054063888536, + "inverse_median_rank": 0.0001367053998632946, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.364337844292, + "variance": 18786714.213176392, + "z_arithmetic_mean_rank": 3.6608451127618764, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.889260262005685 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9938455159247913, + "adjusted_arithmetic_mean_rank_index": 0.006155317355342249, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251317188252537, + "arithmetic_mean_rank": 7341.39013671875, + "count": 118142.0, + "geometric_mean_rank": 5285.39404296875, + "harmonic_mean_rank": 660.1941329808002, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.00013621398829855025, + "inverse_geometric_mean_rank": 0.00018920065485872328, + "inverse_harmonic_mean_rank": 0.0015147061936191088, + "inverse_median_rank": 0.0001367054064758122, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.3642578125, + "variance": 18786714.0, + "z_arithmetic_mean_rank": 3.6612892033609796, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.8892859338809 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.98490719405311, + "adjusted_arithmetic_mean_rank_index": 0.015094854999328944, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.0007080517688152044, + "arithmetic_mean_rank": 7255.564083899037, + "count": 59071.0, + "geometric_mean_rank": 5224.860477100204, + "harmonic_mean_rank": 714.4891053150517, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001378252591303162, + "inverse_geometric_mean_rank": 0.00019139267055701356, + "inverse_harmonic_mean_rank": 0.0013996014670637324, + "inverse_median_rank": 0.00013900472616068947, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.428105454356, + "variance": 18502283.746392652, + "z_arithmetic_mean_rank": 6.3479662286523535, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.290813338188357 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9849086854543545, + "adjusted_arithmetic_mean_rank_index": 0.015093363395606518, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.0007080504080451104, + "arithmetic_mean_rank": 7255.575070677659, + "count": 59071.0, + "geometric_mean_rank": 5224.8727660877985, + "harmonic_mean_rank": 714.4897995008857, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001378250504279603, + "inverse_geometric_mean_rank": 0.00019139222039827103, + "inverse_harmonic_mean_rank": 0.0013996001072353454, + "inverse_median_rank": 0.00013900472616068947, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.428409444852, + "variance": 18502286.361579273, + "z_arithmetic_mean_rank": 6.347338952003668, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.29078202967029 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9849079732733217, + "adjusted_arithmetic_mean_rank_index": 0.015094075673327434, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.000708051087890621, + "arithmetic_mean_rank": 7255.56982421875, + "count": 59071.0, + "geometric_mean_rank": 5224.86328125, + "harmonic_mean_rank": 714.4894411794969, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.00013782514724880457, + "inverse_geometric_mean_rank": 0.0001913925661938265, + "inverse_harmonic_mean_rank": 0.0013996007866103758, + "inverse_median_rank": 0.00013900472549721599, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.42822265625, + "variance": 18502286.0, + "z_arithmetic_mean_rank": 6.347638492139595, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.290797671515698 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.002733849824362, + "adjusted_arithmetic_mean_rank_index": -0.0027342189664276706, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.0009422128039963093, + "arithmetic_mean_rank": 7427.205312251359, + "count": 59071.0, + "geometric_mean_rank": 5346.618954636671, + "harmonic_mean_rank": 613.5675121777332, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013464014497491746, + "inverse_geometric_mean_rank": 0.00018703408798803297, + "inverse_harmonic_mean_rank": 0.0016298124984660664, + "inverse_median_rank": 0.00013466199838405601, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365254207696, + "variance": 19056413.80264382, + "z_arithmetic_mean_rank": -1.150184055662919, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.744791705554526 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0027353239878083, + "adjusted_arithmetic_mean_rank_index": -0.002735693328925093, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.000942211009837779, + "arithmetic_mean_rank": 7427.216231314858, + "count": 59071.0, + "geometric_mean_rank": 5346.631614801664, + "harmonic_mean_rank": 613.5681871516629, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013463994703476778, + "inverse_geometric_mean_rank": 0.00018703364511435402, + "inverse_harmonic_mean_rank": 0.0016298107055423624, + "inverse_median_rank": 0.00013466199838405601, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365015753101, + "variance": 19056411.720761076, + "z_arithmetic_mean_rank": -1.1508042650380617, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.74475029919349 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0027346092792295, + "adjusted_arithmetic_mean_rank_index": -0.0027349785238417468, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.0009422119055397235, + "arithmetic_mean_rank": 7427.2109375, + "count": 59071.0, + "geometric_mean_rank": 5346.62548828125, + "harmonic_mean_rank": 613.5678263684528, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013464003859553486, + "inverse_geometric_mean_rank": 0.00018703386012930423, + "inverse_harmonic_mean_rank": 0.0016298116006278416, + "inverse_median_rank": 0.00013466199743561447, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365234375, + "variance": 19056412.0, + "z_arithmetic_mean_rank": -1.1505035731696012, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.74477097058761 + } + } + }, + "times": { + "evaluation": 29.358933925628662, + "training": 10.431986570358276 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json new file mode 100644 index 0000000..3d243a1 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json new file mode 100644 index 0000000..28f9432 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 20.45071284740388, + 14.57502553498077, + 10.225484043371134, + 7.005688575639661, + 4.622200210598691 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0126994693360527, + "adjusted_arithmetic_mean_rank_index": -0.012700089965743233, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.241758676361911e-05, + "arithmetic_mean_rank": 20723.1052, + "count": 10000.0, + "geometric_mean_rank": 15394.675497871895, + "harmonic_mean_rank": 3067.5569736687403, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255316486064065e-05, + "inverse_geometric_mean_rank": 6.495752379699309e-05, + "inverse_harmonic_mean_rank": 0.00032599231524753683, + "inverse_median_rank": 4.8173037550882774e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.285208500767, + "variance": 138374878.89653295, + "z_arithmetic_mean_rank": -2.1996636049485403, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273479671066601 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0127007105866408, + "adjusted_arithmetic_mean_rank_index": -0.012701331276992134, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.241714190367465e-05, + "arithmetic_mean_rank": 20723.1306, + "count": 10000.0, + "geometric_mean_rank": 15394.705594314819, + "harmonic_mean_rank": 3067.5611586193463, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255257340317104e-05, + "inverse_geometric_mean_rank": 6.495739680590544e-05, + "inverse_harmonic_mean_rank": 0.00032599187050930123, + "inverse_median_rank": 4.8173037550882774e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.28695570858, + "variance": 138374920.00234362, + "z_arithmetic_mean_rank": -2.1998786008410036, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273409455325398 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0127001505883748, + "adjusted_arithmetic_mean_rank_index": -0.01270077125135849, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.2417364271439425e-05, + "arithmetic_mean_rank": 20723.119140625, + "count": 10000.0, + "geometric_mean_rank": 15394.67578125, + "harmonic_mean_rank": 3067.559378276964, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255282308673486e-05, + "inverse_geometric_mean_rank": 6.495752313639969e-05, + "inverse_harmonic_mean_rank": 0.0003259920928162286, + "inverse_median_rank": 4.8173038521781564e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.2861328125, + "variance": 138374896.0, + "z_arithmetic_mean_rank": -2.199781604047473, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273444553377312 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0164361612514943, + "adjusted_arithmetic_mean_rank_index": -0.016436964513890873, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457574579320083, + "arithmetic_mean_rank": 20799.0878, + "count": 5000.0, + "geometric_mean_rank": 15386.013539591448, + "harmonic_mean_rank": 2391.594541454187, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8079031619838636e-05, + "inverse_geometric_mean_rank": 6.499409333202456e-05, + "inverse_harmonic_mean_rank": 0.0004181310764290168, + "inverse_median_rank": 4.781257470714798e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.276661378286, + "variance": 138751358.66529116, + "z_arithmetic_mean_rank": -2.013056919338908, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.6135653501144027 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0164373341138861, + "adjusted_arithmetic_mean_rank_index": -0.01643813743360245, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457539979149638, + "arithmetic_mean_rank": 20799.1118, + "count": 5000.0, + "geometric_mean_rank": 15386.042101334648, + "harmonic_mean_rank": 2391.596519948779, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.80789761416639e-05, + "inverse_geometric_mean_rank": 6.499397268081413e-05, + "inverse_harmonic_mean_rank": 0.00041813073052197663, + "inverse_median_rank": 4.781257470714798e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.275458639244, + "variance": 138751330.33050075, + "z_arithmetic_mean_rank": -2.013200568377002, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.6135614884959946 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0164368338147722, + "adjusted_arithmetic_mean_rank_index": -0.016437637110037873, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457557275534118, + "arithmetic_mean_rank": 20799.1015625, + "count": 5000.0, + "geometric_mean_rank": 15386.03125, + "harmonic_mean_rank": 2391.5954229298945, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.807900040759705e-05, + "inverse_geometric_mean_rank": 6.499401933979243e-05, + "inverse_harmonic_mean_rank": 0.0004181309034384994, + "inverse_median_rank": 4.781257302965969e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.2763671875, + "variance": 138751344.0, + "z_arithmetic_mean_rank": -2.013139293084194, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.61356341889217 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0089629506723694, + "adjusted_arithmetic_mean_rank_index": -0.008963388686288987, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.974057119360112e-05, + "arithmetic_mean_rank": 20647.1226, + "count": 5000.0, + "geometric_mean_rank": 15403.342332628104, + "harmonic_mean_rank": 4276.180466846908, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843289882920539e-05, + "inverse_geometric_mean_rank": 6.49209748381526e-05, + "inverse_harmonic_mean_rank": 0.00023385355406605694, + "inverse_median_rank": 4.848602390360979e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20624.5, + "standard_deviation": 11746.780512837091, + "variance": 137986852.41676924, + "z_arithmetic_mean_rank": -1.0977583272479166, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.44354261668996875 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.008964260307983, + "adjusted_arithmetic_mean_rank_index": -0.008964698385903747, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.9741114911784364e-05, + "arithmetic_mean_rank": 20647.1494, + "count": 5000.0, + "geometric_mean_rank": 15403.373965489793, + "harmonic_mean_rank": 4276.190406428094, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843283596330252e-05, + "inverse_geometric_mean_rank": 6.492084151436119e-05, + "inverse_harmonic_mean_rank": 0.00023385301049662588, + "inverse_median_rank": 4.848484848484849e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20625.0, + "standard_deviation": 11746.78523633933, + "variance": 137986963.38867965, + "z_arithmetic_mean_rank": -1.0979187279299225, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.4435486851026389 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0089636406133329, + "adjusted_arithmetic_mean_rank_index": -0.008964078660969488, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.9740843140100186e-05, + "arithmetic_mean_rank": 20647.13671875, + "count": 5000.0, + "geometric_mean_rank": 15403.35546875, + "harmonic_mean_rank": 4276.1856863784815, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843286660616286e-05, + "inverse_geometric_mean_rank": 6.492091779364273e-05, + "inverse_harmonic_mean_rank": 0.00023385328219395786, + "inverse_median_rank": 4.8485435399925336e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20624.75, + "standard_deviation": 11746.783203125, + "variance": 137986912.0, + "z_arithmetic_mean_rank": -1.0978428293796014, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.4435456518718546 + } + } + }, + "times": { + "evaluation": 8.131263971328735, + "training": 2.695167064666748 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json new file mode 100644 index 0000000..67eab9e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json new file mode 100644 index 0000000..4aa2bbc --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00020909712704460964, + 0.00020904143676822714, + 0.00020899788573421359, + 0.00020894385538492384, + 0.000208891014557524 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9787776633371456, + "adjusted_arithmetic_mean_rank_index": 0.021225210039617082, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.00030412194368257097, + "arithmetic_mean_rank": 7230.086133635794, + "count": 118142.0, + "geometric_mean_rank": 5227.108734838778, + "harmonic_mean_rank": 1005.9795523048979, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.0001383109386965394, + "inverse_geometric_mean_rank": 0.00019131034970345674, + "inverse_harmonic_mean_rank": 0.0009940559902125272, + "inverse_median_rank": 0.0001398014818957081, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.730639893114, + "variance": 18102732.818045266, + "z_arithmetic_mean_rank": 12.625121967053815, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910686640541236 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9787875751334283, + "adjusted_arithmetic_mean_rank_index": 0.02121529690133661, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.000304114239486995, + "arithmetic_mean_rank": 7230.159350611975, + "count": 118142.0, + "geometric_mean_rank": 5227.181095160295, + "harmonic_mean_rank": 1005.9873435907807, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.0001383095380761363, + "inverse_geometric_mean_rank": 0.00019130770137768384, + "inverse_harmonic_mean_rank": 0.0009940482913339552, + "inverse_median_rank": 0.0001398014818957081, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.740975209824, + "variance": 18102820.766129445, + "z_arithmetic_mean_rank": 12.61922546098242, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910435577210379 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9787826604911695, + "adjusted_arithmetic_mean_rank_index": 0.021220212209008493, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.00030411808652413067, + "arithmetic_mean_rank": 7230.123046875, + "count": 118142.0, + "geometric_mean_rank": 5227.14111328125, + "harmonic_mean_rank": 1005.9834466369628, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.00013831023534294218, + "inverse_geometric_mean_rank": 0.00019130916916765273, + "inverse_harmonic_mean_rank": 0.0009940521357160814, + "inverse_median_rank": 0.00013980148651171476, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.73583984375, + "variance": 18102778.0, + "z_arithmetic_mean_rank": 12.62214917098322, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910560943959922 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9770256831014105, + "adjusted_arithmetic_mean_rank_index": 0.02297743597268609, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.435030430268328e-06, + "arithmetic_mean_rank": 7197.5029710009985, + "count": 59071.0, + "geometric_mean_rank": 5222.128776966808, + "harmonic_mean_rank": 1427.6151482423568, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.00013893707359747352, + "inverse_geometric_mean_rank": 0.0001914927882304799, + "inverse_harmonic_mean_rank": 0.0007004688912352706, + "inverse_median_rank": 0.00014072614691809738, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.108553400409, + "variance": 18097439.584114514, + "z_arithmetic_mean_rank": 9.66289424986968, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.19407268266750854 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9770355943703901, + "adjusted_arithmetic_mean_rank_index": 0.022967523358117692, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.425479968338784e-06, + "arithmetic_mean_rank": 7197.575984831813, + "count": 59071.0, + "geometric_mean_rank": 5222.203535726582, + "harmonic_mean_rank": 1427.6345996903804, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.0001389356641885271, + "inverse_geometric_mean_rank": 0.0001914900469042838, + "inverse_harmonic_mean_rank": 0.0007004593473826398, + "inverse_median_rank": 0.00014072614691809738, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.11876984047, + "variance": 18097526.507908992, + "z_arithmetic_mean_rank": 9.658725614760556, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.19385294620268745 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9770305823451633, + "adjusted_arithmetic_mean_rank_index": 0.02297253606379468, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.430247222894309e-06, + "arithmetic_mean_rank": 7197.5390625, + "count": 59071.0, + "geometric_mean_rank": 5222.1630859375, + "harmonic_mean_rank": 1427.6249653122286, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.00013893637515138835, + "inverse_geometric_mean_rank": 0.00019149153376929462, + "inverse_harmonic_mean_rank": 0.000700464111338066, + "inverse_median_rank": 0.00014072614430915564, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.11376953125, + "variance": 18097482.0, + "z_arithmetic_mean_rank": 9.660833650005259, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.1939626309143474 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9805201333855541, + "adjusted_arithmetic_mean_rank_index": 0.019482496911887748, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998077350576884, + "arithmetic_mean_rank": 7262.669296270589, + "count": 59071.0, + "geometric_mean_rank": 5232.093441728144, + "harmonic_mean_rank": 776.6127185361778, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001376904219655856, + "inverse_geometric_mean_rank": 0.00019112808498880768, + "inverse_harmonic_mean_rank": 0.0012876430891897838, + "inverse_median_rank": 0.0001389274798555154, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.103139408192, + "variance": 18105902.72700145, + "z_arithmetic_mean_rank": 8.195560629086003, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.84262048540465 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9805300457062778, + "adjusted_arithmetic_mean_rank_index": 0.019472583252738485, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998018771214614, + "arithmetic_mean_rank": 7262.742716392138, + "count": 59071.0, + "geometric_mean_rank": 5232.1633989701495, + "harmonic_mean_rank": 776.616249201938, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.00013768903003310063, + "inverse_geometric_mean_rank": 0.00019112552948878292, + "inverse_harmonic_mean_rank": 0.0012876372352852705, + "inverse_median_rank": 0.0001389274798555154, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.113591068842, + "variance": 18105991.672898777, + "z_arithmetic_mean_rank": 8.191390322015991, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.842485293436821 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9805250960743923, + "adjusted_arithmetic_mean_rank_index": 0.019477533552955117, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998048039446685, + "arithmetic_mean_rank": 7262.7060546875, + "count": 59071.0, + "geometric_mean_rank": 5232.12841796875, + "harmonic_mean_rank": 776.6144552693261, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001376897271256894, + "inverse_geometric_mean_rank": 0.00019112680456601083, + "inverse_harmonic_mean_rank": 0.001287640160094097, + "inverse_median_rank": 0.00013892748393118382, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.1083984375, + "variance": 18105946.0, + "z_arithmetic_mean_rank": 8.193472728893369, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.842552839919668 + } + } + }, + "times": { + "evaluation": 28.336533308029175, + "training": 11.485997915267944 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json new file mode 100644 index 0000000..512d190 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json new file mode 100644 index 0000000..7c3c728 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002799568278957982, + 0.002799524831899284, + 0.002799194914067409, + 0.0027994709019207786, + 0.002799336377016278 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9900345170286631, + "adjusted_arithmetic_mean_rank_index": 0.009965969989687906, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.224514997842025e-05, + "arithmetic_mean_rank": 20259.307, + "count": 10000.0, + "geometric_mean_rank": 14914.15475604489, + "harmonic_mean_rank": 4322.3528231180835, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.9360029935870956e-05, + "inverse_geometric_mean_rank": 6.705039718021483e-05, + "inverse_harmonic_mean_rank": 0.00023135547719554608, + "inverse_median_rank": 4.975371909050202e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.0, + "standard_deviation": 11779.53928541991, + "variance": 138757545.776751, + "z_arithmetic_mean_rank": 1.726112297901551, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.6667884027621884 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9900373269463334, + "adjusted_arithmetic_mean_rank_index": 0.009963159934695431, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.224635404830653e-05, + "arithmetic_mean_rank": 20259.3645, + "count": 10000.0, + "geometric_mean_rank": 14914.222346617338, + "harmonic_mean_rank": 4322.375312397938, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.935988984254664e-05, + "inverse_geometric_mean_rank": 6.705009331088642e-05, + "inverse_harmonic_mean_rank": 0.00023135427345508016, + "inverse_median_rank": 4.9752481405010074e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.5, + "standard_deviation": 11779.540283187614, + "variance": 138757569.28323975, + "z_arithmetic_mean_rank": 1.7256255946016665, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.6668074075434873 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9900358357047002, + "adjusted_arithmetic_mean_rank_index": 0.009964651249206313, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.2245752311731276e-05, + "arithmetic_mean_rank": 20259.333984375, + "count": 10000.0, + "geometric_mean_rank": 14914.1748046875, + "harmonic_mean_rank": 4322.363974171985, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.9359965487383306e-05, + "inverse_geometric_mean_rank": 6.705030682496727e-05, + "inverse_harmonic_mean_rank": 0.00023135487502702685, + "inverse_median_rank": 4.9753100029192865e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.25, + "standard_deviation": 11779.5390625, + "variance": 138757552.0, + "z_arithmetic_mean_rank": 1.7258838912170802, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.666797909862213 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9904344373197072, + "adjusted_arithmetic_mean_rank_index": 0.009566030165168882, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842764639345146e-05, + "arithmetic_mean_rank": 20267.0208, + "count": 5000.0, + "geometric_mean_rank": 15026.007280375668, + "harmonic_mean_rank": 4873.627784794362, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.934124308985759e-05, + "inverse_geometric_mean_rank": 6.655127881549907e-05, + "inverse_harmonic_mean_rank": 0.00020518596088112915, + "inverse_median_rank": 5.0261359067149175e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.709523334077, + "variance": 135669137.13996735, + "z_arithmetic_mean_rank": 1.1715644453891514, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.7636998765220726 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9904372521894481, + "adjusted_arithmetic_mean_rank_index": 0.009563215157860516, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842836303762437e-05, + "arithmetic_mean_rank": 20267.0784, + "count": 5000.0, + "geometric_mean_rank": 15026.073723288115, + "harmonic_mean_rank": 4873.644802106642, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.93411028596998e-05, + "inverse_geometric_mean_rank": 6.65509845363099e-05, + "inverse_harmonic_mean_rank": 0.00020518524443302643, + "inverse_median_rank": 5.0261359067149175e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.705968913082, + "variance": 135669054.33825344, + "z_arithmetic_mean_rank": 1.1712196876977146, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.763707874766864 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9904358070336128, + "adjusted_arithmetic_mean_rank_index": 0.00956466038432302, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842800477898023e-05, + "arithmetic_mean_rank": 20267.048828125, + "count": 5000.0, + "geometric_mean_rank": 15026.03125, + "harmonic_mean_rank": 4873.636259270741, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.934117532684468e-05, + "inverse_geometric_mean_rank": 6.655117613263428e-05, + "inverse_harmonic_mean_rank": 0.00020518560259365283, + "inverse_median_rank": 5.026135841035284e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.7080078125, + "variance": 135669104.0, + "z_arithmetic_mean_rank": 1.171396686505986, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.7637038763525288 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9896346152799272, + "adjusted_arithmetic_mean_rank_index": 0.010365891270086691, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.606265386805113e-05, + "arithmetic_mean_rank": 20251.5932, + "count": 5000.0, + "geometric_mean_rank": 14803.134853911464, + "harmonic_mean_rank": 3883.1182417302443, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937883109364453e-05, + "inverse_geometric_mean_rank": 6.755325881097192e-05, + "inverse_harmonic_mean_rank": 0.00025752499350996296, + "inverse_median_rank": 4.927807618390578e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.904928592578, + "variance": 141845835.40811378, + "z_arithmetic_mean_rank": 1.2695247142959403, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.17927451251047488 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9896374202457566, + "adjusted_arithmetic_mean_rank_index": 0.01036308616718018, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.606434536364506e-05, + "arithmetic_mean_rank": 20251.6506, + "count": 5000.0, + "geometric_mean_rank": 14803.203571376178, + "harmonic_mean_rank": 3883.143740317606, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937869113740289e-05, + "inverse_geometric_mean_rank": 6.755294522420967e-05, + "inverse_harmonic_mean_rank": 0.0002575233024771339, + "inverse_median_rank": 4.927807618390578e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.910378324415, + "variance": 141845965.21971962, + "z_arithmetic_mean_rank": 1.2691811695516293, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.1792933912119965 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9896360738071831, + "adjusted_arithmetic_mean_rank_index": 0.010364432671553403, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.6063500149141475e-05, + "arithmetic_mean_rank": 20251.623046875, + "count": 5000.0, + "geometric_mean_rank": 14803.1787109375, + "harmonic_mean_rank": 3883.1308616537235, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937875928590074e-05, + "inverse_geometric_mean_rank": 6.755306094419211e-05, + "inverse_harmonic_mean_rank": 0.00025752414746040096, + "inverse_median_rank": 4.927807458443567e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.9072265625, + "variance": 141845888.0, + "z_arithmetic_mean_rank": 1.2693460777621368, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.17928395781329742 + } + } + }, + "times": { + "evaluation": 7.977590084075928, + "training": 3.7082855701446533 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json new file mode 100644 index 0000000..0749f69 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json new file mode 100644 index 0000000..7df7ae6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00021869831097327203, + 1.6626344659595006e-05, + 1.4145980447915058e-05, + 1.1566674962732687e-05, + 1.0039698529621776e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08082387386970409, + "adjusted_arithmetic_mean_rank_index": 0.9193005770502545, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364597304408337, + "arithmetic_mean_rank": 597.0340268490461, + "count": 118142.0, + "geometric_mean_rank": 48.53253129682446, + "harmonic_mean_rank": 4.460666021875165, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.0016749464101362512, + "inverse_geometric_mean_rank": 0.020604736107498915, + "inverse_harmonic_mean_rank": 0.2241817690667687, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.957313717163, + "variance": 2030503.3459160337, + "z_arithmetic_mean_rank": 546.8158801717057, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.146098304282 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08082393460094144, + "adjusted_arithmetic_mean_rank_index": 0.9193005163107946, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364585976837215, + "arithmetic_mean_rank": 597.0344754617325, + "count": 118142.0, + "geometric_mean_rank": 48.53260829072401, + "harmonic_mean_rank": 4.46066827422893, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.001674945151578765, + "inverse_geometric_mean_rank": 0.02060470341939419, + "inverse_harmonic_mean_rank": 0.22418165586923403, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.9584387937286, + "variance": 2030506.5522894603, + "z_arithmetic_mean_rank": 546.8158440428246, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.142406890027 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08082390282298467, + "adjusted_arithmetic_mean_rank_index": 0.9193005480930538, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364591371526105, + "arithmetic_mean_rank": 597.0342407226562, + "count": 118142.0, + "geometric_mean_rank": 48.53256607055664, + "harmonic_mean_rank": 4.460667229528553, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.0016749458154663444, + "inverse_geometric_mean_rank": 0.020604722201824188, + "inverse_harmonic_mean_rank": 0.22418170977889182, + "inverse_median_rank": 0.0181818176060915, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.9578857421875, + "variance": 2030504.875, + "z_arithmetic_mean_rank": 546.8158629474619, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.144164904279 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08628437326551358, + "adjusted_arithmetic_mean_rank_index": 0.9138396759828533, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062081917111348, + "arithmetic_mean_rank": 635.6353202078855, + "count": 59071.0, + "geometric_mean_rank": 60.487063947463966, + "harmonic_mean_rank": 5.230646899912632, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732291271557228, + "inverse_geometric_mean_rank": 0.016532460574852, + "inverse_harmonic_mean_rank": 0.19118094169512823, + "inverse_median_rank": 0.013513513513513514, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.533801090035, + "variance": 2095354.1052981652, + "z_arithmetic_mean_rank": 384.3046787663501, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.7925652603535 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08628443760948097, + "adjusted_arithmetic_mean_rank_index": 0.9138396116301504, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062063063034304, + "arithmetic_mean_rank": 635.6357942137428, + "count": 59071.0, + "geometric_mean_rank": 60.487188318126684, + "harmonic_mean_rank": 5.230652054760578, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732279539684543, + "inverse_geometric_mean_rank": 0.016532426581652198, + "inverse_harmonic_mean_rank": 0.1911807532848355, + "inverse_median_rank": 0.013513513513513514, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.5347027698704, + "variance": 2095356.715723057, + "z_arithmetic_mean_rank": 384.3046517035671, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.788227325427 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08628440569150582, + "adjusted_arithmetic_mean_rank_index": 0.9138396435524588, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062071994722662, + "arithmetic_mean_rank": 635.6355590820312, + "count": 59071.0, + "geometric_mean_rank": 60.487117767333984, + "harmonic_mean_rank": 5.230649339059701, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732285100966692, + "inverse_geometric_mean_rank": 0.016532445326447487, + "inverse_harmonic_mean_rank": 0.19118084253990825, + "inverse_median_rank": 0.013513513840734959, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.5341796875, + "variance": 2095355.375, + "z_arithmetic_mean_rank": 384.30466512812376, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.790282323017 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07539301543164696, + "adjusted_arithmetic_mean_rank_index": 0.9247318309769208, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.25667100161503614, + "arithmetic_mean_rank": 558.4327334902067, + "count": 59071.0, + "geometric_mean_rank": 38.94066665432855, + "harmonic_mean_rank": 3.888287986234258, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.001790725972938578, + "inverse_geometric_mean_rank": 0.025680094510884353, + "inverse_harmonic_mean_rank": 0.25718259643840924, + "inverse_median_rank": 0.025, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.9541273132224, + "variance": 1962672.4668359524, + "z_arithmetic_mean_rank": 389.0002303448466, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.563597631679 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07539307256976503, + "adjusted_arithmetic_mean_rank_index": 0.9247317738310875, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.2566709636040986, + "arithmetic_mean_rank": 558.4331567097222, + "count": 59071.0, + "geometric_mean_rank": 38.94071014034867, + "harmonic_mean_rank": 3.888288560517955, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.0017907246158017935, + "inverse_geometric_mean_rank": 0.0256800658333101, + "inverse_harmonic_mean_rank": 0.25718255845363264, + "inverse_median_rank": 0.025, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.9554829604954, + "variance": 1962676.265237075, + "z_arithmetic_mean_rank": 389.00020630572357, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.562720398939 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07539304093106662, + "adjusted_arithmetic_mean_rank_index": 0.924731805474058, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.25667098218112694, + "arithmetic_mean_rank": 558.4329223632812, + "count": 59071.0, + "geometric_mean_rank": 38.940673828125, + "harmonic_mean_rank": 3.8882882236595933, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.0017907253932207823, + "inverse_geometric_mean_rank": 0.025680089369416237, + "inverse_harmonic_mean_rank": 0.2571825770178754, + "inverse_median_rank": 0.02500000037252903, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.954833984375, + "variance": 1962674.375, + "z_arithmetic_mean_rank": 389.00021961674236, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.563149127577 + } + } + }, + "times": { + "evaluation": 42.46483302116394, + "training": 859.2714371681213 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..2a76f64 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..684147b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002791654386314053, + 7.358028466998594e-05, + 2.835282286703969e-05, + 2.1093817337828796e-05, + 1.892652983966786e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5663228397885496, + "adjusted_arithmetic_mean_rank_index": 0.4337379446939893, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.02716977127692903, + "arithmetic_mean_rank": 4041.092303552207, + "count": 40876.0, + "geometric_mean_rank": 1230.6452823359225, + "harmonic_mean_rank": 35.88982075732999, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.00024745784676112907, + "inverse_geometric_mean_rank": 0.0008125818335742301, + "inverse_harmonic_mean_rank": 0.027863053615160897, + "inverse_median_rank": 0.00033500837520938025, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.5773194486583, + "variance": 15035605.868302641, + "z_arithmetic_mean_rank": 151.69269004908605, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.6232337760049 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5663232752007933, + "adjusted_arithmetic_mean_rank_index": 0.43373750922071796, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.0271697708446242, + "arithmetic_mean_rank": 4041.0954105098344, + "count": 40876.0, + "geometric_mean_rank": 1230.6462909069642, + "harmonic_mean_rank": 35.88982131377605, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.00024745765650552597, + "inverse_geometric_mean_rank": 0.0008125811676261731, + "inverse_harmonic_mean_rank": 0.027863053183164146, + "inverse_median_rank": 0.00033500837520938025, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.578950153829, + "variance": 15035618.51467607, + "z_arithmetic_mean_rank": 151.69253774949414, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.6232256354434 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5663230424952339, + "adjusted_arithmetic_mean_rank_index": 0.4337377419588935, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.027169771060678034, + "arithmetic_mean_rank": 4041.09375, + "count": 40876.0, + "geometric_mean_rank": 1230.645263671875, + "harmonic_mean_rank": 35.88982087682511, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.0002474577631801367, + "inverse_geometric_mean_rank": 0.0008125818567350507, + "inverse_harmonic_mean_rank": 0.02786305339906401, + "inverse_median_rank": 0.0003350083716213703, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.578125, + "variance": 15035612.0, + "z_arithmetic_mean_rank": 151.69261914583115, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.62322970386776 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6753109059937, + "adjusted_arithmetic_mean_rank_index": 0.32473524320845226, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.0050529021797449565, + "arithmetic_mean_rank": 4751.918583031607, + "count": 20438.0, + "geometric_mean_rank": 2508.5323512557943, + "harmonic_mean_rank": 173.2350944902231, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.00021044131597095346, + "inverse_geometric_mean_rank": 0.0003986394672165144, + "inverse_harmonic_mean_rank": 0.005772502407452072, + "inverse_median_rank": 0.00024783147459727387, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.766512579791, + "variance": 14173466.894242203, + "z_arithmetic_mean_rank": 80.2213637364448, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.72758692545682 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6753114066375367, + "adjusted_arithmetic_mean_rank_index": 0.32473474249345724, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.005052901714946772, + "arithmetic_mean_rank": 4751.922105881202, + "count": 20438.0, + "geometric_mean_rank": 2508.5346523933813, + "harmonic_mean_rank": 173.2351084289142, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.0002104411599597462, + "inverse_geometric_mean_rank": 0.0003986391015351949, + "inverse_harmonic_mean_rank": 0.005772501942990055, + "inverse_median_rank": 0.00024783147459727387, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.7675746432033, + "variance": 14173474.891084868, + "z_arithmetic_mean_rank": 80.22124004171013, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.7275807874276 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6753110962611738, + "adjusted_arithmetic_mean_rank_index": 0.3247350529139351, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.005052901947239784, + "arithmetic_mean_rank": 4751.919921875, + "count": 20438.0, + "geometric_mean_rank": 2508.53369140625, + "harmonic_mean_rank": 173.23509892383208, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.00021044125605840236, + "inverse_geometric_mean_rank": 0.0003986392403021455, + "inverse_harmonic_mean_rank": 0.00577250217511506, + "inverse_median_rank": 0.0002478314854670316, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.76708984375, + "variance": 14173471.0, + "z_arithmetic_mean_rank": 80.22131672680848, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.72758385504133 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.46031849371867495, + "adjusted_arithmetic_mean_rank_index": 0.5397561128500036, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.049286170723083396, + "arithmetic_mean_rank": 3330.2660240728055, + "count": 20438.0, + "geometric_mean_rank": 603.7346140572965, + "harmonic_mean_rank": 20.018575306945234, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.00030027631209384077, + "inverse_geometric_mean_rank": 0.0016563569103312942, + "inverse_harmonic_mean_rank": 0.049953604822869715, + "inverse_median_rank": 0.0006263701847792045, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.3930389692055, + "variance": 14887196.843166022, + "z_arithmetic_mean_rank": 133.64077253097005, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883557742102 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4603188656851434, + "adjusted_arithmetic_mean_rank_index": 0.5397557408321139, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.04928617032327123, + "arithmetic_mean_rank": 3330.2687151384675, + "count": 20438.0, + "geometric_mean_rank": 603.7350498140833, + "harmonic_mean_rank": 20.018575467054823, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.00030027606945177743, + "inverse_geometric_mean_rank": 0.0016563557148254757, + "inverse_harmonic_mean_rank": 0.04995360442333823, + "inverse_median_rank": 0.0006263701847792045, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.3951270654115, + "variance": 14887212.956562115, + "z_arithmetic_mean_rank": 133.64068042129867, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883504057462 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.46031864103245473, + "adjusted_arithmetic_mean_rank_index": 0.5397559655158588, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.049286170523086245, + "arithmetic_mean_rank": 3330.26708984375, + "count": 20438.0, + "geometric_mean_rank": 603.7346801757812, + "harmonic_mean_rank": 20.018575172499716, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.0003002762096002698, + "inverse_geometric_mean_rank": 0.0016563567332923412, + "inverse_harmonic_mean_rank": 0.049953604623012966, + "inverse_median_rank": 0.0006263701943680644, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.394287109375, + "variance": 14887206.0, + "z_arithmetic_mean_rank": 133.6407360518113, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883530887553 + } + } + }, + "times": { + "evaluation": 15.475303888320923, + "training": 466.48984026908875 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json new file mode 100644 index 0000000..9cea214 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json new file mode 100644 index 0000000..93f0ec8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001784344254726013, + 3.546414034157781e-06, + 3.241080858006321e-06, + 3.114976756636245e-06, + 2.580594582893777e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.173634136893227, + "adjusted_arithmetic_mean_rank_index": 0.8264062480375431, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.14717656877035404, + "arithmetic_mean_rank": 3553.1158, + "count": 10000.0, + "geometric_mean_rank": 217.95702597254055, + "harmonic_mean_rank": 6.783805257873866, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.0002814431209925666, + "inverse_geometric_mean_rank": 0.00458806040107184, + "inverse_harmonic_mean_rank": 0.14740989193923487, + "inverse_median_rank": 0.0030581039755351682, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.022572537755, + "variance": 49421217.370390356, + "z_arithmetic_mean_rank": 143.13408421621838, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.0034516277847 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.17363431281850722, + "adjusted_arithmetic_mean_rank_index": 0.8264060721036653, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.14717656856459646, + "arithmetic_mean_rank": 3553.1194, + "count": 10000.0, + "geometric_mean_rank": 217.9571490546336, + "harmonic_mean_rank": 6.783805267340243, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.0002814428358360262, + "inverse_geometric_mean_rank": 0.004588057810158537, + "inverse_harmonic_mean_rank": 0.14740989173353358, + "inverse_median_rank": 0.0030581039755351682, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.028042059551, + "variance": 49421294.27214364, + "z_arithmetic_mean_rank": 143.13405374435962, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.003448380151 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.17363422855915533, + "adjusted_arithmetic_mean_rank_index": 0.826406156367135, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.1471765686674477, + "arithmetic_mean_rank": 3553.11767578125, + "count": 10000.0, + "geometric_mean_rank": 217.95701599121094, + "harmonic_mean_rank": 6.783805453179238, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.00028144297539256513, + "inverse_geometric_mean_rank": 0.004588060546666384, + "inverse_harmonic_mean_rank": 0.14740989183635667, + "inverse_median_rank": 0.0030581040773540735, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.025390625, + "variance": 49421256.0, + "z_arithmetic_mean_rank": 143.1340683388458, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.0034500035326 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.18254785061091422, + "adjusted_arithmetic_mean_rank_index": 0.8174920996282304, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.13370304480394207, + "arithmetic_mean_rank": 3735.4326, + "count": 5000.0, + "geometric_mean_rank": 251.60200046021046, + "harmonic_mean_rank": 7.466026264434069, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.00026770661047397827, + "inverse_geometric_mean_rank": 0.003974531196774585, + "inverse_harmonic_mean_rank": 0.1339400592204856, + "inverse_median_rank": 0.0025284450063211127, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.8929429127775, + "variance": 51178184.23865724, + "z_arithmetic_mean_rank": 100.11934541020389, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184846206778 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1825480656356861, + "adjusted_arithmetic_mean_rank_index": 0.81749188459295, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.13370304443324765, + "arithmetic_mean_rank": 3735.437, + "count": 5000.0, + "geometric_mean_rank": 251.60221622118326, + "harmonic_mean_rank": 7.466026285091496, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.00026770629514030086, + "inverse_geometric_mean_rank": 0.003974527788423378, + "inverse_harmonic_mean_rank": 0.1339400588498926, + "inverse_median_rank": 0.0025284450063211127, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.898885644877, + "variance": 51178269.266031004, + "z_arithmetic_mean_rank": 100.11931907454691, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184804834717 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.18254795882961378, + "adjusted_arithmetic_mean_rank_index": 0.817491991404242, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.1337030446185418, + "arithmetic_mean_rank": 3735.434814453125, + "count": 5000.0, + "geometric_mean_rank": 251.6021728515625, + "harmonic_mean_rank": 7.46602602719376, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.0002677064621821046, + "inverse_geometric_mean_rank": 0.003974528517574072, + "inverse_harmonic_mean_rank": 0.13394005903513603, + "inverse_median_rank": 0.0025284450966864824, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.8955078125, + "variance": 51178224.0, + "z_arithmetic_mean_rank": 100.11933215586801, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184825514823 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.16472083645996619, + "adjusted_arithmetic_mean_rank_index": 0.8353199831220769, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009257998678, + "arithmetic_mean_rank": 3370.799, + "count": 5000.0, + "geometric_mean_rank": 188.81115843237254, + "harmonic_mean_rank": 6.215823666567742, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.0002966655680151798, + "inverse_geometric_mean_rank": 0.0052962971484451496, + "inverse_harmonic_mean_rank": 0.1608797246579842, + "inverse_median_rank": 0.003703703703703704, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.113832297522, + "variance": 47597771.670999005, + "z_arithmetic_mean_rank": 102.30276734417879, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080090516244 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.16472097328756763, + "adjusted_arithmetic_mean_rank_index": 0.8353198462877888, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009253916593, + "arithmetic_mean_rank": 3370.8018, + "count": 5000.0, + "geometric_mean_rank": 188.81120976399464, + "harmonic_mean_rank": 6.215823668144484, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.00029666532158609856, + "inverse_geometric_mean_rank": 0.00529629570855435, + "inverse_harmonic_mean_rank": 0.1608797246171745, + "inverse_median_rank": 0.003703703703703704, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.118774431758, + "variance": 47597839.86371676, + "z_arithmetic_mean_rank": 102.30275058589858, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080085960246 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.16472089964347075, + "adjusted_arithmetic_mean_rank_index": 0.8353199199354846, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009255957435, + "arithmetic_mean_rank": 3370.80029296875, + "count": 5000.0, + "geometric_mean_rank": 188.81117248535156, + "harmonic_mean_rank": 6.215823215674801, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.00029666544287465513, + "inverse_geometric_mean_rank": 0.005296296905726194, + "inverse_harmonic_mean_rank": 0.16087972463757735, + "inverse_median_rank": 0.003703703638166189, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.11572265625, + "variance": 47597800.0, + "z_arithmetic_mean_rank": 102.30275960563142, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080088238021 + } + } + }, + "times": { + "evaluation": 6.739559888839722, + "training": 597.220596075058 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json new file mode 100644 index 0000000..c14d63d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json new file mode 100644 index 0000000..04dd837 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001750188616934491, + 3.981787474154627e-06, + 3.557281189895917e-06, + 3.3347453562454225e-06, + 3.199828891147623e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7015087458919147, + "adjusted_arithmetic_mean_rank_index": 0.29850597874848384, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.020728818694365003, + "arithmetic_mean_rank": 14221.369699042407, + "count": 5848.0, + "geometric_mean_rank": 4766.871536494683, + "harmonic_mean_rank": 47.621246158623535, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031671499738417e-05, + "inverse_geometric_mean_rank": 0.0002097811934607639, + "inverse_harmonic_mean_rank": 0.020999030488808704, + "inverse_median_rank": 9.231053263177328e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.902335092624, + "variance": 162509013.94505996, + "z_arithmetic_mean_rank": 39.53722336009449, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.0339912312517 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7015092266860429, + "adjusted_arithmetic_mean_rank_index": 0.29850549793063796, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.0207288182389401, + "arithmetic_mean_rank": 14221.379445964432, + "count": 5848.0, + "geometric_mean_rank": 4766.876381520779, + "harmonic_mean_rank": 47.62124719114347, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031666680434208e-05, + "inverse_geometric_mean_rank": 0.00020978098024034963, + "inverse_harmonic_mean_rank": 0.020999030033509468, + "inverse_median_rank": 9.231053263177328e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.90679032108, + "variance": 162509127.5347143, + "z_arithmetic_mean_rank": 39.537159675599284, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.0339857598217 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7015089110330996, + "adjusted_arithmetic_mean_rank_index": 0.2985058135991524, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.02072881846653769, + "arithmetic_mean_rank": 14221.373046875, + "count": 5848.0, + "geometric_mean_rank": 4766.87451171875, + "harmonic_mean_rank": 47.621243309185644, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031669520074502e-05, + "inverse_geometric_mean_rank": 0.0002097810647683218, + "inverse_harmonic_mean_rank": 0.020999030261044258, + "inverse_median_rank": 9.231053263647482e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.904296875, + "variance": 162509072.0, + "z_arithmetic_mean_rank": 39.53720148600662, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.03398849415677 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8128474155111893, + "adjusted_arithmetic_mean_rank_index": 0.18716181881646976, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.01546072167844193, + "arithmetic_mean_rank": 16474.83242134063, + "count": 2924.0, + "geometric_mean_rank": 7222.092081377499, + "harmonic_mean_rank": 63.56292193252177, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.069864472215527e-05, + "inverse_geometric_mean_rank": 0.00013846403351440873, + "inverse_harmonic_mean_rank": 0.015732442272896097, + "inverse_median_rank": 6.789327177676693e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.96661470027, + "variance": 165764765.32964653, + "z_arithmetic_mean_rank": 17.528919271821408, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.3259227117884 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8128478036065856, + "adjusted_arithmetic_mean_rank_index": 0.18716143070192437, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.015460721633633155, + "arithmetic_mean_rank": 16474.8402872777, + "count": 2924.0, + "geometric_mean_rank": 7222.095610109252, + "harmonic_mean_rank": 63.56292211351025, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.0698615741496806e-05, + "inverse_geometric_mean_rank": 0.0001384639658605783, + "inverse_harmonic_mean_rank": 0.015732442228099688, + "inverse_median_rank": 6.789327177676693e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.96983860059, + "variance": 165764848.3448749, + "z_arithmetic_mean_rank": 17.528882922374834, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.32592233117524 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8128476853587696, + "adjusted_arithmetic_mean_rank_index": 0.1871615489555749, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.015460721656036102, + "arithmetic_mean_rank": 16474.837890625, + "count": 2924.0, + "geometric_mean_rank": 7222.09326171875, + "harmonic_mean_rank": 63.56292443713892, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.069862502044998e-05, + "inverse_geometric_mean_rank": 0.00013846400543116033, + "inverse_harmonic_mean_rank": 0.015732442250496453, + "inverse_median_rank": 6.78932701703161e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.9677734375, + "variance": 165764800.0, + "z_arithmetic_mean_rank": 17.52889399759684, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.32592252146958 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5902194668953142, + "adjusted_arithmetic_mean_rank_index": 0.4098007431862519, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.02599691511975857, + "arithmetic_mean_rank": 11967.906976744185, + "count": 2924.0, + "geometric_mean_rank": 3146.3271292311133, + "harmonic_mean_rank": 38.07258497285074, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355679919163656e-05, + "inverse_geometric_mean_rank": 0.00031783090534657024, + "inverse_harmonic_mean_rank": 0.026265618704721314, + "inverse_median_rank": 0.00013768415255404102, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.531277503775, + "variance": 149097074.07889798, + "z_arithmetic_mean_rank": 38.38054590290827, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129861041845 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5902200403470523, + "adjusted_arithmetic_mean_rank_index": 0.40980016970623157, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.02599691425371758, + "arithmetic_mean_rank": 11967.918604651162, + "count": 2924.0, + "geometric_mean_rank": 3146.331987753468, + "harmonic_mean_rank": 38.07258622784969, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355671800870739e-05, + "inverse_geometric_mean_rank": 0.00031783041455648046, + "inverse_harmonic_mean_rank": 0.026265617838919245, + "inverse_median_rank": 0.00013768415255404102, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.53787504421, + "variance": 149097235.19788918, + "z_arithmetic_mean_rank": 38.38049219271713, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129125248399 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5902197200204955, + "adjusted_arithmetic_mean_rank_index": 0.40980049004858665, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.0259969146865098, + "arithmetic_mean_rank": 11967.912109375, + "count": 2924.0, + "geometric_mean_rank": 3146.3310546875, + "harmonic_mean_rank": 38.0725823227289, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355676254723221e-05, + "inverse_geometric_mean_rank": 0.00031783050508238375, + "inverse_harmonic_mean_rank": 0.026265618271592066, + "inverse_median_rank": 0.00013768415374215692, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.5341796875, + "variance": 149097152.0, + "z_arithmetic_mean_rank": 38.380522194894205, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129492951178 + } + } + }, + "times": { + "evaluation": 5.12530517578125, + "training": 358.1025986671448 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/summary.tsv b/nonasv/results/v1.8.0/summary.tsv new file mode 100644 index 0000000..1909631 --- /dev/null +++ b/nonasv/results/v1.8.0/summary.tsv @@ -0,0 +1,39 @@ +batch_size.evaluation evaluation path training + 29.81851315498352 complex/trouillon2016_complex_fb15k.json 1.640441608428955 + 4.551660537719727 complex/trouillon2016_complex_wn18.json 0.3536832332611084 + 14.673102378845215 conve/dettmers2018_conve_fb15k.json 34.67542872428894 + 6.211552381515503 conve/dettmers2018_conve_fb15k237.json 19.024553966522216 + 3.723423957824707 conve/dettmers2018_conve_wn18.json 33.473473691940306 + 2.1725149154663086 conve/dettmers2018_conve_wn18rr.json 20.13378930091858 + 113.07699227333069 convkb/nguyen2018_convkb_fb15k237.json 5.8175290584564205 + 228.67922687530518 convkb/nguyen2018_convkb_wn18rr.json 2.3075665473937987 + 9.182297706604004 convkb/nguyen2018_transe_fb15k237.json 2.377616119384766 + 20.92709469795227 distmult/yang2014_distmult_fb15k.json 3.150154638290405 + 5.345269203186035 distmult/yang2014_distmult_wn18.json 1.189062738418579 + 34.076919078826904 hole/nickel2016_hole_fb15k.json 0.8448263168334961 + 7.570884943008423 hole/nickel2016_hole_wn18.json 0.4609029293060303 + 28.00254464149475 kg2e/he2015_kg2e_fb15k.json 2.760317659378052 + 7.501351594924927 kg2e/he2015_kg2e_wn18.json 39.696214532852174 + 14.712653875350952 mure/balazevic2019_mure_fb15k237.json 13.239705181121826 + 143.83830213546753 quate/zhang2019_quate_fb15k.json 1.456200647354126 + 33.202207803726196 quate/zhang2019_quate_fb15k237.json 0.8993231296539307 + 50.53933072090149 quate/zhang2019_quate_wn18.json 2.752012538909912 + 13.765980958938599 quate/zhang2019_quate_wn18rr.json 0.9874269962310791 + 88.11779618263245 rotate/sun2019_rotate_fb15k.json 80.48430371284485 + 28.418932676315308 rotate/sun2019_rotate_fb15k237.json 88.11092224121094 + 11.729146718978882 rotate/sun2019_rotate_wn18.json 94.30107221603393 + 7.746156454086304 rotate/sun2019_rotate_wn18rr.json 58.11595721244812 + 29.054805994033813 simple/kazemi2018_simple_fb15k.json 2.0225048065185547 + 8.916642427444458 simple/kazemi2018_simple_wn18.json 0.9781480312347413 + 33.93540859222412 transd/ji2015_transd_fb15k.json 17.45572304725647 + 9.962103366851807 transd/ji2015_transd_wn18.json 5.07728943824768 + 16.924992561340332 transe/bordes2013_transe_fb15k.json 23.030786180496214 + 4.560383558273315 transe/bordes2013_transe_wn18.json 6.884837055206299 + 29.358933925628662 transh/wang2014_transh_fb15k.json 2.086397314071655 + 8.131263971328735 transh/wang2014_transh_wn18.json 0.5390334129333496 + 28.336533308029175 transr/li2015_transr_fb15k.json 2.297199583053589 + 7.977590084075928 transr/li2015_transr_wn18.json 0.7416571140289306 + 42.46483302116394 tucker/balazevic2019_tucker_fb15k.json 171.85428743362428 + 15.475303888320923 tucker/balazevic2019_tucker_fb15k237.json 93.29796805381775 + 6.739559888839722 tucker/balazevic2019_tucker_wn18.json 119.4441192150116 + 5.12530517578125 tucker/balazevic2019_tucker_wn18rr.json 71.62051973342895 From 735e93c19ff4872cb98ca6e561b2206391b64997 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 19 Apr 2022 14:06:51 +0200 Subject: [PATCH 6/7] fail-safe --- nonasv/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nonasv/pipeline.py b/nonasv/pipeline.py index a6ac2e9..498265e 100644 --- a/nonasv/pipeline.py +++ b/nonasv/pipeline.py @@ -156,7 +156,9 @@ def main( { "training": results["times"]["training"] / num_epochs, "evaluation": results["times"]["evaluation"], - "batch_size.evaluation": results["evaluator_metadata"]["batch_size"], + "batch_size.evaluation": results.get("evaluator_metadata", {}).get( + "batch_size", None + ), "path": path.relative_to(configuration_root).as_posix(), } ) From 8190749378c391f63c7a3b99c40ce45fbb9c40d9 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Tue, 31 May 2022 14:42:19 +0200 Subject: [PATCH 7/7] update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 028a987..f1a067d 100644 --- a/.gitignore +++ b/.gitignore @@ -405,3 +405,6 @@ tags pykeen/ html/ + +# slurm +slurm-*.out \ No newline at end of file