diff --git a/.gitignore b/.gitignore index 028a987..f1a067d 100644 --- a/.gitignore +++ b/.gitignore @@ -405,3 +405,6 @@ tags pykeen/ html/ + +# slurm +slurm-*.out \ No newline at end of file diff --git a/nonasv/pipeline.py b/nonasv/pipeline.py new file mode 100644 index 0000000..498265e --- /dev/null +++ b/nonasv/pipeline.py @@ -0,0 +1,179 @@ +import json +import logging +import pathlib +import pprint +from typing import Any, Mapping +import platform +import click +import more_click +import pandas +import torch + +from pykeen.experiments.cli import HERE +from pykeen.pipeline.api import pipeline_from_config +from pykeen.utils import ( + CONFIGURATION_FILE_FORMATS, + format_relative_comparison, + load_configuration, + resolve_device, +) +from pykeen.version import get_git_hash + +logger = logging.getLogger(__name__) + + +def _collect_system_information() -> Mapping[str, Any]: + uname = platform.uname() + result = dict( + system=uname.system, + release=uname.release, + machine=uname.machine, + torch_version=torch.__version__, + ) + if torch.cuda.is_available(): + # e.g. _CudaDeviceProperties(name='Quadro RTX 8000', major=7, minor=5, total_memory=48601MB, multi_processor_count=72) + properties = torch.cuda.get_device_properties(device=torch.device("cuda")) + result["gpu"] = dict( + name=properties.name, + total_memory=properties.total_memory, + compute_capability=(properties.major, properties.minor), + cuda=torch.version.cuda, + cudnn=torch.backends.cudnn.version(), + ) + return result + + +@click.command() +@click.option("-c", "--configuration-root", type=pathlib.Path, default=HERE) +@click.option( + "-o", + "--output-root", + type=pathlib.Path, + default=pathlib.Path(__file__).parent.joinpath("results", get_git_hash()), +) +@click.option("-e", "--num-epochs", type=int, default=5) +@click.option("--debug", is_flag=True) +@more_click.log_level_option() +def main( + configuration_root: pathlib.Path, + output_root: pathlib.Path, + num_epochs: int, + log_level: str, + debug: bool, +): + """ + Run all packaged reproducibility configurations and save results for a given commit. + + Also reduces the number of training epochs to reduce runtime. + """ + logging.basicConfig(level=log_level) + + # make absolute + output_root = output_root.expanduser().resolve() + logger.info(f"Resolved output root: {output_root}") + + device = resolve_device(device=None) + logger.info(f"Running on device: {device}") + + system_information = _collect_system_information() + + configuration_paths = sorted( + path + for ext in CONFIGURATION_FILE_FORMATS + for path in configuration_root.rglob(f"*{ext}") + ) + logger.info( + f"Found {len(configuration_paths)} configurations under {configuration_root}" + ) + + for i, path in enumerate(configuration_paths, start=1): + logger.info( + f"Progress: {format_relative_comparison(part=i, total=len(configuration_paths))}" + ) + reference, model, dataset, *remainder = path.stem.split("_") + if model in { + "nodepiece", # no precomputed anchors... + "rgcn", # too slow + "boxe", + }: + logger.warning(f"Skipping {path} due to explicit model ignore rule") + continue + + output_path = output_root.joinpath( + device.type, model, dataset, "_".join((reference, *remainder)) + ) + if output_path.exists(): + logger.debug( + f"Skipping configuration {path} since output path exists {output_path}" + ) + continue + + # load configuration + configuration = dict(load_configuration(path)) + # reduce number of training epochs + configuration["pipeline"]["training_kwargs"]["num_epochs"] = num_epochs + # discard results + configuration.pop("results", None) + # add system information to metadata + configuration.setdefault("metadata", {}) + configuration["metadata"]["system"] = system_information + + logger.info(f"Running configuration from {path}") + logger.debug(pprint.pformat(configuration, indent=2, sort_dicts=True)) + try: + result = pipeline_from_config(config=configuration) + except TypeError as error: + logger.error("Could not run pipeline", exc_info=error) + continue + + # save results + result.save_to_directory( + directory=output_path, + save_metadata=True, + save_replicates=False, + save_training=False, + ) + + if debug: + break + + data = [] + for i, path in enumerate(configuration_paths, start=1): + reference, model, dataset, *remainder = path.stem.split("_") + output_path = output_root.joinpath( + device.type, + model, + dataset, + "_".join((reference, *remainder)), + "results.json", + ) + if not output_path.exists(): + logger.warning(f"{output_path} is not existing") + continue + + results = json.loads(output_path.read_text()) + data.append( + { + "training": results["times"]["training"] / num_epochs, + "evaluation": results["times"]["evaluation"], + "batch_size.evaluation": results.get("evaluator_metadata", {}).get( + "batch_size", None + ), + "path": path.relative_to(configuration_root).as_posix(), + } + ) + df = pandas.DataFrame.from_records(data=data) + df = df[sorted(df.columns)] + df = df.sort_values(by="path") + output_path = output_root.joinpath("summary.tsv") + df.to_csv(output_path, sep="\t", index=False) + logger.info(f"Written summary to {output_path}") + + # print + print(f"{'path':50} training evaluation") + for _, row in df.iterrows(): + print(f"{row['path']:50} {row['training']:8.2f}s {row['evaluation']:8.2f}s") + + +if __name__ == "__main__": + main() diff --git a/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json new file mode 100644 index 0000000..b62165b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json new file mode 100644 index 0000000..79e0b72 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/fb15k/trouillon2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00027370825300399417, + 5.7055877953340055e-05, + 3.4271220052227624e-05, + 2.3875505992467534e-05, + 1.8529287064680124e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08070188773133745, + "adjusted_arithmetic_mean_rank_index": 0.9194225797048108, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393374384795, + "arithmetic_mean_rank": 596.132933249818, + "count": 118142.0, + "geometric_mean_rank": 59.0259733036713, + "harmonic_mean_rank": 8.15661454878056, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774782002874778, + "inverse_geometric_mean_rank": 0.016941694376732996, + "inverse_harmonic_mean_rank": 0.1225998843048803, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5974290660563, + "variance": 2711283.093406946, + "z_arithmetic_mean_rank": 546.8884494603583, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225642125115 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08070189231482705, + "adjusted_arithmetic_mean_rank_index": 0.9194225751207006, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393369414213, + "arithmetic_mean_rank": 596.1329671073793, + "count": 118142.0, + "geometric_mean_rank": 59.025974985687476, + "harmonic_mean_rank": 8.156614552085223, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774781050145706, + "inverse_geometric_mean_rank": 0.01694169389395903, + "inverse_harmonic_mean_rank": 0.12259988425520878, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5974825402982, + "variance": 2711283.2695080475, + "z_arithmetic_mean_rank": 546.8884467336502, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225625927046 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08070189617277465, + "adjusted_arithmetic_mean_rank_index": 0.9194225712622306, + "adjusted_hits_at_k": 0.25467888530694455, + "adjusted_inverse_harmonic_mean_rank": 0.12199393371896833, + "arithmetic_mean_rank": 596.1329956054688, + "count": 118142.0, + "geometric_mean_rank": 59.02597427368164, + "harmonic_mean_rank": 8.156614672134552, + "hits_at_1": 0.05637283946437338, + "hits_at_10": 0.25518443906485416, + "hits_at_3": 0.12941206344906978, + "hits_at_5": 0.17772680333835553, + "inverse_arithmetic_mean_rank": 0.0016774780815467238, + "inverse_geometric_mean_rank": 0.016941694542765617, + "inverse_harmonic_mean_rank": 0.12259988428001783, + "inverse_median_rank": 0.0181818176060915, + "median_absolute_deviation": 77.0953153622913, + "median_rank": 55.0, + "standard_deviation": 1646.5975341796875, + "variance": 2711283.5, + "z_arithmetic_mean_rank": 546.8884444385656, + "z_hits_at_k": 3359.976035667772, + "z_inverse_harmonic_mean_rank": 3975.5225634017374 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08704475810034824, + "adjusted_arithmetic_mean_rank_index": 0.9130791879154981, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476825209027, + "arithmetic_mean_rank": 641.2368844272147, + "count": 59071.0, + "geometric_mean_rank": 64.55592253351908, + "harmonic_mean_rank": 8.658960015297517, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594860874125335, + "inverse_geometric_mean_rank": 0.01549044550452787, + "inverse_harmonic_mean_rank": 0.11548731005032141, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.3770042289605, + "variance": 2915136.2345698597, + "z_arithmetic_mean_rank": 383.9848643283128, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031892984183 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08704476269634591, + "adjusted_arithmetic_mean_rank_index": 0.9130791833188765, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476816142525, + "arithmetic_mean_rank": 641.2369182847759, + "count": 59071.0, + "geometric_mean_rank": 64.55592524572788, + "harmonic_mean_rank": 8.658960022090657, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594860050710555, + "inverse_geometric_mean_rank": 0.015490444853722812, + "inverse_harmonic_mean_rank": 0.11548730995971913, + "inverse_median_rank": 0.016666666666666666, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.3770524809593, + "variance": 2915136.3993385686, + "z_arithmetic_mean_rank": 383.9848623952569, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031890898167 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08704475715205905, + "adjusted_arithmetic_mean_rank_index": 0.913079188863916, + "adjusted_hits_at_k": 0.24568203089203158, + "adjusted_inverse_harmonic_mean_rank": 0.11487476820670574, + "arithmetic_mean_rank": 641.2368774414062, + "count": 59071.0, + "geometric_mean_rank": 64.55589294433594, + "harmonic_mean_rank": 8.658960279801065, + "hits_at_1": 0.051480421865213045, + "hits_at_10": 0.2461952565556703, + "hits_at_3": 0.12009276971779723, + "hits_at_5": 0.16795043253034483, + "inverse_arithmetic_mean_rank": 0.0015594861470162868, + "inverse_geometric_mean_rank": 0.015490452758967876, + "inverse_harmonic_mean_rank": 0.11548731000496829, + "inverse_median_rank": 0.01666666753590107, + "median_absolute_deviation": 84.50832645481931, + "median_rank": 60.0, + "standard_deviation": 1707.376953125, + "variance": 2915136.25, + "z_arithmetic_mean_rank": 383.984864727159, + "z_hits_at_k": 2288.4226908083533, + "z_inverse_harmonic_mean_rank": 2643.031891939978 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07439344805062242, + "adjusted_arithmetic_mean_rank_index": 0.9257315333259909, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.12911307222450374, + "arithmetic_mean_rank": 551.0289820724213, + "count": 59071.0, + "geometric_mean_rank": 53.9697271406308, + "harmonic_mean_rank": 7.70935969532766, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147865766315915, + "inverse_geometric_mean_rank": 0.018528905980833756, + "inverse_harmonic_mean_rank": 0.1297124585594392, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.201383964887, + "variance": 2503361.219420404, + "z_arithmetic_mean_rank": 389.4207678791215, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.7269258899264 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0743934526216719, + "adjusted_arithmetic_mean_rank_index": 0.9257315287543242, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.12911307221575694, + "arithmetic_mean_rank": 551.0290159299826, + "count": 59071.0, + "geometric_mean_rank": 53.969727949047844, + "harmonic_mean_rank": 7.709359695847163, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147864651234021, + "inverse_geometric_mean_rank": 0.018528905703287735, + "inverse_harmonic_mean_rank": 0.1297124585506984, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.2014431967566, + "variance": 2503361.4068538994, + "z_arithmetic_mean_rank": 389.42076595599167, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.726925688064 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07439345759056779, + "adjusted_arithmetic_mean_rank_index": 0.9257315237847574, + "adjusted_hits_at_k": 0.26367570226177617, + "adjusted_inverse_harmonic_mean_rank": 0.1291130722201289, + "arithmetic_mean_rank": 551.029052734375, + "count": 59071.0, + "geometric_mean_rank": 53.9697265625, + "harmonic_mean_rank": 7.709359214767926, + "hits_at_1": 0.06126525706353372, + "hits_at_10": 0.264173621574038, + "hits_at_3": 0.1387313571803423, + "hits_at_5": 0.18750317414636625, + "inverse_arithmetic_mean_rank": 0.0018147863447666168, + "inverse_geometric_mean_rank": 0.018528906628489494, + "inverse_harmonic_mean_rank": 0.12971245855506738, + "inverse_median_rank": 0.019999999552965164, + "median_absolute_deviation": 71.1649064882689, + "median_rank": 50.0, + "standard_deviation": 1582.201416015625, + "variance": 2503361.25, + "z_arithmetic_mean_rank": 389.42076386548007, + "z_hits_at_k": 2463.574967230824, + "z_inverse_harmonic_mean_rank": 2979.7269257889625 + } + } + }, + "times": { + "evaluation": 29.81851315498352, + "training": 8.202208042144775 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json new file mode 100644 index 0000000..26a4bcb --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with ComplEx as described by Trouillon et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json new file mode 100644 index 0000000..9570d2a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/complex/wn18/trouillon2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00019845912073571508, + 0.00012037810856069427, + 0.00010923121339792283, + 0.00010423280733917918, + 0.00010124416824204406 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23338080058024066, + "adjusted_arithmetic_mean_rank_index": 0.7666566644999107, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.01730672159221292, + "arithmetic_mean_rank": 4775.7257, + "count": 10000.0, + "geometric_mean_rank": 1159.3200186029587, + "harmonic_mean_rank": 56.89714039894227, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.000209392260531211, + "inverse_geometric_mean_rank": 0.0008625745988627474, + "inverse_harmonic_mean_rank": 0.01757557573172149, + "inverse_median_rank": 0.0006228589224540642, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.395222348466, + "variance": 54204863.41005951, + "z_arithmetic_mean_rank": 132.78541860258312, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.16558831999345 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23338082501430735, + "adjusted_arithmetic_mean_rank_index": 0.7666566400646498, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.017306721511205696, + "arithmetic_mean_rank": 4775.7262, + "count": 10000.0, + "geometric_mean_rank": 1159.3202177617752, + "harmonic_mean_rank": 56.897140661113966, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.0002093922386086539, + "inverse_geometric_mean_rank": 0.000862574450681655, + "inverse_harmonic_mean_rank": 0.017575575650736426, + "inverse_median_rank": 0.0006228589224540642, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.395261056931, + "variance": 54204863.980033554, + "z_arithmetic_mean_rank": 132.78541437038052, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.1655870413927 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2333807950062192, + "adjusted_arithmetic_mean_rank_index": 0.7666566700742046, + "adjusted_hits_at_k": 0.032863682896140556, + "adjusted_inverse_harmonic_mean_rank": 0.017306721551698052, + "arithmetic_mean_rank": 4775.7255859375, + "count": 10000.0, + "geometric_mean_rank": 1159.3203125, + "harmonic_mean_rank": 56.897141937862685, + "hits_at_1": 0.0065, + "hits_at_10": 0.0331, + "hits_at_3": 0.0158, + "hits_at_5": 0.0207, + "inverse_arithmetic_mean_rank": 0.0002093922666972503, + "inverse_geometric_mean_rank": 0.0008625743794254959, + "inverse_harmonic_mean_rank": 0.017575575691217704, + "inverse_median_rank": 0.0006228589336387813, + "median_absolute_deviation": 2239.4706510527117, + "median_rank": 1605.5, + "standard_deviation": 7362.39501953125, + "variance": 54204860.0, + "z_arithmetic_mean_rank": 132.78541956805435, + "z_hits_at_k": 210.2129340175632, + "z_inverse_harmonic_mean_rank": 273.1655876805154 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23330198500215898, + "adjusted_arithmetic_mean_rank_index": 0.7667354847984572, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.01738759586962493, + "arithmetic_mean_rank": 4774.0022, + "count": 5000.0, + "geometric_mean_rank": 1156.8772914097908, + "harmonic_mean_rank": 56.63657920733289, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.0002094678548744699, + "inverse_geometric_mean_rank": 0.0008643959108069124, + "inverse_harmonic_mean_rank": 0.017656433598138767, + "inverse_median_rank": 0.0006309148264984228, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.204723369773, + "variance": 54260972.02659516, + "z_arithmetic_mean_rank": 93.90311524197872, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.05759979374437 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2333019947760123, + "adjusted_arithmetic_mean_rank_index": 0.7667354750241262, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.017387595855387426, + "arithmetic_mean_rank": 4774.0024, + "count": 5000.0, + "geometric_mean_rank": 1156.8773531346058, + "harmonic_mean_rank": 56.636579252990074, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.0002094678460991138, + "inverse_geometric_mean_rank": 0.0008643958646873497, + "inverse_harmonic_mean_rank": 0.01765643358390516, + "inverse_median_rank": 0.0006309148264984228, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.204695526336, + "variance": 54260971.61639424, + "z_arithmetic_mean_rank": 93.9031140449034, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.057599634844 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23330199679950533, + "adjusted_arithmetic_mean_rank_index": 0.7667354730005342, + "adjusted_hits_at_k": 0.03356384846492686, + "adjusted_inverse_harmonic_mean_rank": 0.01738759586250523, + "arithmetic_mean_rank": 4774.00244140625, + "count": 5000.0, + "geometric_mean_rank": 1156.8773193359375, + "harmonic_mean_rank": 56.63658078767159, + "hits_at_1": 0.006, + "hits_at_10": 0.0338, + "hits_at_3": 0.0164, + "hits_at_5": 0.0216, + "inverse_arithmetic_mean_rank": 0.00020946784934494644, + "inverse_geometric_mean_rank": 0.000864395871758461, + "inverse_harmonic_mean_rank": 0.017656433591021013, + "inverse_median_rank": 0.0006309148157015443, + "median_absolute_deviation": 2216.490316665875, + "median_rank": 1585.0, + "standard_deviation": 7366.20458984375, + "variance": 54260972.0, + "z_arithmetic_mean_rank": 93.9031137970714, + "z_hits_at_k": 151.8080787783259, + "z_inverse_harmonic_mean_rank": 194.05759971428358 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23345961250403727, + "adjusted_arithmetic_mean_rank_index": 0.7665778478560465, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.017225847315741973, + "arithmetic_mean_rank": 4777.4492, + "count": 5000.0, + "geometric_mean_rank": 1161.7679035740384, + "harmonic_mean_rank": 57.160110137198345, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.00020931672073038474, + "inverse_geometric_mean_rank": 0.0008607571244855542, + "inverse_harmonic_mean_rank": 0.017494717865304207, + "inverse_median_rank": 0.0006142506142506142, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.5833454965605, + "variance": 54148748.85261936, + "z_arithmetic_mean_rank": 93.8838251268764, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685901455532 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.23345965159763768, + "adjusted_arithmetic_mean_rank_index": 0.7665778087605357, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.017225847167965026, + "arithmetic_mean_rank": 4777.45, + "count": 5000.0, + "geometric_mean_rank": 1161.7682407469797, + "harmonic_mean_rank": 57.16011061989465, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.00020931668567959896, + "inverse_geometric_mean_rank": 0.0008607568746732412, + "inverse_harmonic_mean_rank": 0.01749471771756769, + "inverse_median_rank": 0.0006142506142506142, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.583450685329, + "variance": 54148750.4007, + "z_arithmetic_mean_rank": 93.88382033879634, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685736522388 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23345963728113361, + "adjusted_arithmetic_mean_rank_index": 0.7665778230777394, + "adjusted_hits_at_k": 0.03216351733534689, + "adjusted_inverse_harmonic_mean_rank": 0.01722584724183194, + "arithmetic_mean_rank": 4777.44970703125, + "count": 5000.0, + "geometric_mean_rank": 1161.7679443359375, + "harmonic_mean_rank": 57.16011163386548, + "hits_at_1": 0.007, + "hits_at_10": 0.0324, + "hits_at_3": 0.0152, + "hits_at_5": 0.0198, + "inverse_arithmetic_mean_rank": 0.0002093166986014694, + "inverse_geometric_mean_rank": 0.0008607570780441165, + "inverse_harmonic_mean_rank": 0.017494717791414392, + "inverse_median_rank": 0.0006142506026662886, + "median_absolute_deviation": 2263.933587658054, + "median_rank": 1628.0, + "standard_deviation": 7358.58349609375, + "variance": 54148748.0, + "z_arithmetic_mean_rank": 93.88382209224363, + "z_hits_at_k": 145.4778296184482, + "z_inverse_harmonic_mean_rank": 192.25685818964897 + } + } + }, + "times": { + "evaluation": 4.551660537719727, + "training": 1.768416166305542 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json new file mode 100644 index 0000000..a2debf8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json new file mode 100644 index 0000000..bf5bc98 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 9.414209683117643e-05, + 1.3477408468953255e-05, + 1.114161235120257e-05, + 9.765913843640251e-06, + 8.849434282325717e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05739133898406139, + "adjusted_arithmetic_mean_rank_index": 0.9427362845602079, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540388113957916, + "arithmetic_mean_rank": 423.94135023954226, + "count": 118142.0, + "geometric_mean_rank": 37.30836518451274, + "harmonic_mean_rank": 4.23851726046015, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.002358816849158412, + "inverse_geometric_mean_rank": 0.0268036402842737, + "inverse_harmonic_mean_rank": 0.23593156251331063, + "inverse_median_rank": 0.02564102564102564, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.6051548955982, + "variance": 1150481.81830861, + "z_arithmetic_mean_rank": 560.7558442589916, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.311289450006 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05739136763087145, + "adjusted_arithmetic_mean_rank_index": 0.9427362559095191, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540385571304592, + "arithmetic_mean_rank": 423.9415618493, + "count": 118142.0, + "geometric_mean_rank": 37.308392746564536, + "harmonic_mean_rank": 4.2385177169333454, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.0023588156717587257, + "inverse_geometric_mean_rank": 0.026803620482741992, + "inverse_harmonic_mean_rank": 0.23593153710432535, + "inverse_median_rank": 0.02564102564102564, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.60537847533, + "variance": 1150482.2979342062, + "z_arithmetic_mean_rank": 560.7558272170666, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.310460853424 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.057391350697851125, + "adjusted_arithmetic_mean_rank_index": 0.9427362728448321, + "adjusted_hits_at_k": 0.3600643128248548, + "adjusted_inverse_harmonic_mean_rank": 0.23540386809111943, + "arithmetic_mean_rank": 423.9414367675781, + "count": 118142.0, + "geometric_mean_rank": 37.30837631225586, + "harmonic_mean_rank": 4.238517393849456, + "hits_at_1": 0.17041357011054495, + "hits_at_10": 0.3604983833014508, + "hits_at_3": 0.25555687223849266, + "hits_at_5": 0.29802271842359196, + "inverse_arithmetic_mean_rank": 0.0023588163312524557, + "inverse_geometric_mean_rank": 0.02680363319814205, + "inverse_harmonic_mean_rank": 0.23593154947385622, + "inverse_median_rank": 0.025641025975346565, + "median_absolute_deviation": 56.33888430321287, + "median_rank": 39.0, + "standard_deviation": 1072.605224609375, + "variance": 1150482.0, + "z_arithmetic_mean_rank": 560.7558372904836, + "z_hits_at_k": 4750.324947168706, + "z_inverse_harmonic_mean_rank": 7671.310864228486 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.06338498032097208, + "adjusted_arithmetic_mean_rank_index": 0.9367421778295518, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237043022013573, + "arithmetic_mean_rank": 466.94124020246824, + "count": 59071.0, + "geometric_mean_rank": 46.42692536529115, + "harmonic_mean_rank": 4.927991655022916, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415970873902563, + "inverse_geometric_mean_rank": 0.021539225183057283, + "inverse_harmonic_mean_rank": 0.20292242154686638, + "inverse_median_rank": 0.019230769230769232, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.815756992061, + "variance": 1278744.2762615283, + "z_arithmetic_mean_rank": 393.9360603384787, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.126923320398 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.06338501249295578, + "adjusted_arithmetic_mean_rank_index": 0.9367421456532004, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237037988647139, + "arithmetic_mean_rank": 466.9414772053969, + "count": 59071.0, + "geometric_mean_rank": 46.42698260283127, + "harmonic_mean_rank": 4.927992876535459, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415960003915496, + "inverse_geometric_mean_rank": 0.021539198628407454, + "inverse_harmonic_mean_rank": 0.20292237124803492, + "inverse_median_rank": 0.019230769230769232, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.8160192560426, + "variance": 1278744.8694060827, + "z_arithmetic_mean_rank": 393.93604680708717, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.125765246417 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.06338499457588637, + "adjusted_arithmetic_mean_rank_index": 0.9367421635727022, + "adjusted_hits_at_k": 0.32262490314518455, + "adjusted_inverse_harmonic_mean_rank": 0.20237040438327544, + "arithmetic_mean_rank": 466.94134521484375, + "count": 59071.0, + "geometric_mean_rank": 46.426937103271484, + "harmonic_mean_rank": 4.9279921737017185, + "hits_at_1": 0.13949315230823925, + "hits_at_10": 0.3230857781314012, + "hits_at_3": 0.22098830221259164, + "hits_at_5": 0.26163430448104824, + "inverse_arithmetic_mean_rank": 0.0021415965165942907, + "inverse_geometric_mean_rank": 0.021539220586419106, + "inverse_harmonic_mean_rank": 0.2029223957278862, + "inverse_median_rank": 0.01923076994717121, + "median_absolute_deviation": 75.6127131437857, + "median_rank": 52.0, + "standard_deviation": 1130.8157958984375, + "variance": 1278744.5, + "z_arithmetic_mean_rank": 393.9360543429259, + "z_hits_at_k": 3005.112528159395, + "z_inverse_harmonic_mean_rank": 4656.12632886744 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05143023263283102, + "adjusted_arithmetic_mean_rank_index": 0.9486978493854841, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.2684372067255253, + "arithmetic_mean_rank": 380.9414602766163, + "count": 59071.0, + "geometric_mean_rank": 29.980751509802925, + "harmonic_mean_rank": 3.718291753763027, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250752524386856, + "inverse_geometric_mean_rank": 0.03335473427585783, + "inverse_harmonic_mean_rank": 0.2689407034797549, + "inverse_median_rank": 0.03571428571428571, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2182020168108, + "variance": 1018521.3792820444, + "z_arithmetic_mean_rank": 399.0818414336863, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.1091319390425 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05143025777360297, + "adjusted_arithmetic_mean_rank_index": 0.9486978242413174, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.2684372062060287, + "arithmetic_mean_rank": 380.9416464932031, + "count": 59071.0, + "geometric_mean_rank": 29.980758845331998, + "harmonic_mean_rank": 3.718291760940484, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250739692170737, + "inverse_geometric_mean_rank": 0.033354726114802793, + "inverse_harmonic_mean_rank": 0.26894070296061584, + "inverse_median_rank": 0.03571428571428571, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2183812342437, + "variance": 1018521.7410210673, + "z_arithmetic_mean_rank": 399.08183085647215, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.109119949877 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05143023769916584, + "adjusted_arithmetic_mean_rank_index": 0.9486978443184652, + "adjusted_hits_at_k": 0.39750356661856007, + "adjusted_inverse_harmonic_mean_rank": 0.26843720646541763, + "arithmetic_mean_rank": 380.9414978027344, + "count": 59071.0, + "geometric_mean_rank": 29.980751037597656, + "harmonic_mean_rank": 3.718291663785221, + "hits_at_1": 0.20133398791285065, + "hits_at_10": 0.3979109884715004, + "hits_at_3": 0.2901254422643937, + "hits_at_5": 0.3344111323661357, + "inverse_arithmetic_mean_rank": 0.0026250749360769987, + "inverse_geometric_mean_rank": 0.033354733139276505, + "inverse_harmonic_mean_rank": 0.26894070321982627, + "inverse_median_rank": 0.0357142873108387, + "median_absolute_deviation": 40.030259899651256, + "median_rank": 28.0, + "standard_deviation": 1009.2183227539062, + "variance": 1018521.5625, + "z_arithmetic_mean_rank": 399.08183930218024, + "z_hits_at_k": 3713.9555435192497, + "z_inverse_harmonic_mean_rank": 6195.1091259361665 + } + } + }, + "times": { + "evaluation": 14.673102378845215, + "training": 173.3771436214447 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json new file mode 100644 index 0000000..c8b9c51 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json new file mode 100644 index 0000000..cca2c8a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/fb15k237/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001577785732766328, + 1.5780817098527492e-05, + 1.386544718458381e-05, + 1.2271771170897758e-05, + 1.1084717919739344e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.18198389342227317, + "adjusted_arithmetic_mean_rank_index": 0.818130760265559, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.20563247146486757, + "arithmetic_mean_rank": 1298.5768176925335, + "count": 40876.0, + "geometric_mean_rank": 82.93930185831799, + "harmonic_mean_rank": 4.849694078852137, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700738118649919, + "inverse_geometric_mean_rank": 0.01205701009767675, + "inverse_harmonic_mean_rank": 0.20619857329984156, + "inverse_median_rank": 0.010526315789473684, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.477521691501, + "variance": 6332659.117178599, + "z_arithmetic_mean_rank": 286.12773531756534, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.183867427056 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.18198398599023052, + "adjusted_arithmetic_mean_rank_index": 0.8181306676846272, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.2056324713907482, + "arithmetic_mean_rank": 1298.5774782268325, + "count": 40876.0, + "geometric_mean_rank": 82.93931610724847, + "harmonic_mean_rank": 4.849694080594148, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700734201593186, + "inverse_geometric_mean_rank": 0.012057008026288814, + "inverse_harmonic_mean_rank": 0.206198573225775, + "inverse_median_rank": 0.010526315789473684, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.4785795775047, + "variance": 6332664.441472416, + "z_arithmetic_mean_rank": 286.1277029389119, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.1838660313433 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.18198393977321362, + "adjusted_arithmetic_mean_rank_index": 0.8181307139081219, + "adjusted_hits_at_k": 0.31667399392675755, + "adjusted_inverse_harmonic_mean_rank": 0.20563247142779512, + "arithmetic_mean_rank": 1298.5771484375, + "count": 40876.0, + "geometric_mean_rank": 82.93929290771484, + "harmonic_mean_rank": 4.849694433624096, + "hits_at_1": 0.15028378510617477, + "hits_at_10": 0.3171543203835992, + "hits_at_3": 0.22367648497896075, + "hits_at_5": 0.2611556903806635, + "inverse_arithmetic_mean_rank": 0.0007700736168771982, + "inverse_geometric_mean_rank": 0.012057011015713215, + "inverse_harmonic_mean_rank": 0.20619857326279553, + "inverse_median_rank": 0.010526316240429878, + "median_absolute_deviation": 139.36460853952659, + "median_rank": 95.0, + "standard_deviation": 2516.47802734375, + "variance": 6332661.5, + "z_arithmetic_mean_rank": 286.12771910481655, + "z_hits_at_k": 2414.0164581702625, + "z_inverse_harmonic_mean_rank": 3872.1838667289594 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23351524215549604, + "adjusted_arithmetic_mean_rank_index": 0.7665937010171862, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.10774751052365336, + "arithmetic_mean_rank": 1643.162295723652, + "count": 20438.0, + "geometric_mean_rank": 190.31729087252333, + "harmonic_mean_rank": 9.225701931373973, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.0006085826108610884, + "inverse_geometric_mean_rank": 0.00525438332699792, + "inverse_harmonic_mean_rank": 0.10839283638671288, + "inverse_median_rank": 0.003115264797507788, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.7272310550084, + "variance": 7310140.939948384, + "z_arithmetic_mean_rank": 189.37640251104858, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.891462908065 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2335153603630686, + "adjusted_arithmetic_mean_rank_index": 0.7665935827928123, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.10774751039939984, + "arithmetic_mean_rank": 1643.163127507584, + "count": 20438.0, + "geometric_mean_rank": 190.31733690440817, + "harmonic_mean_rank": 9.225701941941985, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.0006085823027911052, + "inverse_geometric_mean_rank": 0.005254382056124903, + "inverse_harmonic_mean_rank": 0.10839283626254922, + "inverse_median_rank": 0.003115264797507788, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.7283336983282, + "variance": 7310146.902443139, + "z_arithmetic_mean_rank": 189.37637330534733, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.8914612671986 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23351530241195553, + "adjusted_arithmetic_mean_rank_index": 0.7665936407521623, + "adjusted_hits_at_k": 0.19269049159897544, + "adjusted_inverse_harmonic_mean_rank": 0.1077475104615027, + "arithmetic_mean_rank": 1643.1627197265625, + "count": 20438.0, + "geometric_mean_rank": 190.3172607421875, + "harmonic_mean_rank": 9.225701024597301, + "hits_at_1": 0.06507486055387024, + "hits_at_10": 0.19326744299833643, + "hits_at_3": 0.11664546433114786, + "hits_at_5": 0.1443389764164791, + "inverse_arithmetic_mean_rank": 0.000608582457061857, + "inverse_geometric_mean_rank": 0.005254384130239487, + "inverse_harmonic_mean_rank": 0.10839283632460717, + "inverse_median_rank": 0.0031152646988630295, + "median_absolute_deviation": 472.950107703287, + "median_rank": 321.0, + "standard_deviation": 2703.727783203125, + "variance": 7310143.5, + "z_arithmetic_mean_rank": 189.37638762340546, + "z_hits_at_k": 1030.091386500863, + "z_inverse_harmonic_mean_rank": 1422.891462087316 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1318632966013177, + "adjusted_arithmetic_mean_rank_index": 0.8682567162207461, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.30351535382197037, + "arithmetic_mean_rank": 953.991339661415, + "count": 20438.0, + "geometric_mean_rank": 36.14452350182288, + "harmonic_mean_rank": 3.289427045621327, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482275450790926, + "inverse_geometric_mean_rank": 0.027666708621834974, + "inverse_harmonic_mean_rank": 0.3040043102129702, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.2331867137273, + "variance": 5117698.991068945, + "z_arithmetic_mean_rank": 214.97579285996775, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.441934546446 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1318633642315847, + "adjusted_arithmetic_mean_rank_index": 0.8682566485811298, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.3035153537979841, + "arithmetic_mean_rank": 953.9918289460809, + "count": 20438.0, + "geometric_mean_rank": 36.14452717879933, + "harmonic_mean_rank": 3.289427045880684, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482270074626807, + "inverse_geometric_mean_rank": 0.02766670580730553, + "inverse_harmonic_mean_rank": 0.30400431018900076, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.2341702664685, + "variance": 5117703.441121218, + "z_arithmetic_mean_rank": 214.97577611275477, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.4419342243714 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.13186332942742457, + "adjusted_arithmetic_mean_rank_index": 0.8682566833901013, + "adjusted_hits_at_k": 0.4406545843034417, + "adjusted_inverse_harmonic_mean_rank": 0.3035153538099757, + "arithmetic_mean_rank": 953.9915771484375, + "count": 20438.0, + "geometric_mean_rank": 36.1445198059082, + "harmonic_mean_rank": 3.289426970315094, + "hits_at_1": 0.2354927096584793, + "hits_at_10": 0.4410411977688619, + "hits_at_3": 0.33070750562677365, + "hits_at_5": 0.37797240434484786, + "inverse_arithmetic_mean_rank": 0.0010482273064553738, + "inverse_geometric_mean_rank": 0.027666712179780006, + "inverse_harmonic_mean_rank": 0.30400431020098395, + "inverse_median_rank": 0.0476190485060215, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 2262.233642578125, + "variance": 5117701.5, + "z_arithmetic_mean_rank": 214.97578473127288, + "z_hits_at_k": 2395.349603279926, + "z_inverse_harmonic_mean_rank": 4075.4419343853883 + } + } + }, + "times": { + "evaluation": 6.211552381515503, + "training": 95.12276983261108 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json new file mode 100644 index 0000000..6655aca --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json new file mode 100644 index 0000000..4179e29 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001197095220165262, + 3.842027433118703e-06, + 3.4459802265337745e-06, + 3.297556057019098e-06, + 3.1852552669447856e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6690416172263688, + "adjusted_arithmetic_mean_rank_index": 0.3309745568824283, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.02654567437482097, + "arithmetic_mean_rank": 13690.7545, + "count": 10000.0, + "geometric_mean_rank": 3642.850837532699, + "harmonic_mean_rank": 37.29673164073776, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304199341241566e-05, + "inverse_geometric_mean_rank": 0.0002745102790641023, + "inverse_harmonic_mean_rank": 0.02681200083783586, + "inverse_median_rank": 0.00010062893081761007, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.265564228193, + "variance": 168798964.49142975, + "z_arithmetic_mean_rank": 57.3250023347874, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.9912410223125 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6690419739637427, + "adjusted_arithmetic_mean_rank_index": 0.3309742001276206, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.026545674264344184, + "arithmetic_mean_rank": 13690.7618, + "count": 10000.0, + "geometric_mean_rank": 3642.8529066914157, + "harmonic_mean_rank": 37.29673179437403, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304195446596696e-05, + "inverse_geometric_mean_rank": 0.00027451012314088735, + "inverse_harmonic_mean_rank": 0.026812000727389294, + "inverse_median_rank": 0.00010062893081761007, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.268889068635, + "variance": 168799050.88586074, + "z_arithmetic_mean_rank": 57.324940544629314, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.9912392785705 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6690418268248473, + "adjusted_arithmetic_mean_rank_index": 0.33097434727370667, + "adjusted_hits_at_k": 0.0460669090682069, + "adjusted_inverse_harmonic_mean_rank": 0.026545674319570493, + "arithmetic_mean_rank": 13690.7587890625, + "count": 10000.0, + "geometric_mean_rank": 3642.852783203125, + "harmonic_mean_rank": 37.29673348542801, + "hits_at_1": 0.0163, + "hits_at_10": 0.0463, + "hits_at_3": 0.026, + "hits_at_5": 0.0319, + "inverse_arithmetic_mean_rank": 7.304197060875595e-05, + "inverse_geometric_mean_rank": 0.00027451012283563614, + "inverse_harmonic_mean_rank": 0.026812000782600494, + "inverse_median_rank": 0.00010062893124995753, + "median_absolute_deviation": 14190.727134426368, + "median_rank": 9937.5, + "standard_deviation": 12992.267578125, + "variance": 168799008.0, + "z_arithmetic_mean_rank": 57.32496603042439, + "z_hits_at_k": 294.6675254551374, + "z_inverse_harmonic_mean_rank": 418.99124015025075 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6834586222406983, + "adjusted_arithmetic_mean_rank_index": 0.31655684766020065, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609226782556, + "arithmetic_mean_rank": 13985.4488, + "count": 5000.0, + "geometric_mean_rank": 3911.6517271442685, + "harmonic_mean_rank": 39.23983846635467, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150288948896657e-05, + "inverse_geometric_mean_rank": 0.00025564648127047285, + "inverse_harmonic_mean_rank": 0.02548430470368597, + "inverse_median_rank": 9.482268158543523e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.836726111675, + "variance": 168605984.80377856, + "z_arithmetic_mean_rank": 38.76913842625525, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.4459661806924 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6834589350040029, + "adjusted_arithmetic_mean_rank_index": 0.31655653488161084, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609157271326, + "arithmetic_mean_rank": 13985.4552, + "count": 5000.0, + "geometric_mean_rank": 3911.6534371979524, + "harmonic_mean_rank": 39.239838573356344, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150285676793701e-05, + "inverse_geometric_mean_rank": 0.0002556463695097522, + "inverse_harmonic_mean_rank": 0.025484304634193756, + "inverse_median_rank": 9.482268158543523e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.840143828995, + "variance": 168606073.56079298, + "z_arithmetic_mean_rank": 38.76910011984509, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.44596540489897 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6834588336002752, + "adjusted_arithmetic_mean_rank_index": 0.3165566362902943, + "adjusted_hits_at_k": 0.041365754883860385, + "adjusted_inverse_harmonic_mean_rank": 0.025217609192021116, + "arithmetic_mean_rank": 13985.453125, + "count": 5000.0, + "geometric_mean_rank": 3911.652099609375, + "harmonic_mean_rank": 39.239838303317676, + "hits_at_1": 0.0156, + "hits_at_10": 0.0416, + "hits_at_3": 0.0246, + "hits_at_5": 0.03, + "inverse_arithmetic_mean_rank": 7.150286546675488e-05, + "inverse_geometric_mean_rank": 0.00025564644602127373, + "inverse_harmonic_mean_rank": 0.02548430466893404, + "inverse_median_rank": 9.482268069405109e-05, + "median_absolute_deviation": 14892.739284888772, + "median_rank": 10546.0, + "standard_deviation": 12984.8388671875, + "variance": 168606048.0, + "z_arithmetic_mean_rank": 38.76911253950152, + "z_hits_at_k": 187.095820751784, + "z_inverse_harmonic_mean_rank": 281.44596579273065 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6546252806566224, + "adjusted_arithmetic_mean_rank_index": 0.34539159759475413, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.02787373950740589, + "arithmetic_mean_rank": 13396.0602, + "count": 5000.0, + "geometric_mean_rank": 3392.5214078812855, + "harmonic_mean_rank": 35.536985383870416, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464881353698307e-05, + "inverse_geometric_mean_rank": 0.00029476601022380137, + "inverse_harmonic_mean_rank": 0.028139696971985744, + "inverse_median_rank": 0.0001077179942909463, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.007916497856, + "variance": 168818254.71817595, + "z_arithmetic_mean_rank": 42.30057578570636, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747511731894 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6546256813660266, + "adjusted_arithmetic_mean_rank_index": 0.34539119686576747, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.027873739355963534, + "arithmetic_mean_rank": 13396.0684, + "count": 5000.0, + "geometric_mean_rank": 3392.5237787159485, + "harmonic_mean_rank": 35.536985575071206, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464876784295906e-05, + "inverse_geometric_mean_rank": 0.0002947658042292321, + "inverse_harmonic_mean_rank": 0.02813969682058482, + "inverse_median_rank": 0.0001077179942909463, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.011191056576, + "variance": 168818339.81092143, + "z_arithmetic_mean_rank": 42.30052670788574, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747342707806 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.654625488494084, + "adjusted_arithmetic_mean_rank_index": 0.3453913897471357, + "adjusted_hits_at_k": 0.05076806319888817, + "adjusted_inverse_harmonic_mean_rank": 0.027873739431666374, + "arithmetic_mean_rank": 13396.064453125, + "count": 5000.0, + "geometric_mean_rank": 3392.524658203125, + "harmonic_mean_rank": 35.53698501304006, + "hits_at_1": 0.017, + "hits_at_10": 0.051, + "hits_at_3": 0.0274, + "hits_at_5": 0.0338, + "inverse_arithmetic_mean_rank": 7.464879308827221e-05, + "inverse_geometric_mean_rank": 0.0002947657194454223, + "inverse_harmonic_mean_rank": 0.028139696896266952, + "inverse_median_rank": 0.00010771799134090543, + "median_absolute_deviation": 13414.584873038686, + "median_rank": 9283.5, + "standard_deviation": 12993.009765625, + "variance": 168818304.0, + "z_arithmetic_mean_rank": 42.300550330327596, + "z_hits_at_k": 229.62748666763034, + "z_inverse_harmonic_mean_rank": 311.09747427199386 + } + } + }, + "times": { + "evaluation": 3.723423957824707, + "training": 167.36736845970154 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json new file mode 100644 index 0000000..99eeec8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvE as described by Dettmers et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json new file mode 100644 index 0000000..eb2b85e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/conve/wn18rr/dettmers2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00018945981818183324, + 4.8747250159544595e-06, + 3.806483352484587e-06, + 3.5589507950760996e-06, + 3.4506096300906347e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.748950246120412, + "adjusted_arithmetic_mean_rank_index": 0.25106213822015777, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347310534693, + "arithmetic_mean_rank": 15183.129787961696, + "count": 5848.0, + "geometric_mean_rank": 5271.940706937559, + "harmonic_mean_rank": 45.24629960213599, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586257339332459e-05, + "inverse_geometric_mean_rank": 0.00018968346868622018, + "inverse_harmonic_mean_rank": 0.02210125488257148, + "inverse_median_rank": 7.812194836139214e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.650059969405, + "variance": 162171312.14987877, + "z_arithmetic_mean_rank": 33.253269759253385, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.2796617047961 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7489507100445709, + "adjusted_arithmetic_mean_rank_index": 0.25106167427311343, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347249979118, + "arithmetic_mean_rank": 15183.139192886458, + "count": 5848.0, + "geometric_mean_rank": 5271.943980987781, + "harmonic_mean_rank": 45.246299726072834, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586253259592824e-05, + "inverse_geometric_mean_rank": 0.00018968335088656127, + "inverse_harmonic_mean_rank": 0.022101254822032613, + "inverse_median_rank": 7.812194836139214e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.654306532657, + "variance": 162171420.30689073, + "z_arithmetic_mean_rank": 33.25320830930188, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.2796609772874 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7489504434858746, + "adjusted_arithmetic_mean_rank_index": 0.25106194084495925, + "adjusted_hits_at_k": 0.03840851855613144, + "adjusted_inverse_harmonic_mean_rank": 0.021831347280254862, + "arithmetic_mean_rank": 15183.1337890625, + "count": 5848.0, + "geometric_mean_rank": 5271.947265625, + "harmonic_mean_rank": 45.24629964608544, + "hits_at_1": 0.012311901504787962, + "hits_at_10": 0.03864569083447333, + "hits_at_3": 0.02308481532147743, + "hits_at_5": 0.028898768809849523, + "inverse_arithmetic_mean_rank": 6.586255767615512e-05, + "inverse_geometric_mean_rank": 0.00018968322547152638, + "inverse_harmonic_mean_rank": 0.022101254852300002, + "inverse_median_rank": 7.812194962752983e-05, + "median_absolute_deviation": 16124.781728466927, + "median_rank": 12800.5, + "standard_deviation": 12734.65234375, + "variance": 162171376.0, + "z_arithmetic_mean_rank": 33.25324361683795, + "z_hits_at_k": 186.99974504247893, + "z_inverse_harmonic_mean_rank": 262.27966134101723 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8731896014158192, + "adjusted_arithmetic_mean_rank_index": 0.12681665555773147, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.01581431335858354, + "arithmetic_mean_rank": 17697.850889192887, + "count": 2924.0, + "geometric_mean_rank": 8745.380971246836, + "harmonic_mean_rank": 62.166104430540265, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.6504035787229144e-05, + "inverse_geometric_mean_rank": 0.000114346076321639, + "inverse_harmonic_mean_rank": 0.016085936366132203, + "inverse_median_rank": 5.902665053271552e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.980565244461, + "variance": 157301276.4989698, + "z_arithmetic_mean_rank": 11.877203008876947, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.32938882569934 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8731901245009187, + "adjusted_arithmetic_mean_rank_index": 0.12681613244682233, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.015814313287617128, + "arithmetic_mean_rank": 17697.86149110807, + "count": 2924.0, + "geometric_mean_rank": 8745.387291427725, + "harmonic_mean_rank": 62.166104704723125, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.6504001938450566e-05, + "inverse_geometric_mean_rank": 0.0001143459936851745, + "inverse_harmonic_mean_rank": 0.016085936295185375, + "inverse_median_rank": 5.902665053271552e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.982420271721, + "variance": 157301323.0304049, + "z_arithmetic_mean_rank": 11.877154016144601, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.3293882228989 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8731898273653809, + "adjusted_arithmetic_mean_rank_index": 0.12681642959702122, + "adjusted_hits_at_k": 0.022672709474702245, + "adjusted_inverse_harmonic_mean_rank": 0.015814313323097975, + "arithmetic_mean_rank": 17697.85546875, + "count": 2924.0, + "geometric_mean_rank": 8745.3857421875, + "harmonic_mean_rank": 62.16610392070862, + "hits_at_1": 0.01094391244870041, + "hits_at_10": 0.022913816689466484, + "hits_at_3": 0.018467852257181942, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 5.650402090395801e-05, + "inverse_geometric_mean_rank": 0.00011434601037763059, + "inverse_harmonic_mean_rank": 0.01608593633065643, + "inverse_median_rank": 5.902664997847751e-05, + "median_absolute_deviation": 16556.96027516131, + "median_rank": 16941.5, + "standard_deviation": 12541.982421875, + "variance": 157301312.0, + "z_arithmetic_mean_rank": 11.877181846189643, + "z_hits_at_k": 78.04650152338807, + "z_inverse_harmonic_mean_rank": 134.3293885242791 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.624766004283246, + "adjusted_arithmetic_mean_rank_index": 0.37525250198790927, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380588003943, + "arithmetic_mean_rank": 12668.408686730507, + "count": 2924.0, + "geometric_mean_rank": 3178.061528576585, + "harmonic_mean_rank": 35.56621163641454, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893651244827991e-05, + "inverse_geometric_mean_rank": 0.0003146572182470891, + "inverse_harmonic_mean_rank": 0.028116573399010757, + "inverse_median_rank": 0.00011536686663590217, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.52628128213, + "variance": 154393703.36683288, + "z_arithmetic_mean_rank": 35.14487495983464, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.60145660877637 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6247664090727081, + "adjusted_arithmetic_mean_rank_index": 0.3752520971784833, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380537859197, + "arithmetic_mean_rank": 12668.416894664842, + "count": 2924.0, + "geometric_mean_rank": 3178.063179193508, + "harmonic_mean_rank": 35.56621169982792, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893646130489583e-05, + "inverse_geometric_mean_rank": 0.00031465705482097066, + "inverse_harmonic_mean_rank": 0.028116573348879844, + "inverse_median_rank": 0.00011536686663590217, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.532628786203, + "variance": 154393861.10903057, + "z_arithmetic_mean_rank": 35.144837046758546, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.6014561827437 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6247662212382751, + "adjusted_arithmetic_mean_rank_index": 0.3752522850221801, + "adjusted_hits_at_k": 0.054144325905684736, + "adjusted_inverse_harmonic_mean_rank": 0.027848380562929834, + "arithmetic_mean_rank": 12668.4130859375, + "count": 2924.0, + "geometric_mean_rank": 3178.061767578125, + "harmonic_mean_rank": 35.56621185760246, + "hits_at_1": 0.013679890560875513, + "hits_at_10": 0.05437756497948017, + "hits_at_3": 0.027701778385772913, + "hits_at_5": 0.038303693570451436, + "inverse_arithmetic_mean_rank": 7.893648580648005e-05, + "inverse_geometric_mean_rank": 0.00031465719803236425, + "inverse_harmonic_mean_rank": 0.028116573373943566, + "inverse_median_rank": 0.0001153668636106886, + "median_absolute_deviation": 12472.391163178376, + "median_rank": 8668.0, + "standard_deviation": 12425.529296875, + "variance": 154393776.0, + "z_arithmetic_mean_rank": 35.144854639561274, + "z_hits_at_k": 186.4231598474591, + "z_inverse_harmonic_mean_rank": 236.6014563957453 + } + } + }, + "times": { + "evaluation": 2.1725149154663086, + "training": 100.6689465045929 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..4dad16b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..baea645 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/fb15k237/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0027069504231270925, + 0.0027013514650969135, + 0.002696092348934795, + 0.0026906675357372457, + 0.002684772505929381 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8925691737374529, + "adjusted_arithmetic_mean_rank_index": 0.10744588383943188, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802826417809, + "arithmetic_mean_rank": 6369.078138761131, + "count": 40876.0, + "geometric_mean_rank": 3537.461831461074, + "harmonic_mean_rank": 84.89284921971634, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.0001570085934280142, + "inverse_geometric_mean_rank": 0.00028268856249029015, + "inverse_harmonic_mean_rank": 0.0117795551591376, + "inverse_median_rank": 0.00016385384237260363, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.34208695286, + "variance": 18934177.95768727, + "z_arithmetic_mean_rank": 37.577425156574975, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.54523866731483 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8925692423063102, + "adjusted_arithmetic_mean_rank_index": 0.10744581526096397, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802744426081, + "arithmetic_mean_rank": 6369.078628045797, + "count": 40876.0, + "geometric_mean_rank": 3537.4622506253572, + "harmonic_mean_rank": 84.89284981019289, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.00015700858136631713, + "inverse_geometric_mean_rank": 0.0002826885289936928, + "inverse_harmonic_mean_rank": 0.011779555077204303, + "inverse_median_rank": 0.00016385384237260363, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.342209829061, + "variance": 18934179.027040057, + "z_arithmetic_mean_rank": 37.57740117238728, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.54523712336098 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8925692402371914, + "adjusted_arithmetic_mean_rank_index": 0.10744581733037284, + "adjusted_hits_at_k": 0.020203732621692873, + "adjusted_inverse_harmonic_mean_rank": 0.011074802785395542, + "arithmetic_mean_rank": 6369.07861328125, + "count": 40876.0, + "geometric_mean_rank": 3537.4609375, + "harmonic_mean_rank": 84.89284887628386, + "hits_at_1": 0.006605342988550739, + "hits_at_10": 0.020892455230453078, + "hits_at_3": 0.011987474312555044, + "hits_at_5": 0.015657109306194343, + "inverse_arithmetic_mean_rank": 0.0001570085878483951, + "inverse_geometric_mean_rank": 0.0002826886484399438, + "inverse_harmonic_mean_rank": 0.011779555118144566, + "inverse_median_rank": 0.00016385383787564933, + "median_absolute_deviation": 5645.749248069332, + "median_rank": 6103.0, + "standard_deviation": 4351.34228515625, + "variance": 18934178.0, + "z_arithmetic_mean_rank": 37.57740189612889, + "z_hits_at_k": 154.01373021025012, + "z_inverse_harmonic_mean_rank": 208.5452378948407 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.984182731634944, + "adjusted_arithmetic_mean_rank_index": 0.015819516529003597, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789023046226185e-05, + "arithmetic_mean_rank": 6925.337899990214, + "count": 20438.0, + "geometric_mean_rank": 4991.1658658403585, + "harmonic_mean_rank": 1248.3012815646675, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.0001443972863766565, + "inverse_geometric_mean_rank": 0.00020035399080684145, + "inverse_harmonic_mean_rank": 0.0008010886592590553, + "inverse_median_rank": 0.0001472211998527788, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232259464963, + "variance": 17290895.52365509, + "z_arithmetic_mean_rank": 3.90799340739639, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286023633405836 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.98418282202897, + "adjusted_arithmetic_mean_rank_index": 0.0158194261221295, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789019902602824e-05, + "arithmetic_mean_rank": 6925.33853606028, + "count": 20438.0, + "geometric_mean_rank": 4991.166433981173, + "harmonic_mean_rank": 1248.301330514943, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.00014439727311423028, + "inverse_geometric_mean_rank": 0.0002003539680006936, + "inverse_harmonic_mean_rank": 0.000801088627845558, + "inverse_median_rank": 0.0001472211998527788, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232367660528, + "variance": 17290896.423459683, + "z_arithmetic_mean_rank": 3.90797107362485, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286019482001478 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9841827996952898, + "adjusted_arithmetic_mean_rank_index": 0.015819448458984087, + "adjusted_hits_at_k": 1.9281290991966858e-05, + "adjusted_inverse_harmonic_mean_rank": 7.789021474093581e-05, + "arithmetic_mean_rank": 6925.33837890625, + "count": 20438.0, + "geometric_mean_rank": 4991.166015625, + "harmonic_mean_rank": 1248.301271763787, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007339269987278598, + "hits_at_3": 0.0001957138663274293, + "hits_at_5": 0.0003914277326548586, + "inverse_arithmetic_mean_rank": 0.00014439727237913758, + "inverse_geometric_mean_rank": 0.00020035398483742028, + "inverse_harmonic_mean_rank": 0.0008010886435490997, + "inverse_median_rank": 0.00014722120249643922, + "median_absolute_deviation": 5302.526834485286, + "median_rank": 6792.5, + "standard_deviation": 4158.232421875, + "variance": 17290896.0, + "z_arithmetic_mean_rank": 3.9079765916367535, + "z_hits_at_k": 0.10307458145250425, + "z_inverse_harmonic_mean_rank": 1.0286021557279845 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8034636814188896, + "adjusted_arithmetic_mean_rank_index": 0.19656348812496072, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481903275268, + "arithmetic_mean_rank": 5812.8183775320485, + "count": 20438.0, + "geometric_mean_rank": 2507.1569539869356, + "harmonic_mean_rank": 43.940550500523216, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.00017203358767671844, + "inverse_geometric_mean_rank": 0.00039885815621147223, + "inverse_harmonic_mean_rank": 0.022758021659016144, + "inverse_median_rank": 0.00018950161076369148, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.506076973391, + "variance": 19958610.547794178, + "z_arithmetic_mean_rank": 48.668085046221506, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.36406123610016 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8034637287600763, + "adjusted_arithmetic_mean_rank_index": 0.19656344077722931, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481770729127, + "arithmetic_mean_rank": 5812.818720031314, + "count": 20438.0, + "geometric_mean_rank": 2507.1572627599153, + "harmonic_mean_rank": 43.94055075625994, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.0001720335775402631, + "inverse_geometric_mean_rank": 0.00039885810708945534, + "inverse_harmonic_mean_rank": 0.022758021526563053, + "inverse_median_rank": 0.00018950161076369148, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.506179077078, + "variance": 19958611.460091874, + "z_arithmetic_mean_rank": 48.66807332317242, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.3640594563414 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8034636789091726, + "adjusted_arithmetic_mean_rank_index": 0.19656349063502465, + "adjusted_hits_at_k": 0.04038770988823111, + "adjusted_inverse_harmonic_mean_rank": 0.022071481836952594, + "arithmetic_mean_rank": 5812.818359375, + "count": 20438.0, + "geometric_mean_rank": 2507.1572265625, + "harmonic_mean_rank": 43.94055321163615, + "hits_at_1": 0.013063900577355906, + "hits_at_10": 0.041050983462178295, + "hits_at_3": 0.023779234758782658, + "hits_at_5": 0.03092279087973383, + "inverse_arithmetic_mean_rank": 0.00017203358584083617, + "inverse_geometric_mean_rank": 0.0003988581011071801, + "inverse_harmonic_mean_rank": 0.02275802159274003, + "inverse_median_rank": 0.00018950161756947637, + "median_absolute_deviation": 5850.348354223105, + "median_rank": 5277.0, + "standard_deviation": 4467.505859375, + "variance": 19958610.0, + "z_arithmetic_mean_rank": 48.668085667700126, + "z_hits_at_k": 219.5431258501117, + "z_inverse_harmonic_mean_rank": 296.3640603455547 + } + } + }, + "times": { + "evaluation": 113.07699227333069, + "training": 29.087645292282104 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json new file mode 100644 index 0000000..407b565 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json new file mode 100644 index 0000000..cd957e6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/convkb/wn18rr/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002713467231609252, + 0.0027108328806589995, + 0.0027068490911217875, + 0.0027012186428765706, + 0.002693269666272096 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9336325239278058, + "adjusted_arithmetic_mean_rank_index": 0.06637074999464865, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.003291821314377242, + "arithmetic_mean_rank": 18927.110123119015, + "count": 5848.0, + "geometric_mean_rank": 12085.68222373154, + "harmonic_mean_rank": 280.35985102877726, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.2834267539793294e-05, + "inverse_geometric_mean_rank": 8.27425362911158e-05, + "inverse_harmonic_mean_rank": 0.0035668445261706036, + "inverse_median_rank": 5.537251861900939e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.122573466682, + "variance": 146533992.51865265, + "z_arithmetic_mean_rank": 8.79082951074308, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.54761785640589 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9336326251476224, + "adjusted_arithmetic_mean_rank_index": 0.066370648769839, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.003291821259997524, + "arithmetic_mean_rank": 18927.1121751026, + "count": 5848.0, + "geometric_mean_rank": 12085.684555207396, + "harmonic_mean_rank": 280.3598553019333, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.283426181176417e-05, + "inverse_geometric_mean_rank": 8.274252032907205e-05, + "inverse_harmonic_mean_rank": 0.0035668444718058904, + "inverse_median_rank": 5.537251861900939e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.12287157998, + "variance": 146533999.73604876, + "z_arithmetic_mean_rank": 8.790816103480935, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.547617203093296 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9336325833680886, + "adjusted_arithmetic_mean_rank_index": 0.06637069055143363, + "adjusted_hits_at_k": 0.007621171231354433, + "adjusted_inverse_harmonic_mean_rank": 0.0032918212871810037, + "arithmetic_mean_rank": 18927.111328125, + "count": 5848.0, + "geometric_mean_rank": 12085.6796875, + "harmonic_mean_rank": 280.3598644266672, + "hits_at_1": 0.0015389876880984952, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.005642954856361149, + "inverse_arithmetic_mean_rank": 5.283426435198635e-05, + "inverse_geometric_mean_rank": 8.274255378637463e-05, + "inverse_harmonic_mean_rank": 0.0035668444989818693, + "inverse_median_rank": 5.537251854548231e-05, + "median_absolute_deviation": 15634.040394141572, + "median_rank": 18059.5, + "standard_deviation": 12105.1220703125, + "variance": 146533984.0, + "z_arithmetic_mean_rank": 8.790821637468042, + "z_hits_at_k": 37.1052342231212, + "z_inverse_harmonic_mean_rank": 39.547617529672955 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9786446205308251, + "adjusted_arithmetic_mean_rank_index": 0.02135643316860325, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0038716852646274304, + "arithmetic_mean_rank": 19835.21853625171, + "count": 2924.0, + "geometric_mean_rank": 13690.242814848845, + "harmonic_mean_rank": 241.1611840217548, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041537597240769e-05, + "inverse_geometric_mean_rank": 7.304472342268248e-05, + "inverse_harmonic_mean_rank": 0.004146604288979571, + "inverse_median_rank": 5.22875816993464e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.606562661385, + "variance": 139774025.93548405, + "z_arithmetic_mean_rank": 2.0001685990965115, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.8867339055593 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9786447892679542, + "adjusted_arithmetic_mean_rank_index": 0.02135626442314853, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0038716852510751894, + "arithmetic_mean_rank": 19835.22195622435, + "count": 2924.0, + "geometric_mean_rank": 13690.245472897186, + "harmonic_mean_rank": 241.1611848097182, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041536727983006e-05, + "inverse_geometric_mean_rank": 7.304470924058426e-05, + "inverse_harmonic_mean_rank": 0.00414660427543107, + "inverse_median_rank": 5.22875816993464e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.607099287705, + "variance": 139774038.62412804, + "z_arithmetic_mean_rank": 2.0001527949892903, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.88673379044433 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9786447274416155, + "adjusted_arithmetic_mean_rank_index": 0.021356326252537805, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.003871685257851033, + "arithmetic_mean_rank": 19835.220703125, + "count": 2924.0, + "geometric_mean_rank": 13690.232421875, + "harmonic_mean_rank": 241.16118319861056, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 5.041537224315107e-05, + "inverse_geometric_mean_rank": 7.304477912839502e-05, + "inverse_harmonic_mean_rank": 0.004146604282205044, + "inverse_median_rank": 5.228758163866587e-05, + "median_absolute_deviation": 15294.52448610379, + "median_rank": 19125.0, + "standard_deviation": 11822.6064453125, + "variance": 139774032.0, + "z_arithmetic_mean_rank": 2.0001585857129527, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 32.886733847999466 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8886403950097489, + "adjusted_arithmetic_mean_rank_index": 0.11136509716598464, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.002711957429127143, + "arithmetic_mean_rank": 18019.00170998632, + "count": 2924.0, + "geometric_mean_rank": 10669.183650606674, + "harmonic_mean_rank": 334.77456423921825, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.54969701482291e-05, + "inverse_geometric_mean_rank": 9.372788328965898e-05, + "inverse_harmonic_mean_rank": 0.0029870847633616357, + "inverse_median_rank": 5.899879052479424e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.407712992797, + "variance": 151644637.3218165, + "z_arithmetic_mean_rank": 10.430076799100135, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.040947604288164 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.888640428742204, + "adjusted_arithmetic_mean_rank_index": 0.11136506343186581, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.0027119573339199534, + "arithmetic_mean_rank": 18019.00239398085, + "count": 2924.0, + "geometric_mean_rank": 10669.185695548289, + "harmonic_mean_rank": 334.77457490652637, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.549696804158507e-05, + "inverse_geometric_mean_rank": 9.372786532502189e-05, + "inverse_harmonic_mean_rank": 0.0029870846681807114, + "inverse_median_rank": 5.899879052479424e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.407582130434, + "variance": 151644634.0988315, + "z_arithmetic_mean_rank": 10.430073639677133, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.04094679540236 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8886404070005824, + "adjusted_arithmetic_mean_rank_index": 0.11136508517455967, + "adjusted_hits_at_k": 0.008305389077967469, + "adjusted_inverse_harmonic_mean_rank": 0.0027119573815110663, + "arithmetic_mean_rank": 18019.001953125, + "count": 2924.0, + "geometric_mean_rank": 10669.185546875, + "harmonic_mean_rank": 334.77456747877113, + "hits_at_1": 0.0003419972640218878, + "hits_at_10": 0.008549931600547196, + "hits_at_3": 0.0027359781121751026, + "hits_at_5": 0.005471956224350205, + "inverse_arithmetic_mean_rank": 5.549697016249411e-05, + "inverse_geometric_mean_rank": 9.372786735184491e-05, + "inverse_harmonic_mean_rank": 0.002987084715758695, + "inverse_median_rank": 5.89987903367728e-05, + "median_absolute_deviation": 16185.568419425656, + "median_rank": 16949.5, + "standard_deviation": 12314.408203125, + "variance": 151644640.0, + "z_arithmetic_mean_rank": 10.43007567602399, + "z_hits_at_k": 28.59610586664832, + "z_inverse_harmonic_mean_rank": 23.040947199739215 + } + } + }, + "times": { + "evaluation": 228.67922687530518, + "training": 11.537832736968994 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json new file mode 100644 index 0000000..03f1778 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult creates two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json new file mode 100644 index 0000000..6a1120c --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/fb15k/yang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 1.9983569268922784e-05, + 1.040764747672111e-05, + 6.409794851883364e-06, + 4.78967919205319e-06, + 3.924975132708391e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.043914466668474816, + "adjusted_arithmetic_mean_rank_index": 0.956214981563262, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.17646159546294732, + "arithmetic_mean_rank": 324.3896836010902, + "count": 118142.0, + "geometric_mean_rank": 40.0771716312718, + "harmonic_mean_rank": 5.64876151716278, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.0030827120915155985, + "inverse_geometric_mean_rank": 0.02495186060534547, + "inverse_harmonic_mean_rank": 0.17702995549762082, + "inverse_median_rank": 0.024390243902439025, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2480600719877, + "variance": 728032.2520166102, + "z_arithmetic_mean_rank": 568.7732063158522, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.507692889807 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.043914504482264104, + "adjusted_arithmetic_mean_rank_index": 0.9562149437443529, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.1764615264839941, + "arithmetic_mean_rank": 324.38996292597045, + "count": 118142.0, + "geometric_mean_rank": 40.077258631969784, + "harmonic_mean_rank": 5.6487637166600075, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.003082709437061749, + "inverse_geometric_mean_rank": 0.02495180643923325, + "inverse_harmonic_mean_rank": 0.17702988656627303, + "inverse_median_rank": 0.024390243902439025, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2481634963606, + "variance": 728032.4285099121, + "z_arithmetic_mean_rank": 568.7731838205111, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.505445012585 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04391448256486668, + "adjusted_arithmetic_mean_rank_index": 0.9562149656647179, + "adjusted_hits_at_k": 0.3139274840441076, + "adjusted_inverse_harmonic_mean_rank": 0.17646156018151202, + "arithmetic_mean_rank": 324.3898010253906, + "count": 118142.0, + "geometric_mean_rank": 40.07722091674805, + "harmonic_mean_rank": 5.648762876681754, + "hits_at_1": 0.10610959692573343, + "hits_at_10": 0.31439284928306616, + "hits_at_3": 0.1927341673579252, + "hits_at_5": 0.23630884867363003, + "inverse_arithmetic_mean_rank": 0.0030827110167592764, + "inverse_geometric_mean_rank": 0.024951830506324768, + "inverse_harmonic_mean_rank": 0.1770299202405348, + "inverse_median_rank": 0.024390242993831635, + "median_absolute_deviation": 59.30408874022408, + "median_rank": 41.0, + "standard_deviation": 853.2481079101562, + "variance": 728032.375, + "z_arithmetic_mean_rank": 568.7731968591232, + "z_hits_at_k": 4141.64221762799, + "z_inverse_harmonic_mean_rank": 5750.506543142948 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.046864358757327716, + "adjusted_arithmetic_mean_rank_index": 0.9532650422909069, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807807836110954, + "arithmetic_mean_rank": 345.2379678691744, + "count": 59071.0, + "geometric_mean_rank": 46.03301841722873, + "harmonic_mean_rank": 6.302757159725196, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.002896552792765086, + "inverse_geometric_mean_rank": 0.021723537460357607, + "inverse_harmonic_mean_rank": 0.15866072175365242, + "inverse_median_rank": 0.020833333333333332, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2229416788517, + "variance": 734831.171740544, + "z_arithmetic_mean_rank": 400.8845593870582, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.051103974374 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04686439782330793, + "adjusted_arithmetic_mean_rank_index": 0.953265003219623, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807805731536828, + "arithmetic_mean_rank": 345.2382556584449, + "count": 59071.0, + "geometric_mean_rank": 46.033083293710646, + "harmonic_mean_rank": 6.302757995183501, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.002896550378209915, + "inverse_geometric_mean_rank": 0.021723506844405247, + "inverse_harmonic_mean_rank": 0.15866070072247565, + "inverse_median_rank": 0.020833333333333332, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2230169884216, + "variance": 734831.3008547318, + "z_arithmetic_mean_rank": 400.8845429560828, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.050619755201 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.046864376441567664, + "adjusted_arithmetic_mean_rank_index": 0.9532650246042661, + "adjusted_hits_at_k": 0.29047220135379653, + "adjusted_inverse_harmonic_mean_rank": 0.15807806772369895, + "arithmetic_mean_rank": 345.23809814453125, + "count": 59071.0, + "geometric_mean_rank": 46.033042907714844, + "harmonic_mean_rank": 6.30275744512894, + "hits_at_1": 0.09011189923989775, + "hits_at_10": 0.2909549525147704, + "hits_at_3": 0.17233498671090722, + "hits_at_5": 0.21372585532664082, + "inverse_arithmetic_mean_rank": 0.0028965517412871122, + "inverse_geometric_mean_rank": 0.02172352559864521, + "inverse_harmonic_mean_rank": 0.15866071112360333, + "inverse_median_rank": 0.02083333395421505, + "median_absolute_deviation": 68.19970205125769, + "median_rank": 48.0, + "standard_deviation": 857.2229614257812, + "variance": 734831.25, + "z_arithmetic_mean_rank": 400.88455194914644, + "z_hits_at_k": 2705.6239083240193, + "z_inverse_harmonic_mean_rank": 3637.0508592294586 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04098058733238346, + "adjusted_arithmetic_mean_rank_index": 0.959148905664551, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.19484504281516982, + "arithmetic_mean_rank": 303.54139933300604, + "count": 59071.0, + "geometric_mean_rank": 34.891904576070026, + "harmonic_mean_rank": 5.117728501747324, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.0032944435328998746, + "inverse_geometric_mean_rank": 0.02865994310570916, + "inverse_harmonic_mean_rank": 0.19539918924158917, + "inverse_median_rank": 0.02857142857142857, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7426172750083, + "variance": 720364.0303788312, + "z_arithmetic_mean_rank": 403.4782114554786, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.717570495163 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.040980623900779035, + "adjusted_arithmetic_mean_rank_index": 0.9591488690912178, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.1948449259031866, + "arithmetic_mean_rank": 303.541670193496, + "count": 59071.0, + "geometric_mean_rank": 34.89200689003693, + "harmonic_mean_rank": 5.1177315617004036, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.0032944405931565803, + "inverse_geometric_mean_rank": 0.028659859066047018, + "inverse_harmonic_mean_rank": 0.19539907241007043, + "inverse_median_rank": 0.02857142857142857, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7427487439139, + "variance": 720364.2535453746, + "z_arithmetic_mean_rank": 403.4781960704398, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.71487235021 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04098060557075636, + "adjusted_arithmetic_mean_rank_index": 0.9591488874237154, + "adjusted_hits_at_k": 0.33738266907397446, + "adjusted_inverse_harmonic_mean_rank": 0.1948449828898034, + "arithmetic_mean_rank": 303.5415344238281, + "count": 59071.0, + "geometric_mean_rank": 34.8919677734375, + "harmonic_mean_rank": 5.117730112424955, + "hits_at_1": 0.12210729461156913, + "hits_at_10": 0.33783074605136193, + "hits_at_3": 0.2131333480049432, + "hits_at_5": 0.25889184202061927, + "inverse_arithmetic_mean_rank": 0.003294442081823945, + "inverse_geometric_mean_rank": 0.028659891337156296, + "inverse_harmonic_mean_rank": 0.19539912935746628, + "inverse_median_rank": 0.02857142873108387, + "median_absolute_deviation": 50.40847542919047, + "median_rank": 35.0, + "standard_deviation": 848.7426147460938, + "variance": 720364.0625, + "z_arithmetic_mean_rank": 403.4782037822386, + "z_hits_at_k": 3152.233940323348, + "z_inverse_harmonic_mean_rank": 4496.716187511824 + } + } + }, + "times": { + "evaluation": 20.92709469795227, + "training": 15.750773191452026 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json new file mode 100644 index 0000000..561f095 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "For each positive triple, DistMult create two negatives where head and tail are corrupted", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with DistMult as described by Yang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json new file mode 100644 index 0000000..c4a98ae --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/distmult/wn18/yang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 2.09204011979469e-05, + 1.6905181007187295e-05, + 1.3378267618860774e-05, + 1.0851657965730716e-05, + 8.871422993938076e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04896868446936024, + "adjusted_arithmetic_mean_rank_index": 0.9510777929270965, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576, + "count": 10000.0, + "geometric_mean_rank": 5.3526386004716215, + "harmonic_mean_rank": 1.6319800035619036, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.00099794662502435, + "inverse_geometric_mean_rank": 0.18682374706035448, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982390329411, + "variance": 20903022.977482237, + "z_arithmetic_mean_rank": 164.72727454841043, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04896868446936024, + "adjusted_arithmetic_mean_rank_index": 0.9510777929270965, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576, + "count": 10000.0, + "geometric_mean_rank": 5.3526386004716215, + "harmonic_mean_rank": 1.6319800035619036, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.00099794662502435, + "inverse_geometric_mean_rank": 0.18682374706035448, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982390329411, + "variance": 20903022.977482237, + "z_arithmetic_mean_rank": 164.72727454841043, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04896868530928128, + "adjusted_arithmetic_mean_rank_index": 0.9510777920871344, + "adjusted_hits_at_k": 0.7553402180539828, + "adjusted_inverse_harmonic_mean_rank": 0.612646630277034, + "arithmetic_mean_rank": 1002.0576171875, + "count": 10000.0, + "geometric_mean_rank": 5.352639198303223, + "harmonic_mean_rank": 1.6319799215470276, + "hits_at_1": 0.5278, + "hits_at_10": 0.7554, + "hits_at_3": 0.6717, + "hits_at_5": 0.7107, + "inverse_arithmetic_mean_rank": 0.000997946597635746, + "inverse_geometric_mean_rank": 0.18682372570037842, + "inverse_harmonic_mean_rank": 0.612752605924971, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4571.982421875, + "variance": 20903022.0, + "z_arithmetic_mean_rank": 164.7272744029285, + "z_hits_at_k": 4831.542585181187, + "z_inverse_harmonic_mean_rank": 9669.883247396065 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04896234274102222, + "adjusted_arithmetic_mean_rank_index": 0.9510841360429304, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046, + "count": 5000.0, + "geometric_mean_rank": 5.329299858068893, + "harmonic_mean_rank": 1.629419167883981, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.000998099020605355, + "inverse_geometric_mean_rank": 0.18764190918736492, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.616893806392, + "variance": 20927119.72509884, + "z_arithmetic_mean_rank": 116.48053990240568, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04896234274102222, + "adjusted_arithmetic_mean_rank_index": 0.9510841360429304, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046, + "count": 5000.0, + "geometric_mean_rank": 5.329299858068893, + "harmonic_mean_rank": 1.629419167883981, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.000998099020605355, + "inverse_geometric_mean_rank": 0.18764190918736492, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.616893806392, + "variance": 20927119.72509884, + "z_arithmetic_mean_rank": 116.48053990240568, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.048962342841242394, + "adjusted_arithmetic_mean_rank_index": 0.9510841359427052, + "adjusted_hits_at_k": 0.7549401188924727, + "adjusted_inverse_harmonic_mean_rank": 0.6136099088029643, + "arithmetic_mean_rank": 1001.9046020507812, + "count": 5000.0, + "geometric_mean_rank": 5.329300403594971, + "harmonic_mean_rank": 1.6294191629877437, + "hits_at_1": 0.5286, + "hits_at_10": 0.755, + "hits_at_3": 0.6732, + "hits_at_5": 0.7098, + "inverse_arithmetic_mean_rank": 0.0009980989852920175, + "inverse_geometric_mean_rank": 0.18764188885688782, + "inverse_harmonic_mean_rank": 0.6137156231558476, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4574.61669921875, + "variance": 20927120.0, + "z_arithmetic_mean_rank": 116.48053989013098, + "z_hits_at_k": 3414.5669904780675, + "z_inverse_harmonic_mean_rank": 6848.311118156339 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04897502590366397, + "adjusted_arithmetic_mean_rank_index": 0.9510714501053755, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2106, + "count": 5000.0, + "geometric_mean_rank": 5.376079550840021, + "harmonic_mean_rank": 1.63454890125634, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977942759735329, + "inverse_geometric_mean_rank": 0.1860091523094647, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.346362779653, + "variance": 20878926.183047645, + "z_arithmetic_mean_rank": 116.47900595429857, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.04897502590366397, + "adjusted_arithmetic_mean_rank_index": 0.9510714501053755, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2106, + "count": 5000.0, + "geometric_mean_rank": 5.376079550840021, + "harmonic_mean_rank": 1.63454890125634, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977942759735329, + "inverse_geometric_mean_rank": 0.1860091523094647, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.346362779653, + "variance": 20878926.183047645, + "z_arithmetic_mean_rank": 116.47900595429857, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.04897502450064657, + "adjusted_arithmetic_mean_rank_index": 0.9510714515084615, + "adjusted_hits_at_k": 0.7557403172109258, + "adjusted_inverse_harmonic_mean_rank": 0.6116833517623125, + "arithmetic_mean_rank": 1002.2105712890625, + "count": 5000.0, + "geometric_mean_rank": 5.376080513000488, + "harmonic_mean_rank": 1.6345490025499922, + "hits_at_1": 0.527, + "hits_at_10": 0.7558, + "hits_at_3": 0.6702, + "hits_at_5": 0.7116, + "inverse_arithmetic_mean_rank": 0.0009977943263947964, + "inverse_geometric_mean_rank": 0.18600912392139435, + "inverse_harmonic_mean_rank": 0.6117895886940944, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 4569.34619140625, + "variance": 20878926.0, + "z_arithmetic_mean_rank": 116.4790061261364, + "z_hits_at_k": 3418.2661043162093, + "z_inverse_harmonic_mean_rank": 6826.96866898661 + } + } + }, + "times": { + "evaluation": 5.345269203186035, + "training": 5.9453136920928955 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json new file mode 100644 index 0000000..8fa85b6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json new file mode 100644 index 0000000..1584b34 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/fb15k/nickel2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 1.8612450855828065e-05, + 1.7000744598844161e-06, + 1.2025011869850476e-06, + 1.0128473853421426e-06, + 9.114429453462934e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.03748523558448788, + "adjusted_arithmetic_mean_rank_index": 0.9626450831264751, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514178450284293, + "arithmetic_mean_rank": 276.8979025240812, + "count": 118142.0, + "geometric_mean_rank": 25.875818197160168, + "harmonic_mean_rank": 4.861676761958902, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.003611439418227562, + "inverse_geometric_mean_rank": 0.03864612096052478, + "inverse_harmonic_mean_rank": 0.20569035107901182, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3979695616902, + "variance": 935858.8315120807, + "z_arithmetic_mean_rank": 572.5979419177419, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.133990894033 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.03748524704321191, + "adjusted_arithmetic_mean_rank_index": 0.9626450716661996, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514173170088618, + "arithmetic_mean_rank": 276.8979871679843, + "count": 118142.0, + "geometric_mean_rank": 25.87585088024041, + "harmonic_mean_rank": 4.861678009119727, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.0036114383142602444, + "inverse_geometric_mean_rank": 0.038646072147665314, + "inverse_harmonic_mean_rank": 0.20569029831349603, + "inverse_median_rank": 0.045454545454545456, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3980848472062, + "variance": 935859.0545660424, + "z_arithmetic_mean_rank": 572.5979351009719, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.132270190683 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.03748523777447217, + "adjusted_arithmetic_mean_rank_index": 0.9626450809361943, + "adjusted_hits_at_k": 0.38729579887774673, + "adjusted_inverse_harmonic_mean_rank": 0.20514175745144342, + "arithmetic_mean_rank": 276.8979187011719, + "count": 118142.0, + "geometric_mean_rank": 25.875835418701172, + "harmonic_mean_rank": 4.8616774504878295, + "hits_at_1": 0.11486177650623826, + "hits_at_10": 0.3877113981479914, + "hits_at_3": 0.22980819691557616, + "hits_at_5": 0.29242775642870444, + "inverse_arithmetic_mean_rank": 0.0036114391405135393, + "inverse_geometric_mean_rank": 0.038646094501018524, + "inverse_harmonic_mean_rank": 0.20569032404628168, + "inverse_median_rank": 0.04545454680919647, + "median_absolute_deviation": 31.134646588617642, + "median_rank": 22.0, + "standard_deviation": 967.3980712890625, + "variance": 935859.0, + "z_arithmetic_mean_rank": 572.597940614925, + "z_hits_at_k": 5109.589675546418, + "z_inverse_harmonic_mean_rank": 6685.133109346518 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.04029054472522748, + "adjusted_arithmetic_mean_rank_index": 0.95983974880713, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.1901638974649306, + "arithmetic_mean_rank": 296.8103299419343, + "count": 59071.0, + "geometric_mean_rank": 29.242573492518023, + "harmonic_mean_rank": 5.243169382262753, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.003369154975824569, + "inverse_geometric_mean_rank": 0.034196716655456434, + "inverse_harmonic_mean_rank": 0.190724336196905, + "inverse_median_rank": 0.038461538461538464, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.321567958725, + "variance": 982718.451100145, + "z_arithmetic_mean_rank": 403.64947597155924, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.279737592333 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0402905585132205, + "adjusted_arithmetic_mean_rank_index": 0.959839735017265, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.19016385703379793, + "arithmetic_mean_rank": 296.810431514618, + "count": 59071.0, + "geometric_mean_rank": 29.24260990758518, + "harmonic_mean_rank": 5.243170492978996, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.0033691538228525827, + "inverse_geometric_mean_rank": 0.03419667407116805, + "inverse_harmonic_mean_rank": 0.19072429579375227, + "inverse_median_rank": 0.038461538461538464, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.3216042325861, + "variance": 982718.5230182679, + "z_arithmetic_mean_rank": 403.6494701723915, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.278807355217 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.040290553459769014, + "adjusted_arithmetic_mean_rank_index": 0.9598397400714026, + "adjusted_hits_at_k": 0.3647046245623646, + "adjusted_inverse_harmonic_mean_rank": 0.19016387690131012, + "arithmetic_mean_rank": 296.8103942871094, + "count": 59071.0, + "geometric_mean_rank": 29.242595672607422, + "harmonic_mean_rank": 5.243169799388333, + "hits_at_1": 0.10323170422034501, + "hits_at_10": 0.36513686919131216, + "hits_at_3": 0.2119652621421679, + "hits_at_5": 0.2733151631088013, + "inverse_arithmetic_mean_rank": 0.0033691541757434607, + "inverse_geometric_mean_rank": 0.03419668972492218, + "inverse_harmonic_mean_rank": 0.19072431564751535, + "inverse_median_rank": 0.03846153989434242, + "median_absolute_deviation": 37.065055462640046, + "median_rank": 26.0, + "standard_deviation": 991.3215942382812, + "variance": 982718.5, + "z_arithmetic_mean_rank": 403.6494722978504, + "z_hits_at_k": 3397.0670759312984, + "z_inverse_harmonic_mean_rank": 4375.279264465766 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.034695154364071244, + "adjusted_arithmetic_mean_rank_index": 0.9654351873326302, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.22011961471257302, + "arithmetic_mean_rank": 256.9854751062281, + "count": 59071.0, + "geometric_mean_rank": 22.896684094640214, + "harmonic_mean_rank": 4.531933604744529, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912705069679048, + "inverse_geometric_mean_rank": 0.04367444630264544, + "inverse_harmonic_mean_rank": 0.22065636596111857, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.4469228517189, + "variance": 888206.2023926738, + "z_arithmetic_mean_rank": 406.1226159573892, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.014994415809 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.034695163506170135, + "adjusted_arithmetic_mean_rank_index": 0.9654351781892969, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.22011954953983928, + "arithmetic_mean_rank": 256.9855428213506, + "count": 59071.0, + "geometric_mean_rank": 22.896713422380326, + "harmonic_mean_rank": 4.531934942368793, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912694816267274, + "inverse_geometric_mean_rank": 0.043674390361306305, + "inverse_harmonic_mean_rank": 0.22065630083323984, + "inverse_median_rank": 0.05263157894736842, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.447120656698, + "variance": 888206.5752341007, + "z_arithmetic_mean_rank": 406.12261211112946, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.013490331473 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.03469515828527225, + "adjusted_arithmetic_mean_rank_index": 0.9654351834108997, + "adjusted_hits_at_k": 0.40988687913056115, + "adjusted_inverse_harmonic_mean_rank": 0.2201195811734192, + "arithmetic_mean_rank": 256.9855041503906, + "count": 59071.0, + "geometric_mean_rank": 22.89670181274414, + "harmonic_mean_rank": 4.531934150162851, + "hits_at_1": 0.1264918487921315, + "hits_at_10": 0.41028592710467066, + "hits_at_3": 0.24765113168898445, + "hits_at_5": 0.3115403497486076, + "inverse_arithmetic_mean_rank": 0.0038912701420485973, + "inverse_geometric_mean_rank": 0.04367441311478615, + "inverse_harmonic_mean_rank": 0.220656332445048, + "inverse_median_rank": 0.05263157933950424, + "median_absolute_deviation": 26.686839933100835, + "median_rank": 19.0, + "standard_deviation": 942.447021484375, + "variance": 888206.4375, + "z_arithmetic_mean_rank": 406.1226143076633, + "z_hits_at_k": 3829.6553158315073, + "z_inverse_harmonic_mean_rank": 5080.0142203848145 + } + } + }, + "times": { + "evaluation": 34.076919078826904, + "training": 4.2241315841674805 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json new file mode 100644 index 0000000..e53e97a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with HolE as described by Nickel et al., 2016" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json new file mode 100644 index 0000000..2bd19a4 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/hole/wn18/nickel2016/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.795345775860285e-05, + 2.2625781074151614e-06, + 2.8825618770567654e-07, + 2.830078377052869e-07, + 2.627437319533504e-07 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05202866037834784, + "adjusted_arithmetic_mean_rank_index": 0.9480176674754901, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824278747, + "arithmetic_mean_rank": 1064.6746, + "count": 10000.0, + "geometric_mean_rank": 2.5872329490836847, + "harmonic_mean_rank": 1.2936486730420298, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.0009392541157645726, + "inverse_geometric_mean_rank": 0.386513321250863, + "inverse_harmonic_mean_rank": 0.7730074021167497, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.040841194875, + "variance": 26092081.235314842, + "z_arithmetic_mean_rank": 164.19725888705435, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.002853722233 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.052028670151974515, + "adjusted_arithmetic_mean_rank_index": 0.9480176577013858, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824037771, + "arithmetic_mean_rank": 1064.6748, + "count": 10000.0, + "geometric_mean_rank": 2.587233083873683, + "harmonic_mean_rank": 1.2936486730823467, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.000939253939324947, + "inverse_geometric_mean_rank": 0.38651330111424287, + "inverse_harmonic_mean_rank": 0.7730074020926587, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.041468042811, + "variance": 26092087.63924496, + "z_arithmetic_mean_rank": 164.19725719417332, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.002853341883 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05202866441569558, + "adjusted_arithmetic_mean_rank_index": 0.9480176634379451, + "adjusted_hits_at_k": 0.895374435030444, + "adjusted_inverse_harmonic_mean_rank": 0.7729452824158229, + "arithmetic_mean_rank": 1064.6746826171875, + "count": 10000.0, + "geometric_mean_rank": 2.587232828140259, + "harmonic_mean_rank": 1.2936486399760676, + "hits_at_1": 0.7071, + "hits_at_10": 0.8954, + "hits_at_3": 0.8185, + "hits_at_5": 0.8575, + "inverse_arithmetic_mean_rank": 0.0009392540669068694, + "inverse_geometric_mean_rank": 0.386513352394104, + "inverse_harmonic_mean_rank": 0.7730074021047012, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.041015625, + "variance": 26092084.0, + "z_arithmetic_mean_rank": 164.19725818774901, + "z_hits_at_k": 5727.273100428186, + "z_inverse_harmonic_mean_rank": 12200.00285353201 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05201507874896203, + "adjusted_arithmetic_mean_rank_index": 0.9480312508427307, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076970289, + "arithmetic_mean_rank": 1064.372, + "count": 5000.0, + "geometric_mean_rank": 2.602302684720317, + "harmonic_mean_rank": 1.2978888205900572, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395211448628862, + "inverse_geometric_mean_rank": 0.3842750521957346, + "inverse_harmonic_mean_rank": 0.7704820198276857, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.4860026446195, + "variance": 26096629.239216004, + "z_arithmetic_mean_rank": 116.10664898896995, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384857467 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05201509829666856, + "adjusted_arithmetic_mean_rank_index": 0.9480312312940687, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076488339, + "arithmetic_mean_rank": 1064.3724, + "count": 5000.0, + "geometric_mean_rank": 2.6023029558705306, + "harmonic_mean_rank": 1.2978888206712202, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395207917830263, + "inverse_geometric_mean_rank": 0.3842750121557145, + "inverse_harmonic_mean_rank": 0.7704820197795038, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.4872562548535, + "variance": 26096642.04731824, + "z_arithmetic_mean_rank": 116.10664659481931, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384319577 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05201508815056893, + "adjusted_arithmetic_mean_rank_index": 0.9480312414406642, + "adjusted_hits_at_k": 0.8943741900205923, + "adjusted_inverse_harmonic_mean_rank": 0.7704192076729255, + "arithmetic_mean_rank": 1064.3721923828125, + "count": 5000.0, + "geometric_mean_rank": 2.6023025512695312, + "harmonic_mean_rank": 1.297888801348993, + "hits_at_1": 0.703, + "hits_at_10": 0.8944, + "hits_at_3": 0.8156, + "hits_at_5": 0.8562, + "inverse_arithmetic_mean_rank": 0.0009395209490321577, + "inverse_geometric_mean_rank": 0.38427507877349854, + "inverse_harmonic_mean_rank": 0.7704820198035889, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5108.48681640625, + "variance": 26096636.0, + "z_arithmetic_mean_rank": 116.10664783748636, + "z_hits_at_k": 4045.2222765165366, + "z_inverse_harmonic_mean_rank": 8598.411384588457 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05204224137802127, + "adjusted_arithmetic_mean_rank_index": 0.9480040847380733, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9772, + "count": 5000.0, + "geometric_mean_rank": 2.57225048113251, + "harmonic_mean_rank": 1.2894361400416687, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872384122401, + "inverse_geometric_mean_rank": 0.3887646274478371, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595623018737, + "variance": 26087533.048280153, + "z_arithmetic_mean_rank": 116.10334157194073, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05204224137802127, + "adjusted_arithmetic_mean_rank_index": 0.9480040847380733, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9772, + "count": 5000.0, + "geometric_mean_rank": 2.57225048113251, + "harmonic_mean_rank": 1.2894361400416687, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872384122401, + "inverse_geometric_mean_rank": 0.3887646274478371, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595623018737, + "variance": 26087533.048280153, + "z_arithmetic_mean_rank": 116.10334157194073, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05204224005135856, + "adjusted_arithmetic_mean_rank_index": 0.9480040860648008, + "adjusted_hits_at_k": 0.8963746800288777, + "adjusted_inverse_harmonic_mean_rank": 0.7754713571293268, + "arithmetic_mean_rank": 1064.9771728515625, + "count": 5000.0, + "geometric_mean_rank": 2.5722501277923584, + "harmonic_mean_rank": 1.2894360933309963, + "hits_at_1": 0.7112, + "hits_at_10": 0.8964, + "hits_at_3": 0.8214, + "hits_at_5": 0.8588, + "inverse_arithmetic_mean_rank": 0.0009389872429892421, + "inverse_geometric_mean_rank": 0.3887646794319153, + "inverse_harmonic_mean_rank": 0.7755327844058136, + "inverse_median_rank": 1.0, + "median_absolute_deviation": 0.0, + "median_rank": 1.0, + "standard_deviation": 5107.595703125, + "variance": 26087534.0, + "z_arithmetic_mean_rank": 116.10334173442685, + "z_hits_at_k": 4054.3651248062633, + "z_inverse_harmonic_mean_rank": 8654.998772756573 + } + } + }, + "times": { + "evaluation": 7.570884943008423, + "training": 2.3045146465301514 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json new file mode 100644 index 0000000..f9417f7 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json new file mode 100644 index 0000000..b403d8e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/fb15k/he2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.008025066050702042, + 0.006834184265635721, + 0.006292605368404343, + 0.00592642079037386, + 0.005657277270934352 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9204867973289346, + "adjusted_arithmetic_mean_rank_index": 0.07952396828054997, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.6141685120576216e-05, + "arithmetic_mean_rank": 6799.500110037074, + "count": 118142.0, + "geometric_mean_rank": 4816.9160901851965, + "harmonic_mean_rank": 1528.9884148398814, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00014706963509329918, + "inverse_geometric_mean_rank": 0.00020760170641908625, + "inverse_harmonic_mean_rank": 0.0006540271922889108, + "inverse_median_rank": 0.00015332720024532353, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.87870870036, + "variance": 18035978.76641244, + "z_arithmetic_mean_rank": 47.302231496041486, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.1777805690503058 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9204871410906554, + "adjusted_arithmetic_mean_rank_index": 0.07952362447228589, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.6142143183925403e-05, + "arithmetic_mean_rank": 6799.502649354167, + "count": 118142.0, + "geometric_mean_rank": 4816.919034257423, + "harmonic_mean_rank": 1528.9894849646314, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.00014706958016921757, + "inverse_geometric_mean_rank": 0.00020760157953415967, + "inverse_harmonic_mean_rank": 0.0006540267345416912, + "inverse_median_rank": 0.00015332720024532353, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.879017549971, + "variance": 18035981.389706206, + "z_arithmetic_mean_rank": 47.30202699294068, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.1777954963595967 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9204869807367789, + "adjusted_arithmetic_mean_rank_index": 0.07952378484787326, + "adjusted_hits_at_k": -0.0002298465007639259, + "adjusted_inverse_harmonic_mean_rank": -3.614191435791117e-05, + "arithmetic_mean_rank": 6799.50146484375, + "count": 118142.0, + "geometric_mean_rank": 4816.91455078125, + "harmonic_mean_rank": 1528.9889966880858, + "hits_at_1": 3.3857561239863894e-05, + "hits_at_10": 0.0004486126864281966, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.00016928780619931947, + "inverse_arithmetic_mean_rank": 0.0001470696006435901, + "inverse_geometric_mean_rank": 0.0002076017699437216, + "inverse_harmonic_mean_rank": 0.0006540269632097826, + "inverse_median_rank": 0.00015332720067817718, + "median_absolute_deviation": 5387.776462049357, + "median_rank": 6522.0, + "standard_deviation": 4246.87841796875, + "variance": 18035978.0, + "z_arithmetic_mean_rank": 47.3021223871134, + "z_hits_at_k": -3.0323626299766397, + "z_inverse_harmonic_mean_rank": -1.177788039406984 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9334855474142308, + "adjusted_arithmetic_mean_rank_index": 0.06652348282188125, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.608883349467907e-05, + "arithmetic_mean_rank": 6876.753720099541, + "count": 59071.0, + "geometric_mean_rank": 4918.491902159602, + "harmonic_mean_rank": 1623.3673438831504, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.0001454174514171092, + "inverse_geometric_mean_rank": 0.00020331435323923616, + "inverse_harmonic_mean_rank": 0.0006160035211796029, + "inverse_median_rank": 0.00015015015015015014, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.158683323923, + "variance": 17928099.75556737, + "z_arithmetic_mean_rank": 27.975679288367417, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.7506473935605982 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9334858415580818, + "adjusted_arithmetic_mean_rank_index": 0.06652318863809625, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.608907332910548e-05, + "arithmetic_mean_rank": 6876.75588698346, + "count": 59071.0, + "geometric_mean_rank": 4918.49423226719, + "harmonic_mean_rank": 1623.3679754868276, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.00014541740559568668, + "inverse_geometric_mean_rank": 0.00020331425692026237, + "inverse_harmonic_mean_rank": 0.0006160032815111514, + "inverse_median_rank": 0.00015015015015015014, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.158920923222, + "variance": 17928101.767633703, + "z_arithmetic_mean_rank": 27.97555557278804, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.750652911656939 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9334856389650233, + "adjusted_arithmetic_mean_rank_index": 0.06652339125865947, + "adjusted_hits_at_k": -0.0003420408236310326, + "adjusted_inverse_harmonic_mean_rank": -7.60889534720333e-05, + "arithmetic_mean_rank": 6876.75439453125, + "count": 59071.0, + "geometric_mean_rank": 4918.4921875, + "harmonic_mean_rank": 1623.3677168983902, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.00033857561239863894, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001523590255793875, + "inverse_arithmetic_mean_rank": 0.00014541743439622223, + "inverse_geometric_mean_rank": 0.00020331433915998787, + "inverse_harmonic_mean_rank": 0.0006160034012852777, + "inverse_median_rank": 0.00015015015378594398, + "median_absolute_deviation": 5361.089622116257, + "median_rank": 6660.0, + "standard_deviation": 4234.15869140625, + "variance": 17928100.0, + "z_arithmetic_mean_rank": 27.97564078251628, + "z_hits_at_k": -3.1859634957349265, + "z_inverse_harmonic_mean_rank": -1.7506501539924884 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9075586077133286, + "adjusted_arithmetic_mean_rank_index": 0.0924538743206843, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.805311688501677e-06, + "arithmetic_mean_rank": 6722.246499974607, + "count": 59071.0, + "geometric_mean_rank": 4717.438003648486, + "harmonic_mean_rank": 1444.9804962161888, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014875979332263067, + "inverse_geometric_mean_rank": 0.00021197946835265155, + "inverse_harmonic_mean_rank": 0.0006920508633982187, + "inverse_median_rank": 0.00015683814303638644, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.159407152596, + "variance": 18131921.536722146, + "z_arithmetic_mean_rank": 38.891900551347305, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08782061726419244 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9075590008235809, + "adjusted_arithmetic_mean_rank_index": 0.09245348115735175, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.804635397057607e-06, + "arithmetic_mean_rank": 6722.249411724873, + "count": 59071.0, + "geometric_mean_rank": 4717.441535332689, + "harmonic_mean_rank": 1444.9819073210315, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.00014875972888715063, + "inverse_geometric_mean_rank": 0.00021197930965549883, + "inverse_harmonic_mean_rank": 0.000692050187572231, + "inverse_median_rank": 0.00015683814303638644, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.159800469012, + "variance": 18131924.8863303, + "z_arithmetic_mean_rank": 38.89173516218062, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08780500951982617 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9075588824796018, + "adjusted_arithmetic_mean_rank_index": 0.09245359951731047, + "adjusted_hits_at_k": -0.00011765264503880795, + "adjusted_inverse_harmonic_mean_rank": 3.8049731916007037e-06, + "arithmetic_mean_rank": 6722.24853515625, + "count": 59071.0, + "geometric_mean_rank": 4717.43994140625, + "harmonic_mean_rank": 1444.9812400101264, + "hits_at_1": 1.6928780619931947e-05, + "hits_at_10": 0.0005586497604577542, + "hits_at_3": 0.00010157268371959168, + "hits_at_5": 0.0001862165868192514, + "inverse_arithmetic_mean_rank": 0.000148759747389704, + "inverse_geometric_mean_rank": 0.0002119793789461255, + "inverse_harmonic_mean_rank": 0.0006920505251342877, + "inverse_median_rank": 0.00015683814126532525, + "median_absolute_deviation": 5417.428506419469, + "median_rank": 6376.0, + "standard_deviation": 4258.15966796875, + "variance": 18131922.0, + "z_arithmetic_mean_rank": 38.89178495180573, + "z_hits_at_k": -1.099252258208999, + "z_inverse_harmonic_mean_rank": 0.08781280528735103 + } + } + }, + "times": { + "evaluation": 28.00254464149475, + "training": 13.801588296890259 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json new file mode 100644 index 0000000..b5fa871 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with KG2E as described by He et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json new file mode 100644 index 0000000..a66458d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/kg2e/wn18/he2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.1859842339811534, + 0.1488544849477889, + 0.13528452142724368, + 0.12644040748196536, + 0.1193752745758366 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6755986993844032, + "adjusted_arithmetic_mean_rank_index": 0.32441715427636864, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160908295552288, + "arithmetic_mean_rank": 13824.9336, + "count": 10000.0, + "geometric_mean_rank": 5556.814652149579, + "harmonic_mean_rank": 87.47801345454457, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233307796863487e-05, + "inverse_geometric_mean_rank": 0.00017995921451386963, + "inverse_harmonic_mean_rank": 0.011431443862401164, + "inverse_median_rank": 9.026085386767759e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.625858922253, + "variance": 143152271.94399107, + "z_arithmetic_mean_rank": 56.189256060985315, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.161387036573 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6755991538580438, + "adjusted_arithmetic_mean_rank_index": 0.3244166997805177, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160907981794924, + "arithmetic_mean_rank": 13824.9429, + "count": 10000.0, + "geometric_mean_rank": 5556.819712994077, + "harmonic_mean_rank": 87.4780158548855, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233302931037784e-05, + "inverse_geometric_mean_rank": 0.00017995905061695598, + "inverse_harmonic_mean_rank": 0.011431443548729641, + "inverse_median_rank": 9.026085386767759e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.629322851568, + "variance": 143152354.8332396, + "z_arithmetic_mean_rank": 56.189177342016805, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.16138208429393 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6755988899701234, + "adjusted_arithmetic_mean_rank_index": 0.3244169636813343, + "adjusted_hits_at_k": 0.020260603368259057, + "adjusted_inverse_harmonic_mean_rank": 0.011160908138604079, + "arithmetic_mean_rank": 13824.9375, + "count": 10000.0, + "geometric_mean_rank": 5556.8173828125, + "harmonic_mean_rank": 87.4780140344442, + "hits_at_1": 0.0054, + "hits_at_10": 0.0205, + "hits_at_3": 0.0121, + "hits_at_5": 0.0157, + "inverse_arithmetic_mean_rank": 7.233305950649083e-05, + "inverse_geometric_mean_rank": 0.00017995912639889866, + "inverse_harmonic_mean_rank": 0.011431443705495894, + "inverse_median_rank": 9.026085172081366e-05, + "median_absolute_deviation": 13728.896543361874, + "median_rank": 11079.0, + "standard_deviation": 11964.626953125, + "variance": 143152304.0, + "z_arithmetic_mean_rank": 56.189223049804966, + "z_hits_at_k": 129.59718764533335, + "z_inverse_harmonic_mean_rank": 176.16138455933606 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6871711031180322, + "adjusted_arithmetic_mean_rank_index": 0.31284418534778335, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.01172720074827386, + "arithmetic_mean_rank": 14061.4164, + "count": 5000.0, + "geometric_mean_rank": 5711.947514522259, + "harmonic_mean_rank": 83.35009273134703, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111659107115269e-05, + "inverse_geometric_mean_rank": 0.00017507163668040794, + "inverse_harmonic_mean_rank": 0.01199758713194462, + "inverse_median_rank": 8.702462796971542e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.630931162526, + "variance": 144183200.57901102, + "z_arithmetic_mean_rank": 38.31444373181431, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.8836740037781 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6871715038460162, + "adjusted_arithmetic_mean_rank_index": 0.3128437846002152, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.011727200219012527, + "arithmetic_mean_rank": 14061.4246, + "count": 5000.0, + "geometric_mean_rank": 5711.953444634062, + "harmonic_mean_rank": 83.35009640724563, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111654959910676e-05, + "inverse_geometric_mean_rank": 0.0001750714549222075, + "inverse_harmonic_mean_rank": 0.011997586602828092, + "inverse_median_rank": 8.702462796971542e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.632850995855, + "variance": 144183246.68431485, + "z_arithmetic_mean_rank": 38.31439465172629, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.88366809685547 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6871713229533725, + "adjusted_arithmetic_mean_rank_index": 0.31284396550169946, + "adjusted_hits_at_k": 0.02096076886511117, + "adjusted_inverse_harmonic_mean_rank": 0.011727200483508837, + "arithmetic_mean_rank": 14061.4208984375, + "count": 5000.0, + "geometric_mean_rank": 5711.947265625, + "harmonic_mean_rank": 83.35008958074279, + "hits_at_1": 0.0058, + "hits_at_10": 0.0212, + "hits_at_3": 0.0128, + "hits_at_5": 0.0166, + "inverse_arithmetic_mean_rank": 7.111657032510266e-05, + "inverse_geometric_mean_rank": 0.00017507164739072323, + "inverse_harmonic_mean_rank": 0.011997586867252037, + "inverse_median_rank": 8.702462946530432e-05, + "median_absolute_deviation": 14010.590964877938, + "median_rank": 11491.0, + "standard_deviation": 12007.6318359375, + "variance": 144183216.0, + "z_arithmetic_mean_rank": 38.31441680697182, + "z_hits_at_k": 94.80480328273974, + "z_inverse_harmonic_mean_rank": 130.8836710488173 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6640268322053902, + "adjusted_arithmetic_mean_rank_index": 0.3359895865979101, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615849420157, + "arithmetic_mean_rank": 13588.4508, + "count": 5000.0, + "geometric_mean_rank": 5405.8950996728145, + "harmonic_mean_rank": 92.036109949627, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359190644455216e-05, + "inverse_geometric_mean_rank": 0.00018498324173188709, + "inverse_harmonic_mean_rank": 0.010865300592857712, + "inverse_median_rank": 9.43040362127499e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.773685842129, + "variance": 142009495.07957938, + "z_arithmetic_mean_rank": 41.149098791246644, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600139198488 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6640273404221955, + "adjusted_arithmetic_mean_rank_index": 0.3359890783562687, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615751166754, + "arithmetic_mean_rank": 13588.4612, + "count": 5000.0, + "geometric_mean_rank": 5405.899334093384, + "harmonic_mean_rank": 92.03611078166908, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359185012060085e-05, + "inverse_geometric_mean_rank": 0.00018498309683521116, + "inverse_harmonic_mean_rank": 0.01086530049463119, + "inverse_median_rank": 9.43040362127499e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.778750706693, + "variance": 142009615.79329458, + "z_arithmetic_mean_rank": 41.14903654620588, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600029538334 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6640270889862069, + "adjusted_arithmetic_mean_rank_index": 0.33598932980454477, + "adjusted_hits_at_k": 0.01956043787939957, + "adjusted_inverse_harmonic_mean_rank": 0.010594615800288752, + "arithmetic_mean_rank": 13588.4560546875, + "count": 5000.0, + "geometric_mean_rank": 5405.8955078125, + "harmonic_mean_rank": 92.03610861232633, + "hits_at_1": 0.005, + "hits_at_10": 0.0198, + "hits_at_3": 0.0114, + "hits_at_5": 0.0148, + "inverse_arithmetic_mean_rank": 7.359188020927832e-05, + "inverse_geometric_mean_rank": 0.00018498323333915323, + "inverse_harmonic_mean_rank": 0.010865300543739749, + "inverse_median_rank": 9.430403588339686e-05, + "median_absolute_deviation": 13467.217251795635, + "median_rank": 10604.0, + "standard_deviation": 11916.7763671875, + "variance": 142009552.0, + "z_arithmetic_mean_rank": 41.1490673414161, + "z_hits_at_k": 88.4732232302926, + "z_inverse_harmonic_mean_rank": 118.24600084363162 + } + } + }, + "times": { + "evaluation": 7.501351594924927, + "training": 198.48107266426086 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..2acde2b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "The number of epochs has been inferred from the convergence experiment plot, Figure 2b. The publication favours SGD optimizer over others to ensure comaparabiltiy to RSGD, which is used to optimize the hyperbolic models studied in the same paper. The evaluation uses the non-deterministic variant of rank computation. While it is not guaranteed to yield the same results as average ranking definition due to the unknown properties of the employed sort algorithm (e.g. stability), it is closest to the average rank definition.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with MuRE as described by Bala\u017eevi\u0107 et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..26d64c8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/mure/fb15k237/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.000975197495752765, + 0.0007105042391938026, + 0.0006824577425091475, + 0.0006504202312117928, + 0.0006158363464985977 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1223330365810328, + "adjusted_arithmetic_mean_rank_index": 0.8777899778110253, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.1809697637390021, + "arithmetic_mean_rank": 872.9280506898914, + "count": 40876.0, + "geometric_mean_rank": 68.13149559739709, + "harmonic_mean_rank": 5.508020077803989, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.001145569785745436, + "inverse_geometric_mean_rank": 0.014677499609126505, + "inverse_harmonic_mean_rank": 0.18155344132272905, + "inverse_median_rank": 0.014285714285714285, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1981.9991308986464, + "variance": 3928320.5548829897, + "z_arithmetic_mean_rank": 306.9925623551912, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.770157360449 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.12233337256843348, + "adjusted_arithmetic_mean_rank_index": 0.8777896417765324, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.18096974666989954, + "arithmetic_mean_rank": 872.9304481847539, + "count": 40876.0, + "geometric_mean_rank": 68.13161233034441, + "harmonic_mean_rank": 5.508020595282298, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.0011455666394493231, + "inverse_geometric_mean_rank": 0.014677474461508093, + "inverse_harmonic_mean_rank": 0.18155342426579069, + "inverse_median_rank": 0.014285714285714285, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1982.0041488880931, + "variance": 3928340.446209614, + "z_arithmetic_mean_rank": 306.9924448326714, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.7698359389187 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.12233320609058031, + "adjusted_arithmetic_mean_rank_index": 0.8777898082777192, + "adjusted_hits_at_k": 0.30771378619449874, + "adjusted_inverse_harmonic_mean_rank": 0.1809697551097282, + "arithmetic_mean_rank": 872.9292602539062, + "count": 40876.0, + "geometric_mean_rank": 68.13153839111328, + "harmonic_mean_rank": 5.508020259533576, + "hits_at_1": 0.11840688912809473, + "hits_at_10": 0.30820041099911927, + "hits_at_3": 0.1975976122908308, + "hits_at_5": 0.24456894020941383, + "inverse_arithmetic_mean_rank": 0.001145568210631609, + "inverse_geometric_mean_rank": 0.01467749010771513, + "inverse_harmonic_mean_rank": 0.18155343269960478, + "inverse_median_rank": 0.014285714365541935, + "median_absolute_deviation": 102.29955307688654, + "median_rank": 70.0, + "standard_deviation": 1982.0015869140625, + "variance": 3928330.5, + "z_arithmetic_mean_rank": 306.9925030637145, + "z_hits_at_k": 2345.712494633238, + "z_inverse_harmonic_mean_rank": 3407.769994866 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.17011602215789823, + "adjusted_arithmetic_mean_rank_index": 0.8300019321687586, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.10238267347048406, + "arithmetic_mean_rank": 1197.0449163323221, + "count": 20438.0, + "geometric_mean_rank": 153.8541949078438, + "harmonic_mean_rank": 9.705733847342007, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353905407860078, + "inverse_geometric_mean_rank": 0.00649966028289956, + "inverse_harmonic_mean_rank": 0.1030318794774965, + "inverse_median_rank": 0.0043859649122807015, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.4322111858924, + "variance": 5123125.374633859, + "z_arithmetic_mean_rank": 205.04053161769326, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0445282016553 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.17011651584834828, + "adjusted_arithmetic_mean_rank_index": 0.8300014384081386, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.1023826397223952, + "arithmetic_mean_rank": 1197.0483902534495, + "count": 20438.0, + "geometric_mean_rank": 153.8546405130579, + "harmonic_mean_rank": 9.705737024156553, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353881164221534, + "inverse_geometric_mean_rank": 0.006499641458101671, + "inverse_harmonic_mean_rank": 0.10303184575381609, + "inverse_median_rank": 0.0043859649122807015, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.438376850168, + "variance": 5123153.285798122, + "z_arithmetic_mean_rank": 205.04040964094105, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0440825313349 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.17011626581502265, + "adjusted_arithmetic_mean_rank_index": 0.8300016884770023, + "adjusted_hits_at_k": 0.192592564681411, + "adjusted_inverse_harmonic_mean_rank": 0.1023826564070964, + "arithmetic_mean_rank": 1197.046630859375, + "count": 20438.0, + "geometric_mean_rank": 153.8544464111328, + "harmonic_mean_rank": 9.705735764932905, + "hits_at_1": 0.05974165769644779, + "hits_at_10": 0.1931695860651727, + "hits_at_3": 0.1055876308836481, + "hits_at_5": 0.13925041589196593, + "inverse_arithmetic_mean_rank": 0.0008353893645107746, + "inverse_geometric_mean_rank": 0.006499649491161108, + "inverse_harmonic_mean_rank": 0.10303186242645, + "inverse_median_rank": 0.004385964944958687, + "median_absolute_deviation": 335.06810138226604, + "median_rank": 228.0, + "standard_deviation": 2263.435302734375, + "variance": 5123139.0, + "z_arithmetic_mean_rank": 205.0404714170054, + "z_hits_at_k": 1029.5678854528737, + "z_inverse_harmonic_mean_rank": 1352.0443028660675 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0758581855656241, + "adjusted_arithmetic_mean_rank_index": 0.9242695695064783, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.2595551852130943, + "arithmetic_mean_rank": 548.8111850474606, + "count": 20438.0, + "geometric_mean_rank": 30.1707775671542, + "harmonic_mean_rank": 3.8450446518083097, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.0018221202979190757, + "inverse_geometric_mean_rank": 0.03314465455105349, + "inverse_harmonic_mean_rank": 0.2600750031679616, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.5251807712627, + "variance": 2523412.249944373, + "z_arithmetic_mean_rank": 228.8442805094061, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168288279998 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07585836816734498, + "adjusted_arithmetic_mean_rank_index": 0.9242693868795142, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.25955518482262385, + "arithmetic_mean_rank": 548.8125061160583, + "count": 20438.0, + "geometric_mean_rank": 30.170793570171003, + "harmonic_mean_rank": 3.845044657577116, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.001822115911820217, + "inverse_geometric_mean_rank": 0.03314463697065865, + "inverse_harmonic_mean_rank": 0.2600750027777653, + "inverse_median_rank": 0.047619047619047616, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.5284781455484, + "variance": 2523422.725879412, + "z_arithmetic_mean_rank": 228.84423529193106, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168283036969 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.075858274521055, + "adjusted_arithmetic_mean_rank_index": 0.92426948053875, + "adjusted_hits_at_k": 0.4228323039013454, + "adjusted_inverse_harmonic_mean_rank": 0.25955518501775515, + "arithmetic_mean_rank": 548.8118286132812, + "count": 20438.0, + "geometric_mean_rank": 30.170780181884766, + "harmonic_mean_rank": 3.845044527972988, + "hits_at_1": 0.17707212055974167, + "hits_at_10": 0.42323123593306583, + "hits_at_3": 0.2896075936980135, + "hits_at_5": 0.3498874645268617, + "inverse_arithmetic_mean_rank": 0.0018221181817352772, + "inverse_geometric_mean_rank": 0.03314465284347534, + "inverse_harmonic_mean_rank": 0.2600750029727596, + "inverse_median_rank": 0.0476190485060215, + "median_absolute_deviation": 29.65204437011204, + "median_rank": 21.0, + "standard_deviation": 1588.52685546875, + "variance": 2523417.75, + "z_arithmetic_mean_rank": 228.8442584814649, + "z_hits_at_k": 2298.469657373571, + "z_inverse_harmonic_mean_rank": 3485.168285657088 + } + } + }, + "times": { + "evaluation": 14.712653875350952, + "training": 66.19852590560913 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json new file mode 100644 index 0000000..1e46a74 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json new file mode 100644 index 0000000..734222d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 6.981585166240256e-05, + 4.908339620017992e-05, + 4.2677227727666245e-05, + 3.223726586237798e-05, + 2.246727524548819e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.19171643534750452, + "adjusted_arithmetic_mean_rank_index": 0.808393001386489, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.09786852485330108, + "arithmetic_mean_rank": 1416.1810109867786, + "count": 118142.0, + "geometric_mean_rank": 142.07525085299747, + "harmonic_mean_rank": 10.153199039470467, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061244235319972, + "inverse_geometric_mean_rank": 0.007038523557031624, + "inverse_harmonic_mean_rank": 0.09849112541894524, + "inverse_median_rank": 0.006172839506172839, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.2438173108694, + "variance": 8295804.447157488, + "z_arithmetic_mean_rank": 480.846136305248, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.3268537224485 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.19171644451448372, + "adjusted_arithmetic_mean_rank_index": 0.8083929922182685, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.09786852189072592, + "arithmetic_mean_rank": 1416.181078701901, + "count": 118142.0, + "geometric_mean_rank": 142.07527498310955, + "harmonic_mean_rank": 10.153199344664023, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061243897684464, + "inverse_geometric_mean_rank": 0.007038522361606436, + "inverse_harmonic_mean_rank": 0.09849112245841468, + "inverse_median_rank": 0.006172839506172839, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.2439285569008, + "variance": 8295805.087988889, + "z_arithmetic_mean_rank": 480.84613085183196, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.3267571784318 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.19171643795844864, + "adjusted_arithmetic_mean_rank_index": 0.8083929987751913, + "adjusted_hits_at_k": 0.1871888446010525, + "adjusted_inverse_harmonic_mean_rank": 0.0978685233581836, + "arithmetic_mean_rank": 1416.1810302734375, + "count": 118142.0, + "geometric_mean_rank": 142.07528686523438, + "harmonic_mean_rank": 10.153199094165512, + "hits_at_1": 0.05242843357992924, + "hits_at_10": 0.18774017707504528, + "hits_at_3": 0.10307934519476562, + "hits_at_5": 0.13531174349511604, + "inverse_arithmetic_mean_rank": 0.0007061244104988873, + "inverse_geometric_mean_rank": 0.007038521580398083, + "inverse_harmonic_mean_rank": 0.0984911239248596, + "inverse_median_rank": 0.006172839552164078, + "median_absolute_deviation": 235.73375274239072, + "median_rank": 162.0, + "standard_deviation": 2880.243896484375, + "variance": 8295805.5, + "z_arithmetic_mean_rank": 480.846134752003, + "z_hits_at_k": 2469.5805906557616, + "z_inverse_harmonic_mean_rank": 3189.326804999753 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.1892347211170444, + "adjusted_arithmetic_mean_rank_index": 0.8108753512298704, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794790708412, + "arithmetic_mean_rank": 1394.0446919808367, + "count": 59071.0, + "geometric_mean_rank": 143.2472431101072, + "harmonic_mean_rank": 10.589286510880694, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173371167742637, + "inverse_geometric_mean_rank": 0.00698093714258325, + "inverse_harmonic_mean_rank": 0.09443506878130845, + "inverse_median_rank": 0.006211180124223602, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.4004548954404, + "variance": 8153311.757817088, + "z_arithmetic_mean_rank": 341.0042259751849, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3277329424395 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.18923473260703857, + "adjusted_arithmetic_mean_rank_index": 0.8108753397383164, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794198527219, + "arithmetic_mean_rank": 1394.0447766247398, + "count": 59071.0, + "geometric_mean_rank": 143.24729012298587, + "harmonic_mean_rank": 10.58928717445167, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173370732188383, + "inverse_geometric_mean_rank": 0.006980934851482661, + "inverse_harmonic_mean_rank": 0.09443506286359465, + "inverse_median_rank": 0.006211180124223602, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.4005966875957, + "variance": 8153312.5675638765, + "z_arithmetic_mean_rank": 341.0042211425451, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3275966937404 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1892347523240481, + "adjusted_arithmetic_mean_rank_index": 0.8108753200186299, + "adjusted_hits_at_k": 0.18108864209826847, + "adjusted_inverse_harmonic_mean_rank": 0.09380794491851856, + "arithmetic_mean_rank": 1394.044921875, + "count": 59071.0, + "geometric_mean_rank": 143.24728393554688, + "harmonic_mean_rank": 10.5892863676429, + "hits_at_1": 0.048958033552843186, + "hits_at_10": 0.18164581605186977, + "hits_at_3": 0.09903336662660188, + "hits_at_5": 0.1299283912579777, + "inverse_arithmetic_mean_rank": 0.0007173370104283094, + "inverse_geometric_mean_rank": 0.006980935111641884, + "inverse_harmonic_mean_rank": 0.09443506579481109, + "inverse_median_rank": 0.006211180239915848, + "median_absolute_deviation": 234.25115052388512, + "median_rank": 161.0, + "standard_deviation": 2855.400390625, + "variance": 8153312.0, + "z_arithmetic_mean_rank": 341.00421284965967, + "z_hits_at_k": 1686.7629924773278, + "z_inverse_harmonic_mean_rank": 2158.3276641816997 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.19418467821182647, + "adjusted_arithmetic_mean_rank_index": 0.8059241281789679, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.1019290863931256, + "arithmetic_mean_rank": 1438.3173299927207, + "count": 59071.0, + "geometric_mean_rank": 140.9128473727532, + "harmonic_mean_rank": 9.751608771153105, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952568665810771, + "inverse_geometric_mean_rank": 0.0070965850072898275, + "inverse_harmonic_mean_rank": 0.10254718205658207, + "inverse_median_rank": 0.006134969325153374, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.70602699475, + "variance": 8437317.103259625, + "z_arithmetic_mean_rank": 339.02225596678153, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632272404034 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.19418468506840061, + "adjusted_arithmetic_mean_rank_index": 0.8059241213214678, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.10192908638977595, + "arithmetic_mean_rank": 1438.3173807790624, + "count": 59071.0, + "geometric_mean_rank": 140.91284899139066, + "harmonic_mean_rank": 9.751608771471417, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952568420318689, + "inverse_geometric_mean_rank": 0.007096584925772787, + "inverse_harmonic_mean_rank": 0.10254718205323472, + "inverse_median_rank": 0.006134969325153374, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.7061084857787, + "variance": 8437317.576674597, + "z_arithmetic_mean_rank": 339.02225308208676, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632271630986 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.1941846688624345, + "adjusted_arithmetic_mean_rank_index": 0.8059241375296222, + "adjusted_hits_at_k": 0.19328902170450568, + "adjusted_inverse_harmonic_mean_rank": 0.10192908639145049, + "arithmetic_mean_rank": 1438.3172607421875, + "count": 59071.0, + "geometric_mean_rank": 140.9127655029297, + "harmonic_mean_rank": 9.751608993533074, + "hits_at_1": 0.05589883360701529, + "hits_at_10": 0.1938345380982208, + "hits_at_3": 0.10712532376292935, + "hits_at_5": 0.14069509573225442, + "inverse_arithmetic_mean_rank": 0.0006952569237910211, + "inverse_geometric_mean_rank": 0.0070965890772640705, + "inverse_harmonic_mean_rank": 0.1025471820549081, + "inverse_median_rank": 0.0061349691823124886, + "median_absolute_deviation": 237.21635496089633, + "median_rank": 163.0, + "standard_deviation": 2904.705810546875, + "variance": 8437316.0, + "z_arithmetic_mean_rank": 339.0222599002535, + "z_hits_at_k": 1805.938094512038, + "z_inverse_harmonic_mean_rank": 2352.3632272017444 + } + } + }, + "times": { + "evaluation": 143.83830213546753, + "training": 7.28100323677063 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json new file mode 100644 index 0000000..45d8a38 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json new file mode 100644 index 0000000..1630779 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/fb15k237/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 6.97649384102165e-05, + 4.809209213251929e-05, + 4.411240719934941e-05, + 3.876147011645326e-05, + 3.176101996801703e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.28883692078052037, + "adjusted_arithmetic_mean_rank_index": 0.7112627563151099, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011291035, + "arithmetic_mean_rank": 2061.044647225756, + "count": 40876.0, + "geometric_mean_rank": 312.6961306089733, + "harmonic_mean_rank": 16.044739074726174, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.00048519084792560796, + "inverse_geometric_mean_rank": 0.003197992882267228, + "inverse_harmonic_mean_rank": 0.062325725294916735, + "inverse_median_rank": 0.002197802197802198, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.958078763932, + "variance": 10536243.849092837, + "z_arithmetic_mean_rank": 248.7524141179013, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389978492628 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2888369413511776, + "adjusted_arithmetic_mean_rank_index": 0.7112627357415695, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011019156, + "arithmetic_mean_rank": 2061.044794011156, + "count": 40876.0, + "geometric_mean_rank": 312.696136716117, + "harmonic_mean_rank": 16.044739075425582, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.0004851908133708361, + "inverse_geometric_mean_rank": 0.0031979928198085025, + "inverse_harmonic_mean_rank": 0.06232572529219989, + "inverse_median_rank": 0.002197802197802198, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.958340975742, + "variance": 10536245.55134999, + "z_arithmetic_mean_rank": 248.75240692264498, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389977980665 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2888369250560297, + "adjusted_arithmetic_mean_rank_index": 0.7112627520390014, + "adjusted_hits_at_k": 0.12278097414673175, + "adjusted_inverse_harmonic_mean_rank": 0.06165702011155085, + "arithmetic_mean_rank": 2061.044677734375, + "count": 40876.0, + "geometric_mean_rank": 312.696044921875, + "harmonic_mean_rank": 16.04473833353234, + "hits_at_1": 0.030457970447206186, + "hits_at_10": 0.12339759271944417, + "hits_at_3": 0.06365593502299638, + "hits_at_5": 0.08564928075154124, + "inverse_arithmetic_mean_rank": 0.0004851908306591213, + "inverse_geometric_mean_rank": 0.0031979938503354788, + "inverse_harmonic_mean_rank": 0.06232572529355821, + "inverse_median_rank": 0.002197802299633622, + "median_absolute_deviation": 664.2057938905097, + "median_rank": 455.0, + "standard_deviation": 3245.9580078125, + "variance": 10536244.0, + "z_arithmetic_mean_rank": 248.75241262240291, + "z_hits_at_k": 935.9634767133426, + "z_inverse_harmonic_mean_rank": 1161.0389978236628 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.280102424914663, + "adjusted_arithmetic_mean_rank_index": 0.7199998966579886, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280967176, + "arithmetic_mean_rank": 1970.9794500440357, + "count": 20438.0, + "geometric_mean_rank": 345.32466566691244, + "harmonic_mean_rank": 20.285374139606326, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619615758337, + "inverse_geometric_mean_rank": 0.002895825579295756, + "inverse_harmonic_mean_rank": 0.04929660124175589, + "inverse_median_rank": 0.0020964360587002098, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9989276446954, + "variance": 9815682.28062281, + "z_arithmetic_mean_rank": 177.86604567255606, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138905064652 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2801024457748228, + "adjusted_arithmetic_mean_rank_index": 0.7199998757948638, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280770392, + "arithmetic_mean_rank": 1970.9795968294354, + "count": 20438.0, + "geometric_mean_rank": 345.3246714815715, + "harmonic_mean_rank": 20.285374140415495, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619237909026, + "inverse_geometric_mean_rank": 0.002895825530535155, + "inverse_harmonic_mean_rank": 0.049296601239789475, + "inverse_median_rank": 0.0020964360587002098, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9992511936284, + "variance": 9815684.307979835, + "z_arithmetic_mean_rank": 177.86604051860877, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138904804784 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2801024309038104, + "adjusted_arithmetic_mean_rank_index": 0.7199998906679899, + "adjusted_hits_at_k": 0.09760345464392997, + "adjusted_inverse_harmonic_mean_rank": 0.04860850280868778, + "arithmetic_mean_rank": 1970.9794921875, + "count": 20438.0, + "geometric_mean_rank": 345.3247375488281, + "harmonic_mean_rank": 20.2853743489536, + "hits_at_1": 0.022800665427145515, + "hits_at_10": 0.09824836089636951, + "hits_at_3": 0.047558469517565316, + "hits_at_5": 0.06610235835208925, + "inverse_arithmetic_mean_rank": 0.0005073619540780783, + "inverse_geometric_mean_rank": 0.0028958250768482685, + "inverse_harmonic_mean_rank": 0.04929660124077262, + "inverse_median_rank": 0.0020964359864592552, + "median_absolute_deviation": 687.9274293865993, + "median_rank": 477.0, + "standard_deviation": 3132.9990234375, + "variance": 9815683.0, + "z_arithmetic_mean_rank": 177.86604419280948, + "z_hits_at_k": 521.7718689030245, + "z_inverse_harmonic_mean_rank": 641.9138904934711 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.297332296051859, + "adjusted_arithmetic_mean_rank_index": 0.702764842029959, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526033128848, + "arithmetic_mean_rank": 2151.1098444074764, + "count": 20438.0, + "geometric_mean_rank": 283.1505531439164, + "harmonic_mean_rank": 13.270546071704294, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.000464876306804988, + "inverse_geometric_mean_rank": 0.0035316900811128977, + "inverse_harmonic_mean_rank": 0.0753548493480776, + "inverse_median_rank": 0.002331002331002331, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.6977105125507, + "variance": 11240581.9380761, + "z_arithmetic_mean_rank": 174.00087587815491, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038917634 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.29733231634093904, + "adjusted_arithmetic_mean_rank_index": 0.7027648217380741, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526032781874, + "arithmetic_mean_rank": 2151.109991192876, + "count": 20438.0, + "geometric_mean_rank": 283.15055943636077, + "harmonic_mean_rank": 13.270546072314913, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.00046487627508320033, + "inverse_geometric_mean_rank": 0.0035316900026282805, + "inverse_harmonic_mean_rank": 0.07535484934461029, + "inverse_median_rank": 0.002331002331002331, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.697915893017, + "variance": 11240583.31523338, + "z_arithmetic_mean_rank": 174.00087085399105, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038451735 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2973322986606437, + "adjusted_arithmetic_mean_rank_index": 0.7027648394208137, + "adjusted_hits_at_k": 0.14795790231516903, + "adjusted_inverse_harmonic_mean_rank": 0.07470526032955348, + "arithmetic_mean_rank": 2151.10986328125, + "count": 20438.0, + "geometric_mean_rank": 283.1506042480469, + "harmonic_mean_rank": 13.270544968026048, + "hits_at_1": 0.038115275467266854, + "hits_at_10": 0.14854682454251883, + "hits_at_3": 0.07975340052842744, + "hits_at_5": 0.10519620315099325, + "inverse_arithmetic_mean_rank": 0.00046487629879266024, + "inverse_geometric_mean_rank": 0.0035316895227879286, + "inverse_harmonic_mean_rank": 0.0753548493463438, + "inverse_median_rank": 0.0023310023825615644, + "median_absolute_deviation": 630.1059428648808, + "median_rank": 429.0, + "standard_deviation": 3352.697998046875, + "variance": 11240584.0, + "z_arithmetic_mean_rank": 174.00087523214427, + "z_hits_at_k": 804.282799356326, + "z_inverse_harmonic_mean_rank": 1003.1023038684666 + } + } + }, + "times": { + "evaluation": 33.202207803726196, + "training": 4.496615648269653 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json new file mode 100644 index 0000000..2af423b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json new file mode 100644 index 0000000..71c3e23 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.946888238228267e-05, + 5.3340319112503506e-05, + 5.038819471869176e-05, + 5.192173830135024e-05, + 5.254753102436471e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.48480670674081655, + "adjusted_arithmetic_mean_rank_index": 0.5152184710241546, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.1720038860548289, + "arithmetic_mean_rank": 9920.7126, + "count": 10000.0, + "geometric_mean_rank": 314.4064429963176, + "harmonic_mean_rank": 5.806175348825523, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079921073411601, + "inverse_geometric_mean_rank": 0.0031805963976753245, + "inverse_harmonic_mean_rank": 0.17223041674107908, + "inverse_median_rank": 0.0024691358024691358, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.096689350818, + "variance": 236516614.98040128, + "z_arithmetic_mean_rank": 89.23616465442358, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.8725122286223 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4848067165144432, + "adjusted_arithmetic_mean_rank_index": 0.5152184612500503, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.17200388599540753, + "arithmetic_mean_rank": 9920.7128, + "count": 10000.0, + "geometric_mean_rank": 314.40646818713793, + "harmonic_mean_rank": 5.806175350828168, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079920870201988, + "inverse_geometric_mean_rank": 0.0031805961428401333, + "inverse_harmonic_mean_rank": 0.17223041668167396, + "inverse_median_rank": 0.0024691358024691358, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.0967778643, + "variance": 236516617.70291612, + "z_arithmetic_mean_rank": 89.23616296154255, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.872511290728 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48480672094311783, + "adjusted_arithmetic_mean_rank_index": 0.5152184568211593, + "adjusted_hits_at_k": 0.258518821109638, + "adjusted_inverse_harmonic_mean_rank": 0.17200388602510677, + "arithmetic_mean_rank": 9920.712890625, + "count": 10000.0, + "geometric_mean_rank": 314.4066162109375, + "harmonic_mean_rank": 5.806175241595575, + "hits_at_1": 0.1241, + "hits_at_10": 0.2587, + "hits_at_3": 0.1935, + "hits_at_5": 0.2207, + "inverse_arithmetic_mean_rank": 0.00010079921048600227, + "inverse_geometric_mean_rank": 0.0031805946491658688, + "inverse_harmonic_mean_rank": 0.1722304167113651, + "inverse_median_rank": 0.0024691359139978886, + "median_absolute_deviation": 598.9712962762632, + "median_rank": 405.0, + "standard_deviation": 15379.0966796875, + "variance": 236516624.0, + "z_arithmetic_mean_rank": 89.23616219445582, + "z_hits_at_k": 1653.6186785870127, + "z_inverse_harmonic_mean_rank": 2714.872511759495 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4848100783439825, + "adjusted_arithmetic_mean_rank_index": 0.5152150998399522, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.17161366984021761, + "arithmetic_mean_rank": 9920.5516, + "count": 5000.0, + "geometric_mean_rank": 315.7243009873847, + "harmonic_mean_rank": 5.819356283506532, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084659808634, + "inverse_geometric_mean_rank": 0.0031673203388926235, + "inverse_harmonic_mean_rank": 0.17184031210363296, + "inverse_median_rank": 0.0024509803921568627, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.899898307129, + "variance": 236633609.2813375, + "z_arithmetic_mean_rank": 63.09907895732225, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.327289103086 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.48481008811783566, + "adjusted_arithmetic_mean_rank_index": 0.5152150900656214, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.1716136697215684, + "arithmetic_mean_rank": 9920.5518, + "count": 5000.0, + "geometric_mean_rank": 315.7243496164708, + "harmonic_mean_rank": 5.819356287523477, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084456592426, + "inverse_geometric_mean_rank": 0.0031673198510496882, + "inverse_harmonic_mean_rank": 0.1718403119850162, + "inverse_median_rank": 0.0024509803921568627, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.899786207956, + "variance": 236633605.83251676, + "z_arithmetic_mean_rank": 63.099077760246935, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.3272877788788 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48481008605616355, + "adjusted_arithmetic_mean_rank_index": 0.5152150921273941, + "adjusted_hits_at_k": 0.25881889020131527, + "adjusted_inverse_harmonic_mean_rank": 0.17161366978087017, + "arithmetic_mean_rank": 9920.5517578125, + "count": 5000.0, + "geometric_mean_rank": 315.7243347167969, + "harmonic_mean_rank": 5.819356435439596, + "hits_at_1": 0.124, + "hits_at_10": 0.259, + "hits_at_3": 0.194, + "hits_at_5": 0.2198, + "inverse_arithmetic_mean_rank": 0.00010080084757646546, + "inverse_geometric_mean_rank": 0.003167320042848587, + "inverse_harmonic_mean_rank": 0.17184031204430175, + "inverse_median_rank": 0.0024509804788976908, + "median_absolute_deviation": 603.41910293178, + "median_rank": 408.0, + "standard_deviation": 15382.900390625, + "variance": 236633616.0, + "z_arithmetic_mean_rank": 63.09907801275501, + "z_hits_at_k": 1170.6285265248341, + "z_inverse_harmonic_mean_rank": 1915.3272884407274 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4848033352939752, + "adjusted_arithmetic_mean_rank_index": 0.5152218420520442, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226489954, + "arithmetic_mean_rank": 9920.8736, + "count": 5000.0, + "geometric_mean_rank": 313.0940858478499, + "harmonic_mean_rank": 5.793053989259962, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757492324062, + "inverse_geometric_mean_rank": 0.003193928104045876, + "inverse_harmonic_mean_rank": 0.1726205213785252, + "inverse_median_rank": 0.002509410288582183, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.29253795267, + "variance": 236399620.62762305, + "z_arithmetic_mean_rank": 63.099915365470764, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.082340135151 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4848033450673752, + "adjusted_arithmetic_mean_rank_index": 0.5152218322781665, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226470606, + "arithmetic_mean_rank": 9920.8738, + "count": 5000.0, + "geometric_mean_rank": 313.0940877952246, + "harmonic_mean_rank": 5.793053989266453, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757289121045, + "inverse_geometric_mean_rank": 0.0031939280841803627, + "inverse_harmonic_mean_rank": 0.17262052137833178, + "inverse_median_rank": 0.002509410288582183, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.292827178075, + "variance": 236399629.52147353, + "z_arithmetic_mean_rank": 63.09991416845076, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.0823401329917 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.48480335598609564, + "adjusted_arithmetic_mean_rank_index": 0.5152218213589126, + "adjusted_hits_at_k": 0.2582187520213862, + "adjusted_inverse_harmonic_mean_rank": 0.17239410226480278, + "arithmetic_mean_rank": 9920.8740234375, + "count": 5000.0, + "geometric_mean_rank": 313.0939636230469, + "harmonic_mean_rank": 5.793054034867086, + "hits_at_1": 0.1242, + "hits_at_10": 0.2584, + "hits_at_3": 0.193, + "hits_at_5": 0.2216, + "inverse_arithmetic_mean_rank": 0.00010079757339553908, + "inverse_geometric_mean_rank": 0.003193929325789213, + "inverse_harmonic_mean_rank": 0.17262052137842848, + "inverse_median_rank": 0.002509410260245204, + "median_absolute_deviation": 589.3343818559767, + "median_rank": 398.5, + "standard_deviation": 15375.29296875, + "variance": 236399632.0, + "z_arithmetic_mean_rank": 63.09991283115496, + "z_hits_at_k": 1167.941404517113, + "z_inverse_harmonic_mean_rank": 1924.0823401340713 + } + } + }, + "times": { + "evaluation": 50.53933072090149, + "training": 13.76006269454956 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json new file mode 100644 index 0000000..669b607 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Data Set with QuatE^1 as described by Zhang et al., 2019." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json new file mode 100644 index 0000000..d822612 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/quate/wn18rr/zhang2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 8.776860787075418e-05, + 5.3628090098298616e-05, + 4.521653484294621e-05, + 4.545193286306426e-05, + 4.5890227263061786e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.135452151724743, + "adjusted_arithmetic_mean_rank_index": -0.135458833609716, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.003784262744786251, + "arithmetic_mean_rank": 23018.508207934337, + "count": 5848.0, + "geometric_mean_rank": 13912.042658983803, + "harmonic_mean_rank": 246.35699128213304, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.3443301840703396e-05, + "inverse_geometric_mean_rank": 7.188017061996592e-05, + "inverse_harmonic_mean_rank": 0.004059150076462736, + "inverse_median_rank": 4.062150908906266e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.646932078893, + "variance": 194062923.94623905, + "z_arithmetic_mean_rank": -17.94157082876319, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.46376082000466 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.1354521938996665, + "adjusted_arithmetic_mean_rank_index": -0.1354588757867199, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.0037842626820186074, + "arithmetic_mean_rank": 23018.509062927496, + "count": 5848.0, + "geometric_mean_rank": 13912.04449785986, + "harmonic_mean_rank": 246.35699509056116, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.344330022705736e-05, + "inverse_geometric_mean_rank": 7.188016111893788e-05, + "inverse_harmonic_mean_rank": 0.004059150013712412, + "inverse_median_rank": 4.062150908906266e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.646769802916, + "variance": 194062919.42502043, + "z_arithmetic_mean_rank": -17.941576415122427, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.46376006592036 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.1354522285621818, + "adjusted_arithmetic_mean_rank_index": -0.13545891045094516, + "adjusted_hits_at_k": 0.007279089594412466, + "adjusted_inverse_harmonic_mean_rank": 0.003784262713393222, + "arithmetic_mean_rank": 23018.509765625, + "count": 5848.0, + "geometric_mean_rank": 13912.0439453125, + "harmonic_mean_rank": 246.35699470836892, + "hits_at_1": 0.002051983584131327, + "hits_at_10": 0.007523939808481532, + "hits_at_3": 0.003761969904240766, + "hits_at_5": 0.005129958960328317, + "inverse_arithmetic_mean_rank": 4.344329863670282e-05, + "inverse_geometric_mean_rank": 7.188016752479598e-05, + "inverse_harmonic_mean_rank": 0.00405915004507837, + "inverse_median_rank": 4.062150765093975e-05, + "median_absolute_deviation": 19471.756236743324, + "median_rank": 24617.5, + "standard_deviation": 13930.6474609375, + "variance": 194062928.0, + "z_arithmetic_mean_rank": -17.94158100641141, + "z_hits_at_k": 35.43973965846167, + "z_inverse_harmonic_mean_rank": 45.4637604428519 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.1362039608020613, + "adjusted_arithmetic_mean_rank_index": -0.13621068126497105, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.004292715311898555, + "arithmetic_mean_rank": 23028.639192886458, + "count": 2924.0, + "geometric_mean_rank": 14080.544453702405, + "harmonic_mean_rank": 218.9372805893627, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.3424189837013896e-05, + "inverse_geometric_mean_rank": 7.101998103042495e-05, + "inverse_harmonic_mean_rank": 0.004567518137194704, + "inverse_median_rank": 4.088808929958703e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.226635277537, + "variance": 191690300.58219853, + "z_arithmetic_mean_rank": -12.757014496609344, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.463032644858544 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.136203994549487, + "adjusted_arithmetic_mean_rank_index": -0.1362107150140619, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.004292715310077607, + "arithmetic_mean_rank": 23028.639876880985, + "count": 2924.0, + "geometric_mean_rank": 14080.544938681376, + "harmonic_mean_rank": 218.9372806766231, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.342418854723263e-05, + "inverse_geometric_mean_rank": 7.101997858426982e-05, + "inverse_harmonic_mean_rank": 0.004567518135374258, + "inverse_median_rank": 4.088808929958703e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.226529869527, + "variance": 191690297.663403, + "z_arithmetic_mean_rank": -12.75701765743078, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.46303262939111 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.136204031460734, + "adjusted_arithmetic_mean_rank_index": -0.13621075192713006, + "adjusted_hits_at_k": 0.006936953309436689, + "adjusted_inverse_harmonic_mean_rank": 0.0042927153109880555, + "arithmetic_mean_rank": 23028.640625, + "count": 2924.0, + "geometric_mean_rank": 14080.5458984375, + "harmonic_mean_rank": 218.93728156515598, + "hits_at_1": 0.0027359781121751026, + "hits_at_10": 0.007181942544459644, + "hits_at_3": 0.004103967168262654, + "hits_at_5": 0.005813953488372093, + "inverse_arithmetic_mean_rank": 4.342418833402917e-05, + "inverse_geometric_mean_rank": 7.101997471181676e-05, + "inverse_harmonic_mean_rank": 0.0045675181362844555, + "inverse_median_rank": 4.088808782398701e-05, + "median_absolute_deviation": 19322.013412674256, + "median_rank": 24457.0, + "standard_deviation": 13845.2265625, + "variance": 191690288.0, + "z_arithmetic_mean_rank": -12.757021114579231, + "z_hits_at_k": 23.87914587961846, + "z_inverse_harmonic_mean_rank": 36.463032637124606 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.1347006761552578, + "adjusted_arithmetic_mean_rank_index": -0.13470731949585657, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758102346691497, + "arithmetic_mean_rank": 23008.377222982217, + "count": 2924.0, + "geometric_mean_rank": 13745.557324276184, + "harmonic_mean_rank": 281.6281020828012, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.346243067508199e-05, + "inverse_geometric_mean_rank": 7.275077877227202e-05, + "inverse_harmonic_mean_rank": 0.00355078201573077, + "inverse_median_rank": 4.030389134070895e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.539305947788, + "variance": 196435342.03656742, + "z_arithmetic_mean_rank": -12.61623007115594, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.831473742158146 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.1347007267539404, + "adjusted_arithmetic_mean_rank_index": -0.1347073700970347, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758101109548166, + "arithmetic_mean_rank": 23008.378248974008, + "count": 2924.0, + "geometric_mean_rank": 13745.560484575846, + "harmonic_mean_rank": 281.6281118924212, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.346242873700115e-05, + "inverse_geometric_mean_rank": 7.275076204583429e-05, + "inverse_harmonic_mean_rank": 0.003550781892050567, + "inverse_median_rank": 4.030389134070895e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.539087735984, + "variance": 196435335.9198552, + "z_arithmetic_mean_rank": -12.61623481029045, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.831472691073998 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.13470066284675, + "adjusted_arithmetic_mean_rank_index": -0.13470730618669258, + "adjusted_hits_at_k": 0.007621225841732883, + "adjusted_inverse_harmonic_mean_rank": 0.0032758101727935946, + "arithmetic_mean_rank": 23008.376953125, + "count": 2924.0, + "geometric_mean_rank": 13745.55859375, + "harmonic_mean_rank": 281.62810942326075, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.00786593707250342, + "hits_at_3": 0.003419972640218878, + "hits_at_5": 0.004445964432284542, + "inverse_arithmetic_mean_rank": 4.3462430767249316e-05, + "inverse_geometric_mean_rank": 7.27507722331211e-05, + "inverse_harmonic_mean_rank": 0.003550781953872285, + "inverse_median_rank": 4.03038902732078e-05, + "median_absolute_deviation": 19577.762295366472, + "median_rank": 24811.5, + "standard_deviation": 14015.5390625, + "variance": 196435328.0, + "z_arithmetic_mean_rank": -12.61622882466483, + "z_hits_at_k": 26.240478195292937, + "z_inverse_harmonic_mean_rank": 27.83147321645984 + } + } + }, + "times": { + "evaluation": 13.765980958938599, + "training": 4.9371349811553955 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json new file mode 100644 index 0000000..5dda922 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json new file mode 100644 index 0000000..df07c04 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002959653189315609, + 0.002774793042292991, + 0.0025934358465079417, + 0.002415573638258217, + 0.0022410840366893685 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.46136554119551004, + "adjusted_arithmetic_mean_rank_index": 0.5387073866711023, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.06391836136475008, + "arithmetic_mean_rank": 3408.039156269574, + "count": 118142.0, + "geometric_mean_rank": 767.5402133699725, + "harmonic_mean_rank": 15.488413386485405, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.0002934238587489106, + "inverse_geometric_mean_rank": 0.0013028633322147206, + "inverse_harmonic_mean_rank": 0.06456439242979928, + "inverse_median_rank": 0.0005056890012642225, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6833542856266, + "variance": 13910537.923235284, + "z_arithmetic_mean_rank": 320.43246915252945, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.963308705239 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4613685399435877, + "adjusted_arithmetic_mean_rank_index": 0.5387043875170122, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.0639183208909707, + "arithmetic_mean_rank": 3408.061307579015, + "count": 118142.0, + "geometric_mean_rank": 767.5465563380474, + "harmonic_mean_rank": 15.488423089084028, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.00029342195158759337, + "inverse_geometric_mean_rank": 0.0013028525654143826, + "inverse_harmonic_mean_rank": 0.06456435198395262, + "inverse_median_rank": 0.0005056890012642225, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6976699738793, + "variance": 13910644.709408585, + "z_arithmetic_mean_rank": 320.43068520381394, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.961989750941 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4613670488334821, + "adjusted_arithmetic_mean_rank_index": 0.5387058788290051, + "adjusted_hits_at_k": 0.10635834027671416, + "adjusted_inverse_harmonic_mean_rank": 0.0639183407927078, + "arithmetic_mean_rank": 3408.05029296875, + "count": 118142.0, + "geometric_mean_rank": 767.54345703125, + "harmonic_mean_rank": 15.48841803698083, + "hits_at_1": 0.042829814968427825, + "hits_at_10": 0.10696450034704, + "hits_at_3": 0.06818066394677591, + "hits_at_5": 0.08373821333649337, + "inverse_arithmetic_mean_rank": 0.00029342289781197906, + "inverse_geometric_mean_rank": 0.0013028578832745552, + "inverse_harmonic_mean_rank": 0.06456437187195466, + "inverse_median_rank": 0.0005056890076957643, + "median_absolute_deviation": 2884.4026161026486, + "median_rank": 1977.5, + "standard_deviation": 3729.6904296875, + "variance": 13910590.0, + "z_arithmetic_mean_rank": 320.43157226197576, + "z_hits_at_k": 1403.1845399843735, + "z_inverse_harmonic_mean_rank": 2082.9626383061805 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4984113221667289, + "adjusted_arithmetic_mean_rank_index": 0.5016567752769987, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.041081447926066904, + "arithmetic_mean_rank": 3671.671107650116, + "count": 59071.0, + "geometric_mean_rank": 1054.047269125659, + "harmonic_mean_rank": 23.954931595237145, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.00027235554892605945, + "inverse_geometric_mean_rank": 0.0009487240556388978, + "inverse_harmonic_mean_rank": 0.041745057631424234, + "inverse_median_rank": 0.00042283298097251583, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.175230589348, + "variance": 14176544.517043548, + "z_arithmetic_mean_rank": 210.96593958501688, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.1995310257179 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.49841446353113766, + "adjusted_arithmetic_mean_rank_index": 0.5016536334861073, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.04108138464536481, + "arithmetic_mean_rank": 3671.6942492932235, + "count": 59071.0, + "geometric_mean_rank": 1054.0570794994069, + "harmonic_mean_rank": 23.954967883080986, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.0002723538323466049, + "inverse_geometric_mean_rank": 0.0009487152256260357, + "inverse_harmonic_mean_rank": 0.041744994394514894, + "inverse_median_rank": 0.00042283298097251583, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.1876813762074, + "variance": 14176638.27598714, + "z_arithmetic_mean_rank": 210.96461834129, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.198075067056 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.49841288452990823, + "adjusted_arithmetic_mean_rank_index": 0.5016552127017074, + "adjusted_hits_at_k": 0.07201000836025624, + "adjusted_inverse_harmonic_mean_rank": 0.04108141567445039, + "arithmetic_mean_rank": 3671.6826171875, + "count": 59071.0, + "geometric_mean_rank": 1054.052734375, + "harmonic_mean_rank": 23.954949947576385, + "hits_at_1": 0.025850248006636083, + "hits_at_10": 0.07264139764012797, + "hits_at_3": 0.04159401398317279, + "hits_at_5": 0.05302094090162686, + "inverse_arithmetic_mean_rank": 0.00027235469315201044, + "inverse_geometric_mean_rank": 0.0009487191564403474, + "inverse_harmonic_mean_rank": 0.041745025402127114, + "inverse_median_rank": 0.0004228329926263541, + "median_absolute_deviation": 3340.302798293121, + "median_rank": 2365.0, + "standard_deviation": 3765.18115234375, + "variance": 14176590.0, + "z_arithmetic_mean_rank": 210.96528246209795, + "z_hits_at_k": 670.7423269768094, + "z_inverse_harmonic_mean_rank": 945.1987889824271 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.4245208539977861, + "adjusted_arithmetic_mean_rank_index": 0.5755568509144449, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.0867551881570143, + "arithmetic_mean_rank": 3144.4072048890316, + "count": 59071.0, + "geometric_mean_rank": 558.9103984195146, + "harmonic_mean_rank": 11.443778283670872, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.0003180249677729926, + "inverse_geometric_mean_rank": 0.001789195554113499, + "inverse_harmonic_mean_rank": 0.0873837272281743, + "inverse_median_rank": 0.0006397952655150352, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3674.9867643094444, + "variance": 13505527.7178496, + "z_arithmetic_mean_rank": 242.11532477015106, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1734876125245 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4245237109036899, + "adjusted_arithmetic_mean_rank_index": 0.5755539936227831, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.08675517049007114, + "arithmetic_mean_rank": 3144.428365864807, + "count": 59071.0, + "geometric_mean_rank": 558.914434146379, + "harmonic_mean_rank": 11.443780595742925, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.0003180228275688423, + "inverse_geometric_mean_rank": 0.0017891826349542821, + "inverse_harmonic_mean_rank": 0.08738370957339037, + "inverse_median_rank": 0.0006397952655150352, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3675.002923388804, + "variance": 13505646.486916255, + "z_arithmetic_mean_rank": 242.11412281399757, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1730798872231 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.4245222742466193, + "adjusted_arithmetic_mean_rank_index": 0.5755554304738404, + "adjusted_hits_at_k": 0.14070652917748577, + "adjusted_inverse_harmonic_mean_rank": 0.08675517926450192, + "arithmetic_mean_rank": 3144.417724609375, + "count": 59071.0, + "geometric_mean_rank": 558.91259765625, + "harmonic_mean_rank": 11.443779172897282, + "hits_at_1": 0.05980938193021956, + "hits_at_10": 0.14128760305395202, + "hits_at_3": 0.09476731391037904, + "hits_at_5": 0.11445548577135989, + "inverse_arithmetic_mean_rank": 0.00031802390003576875, + "inverse_geometric_mean_rank": 0.001789188478142023, + "inverse_harmonic_mean_rank": 0.08738371834178216, + "inverse_median_rank": 0.0006397952674888074, + "median_absolute_deviation": 2306.9290519947167, + "median_rank": 1563.0, + "standard_deviation": 3674.994873046875, + "variance": 13505587.0, + "z_arithmetic_mean_rank": 242.1147272437074, + "z_hits_at_k": 1314.6493212463074, + "z_inverse_harmonic_mean_rank": 2002.1732823873047 + } + } + }, + "times": { + "evaluation": 88.11779618263245, + "training": 402.42151856422424 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json new file mode 100644 index 0000000..f25d2fd --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with RotatE as described by Sun et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json new file mode 100644 index 0000000..4c66513 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/fb15k237/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.013367026912761475, + 0.013179833175412171, + 0.012994235904604514, + 0.012810520019806017, + 0.012628654588765545 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5276142010546685, + "adjusted_arithmetic_mean_rank_index": 0.4724520088567171, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.07283633819545277, + "arithmetic_mean_rank": 3764.880272042274, + "count": 40876.0, + "geometric_mean_rank": 793.899945390293, + "harmonic_mean_rank": 13.605983362668397, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.00026561269621929996, + "inverse_geometric_mean_rank": 0.0012596045708359194, + "inverse_harmonic_mean_rank": 0.07349707649531335, + "inverse_median_rank": 0.0004452359750667854, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.336655248859, + "variance": 16058747.068501113, + "z_arithmetic_mean_rank": 165.23229525867953, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5523220323657 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5276190317306638, + "adjusted_arithmetic_mean_rank_index": 0.47244717750365106, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.072836276435286, + "arithmetic_mean_rank": 3764.914742146981, + "count": 40876.0, + "geometric_mean_rank": 793.912000370915, + "harmonic_mean_rank": 13.605994787744137, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.00026561026437207973, + "inverse_geometric_mean_rank": 0.0012595854446497859, + "inverse_harmonic_mean_rank": 0.07349701477915965, + "inverse_median_rank": 0.0004452359750667854, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.35491441866, + "variance": 16058893.410115385, + "z_arithmetic_mean_rank": 165.23060557265566, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5511590510541 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5276166099241587, + "adjusted_arithmetic_mean_rank_index": 0.4724495996495982, + "adjusted_hits_at_k": 0.11342906334421032, + "adjusted_inverse_harmonic_mean_rank": 0.07283630710675247, + "arithmetic_mean_rank": 3764.8974609375, + "count": 40876.0, + "geometric_mean_rank": 793.9061279296875, + "harmonic_mean_rank": 13.605989531484779, + "hits_at_1": 0.05210881690967805, + "hits_at_10": 0.11405225560230943, + "hits_at_3": 0.07735590566591644, + "hits_at_5": 0.0932087288384382, + "inverse_arithmetic_mean_rank": 0.0002656114811543375, + "inverse_geometric_mean_rank": 0.001259594806469977, + "inverse_harmonic_mean_rank": 0.07349704542876827, + "inverse_median_rank": 0.0004452359862625599, + "median_absolute_deviation": 3295.8247317379532, + "median_rank": 2246.0, + "standard_deviation": 4007.345703125, + "variance": 16058820.0, + "z_arithmetic_mean_rank": 165.23145267823844, + "z_hits_at_k": 864.6735475572135, + "z_inverse_harmonic_mean_rank": 1371.5517366133274 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6411589651715538, + "adjusted_arithmetic_mean_rank_index": 0.35889203816598525, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873180036128378, + "arithmetic_mean_rank": 4511.603728349153, + "count": 20438.0, + "geometric_mean_rank": 1934.0093597003513, + "harmonic_mean_rank": 86.29191849008801, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022165067240200888, + "inverse_geometric_mean_rank": 0.0005170605793525924, + "inverse_harmonic_mean_rank": 0.011588570720152269, + "inverse_median_rank": 0.0002860411899313501, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.2526904656297, + "variance": 15778791.436911434, + "z_arithmetic_mean_rank": 88.65932890858505, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58897920591045 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6411647017155157, + "adjusted_arithmetic_mean_rank_index": 0.35888630080666817, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873108507002505, + "arithmetic_mean_rank": 4511.644094334084, + "count": 20438.0, + "geometric_mean_rank": 1934.0403711432396, + "harmonic_mean_rank": 86.29245073513225, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022164868927844792, + "inverse_geometric_mean_rank": 0.0005170522885253349, + "inverse_harmonic_mean_rank": 0.011588499242760177, + "inverse_median_rank": 0.0002860411899313501, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.267678960659, + "variance": 15778910.5133155, + "z_arithmetic_mean_rank": 88.65791157308394, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58803460698596 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6411618493738521, + "adjusted_arithmetic_mean_rank_index": 0.35888915355374507, + "adjusted_hits_at_k": 0.02347277804766899, + "adjusted_inverse_harmonic_mean_rank": 0.010873143966700935, + "arithmetic_mean_rank": 4511.6240234375, + "count": 20438.0, + "geometric_mean_rank": 1934.02490234375, + "harmonic_mean_rank": 86.29218338315411, + "hits_at_1": 0.004061062726294158, + "hits_at_10": 0.024170662491437518, + "hits_at_3": 0.00949212251688032, + "hits_at_5": 0.015510323906448771, + "inverse_arithmetic_mean_rank": 0.00022164967958815396, + "inverse_geometric_mean_rank": 0.0005170564400032163, + "inverse_harmonic_mean_rank": 0.01158853467681221, + "inverse_median_rank": 0.00028604117687791586, + "median_absolute_deviation": 4356.626619078711, + "median_rank": 3496.0, + "standard_deviation": 3972.260009765625, + "variance": 15778850.0, + "z_arithmetic_mean_rank": 88.65861630490991, + "z_hits_at_k": 125.4815755749462, + "z_inverse_harmonic_mean_rank": 143.58850288047023 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.417177903862162, + "adjusted_arithmetic_mean_rank_index": 0.5829026665413757, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.1347981805690344, + "arithmetic_mean_rank": 3018.156815735395, + "count": 20438.0, + "geometric_mean_rank": 325.8914545213797, + "harmonic_mean_rank": 7.385219894424195, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.0003313280459074964, + "inverse_geometric_mean_rank": 0.0030685063573349888, + "inverse_harmonic_mean_rank": 0.13540558227047444, + "inverse_median_rank": 0.0010378827192527244, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.731777005309, + "variance": 15223510.859693006, + "z_arithmetic_mean_rank": 144.3236321227174, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9979156699924 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.41718185346975445, + "adjusted_arithmetic_mean_rank_index": 0.5828987163877821, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.13479812857761928, + "arithmetic_mean_rank": 3018.1853899598786, + "count": 20438.0, + "geometric_mean_rank": 325.89612592231913, + "harmonic_mean_rank": 7.385222728122889, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.000331324909108149, + "inverse_geometric_mean_rank": 0.003068462373309589, + "inverse_harmonic_mean_rank": 0.1354055303155591, + "inverse_median_rank": 0.0010378827192527244, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.7517676140965, + "variance": 15223666.856079726, + "z_arithmetic_mean_rank": 144.32265408547934, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9972175570608 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.41717988415761514, + "adjusted_arithmetic_mean_rank_index": 0.5829006859721628, + "adjusted_hits_at_k": 0.20338323587333643, + "adjusted_inverse_harmonic_mean_rank": 0.1347981544609555, + "arithmetic_mean_rank": 3018.171142578125, + "count": 20438.0, + "geometric_mean_rank": 325.89361572265625, + "harmonic_mean_rank": 7.385221466974059, + "hits_at_1": 0.10015657109306195, + "hits_at_10": 0.20393384871318132, + "hits_at_3": 0.14521968881495254, + "hits_at_5": 0.17090713377042763, + "inverse_arithmetic_mean_rank": 0.0003313264751341194, + "inverse_geometric_mean_rank": 0.0030684859957545996, + "inverse_harmonic_mean_rank": 0.13540555618072433, + "inverse_median_rank": 0.0010378827573731542, + "median_absolute_deviation": 1427.0046353116418, + "median_rank": 963.5, + "standard_deviation": 3901.741943359375, + "variance": 15223590.0, + "z_arithmetic_mean_rank": 144.32314174420506, + "z_hits_at_k": 1105.5687849771882, + "z_inverse_harmonic_mean_rank": 1809.9975651046645 + } + } + }, + "times": { + "evaluation": 28.418932676315308, + "training": 440.5546112060547 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json new file mode 100644 index 0000000..6ee6351 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with RotatE as described by Sun et al., 2019;" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json new file mode 100644 index 0000000..e447740 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.013387655595250347, + 0.013083627746816562, + 0.012784079545507152, + 0.0124903927645521, + 0.012203190229969347 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7891292153102102, + "adjusted_arithmetic_mean_rank_index": 0.21088109005502975, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.01896367615359624, + "arithmetic_mean_rank": 16148.135, + "count": 10000.0, + "geometric_mean_rank": 6089.0981506932985, + "harmonic_mean_rank": 51.99646411776393, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192665592652031e-05, + "inverse_geometric_mean_rank": 0.0001642279324872011, + "inverse_harmonic_mean_rank": 0.019232076968448378, + "inverse_median_rank": 6.963788300835655e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.230432867844, + "variance": 165900336.00377497, + "z_arithmetic_mean_rank": 36.524738015017164, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.3186043703253 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7891330123641752, + "adjusted_arithmetic_mean_rank_index": 0.21087729281550072, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.01896367431133423, + "arithmetic_mean_rank": 16148.2127, + "count": 10000.0, + "geometric_mean_rank": 6089.140137166568, + "harmonic_mean_rank": 51.99646909720074, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192635795539155e-05, + "inverse_geometric_mean_rank": 0.00016422680008565635, + "inverse_harmonic_mean_rank": 0.019232075126690392, + "inverse_median_rank": 6.963788300835655e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.246949242031, + "variance": 165900761.47345868, + "z_arithmetic_mean_rank": 36.52408033073195, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.31857529245616 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7891311604909889, + "adjusted_arithmetic_mean_rank_index": 0.21087914477918912, + "adjusted_hits_at_k": 0.03616448943915714, + "adjusted_inverse_harmonic_mean_rank": 0.018963675232121602, + "arithmetic_mean_rank": 16148.1748046875, + "count": 10000.0, + "geometric_mean_rank": 6089.12109375, + "harmonic_mean_rank": 51.99646808658853, + "hits_at_1": 0.0108, + "hits_at_10": 0.0364, + "hits_at_3": 0.0193, + "hits_at_5": 0.0251, + "inverse_arithmetic_mean_rank": 6.192650471348315e-05, + "inverse_geometric_mean_rank": 0.00016422731277998537, + "inverse_harmonic_mean_rank": 0.01923207604722585, + "inverse_median_rank": 6.963788473512977e-05, + "median_absolute_deviation": 16992.845327401956, + "median_rank": 14360.0, + "standard_deviation": 12880.23828125, + "variance": 165900528.0, + "z_arithmetic_mean_rank": 36.52440109201268, + "z_hits_at_k": 231.3265818769568, + "z_inverse_harmonic_mean_rank": 299.31858982596697 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7980273197378187, + "adjusted_arithmetic_mean_rank_index": 0.2019825510012615, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.0189419066190694, + "arithmetic_mean_rank": 16329.8404, + "count": 5000.0, + "geometric_mean_rank": 6326.548079401762, + "harmonic_mean_rank": 52.0553560301053, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123758564106971e-05, + "inverse_geometric_mean_rank": 0.00015806407972395587, + "inverse_harmonic_mean_rank": 0.019210319096111216, + "inverse_median_rank": 6.911088842047065e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.251613807723, + "variance": 166210151.67372784, + "z_arithmetic_mean_rank": 24.737071831918513, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40478313251361 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.79803106312362, + "adjusted_arithmetic_mean_rank_index": 0.20197880743251473, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.018941905090615847, + "arithmetic_mean_rank": 16329.917, + "count": 5000.0, + "geometric_mean_rank": 6326.5873648947, + "harmonic_mean_rank": 52.05536017071492, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123729838920798e-05, + "inverse_geometric_mean_rank": 0.00015806309821134415, + "inverse_harmonic_mean_rank": 0.019210317568075837, + "inverse_median_rank": 6.911088842047065e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.268639789934, + "variance": 166210590.680511, + "z_arithmetic_mean_rank": 24.736613352071913, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40476607391446 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7980291537861127, + "adjusted_arithmetic_mean_rank_index": 0.20198071686333463, + "adjusted_hits_at_k": 0.03476414176014741, + "adjusted_inverse_harmonic_mean_rank": 0.01894190585454245, + "arithmetic_mean_rank": 16329.8779296875, + "count": 5000.0, + "geometric_mean_rank": 6326.56640625, + "harmonic_mean_rank": 52.05535901882765, + "hits_at_1": 0.0116, + "hits_at_10": 0.035, + "hits_at_3": 0.0184, + "hits_at_5": 0.0246, + "inverse_arithmetic_mean_rank": 6.123744242358953e-05, + "inverse_geometric_mean_rank": 0.00015806361625436693, + "inverse_harmonic_mean_rank": 0.019210318331793434, + "inverse_median_rank": 6.911088712513447e-05, + "median_absolute_deviation": 17057.33852390695, + "median_rank": 14469.5, + "standard_deviation": 12892.259765625, + "variance": 166210352.0, + "z_arithmetic_mean_rank": 24.736847202605617, + "z_hits_at_k": 157.23696215885795, + "z_inverse_harmonic_mean_rank": 211.40477459986386 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7802315234433033, + "adjusted_arithmetic_mean_rank_index": 0.2197792165077822, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.018985445687869747, + "arithmetic_mean_rank": 15966.4296, + "count": 5000.0, + "geometric_mean_rank": 5860.560265003558, + "harmonic_mean_rank": 51.93770530750026, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263141009308681e-05, + "inverse_geometric_mean_rank": 0.00017063215030336233, + "inverse_harmonic_mean_rank": 0.01925383484078553, + "inverse_median_rank": 7.000840100812098e-05, + "median_absolute_deviation": 16935.02384088024, + "median_rank": 14284.0, + "standard_deviation": 12865.631994933006, + "variance": 165524486.62904385, + "z_arithmetic_mean_rank": 26.91665769738404, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.89565239009264 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7802353741629438, + "adjusted_arithmetic_mean_rank_index": 0.21977536559995947, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.018985443531799297, + "arithmetic_mean_rank": 15966.5084, + "count": 5000.0, + "geometric_mean_rank": 5860.6046943714355, + "harmonic_mean_rank": 51.937711121964185, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263110098636218e-05, + "inverse_geometric_mean_rank": 0.00017063085673742963, + "inverse_harmonic_mean_rank": 0.019253832685304943, + "inverse_median_rank": 7.000840100812098e-05, + "median_absolute_deviation": 16935.76514198949, + "median_rank": 14284.0, + "standard_deviation": 12865.648035024487, + "variance": 165524899.36112943, + "z_arithmetic_mean_rank": 26.91618607149813, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.89562832629335 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7802334365863733, + "adjusted_arithmetic_mean_rank_index": 0.21977730327121803, + "adjusted_hits_at_k": 0.03756483710218146, + "adjusted_inverse_harmonic_mean_rank": 0.01898544460944743, + "arithmetic_mean_rank": 15966.46875, + "count": 5000.0, + "geometric_mean_rank": 5860.5830078125, + "harmonic_mean_rank": 51.937706135894416, + "hits_at_1": 0.01, + "hits_at_10": 0.0378, + "hits_at_3": 0.0202, + "hits_at_5": 0.0256, + "inverse_arithmetic_mean_rank": 6.263125396799296e-05, + "inverse_geometric_mean_rank": 0.0001706314942566678, + "inverse_harmonic_mean_rank": 0.01925383376265825, + "inverse_median_rank": 7.000839832471684e-05, + "median_absolute_deviation": 16935.394491434865, + "median_rank": 14284.0, + "standard_deviation": 12865.6396484375, + "variance": 165524688.0, + "z_arithmetic_mean_rank": 26.91642338071611, + "z_hits_at_k": 169.90837521337207, + "z_inverse_harmonic_mean_rank": 211.8956403538727 + } + } + }, + "times": { + "evaluation": 11.729146718978882, + "training": 471.5053610801697 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json new file mode 100644 index 0000000..0acad86 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Could not find number of epochs. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json new file mode 100644 index 0000000..9d80498 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/rotate/wn18rr/sun2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.019358374847747738, + 0.01926067069163421, + 0.019161655377482285, + 0.01906288573317298, + 0.01896479291372804 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8705811229150148, + "adjusted_arithmetic_mean_rank_index": 0.12942526134716914, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.003645520836678461, + "arithmetic_mean_rank": 17648.897571819427, + "count": 5848.0, + "geometric_mean_rank": 10309.546076029572, + "harmonic_mean_rank": 255.07299036983989, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.6660762856753884e-05, + "inverse_geometric_mean_rank": 9.699748103605368e-05, + "inverse_harmonic_mean_rank": 0.003920446451621798, + "inverse_median_rank": 6.00817111271329e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.257654460096, + "variance": 144582772.14084223, + "z_arithmetic_mean_rank": 17.14242203045866, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.796929167098405 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8705851548377035, + "adjusted_arithmetic_mean_rank_index": 0.12942122922558474, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.0036455178545416692, + "arithmetic_mean_rank": 17648.979309165527, + "count": 5848.0, + "geometric_mean_rank": 10309.62711068822, + "harmonic_mean_rank": 255.07318434092105, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.666050044495642e-05, + "inverse_geometric_mean_rank": 9.699671862654254e-05, + "inverse_harmonic_mean_rank": 0.003920443470307872, + "inverse_median_rank": 6.00817111271329e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.267924256845, + "variance": 144583019.114312, + "z_arithmetic_mean_rank": 17.141887974516578, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.796893339999976 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8705830924839433, + "adjusted_arithmetic_mean_rank_index": 0.12942329168108146, + "adjusted_hits_at_k": 0.008134293686767383, + "adjusted_inverse_harmonic_mean_rank": 0.0036455193447481517, + "arithmetic_mean_rank": 17648.9375, + "count": 5848.0, + "geometric_mean_rank": 10309.5869140625, + "harmonic_mean_rank": 255.0730929684683, + "hits_at_1": 0.0011969904240766074, + "hits_at_10": 0.008378932968536252, + "hits_at_3": 0.004274965800273598, + "hits_at_5": 0.005984952120383037, + "inverse_arithmetic_mean_rank": 5.6660635891603306e-05, + "inverse_geometric_mean_rank": 9.699709335109219e-05, + "inverse_harmonic_mean_rank": 0.00392044496010316, + "inverse_median_rank": 6.00817111262586e-05, + "median_absolute_deviation": 15450.197719046877, + "median_rank": 16644.0, + "standard_deviation": 12024.2626953125, + "variance": 144582896.0, + "z_arithmetic_mean_rank": 17.142161147482767, + "z_hits_at_k": 39.603476070110496, + "z_inverse_harmonic_mean_rank": 43.79691124319425 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9373203914542778, + "adjusted_arithmetic_mean_rank_index": 0.0626827012310005, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593373166947965, + "arithmetic_mean_rank": 18997.656976744187, + "count": 2924.0, + "geometric_mean_rank": 12721.473902043308, + "harmonic_mean_rank": 544.9904552697009, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.2638070116969746e-05, + "inverse_geometric_mean_rank": 7.860724375965439e-05, + "inverse_harmonic_mean_rank": 0.0018348945203180253, + "inverse_median_rank": 5.425935973955508e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.0, + "standard_deviation": 11759.070452955062, + "variance": 138275737.91756076, + "z_arithmetic_mean_rank": 5.870641867908702, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245268635772078 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9373248123670547, + "adjusted_arithmetic_mean_rank_index": 0.06267828010009058, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593357925949337, + "arithmetic_mean_rank": 18997.74658002736, + "count": 2924.0, + "geometric_mean_rank": 12721.566306576726, + "harmonic_mean_rank": 544.9909078250481, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.263782184837207e-05, + "inverse_geometric_mean_rank": 7.860667278706282e-05, + "inverse_harmonic_mean_rank": 0.0018348929966387952, + "inverse_median_rank": 5.4257887740430265e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.5, + "standard_deviation": 11759.076669307115, + "variance": 138275884.1146429, + "z_arithmetic_mean_rank": 5.870227800299716, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245255689816194 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.937322571986191, + "adjusted_arithmetic_mean_rank_index": 0.06268052059149731, + "adjusted_hits_at_k": 0.0028319734402369786, + "adjusted_inverse_harmonic_mean_rank": 0.0015593365544549359, + "arithmetic_mean_rank": 18997.701171875, + "count": 2924.0, + "geometric_mean_rank": 12721.5244140625, + "harmonic_mean_rank": 544.9906554581698, + "hits_at_1": 0.0010259917920656635, + "hits_at_10": 0.0030779753761969904, + "hits_at_3": 0.0010259917920656635, + "hits_at_5": 0.0023939808481532147, + "inverse_arithmetic_mean_rank": 5.263794810161926e-05, + "inverse_geometric_mean_rank": 7.860692858230323e-05, + "inverse_harmonic_mean_rank": 0.0018348937582885335, + "inverse_median_rank": 5.425862400443293e-05, + "median_absolute_deviation": 15111.423112118347, + "median_rank": 18430.25, + "standard_deviation": 11759.0732421875, + "variance": 138275808.0, + "z_arithmetic_mean_rank": 5.870437636863841, + "z_hits_at_k": 9.748531363852472, + "z_inverse_harmonic_mean_rank": 13.245262161180847 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8038714603881213, + "adjusted_arithmetic_mean_rank_index": 0.19613821253058217, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.0057317041228105246, + "arithmetic_mean_rank": 16300.138166894665, + "count": 2924.0, + "geometric_mean_rank": 8354.907702691988, + "harmonic_mean_rank": 166.50021132920975, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134917322547517e-05, + "inverse_geometric_mean_rank": 0.00011969013130782949, + "inverse_harmonic_mean_rank": 0.0060059983829255734, + "inverse_median_rank": 6.83900971139379e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.723008761996, + "variance": 147251502.49937776, + "z_arithmetic_mean_rank": 18.369638890388757, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.69689065121688 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8038751034932807, + "adjusted_arithmetic_mean_rank_index": 0.19613456924574735, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.005731699682636964, + "arithmetic_mean_rank": 16300.212038303693, + "count": 2924.0, + "geometric_mean_rank": 8354.978357223932, + "harmonic_mean_rank": 166.5003343872566, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134889519535763e-05, + "inverse_geometric_mean_rank": 0.00011968911913881547, + "inverse_harmonic_mean_rank": 0.006005993943976948, + "inverse_median_rank": 6.83900971139379e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.735588882228, + "variance": 147251807.8120849, + "z_arithmetic_mean_rank": 18.369297672703908, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.69685292724479 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8038732672486356, + "adjusted_arithmetic_mean_rank_index": 0.1961364055809548, + "adjusted_hits_at_k": 0.013436613349726862, + "adjusted_inverse_harmonic_mean_rank": 0.005731701901189846, + "arithmetic_mean_rank": 16300.1748046875, + "count": 2924.0, + "geometric_mean_rank": 8354.947265625, + "harmonic_mean_rank": 166.50025295140748, + "hits_at_1": 0.0013679890560875513, + "hits_at_10": 0.013679890560875513, + "hits_at_3": 0.007523939808481532, + "hits_at_5": 0.009575923392612859, + "inverse_arithmetic_mean_rank": 6.134903378551826e-05, + "inverse_geometric_mean_rank": 0.00011968956096097827, + "inverse_harmonic_mean_rank": 0.006005996161917786, + "inverse_median_rank": 6.839009438408539e-05, + "median_absolute_deviation": 15313.05701383511, + "median_rank": 14622.0, + "standard_deviation": 12134.7294921875, + "variance": 147251648.0, + "z_arithmetic_mean_rank": 18.369469657623156, + "z_hits_at_k": 46.263313401813704, + "z_inverse_harmonic_mean_rank": 48.696871776198755 + } + } + }, + "times": { + "evaluation": 7.746156454086304, + "training": 290.5797860622406 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json new file mode 100644 index 0000000..8ca03b4 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json new file mode 100644 index 0000000..b7ad605 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/fb15k/kazemi2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0005177027531600979, + 9.552033202193208e-05, + 7.630582763936014e-05, + 6.830938954821957e-05, + 6.340357705154206e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6174970603345358, + "adjusted_arithmetic_mean_rank_index": 0.3825547282632912, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.0015853138783117442, + "arithmetic_mean_rank": 4561.35964347988, + "count": 118142.0, + "geometric_mean_rank": 2305.144626685176, + "harmonic_mean_rank": 439.683412149987, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.00021923287751042053, + "inverse_geometric_mean_rank": 0.0004338122599439721, + "inverse_harmonic_mean_rank": 0.0022743637179991564, + "inverse_median_rank": 0.0002976633427593392, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.257051319545, + "variance": 16918883.57022996, + "z_arithmetic_mean_rank": 227.55016767242148, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.66200401260044 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.617497263153951, + "adjusted_arithmetic_mean_rank_index": 0.3825545254164153, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.001585313588680166, + "arithmetic_mean_rank": 4561.361141676965, + "count": 118142.0, + "geometric_mean_rank": 2305.145527422239, + "harmonic_mean_rank": 439.6834681033637, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.0002192328055025159, + "inverse_geometric_mean_rank": 0.00043381209043155894, + "inverse_harmonic_mean_rank": 0.0022743634285674655, + "inverse_median_rank": 0.0002976633427593392, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.25777071006, + "variance": 16918889.48830669, + "z_arithmetic_mean_rank": 227.55004701559199, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.66199457412396 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6174971561916859, + "adjusted_arithmetic_mean_rank_index": 0.38255463239316256, + "adjusted_hits_at_k": 0.001387949404711147, + "adjusted_inverse_harmonic_mean_rank": 0.00158531373336889, + "arithmetic_mean_rank": 4561.3603515625, + "count": 118142.0, + "geometric_mean_rank": 2305.14501953125, + "harmonic_mean_rank": 439.68341606952293, + "hits_at_1": 0.00024546731898901324, + "hits_at_10": 0.0020653112356316975, + "hits_at_3": 0.0005755785410776862, + "hits_at_5": 0.0008718322019264953, + "inverse_arithmetic_mean_rank": 0.00021923283929936588, + "inverse_geometric_mean_rank": 0.00043381217983551323, + "inverse_harmonic_mean_rank": 0.0022743635731563338, + "inverse_median_rank": 0.0002976633550133556, + "median_absolute_deviation": 4103.101639714253, + "median_rank": 3359.5, + "standard_deviation": 4113.25732421875, + "variance": 16918886.0, + "z_arithmetic_mean_rank": 227.5501106472111, + "z_hits_at_k": 18.31120288173195, + "z_inverse_harmonic_mean_rank": 51.661999289221434 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6461976610370368, + "adjusted_arithmetic_mean_rank_index": 0.3538503724133438, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.0012519077863274576, + "arithmetic_mean_rank": 4760.375971288788, + "count": 59071.0, + "geometric_mean_rank": 2569.324969383426, + "harmonic_mean_rank": 514.6465541118973, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.0002100674413179318, + "inverse_geometric_mean_rank": 0.0003892072867061169, + "inverse_harmonic_mean_rank": 0.0019430811146217727, + "inverse_median_rank": 0.0002697599136768276, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.129880173063, + "variance": 16639300.579320708, + "z_arithmetic_mean_rank": 148.80767083723669, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803820514157955 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6461979000289159, + "adjusted_arithmetic_mean_rank_index": 0.3538501333890185, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.001251907536445022, + "arithmetic_mean_rank": 4760.377731881973, + "count": 59071.0, + "geometric_mean_rank": 2569.3260504361556, + "harmonic_mean_rank": 514.6466202502345, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.00021006736362592362, + "inverse_geometric_mean_rank": 0.00038920712294581887, + "inverse_harmonic_mean_rank": 0.0019430808649122655, + "inverse_median_rank": 0.0002697599136768276, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.130608036787, + "variance": 16639306.51742257, + "z_arithmetic_mean_rank": 148.80757031832837, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803814764877608 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6461977943164811, + "adjusted_arithmetic_mean_rank_index": 0.35385023911580515, + "adjusted_hits_at_k": 0.0011317658443619886, + "adjusted_inverse_harmonic_mean_rank": 0.0012519076612958744, + "arithmetic_mean_rank": 4760.376953125, + "count": 59071.0, + "geometric_mean_rank": 2569.32470703125, + "harmonic_mean_rank": 514.6466113756944, + "hits_at_1": 0.00020314536743918336, + "hits_at_10": 0.0018113795263327182, + "hits_at_3": 0.00042321951549829863, + "hits_at_5": 0.0006771512247972779, + "inverse_arithmetic_mean_rank": 0.0002100674028042704, + "inverse_geometric_mean_rank": 0.00038920732913538814, + "inverse_harmonic_mean_rank": 0.001943080989676716, + "inverse_median_rank": 0.0002697599120438099, + "median_absolute_deviation": 4281.755207044179, + "median_rank": 3707.0, + "standard_deviation": 4079.130126953125, + "variance": 16639303.0, + "z_arithmetic_mean_rank": 148.8076147805025, + "z_hits_at_k": 10.541913177435609, + "z_inverse_harmonic_mean_rank": 28.803817637438655 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5889522536778217, + "adjusted_arithmetic_mean_rank_index": 0.4111032486444999, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.001918718705307047, + "arithmetic_mean_rank": 4362.3433156709725, + "count": 59071.0, + "geometric_mean_rank": 2068.1275483850927, + "harmonic_mean_rank": 383.78193993408456, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.0002292345942621414, + "inverse_geometric_mean_rank": 0.0004835291714869592, + "inverse_harmonic_mean_rank": 0.002605646321376541, + "inverse_median_rank": 0.0003324468085106383, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.54172953822, + "variance": 17119251.56367013, + "z_arithmetic_mean_rank": 172.93582102530266, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.281014237433794 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5889524205211264, + "adjusted_arithmetic_mean_rank_index": 0.4111030817786667, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.0019187183759264768, + "arithmetic_mean_rank": 4362.344551471958, + "count": 59071.0, + "geometric_mean_rank": 2068.1282944578934, + "harmonic_mean_rank": 383.7819884146966, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.0002292345293226727, + "inverse_geometric_mean_rank": 0.0004835289970548584, + "inverse_harmonic_mean_rank": 0.002605645992222666, + "inverse_median_rank": 0.0003324468085106383, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.542417043861, + "variance": 17119257.25283716, + "z_arithmetic_mean_rank": 172.93575083106327, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.28100663584755 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.588952378237803, + "adjusted_arithmetic_mean_rank_index": 0.41110312406769955, + "adjusted_hits_at_k": 0.001644131898392271, + "adjusted_inverse_harmonic_mean_rank": 0.0019187185404529977, + "arithmetic_mean_rank": 4362.34423828125, + "count": 59071.0, + "geometric_mean_rank": 2068.12646484375, + "harmonic_mean_rank": 383.7819330855535, + "hits_at_1": 0.0002877892705388431, + "hits_at_10": 0.0023192429449306767, + "hits_at_3": 0.0007279375666570737, + "hits_at_5": 0.0010665131790557126, + "inverse_arithmetic_mean_rank": 0.00022923454525880516, + "inverse_geometric_mean_rank": 0.00048352943849749863, + "inverse_harmonic_mean_rank": 0.002605646156635952, + "inverse_median_rank": 0.0003324467979837209, + "median_absolute_deviation": 3817.700712651925, + "median_rank": 3008.0, + "standard_deviation": 4137.54248046875, + "variance": 17119256.0, + "z_arithmetic_mean_rank": 172.93576862048388, + "z_hits_at_k": 15.361454062550017, + "z_inverse_harmonic_mean_rank": 44.28101043286125 + } + } + }, + "times": { + "evaluation": 29.054805994033813, + "training": 10.112524032592773 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json new file mode 100644 index 0000000..99e152d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with SimplE as described by Kazemi et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json new file mode 100644 index 0000000..48bb631 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/simple/wn18/kazemi2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.007010572051467946, + 0.0005912138870406634, + 0.0003330073621221427, + 0.00022124117972636016, + 0.00013660546551546888 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.055875071158110745, + "adjusted_arithmetic_mean_rank_index": 0.9441710687196261, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846, + "count": 10000.0, + "geometric_mean_rank": 55.061474085931685, + "harmonic_mean_rank": 5.073295495004501, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745963519186807, + "inverse_geometric_mean_rank": 0.018161518858709633, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.387928648988, + "variance": 17634858.97528284, + "z_arithmetic_mean_rank": 163.5310255525711, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.055875071158110745, + "adjusted_arithmetic_mean_rank_index": 0.9441710687196261, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846, + "count": 10000.0, + "geometric_mean_rank": 55.061474085931685, + "harmonic_mean_rank": 5.073295495004501, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745963519186807, + "inverse_geometric_mean_rank": 0.018161518858709633, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.01694915254237288, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.387928648988, + "variance": 17634858.97528284, + "z_arithmetic_mean_rank": 163.5310255525711, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05587507328654702, + "adjusted_arithmetic_mean_rank_index": 0.9441710665910859, + "adjusted_hits_at_k": 0.32573524525833936, + "adjusted_inverse_harmonic_mean_rank": 0.19689081522774649, + "arithmetic_mean_rank": 1143.3846435546875, + "count": 10000.0, + "geometric_mean_rank": 55.06148910522461, + "harmonic_mean_rank": 5.073295558172437, + "hits_at_1": 0.1307, + "hits_at_10": 0.3259, + "hits_at_3": 0.2176, + "hits_at_5": 0.2635, + "inverse_arithmetic_mean_rank": 0.0008745962986722589, + "inverse_geometric_mean_rank": 0.018161514773964882, + "inverse_harmonic_mean_rank": 0.19711053712220497, + "inverse_median_rank": 0.016949152573943138, + "median_absolute_deviation": 85.99092867332492, + "median_rank": 59.0, + "standard_deviation": 4199.38818359375, + "variance": 17634860.0, + "z_arithmetic_mean_rank": 163.5310251839066, + "z_hits_at_k": 2083.5693259055724, + "z_inverse_harmonic_mean_rank": 3107.6824741140026 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.05311946552504412, + "adjusted_arithmetic_mean_rank_index": 0.9469268100934394, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708, + "count": 5000.0, + "geometric_mean_rank": 58.671546702675336, + "harmonic_mean_rank": 5.294065622093927, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.000919987915038748, + "inverse_geometric_mean_rank": 0.017044036781024584, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925373134328358, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.5450086197116, + "variance": 15614707.955147358, + "z_arithmetic_mean_rank": 115.97138666054661, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.05311946552504412, + "adjusted_arithmetic_mean_rank_index": 0.9469268100934394, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708, + "count": 5000.0, + "geometric_mean_rank": 58.671546702675336, + "harmonic_mean_rank": 5.294065622093927, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.000919987915038748, + "inverse_geometric_mean_rank": 0.017044036781024584, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925373134328358, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.5450086197116, + "variance": 15614707.955147358, + "z_arithmetic_mean_rank": 115.97138666054661, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.05311946675632055, + "adjusted_arithmetic_mean_rank_index": 0.9469268088621028, + "adjusted_hits_at_k": 0.3108315996608721, + "adjusted_inverse_harmonic_mean_rank": 0.18866876903099505, + "arithmetic_mean_rank": 1086.9708251953125, + "count": 5000.0, + "geometric_mean_rank": 58.67153549194336, + "harmonic_mean_rank": 5.294065567053748, + "hits_at_1": 0.1248, + "hits_at_10": 0.311, + "hits_at_3": 0.2076, + "hits_at_5": 0.2522, + "inverse_arithmetic_mean_rank": 0.0009199879132211208, + "inverse_geometric_mean_rank": 0.01704403944313526, + "inverse_harmonic_mean_rank": 0.18889074510649465, + "inverse_median_rank": 0.014925372786819935, + "median_absolute_deviation": 97.85174642136973, + "median_rank": 67.0, + "standard_deviation": 3951.545166015625, + "variance": 15614709.0, + "z_arithmetic_mean_rank": 115.97138650974318, + "z_hits_at_k": 1405.8801396812214, + "z_inverse_harmonic_mean_rank": 2105.673995917701 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.0586305490274944, + "adjusted_arithmetic_mean_rank_index": 0.9414154551220333, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.7984, + "count": 5000.0, + "geometric_mean_rank": 51.673529997079186, + "harmonic_mean_rank": 4.870201125174862, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334733568572854, + "inverse_geometric_mean_rank": 0.019352267980463585, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.679208081433, + "variance": 19648644.96175744, + "z_arithmetic_mean_rank": 115.29642319772988, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.0586305490274944, + "adjusted_arithmetic_mean_rank_index": 0.9414154551220333, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.7984, + "count": 5000.0, + "geometric_mean_rank": 51.673529997079186, + "harmonic_mean_rank": 4.870201125174862, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334733568572854, + "inverse_geometric_mean_rank": 0.019352267980463585, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.02, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.679208081433, + "variance": 19648644.96175744, + "z_arithmetic_mean_rank": 115.29642319772988, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.0586305460878389, + "adjusted_arithmetic_mean_rank_index": 0.9414154580618325, + "adjusted_hits_at_k": 0.34063889068567665, + "adjusted_inverse_harmonic_mean_rank": 0.20511286132882536, + "arithmetic_mean_rank": 1199.79833984375, + "count": 5000.0, + "geometric_mean_rank": 51.67353439331055, + "harmonic_mean_rank": 4.870200709104077, + "hits_at_1": 0.1366, + "hits_at_10": 0.3408, + "hits_at_3": 0.2276, + "hits_at_5": 0.2748, + "inverse_arithmetic_mean_rank": 0.0008334734011441469, + "inverse_geometric_mean_rank": 0.019352266564965248, + "inverse_harmonic_mean_rank": 0.2053303291379153, + "inverse_median_rank": 0.019999999552965164, + "median_absolute_deviation": 72.6475087067745, + "median_rank": 50.0, + "standard_deviation": 4432.67919921875, + "variance": 19648646.0, + "z_arithmetic_mean_rank": 115.29642355777105, + "z_hits_at_k": 1540.73343359521, + "z_inverse_harmonic_mean_rank": 2289.2548470768475 + } + } + }, + "times": { + "evaluation": 8.916642427444458, + "training": 4.890740156173706 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json new file mode 100644 index 0000000..87c1425 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json new file mode 100644 index 0000000..a6230c9 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/fb15k/ji2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002552040782367046, + 0.0013616517112822108, + 0.0010789413683163901, + 0.0009065022953803493, + 0.000778262092445874 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.23547394843537406, + "adjusted_arithmetic_mean_rank_index": 0.7646295637944857, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693621079345207, + "arithmetic_mean_rank": 1739.4113016539418, + "count": 118142.0, + "geometric_mean_rank": 208.93563489743488, + "harmonic_mean_rank": 8.507332970126315, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.00057490715338525, + "inverse_geometric_mean_rank": 0.0047861629754584155, + "inverse_harmonic_mean_rank": 0.11754565191130073, + "inverse_median_rank": 0.0020491803278688526, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.43720587806, + "variance": 6705185.043185574, + "z_arithmetic_mean_rank": 454.8148868492797, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7021416245047 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.2354740939611692, + "adjusted_arithmetic_mean_rank_index": 0.7646294182489872, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693620537414412, + "arithmetic_mean_rank": 1739.4123766315113, + "count": 118142.0, + "geometric_mean_rank": 208.9357986842343, + "harmonic_mean_rank": 8.507333362076508, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.0005749067980857805, + "inverse_geometric_mean_rank": 0.00478615922353883, + "inverse_harmonic_mean_rank": 0.11754564649573288, + "inverse_median_rank": 0.0020491803278688526, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.438397730749, + "variance": 6705191.215642387, + "z_arithmetic_mean_rank": 454.8148002763004, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7019650207967 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.23547402473044568, + "adjusted_arithmetic_mean_rank_index": 0.7646294874890841, + "adjusted_hits_at_k": 0.21288723610477697, + "adjusted_inverse_harmonic_mean_rank": 0.11693620806333842, + "arithmetic_mean_rank": 1739.411865234375, + "count": 118142.0, + "geometric_mean_rank": 208.9356231689453, + "harmonic_mean_rank": 8.507333741808493, + "hits_at_1": 0.06665707369098203, + "hits_at_10": 0.21342113727548204, + "hits_at_3": 0.1354471737400755, + "hits_at_5": 0.16716324423151802, + "inverse_arithmetic_mean_rank": 0.0005749069387093186, + "inverse_geometric_mean_rank": 0.004786163102835417, + "inverse_harmonic_mean_rank": 0.11754564918307125, + "inverse_median_rank": 0.0020491802133619785, + "median_absolute_deviation": 720.5446781937226, + "median_rank": 488.0, + "standard_deviation": 2589.437744140625, + "variance": 6705187.5, + "z_arithmetic_mean_rank": 454.8148414615074, + "z_hits_at_k": 2808.619217684682, + "z_inverse_harmonic_mean_rank": 3810.7020526559136 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.26508252584909736, + "adjusted_arithmetic_mean_rank_index": 0.7350172491329787, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.0895138719752979, + "arithmetic_mean_rank": 1952.7964314130454, + "count": 59071.0, + "geometric_mean_rank": 302.6048275128738, + "harmonic_mean_rank": 11.09336609853906, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120861467758823, + "inverse_geometric_mean_rank": 0.0033046399431861566, + "inverse_harmonic_mean_rank": 0.09014396452053404, + "inverse_median_rank": 0.0012755102040816326, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.174746370657, + "variance": 7204794.06905398, + "z_arithmetic_mean_rank": 309.10298079580843, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.5298871552714 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.26508269130501355, + "adjusted_arithmetic_mean_rank_index": 0.7350170836545997, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.08951386824429253, + "arithmetic_mean_rank": 1952.79765028525, + "count": 59071.0, + "geometric_mean_rank": 302.6050304340035, + "harmonic_mean_rank": 11.09336655736919, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120858271485156, + "inverse_geometric_mean_rank": 0.003304637727157991, + "inverse_harmonic_mean_rank": 0.09014396079211066, + "inverse_median_rank": 0.0012755102040816326, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.1761261754814, + "variance": 7204801.476330414, + "z_arithmetic_mean_rank": 309.102911205795, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.52980131252 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.2650826192047391, + "adjusted_arithmetic_mean_rank_index": 0.7350171557646628, + "adjusted_hits_at_k": 0.1697216964175407, + "adjusted_inverse_harmonic_mean_rank": 0.08951387009760452, + "arithmetic_mean_rank": 1952.797119140625, + "count": 59071.0, + "geometric_mean_rank": 302.6047668457031, + "harmonic_mean_rank": 11.093367293323608, + "hits_at_1": 0.04784073403192768, + "hits_at_10": 0.17028660425589545, + "hits_at_3": 0.10321477543972507, + "hits_at_5": 0.12986067613549795, + "inverse_arithmetic_mean_rank": 0.0005120859714224935, + "inverse_geometric_mean_rank": 0.003304640529677272, + "inverse_harmonic_mean_rank": 0.09014396264414008, + "inverse_median_rank": 0.0012755101779475808, + "median_absolute_deviation": 1156.4297304343695, + "median_rank": 784.0, + "standard_deviation": 2684.175537109375, + "variance": 7204798.0, + "z_arithmetic_mean_rank": 309.1029415308459, + "z_hits_at_k": 1580.8847712394263, + "z_inverse_harmonic_mean_rank": 2059.5298439534126 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.2060260937923374, + "adjusted_arithmetic_mean_rank_index": 0.7940811136940227, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.14435844556744704, + "arithmetic_mean_rank": 1526.0261718948384, + "count": 59071.0, + "geometric_mean_rank": 144.2610809906429, + "harmonic_mean_rank": 6.899057304639579, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.0006552967559909661, + "inverse_geometric_mean_rank": 0.006931876519522699, + "inverse_harmonic_mean_rank": 0.14494733930206743, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.7534430493765, + "variance": 6114509.590112546, + "z_arithmetic_mean_rank": 334.04034098527376, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.566198726866 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.20602621949619718, + "adjusted_arithmetic_mean_rank_index": 0.7940809879731895, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.1443584384598429, + "arithmetic_mean_rank": 1526.0271029777725, + "count": 59071.0, + "geometric_mean_rank": 144.26121042736472, + "harmonic_mean_rank": 6.899057642707337, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.0006552963561713135, + "inverse_geometric_mean_rank": 0.00693187029997574, + "inverse_harmonic_mean_rank": 0.1449473321993551, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.754416623252, + "variance": 6114514.4049298, + "z_arithmetic_mean_rank": 334.040288099203, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.5660346945256 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.20602616959994496, + "adjusted_arithmetic_mean_rank_index": 0.794081037876179, + "adjusted_hits_at_k": 0.25605259606423786, + "adjusted_inverse_harmonic_mean_rank": 0.14435844198491637, + "arithmetic_mean_rank": 1526.0267333984375, + "count": 59071.0, + "geometric_mean_rank": 144.26116943359375, + "harmonic_mean_rank": 6.8990578575211945, + "hits_at_1": 0.0854734133500364, + "hits_at_10": 0.25655567029506865, + "hits_at_3": 0.16767957204042594, + "hits_at_5": 0.20446581232753805, + "inverse_arithmetic_mean_rank": 0.000655296491459012, + "inverse_geometric_mean_rank": 0.006931872107088566, + "inverse_harmonic_mean_rank": 0.14494733572200244, + "inverse_median_rank": 0.00390625, + "median_absolute_deviation": 378.0635657189285, + "median_rank": 256.0, + "standard_deviation": 2472.754150390625, + "variance": 6114513.0, + "z_arithmetic_mean_rank": 334.0403090915315, + "z_hits_at_k": 2392.350757189078, + "z_inverse_harmonic_mean_rank": 3331.566116047685 + } + } + }, + "times": { + "evaluation": 33.93540859222412, + "training": 87.27861523628235 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json new file mode 100644 index 0000000..f9e9946 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "We assumed the learning rate for Adadelta as the default from PyTorch since it was not explicitly stated in the original paper", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransD as described by Ji et al., 2015" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json new file mode 100644 index 0000000..ce10c4c --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transd/wn18/ji2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.004478471603307385, + 0.0036625902134173917, + 0.0031688402850889408, + 0.0027868693412893444, + 0.002493769001652959 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.596269284526057, + "adjusted_arithmetic_mean_rank_index": 0.40375044600459786, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145151990078, + "arithmetic_mean_rank": 12201.5973, + "count": 10000.0, + "geometric_mean_rank": 3947.2342081152287, + "harmonic_mean_rank": 63.71653149146111, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.19564828614693e-05, + "inverse_geometric_mean_rank": 0.0002533419471142787, + "inverse_harmonic_mean_rank": 0.01569451407024586, + "inverse_median_rank": 0.00011169440411035407, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.701157837608, + "variance": 131599927.45473272, + "z_arithmetic_mean_rank": 69.92983230462272, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.46718862459554 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5962695093194706, + "adjusted_arithmetic_mean_rank_index": 0.40375022120019854, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145101858308, + "arithmetic_mean_rank": 12201.6019, + "count": 10000.0, + "geometric_mean_rank": 3947.2356013178296, + "harmonic_mean_rank": 63.71653169493022, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.195645196390157e-05, + "inverse_geometric_mean_rank": 0.0002533418576955829, + "inverse_harmonic_mean_rank": 0.015694514020127805, + "inverse_median_rank": 0.00011169440411035407, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.704064428108, + "variance": 131599994.14181638, + "z_arithmetic_mean_rank": 69.92979336835873, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.4671878333264 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5962693973809026, + "adjusted_arithmetic_mean_rank_index": 0.40375033314423703, + "adjusted_hits_at_k": 0.037064709405434396, + "adjusted_inverse_harmonic_mean_rank": 0.015425145126921166, + "arithmetic_mean_rank": 12201.599609375, + "count": 10000.0, + "geometric_mean_rank": 3947.233642578125, + "harmonic_mean_rank": 63.71652988749175, + "hits_at_1": 0.0041, + "hits_at_10": 0.0373, + "hits_at_3": 0.0165, + "hits_at_5": 0.0244, + "inverse_arithmetic_mean_rank": 8.195646660169587e-05, + "inverse_geometric_mean_rank": 0.0002533419756218791, + "inverse_harmonic_mean_rank": 0.015694514045183804, + "inverse_median_rank": 0.00011169440404046327, + "median_absolute_deviation": 11985.356334399286, + "median_rank": 8953.0, + "standard_deviation": 11471.7021484375, + "variance": 131599960.0, + "z_arithmetic_mean_rank": 69.92981275713693, + "z_hits_at_k": 237.0848494749732, + "z_inverse_harmonic_mean_rank": 243.46718822891316 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6025079727031526, + "adjusted_arithmetic_mean_rank_index": 0.397511453389935, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001105085272, + "arithmetic_mean_rank": 12328.9752, + "count": 5000.0, + "geometric_mean_rank": 4126.3291122784585, + "harmonic_mean_rank": 69.97528513726976, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.11097421949555e-05, + "inverse_geometric_mean_rank": 0.00024234615630255055, + "inverse_harmonic_mean_rank": 0.014290759916709316, + "inverse_median_rank": 0.00011035700491088672, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.727300846267, + "variance": 131646418.17418496, + "z_arithmetic_mean_rank": 48.68375672934111, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407298857802 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6025082268233375, + "adjusted_arithmetic_mean_rank_index": 0.3975111992573307, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001033309185, + "arithmetic_mean_rank": 12328.9804, + "count": 5000.0, + "geometric_mean_rank": 4126.33083194677, + "harmonic_mean_rank": 69.97528548862813, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.110970798526048e-05, + "inverse_geometric_mean_rank": 0.000242346055303619, + "inverse_harmonic_mean_rank": 0.014290759844952868, + "inverse_median_rank": 0.00011035700491088672, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.7308259875, + "variance": 131646499.06721582, + "z_arithmetic_mean_rank": 48.68372560538286, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407218750725 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6025080870114209, + "adjusted_arithmetic_mean_rank_index": 0.3975113390760803, + "adjusted_hits_at_k": 0.03636453282044146, + "adjusted_inverse_harmonic_mean_rank": 0.014021001069192063, + "arithmetic_mean_rank": 12328.9775390625, + "count": 5000.0, + "geometric_mean_rank": 4126.33056640625, + "harmonic_mean_rank": 69.97528079642154, + "hits_at_1": 0.003, + "hits_at_10": 0.0366, + "hits_at_3": 0.0144, + "hits_at_5": 0.022, + "inverse_arithmetic_mean_rank": 8.110972703434527e-05, + "inverse_geometric_mean_rank": 0.00024234606826212257, + "inverse_harmonic_mean_rank": 0.014290759880825928, + "inverse_median_rank": 0.0001103570029954426, + "median_absolute_deviation": 12044.66042313951, + "median_rank": 9061.5, + "standard_deviation": 11473.7294921875, + "variance": 131646464.0, + "z_arithmetic_mean_rank": 48.683742729171186, + "z_hits_at_k": 164.4754733329006, + "z_inverse_harmonic_mean_rank": 156.48407258798497 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5900308856058019, + "adjusted_arithmetic_mean_rank_index": 0.40998914933416863, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.016829289182556138, + "arithmetic_mean_rank": 12074.2194, + "count": 5000.0, + "geometric_mean_rank": 3775.912553206831, + "harmonic_mean_rank": 58.48545518832573, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282108903868353e-05, + "inverse_geometric_mean_rank": 0.0002648366417147859, + "inverse_harmonic_mean_rank": 0.017098268223782413, + "inverse_median_rank": 0.00011267605633802817, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.259958531791, + "variance": 131520986.47646362, + "z_arithmetic_mean_rank": 50.211925256721074, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.8308926335976 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5900310810738039, + "adjusted_arithmetic_mean_rank_index": 0.40998895385661416, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.01682928915406868, + "arithmetic_mean_rank": 12074.2234, + "count": 5000.0, + "geometric_mean_rank": 3775.913645043353, + "harmonic_mean_rank": 58.4854552857418, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282106160136145e-05, + "inverse_geometric_mean_rank": 0.00026483656513509026, + "inverse_harmonic_mean_rank": 0.01709826819530275, + "inverse_median_rank": 0.00011267605633802817, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.26223330686, + "variance": 131521038.65189244, + "z_arithmetic_mean_rank": 50.21190131632078, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.83089231565046 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5900309492856118, + "adjusted_arithmetic_mean_rank_index": 0.4099890856512466, + "adjusted_hits_at_k": 0.03776488598243459, + "adjusted_inverse_harmonic_mean_rank": 0.016829289168311508, + "arithmetic_mean_rank": 12074.220703125, + "count": 5000.0, + "geometric_mean_rank": 3775.914306640625, + "harmonic_mean_rank": 58.48545551786481, + "hits_at_1": 0.0052, + "hits_at_10": 0.038, + "hits_at_3": 0.0186, + "hits_at_5": 0.0268, + "inverse_arithmetic_mean_rank": 8.282108319690451e-05, + "inverse_geometric_mean_rank": 0.0002648365043569356, + "inverse_harmonic_mean_rank": 0.017098268209541682, + "inverse_median_rank": 0.00011267605441389605, + "median_absolute_deviation": 11961.634698903197, + "median_rank": 8875.0, + "standard_deviation": 11468.26171875, + "variance": 131521016.0, + "z_arithmetic_mean_rank": 50.21191745738754, + "z_hits_at_k": 170.81321023540627, + "z_inverse_harmonic_mean_rank": 187.83089247461402 + } + } + }, + "times": { + "evaluation": 9.962103366851807, + "training": 25.386447191238403 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json new file mode 100644 index 0000000..d1f10e5 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Batch_size is not mentioned in the paper. We found out that this value works.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json new file mode 100644 index 0000000..c28d148 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k/bordes2013/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.038617746736009165, + 0.033566348636352834, + 0.0312234644151795, + 0.029337170743096344, + 0.027740906089835815 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.685622737188115, + "adjusted_arithmetic_mean_rank_index": 0.3144198276027439, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.03358499005785345, + "arithmetic_mean_rank": 5064.593963196831, + "count": 118142.0, + "geometric_mean_rank": 1813.7212167611826, + "harmonic_mean_rank": 29.19541337015125, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.00019744919479562547, + "inverse_geometric_mean_rank": 0.0005513526504286753, + "inverse_harmonic_mean_rank": 0.034251955515121356, + "inverse_median_rank": 0.00024274790629930817, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.964777631248, + "variance": 19079116.298627198, + "z_arithmetic_mean_rank": 187.02235054142858, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4633203991789 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6856238337880042, + "adjusted_arithmetic_mean_rank_index": 0.31441873085438155, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.03358498142594515, + "arithmetic_mean_rank": 5064.6020636183575, + "count": 118142.0, + "geometric_mean_rank": 1813.7260518976943, + "harmonic_mean_rank": 29.195420722673642, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.00019744887899160224, + "inverse_geometric_mean_rank": 0.0005513511806006778, + "inverse_harmonic_mean_rank": 0.03425194688917031, + "inverse_median_rank": 0.00024274790629930817, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.967432633831, + "variance": 19079139.49254978, + "z_arithmetic_mean_rank": 187.02169817653726, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4630391036653 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6856233032390106, + "adjusted_arithmetic_mean_rank_index": 0.31441926147520827, + "adjusted_hits_at_k": 0.05589327218864882, + "adjusted_inverse_harmonic_mean_rank": 0.033584985710982486, + "arithmetic_mean_rank": 5064.59814453125, + "count": 118142.0, + "geometric_mean_rank": 1813.7232666015625, + "harmonic_mean_rank": 29.195417225218765, + "hits_at_1": 0.021254084068324557, + "hits_at_10": 0.056533662880262736, + "hits_at_3": 0.0366423456518427, + "hits_at_5": 0.043506966193225104, + "inverse_arithmetic_mean_rank": 0.0001974490296561271, + "inverse_geometric_mean_rank": 0.0005513520445674658, + "inverse_harmonic_mean_rank": 0.03425195117125036, + "inverse_median_rank": 0.0002427479048492387, + "median_absolute_deviation": 5186.883861441848, + "median_rank": 4119.5, + "standard_deviation": 4367.96630859375, + "variance": 19079128.0, + "z_arithmetic_mean_rank": 187.0220137989807, + "z_hits_at_k": 737.3993917186157, + "z_inverse_harmonic_mean_rank": 1094.4631787439089 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.731205300619267, + "adjusted_arithmetic_mean_rank_index": 0.26883119189486704, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.025005942011049314, + "arithmetic_mean_rank": 5386.605914915948, + "count": 59071.0, + "geometric_mean_rank": 2308.3841018965627, + "harmonic_mean_rank": 38.93978400157192, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564565810001414, + "inverse_geometric_mean_rank": 0.0004332034686854768, + "inverse_harmonic_mean_rank": 0.025680676604668173, + "inverse_median_rank": 0.000216076058772688, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.978304404924, + "variance": 18835411.68270544, + "z_arithmetic_mean_rank": 113.05384036036361, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3352390168044 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.731206507068656, + "adjusted_arithmetic_mean_rank_index": 0.2688299852816862, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.025005932334410102, + "arithmetic_mean_rank": 5386.614802525774, + "count": 59071.0, + "geometric_mean_rank": 2308.390778038668, + "harmonic_mean_rank": 38.93979866417694, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564535179517601, + "inverse_geometric_mean_rank": 0.0004332022158092545, + "inverse_harmonic_mean_rank": 0.02568066693472558, + "inverse_median_rank": 0.000216076058772688, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.980567193157, + "variance": 18835431.323614236, + "z_arithmetic_mean_rank": 113.05333293318245, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3350163772601 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7312059028722491, + "adjusted_arithmetic_mean_rank_index": 0.26883058956012096, + "adjusted_hits_at_k": 0.03951850043829517, + "adjusted_inverse_harmonic_mean_rank": 0.02500593713381818, + "arithmetic_mean_rank": 5386.6103515625, + "count": 59071.0, + "geometric_mean_rank": 2308.387939453125, + "harmonic_mean_rank": 38.93978953297726, + "hits_at_1": 0.016996495742411674, + "hits_at_10": 0.04017199641109851, + "hits_at_3": 0.02659511435391309, + "hits_at_5": 0.03113202756005485, + "inverse_arithmetic_mean_rank": 0.00018564550555311143, + "inverse_geometric_mean_rank": 0.00043320274562574923, + "inverse_harmonic_mean_rank": 0.025680671730812277, + "inverse_median_rank": 0.0002160760632250458, + "median_absolute_deviation": 5303.268135594538, + "median_rank": 4628.0, + "standard_deviation": 4339.9794921875, + "variance": 18835422.0, + "z_arithmetic_mean_rank": 113.05358705547087, + "z_hits_at_k": 368.0978734234637, + "z_inverse_harmonic_mean_rank": 575.3351268017581 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6402876073228272, + "adjusted_arithmetic_mean_rank_index": 0.35976096336838814, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216400555455868, + "arithmetic_mean_rank": 4742.582011477713, + "count": 59071.0, + "geometric_mean_rank": 1425.059481837078, + "harmonic_mean_rank": 23.35180920857272, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085560514923302, + "inverse_geometric_mean_rank": 0.0007017250948085874, + "inverse_harmonic_mean_rank": 0.04282323442557454, + "inverse_median_rank": 0.0002795638803466592, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.1204832951535, + "variance": 19115437.520449042, + "z_arithmetic_mean_rank": 151.33803436996612, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.0790266986371 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6402885946695075, + "adjusted_arithmetic_mean_rank_index": 0.3597599758883899, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216399796737732, + "arithmetic_mean_rank": 4742.589324710941, + "count": 59071.0, + "geometric_mean_rank": 1425.062958416174, + "harmonic_mean_rank": 23.35181334306899, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085528000275454, + "inverse_geometric_mean_rank": 0.0007017233828822606, + "inverse_harmonic_mean_rank": 0.042823226843615045, + "inverse_median_rank": 0.0002795638803466592, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.123426145244, + "variance": 19115463.253448024, + "z_arithmetic_mean_rank": 151.33761897391952, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.0788515984075 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6402881373681507, + "adjusted_arithmetic_mean_rank_index": 0.35976043325149454, + "adjusted_hits_at_k": 0.07226797575958617, + "adjusted_inverse_harmonic_mean_rank": 0.04216400173804585, + "arithmetic_mean_rank": 4742.5859375, + "count": 59071.0, + "geometric_mean_rank": 1425.0611572265625, + "harmonic_mean_rank": 23.351808771264732, + "hits_at_1": 0.025511672394237444, + "hits_at_10": 0.07289532934942695, + "hits_at_3": 0.046689576949772306, + "hits_at_5": 0.05588190482639536, + "inverse_arithmetic_mean_rank": 0.00021085543266963214, + "inverse_geometric_mean_rank": 0.0007017242605797946, + "inverse_harmonic_mean_rank": 0.04282323061168842, + "inverse_median_rank": 0.0002795638865791261, + "median_absolute_deviation": 4900.000332161015, + "median_rank": 3577.0, + "standard_deviation": 4372.1220703125, + "variance": 19115450.0, + "z_arithmetic_mean_rank": 151.33781136953849, + "z_hits_at_k": 675.2141910937455, + "z_inverse_harmonic_mean_rank": 973.078938619515 + } + } + }, + "times": { + "evaluation": 16.924992561340332, + "training": 115.15393090248108 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json new file mode 100644 index 0000000..a0879b0 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "TransE was used to initialize the emebddigns. Could not find batch size and number of epochs.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15k-237 Dataset with ConvKB as described by Nguyen et al., 2018" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json new file mode 100644 index 0000000..c4959c0 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/fb15k237/nguyen2018/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00629558999520504, + 0.005996476387571496, + 0.005775562846809344, + 0.005573130315875932, + 0.0054310510201602494 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.948802150747234, + "adjusted_arithmetic_mean_rank_index": 0.05120502517776149, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.00032750514360668293, + "arithmetic_mean_rank": 6770.338046775614, + "count": 40876.0, + "geometric_mean_rank": 4738.385571789443, + "harmonic_mean_rank": 961.6156853712986, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770311217713152, + "inverse_geometric_mean_rank": 0.00021104234445453788, + "inverse_harmonic_mean_rank": 0.0010399164814100142, + "inverse_median_rank": 0.00015076134479119553, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.817822927988, + "variance": 17362370.76947034, + "z_arithmetic_mean_rank": 17.908112740115257, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.167120029921174 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9488038203989086, + "adjusted_arithmetic_mean_rank_index": 0.05120335529206732, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.0003275041272765357, + "arithmetic_mean_rank": 6770.3499608572265, + "count": 40876.0, + "geometric_mean_rank": 4738.39669939512, + "harmonic_mean_rank": 961.616624507759, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770285225749026, + "inverse_geometric_mean_rank": 0.00021104184884470628, + "inverse_harmonic_mean_rank": 0.0010399154658041494, + "inverse_median_rank": 0.00015076134479119553, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.819516552299, + "variance": 17362384.88352114, + "z_arithmetic_mean_rank": 17.90752872514463, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.167100891809513 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9488029500029765, + "adjusted_arithmetic_mean_rank_index": 0.05120422580999462, + "adjusted_hits_at_k": 0.0004716904572104511, + "adjusted_inverse_harmonic_mean_rank": 0.000327504635134217, + "arithmetic_mean_rank": 6770.34375, + "count": 40876.0, + "geometric_mean_rank": 4738.38818359375, + "harmonic_mean_rank": 961.6161284897561, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0011742831979645759, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0006605342988550739, + "inverse_arithmetic_mean_rank": 0.00014770298730582, + "inverse_geometric_mean_rank": 0.00021104222105350345, + "inverse_harmonic_mean_rank": 0.0010399159732999085, + "inverse_median_rank": 0.00015076134877745062, + "median_absolute_deviation": 5296.596425611263, + "median_rank": 6633.0, + "standard_deviation": 4166.81884765625, + "variance": 17362378.0, + "z_arithmetic_mean_rank": 17.90783317442731, + "z_hits_at_k": 3.5957121478413656, + "z_inverse_harmonic_mean_rank": 6.16711045507696 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9754310949337636, + "adjusted_arithmetic_mean_rank_index": 0.024572397131068624, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.000123487588625063, + "arithmetic_mean_rank": 6863.75579802329, + "count": 20438.0, + "geometric_mean_rank": 4989.246896709227, + "harmonic_mean_rank": 1181.1213732617991, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.000145692829032174, + "inverse_geometric_mean_rank": 0.00020043105115915853, + "inverse_harmonic_mean_rank": 0.0008466530389153723, + "inverse_median_rank": 0.0001485663348685188, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.871604112257, + "variance": 16874609.115871806, + "z_arithmetic_mean_rank": 6.070271857934642, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.63075169696039 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9754327220262328, + "adjusted_arithmetic_mean_rank_index": 0.02457076980733508, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0001234869334234348, + "arithmetic_mean_rank": 6863.76724728447, + "count": 20438.0, + "geometric_mean_rank": 4989.257177530884, + "harmonic_mean_rank": 1181.122286638947, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.0001456925860059769, + "inverse_geometric_mean_rank": 0.00020043063815260902, + "inverse_harmonic_mean_rank": 0.0008466523841876217, + "inverse_median_rank": 0.0001485663348685188, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.873714924148, + "variance": 16874626.45776472, + "z_arithmetic_mean_rank": 6.069869850047041, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.630743044502447 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9754318669566006, + "adjusted_arithmetic_mean_rank_index": 0.024571624998501296, + "adjusted_hits_at_k": 6.82447497741736e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00012348726092708357, + "arithmetic_mean_rank": 6863.76123046875, + "count": 20438.0, + "geometric_mean_rank": 4989.2529296875, + "harmonic_mean_rank": 1181.1218161462336, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0007828554653097172, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0005871415989822879, + "inverse_arithmetic_mean_rank": 0.00014569271297659725, + "inverse_geometric_mean_rank": 0.00020043080439791083, + "inverse_harmonic_mean_rank": 0.0008466527114544019, + "inverse_median_rank": 0.00014856633788440377, + "median_absolute_deviation": 5204.675088063916, + "median_rank": 6731.0, + "standard_deviation": 4107.87255859375, + "variance": 16874618.0, + "z_arithmetic_mean_rank": 6.070081113231595, + "z_hits_at_k": 0.364825105447272, + "z_inverse_harmonic_mean_rank": 1.6307473694482741 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9229022158891286, + "adjusted_arithmetic_mean_rank_index": 0.07710844225101199, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315183662818469, + "arithmetic_mean_rank": 6676.920295527938, + "count": 20438.0, + "geometric_mean_rank": 4500.13765439255, + "harmonic_mean_rank": 810.9116768895075, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00014976964764275817, + "inverse_geometric_mean_rank": 0.0002222154246823778, + "inverse_harmonic_mean_rank": 0.0012331799239046562, + "inverse_median_rank": 0.00015339776039269826, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.875639960587, + "variance": 17832678.670572538, + "z_arithmetic_mean_rank": 19.091644440437506, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136944512524542 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9229039269348835, + "adjusted_arithmetic_mean_rank_index": 0.0771067309687189, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315169888308492, + "arithmetic_mean_rank": 6676.932674429983, + "count": 20438.0, + "geometric_mean_rank": 4500.149517638199, + "harmonic_mean_rank": 810.9125820357525, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.00014976936997277286, + "inverse_geometric_mean_rank": 0.00022221483888047063, + "inverse_harmonic_mean_rank": 0.001233178547420677, + "inverse_median_rank": 0.00015339776039269826, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.876949482508, + "variance": 17832689.730470695, + "z_arithmetic_mean_rank": 19.091220735949094, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136926016848831 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9229030416322332, + "adjusted_arithmetic_mean_rank_index": 0.07710761639375507, + "adjusted_hits_at_k": 0.0008751266890782155, + "adjusted_inverse_harmonic_mean_rank": 0.0005315176770387331, + "arithmetic_mean_rank": 6676.92626953125, + "count": 20438.0, + "geometric_mean_rank": 4500.14208984375, + "harmonic_mean_rank": 810.9121597190133, + "hits_at_1": 0.00014678539974557198, + "hits_at_10": 0.0015657109306194345, + "hits_at_3": 0.0003914277326548586, + "hits_at_5": 0.0007339269987278598, + "inverse_arithmetic_mean_rank": 0.0001497695193393156, + "inverse_geometric_mean_rank": 0.00022221521066967398, + "inverse_harmonic_mean_rank": 0.0012331792351454151, + "inverse_median_rank": 0.00015339776291511953, + "median_absolute_deviation": 5393.70687092338, + "median_rank": 6519.0, + "standard_deviation": 4222.8759765625, + "variance": 17832682.0, + "z_arithmetic_mean_rank": 19.091439962527602, + "z_hits_at_k": 4.757091931352016, + "z_inverse_harmonic_mean_rank": 7.136935257736431 + } + } + }, + "times": { + "evaluation": 9.182297706604004, + "training": 11.888080596923828 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json new file mode 100644 index 0000000..ed2916a --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Num_epochs varies from the value reported in the paper, and batch_size is not mentioned in the paper. We found out that these values work.", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WordNet Dataset with TransE as described by Bordes et al., 2013" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json new file mode 100644 index 0000000..fc1ce94 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transe/wn18/bordes2013/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.06581889594586139, + 0.06439249604551456, + 0.06347951248280359, + 0.06278877398983285, + 0.06220475568033866 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9305804659508103, + "adjusted_arithmetic_mean_rank_index": 0.06942292661802851, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213703543287622, + "arithmetic_mean_rank": 19042.6849, + "count": 10000.0, + "geometric_mean_rank": 12709.213673151666, + "harmonic_mean_rank": 2194.5404157572866, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.2513603268203e-05, + "inverse_geometric_mean_rank": 7.868307400579073e-05, + "inverse_harmonic_mean_rank": 0.0004556762740935543, + "inverse_median_rank": 5.388221348132981e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.146736336086, + "variance": 145664302.943212, + "z_arithmetic_mean_rank": 12.024094745989629, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.874811972549873 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9305812576145713, + "adjusted_arithmetic_mean_rank_index": 0.06942213491557847, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213673413773629, + "arithmetic_mean_rank": 19042.7011, + "count": 10000.0, + "geometric_mean_rank": 12709.230306238696, + "harmonic_mean_rank": 2194.5418664009517, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.251355859384886e-05, + "inverse_geometric_mean_rank": 7.868297103004899e-05, + "inverse_harmonic_mean_rank": 0.0004556759728808454, + "inverse_median_rank": 5.388221348132981e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.149000246818, + "variance": 145664357.5901588, + "z_arithmetic_mean_rank": 12.023957622625176, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.87480721697189 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9305809747902494, + "adjusted_arithmetic_mean_rank_index": 0.06942241775372227, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": 0.00018213688474468513, + "arithmetic_mean_rank": 19042.6953125, + "count": 10000.0, + "geometric_mean_rank": 12709.228515625, + "harmonic_mean_rank": 2194.541126665028, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 5.251357288216241e-05, + "inverse_geometric_mean_rank": 7.868298416724429e-05, + "inverse_harmonic_mean_rank": 0.00045567612344658983, + "inverse_median_rank": 5.388221325119957e-05, + "median_absolute_deviation": 15599.940543115943, + "median_rank": 18559.0, + "standard_deviation": 12069.1474609375, + "variance": 145664320.0, + "z_arithmetic_mean_rank": 12.024006610370321, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": 2.874809594119726 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9428129206391556, + "adjusted_arithmetic_mean_rank_index": 0.05718987418803789, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0002227443657399582, + "arithmetic_mean_rank": 19292.5532, + "count": 5000.0, + "geometric_mean_rank": 12902.47875932424, + "harmonic_mean_rank": 2014.9983994265895, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833471165444296e-05, + "inverse_geometric_mean_rank": 7.750448721160107e-05, + "inverse_harmonic_mean_rank": 0.000496278309841125, + "inverse_median_rank": 5.309405612041732e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.543824106264, + "variance": 147053065.11796975, + "z_arithmetic_mean_rank": 7.0041200036088185, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859812309410803 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9428137709643899, + "adjusted_arithmetic_mean_rank_index": 0.0571890238212468, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.0002227442098313817, + "arithmetic_mean_rank": 19292.5706, + "count": 5000.0, + "geometric_mean_rank": 12902.494552544491, + "harmonic_mean_rank": 2014.9990322764907, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833424416754504e-05, + "inverse_geometric_mean_rank": 7.750439234270328e-05, + "inverse_harmonic_mean_rank": 0.0004962781539752044, + "inverse_median_rank": 5.309405612041732e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.546118389013, + "variance": 147053120.76141563, + "z_arithmetic_mean_rank": 7.004015858056192, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859794908934996 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9428133751233326, + "adjusted_arithmetic_mean_rank_index": 0.05718941968164948, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00022274428777780813, + "arithmetic_mean_rank": 19292.5625, + "count": 5000.0, + "geometric_mean_rank": 12902.4775390625, + "harmonic_mean_rank": 2014.9986899134094, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 5.1833445468219e-05, + "inverse_geometric_mean_rank": 7.7504497312475e-05, + "inverse_harmonic_mean_rank": 0.000496278231900305, + "inverse_median_rank": 5.3094056056579575e-05, + "median_absolute_deviation": 15824.554779219543, + "median_rank": 18834.5, + "standard_deviation": 12126.5458984375, + "variance": 147053104.0, + "z_arithmetic_mean_rank": 7.004064339606558, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 2.4859803608295463 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9183485784203501, + "adjusted_arithmetic_mean_rank_index": 0.08165541183470171, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00014152970559830536, + "arithmetic_mean_rank": 18792.8166, + "count": 5000.0, + "geometric_mean_rank": 12518.843487581595, + "harmonic_mean_rank": 2409.2075769020707, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321182137221517e-05, + "inverse_geometric_mean_rank": 7.987958320526789e-05, + "inverse_harmonic_mean_rank": 0.0004150742383459837, + "inverse_median_rank": 5.4580683895969216e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.276376702497, + "variance": 144150672.43376443, + "z_arithmetic_mean_rank": 10.000448652139795, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.579606283327427 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9183493114253579, + "adjusted_arithmetic_mean_rank_index": 0.08165467879387245, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.00014152925891660332, + "arithmetic_mean_rank": 18792.8316, + "count": 5000.0, + "geometric_mean_rank": 12518.860931831336, + "harmonic_mean_rank": 2409.210168861732, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321177889978006e-05, + "inverse_geometric_mean_rank": 7.98794718980646e-05, + "inverse_harmonic_mean_rank": 0.00041507379178648627, + "inverse_median_rank": 5.4580683895969216e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.278561021372, + "variance": 144150724.88484144, + "z_arithmetic_mean_rank": 10.000358875638653, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5796012979342877 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9183490461691707, + "adjusted_arithmetic_mean_rank_index": 0.08165494406302254, + "adjusted_hits_at_k": -0.00024440126566051376, + "adjusted_inverse_harmonic_mean_rank": 0.0001415294821840739, + "arithmetic_mean_rank": 18792.826171875, + "count": 5000.0, + "geometric_mean_rank": 12518.8505859375, + "harmonic_mean_rank": 2409.2088752176546, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 5.321179560269229e-05, + "inverse_geometric_mean_rank": 7.987953722476959e-05, + "inverse_harmonic_mean_rank": 0.00041507401499287465, + "inverse_median_rank": 5.458068335428834e-05, + "median_absolute_deviation": 15256.718129531897, + "median_rank": 18321.5, + "standard_deviation": 12006.27734375, + "variance": 144150704.0, + "z_arithmetic_mean_rank": 10.000391363510005, + "z_hits_at_k": -1.1054439510948424, + "z_inverse_harmonic_mean_rank": 1.5796037898118622 + } + } + }, + "times": { + "evaluation": 4.560383558273315, + "training": 34.424185276031494 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json new file mode 100644 index 0000000..80f6ec5 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json new file mode 100644 index 0000000..9b4f3bc --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/fb15k/wang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 13.100006539226563, + 0.20497526620080622, + 0.0027243127839496222, + 0.0026737430891539487, + 0.0026241765188342097 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9938447796648451, + "adjusted_arithmetic_mean_rank_index": 0.0061560537149736305, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251325085160447, + "arithmetic_mean_rank": 7341.384698075198, + "count": 118142.0, + "geometric_mean_rank": 5285.389111711265, + "harmonic_mean_rank": 660.1936951360921, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.0001362140850978951, + "inverse_geometric_mean_rank": 0.00018920082871178186, + "inverse_harmonic_mean_rank": 0.0015147069827648996, + "inverse_median_rank": 0.0001367053998632946, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.364307754461, + "variance": 18786713.952335812, + "z_arithmetic_mean_rank": 3.6617272028031493, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.889311668222454 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9938462624237342, + "adjusted_arithmetic_mean_rank_index": 0.006154570755327993, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251309310513212, + "arithmetic_mean_rank": 7341.395650996259, + "count": 118142.0, + "geometric_mean_rank": 5285.401584977365, + "harmonic_mean_rank": 660.1943822093159, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.0001362138818746672, + "inverse_geometric_mean_rank": 0.00018920038220790797, + "inverse_harmonic_mean_rank": 0.0015147054063888536, + "inverse_median_rank": 0.0001367053998632946, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.364337844292, + "variance": 18786714.213176392, + "z_arithmetic_mean_rank": 3.6608451127618764, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.889260262005685 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9938455159247913, + "adjusted_arithmetic_mean_rank_index": 0.006155317355342249, + "adjusted_hits_at_k": 0.0013964195403419067, + "adjusted_inverse_harmonic_mean_rank": 0.0008251317188252537, + "arithmetic_mean_rank": 7341.39013671875, + "count": 118142.0, + "geometric_mean_rank": 5285.39404296875, + "harmonic_mean_rank": 660.1941329808002, + "hits_at_1": 0.0006009717120075841, + "hits_at_10": 0.0020737756259416634, + "hits_at_3": 0.0011257639112254745, + "hits_at_5": 0.0014897326945540112, + "inverse_arithmetic_mean_rank": 0.00013621398829855025, + "inverse_geometric_mean_rank": 0.00018920065485872328, + "inverse_harmonic_mean_rank": 0.0015147061936191088, + "inverse_median_rank": 0.0001367054064758122, + "median_absolute_deviation": 5602.75378373267, + "median_rank": 7315.0, + "standard_deviation": 4334.3642578125, + "variance": 18786714.0, + "z_arithmetic_mean_rank": 3.6612892033609796, + "z_hits_at_k": 18.42294929802362, + "z_inverse_harmonic_mean_rank": 26.8892859338809 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.98490719405311, + "adjusted_arithmetic_mean_rank_index": 0.015094854999328944, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.0007080517688152044, + "arithmetic_mean_rank": 7255.564083899037, + "count": 59071.0, + "geometric_mean_rank": 5224.860477100204, + "harmonic_mean_rank": 714.4891053150517, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001378252591303162, + "inverse_geometric_mean_rank": 0.00019139267055701356, + "inverse_harmonic_mean_rank": 0.0013996014670637324, + "inverse_median_rank": 0.00013900472616068947, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.428105454356, + "variance": 18502283.746392652, + "z_arithmetic_mean_rank": 6.3479662286523535, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.290813338188357 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9849086854543545, + "adjusted_arithmetic_mean_rank_index": 0.015093363395606518, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.0007080504080451104, + "arithmetic_mean_rank": 7255.575070677659, + "count": 59071.0, + "geometric_mean_rank": 5224.8727660877985, + "harmonic_mean_rank": 714.4897995008857, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001378250504279603, + "inverse_geometric_mean_rank": 0.00019139222039827103, + "inverse_harmonic_mean_rank": 0.0013996001072353454, + "inverse_median_rank": 0.00013900472616068947, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.428409444852, + "variance": 18502286.361579273, + "z_arithmetic_mean_rank": 6.347338952003668, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.29078202967029 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9849079732733217, + "adjusted_arithmetic_mean_rank_index": 0.015094075673327434, + "adjusted_hits_at_k": 0.0010978852313046777, + "adjusted_inverse_harmonic_mean_rank": 0.000708051087890621, + "arithmetic_mean_rank": 7255.56982421875, + "count": 59071.0, + "geometric_mean_rank": 5224.86328125, + "harmonic_mean_rank": 714.4894411794969, + "hits_at_1": 0.0005417209798378223, + "hits_at_10": 0.0017775219650928544, + "hits_at_3": 0.0009987980565759847, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.00013782514724880457, + "inverse_geometric_mean_rank": 0.0001913925661938265, + "inverse_harmonic_mean_rank": 0.0013996007866103758, + "inverse_median_rank": 0.00013900472549721599, + "median_absolute_deviation": 5513.797650622334, + "median_rank": 7194.0, + "standard_deviation": 4301.42822265625, + "variance": 18502286.0, + "z_arithmetic_mean_rank": 6.347638492139595, + "z_hits_at_k": 10.226329805638587, + "z_inverse_harmonic_mean_rank": 16.290797671515698 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.002733849824362, + "adjusted_arithmetic_mean_rank_index": -0.0027342189664276706, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.0009422128039963093, + "arithmetic_mean_rank": 7427.205312251359, + "count": 59071.0, + "geometric_mean_rank": 5346.618954636671, + "harmonic_mean_rank": 613.5675121777332, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013464014497491746, + "inverse_geometric_mean_rank": 0.00018703408798803297, + "inverse_harmonic_mean_rank": 0.0016298124984660664, + "inverse_median_rank": 0.00013466199838405601, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365254207696, + "variance": 19056413.80264382, + "z_arithmetic_mean_rank": -1.150184055662919, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.744791705554526 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0027353239878083, + "adjusted_arithmetic_mean_rank_index": -0.002735693328925093, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.000942211009837779, + "arithmetic_mean_rank": 7427.216231314858, + "count": 59071.0, + "geometric_mean_rank": 5346.631614801664, + "harmonic_mean_rank": 613.5681871516629, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013463994703476778, + "inverse_geometric_mean_rank": 0.00018703364511435402, + "inverse_harmonic_mean_rank": 0.0016298107055423624, + "inverse_median_rank": 0.00013466199838405601, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365015753101, + "variance": 19056411.720761076, + "z_arithmetic_mean_rank": -1.1508042650380617, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.74475029919349 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0027346092792295, + "adjusted_arithmetic_mean_rank_index": -0.0027349785238417468, + "adjusted_hits_at_k": 0.0016949526063758595, + "adjusted_inverse_harmonic_mean_rank": 0.0009422119055397235, + "arithmetic_mean_rank": 7427.2109375, + "count": 59071.0, + "geometric_mean_rank": 5346.62548828125, + "harmonic_mean_rank": 613.5678263684528, + "hits_at_1": 0.0006602224441773459, + "hits_at_10": 0.0023700292867904725, + "hits_at_3": 0.001252729765874964, + "hits_at_5": 0.0016928780619931945, + "inverse_arithmetic_mean_rank": 0.00013464003859553486, + "inverse_geometric_mean_rank": 0.00018703386012930423, + "inverse_harmonic_mean_rank": 0.0016298116006278416, + "inverse_median_rank": 0.00013466199743561447, + "median_absolute_deviation": 5699.122927935534, + "median_rank": 7426.0, + "standard_deviation": 4365.365234375, + "variance": 19056412.0, + "z_arithmetic_mean_rank": -1.1505035731696012, + "z_hits_at_k": 15.83628212949499, + "z_inverse_harmonic_mean_rank": 21.74477097058761 + } + } + }, + "times": { + "evaluation": 29.358933925628662, + "training": 10.431986570358276 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json new file mode 100644 index 0000000..3d243a1 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransH as described by Wang et al., 2014" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json new file mode 100644 index 0000000..28f9432 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transh/wn18/wang2014/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 20.45071284740388, + 14.57502553498077, + 10.225484043371134, + 7.005688575639661, + 4.622200210598691 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0126994693360527, + "adjusted_arithmetic_mean_rank_index": -0.012700089965743233, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.241758676361911e-05, + "arithmetic_mean_rank": 20723.1052, + "count": 10000.0, + "geometric_mean_rank": 15394.675497871895, + "harmonic_mean_rank": 3067.5569736687403, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255316486064065e-05, + "inverse_geometric_mean_rank": 6.495752379699309e-05, + "inverse_harmonic_mean_rank": 0.00032599231524753683, + "inverse_median_rank": 4.8173037550882774e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.285208500767, + "variance": 138374878.89653295, + "z_arithmetic_mean_rank": -2.1996636049485403, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273479671066601 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0127007105866408, + "adjusted_arithmetic_mean_rank_index": -0.012701331276992134, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.241714190367465e-05, + "arithmetic_mean_rank": 20723.1306, + "count": 10000.0, + "geometric_mean_rank": 15394.705594314819, + "harmonic_mean_rank": 3067.5611586193463, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255257340317104e-05, + "inverse_geometric_mean_rank": 6.495739680590544e-05, + "inverse_harmonic_mean_rank": 0.00032599187050930123, + "inverse_median_rank": 4.8173037550882774e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.28695570858, + "variance": 138374920.00234362, + "z_arithmetic_mean_rank": -2.1998786008410036, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273409455325398 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0127001505883748, + "adjusted_arithmetic_mean_rank_index": -0.01270077125135849, + "adjusted_hits_at_k": -4.435809332781619e-05, + "adjusted_inverse_harmonic_mean_rank": 5.2417364271439425e-05, + "arithmetic_mean_rank": 20723.119140625, + "count": 10000.0, + "geometric_mean_rank": 15394.67578125, + "harmonic_mean_rank": 3067.559378276964, + "hits_at_1": 0.0001, + "hits_at_10": 0.0002, + "hits_at_3": 0.0001, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8255282308673486e-05, + "inverse_geometric_mean_rank": 6.495752313639969e-05, + "inverse_harmonic_mean_rank": 0.0003259920928162286, + "inverse_median_rank": 4.8173038521781564e-05, + "median_absolute_deviation": 15022.466979008012, + "median_rank": 20758.5, + "standard_deviation": 11763.2861328125, + "variance": 138374896.0, + "z_arithmetic_mean_rank": -2.199781604047473, + "z_hits_at_k": -0.28373706548148925, + "z_inverse_harmonic_mean_rank": 0.8273444553377312 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0164361612514943, + "adjusted_arithmetic_mean_rank_index": -0.016436964513890873, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457574579320083, + "arithmetic_mean_rank": 20799.0878, + "count": 5000.0, + "geometric_mean_rank": 15386.013539591448, + "harmonic_mean_rank": 2391.594541454187, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.8079031619838636e-05, + "inverse_geometric_mean_rank": 6.499409333202456e-05, + "inverse_harmonic_mean_rank": 0.0004181310764290168, + "inverse_median_rank": 4.781257470714798e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.276661378286, + "variance": 138751358.66529116, + "z_arithmetic_mean_rank": -2.013056919338908, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.6135653501144027 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.0164373341138861, + "adjusted_arithmetic_mean_rank_index": -0.01643813743360245, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457539979149638, + "arithmetic_mean_rank": 20799.1118, + "count": 5000.0, + "geometric_mean_rank": 15386.042101334648, + "harmonic_mean_rank": 2391.596519948779, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.80789761416639e-05, + "inverse_geometric_mean_rank": 6.499397268081413e-05, + "inverse_harmonic_mean_rank": 0.00041813073052197663, + "inverse_median_rank": 4.781257470714798e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.275458639244, + "variance": 138751330.33050075, + "z_arithmetic_mean_rank": -2.013200568377002, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.6135614884959946 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0164368338147722, + "adjusted_arithmetic_mean_rank_index": -0.016437637110037873, + "adjusted_hits_at_k": -4.436380124831586e-05, + "adjusted_inverse_harmonic_mean_rank": 0.00014457557275534118, + "arithmetic_mean_rank": 20799.1015625, + "count": 5000.0, + "geometric_mean_rank": 15386.03125, + "harmonic_mean_rank": 2391.5954229298945, + "hits_at_1": 0.0002, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.807900040759705e-05, + "inverse_geometric_mean_rank": 6.499401933979243e-05, + "inverse_harmonic_mean_rank": 0.0004181309034384994, + "inverse_median_rank": 4.781257302965969e-05, + "median_absolute_deviation": 15121.801327647887, + "median_rank": 20915.0, + "standard_deviation": 11779.2763671875, + "variance": 138751344.0, + "z_arithmetic_mean_rank": -2.013139293084194, + "z_hits_at_k": -0.20065587657052963, + "z_inverse_harmonic_mean_rank": 1.61356341889217 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 1.0089629506723694, + "adjusted_arithmetic_mean_rank_index": -0.008963388686288987, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.974057119360112e-05, + "arithmetic_mean_rank": 20647.1226, + "count": 5000.0, + "geometric_mean_rank": 15403.342332628104, + "harmonic_mean_rank": 4276.180466846908, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843289882920539e-05, + "inverse_geometric_mean_rank": 6.49209748381526e-05, + "inverse_harmonic_mean_rank": 0.00023385355406605694, + "inverse_median_rank": 4.848602390360979e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20624.5, + "standard_deviation": 11746.780512837091, + "variance": 137986852.41676924, + "z_arithmetic_mean_rank": -1.0977583272479166, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.44354261668996875 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 1.008964260307983, + "adjusted_arithmetic_mean_rank_index": -0.008964698385903747, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.9741114911784364e-05, + "arithmetic_mean_rank": 20647.1494, + "count": 5000.0, + "geometric_mean_rank": 15403.373965489793, + "harmonic_mean_rank": 4276.190406428094, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843283596330252e-05, + "inverse_geometric_mean_rank": 6.492084151436119e-05, + "inverse_harmonic_mean_rank": 0.00023385301049662588, + "inverse_median_rank": 4.848484848484849e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20625.0, + "standard_deviation": 11746.78523633933, + "variance": 137986963.38867965, + "z_arithmetic_mean_rank": -1.0979187279299225, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.4435486851026389 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 1.0089636406133329, + "adjusted_arithmetic_mean_rank_index": -0.008964078660969488, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -3.9740843140100186e-05, + "arithmetic_mean_rank": 20647.13671875, + "count": 5000.0, + "geometric_mean_rank": 15403.35546875, + "harmonic_mean_rank": 4276.1856863784815, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.843286660616286e-05, + "inverse_geometric_mean_rank": 6.492091779364273e-05, + "inverse_harmonic_mean_rank": 0.00023385328219395786, + "inverse_median_rank": 4.8485435399925336e-05, + "median_absolute_deviation": 14937.21735144394, + "median_rank": 20624.75, + "standard_deviation": 11746.783203125, + "variance": 137986912.0, + "z_arithmetic_mean_rank": -1.0978428293796014, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.4435456518718546 + } + } + }, + "times": { + "evaluation": 8.131263971328735, + "training": 2.695167064666748 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json new file mode 100644 index 0000000..67eab9e --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json new file mode 100644 index 0000000..4aa2bbc --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/fb15k/li2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00020909712704460964, + 0.00020904143676822714, + 0.00020899788573421359, + 0.00020894385538492384, + 0.000208891014557524 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9787776633371456, + "adjusted_arithmetic_mean_rank_index": 0.021225210039617082, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.00030412194368257097, + "arithmetic_mean_rank": 7230.086133635794, + "count": 118142.0, + "geometric_mean_rank": 5227.108734838778, + "harmonic_mean_rank": 1005.9795523048979, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.0001383109386965394, + "inverse_geometric_mean_rank": 0.00019131034970345674, + "inverse_harmonic_mean_rank": 0.0009940559902125272, + "inverse_median_rank": 0.0001398014818957081, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.730639893114, + "variance": 18102732.818045266, + "z_arithmetic_mean_rank": 12.625121967053815, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910686640541236 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9787875751334283, + "adjusted_arithmetic_mean_rank_index": 0.02121529690133661, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.000304114239486995, + "arithmetic_mean_rank": 7230.159350611975, + "count": 118142.0, + "geometric_mean_rank": 5227.181095160295, + "harmonic_mean_rank": 1005.9873435907807, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.0001383095380761363, + "inverse_geometric_mean_rank": 0.00019130770137768384, + "inverse_harmonic_mean_rank": 0.0009940482913339552, + "inverse_median_rank": 0.0001398014818957081, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.740975209824, + "variance": 18102820.766129445, + "z_arithmetic_mean_rank": 12.61922546098242, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910435577210379 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9787826604911695, + "adjusted_arithmetic_mean_rank_index": 0.021220212209008493, + "adjusted_hits_at_k": 0.0017860457793568458, + "adjusted_inverse_harmonic_mean_rank": 0.00030411808652413067, + "arithmetic_mean_rank": 7230.123046875, + "count": 118142.0, + "geometric_mean_rank": 5227.14111328125, + "harmonic_mean_rank": 1005.9834466369628, + "hits_at_1": 5.9250732169761814e-05, + "hits_at_10": 0.0024631375802000983, + "hits_at_3": 0.00017775219650928545, + "hits_at_5": 0.0007702595182069035, + "inverse_arithmetic_mean_rank": 0.00013831023534294218, + "inverse_geometric_mean_rank": 0.00019130916916765273, + "inverse_harmonic_mean_rank": 0.0009940521357160814, + "inverse_median_rank": 0.00013980148651171476, + "median_absolute_deviation": 5399.637279797403, + "median_rank": 7153.0, + "standard_deviation": 4254.73583984375, + "variance": 18102778.0, + "z_arithmetic_mean_rank": 12.62214917098322, + "z_hits_at_k": 23.563284447440346, + "z_inverse_harmonic_mean_rank": 9.910560943959922 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9770256831014105, + "adjusted_arithmetic_mean_rank_index": 0.02297743597268609, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.435030430268328e-06, + "arithmetic_mean_rank": 7197.5029710009985, + "count": 59071.0, + "geometric_mean_rank": 5222.128776966808, + "harmonic_mean_rank": 1427.6151482423568, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.00013893707359747352, + "inverse_geometric_mean_rank": 0.0001914927882304799, + "inverse_harmonic_mean_rank": 0.0007004688912352706, + "inverse_median_rank": 0.00014072614691809738, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.108553400409, + "variance": 18097439.584114514, + "z_arithmetic_mean_rank": 9.66289424986968, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.19407268266750854 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9770355943703901, + "adjusted_arithmetic_mean_rank_index": 0.022967523358117692, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.425479968338784e-06, + "arithmetic_mean_rank": 7197.575984831813, + "count": 59071.0, + "geometric_mean_rank": 5222.203535726582, + "harmonic_mean_rank": 1427.6345996903804, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.0001389356641885271, + "inverse_geometric_mean_rank": 0.0001914900469042838, + "inverse_harmonic_mean_rank": 0.0007004593473826398, + "inverse_median_rank": 0.00014072614691809738, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.11876984047, + "variance": 18097526.507908992, + "z_arithmetic_mean_rank": 9.658725614760556, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.19385294620268745 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9770305823451633, + "adjusted_arithmetic_mean_rank_index": 0.02297253606379468, + "adjusted_hits_at_k": -0.0002065183714017893, + "adjusted_inverse_harmonic_mean_rank": 8.430247222894309e-06, + "arithmetic_mean_rank": 7197.5390625, + "count": 59071.0, + "geometric_mean_rank": 5222.1630859375, + "harmonic_mean_rank": 1427.6249653122286, + "hits_at_1": 6.771512247972779e-05, + "hits_at_10": 0.0004740058573580945, + "hits_at_3": 0.0001523590255793875, + "hits_at_5": 0.0002539317092989792, + "inverse_arithmetic_mean_rank": 0.00013893637515138835, + "inverse_geometric_mean_rank": 0.00019149153376929462, + "inverse_harmonic_mean_rank": 0.000700464111338066, + "inverse_median_rank": 0.00014072614430915564, + "median_absolute_deviation": 5424.841517511997, + "median_rank": 7106.0, + "standard_deviation": 4254.11376953125, + "variance": 18097482.0, + "z_arithmetic_mean_rank": 9.660833650005259, + "z_hits_at_k": -1.923630008546831, + "z_inverse_harmonic_mean_rank": 0.1939626309143474 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9805201333855541, + "adjusted_arithmetic_mean_rank_index": 0.019482496911887748, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998077350576884, + "arithmetic_mean_rank": 7262.669296270589, + "count": 59071.0, + "geometric_mean_rank": 5232.093441728144, + "harmonic_mean_rank": 776.6127185361778, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001376904219655856, + "inverse_geometric_mean_rank": 0.00019112808498880768, + "inverse_harmonic_mean_rank": 0.0012876430891897838, + "inverse_median_rank": 0.0001389274798555154, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.103139408192, + "variance": 18105902.72700145, + "z_arithmetic_mean_rank": 8.195560629086003, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.84262048540465 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9805300457062778, + "adjusted_arithmetic_mean_rank_index": 0.019472583252738485, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998018771214614, + "arithmetic_mean_rank": 7262.742716392138, + "count": 59071.0, + "geometric_mean_rank": 5232.1633989701495, + "harmonic_mean_rank": 776.616249201938, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.00013768903003310063, + "inverse_geometric_mean_rank": 0.00019112552948878292, + "inverse_harmonic_mean_rank": 0.0012876372352852705, + "inverse_median_rank": 0.0001389274798555154, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.113591068842, + "variance": 18105991.672898777, + "z_arithmetic_mean_rank": 8.191390322015991, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.842485293436821 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9805250960743923, + "adjusted_arithmetic_mean_rank_index": 0.019477533552955117, + "adjusted_hits_at_k": 0.003778601633703001, + "adjusted_inverse_harmonic_mean_rank": 0.0005998048039446685, + "arithmetic_mean_rank": 7262.7060546875, + "count": 59071.0, + "geometric_mean_rank": 5232.12841796875, + "harmonic_mean_rank": 776.6144552693261, + "hits_at_1": 5.078634185979584e-05, + "hits_at_10": 0.004452269303042102, + "hits_at_3": 0.00020314536743918336, + "hits_at_5": 0.001286587327114828, + "inverse_arithmetic_mean_rank": 0.0001376897271256894, + "inverse_geometric_mean_rank": 0.00019112680456601083, + "inverse_harmonic_mean_rank": 0.001287640160094097, + "inverse_median_rank": 0.00013892748393118382, + "median_absolute_deviation": 5369.98523542729, + "median_rank": 7198.0, + "standard_deviation": 4255.1083984375, + "variance": 18105946.0, + "z_arithmetic_mean_rank": 8.193472728893369, + "z_hits_at_k": 35.30423287423883, + "z_inverse_harmonic_mean_rank": 13.842552839919668 + } + } + }, + "times": { + "evaluation": 28.336533308029175, + "training": 11.485997915267944 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json new file mode 100644 index 0000000..512d190 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/metadata.json @@ -0,0 +1,19 @@ +{ + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json new file mode 100644 index 0000000..7c3c728 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/transr/wn18/li2015/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002799568278957982, + 0.002799524831899284, + 0.002799194914067409, + 0.0027994709019207786, + 0.002799336377016278 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9900345170286631, + "adjusted_arithmetic_mean_rank_index": 0.009965969989687906, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.224514997842025e-05, + "arithmetic_mean_rank": 20259.307, + "count": 10000.0, + "geometric_mean_rank": 14914.15475604489, + "harmonic_mean_rank": 4322.3528231180835, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.9360029935870956e-05, + "inverse_geometric_mean_rank": 6.705039718021483e-05, + "inverse_harmonic_mean_rank": 0.00023135547719554608, + "inverse_median_rank": 4.975371909050202e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.0, + "standard_deviation": 11779.53928541991, + "variance": 138757545.776751, + "z_arithmetic_mean_rank": 1.726112297901551, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.6667884027621884 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9900373269463334, + "adjusted_arithmetic_mean_rank_index": 0.009963159934695431, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.224635404830653e-05, + "arithmetic_mean_rank": 20259.3645, + "count": 10000.0, + "geometric_mean_rank": 14914.222346617338, + "harmonic_mean_rank": 4322.375312397938, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.935988984254664e-05, + "inverse_geometric_mean_rank": 6.705009331088642e-05, + "inverse_harmonic_mean_rank": 0.00023135427345508016, + "inverse_median_rank": 4.9752481405010074e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.5, + "standard_deviation": 11779.540283187614, + "variance": 138757569.28323975, + "z_arithmetic_mean_rank": 1.7256255946016665, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.6668074075434873 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9900358357047002, + "adjusted_arithmetic_mean_rank_index": 0.009964651249206313, + "adjusted_hits_at_k": -0.00014438253402528848, + "adjusted_inverse_harmonic_mean_rank": -4.2245752311731276e-05, + "arithmetic_mean_rank": 20259.333984375, + "count": 10000.0, + "geometric_mean_rank": 14914.1748046875, + "harmonic_mean_rank": 4322.363974171985, + "hits_at_1": 0.0, + "hits_at_10": 0.0001, + "hits_at_3": 0.0001, + "hits_at_5": 0.0001, + "inverse_arithmetic_mean_rank": 4.9359965487383306e-05, + "inverse_geometric_mean_rank": 6.705030682496727e-05, + "inverse_harmonic_mean_rank": 0.00023135487502702685, + "inverse_median_rank": 4.9753100029192865e-05, + "median_absolute_deviation": 15158.125082001274, + "median_rank": 20099.25, + "standard_deviation": 11779.5390625, + "variance": 138757552.0, + "z_arithmetic_mean_rank": 1.7258838912170802, + "z_hits_at_k": -0.9235445763722028, + "z_inverse_harmonic_mean_rank": -0.666797909862213 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9904344373197072, + "adjusted_arithmetic_mean_rank_index": 0.009566030165168882, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842764639345146e-05, + "arithmetic_mean_rank": 20267.0208, + "count": 5000.0, + "geometric_mean_rank": 15026.007280375668, + "harmonic_mean_rank": 4873.627784794362, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.934124308985759e-05, + "inverse_geometric_mean_rank": 6.655127881549907e-05, + "inverse_harmonic_mean_rank": 0.00020518596088112915, + "inverse_median_rank": 5.0261359067149175e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.709523334077, + "variance": 135669137.13996735, + "z_arithmetic_mean_rank": 1.1715644453891514, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.7636998765220726 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9904372521894481, + "adjusted_arithmetic_mean_rank_index": 0.009563215157860516, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842836303762437e-05, + "arithmetic_mean_rank": 20267.0784, + "count": 5000.0, + "geometric_mean_rank": 15026.073723288115, + "harmonic_mean_rank": 4873.644802106642, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.93411028596998e-05, + "inverse_geometric_mean_rank": 6.65509845363099e-05, + "inverse_harmonic_mean_rank": 0.00020518524443302643, + "inverse_median_rank": 5.0261359067149175e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.705968913082, + "variance": 135669054.33825344, + "z_arithmetic_mean_rank": 1.1712196876977146, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.763707874766864 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9904358070336128, + "adjusted_arithmetic_mean_rank_index": 0.00956466038432302, + "adjusted_hits_at_k": -0.0002444126837850729, + "adjusted_inverse_harmonic_mean_rank": -6.842800477898023e-05, + "arithmetic_mean_rank": 20267.048828125, + "count": 5000.0, + "geometric_mean_rank": 15026.03125, + "harmonic_mean_rank": 4873.636259270741, + "hits_at_1": 0.0, + "hits_at_10": 0.0, + "hits_at_3": 0.0, + "hits_at_5": 0.0, + "inverse_arithmetic_mean_rank": 4.934117532684468e-05, + "inverse_geometric_mean_rank": 6.655117613263428e-05, + "inverse_harmonic_mean_rank": 0.00020518560259365283, + "inverse_median_rank": 5.026135841035284e-05, + "median_absolute_deviation": 14680.72716764247, + "median_rank": 19896.0, + "standard_deviation": 11647.7080078125, + "variance": 135669104.0, + "z_arithmetic_mean_rank": 1.171396686505986, + "z_hits_at_k": -1.1054697733258656, + "z_inverse_harmonic_mean_rank": -0.7637038763525288 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.9896346152799272, + "adjusted_arithmetic_mean_rank_index": 0.010365891270086691, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.606265386805113e-05, + "arithmetic_mean_rank": 20251.5932, + "count": 5000.0, + "geometric_mean_rank": 14803.134853911464, + "harmonic_mean_rank": 3883.1182417302443, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937883109364453e-05, + "inverse_geometric_mean_rank": 6.755325881097192e-05, + "inverse_harmonic_mean_rank": 0.00025752499350996296, + "inverse_median_rank": 4.927807618390578e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.904928592578, + "variance": 141845835.40811378, + "z_arithmetic_mean_rank": 1.2695247142959403, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.17927451251047488 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.9896374202457566, + "adjusted_arithmetic_mean_rank_index": 0.01036308616718018, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.606434536364506e-05, + "arithmetic_mean_rank": 20251.6506, + "count": 5000.0, + "geometric_mean_rank": 14803.203571376178, + "harmonic_mean_rank": 3883.143740317606, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937869113740289e-05, + "inverse_geometric_mean_rank": 6.755294522420967e-05, + "inverse_harmonic_mean_rank": 0.0002575233024771339, + "inverse_median_rank": 4.927807618390578e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.910378324415, + "variance": 141845965.21971962, + "z_arithmetic_mean_rank": 1.2691811695516293, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.1792933912119965 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.9896360738071831, + "adjusted_arithmetic_mean_rank_index": 0.010364432671553403, + "adjusted_hits_at_k": -4.4352385407381626e-05, + "adjusted_inverse_harmonic_mean_rank": -1.6063500149141475e-05, + "arithmetic_mean_rank": 20251.623046875, + "count": 5000.0, + "geometric_mean_rank": 14803.1787109375, + "harmonic_mean_rank": 3883.1308616537235, + "hits_at_1": 0.0, + "hits_at_10": 0.0002, + "hits_at_3": 0.0002, + "hits_at_5": 0.0002, + "inverse_arithmetic_mean_rank": 4.937875928590074e-05, + "inverse_geometric_mean_rank": 6.755306094419211e-05, + "inverse_harmonic_mean_rank": 0.00025752414746040096, + "inverse_median_rank": 4.927807458443567e-05, + "median_absolute_deviation": 15595.492736460426, + "median_rank": 20293.0, + "standard_deviation": 11909.9072265625, + "variance": 141845888.0, + "z_arithmetic_mean_rank": 1.2693460777621368, + "z_hits_at_k": -0.20060892906062588, + "z_inverse_harmonic_mean_rank": -0.17928395781329742 + } + } + }, + "times": { + "evaluation": 7.977590084075928, + "training": 3.7082855701446533 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json new file mode 100644 index 0000000..0749f69 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json new file mode 100644 index 0000000..7df7ae6 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.00021869831097327203, + 1.6626344659595006e-05, + 1.4145980447915058e-05, + 1.1566674962732687e-05, + 1.0039698529621776e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08082387386970409, + "adjusted_arithmetic_mean_rank_index": 0.9193005770502545, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364597304408337, + "arithmetic_mean_rank": 597.0340268490461, + "count": 118142.0, + "geometric_mean_rank": 48.53253129682446, + "harmonic_mean_rank": 4.460666021875165, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.0016749464101362512, + "inverse_geometric_mean_rank": 0.020604736107498915, + "inverse_harmonic_mean_rank": 0.2241817690667687, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.957313717163, + "variance": 2030503.3459160337, + "z_arithmetic_mean_rank": 546.8158801717057, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.146098304282 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08082393460094144, + "adjusted_arithmetic_mean_rank_index": 0.9193005163107946, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364585976837215, + "arithmetic_mean_rank": 597.0344754617325, + "count": 118142.0, + "geometric_mean_rank": 48.53260829072401, + "harmonic_mean_rank": 4.46066827422893, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.001674945151578765, + "inverse_geometric_mean_rank": 0.02060470341939419, + "inverse_harmonic_mean_rank": 0.22418165586923403, + "inverse_median_rank": 0.01818181818181818, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.9584387937286, + "variance": 2030506.5522894603, + "z_arithmetic_mean_rank": 546.8158440428246, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.142406890027 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08082390282298467, + "adjusted_arithmetic_mean_rank_index": 0.9193005480930538, + "adjusted_hits_at_k": 0.3392531895800786, + "adjusted_inverse_harmonic_mean_rank": 0.22364591371526105, + "arithmetic_mean_rank": 597.0342407226562, + "count": 118142.0, + "geometric_mean_rank": 48.53256607055664, + "harmonic_mean_rank": 4.460667229528553, + "hits_at_1": 0.1631595876149041, + "hits_at_10": 0.3397013763098644, + "hits_at_3": 0.24314807604408253, + "hits_at_5": 0.2834301095292106, + "inverse_arithmetic_mean_rank": 0.0016749458154663444, + "inverse_geometric_mean_rank": 0.020604722201824188, + "inverse_harmonic_mean_rank": 0.22418170977889182, + "inverse_median_rank": 0.0181818176060915, + "median_absolute_deviation": 80.06051979930251, + "median_rank": 55.0, + "standard_deviation": 1424.9578857421875, + "variance": 2030504.875, + "z_arithmetic_mean_rank": 546.8158629474619, + "z_hits_at_k": 4475.764002340077, + "z_inverse_harmonic_mean_rank": 7288.144164904279 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.08628437326551358, + "adjusted_arithmetic_mean_rank_index": 0.9138396759828533, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062081917111348, + "arithmetic_mean_rank": 635.6353202078855, + "count": 59071.0, + "geometric_mean_rank": 60.487063947463966, + "harmonic_mean_rank": 5.230646899912632, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732291271557228, + "inverse_geometric_mean_rank": 0.016532460574852, + "inverse_harmonic_mean_rank": 0.19118094169512823, + "inverse_median_rank": 0.013513513513513514, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.533801090035, + "variance": 2095354.1052981652, + "z_arithmetic_mean_rank": 384.3046787663501, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.7925652603535 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.08628443760948097, + "adjusted_arithmetic_mean_rank_index": 0.9138396116301504, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062063063034304, + "arithmetic_mean_rank": 635.6357942137428, + "count": 59071.0, + "geometric_mean_rank": 60.487188318126684, + "harmonic_mean_rank": 5.230652054760578, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732279539684543, + "inverse_geometric_mean_rank": 0.016532426581652198, + "inverse_harmonic_mean_rank": 0.1911807532848355, + "inverse_median_rank": 0.013513513513513514, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.5347027698704, + "variance": 2095356.715723057, + "z_arithmetic_mean_rank": 384.3046517035671, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.788227325427 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.08628440569150582, + "adjusted_arithmetic_mean_rank_index": 0.9138396435524588, + "adjusted_hits_at_k": 0.3011107138537921, + "adjusted_inverse_harmonic_mean_rank": 0.19062071994722662, + "arithmetic_mean_rank": 635.6355590820312, + "count": 59071.0, + "geometric_mean_rank": 60.487117767333984, + "harmonic_mean_rank": 5.230649339059701, + "hits_at_1": 0.13214606151918878, + "hits_at_10": 0.30158622674408764, + "hits_at_3": 0.20822400162516294, + "hits_at_5": 0.24743105754092531, + "inverse_arithmetic_mean_rank": 0.0015732285100966692, + "inverse_geometric_mean_rank": 0.016532445326447487, + "inverse_harmonic_mean_rank": 0.19118084253990825, + "inverse_median_rank": 0.013513513840734959, + "median_absolute_deviation": 108.22996195090894, + "median_rank": 74.0, + "standard_deviation": 1447.5341796875, + "variance": 2095355.375, + "z_arithmetic_mean_rank": 384.30466512812376, + "z_hits_at_k": 2804.717087068285, + "z_inverse_harmonic_mean_rank": 4385.790282323017 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.07539301543164696, + "adjusted_arithmetic_mean_rank_index": 0.9247318309769208, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.25667100161503614, + "arithmetic_mean_rank": 558.4327334902067, + "count": 59071.0, + "geometric_mean_rank": 38.94066665432855, + "harmonic_mean_rank": 3.888287986234258, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.001790725972938578, + "inverse_geometric_mean_rank": 0.025680094510884353, + "inverse_harmonic_mean_rank": 0.25718259643840924, + "inverse_median_rank": 0.025, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.9541273132224, + "variance": 1962672.4668359524, + "z_arithmetic_mean_rank": 389.0002303448466, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.563597631679 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.07539307256976503, + "adjusted_arithmetic_mean_rank_index": 0.9247317738310875, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.2566709636040986, + "arithmetic_mean_rank": 558.4331567097222, + "count": 59071.0, + "geometric_mean_rank": 38.94071014034867, + "harmonic_mean_rank": 3.888288560517955, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.0017907246158017935, + "inverse_geometric_mean_rank": 0.0256800658333101, + "inverse_harmonic_mean_rank": 0.25718255845363264, + "inverse_median_rank": 0.025, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.9554829604954, + "variance": 1962676.265237075, + "z_arithmetic_mean_rank": 389.00020630572357, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.562720398939 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.07539304093106662, + "adjusted_arithmetic_mean_rank_index": 0.924731805474058, + "adjusted_hits_at_k": 0.3773955064930534, + "adjusted_inverse_harmonic_mean_rank": 0.25667098218112694, + "arithmetic_mean_rank": 558.4329223632812, + "count": 59071.0, + "geometric_mean_rank": 38.940673828125, + "harmonic_mean_rank": 3.8882882236595933, + "hits_at_1": 0.19417311371061943, + "hits_at_10": 0.3778165258756412, + "hits_at_3": 0.27807215046300215, + "hits_at_5": 0.3194291615174959, + "inverse_arithmetic_mean_rank": 0.0017907253932207823, + "inverse_geometric_mean_rank": 0.025680089369416237, + "inverse_harmonic_mean_rank": 0.2571825770178754, + "inverse_median_rank": 0.02500000037252903, + "median_absolute_deviation": 57.82148652171848, + "median_rank": 40.0, + "standard_deviation": 1400.954833984375, + "variance": 1962674.375, + "z_arithmetic_mean_rank": 389.00021961674236, + "z_hits_at_k": 3526.0819050313557, + "z_inverse_harmonic_mean_rank": 5923.563149127577 + } + } + }, + "times": { + "evaluation": 42.46483302116394, + "training": 859.2714371681213 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json new file mode 100644 index 0000000..2a76f64 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn FB15K-237 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json new file mode 100644 index 0000000..684147b --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/fb15k237/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.002791654386314053, + 7.358028466998594e-05, + 2.835282286703969e-05, + 2.1093817337828796e-05, + 1.892652983966786e-05 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5663228397885496, + "adjusted_arithmetic_mean_rank_index": 0.4337379446939893, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.02716977127692903, + "arithmetic_mean_rank": 4041.092303552207, + "count": 40876.0, + "geometric_mean_rank": 1230.6452823359225, + "harmonic_mean_rank": 35.88982075732999, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.00024745784676112907, + "inverse_geometric_mean_rank": 0.0008125818335742301, + "inverse_harmonic_mean_rank": 0.027863053615160897, + "inverse_median_rank": 0.00033500837520938025, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.5773194486583, + "variance": 15035605.868302641, + "z_arithmetic_mean_rank": 151.69269004908605, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.6232337760049 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5663232752007933, + "adjusted_arithmetic_mean_rank_index": 0.43373750922071796, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.0271697708446242, + "arithmetic_mean_rank": 4041.0954105098344, + "count": 40876.0, + "geometric_mean_rank": 1230.6462909069642, + "harmonic_mean_rank": 35.88982131377605, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.00024745765650552597, + "inverse_geometric_mean_rank": 0.0008125811676261731, + "inverse_harmonic_mean_rank": 0.027863053183164146, + "inverse_median_rank": 0.00033500837520938025, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.578950153829, + "variance": 15035618.51467607, + "z_arithmetic_mean_rank": 151.69253774949414, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.6232256354434 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5663230424952339, + "adjusted_arithmetic_mean_rank_index": 0.4337377419588935, + "adjusted_hits_at_k": 0.060549148858748736, + "adjusted_inverse_harmonic_mean_rank": 0.027169771060678034, + "arithmetic_mean_rank": 4041.09375, + "count": 40876.0, + "geometric_mean_rank": 1230.645263671875, + "harmonic_mean_rank": 35.88982087682511, + "hits_at_1": 0.010935512281045112, + "hits_at_10": 0.06120951169390351, + "hits_at_3": 0.028231725217731676, + "hits_at_5": 0.04119776886192387, + "inverse_arithmetic_mean_rank": 0.0002474577631801367, + "inverse_geometric_mean_rank": 0.0008125818567350507, + "inverse_harmonic_mean_rank": 0.02786305339906401, + "inverse_median_rank": 0.0003350083716213703, + "median_absolute_deviation": 4191.316471715337, + "median_rank": 2985.0, + "standard_deviation": 3877.578125, + "variance": 15035612.0, + "z_arithmetic_mean_rank": 151.69261914583115, + "z_hits_at_k": 461.56818897805357, + "z_inverse_harmonic_mean_rank": 511.62322970386776 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.6753109059937, + "adjusted_arithmetic_mean_rank_index": 0.32473524320845226, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.0050529021797449565, + "arithmetic_mean_rank": 4751.918583031607, + "count": 20438.0, + "geometric_mean_rank": 2508.5323512557943, + "harmonic_mean_rank": 173.2350944902231, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.00021044131597095346, + "inverse_geometric_mean_rank": 0.0003986394672165144, + "inverse_harmonic_mean_rank": 0.005772502407452072, + "inverse_median_rank": 0.00024783147459727387, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.766512579791, + "variance": 14173466.894242203, + "z_arithmetic_mean_rank": 80.2213637364448, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.72758692545682 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.6753114066375367, + "adjusted_arithmetic_mean_rank_index": 0.32473474249345724, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.005052901714946772, + "arithmetic_mean_rank": 4751.922105881202, + "count": 20438.0, + "geometric_mean_rank": 2508.5346523933813, + "harmonic_mean_rank": 173.2351084289142, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.0002104411599597462, + "inverse_geometric_mean_rank": 0.0003986391015351949, + "inverse_harmonic_mean_rank": 0.005772501942990055, + "inverse_median_rank": 0.00024783147459727387, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.7675746432033, + "variance": 14173474.891084868, + "z_arithmetic_mean_rank": 80.22124004171013, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.7275807874276 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.6753110962611738, + "adjusted_arithmetic_mean_rank_index": 0.3247350529139351, + "adjusted_hits_at_k": 0.007706544319798423, + "adjusted_inverse_harmonic_mean_rank": 0.005052901947239784, + "arithmetic_mean_rank": 4751.919921875, + "count": 20438.0, + "geometric_mean_rank": 2508.53369140625, + "harmonic_mean_rank": 173.23509892383208, + "hits_at_1": 0.002299637929347294, + "hits_at_10": 0.00841569625207946, + "hits_at_3": 0.004403561992367159, + "hits_at_5": 0.006164986789314023, + "inverse_arithmetic_mean_rank": 0.00021044125605840236, + "inverse_geometric_mean_rank": 0.0003986392403021455, + "inverse_harmonic_mean_rank": 0.00577250217511506, + "inverse_median_rank": 0.0002478314854670316, + "median_absolute_deviation": 4339.576693565897, + "median_rank": 4035.0, + "standard_deviation": 3764.76708984375, + "variance": 14173471.0, + "z_arithmetic_mean_rank": 80.22131672680848, + "z_hits_at_k": 41.19790684863103, + "z_inverse_harmonic_mean_rank": 66.72758385504133 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.46031849371867495, + "adjusted_arithmetic_mean_rank_index": 0.5397561128500036, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.049286170723083396, + "arithmetic_mean_rank": 3330.2660240728055, + "count": 20438.0, + "geometric_mean_rank": 603.7346140572965, + "harmonic_mean_rank": 20.018575306945234, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.00030027631209384077, + "inverse_geometric_mean_rank": 0.0016563569103312942, + "inverse_harmonic_mean_rank": 0.049953604822869715, + "inverse_median_rank": 0.0006263701847792045, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.3930389692055, + "variance": 14887196.843166022, + "z_arithmetic_mean_rank": 133.64077253097005, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883557742102 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.4603188656851434, + "adjusted_arithmetic_mean_rank_index": 0.5397557408321139, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.04928617032327123, + "arithmetic_mean_rank": 3330.2687151384675, + "count": 20438.0, + "geometric_mean_rank": 603.7350498140833, + "harmonic_mean_rank": 20.018575467054823, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.00030027606945177743, + "inverse_geometric_mean_rank": 0.0016563557148254757, + "inverse_harmonic_mean_rank": 0.04995360442333823, + "inverse_median_rank": 0.0006263701847792045, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.3951270654115, + "variance": 14887212.956562115, + "z_arithmetic_mean_rank": 133.64068042129867, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883504057462 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.46031864103245473, + "adjusted_arithmetic_mean_rank_index": 0.5397559655158588, + "adjusted_hits_at_k": 0.11339051230450989, + "adjusted_inverse_harmonic_mean_rank": 0.049286170523086245, + "arithmetic_mean_rank": 3330.26708984375, + "count": 20438.0, + "geometric_mean_rank": 603.7346801757812, + "harmonic_mean_rank": 20.018575172499716, + "hits_at_1": 0.01957138663274293, + "hits_at_10": 0.11400332713572757, + "hits_at_3": 0.05205988844309619, + "hits_at_5": 0.07623055093453371, + "inverse_arithmetic_mean_rank": 0.0003002762096002698, + "inverse_geometric_mean_rank": 0.0016563567332923412, + "inverse_harmonic_mean_rank": 0.049953604623012966, + "inverse_median_rank": 0.0006263701943680644, + "median_absolute_deviation": 2353.631021877643, + "median_rank": 1596.5, + "standard_deviation": 3858.394287109375, + "variance": 14887206.0, + "z_arithmetic_mean_rank": 133.6407360518113, + "z_hits_at_k": 616.3782888896045, + "z_inverse_harmonic_mean_rank": 661.7883530887553 + } + } + }, + "times": { + "evaluation": 15.475303888320923, + "training": 466.48984026908875 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json new file mode 100644 index 0000000..9cea214 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18 Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json new file mode 100644 index 0000000..93f0ec8 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001784344254726013, + 3.546414034157781e-06, + 3.241080858006321e-06, + 3.114976756636245e-06, + 2.580594582893777e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.173634136893227, + "adjusted_arithmetic_mean_rank_index": 0.8264062480375431, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.14717656877035404, + "arithmetic_mean_rank": 3553.1158, + "count": 10000.0, + "geometric_mean_rank": 217.95702597254055, + "harmonic_mean_rank": 6.783805257873866, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.0002814431209925666, + "inverse_geometric_mean_rank": 0.00458806040107184, + "inverse_harmonic_mean_rank": 0.14740989193923487, + "inverse_median_rank": 0.0030581039755351682, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.022572537755, + "variance": 49421217.370390356, + "z_arithmetic_mean_rank": 143.13408421621838, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.0034516277847 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.17363431281850722, + "adjusted_arithmetic_mean_rank_index": 0.8264060721036653, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.14717656856459646, + "arithmetic_mean_rank": 3553.1194, + "count": 10000.0, + "geometric_mean_rank": 217.9571490546336, + "harmonic_mean_rank": 6.783805267340243, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.0002814428358360262, + "inverse_geometric_mean_rank": 0.004588057810158537, + "inverse_harmonic_mean_rank": 0.14740989173353358, + "inverse_median_rank": 0.0030581039755351682, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.028042059551, + "variance": 49421294.27214364, + "z_arithmetic_mean_rank": 143.13405374435962, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.003448380151 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.17363422855915533, + "adjusted_arithmetic_mean_rank_index": 0.826406156367135, + "adjusted_hits_at_k": 0.22100965584808593, + "adjusted_inverse_harmonic_mean_rank": 0.1471765686674477, + "arithmetic_mean_rank": 3553.11767578125, + "count": 10000.0, + "geometric_mean_rank": 217.95701599121094, + "harmonic_mean_rank": 6.783805453179238, + "hits_at_1": 0.1094, + "hits_at_10": 0.2212, + "hits_at_3": 0.1561, + "hits_at_5": 0.1819, + "inverse_arithmetic_mean_rank": 0.00028144297539256513, + "inverse_geometric_mean_rank": 0.004588060546666384, + "inverse_harmonic_mean_rank": 0.14740989183635667, + "inverse_median_rank": 0.0030581040773540735, + "median_absolute_deviation": 483.3283232328262, + "median_rank": 327.0, + "standard_deviation": 7030.025390625, + "variance": 49421256.0, + "z_arithmetic_mean_rank": 143.1340683388458, + "z_hits_at_k": 1413.6908620029953, + "z_inverse_harmonic_mean_rank": 2323.0034500035326 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.18254785061091422, + "adjusted_arithmetic_mean_rank_index": 0.8174920996282304, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.13370304480394207, + "arithmetic_mean_rank": 3735.4326, + "count": 5000.0, + "geometric_mean_rank": 251.60200046021046, + "harmonic_mean_rank": 7.466026264434069, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.00026770661047397827, + "inverse_geometric_mean_rank": 0.003974531196774585, + "inverse_harmonic_mean_rank": 0.1339400592204856, + "inverse_median_rank": 0.0025284450063211127, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.8929429127775, + "variance": 51178184.23865724, + "z_arithmetic_mean_rank": 100.11934541020389, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184846206778 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.1825480656356861, + "adjusted_arithmetic_mean_rank_index": 0.81749188459295, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.13370304443324765, + "arithmetic_mean_rank": 3735.437, + "count": 5000.0, + "geometric_mean_rank": 251.60221622118326, + "harmonic_mean_rank": 7.466026285091496, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.00026770629514030086, + "inverse_geometric_mean_rank": 0.003974527788423378, + "inverse_harmonic_mean_rank": 0.1339400588498926, + "inverse_median_rank": 0.0025284450063211127, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.898885644877, + "variance": 51178269.266031004, + "z_arithmetic_mean_rank": 100.11931907454691, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184804834717 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.18254795882961378, + "adjusted_arithmetic_mean_rank_index": 0.817491991404242, + "adjusted_hits_at_k": 0.20800647403697897, + "adjusted_inverse_harmonic_mean_rank": 0.1337030446185418, + "arithmetic_mean_rank": 3735.434814453125, + "count": 5000.0, + "geometric_mean_rank": 251.6021728515625, + "harmonic_mean_rank": 7.46602602719376, + "hits_at_1": 0.096, + "hits_at_10": 0.2082, + "hits_at_3": 0.1422, + "hits_at_5": 0.1676, + "inverse_arithmetic_mean_rank": 0.0002677064621821046, + "inverse_geometric_mean_rank": 0.003974528517574072, + "inverse_harmonic_mean_rank": 0.13394005903513603, + "inverse_median_rank": 0.0025284450966864824, + "median_absolute_deviation": 584.88657520046, + "median_rank": 395.5, + "standard_deviation": 7153.8955078125, + "variance": 51178224.0, + "z_arithmetic_mean_rank": 100.11933215586801, + "z_hits_at_k": 940.8057967489788, + "z_inverse_harmonic_mean_rank": 1492.2184825514823 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.16472083645996619, + "adjusted_arithmetic_mean_rank_index": 0.8353199831220769, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009257998678, + "arithmetic_mean_rank": 3370.799, + "count": 5000.0, + "geometric_mean_rank": 188.81115843237254, + "harmonic_mean_rank": 6.215823666567742, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.0002966655680151798, + "inverse_geometric_mean_rank": 0.0052962971484451496, + "inverse_harmonic_mean_rank": 0.1608797246579842, + "inverse_median_rank": 0.003703703703703704, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.113832297522, + "variance": 47597771.670999005, + "z_arithmetic_mean_rank": 102.30276734417879, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080090516244 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.16472097328756763, + "adjusted_arithmetic_mean_rank_index": 0.8353198462877888, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009253916593, + "arithmetic_mean_rank": 3370.8018, + "count": 5000.0, + "geometric_mean_rank": 188.81120976399464, + "harmonic_mean_rank": 6.215823668144484, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.00029666532158609856, + "inverse_geometric_mean_rank": 0.00529629570855435, + "inverse_harmonic_mean_rank": 0.1608797246171745, + "inverse_median_rank": 0.003703703703703704, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.118774431758, + "variance": 47597839.86371676, + "z_arithmetic_mean_rank": 102.30275058589858, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080085960246 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.16472089964347075, + "adjusted_arithmetic_mean_rank_index": 0.8353199199354846, + "adjusted_hits_at_k": 0.2340128375107572, + "adjusted_inverse_harmonic_mean_rank": 0.16065009255957435, + "arithmetic_mean_rank": 3370.80029296875, + "count": 5000.0, + "geometric_mean_rank": 188.81117248535156, + "harmonic_mean_rank": 6.215823215674801, + "hits_at_1": 0.1228, + "hits_at_10": 0.2342, + "hits_at_3": 0.17, + "hits_at_5": 0.1962, + "inverse_arithmetic_mean_rank": 0.00029666544287465513, + "inverse_geometric_mean_rank": 0.005296296905726194, + "inverse_harmonic_mean_rank": 0.16087972463757735, + "inverse_median_rank": 0.003703703638166189, + "median_absolute_deviation": 398.8199967780069, + "median_rank": 270.0, + "standard_deviation": 6899.11572265625, + "variance": 47597800.0, + "z_arithmetic_mean_rank": 102.30275960563142, + "z_hits_at_k": 1058.4563668509725, + "z_inverse_harmonic_mean_rank": 1793.0080088238021 + } + } + }, + "times": { + "evaluation": 6.739559888839722, + "training": 597.220596075058 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json new file mode 100644 index 0000000..c14d63d --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/metadata.json @@ -0,0 +1,20 @@ +{ + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 .", + "system": { + "gpu": { + "compute_capability": [ + 7, + 5 + ], + "cuda": "11.3", + "cudnn": 8200, + "name": "Quadro RTX 8000", + "total_memory": 50962169856 + }, + "machine": "x86_64", + "release": "5.4.0-81-generic", + "system": "Linux", + "torch_version": "1.11.0+cu113" + }, + "title": "Learn WN18RR Dataset with TuckER as described by Balazevic et al., 2019" +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json new file mode 100644 index 0000000..04dd837 --- /dev/null +++ b/nonasv/results/v1.8.0/cuda/tucker/wn18rr/balazevic2019/results.json @@ -0,0 +1,246 @@ +{ + "losses": [ + 0.0001750188616934491, + 3.981787474154627e-06, + 3.557281189895917e-06, + 3.3347453562454225e-06, + 3.199828891147623e-06 + ], + "metrics": { + "both": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.7015087458919147, + "adjusted_arithmetic_mean_rank_index": 0.29850597874848384, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.020728818694365003, + "arithmetic_mean_rank": 14221.369699042407, + "count": 5848.0, + "geometric_mean_rank": 4766.871536494683, + "harmonic_mean_rank": 47.621246158623535, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031671499738417e-05, + "inverse_geometric_mean_rank": 0.0002097811934607639, + "inverse_harmonic_mean_rank": 0.020999030488808704, + "inverse_median_rank": 9.231053263177328e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.902335092624, + "variance": 162509013.94505996, + "z_arithmetic_mean_rank": 39.53722336009449, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.0339912312517 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.7015092266860429, + "adjusted_arithmetic_mean_rank_index": 0.29850549793063796, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.0207288182389401, + "arithmetic_mean_rank": 14221.379445964432, + "count": 5848.0, + "geometric_mean_rank": 4766.876381520779, + "harmonic_mean_rank": 47.62124719114347, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031666680434208e-05, + "inverse_geometric_mean_rank": 0.00020978098024034963, + "inverse_harmonic_mean_rank": 0.020999030033509468, + "inverse_median_rank": 9.231053263177328e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.90679032108, + "variance": 162509127.5347143, + "z_arithmetic_mean_rank": 39.537159675599284, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.0339857598217 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.7015089110330996, + "adjusted_arithmetic_mean_rank_index": 0.2985058135991524, + "adjusted_hits_at_k": 0.037724355282247504, + "adjusted_inverse_harmonic_mean_rank": 0.02072881846653769, + "arithmetic_mean_rank": 14221.373046875, + "count": 5848.0, + "geometric_mean_rank": 4766.87451171875, + "harmonic_mean_rank": 47.621243309185644, + "hits_at_1": 0.011969904240766074, + "hits_at_10": 0.03796169630642955, + "hits_at_3": 0.02222982216142271, + "hits_at_5": 0.027701778385772913, + "inverse_arithmetic_mean_rank": 7.031669520074502e-05, + "inverse_geometric_mean_rank": 0.0002097810647683218, + "inverse_harmonic_mean_rank": 0.020999030261044258, + "inverse_median_rank": 9.231053263647482e-05, + "median_absolute_deviation": 14625.870885557762, + "median_rank": 10833.0, + "standard_deviation": 12747.904296875, + "variance": 162509072.0, + "z_arithmetic_mean_rank": 39.53720148600662, + "z_hits_at_k": 183.66875591315986, + "z_inverse_harmonic_mean_rank": 249.03398849415677 + } + }, + "head": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.8128474155111893, + "adjusted_arithmetic_mean_rank_index": 0.18716181881646976, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.01546072167844193, + "arithmetic_mean_rank": 16474.83242134063, + "count": 2924.0, + "geometric_mean_rank": 7222.092081377499, + "harmonic_mean_rank": 63.56292193252177, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.069864472215527e-05, + "inverse_geometric_mean_rank": 0.00013846403351440873, + "inverse_harmonic_mean_rank": 0.015732442272896097, + "inverse_median_rank": 6.789327177676693e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.96661470027, + "variance": 165764765.32964653, + "z_arithmetic_mean_rank": 17.528919271821408, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.3259227117884 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.8128478036065856, + "adjusted_arithmetic_mean_rank_index": 0.18716143070192437, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.015460721633633155, + "arithmetic_mean_rank": 16474.8402872777, + "count": 2924.0, + "geometric_mean_rank": 7222.095610109252, + "harmonic_mean_rank": 63.56292211351025, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.0698615741496806e-05, + "inverse_geometric_mean_rank": 0.0001384639658605783, + "inverse_harmonic_mean_rank": 0.015732442228099688, + "inverse_median_rank": 6.789327177676693e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.96983860059, + "variance": 165764848.3448749, + "z_arithmetic_mean_rank": 17.528882922374834, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.32592233117524 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.8128476853587696, + "adjusted_arithmetic_mean_rank_index": 0.1871615489555749, + "adjusted_hits_at_k": 0.024725199409302102, + "adjusted_inverse_harmonic_mean_rank": 0.015460721656036102, + "arithmetic_mean_rank": 16474.837890625, + "count": 2924.0, + "geometric_mean_rank": 7222.09326171875, + "harmonic_mean_rank": 63.56292443713892, + "hits_at_1": 0.010601915184678522, + "hits_at_10": 0.02496580027359781, + "hits_at_3": 0.016415868673050615, + "hits_at_5": 0.019493844049247606, + "inverse_arithmetic_mean_rank": 6.069862502044998e-05, + "inverse_geometric_mean_rank": 0.00013846400543116033, + "inverse_harmonic_mean_rank": 0.015732442250496453, + "inverse_median_rank": 6.78932701703161e-05, + "median_absolute_deviation": 16837.91339556812, + "median_rank": 14729.0, + "standard_deviation": 12874.9677734375, + "variance": 165764800.0, + "z_arithmetic_mean_rank": 17.52889399759684, + "z_hits_at_k": 85.11180878127107, + "z_inverse_harmonic_mean_rank": 131.32592252146958 + } + }, + "tail": { + "optimistic": { + "adjusted_arithmetic_mean_rank": 0.5902194668953142, + "adjusted_arithmetic_mean_rank_index": 0.4098007431862519, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.02599691511975857, + "arithmetic_mean_rank": 11967.906976744185, + "count": 2924.0, + "geometric_mean_rank": 3146.3271292311133, + "harmonic_mean_rank": 38.07258497285074, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355679919163656e-05, + "inverse_geometric_mean_rank": 0.00031783090534657024, + "inverse_harmonic_mean_rank": 0.026265618704721314, + "inverse_median_rank": 0.00013768415255404102, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.531277503775, + "variance": 149097074.07889798, + "z_arithmetic_mean_rank": 38.38054590290827, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129861041845 + }, + "pessimistic": { + "adjusted_arithmetic_mean_rank": 0.5902200403470523, + "adjusted_arithmetic_mean_rank_index": 0.40980016970623157, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.02599691425371758, + "arithmetic_mean_rank": 11967.918604651162, + "count": 2924.0, + "geometric_mean_rank": 3146.331987753468, + "harmonic_mean_rank": 38.07258622784969, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355671800870739e-05, + "inverse_geometric_mean_rank": 0.00031783041455648046, + "inverse_harmonic_mean_rank": 0.026265617838919245, + "inverse_median_rank": 0.00013768415255404102, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.53787504421, + "variance": 149097235.19788918, + "z_arithmetic_mean_rank": 38.38049219271713, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129125248399 + }, + "realistic": { + "adjusted_arithmetic_mean_rank": 0.5902197200204955, + "adjusted_arithmetic_mean_rank_index": 0.40980049004858665, + "adjusted_hits_at_k": 0.050723509724511805, + "adjusted_inverse_harmonic_mean_rank": 0.0259969146865098, + "arithmetic_mean_rank": 11967.912109375, + "count": 2924.0, + "geometric_mean_rank": 3146.3310546875, + "harmonic_mean_rank": 38.0725823227289, + "hits_at_1": 0.013337893296853625, + "hits_at_10": 0.05095759233926129, + "hits_at_3": 0.0280437756497948, + "hits_at_5": 0.035909712722298225, + "inverse_arithmetic_mean_rank": 8.355676254723221e-05, + "inverse_geometric_mean_rank": 0.00031783050508238375, + "inverse_harmonic_mean_rank": 0.026265618271592066, + "inverse_median_rank": 0.00013768415374215692, + "median_absolute_deviation": 10454.569543792251, + "median_rank": 7263.0, + "standard_deviation": 12210.5341796875, + "variance": 149097152.0, + "z_arithmetic_mean_rank": 38.380522194894205, + "z_hits_at_k": 174.64502149068215, + "z_inverse_harmonic_mean_rank": 220.87129492951178 + } + } + }, + "times": { + "evaluation": 5.12530517578125, + "training": 358.1025986671448 + } +} \ No newline at end of file diff --git a/nonasv/results/v1.8.0/summary.tsv b/nonasv/results/v1.8.0/summary.tsv new file mode 100644 index 0000000..1909631 --- /dev/null +++ b/nonasv/results/v1.8.0/summary.tsv @@ -0,0 +1,39 @@ +batch_size.evaluation evaluation path training + 29.81851315498352 complex/trouillon2016_complex_fb15k.json 1.640441608428955 + 4.551660537719727 complex/trouillon2016_complex_wn18.json 0.3536832332611084 + 14.673102378845215 conve/dettmers2018_conve_fb15k.json 34.67542872428894 + 6.211552381515503 conve/dettmers2018_conve_fb15k237.json 19.024553966522216 + 3.723423957824707 conve/dettmers2018_conve_wn18.json 33.473473691940306 + 2.1725149154663086 conve/dettmers2018_conve_wn18rr.json 20.13378930091858 + 113.07699227333069 convkb/nguyen2018_convkb_fb15k237.json 5.8175290584564205 + 228.67922687530518 convkb/nguyen2018_convkb_wn18rr.json 2.3075665473937987 + 9.182297706604004 convkb/nguyen2018_transe_fb15k237.json 2.377616119384766 + 20.92709469795227 distmult/yang2014_distmult_fb15k.json 3.150154638290405 + 5.345269203186035 distmult/yang2014_distmult_wn18.json 1.189062738418579 + 34.076919078826904 hole/nickel2016_hole_fb15k.json 0.8448263168334961 + 7.570884943008423 hole/nickel2016_hole_wn18.json 0.4609029293060303 + 28.00254464149475 kg2e/he2015_kg2e_fb15k.json 2.760317659378052 + 7.501351594924927 kg2e/he2015_kg2e_wn18.json 39.696214532852174 + 14.712653875350952 mure/balazevic2019_mure_fb15k237.json 13.239705181121826 + 143.83830213546753 quate/zhang2019_quate_fb15k.json 1.456200647354126 + 33.202207803726196 quate/zhang2019_quate_fb15k237.json 0.8993231296539307 + 50.53933072090149 quate/zhang2019_quate_wn18.json 2.752012538909912 + 13.765980958938599 quate/zhang2019_quate_wn18rr.json 0.9874269962310791 + 88.11779618263245 rotate/sun2019_rotate_fb15k.json 80.48430371284485 + 28.418932676315308 rotate/sun2019_rotate_fb15k237.json 88.11092224121094 + 11.729146718978882 rotate/sun2019_rotate_wn18.json 94.30107221603393 + 7.746156454086304 rotate/sun2019_rotate_wn18rr.json 58.11595721244812 + 29.054805994033813 simple/kazemi2018_simple_fb15k.json 2.0225048065185547 + 8.916642427444458 simple/kazemi2018_simple_wn18.json 0.9781480312347413 + 33.93540859222412 transd/ji2015_transd_fb15k.json 17.45572304725647 + 9.962103366851807 transd/ji2015_transd_wn18.json 5.07728943824768 + 16.924992561340332 transe/bordes2013_transe_fb15k.json 23.030786180496214 + 4.560383558273315 transe/bordes2013_transe_wn18.json 6.884837055206299 + 29.358933925628662 transh/wang2014_transh_fb15k.json 2.086397314071655 + 8.131263971328735 transh/wang2014_transh_wn18.json 0.5390334129333496 + 28.336533308029175 transr/li2015_transr_fb15k.json 2.297199583053589 + 7.977590084075928 transr/li2015_transr_wn18.json 0.7416571140289306 + 42.46483302116394 tucker/balazevic2019_tucker_fb15k.json 171.85428743362428 + 15.475303888320923 tucker/balazevic2019_tucker_fb15k237.json 93.29796805381775 + 6.739559888839722 tucker/balazevic2019_tucker_wn18.json 119.4441192150116 + 5.12530517578125 tucker/balazevic2019_tucker_wn18rr.json 71.62051973342895