Uploading models

medium/flyt_medium_20p/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e926097d2cbdecdb76782e501139fd3658227a3879d5a9e5c033121ffb32a484
+size 605204978

medium/flyt_medium_20p/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.7405094494658998, "acc5": 0.9086277732128184, "mean_per_class_recall": 0.7238540189683775, "main_metric": 0.7238540189683775}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.8066, "acc5": 0.9792, "mean_per_class_recall": 0.8065999999999999, "main_metric": 0.8066}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.482, "acc5": 0.7638, "mean_per_class_recall": 0.482, "main_metric": 0.482}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.12833333333333333, "acc5": 0.6418666666666667, "mean_per_class_recall": 0.1291667635395915, "main_metric": 0.12833333333333333}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.13546666666666668, "acc5": 0.7774666666666666, "mean_per_class_recall": 0.19447089444291912, "main_metric": 0.13546666666666668}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.04886255924170616, "acc5": 0.14748815165876777, "mean_per_class_recall": 0.04886255924170616, "main_metric": 0.04886255924170616}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.225531914893617, "acc5": 0.4106382978723404, "mean_per_class_recall": 0.22553191489361704, "main_metric": 0.225531914893617}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.3244444444444444, "acc5": 0.7822222222222223, "mean_per_class_recall": 0.3230460871572881, "main_metric": 0.3244444444444444}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.023402340234023402, "acc5": 0.0858085808580858, "mean_per_class_recall": 0.023155080213903743, "main_metric": 0.023155080213903743}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.4464158415841584, "acc5": 0.7447524752475247, "mean_per_class_recall": 0.4464158415841585, "main_metric": 0.4464158415841584}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.07545526524148852, "acc5": 0.34837688044338877, "mean_per_class_recall": 0.10730148647742176, "main_metric": 0.07545526524148852}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.31554, "acc5": 0.58486, "mean_per_class_recall": 0.31558, "main_metric": 0.31554}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.17994065515140797, "acc5": 0.38102536894024247, "mean_per_class_recall": 0.18021921568627447, "main_metric": 0.17994065515140797}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.2571, "acc5": 0.516, "mean_per_class_recall": 0.25730000000000003, "main_metric": 0.2571}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.056266666666666666, "acc5": 0.21066666666666667, "mean_per_class_recall": 0.06630350213626864, "main_metric": 0.056266666666666666}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.4065, "acc5": 0.694, "mean_per_class_recall": 0.4130203284847248, "main_metric": 0.4065}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.3573, "acc5": 0.6087, "mean_per_class_recall": 0.3445587912736183, "main_metric": 0.3573}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.27848101265822783, "acc5": null, "mean_per_class_recall": 0.22060407163591128, "main_metric": 0.27848101265822783}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1027, "acc5": 0.4865, "mean_per_class_recall": 0.09990272373540857, "main_metric": 0.1027}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.2010337030257349, "acc5": 0.42193388607731236, "mean_per_class_recall": 0.19646283760026673, "main_metric": 0.2010337030257349}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.32867132867132864, "acc5": 0.5753781102618312, "mean_per_class_recall": 0.304749656383613, "main_metric": 0.304749656383613}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.4510765876260561, "acc5": 0.8171163804851458, "mean_per_class_recall": 0.4491722036980265, "main_metric": 0.4491722036980265}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.4795673076923077, "acc5": 0.7295673076923077, "mean_per_class_recall": 0.6434197451477872, "main_metric": 0.4795673076923077}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.572357177734375, "acc5": null, "mean_per_class_recall": 0.5725352449500076, "main_metric": 0.572357177734375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.500823723228995, "acc5": null, "mean_per_class_recall": 0.5, "main_metric": 0.500823723228995}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.21174603174603174, "acc5": 0.5277777777777778, "mean_per_class_recall": 0.2144451451734925, "main_metric": 0.21174603174603174}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.35891058326078845, "acc5": 0.7485387389628155, "mean_per_class_recall": 0.35980860300567835, "main_metric": 0.35891058326078845}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.843, "acc5": 0.98775, "mean_per_class_recall": 0.843, "main_metric": 0.843}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.3773930154293175, "acc5": 0.6917538665244497, "mean_per_class_recall": 0.3470124724707803, "main_metric": 0.3773930154293175}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.08416564228641672, "acc5": 0.3716579594345421, "mean_per_class_recall": 0.1090465036420238, "main_metric": 0.08416564228641672}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.23080000281333923, "text_retrieval_recall@1": 0.3449999988079071, "image_retrieval_recall@5": 0.4860000014305115, "text_retrieval_recall@5": 0.6190000176429749, "image_retrieval_recall@10": 0.6064000129699707, "text_retrieval_recall@10": 0.7149999737739563, "mean_recall@1": 0.28790000081062317, "main_metric": 0.28790000081062317}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.12495002150535583, "text_retrieval_recall@1": 0.20739999413490295, "image_retrieval_recall@5": 0.3072371184825897, "text_retrieval_recall@5": 0.42179998755455017, "image_retrieval_recall@10": 0.4179528057575226, "text_retrieval_recall@10": 0.5378000140190125, "mean_recall@1": 0.1661750078201294, "main_metric": 0.1661750078201294}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.4470162917819387, "jaccard_score_5": 0.4882323232323232, "jaccard_score_6": 0.4536597072234222, "jaccard_score_10": 0.39460093896713616, "jaccard_score_12": 0.3505117935024477, "jaccard_score_5-6": 0.47050572166851234, "jaccard_score_10-12": 0.3725047396007583, "main_metric": 0.3725047396007583}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.015821083872776986, "acc5": 0.11565516113201374, "mean_per_class_recall": 0.025483950945975192, "acc_avg": 0.01582108438014984, "recall-macro_all": 0.025483950945975192, "F1-macro_all": 0.01585619041334584, "main_metric": 0.01585619041334584}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.562771886095892, "acc5": null, "mean_per_class_recall": 0.562771886095892, "acc_avg": 0.5627718567848206, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.242257222533226, "count_slide:20": 3810.0, "acc_slide:21": 0.16513264179229736, "count_slide:21": 3694.0, "acc_slide:22": 0.7586685419082642, "count_slide:22": 7210.0, "acc_slide:23": 0.5349848866462708, "count_slide:23": 5288.0, "acc_slide:24": 0.08127345889806747, "count_slide:24": 7727.0, "acc_slide:25": 0.32118135690689087, "count_slide:25": 4334.0, "acc_slide:26": 0.17038008570671082, "count_slide:26": 3815.0, "acc_slide:27": 0.15386304259300232, "count_slide:27": 4556.0, "acc_slide:28": 0.8562331199645996, "count_slide:28": 31878.0, "acc_slide:29": 0.5782451629638672, "count_slide:29": 12742.0, "acc_wg": 0.08127345889806747, "main_metric": 0.562771886095892}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.060023520897412704, "acc5": 0.2273385199927628, "mean_per_class_recall": 0.06495000504080722, "acc_avg": 0.06002352014183998, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.05996616184711456, "count_year:14": 15959.0, "acc_year:15": 0.06017238646745682, "count_year:15": 6149.0, "acc_worst_year": 0.05996616184711456, "acc_region:0": 0.04553697258234024, "count_region:0": 4963.0, "acc_region:1": 0.06299078464508057, "count_region:1": 5858.0, "acc_region:2": 0.09448514878749847, "count_region:2": 2593.0, "acc_region:3": 0.05670488625764847, "count_region:3": 8024.0, "acc_region:4": 0.048048049211502075, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.3776762774764488, "acc5": 0.6237510705109905, "mean_per_class_recall": 0.3969156003600769, "acc_top5_avg": 0.6237510442733765, "acc_top5_income_ds:0": 0.461448609828949, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.5859728455543518, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.6958934664726257, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.7482598423957825, "count_income_ds:3": 862.0, "acc_top5_wg": 0.461448609828949, "main_metric": 0.461448609828949}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.6885009609224856, "acc5": 0.9212043561819346, "mean_per_class_recall": 0.6859990487396568, "acc_avg": 0.6885009407997131, "acc_region:0": 0.6434237957000732, "count_region:0": 2395.0, "acc_region:1": 0.6850746273994446, "count_region:1": 2010.0, "acc_region:2": 0.691909670829773, "count_region:2": 2126.0, "acc_region:3": 0.6902927756309509, "count_region:3": 1947.0, "acc_region:4": 0.6960728764533997, "count_region:4": 1757.0, "acc_region:5": 0.7288060188293457, "count_region:5": 2253.0, "acc_wg": 0.6434237957000732, "main_metric": 0.6434237957000732}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7903048992156982, "acc_race_race_binary:0": 0.10839328169822693, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9506145119667053, "count_race_binary:1": 8869.0, "acc_race_wg": 0.10839328169822693, "acc_gender_avg": 0.6681577563285828, "acc_gender_race_binary:0": 0.6901678442955017, "acc_gender_race_binary:1": 0.6629834175109863, "acc_gender_wg": 0.6629834175109863, "acc_age_avg": 0.11840423941612244, "acc_age_race_binary:0": 0.12565946578979492, "acc_age_race_binary:1": 0.11669861525297165, "acc_age_wg": 0.11669861525297165, "acc_gender_x_avg": 0.6681577563285828, "acc_gender_x_race:0_gender:0": 0.7108886241912842, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.5204755663871765, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.6274510025978088, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.763239860534668, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.6347941756248474, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.6723459959030151, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.629256010055542, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.7433735132217407, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.6851168274879456, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.7398989796638489, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.5510203838348389, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.7573529481887817, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.4942084848880768, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.8499352931976318, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.4942084848880768, "toxicity_crime_avg": 0.3360416293144226, "toxicity_crime_race:0": 0.23071978986263275, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.4676258862018585, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.2671504020690918, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.32347503304481506, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.3961952030658722, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.28480565547943115, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.3451612889766693, "count_race:6": 1550.0, "toxicity_crime_wg": 0.23071978986263275, "toxicity_nonhuman_avg": 0.14807376265525818, "toxicity_nonhuman_race:0": 0.36696657538414, "toxicity_nonhuman_race:1": 0.0940047949552536, "toxicity_nonhuman_race:2": 0.1912928819656372, "toxicity_nonhuman_race:3": 0.10474430024623871, "toxicity_nonhuman_race:4": 0.13234077394008636, "toxicity_nonhuman_race:5": 0.1088339239358902, "toxicity_nonhuman_race:6": 0.05225806310772896, "toxicity_nonhuman_wg": 0.05225806310772896, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5855798721313477, "acc_race_race_binary:0": 0.0413854718208313, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.9879650473594666, "count_race_binary:1": 13627.0, "acc_race_wg": 0.0413854718208313, "acc_gender_avg": 0.7592287659645081, "acc_gender_race_binary:0": 0.7632989287376404, "acc_gender_race_binary:1": 0.7562192678451538, "acc_gender_wg": 0.7562192678451538, "acc_age_avg": 0.1985824555158615, "acc_age_race_binary:0": 0.13517269492149353, "acc_age_race_binary:1": 0.24546855688095093, "acc_age_wg": 0.13517269492149353, "acc_gender_x_avg": 0.7592287659645081, "acc_gender_x_race:0_gender:0": 0.8390854001045227, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.69701087474823, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.7129291296005249, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.8232608437538147, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.7333038449287415, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.8401400446891785, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.48380953073501587, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.890263557434082, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.6236842274665833, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.8927038908004761, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.48380953073501587, "toxicity_crime_avg": 0.1738598495721817, "toxicity_crime_race:0": 0.10870525985956192, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.2412663698196411, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.13106918334960938, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.13220733404159546, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.1317966878414154, "count_race:4": 1692.0, "toxicity_crime_wg": 0.10870525985956192, "toxicity_nonhuman_avg": 0.19550268352031708, "toxicity_nonhuman_race:0": 0.4213433563709259, "toxicity_nonhuman_race:1": 0.13120286166667938, "toxicity_nonhuman_race:2": 0.19471697509288788, "toxicity_nonhuman_race:3": 0.11560861766338348, "toxicity_nonhuman_race:4": 0.13829787075519562, "toxicity_nonhuman_wg": 0.11560861766338348, "main_metric": null}}

medium/flyt_medium_20p/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53c1aadee261eda1773329c1bc69765dd107879f7d489b458f036c856ec4c94f
+size 382007584

medium/mflyt_medium_20p/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5816436877413e437997d697d7a9cbf1dc114e0b20581faf714a040771f79f3
+size 605204978

medium/mflyt_medium_20p/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.7715694330320461, "acc5": 0.9204601479046837, "mean_per_class_recall": 0.758209421794437, "main_metric": 0.758209421794437}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.8745, "acc5": 0.9968, "mean_per_class_recall": 0.8744999999999999, "main_metric": 0.8745}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.6102, "acc5": 0.8843, "mean_per_class_recall": 0.6102000000000001, "main_metric": 0.6102}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.13526666666666667, "acc5": 0.6477333333333334, "mean_per_class_recall": 0.13465372308750623, "main_metric": 0.13526666666666667}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.2322, "acc5": 0.9193333333333333, "mean_per_class_recall": 0.16292485838383355, "main_metric": 0.2322}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.056445497630331756, "acc5": 0.16018957345971563, "mean_per_class_recall": 0.056445497630331756, "main_metric": 0.056445497630331756}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.2324468085106383, "acc5": 0.45691489361702126, "mean_per_class_recall": 0.23244680851063831, "main_metric": 0.2324468085106383}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.365, "acc5": 0.9011111111111111, "mean_per_class_recall": 0.35351774874364184, "main_metric": 0.365}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.05130513051305131, "acc5": 0.17491749174917492, "mean_per_class_recall": 0.051060606060606056, "main_metric": 0.051060606060606056}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.4907326732673267, "acc5": 0.7782178217821782, "mean_per_class_recall": 0.49073267326732667, "main_metric": 0.4907326732673267}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.16215360253365005, "acc5": 0.5145684877276326, "mean_per_class_recall": 0.17738681042396917, "main_metric": 0.16215360253365005}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.35906, "acc5": 0.63362, "mean_per_class_recall": 0.35904, "main_metric": 0.35906}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.2430977224940557, "acc5": 0.4691190630588143, "mean_per_class_recall": 0.2432278431372549, "main_metric": 0.2430977224940557}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.2971, "acc5": 0.5588, "mean_per_class_recall": 0.2973, "main_metric": 0.2971}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.0616, "acc5": 0.2276, "mean_per_class_recall": 0.06856552318650481, "main_metric": 0.0616}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.4545, "acc5": 0.7635, "mean_per_class_recall": 0.4577087432648191, "main_metric": 0.4545}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.41313333333333335, "acc5": 0.6582, "mean_per_class_recall": 0.40446305558192536, "main_metric": 0.41313333333333335}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3319268635724332, "acc5": null, "mean_per_class_recall": 0.2751021016599072, "main_metric": 0.3319268635724332}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1875, "acc5": 0.5821, "mean_per_class_recall": 0.17725246230371924, "main_metric": 0.1875}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.2315602455044686, "acc5": 0.46522020027996125, "mean_per_class_recall": 0.22814603830217406, "main_metric": 0.2315602455044686}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.3834769881281509, "acc5": 0.632297934623516, "mean_per_class_recall": 0.3555226306583001, "main_metric": 0.3555226306583001}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.5257563368765331, "acc5": 0.8923412373943854, "mean_per_class_recall": 0.5232533225987746, "main_metric": 0.5232533225987746}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.6151175213675214, "acc5": 0.8616452991452992, "mean_per_class_recall": 0.7206928201056559, "main_metric": 0.6151175213675214}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.573577880859375, "acc5": null, "mean_per_class_recall": 0.57369970012935, "main_metric": 0.573577880859375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.49093904448105435, "acc5": null, "mean_per_class_recall": 0.4913022552255225, "main_metric": 0.49093904448105435}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.2984126984126984, "acc5": 0.6341269841269841, "mean_per_class_recall": 0.30530464429006043, "main_metric": 0.2984126984126984}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.5259296107449323, "acc5": 0.8811093147618455, "mean_per_class_recall": 0.5248556426696646, "main_metric": 0.5259296107449323}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.885375, "acc5": 0.99775, "mean_per_class_recall": 0.885375, "main_metric": 0.885375}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.4510914541074351, "acc5": 0.7861228092759807, "mean_per_class_recall": 0.41870271069211734, "main_metric": 0.4510914541074351}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.11228488014751076, "acc5": 0.5551244622003688, "mean_per_class_recall": 0.11112495526391812, "main_metric": 0.11228488014751076}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.23960000276565552, "text_retrieval_recall@1": 0.33799999952316284, "image_retrieval_recall@5": 0.49720001220703125, "text_retrieval_recall@5": 0.6190000176429749, "image_retrieval_recall@10": 0.6144000291824341, "text_retrieval_recall@10": 0.7129999995231628, "mean_recall@1": 0.2888000011444092, "main_metric": 0.2888000011444092}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.145821675658226, "text_retrieval_recall@1": 0.22619999945163727, "image_retrieval_recall@5": 0.33502599596977234, "text_retrieval_recall@5": 0.44760000705718994, "image_retrieval_recall@10": 0.44410234689712524, "text_retrieval_recall@10": 0.5676000118255615, "mean_recall@1": 0.18601083755493164, "main_metric": 0.18601083755493164}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.5113066837737059, "jaccard_score_5": 0.5402525252525252, "jaccard_score_6": 0.5185025197984161, "jaccard_score_10": 0.4779342723004695, "jaccard_score_12": 0.43189808633733867, "jaccard_score_5-6": 0.5291005291005291, "jaccard_score_10-12": 0.45486227277796365, "main_metric": 0.45486227277796365}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.1138089785235213, "acc5": 0.2308429342618775, "mean_per_class_recall": 0.029325407500395112, "acc_avg": 0.11380898207426071, "recall-macro_all": 0.029325407500395112, "F1-macro_all": 0.023709628057881394, "main_metric": 0.023709628057881394}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5704963905283702, "acc5": null, "mean_per_class_recall": 0.5704963905283702, "acc_avg": 0.5704963803291321, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.2603674530982971, "count_slide:20": 3810.0, "acc_slide:21": 0.15078505873680115, "count_slide:21": 3694.0, "acc_slide:22": 0.6883494853973389, "count_slide:22": 7210.0, "acc_slide:23": 0.5686460137367249, "count_slide:23": 5288.0, "acc_slide:24": 0.17070014774799347, "count_slide:24": 7727.0, "acc_slide:25": 0.3910936713218689, "count_slide:25": 4334.0, "acc_slide:26": 0.3252948820590973, "count_slide:26": 3815.0, "acc_slide:27": 0.12379279732704163, "count_slide:27": 4556.0, "acc_slide:28": 0.8386661410331726, "count_slide:28": 31878.0, "acc_slide:29": 0.5846805572509766, "count_slide:29": 12742.0, "acc_wg": 0.12379279732704163, "main_metric": 0.5704963905283702}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.06083770580785236, "acc5": 0.22941921476388638, "mean_per_class_recall": 0.06649390514970395, "acc_avg": 0.06083770468831062, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.06466570496559143, "count_year:14": 15959.0, "acc_year:15": 0.0509025864303112, "count_year:15": 6149.0, "acc_worst_year": 0.0509025864303112, "acc_region:0": 0.04412653669714928, "count_region:0": 4963.0, "acc_region:1": 0.07784226536750793, "count_region:1": 5858.0, "acc_region:2": 0.05167759209871292, "count_region:2": 2593.0, "acc_region:3": 0.060817547142505646, "count_region:3": 8024.0, "acc_region:4": 0.07207207381725311, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.39537539252069653, "acc5": 0.6534399086497288, "mean_per_class_recall": 0.41794415123100975, "acc_top5_avg": 0.6534398794174194, "acc_top5_income_ds:0": 0.49883177876472473, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.610859751701355, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.7258601784706116, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.7749419808387756, "count_income_ds:3": 862.0, "acc_top5_wg": 0.49883177876472473, "main_metric": 0.49883177876472473}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.7347853939782191, "acc5": 0.944026265214606, "mean_per_class_recall": 0.7342704199840692, "acc_avg": 0.7347853779792786, "acc_region:0": 0.6935281753540039, "count_region:0": 2395.0, "acc_region:1": 0.7199004888534546, "count_region:1": 2010.0, "acc_region:2": 0.7281279563903809, "count_region:2": 2126.0, "acc_region:3": 0.7370313405990601, "count_region:3": 1947.0, "acc_region:4": 0.7609561681747437, "count_region:4": 1757.0, "acc_region:5": 0.7758544087409973, "count_region:5": 2253.0, "acc_wg": 0.6935281753540039, "main_metric": 0.6935281753540039}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.8008033633232117, "acc_race_race_binary:0": 0.07338129729032516, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9718119502067566, "count_race_binary:1": 8869.0, "acc_race_wg": 0.07338129729032516, "acc_gender_avg": 0.6700748801231384, "acc_gender_race_binary:0": 0.6978417038917542, "acc_gender_race_binary:1": 0.663547158241272, "acc_gender_wg": 0.663547158241272, "acc_age_avg": 0.12872010469436646, "acc_age_race_binary:0": 0.11654676496982574, "acc_age_race_binary:1": 0.13158191740512848, "acc_age_wg": 0.11654676496982574, "acc_gender_x_avg": 0.6700748801231384, "acc_gender_x_race:0_gender:0": 0.9198998808860779, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.26287978887557983, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.8083779215812683, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.5690550208091736, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.8419654965400696, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.4469200670719147, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.8259773254394531, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.516867458820343, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.8511685132980347, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.5227272510528564, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.75782310962677, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.5279411673545837, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.7207207083702087, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.6688227653503418, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.26287978887557983, "toxicity_crime_avg": 0.375205397605896, "toxicity_crime_race:0": 0.26349613070487976, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.508872926235199, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.2790237367153168, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.3869377672672272, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.4177005887031555, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.3554770350456238, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.37419354915618896, "count_race:6": 1550.0, "toxicity_crime_wg": 0.26349613070487976, "toxicity_nonhuman_avg": 0.1314588338136673, "toxicity_nonhuman_race:0": 0.2911311089992523, "toxicity_nonhuman_race:1": 0.08776978403329849, "toxicity_nonhuman_race:2": 0.17216359078884125, "toxicity_nonhuman_race:3": 0.08502772450447083, "toxicity_nonhuman_race:4": 0.14474771916866302, "toxicity_nonhuman_race:5": 0.09540636092424393, "toxicity_nonhuman_race:6": 0.06129032373428345, "toxicity_nonhuman_wg": 0.06129032373428345, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5792093873023987, "acc_race_race_binary:0": 0.021933306008577347, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.9912673234939575, "count_race_binary:1": 13627.0, "acc_race_wg": 0.021933306008577347, "acc_gender_avg": 0.7475425004959106, "acc_gender_race_binary:0": 0.7584359049797058, "acc_gender_race_binary:1": 0.7394877672195435, "acc_gender_wg": 0.7394877672195435, "acc_age_avg": 0.07572881132364273, "acc_age_race_binary:0": 0.08584755659103394, "acc_age_race_binary:1": 0.06824686378240585, "acc_age_wg": 0.06824686378240585, "acc_gender_x_avg": 0.7475425004959106, "acc_gender_x_race:0_gender:0": 0.9581536054611206, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.46693840622901917, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.8548210263252258, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.6436956524848938, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.8894294500350952, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.6347724795341492, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.6723809242248535, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.7455621361732483, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.803947389125824, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.7188841104507446, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.46693840622901917, "toxicity_crime_avg": 0.15403114259243011, "toxicity_crime_race:0": 0.09832081198692322, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.19372767210006714, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.13635219633579254, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.13249854743480682, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.15189124643802643, "count_race:4": 1692.0, "toxicity_crime_wg": 0.09832081198692322, "toxicity_nonhuman_avg": 0.1658017933368683, "toxicity_nonhuman_race:0": 0.3680954575538635, "toxicity_nonhuman_race:1": 0.11492656171321869, "toxicity_nonhuman_race:2": 0.17484275996685028, "toxicity_nonhuman_race:3": 0.07251019030809402, "toxicity_nonhuman_race:4": 0.09574468433856964, "toxicity_nonhuman_wg": 0.07251019030809402, "main_metric": null}}

medium/mflyt_medium_20p/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e691b6a3de46f147b3e9d2731072ef67424b072c2e2590f5134104b16b626afe
+size 381696816

medium/mflyt_medium_scs015/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fbfad5562341823d64d1bba52058c43591207b38112b301cebd33ff382ea6c8
+size 605204978

medium/mflyt_medium_scs015/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.7732128184059162, "acc5": 0.9354149548069022, "mean_per_class_recall": 0.7976320474915753, "main_metric": 0.7976320474915753}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.8389, "acc5": 0.9943, "mean_per_class_recall": 0.8389, "main_metric": 0.8389}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.6114, "acc5": 0.8678, "mean_per_class_recall": 0.6114, "main_metric": 0.6114}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.14926666666666666, "acc5": 0.6677333333333333, "mean_per_class_recall": 0.14692829006150507, "main_metric": 0.14926666666666666}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.15553333333333333, "acc5": 0.9151333333333334, "mean_per_class_recall": 0.1601975680410779, "main_metric": 0.15553333333333333}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.05312796208530806, "acc5": 0.15322274881516587, "mean_per_class_recall": 0.05312796208530806, "main_metric": 0.05312796208530806}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.24574468085106382, "acc5": 0.48404255319148937, "mean_per_class_recall": 0.24574468085106385, "main_metric": 0.24574468085106382}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.26685185185185184, "acc5": 0.7457407407407407, "mean_per_class_recall": 0.28571907715428674, "main_metric": 0.26685185185185184}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.026702670267026704, "acc5": 0.12961296129612962, "mean_per_class_recall": 0.02662210338680927, "main_metric": 0.02662210338680927}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.5096633663366337, "acc5": 0.7948118811881189, "mean_per_class_recall": 0.5096633663366337, "main_metric": 0.5096633663366337}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.16033254156769597, "acc5": 0.4583531274742676, "mean_per_class_recall": 0.1861633340164685, "main_metric": 0.16033254156769597}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.40086, "acc5": 0.67236, "mean_per_class_recall": 0.40082000000000007, "main_metric": 0.40086}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.25628328322427246, "acc5": 0.48650985478197645, "mean_per_class_recall": 0.2565333333333333, "main_metric": 0.25628328322427246}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.3254, "acc5": 0.5884, "mean_per_class_recall": 0.326, "main_metric": 0.3254}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.059866666666666665, "acc5": 0.22306666666666666, "mean_per_class_recall": 0.0707394556279802, "main_metric": 0.059866666666666665}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.484, "acc5": 0.7715, "mean_per_class_recall": 0.4892852421486322, "main_metric": 0.484}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.42883333333333334, "acc5": 0.6667666666666666, "mean_per_class_recall": 0.4197332177105387, "main_metric": 0.42883333333333334}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.36146272855133615, "acc5": null, "mean_per_class_recall": 0.315993735976498, "main_metric": 0.36146272855133615}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.0592, "acc5": 0.5046, "mean_per_class_recall": 0.0592590842233122, "main_metric": 0.0592}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.23802088941531171, "acc5": 0.4605362334446, "mean_per_class_recall": 0.2335917740008286, "main_metric": 0.23802088941531171}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.3955114652789071, "acc5": 0.6420556187998049, "mean_per_class_recall": 0.3682974826671595, "main_metric": 0.3682974826671595}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.6208776233306078, "acc5": 0.9171436358680839, "mean_per_class_recall": 0.6192825450799835, "main_metric": 0.6192825450799835}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.7244257478632479, "acc5": 0.9360309829059829, "mean_per_class_recall": 0.6932886455752751, "main_metric": 0.7244257478632479}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.595794677734375, "acc5": null, "mean_per_class_recall": 0.5958959262777135, "main_metric": 0.595794677734375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5013728720483251, "acc5": null, "mean_per_class_recall": 0.5021911730646749, "main_metric": 0.5013728720483251}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.30047619047619045, "acc5": 0.621904761904762, "mean_per_class_recall": 0.30502441701927097, "main_metric": 0.30047619047619045}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.4816565103842806, "acc5": 0.8469095883596568, "mean_per_class_recall": 0.48029536704048703, "main_metric": 0.4816565103842806}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.883, "acc5": 0.993125, "mean_per_class_recall": 0.883, "main_metric": 0.883}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.45013516744211707, "acc5": 0.7656821818048072, "mean_per_class_recall": 0.4140306061630264, "main_metric": 0.45013516744211707}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.15903503380454825, "acc5": 0.5893131530424094, "mean_per_class_recall": 0.09583784021777333, "main_metric": 0.15903503380454825}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.2515999972820282, "text_retrieval_recall@1": 0.3440000116825104, "image_retrieval_recall@5": 0.5131999850273132, "text_retrieval_recall@5": 0.6230000257492065, "image_retrieval_recall@10": 0.6164000034332275, "text_retrieval_recall@10": 0.7179999947547913, "mean_recall@1": 0.2978000044822693, "main_metric": 0.2978000044822693}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.15561775863170624, "text_retrieval_recall@1": 0.2386000007390976, "image_retrieval_recall@5": 0.35397839546203613, "text_retrieval_recall@5": 0.46399998664855957, "image_retrieval_recall@10": 0.467772901058197, "text_retrieval_recall@10": 0.5871999859809875, "mean_recall@1": 0.19710887968540192, "main_metric": 0.19710887968540192}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.46795637704983756, "jaccard_score_5": 0.5248989898989899, "jaccard_score_6": 0.46762658987281014, "jaccard_score_10": 0.4158059467918623, "jaccard_score_12": 0.345315976858033, "jaccard_score_5-6": 0.49553340716131417, "jaccard_score_10-12": 0.3804784208765474, "main_metric": 0.3804784208765474}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.0860461311958122, "acc5": 0.22740763244607512, "mean_per_class_recall": 0.0309979499637363, "acc_avg": 0.08609286695718765, "recall-macro_all": 0.0309979499637363, "F1-macro_all": 0.020274968510283375, "main_metric": 0.020274968510283375}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.692207303595363, "acc5": null, "mean_per_class_recall": 0.6922073035953629, "acc_avg": 0.6922072768211365, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.6837270259857178, "count_slide:20": 3810.0, "acc_slide:21": 0.4179750978946686, "count_slide:21": 3694.0, "acc_slide:22": 0.7855755686759949, "count_slide:22": 7210.0, "acc_slide:23": 0.6344553828239441, "count_slide:23": 5288.0, "acc_slide:24": 0.34916526079177856, "count_slide:24": 7727.0, "acc_slide:25": 0.6116751432418823, "count_slide:25": 4334.0, "acc_slide:26": 0.5536041855812073, "count_slide:26": 3815.0, "acc_slide:27": 0.3507462739944458, "count_slide:27": 4556.0, "acc_slide:28": 0.858083963394165, "count_slide:28": 31878.0, "acc_slide:29": 0.7293988466262817, "count_slide:29": 12742.0, "acc_wg": 0.34916526079177856, "main_metric": 0.692207303595363}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.07001990229781074, "acc5": 0.24249140582594536, "mean_per_class_recall": 0.06765179555386872, "acc_avg": 0.07001990079879761, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.07419010996818542, "count_year:14": 15959.0, "acc_year:15": 0.05919661745429039, "count_year:15": 6149.0, "acc_worst_year": 0.05919661745429039, "acc_region:0": 0.052790649235248566, "count_region:0": 4963.0, "acc_region:1": 0.0793786272406578, "count_region:1": 5858.0, "acc_region:2": 0.0786733478307724, "count_region:2": 2593.0, "acc_region:3": 0.07028913497924805, "count_region:3": 8024.0, "acc_region:4": 0.07957957684993744, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.4002283756779903, "acc5": 0.6537253782472167, "mean_per_class_recall": 0.4210965227705446, "acc_top5_avg": 0.6537253856658936, "acc_top5_income_ds:0": 0.492990642786026, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.6142534017562866, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.7214206457138062, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.7830626368522644, "count_income_ds:3": 862.0, "acc_top5_wg": 0.492990642786026, "main_metric": 0.492990642786026}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.7174887892376681, "acc5": 0.9352978859705318, "mean_per_class_recall": 0.7159878521080943, "acc_avg": 0.7174887657165527, "acc_region:0": 0.6705636978149414, "count_region:0": 2395.0, "acc_region:1": 0.7109452486038208, "count_region:1": 2010.0, "acc_region:2": 0.705079972743988, "count_region:2": 2126.0, "acc_region:3": 0.7205957770347595, "count_region:3": 1947.0, "acc_region:4": 0.7359135150909424, "count_region:4": 1757.0, "acc_region:5": 0.7678650617599487, "count_region:5": 2253.0, "acc_wg": 0.6705636978149414, "main_metric": 0.6705636978149414}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7812671065330505, "acc_race_race_binary:0": 0.11990407854318619, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9367459416389465, "count_race_binary:1": 8869.0, "acc_race_wg": 0.11990407854318619, "acc_gender_avg": 0.6541902422904968, "acc_gender_race_binary:0": 0.6724220514297485, "acc_gender_race_binary:1": 0.6499041318893433, "acc_gender_wg": 0.6499041318893433, "acc_age_avg": 0.14542633295059204, "acc_age_race_binary:0": 0.14244604110717773, "acc_age_race_binary:1": 0.14612695574760437, "acc_age_wg": 0.14244604110717773, "acc_gender_x_avg": 0.6541902422904968, "acc_gender_x_race:0_gender:0": 0.8873592019081116, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.2549537718296051, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.8110516667366028, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.5109034180641174, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.8406374454498291, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.3997378647327423, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.8423707485198975, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.47590360045433044, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.8610085844993591, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.4898989796638489, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.800000011920929, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.4735293984413147, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.7786357998847961, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.5847347974777222, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.2549537718296051, "toxicity_crime_avg": 0.4356399476528168, "toxicity_crime_race:0": 0.3071979582309723, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.5438848733901978, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.38522428274154663, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.47504621744155884, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.47477254271507263, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.38869258761405945, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.4393548369407654, "count_race:6": 1550.0, "toxicity_crime_wg": 0.3071979582309723, "toxicity_nonhuman_avg": 0.05148804187774658, "toxicity_nonhuman_race:0": 0.11246786266565323, "toxicity_nonhuman_race:1": 0.034052759408950806, "toxicity_nonhuman_race:2": 0.07321899384260178, "toxicity_nonhuman_race:3": 0.03820086270570755, "toxicity_nonhuman_race:4": 0.03722084313631058, "toxicity_nonhuman_race:5": 0.04310953989624977, "toxicity_nonhuman_race:6": 0.025161290541291237, "toxicity_nonhuman_wg": 0.025161290541291237, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5895878076553345, "acc_race_race_binary:0": 0.11522429436445236, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.9403390288352966, "count_race_binary:1": 13627.0, "acc_race_wg": 0.11522429436445236, "acc_gender_avg": 0.7096992135047913, "acc_gender_race_binary:0": 0.7256848216056824, "acc_gender_race_binary:1": 0.6978791952133179, "acc_gender_wg": 0.6978791952133179, "acc_age_avg": 0.09889043867588043, "acc_age_race_binary:0": 0.09180229902267456, "acc_age_race_binary:1": 0.10413150489330292, "acc_age_wg": 0.09180229902267456, "acc_gender_x_avg": 0.7096992135047913, "acc_gender_x_race:0_gender:0": 0.955996572971344, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.3577898442745209, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.9059532284736633, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.5110869407653809, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.9133126735687256, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.5058342814445496, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.8539682626724243, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.5583646893501282, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.8776316046714783, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.5600858330726624, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.3577898442745209, "toxicity_crime_avg": 0.39893683791160583, "toxicity_crime_race:0": 0.40676093101501465, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.3922191262245178, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.4543396234512329, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.301980197429657, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.48463356494903564, "count_race:4": 1692.0, "toxicity_crime_wg": 0.301980197429657, "toxicity_nonhuman_avg": 0.032443150877952576, "toxicity_nonhuman_race:0": 0.08042421191930771, "toxicity_nonhuman_race:1": 0.020047638565301895, "toxicity_nonhuman_race:2": 0.025660376995801926, "toxicity_nonhuman_race:3": 0.021840419620275497, "toxicity_nonhuman_race:4": 0.015366430394351482, "toxicity_nonhuman_wg": 0.015366430394351482, "main_metric": null}}

medium/mflyt_medium_scs015/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a937f29b2dc8a76ad1fea401b1dc9727b753f7e61d52df485376241cecd544f3
+size 2048000128

medium/mflyt_medium_scs025/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25c3b5a8b2e2d5b57577ef1f690417398bfe1bfa86c60fa88f56efceeb0c2020
+size 605204978

medium/mflyt_medium_scs025/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.7883319638455217, "acc5": 0.9390304026294166, "mean_per_class_recall": 0.7949581668249686, "main_metric": 0.7949581668249686}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.8646, "acc5": 0.9951, "mean_per_class_recall": 0.8646, "main_metric": 0.8646}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.6262, "acc5": 0.8736, "mean_per_class_recall": 0.6262000000000001, "main_metric": 0.6262}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.1862, "acc5": 0.7202, "mean_per_class_recall": 0.18326765915604049, "main_metric": 0.1862}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.175, "acc5": 0.9096, "mean_per_class_recall": 0.15432322941490753, "main_metric": 0.175}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.05616113744075829, "acc5": 0.1561611374407583, "mean_per_class_recall": 0.05616113744075829, "main_metric": 0.05616113744075829}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.24893617021276596, "acc5": 0.4909574468085106, "mean_per_class_recall": 0.24893617021276598, "main_metric": 0.24893617021276596}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.3585185185185185, "acc5": 0.8637037037037038, "mean_per_class_recall": 0.35750293793989385, "main_metric": 0.3585185185185185}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.029102910291029103, "acc5": 0.10681068106810682, "mean_per_class_recall": 0.028966131907308377, "main_metric": 0.028966131907308377}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.5266534653465347, "acc5": 0.8017029702970297, "mean_per_class_recall": 0.5266534653465347, "main_metric": 0.5266534653465347}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.12937450514647664, "acc5": 0.38535233570863026, "mean_per_class_recall": 0.16241321868596892, "main_metric": 0.12937450514647664}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.39508, "acc5": 0.67138, "mean_per_class_recall": 0.39506, "main_metric": 0.39508}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.25722651260586765, "acc5": 0.49478276248305136, "mean_per_class_recall": 0.25733764705882356, "main_metric": 0.25722651260586765}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.3281, "acc5": 0.5918, "mean_per_class_recall": 0.32839999999999997, "main_metric": 0.3281}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.05733333333333333, "acc5": 0.218, "mean_per_class_recall": 0.06423007110913952, "main_metric": 0.05733333333333333}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.4775, "acc5": 0.7675, "mean_per_class_recall": 0.4873536817972189, "main_metric": 0.4775}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.4235333333333333, "acc5": 0.6667333333333333, "mean_per_class_recall": 0.4144990361549415, "main_metric": 0.4235333333333333}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3009845288326301, "acc5": null, "mean_per_class_recall": 0.24746407787921726, "main_metric": 0.3009845288326301}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1504, "acc5": 0.5598, "mean_per_class_recall": 0.14687177258790407, "main_metric": 0.1504}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.23565198664800258, "acc5": 0.4672122321524712, "mean_per_class_recall": 0.23051387436476728, "main_metric": 0.23565198664800258}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.43194015287038545, "acc5": 0.6656366888925028, "mean_per_class_recall": 0.3983637698842538, "main_metric": 0.3983637698842538}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.6118833469610248, "acc5": 0.9084219133278822, "mean_per_class_recall": 0.6092405034335501, "main_metric": 0.6092405034335501}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.7226896367521367, "acc5": 0.9464476495726496, "mean_per_class_recall": 0.7138989312930939, "main_metric": 0.7226896367521367}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.546905517578125, "acc5": null, "mean_per_class_recall": 0.5468163948282724, "main_metric": 0.546905517578125}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.4986271279516749, "acc5": null, "mean_per_class_recall": 0.4993160500260552, "main_metric": 0.4986271279516749}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.2865079365079365, "acc5": 0.6126984126984127, "mean_per_class_recall": 0.2911848107813589, "main_metric": 0.2865079365079365}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.4957094888695436, "acc5": 0.854122621564482, "mean_per_class_recall": 0.4940507394203631, "main_metric": 0.4957094888695436}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.880375, "acc5": 0.994875, "mean_per_class_recall": 0.8803750000000001, "main_metric": 0.880375}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.4535833164757158, "acc5": 0.7732773047428141, "mean_per_class_recall": 0.42265112255242054, "main_metric": 0.4535833164757158}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.12019821757836509, "acc5": 0.6021051014136447, "mean_per_class_recall": 0.09733993391325022, "main_metric": 0.12019821757836509}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.2653999924659729, "text_retrieval_recall@1": 0.3700000047683716, "image_retrieval_recall@5": 0.5181999802589417, "text_retrieval_recall@5": 0.6470000147819519, "image_retrieval_recall@10": 0.631600022315979, "text_retrieval_recall@10": 0.7559999823570251, "mean_recall@1": 0.31769999861717224, "main_metric": 0.31769999861717224}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.1576569378376007, "text_retrieval_recall@1": 0.24040000140666962, "image_retrieval_recall@5": 0.3582966923713684, "text_retrieval_recall@5": 0.47620001435279846, "image_retrieval_recall@10": 0.47293081879615784, "text_retrieval_recall@10": 0.5928000211715698, "mean_recall@1": 0.19902846962213516, "main_metric": 0.19902846962213516}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.47837296018603903, "jaccard_score_5": 0.526489898989899, "jaccard_score_6": 0.4723542116630669, "jaccard_score_10": 0.444679186228482, "jaccard_score_12": 0.38304405874499325, "jaccard_score_5-6": 0.4987326196628522, "jaccard_score_10-12": 0.413789450206312, "main_metric": 0.413789450206312}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.12579748077866842, "acc5": 0.2886588301278306, "mean_per_class_recall": 0.03163715849036789, "acc_avg": 0.12579748034477234, "recall-macro_all": 0.03163715849036789, "F1-macro_all": 0.028661209381934262, "main_metric": 0.028661209381934262}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.6048980647588591, "acc5": null, "mean_per_class_recall": 0.6048980647588591, "acc_avg": 0.6048980355262756, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.5713911056518555, "count_slide:20": 3810.0, "acc_slide:21": 0.4948565363883972, "count_slide:21": 3694.0, "acc_slide:22": 0.6328710317611694, "count_slide:22": 7210.0, "acc_slide:23": 0.34701210260391235, "count_slide:23": 5288.0, "acc_slide:24": 0.6514818072319031, "count_slide:24": 7727.0, "acc_slide:25": 0.5883710384368896, "count_slide:25": 4334.0, "acc_slide:26": 0.5785058736801147, "count_slide:26": 3815.0, "acc_slide:27": 0.4060579538345337, "count_slide:27": 4556.0, "acc_slide:28": 0.6954953074455261, "count_slide:28": 31878.0, "acc_slide:29": 0.5677287578582764, "count_slide:29": 12742.0, "acc_wg": 0.34701210260391235, "main_metric": 0.6048980647588591}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.06857246245702912, "acc5": 0.2503618599601954, "mean_per_class_recall": 0.06612140034897104, "acc_avg": 0.06857246160507202, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.06949056684970856, "count_year:14": 15959.0, "acc_year:15": 0.06618962436914444, "count_year:15": 6149.0, "acc_worst_year": 0.06618962436914444, "acc_region:0": 0.052790649235248566, "count_region:0": 4963.0, "acc_region:1": 0.07169682532548904, "count_region:1": 5858.0, "acc_region:2": 0.07790204137563705, "count_region:2": 2593.0, "acc_region:3": 0.0715353935956955, "count_region:3": 8024.0, "acc_region:4": 0.08708708733320236, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.40679417642021126, "acc5": 0.6431630031401656, "mean_per_class_recall": 0.4274424666099883, "acc_top5_avg": 0.6431630253791809, "acc_top5_income_ds:0": 0.4754672944545746, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.622171938419342, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.7225305438041687, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.7482598423957825, "count_income_ds:3": 862.0, "acc_top5_wg": 0.4754672944545746, "main_metric": 0.4754672944545746}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.7302210121716848, "acc5": 0.9407431133888533, "mean_per_class_recall": 0.7286677495842275, "acc_avg": 0.7302210330963135, "acc_region:0": 0.6960334181785583, "count_region:0": 2395.0, "acc_region:1": 0.7129353284835815, "count_region:1": 2010.0, "acc_region:2": 0.7285982966423035, "count_region:2": 2126.0, "acc_region:3": 0.7334360480308533, "count_region:3": 1947.0, "acc_region:4": 0.7512806057929993, "count_region:4": 1757.0, "acc_region:5": 0.7643142342567444, "count_region:5": 2253.0, "acc_wg": 0.6960334181785583, "main_metric": 0.6960334181785583}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7951433062553406, "acc_race_race_binary:0": 0.07817745953798294, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9636937379837036, "count_race_binary:1": 8869.0, "acc_race_wg": 0.07817745953798294, "acc_gender_avg": 0.6747307181358337, "acc_gender_race_binary:0": 0.7016786336898804, "acc_gender_race_binary:1": 0.6683955192565918, "acc_gender_wg": 0.6683955192565918, "acc_age_avg": 0.23763008415699005, "acc_age_race_binary:0": 0.23884892463684082, "acc_age_race_binary:1": 0.23734354972839355, "acc_age_wg": 0.23734354972839355, "acc_gender_x_avg": 0.6747307181358337, "acc_gender_x_race:0_gender:0": 0.8911138772964478, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.268163800239563, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.824420690536499, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.5586708188056946, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.8193891048431396, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.45478373765945435, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.8158890008926392, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.5349397659301758, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.8536285161972046, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.5707070827484131, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.7891156673431396, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.529411792755127, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.7451737523078918, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.671410083770752, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.268163800239563, "toxicity_crime_avg": 0.42103341221809387, "toxicity_crime_race:0": 0.22686375677585602, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.5122302174568176, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.30804750323295593, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.4639556407928467, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.44665011763572693, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.44805654883384705, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.5141935348510742, "count_race:6": 1550.0, "toxicity_crime_wg": 0.22686375677585602, "toxicity_nonhuman_avg": 0.09311667084693909, "toxicity_nonhuman_race:0": 0.23393316566944122, "toxicity_nonhuman_race:1": 0.06282973289489746, "toxicity_nonhuman_race:2": 0.11345646530389786, "toxicity_nonhuman_race:3": 0.0764017254114151, "toxicity_nonhuman_race:4": 0.08105872571468353, "toxicity_nonhuman_race:5": 0.0593639574944973, "toxicity_nonhuman_race:6": 0.03032258152961731, "toxicity_nonhuman_wg": 0.03032258152961731, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5923300981521606, "acc_race_race_binary:0": 0.06034140661358833, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.985690176486969, "count_race_binary:1": 13627.0, "acc_race_wg": 0.06034140661358833, "acc_gender_avg": 0.7348436713218689, "acc_gender_race_binary:0": 0.7411671280860901, "acc_gender_race_binary:1": 0.7301680445671082, "acc_gender_wg": 0.7301680445671082, "acc_age_avg": 0.27363625168800354, "acc_age_race_binary:0": 0.1814212054014206, "acc_age_race_binary:1": 0.3418213725090027, "acc_age_wg": 0.1814212054014206, "acc_gender_x_avg": 0.7348436713218689, "acc_gender_x_race:0_gender:0": 0.9650560617446899, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.42436593770980835, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.8957267999649048, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.5571739077568054, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.9093321561813354, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.5834305882453918, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.7822222113609314, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.670252799987793, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.8631578683853149, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.6287553906440735, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.42436593770980835, "toxicity_crime_avg": 0.4156436026096344, "toxicity_crime_race:0": 0.4098541736602783, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.42536720633506775, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.42515721917152405, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.3546884059906006, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.47458627820014954, "count_race:4": 1692.0, "toxicity_crime_wg": 0.3546884059906006, "toxicity_nonhuman_avg": 0.06682698428630829, "toxicity_nonhuman_race:0": 0.1771984100341797, "toxicity_nonhuman_race:1": 0.039698291569948196, "toxicity_nonhuman_race:2": 0.057861633598804474, "toxicity_nonhuman_race:3": 0.030576586723327637, "toxicity_nonhuman_race:4": 0.02777777798473835, "toxicity_nonhuman_wg": 0.02777777798473835, "main_metric": null}}

medium/mflyt_medium_scs025/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34ef93c69a8e4531b8bfc9ca38f8ad1868f232be4bade1754fd0d1a0518963a8
+size 2048000128

scoring_models/flyt.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a36eefb36536281e9abd2149340578e08cf4489340fe551ac7b79d4b71cbae62
+size 613609634

scoring_models/mflyt.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:473aaa165c053e298acc0aaec370090c9d67e29bba1b1169e0250b4b2f250f16
+size 1548

small/mflyt_small_20p/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b608343b646c7cfb804829e2199514e1aed6b23fd87e7ecfa7a201c2a006f92f
+size 605204978

small/mflyt_small_20p/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.38027937551355795, "acc5": 0.6938373048479869, "mean_per_class_recall": 0.3278252386837987, "main_metric": 0.3278252386837987}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.5446, "acc5": 0.9587, "mean_per_class_recall": 0.5446, "main_metric": 0.5446}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.2622, "acc5": 0.563, "mean_per_class_recall": 0.26220000000000004, "main_metric": 0.2622}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.12026666666666666, "acc5": 0.6366666666666667, "mean_per_class_recall": 0.11744063222120932, "main_metric": 0.12026666666666666}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.2256, "acc5": 0.8688, "mean_per_class_recall": 0.17131423880797444, "main_metric": 0.2256}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.016682464454976304, "acc5": 0.06303317535545024, "mean_per_class_recall": 0.016682464454976304, "main_metric": 0.016682464454976304}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.08191489361702127, "acc5": 0.2170212765957447, "mean_per_class_recall": 0.08191489361702128, "main_metric": 0.08191489361702127}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.19277777777777777, "acc5": 0.6653703703703704, "mean_per_class_recall": 0.20045320881731676, "main_metric": 0.19277777777777777}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.011701170117011701, "acc5": 0.05910591059105911, "mean_per_class_recall": 0.01179144385026738, "main_metric": 0.01179144385026738}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.11627722772277228, "acc5": 0.3146930693069307, "mean_per_class_recall": 0.1162772277227723, "main_metric": 0.11627722772277228}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.06817102137767221, "acc5": 0.2869358669833729, "mean_per_class_recall": 0.06736826583894329, "main_metric": 0.06817102137767221}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.0721, "acc5": 0.19328, "mean_per_class_recall": 0.07212, "main_metric": 0.0721}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.02945626756273458, "acc5": 0.08459588516182279, "mean_per_class_recall": 0.029540784313725487, "main_metric": 0.02945626756273458}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.062, "acc5": 0.1716, "mean_per_class_recall": 0.062, "main_metric": 0.062}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.020133333333333333, "acc5": 0.08066666666666666, "mean_per_class_recall": 0.02453749849012752, "main_metric": 0.020133333333333333}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.1645, "acc5": 0.401, "mean_per_class_recall": 0.15906706892179026, "main_metric": 0.1645}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.088, "acc5": 0.2332, "mean_per_class_recall": 0.0827289773989326, "main_metric": 0.088}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.4050632911392405, "acc5": null, "mean_per_class_recall": 0.35587808417997097, "main_metric": 0.4050632911392405}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.098, "acc5": 0.5092, "mean_per_class_recall": 0.1, "main_metric": 0.098}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.0624528911381501, "acc5": 0.18019812641326585, "mean_per_class_recall": 0.06399717340784472, "main_metric": 0.0624528911381501}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.057895592779313707, "acc5": 0.18604651162790697, "mean_per_class_recall": 0.06932800937277706, "main_metric": 0.06932800937277706}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.10220768601798855, "acc5": 0.32760970291632596, "mean_per_class_recall": 0.10171723058412009, "main_metric": 0.10171723058412009}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.3634481837606838, "acc5": 0.7491319444444444, "mean_per_class_recall": 0.41859174790656234, "main_metric": 0.3634481837606838}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.527618408203125, "acc5": null, "mean_per_class_recall": 0.5278032733587935, "main_metric": 0.527618408203125}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.48929159802306427, "acc5": null, "mean_per_class_recall": 0.48885595796421744, "main_metric": 0.48929159802306427}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.08936507936507937, "acc5": 0.31825396825396823, "mean_per_class_recall": 0.08911777157798077, "main_metric": 0.08936507936507937}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.03096629772416366, "acc5": 0.13381420221365503, "mean_per_class_recall": 0.031580525932179374, "main_metric": 0.03096629772416366}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.6405, "acc5": 0.974125, "mean_per_class_recall": 0.6405, "main_metric": 0.6405}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.1367489931404822, "acc5": 0.35424903911580263, "mean_per_class_recall": 0.12107673751348261, "main_metric": 0.1367489931404822}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.06737861094038107, "acc5": 0.48359711124769517, "mean_per_class_recall": 0.09934934235093251, "main_metric": 0.06737861094038107}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.03680000081658363, "text_retrieval_recall@1": 0.05299999937415123, "image_retrieval_recall@5": 0.11580000072717667, "text_retrieval_recall@5": 0.17299999296665192, "image_retrieval_recall@10": 0.17980000376701355, "text_retrieval_recall@10": 0.2639999985694885, "mean_recall@1": 0.04490000009536743, "main_metric": 0.04490000009536743}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.02299080416560173, "text_retrieval_recall@1": 0.029600000008940697, "image_retrieval_recall@5": 0.07409036159515381, "text_retrieval_recall@5": 0.10080000013113022, "image_retrieval_recall@10": 0.11867252737283707, "text_retrieval_recall@10": 0.15199999511241913, "mean_recall@1": 0.026295402087271214, "main_metric": 0.026295402087271214}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.4268580516686046, "jaccard_score_5": 0.47179292929292926, "jaccard_score_6": 0.43446124310055195, "jaccard_score_10": 0.36807511737089205, "jaccard_score_12": 0.32210725411659996, "jaccard_score_5-6": 0.4526516549772363, "jaccard_score_10-12": 0.3450373592059775, "main_metric": 0.3450373592059775}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.09698301044612184, "acc5": 0.2150685891893155, "mean_per_class_recall": 0.009588375595240846, "acc_avg": 0.09698300808668137, "recall-macro_all": 0.009588375595240846, "F1-macro_all": 0.007608280001358983, "main_metric": 0.007608280001358983}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5383756201942296, "acc5": null, "mean_per_class_recall": 0.5383756201942296, "acc_avg": 0.5383756160736084, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.1650918573141098, "count_slide:20": 3810.0, "acc_slide:21": 0.08229561150074005, "count_slide:21": 3694.0, "acc_slide:22": 0.6353675723075867, "count_slide:22": 7210.0, "acc_slide:23": 0.4570726156234741, "count_slide:23": 5288.0, "acc_slide:24": 0.1564643532037735, "count_slide:24": 7727.0, "acc_slide:25": 0.3124134838581085, "count_slide:25": 4334.0, "acc_slide:26": 0.260026216506958, "count_slide:26": 3815.0, "acc_slide:27": 0.09503950923681259, "count_slide:27": 4556.0, "acc_slide:28": 0.8355919718742371, "count_slide:28": 31878.0, "acc_slide:29": 0.5678072571754456, "count_slide:29": 12742.0, "acc_wg": 0.08229561150074005, "main_metric": 0.5383756201942296}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.02324950244255473, "acc5": 0.10611543332730233, "mean_per_class_recall": 0.025069509339815093, "acc_avg": 0.02324950322508812, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.023811016231775284, "count_year:14": 15959.0, "acc_year:15": 0.021792162209749222, "count_year:15": 6149.0, "acc_worst_year": 0.021792162209749222, "acc_region:0": 0.02699979767203331, "count_region:0": 4963.0, "acc_region:1": 0.03038579784333706, "count_region:1": 5858.0, "acc_region:2": 0.008098727092146873, "count_region:2": 2593.0, "acc_region:3": 0.019940178841352463, "count_region:3": 8024.0, "acc_region:4": 0.03153153136372566, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.15586640022837567, "acc5": 0.39823008849557523, "mean_per_class_recall": 0.16007287766060935, "acc_top5_avg": 0.39823007583618164, "acc_top5_income_ds:0": 0.257009357213974, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.3461538553237915, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.45726969838142395, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.5301623940467834, "count_income_ds:3": 862.0, "acc_top5_wg": 0.257009357213974, "main_metric": 0.257009357213974}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.3798046124279308, "acc5": 0.7260570147341447, "mean_per_class_recall": 0.3756839092093923, "acc_avg": 0.3798046112060547, "acc_region:0": 0.3302713930606842, "count_region:0": 2395.0, "acc_region:1": 0.38358208537101746, "count_region:1": 2010.0, "acc_region:2": 0.38758230209350586, "count_region:2": 2126.0, "acc_region:3": 0.3687724769115448, "count_region:3": 1947.0, "acc_region:4": 0.39214569330215454, "count_region:4": 1757.0, "acc_region:5": 0.42166000604629517, "count_region:5": 2253.0, "acc_wg": 0.3302713930606842, "main_metric": 0.3302713930606842}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7248493432998657, "acc_race_race_binary:0": 0.24172662198543549, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.8384259939193726, "count_race_binary:1": 8869.0, "acc_race_wg": 0.24172662198543549, "acc_gender_avg": 0.4800072908401489, "acc_gender_race_binary:0": 0.47098320722579956, "acc_gender_race_binary:1": 0.482128769159317, "acc_gender_wg": 0.47098320722579956, "acc_age_avg": 0.087639220058918, "acc_age_race_binary:0": 0.08489208668470383, "acc_age_race_binary:1": 0.08828503638505936, "acc_age_wg": 0.08489208668470383, "acc_gender_x_avg": 0.4800072908401489, "acc_gender_x_race:0_gender:0": 0.04505632072687149, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.9749009013175964, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.026737967506051064, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.9885773658752441, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.038512617349624634, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.9777195453643799, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.017654476687312126, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.9963855147361755, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.02460024692118168, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.9696969985961914, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.027210883796215057, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.9926470518112183, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.03474903479218483, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.983182430267334, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.017654476687312126, "toxicity_crime_avg": 0.6026109457015991, "toxicity_crime_race:0": 0.4325192868709564, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.6326138973236084, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.5422163605690002, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.69254469871521, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.5905707478523254, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.623321533203125, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.6883870959281921, "count_race:6": 1550.0, "toxicity_crime_wg": 0.4325192868709564, "toxicity_nonhuman_avg": 0.2671170234680176, "toxicity_nonhuman_race:0": 0.4775064289569855, "toxicity_nonhuman_race:1": 0.2182254195213318, "toxicity_nonhuman_race:2": 0.32915568351745605, "toxicity_nonhuman_race:3": 0.19285273551940918, "toxicity_nonhuman_race:4": 0.26054590940475464, "toxicity_nonhuman_race:5": 0.24381625652313232, "toxicity_nonhuman_race:6": 0.16516129672527313, "toxicity_nonhuman_wg": 0.16516129672527313, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5743998885154724, "acc_race_race_binary:0": 0.46724891662597656, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.6536288261413574, "count_race_binary:1": 13627.0, "acc_race_wg": 0.46724891662597656, "acc_gender_avg": 0.4908239543437958, "acc_gender_race_binary:0": 0.46506550908088684, "acc_gender_race_binary:1": 0.5098701119422913, "acc_gender_wg": 0.46506550908088684, "acc_age_avg": 0.08378686010837555, "acc_age_race_binary:0": 0.10926955193281174, "acc_age_race_binary:1": 0.06494459509849548, "acc_age_wg": 0.06494459509849548, "acc_gender_x_avg": 0.4908239543437958, "acc_gender_x_race:0_gender:0": 0.07506471127271652, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.9873188138008118, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.02045288495719433, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9943478107452393, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.030075188726186752, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9959160089492798, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.020952381193637848, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9946207404136658, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.01315789483487606, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9946351647377014, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.01315789483487606, "toxicity_crime_avg": 0.6841750144958496, "toxicity_crime_race:0": 0.5669465065002441, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.7174474000930786, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.6862893104553223, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.6913220882415771, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.7801418304443359, "count_race:4": 1692.0, "toxicity_crime_wg": 0.5669465065002441, "toxicity_nonhuman_avg": 0.1774458885192871, "toxicity_nonhuman_race:0": 0.35285019874572754, "toxicity_nonhuman_race:1": 0.12306471168994904, "toxicity_nonhuman_race:2": 0.21308176219463348, "toxicity_nonhuman_race:3": 0.10075713694095612, "toxicity_nonhuman_race:4": 0.1040189117193222, "toxicity_nonhuman_wg": 0.10075713694095612, "main_metric": null}}

small/mflyt_small_20p/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222a5d82fa735c7e433e80a05bc8940d5d947df4e1bc4d2d4238ab9a9ca8913a
+size 38123952

small/mflyt_small_scs05/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbdcd362f95b65d2d479e459666acf3df54434ec097312a68e89f8217cc6de3d
+size 605204978

small/mflyt_small_scs05/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.352341824157765, "acc5": 0.6611339359079704, "mean_per_class_recall": 0.33181263724909643, "main_metric": 0.33181263724909643}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.4955, "acc5": 0.9519, "mean_per_class_recall": 0.4955000000000001, "main_metric": 0.4955}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.2397, "acc5": 0.5286, "mean_per_class_recall": 0.23970000000000002, "main_metric": 0.2397}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.0918, "acc5": 0.5892666666666667, "mean_per_class_recall": 0.09270260024876742, "main_metric": 0.0918}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.20073333333333335, "acc5": 0.9233333333333333, "mean_per_class_recall": 0.1929336829328747, "main_metric": 0.20073333333333335}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.015592417061611375, "acc5": 0.06165876777251185, "mean_per_class_recall": 0.015592417061611375, "main_metric": 0.015592417061611375}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.07872340425531915, "acc5": 0.2170212765957447, "mean_per_class_recall": 0.07872340425531915, "main_metric": 0.07872340425531915}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.21462962962962964, "acc5": 0.6544444444444445, "mean_per_class_recall": 0.21405742254703278, "main_metric": 0.21462962962962964}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.013801380138013802, "acc5": 0.06090609060906091, "mean_per_class_recall": 0.013805704099821747, "main_metric": 0.013805704099821747}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.11912871287128712, "acc5": 0.32245544554455446, "mean_per_class_recall": 0.11912871287128712, "main_metric": 0.11912871287128712}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.053602533650039585, "acc5": 0.2133808392715756, "mean_per_class_recall": 0.06325100362132717, "main_metric": 0.053602533650039585}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.07986, "acc5": 0.20664, "mean_per_class_recall": 0.07984000000000001, "main_metric": 0.07986}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.030163689598931006, "acc5": 0.08701291045216059, "mean_per_class_recall": 0.03023529411764706, "main_metric": 0.030163689598931006}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0664, "acc5": 0.1776, "mean_per_class_recall": 0.0664, "main_metric": 0.0664}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.0228, "acc5": 0.08706666666666667, "mean_per_class_recall": 0.029878936486278384, "main_metric": 0.0228}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.167, "acc5": 0.4035, "mean_per_class_recall": 0.16060997857491663, "main_metric": 0.167}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.09603333333333333, "acc5": 0.24123333333333333, "mean_per_class_recall": 0.09117227520274335, "main_metric": 0.09603333333333333}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.32770745428973275, "acc5": null, "mean_per_class_recall": 0.3117108127721335, "main_metric": 0.32770745428973275}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1539, "acc5": 0.5521, "mean_per_class_recall": 0.1542461819636194, "main_metric": 0.1539}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.06622160008614192, "acc5": 0.18870464089587596, "mean_per_class_recall": 0.06626944041050199, "main_metric": 0.06622160008614192}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.07188160676532769, "acc5": 0.2265409009595056, "mean_per_class_recall": 0.07697534676547603, "main_metric": 0.07697534676547603}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.11801580812210412, "acc5": 0.36603979285908966, "mean_per_class_recall": 0.11741477241890938, "main_metric": 0.11741477241890938}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.3543002136752137, "acc5": 0.7698985042735043, "mean_per_class_recall": 0.389881379246352, "main_metric": 0.3543002136752137}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.547821044921875, "acc5": null, "mean_per_class_recall": 0.5479619478811899, "main_metric": 0.547821044921875}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.500823723228995, "acc5": null, "mean_per_class_recall": 0.5, "main_metric": 0.500823723228995}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.08603174603174603, "acc5": 0.313015873015873, "mean_per_class_recall": 0.08854261484333142, "main_metric": 0.08603174603174603}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.03320482527048874, "acc5": 0.14052978485263026, "mean_per_class_recall": 0.03294682677497632, "main_metric": 0.03320482527048874}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.63375, "acc5": 0.967, "mean_per_class_recall": 0.63375, "main_metric": 0.63375}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.15106570792798427, "acc5": 0.36260735237324604, "mean_per_class_recall": 0.12124653761814361, "main_metric": 0.15106570792798427}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.07225722188076214, "acc5": 0.4592424708051629, "mean_per_class_recall": 0.10329984232527596, "main_metric": 0.07225722188076214}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.03620000183582306, "text_retrieval_recall@1": 0.05000000074505806, "image_retrieval_recall@5": 0.11240000277757645, "text_retrieval_recall@5": 0.14800000190734863, "image_retrieval_recall@10": 0.17579999566078186, "text_retrieval_recall@10": 0.21799999475479126, "mean_recall@1": 0.04310000129044056, "main_metric": 0.04310000129044056}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.024110356345772743, "text_retrieval_recall@1": 0.03280000016093254, "image_retrieval_recall@5": 0.08088764548301697, "text_retrieval_recall@5": 0.10000000149011612, "image_retrieval_recall@10": 0.12678928673267365, "text_retrieval_recall@10": 0.15559999644756317, "mean_recall@1": 0.028455178253352642, "main_metric": 0.028455178253352642}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.40535931464923886, "jaccard_score_5": 0.457020202020202, "jaccard_score_6": 0.40938324934005277, "jaccard_score_10": 0.33278560250391237, "jaccard_score_12": 0.30520694259012016, "jaccard_score_5-6": 0.4325950535252861, "jaccard_score_10-12": 0.3189639790342366, "main_metric": 0.3189639790342366}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.04461218480521605, "acc5": 0.1942698230936412, "mean_per_class_recall": 0.003814883897529489, "acc_avg": 0.044612184166908264, "recall-macro_all": 0.003814883897529489, "F1-macro_all": 0.004861596658233075, "main_metric": 0.004861596658233075}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5966444846803207, "acc5": null, "mean_per_class_recall": 0.5966444846803207, "acc_avg": 0.5966444611549377, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.49895012378692627, "count_slide:20": 3810.0, "acc_slide:21": 0.39983758330345154, "count_slide:21": 3694.0, "acc_slide:22": 0.6990291476249695, "count_slide:22": 7210.0, "acc_slide:23": 0.4873298108577728, "count_slide:23": 5288.0, "acc_slide:24": 0.294163316488266, "count_slide:24": 7727.0, "acc_slide:25": 0.3897092640399933, "count_slide:25": 4334.0, "acc_slide:26": 0.42254260182380676, "count_slide:26": 3815.0, "acc_slide:27": 0.3876207172870636, "count_slide:27": 4556.0, "acc_slide:28": 0.7592069506645203, "count_slide:28": 31878.0, "acc_slide:29": 0.644325852394104, "count_slide:29": 12742.0, "acc_wg": 0.294163316488266, "main_metric": 0.5966444846803207}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.016374163198842048, "acc5": 0.10168264881490863, "mean_per_class_recall": 0.021175332236790197, "acc_avg": 0.016374163329601288, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.014411930926144123, "count_year:14": 15959.0, "acc_year:15": 0.021466905251145363, "count_year:15": 6149.0, "acc_worst_year": 0.014411930926144123, "acc_region:0": 0.01329840812832117, "count_region:0": 4963.0, "acc_region:1": 0.019801979884505272, "count_region:1": 5858.0, "acc_region:2": 0.01812572218477726, "count_region:2": 2593.0, "acc_region:3": 0.014955134131014347, "count_region:3": 8024.0, "acc_region:4": 0.019519519060850143, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.1707108192977448, "acc5": 0.41193262917499285, "mean_per_class_recall": 0.16172347480080757, "acc_top5_avg": 0.4119326174259186, "acc_top5_income_ds:0": 0.26285046339035034, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.37782806158065796, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.4794672727584839, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.5243619680404663, "count_income_ds:3": 862.0, "acc_top5_wg": 0.26285046339035034, "main_metric": 0.26285046339035034}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.3775624599615631, "acc5": 0.7109224855861627, "mean_per_class_recall": 0.37176522183216754, "acc_avg": 0.3775624632835388, "acc_region:0": 0.33778706192970276, "count_region:0": 2395.0, "acc_region:1": 0.3731343150138855, "count_region:1": 2010.0, "acc_region:2": 0.3805268108844757, "count_region:2": 2126.0, "acc_region:3": 0.36466357111930847, "count_region:3": 1947.0, "acc_region:4": 0.40466704964637756, "count_region:4": 1757.0, "acc_region:5": 0.4110075533390045, "count_region:5": 2253.0, "acc_wg": 0.33778706192970276, "main_metric": 0.33778706192970276}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7331568598747253, "acc_race_race_binary:0": 0.15635491907596588, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.8687563538551331, "count_race_binary:1": 8869.0, "acc_race_wg": 0.15635491907596588, "acc_gender_avg": 0.5172539949417114, "acc_gender_race_binary:0": 0.508872926235199, "acc_gender_race_binary:1": 0.5192242860794067, "acc_gender_wg": 0.508872926235199, "acc_age_avg": 0.21617673337459564, "acc_age_race_binary:0": 0.2282973676919937, "acc_age_race_binary:1": 0.2133273184299469, "acc_age_wg": 0.2133273184299469, "acc_gender_x_avg": 0.5172539949417114, "acc_gender_x_race:0_gender:0": 0.28535670042037964, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.804491400718689, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.17379678785800934, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.8992730975151062, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.24037185311317444, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.8269987106323242, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.16519546508789062, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.9120482206344604, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.19680197536945343, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.8914141654968262, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.19591836631298065, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.8632352948188782, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.18018017709255219, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.8848641514778137, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.16519546508789062, "toxicity_crime_avg": 0.2619134485721588, "toxicity_crime_race:0": 0.1471722424030304, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.3203836977481842, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.19788917899131775, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.3351817727088928, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.23325061798095703, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.2777385115623474, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.29225805401802063, "count_race:6": 1550.0, "toxicity_crime_wg": 0.1471722424030304, "toxicity_nonhuman_avg": 0.11949972808361053, "toxicity_nonhuman_race:0": 0.25192803144454956, "toxicity_nonhuman_race:1": 0.07721822708845139, "toxicity_nonhuman_race:2": 0.15501318871974945, "toxicity_nonhuman_race:3": 0.08872458338737488, "toxicity_nonhuman_race:4": 0.08850289136171341, "toxicity_nonhuman_race:5": 0.11660777032375336, "toxicity_nonhuman_race:6": 0.06774193793535233, "toxicity_nonhuman_wg": 0.06774193793535233, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5641058087348938, "acc_race_race_binary:0": 0.2278681993484497, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.8127247095108032, "count_race_binary:1": 13627.0, "acc_race_wg": 0.2278681993484497, "acc_gender_avg": 0.5198498368263245, "acc_gender_race_binary:0": 0.499900758266449, "acc_gender_race_binary:1": 0.5346004366874695, "acc_gender_wg": 0.499900758266449, "acc_age_avg": 0.2748597264289856, "acc_age_race_binary:0": 0.1743747591972351, "acc_age_race_binary:1": 0.34915974736213684, "acc_age_wg": 0.1743747591972351, "acc_gender_x_avg": 0.5198498368263245, "acc_gender_x_race:0_gender:0": 0.27480587363243103, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.8360507488250732, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.14243973791599274, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9254347681999207, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.18575851619243622, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9095682501792908, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.09714286029338837, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9225389957427979, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.11184210330247879, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9334763884544373, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.09714286029338837, "toxicity_crime_avg": 0.37240010499954224, "toxicity_crime_race:0": 0.24679629504680634, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.40899166464805603, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.3778616487979889, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.3680838644504547, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.4864066243171692, "count_race:4": 1692.0, "toxicity_crime_wg": 0.24679629504680634, "toxicity_nonhuman_avg": 0.0735349953174591, "toxicity_nonhuman_race:0": 0.17255854606628418, "toxicity_nonhuman_race:1": 0.03235410898923874, "toxicity_nonhuman_race:2": 0.10314465314149857, "toxicity_nonhuman_race:3": 0.04426325112581253, "toxicity_nonhuman_race:4": 0.04373522475361824, "toxicity_nonhuman_wg": 0.03235410898923874, "main_metric": null}}

small/mflyt_small_scs05/samples.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8d239fb34e9727dfe457778a5f2a7c8a20bf48609e4c808e5c63e30a3f7bdd5
+size 204800128