Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021eidolonI-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	vision_transformer_vit_large_patch16_224	.789
2	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.783
3	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.777
4	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.776
5	resnext101_32x16d_wsl	.731
6	vit_base_patch16_clip_224:openai_ft_in1k	.724
7	SWSL_resnext101_32x8d	.715
8	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.698
9	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.689
10	ViT_B_16_imagenet1k	.677
11	resnext101_32x48d_wsl	.667
12	resnet_101_v2	.663
13	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.661
14	resnext101_32x8d_wsl	.655
15	CLIP_ViT-B_32	.651
16	SWSL_resnet50	.647
17	AdvProp_efficientnet-b7	.646
18	resnet_152_v2	.638
19	ViT_L_32_imagenet1k	.637
20	resnext101_32x32d_wsl	.637
21	vit_large_patch14_clip_224:laion2b_ft_in1k	.636
22	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.629
23	resnet101	.629
24	effnetv2m_custom384	.621
25	AT_efficientnet-b4	.619
26	AdvProp_efficientnet-b4	.619
27	vit_large_patch14_clip_224:openai_ft_in1k	.619
28	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.615
29	effnetb1_272x240	.614
30	vit_relpos_base_patch16_clsgap_224:sw_in1k	.610
31	resnet152_imagenet_full	.607
32	focalnet_tiny_lrf_in1k	.603
33	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.593
34	pnasnet_large	.593
35	pnasnet_large-tf	.584
36	efficientnet_b0	.583
37	fixres_resnext101_32x48d_wsl	.582
38	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.577
39	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.575
40	vit_large_patch14_clip_336:laion2b_ft_in1k	.574
41	convnext_xlarge:fb_in22k_ft_in1k	.574
42	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.567
43	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.556
44	AdvProp_efficientnet-b6	.553
45	cvt_cvt-13-384-in1k_4_LucyV4	.551
46	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.549
47	resnet_101_v1	.546
48	AdvProp_efficientnet-b2	.541
49	AT_efficientnet-b2	.541
50	cvt_cvt-13-224-in1k_4_LucyV4	.537
51	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.529
52	vonegrcnn_62e_nobn	.527
53	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.527
54	antialias-resnet152	.524
55	ViT_B_32_imagenet1k	.522
56	resnet-152_v2-tf	.520
57	inception_v4	.519
58	CLIP_resnet50_float32	.513
59	CLIP_resnet50	.513
60	custom_model_cv_18_dagger_408	.512
61	nasnet_large	.505
62	nasnet_large-tf	.499
63	convnext_tiny_imagenet_full_seed-0	.498
64	vonegrcnn_47e	.497
65	convnext_large:fb_in22k_ft_in1k	.492
66	resnet_50_v2	.491
67	vonegrcnn_52e_full	.487
68	convnext_femto_ols:d1_in1k	.482
69	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.481
70	vonegrcnn_62e_nobn	.476
71	convnext_base:clip_laiona_augreg_ft_in1k_384	.472
72	convnext_tiny:in12k_ft_in1k	.471
73	resnet_152_v1	.470
74	cvt_cvt-21-384-in1k_4_LucyV4	.464
75	resnet50_linf_4_robust	.459
76	resnet-152_v2_pytorch	.459
77	resnet-152_v1-tf	.459
78	vonegrcnn_47e	.456
79	efficientnet-b7	.453
80	densenet-201-keras	.448
81	densenet_201_pytorch	.446
82	inception_v3_pytorch	.445
83	inception_v4-tf	.443
84	resnet50_imagenet_full	.443
85	densenet-201	.443
86	resnet34_imagenet_full	.442
87	antialiased-rnext101_32x8d	.436
88	vonegrcnn_52e_full	.435
89	resnet_SIN_IN_FT_IN	.432
90	resnet50-SIN_IN_IN	.432
91	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.431
92	resnet50-VITO-8deg-cc	.430
93	cvt_cvt-21-224-in1k_4_LucyV4	.429
94	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.424
95	voneresnet-50-non_stochastic	.417
96	convnext_small_imagenet_100_seed-0	.417
97	deit_large_imagenet_full_seed-0	.416
98	resnet101_imagenet_full	.412
99	CLIP-RN50	.409
100	BiT-S-R50x3	.408
101	convnext_base_imagenet_full_seed-0	.406
102	voneresnet-50-robust	.403
103	xception-keras	.401
104	resnet50-sup	.400
105	resnet-50-pytorch	.400
106	resnet_50_v1	.400
107	resnet50_tutorial	.400
108	resnet50-moclr8deg	.394
109	resnet50_random_linf8_perturb	.393
110	ReAlnet03	.384
111	efficientnet_b2_imagenet_full	.383
112	resnet-50_v2-tf	.378
113	inception_v3-tf	.377
114	resnet50_random_l2_perturb	.371
115	convnext_small_imagenet_full_seed-0	.367
116	resnet-101_v1-tf	.363
117	resnet-50_v1-tf	.359
118	ReAlnet02	.355
119	resnet50-SIN_IN	.355
120	convnext_large_imagenet_full_seed-0	.354
121	Res2Net50_26w_4s	.350
122	AdvProp_efficientnet-b8	.349
123	densenet-121-keras	.346
124	resnet50	.344
125	inception_v3	.341
126	deit_small_imagenet_full_seed-0	.338
127	BiT-S-R152x2	.336
128	tv_efficientnet-b1	.336
129	nasnet_mobile	.335
130	ReAlnet04	.334
131	grcnn_109	.333
132	blt_vs	.331
133	ReAlnet07	.329
134	ReAlnet01	.328
135	xception	.327
136	ReAlnet06	.326
137	grcnn	.325
138	inception_v1	.324
139	ReAlnet08	.324
140	mobilenet_v2_1.0_192-tf	.318
141	antialiased-r50	.318
142	ReAlnet05	.317
143	inception_v2-tf	.309
144	densenet-121	.304
145	mobilenet_v2_1_0_192	.301
146	deit_base_imagenet_full_seed-0	.299
147	vonealexnet_gaussian_noise_std4_fixed	.297
148	ReAlnet10	.297
149	CORnet-S	.297
150	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.296
151	resnet18_imagenet_full	.295
152	vgg_19	.295
153	cornet_s	.293
154	resnet50_robust_l2_eps1	.291
155	alexnet	.290
156	regnet_y_400mf	.288
157	mobilenet_v1_0.75_224	.287
158	resnet50_linf_8_robust	.287
159	texture_shape_resnet50_trained_on_SIN	.286
160	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.281
161	voneresnet-50	.281
162	alexnet	.281
163	alexnet-baseline	.281
164	alexnet	.281
165	alexnet	.281
166	alexnet	.281
167	alexnet	.281
168	alexnet	.281
169	alexnet	.281
170	alexnet	.281
171	alexnet	.281
172	ReAlnet09	.277
173	mobilenet_v2_1_0_224	.276
174	densenet-169-keras	.274
175	resnet50_l2_3_robust	.272
176	mobilenet_v1_1.0_192	.269
177	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.264
178	alexnet_random_linf8_perturb	.260
179	resnet50_moco_v2	.256
180	vgg_16	.255
181	mobilenet_v2_0_5_224	.253
182	alexnet_reduced_aliasing_early_checkpoint	.252
183	resnet50_byol	.251
184	alexnet_l2_3_robust	.250
185	efficientnet_b1_imagenet_full	.249
186	mobilenet_v2_1_3_224	.247
187	resnet-18	.245
188	resnet50-SIN	.243
189	resnet50_robust_l2_eps3	.242
190	imagenet_l2_3_0	.242
191	resnet-50-robust	.242
192	densenet-169	.241
193	mobilenet_v2_1_0_128	.239
194	BiT-S-R152x4	.238
195	mobilenet_v2_1_0_160	.235
196	mobilenet_v1_1.0_224	.230
197	mobilenet_v2_1.0_160-tf	.229
198	inception_v1-tf	.223
199	cv_18_dagger_408_pretrained	.223
200	shufflenet_v2_x1_0	.222
201	alexnet_linf_8_robust	.218
202	efficientnet_b0_imagenet_full	.217
203	BiT-S-R101x3	.213
204	resnet50-vitoimagevidnet8	.210
205	alexnet_random_l2_3_perturb	.210
206	mobilenet_v2_1.4_224-tf	.208
207	BiT-S-R50x1	.208
208	resnet-34	.208
209	mobilenet_v2_1.3_224-tf	.206
210	mobilenet_v2_0_75_160	.204
211	mobilenet_v1_1.0_160	.198
212	vgg-16-keras	.197
213	mobilenet_v2_0.5_192-tf	.196
214	resnet50_simclr	.196
215	mobilenet_v2_1.0_224-tf	.194
216	mobilenet_v1_1.0_128	.192
217	BiT-S-R101x1	.188
218	squeezenet1_1	.182
219	mobilenet_v2_0.75_96	.181
220	mobilenet_v2_0_5_192	.180
221	alexnet_training_seed_01	.180
222	alexnet_training_seed_05	.180
223	alexnet_robust_correct	.179
224	vgg-19-keras	.179
225	deit_small_imagenet_100_seed-0	.171
226	mobilenet_v2_0.75_128	.169
227	konkle_alexnetgn_ipcl_ref01_primary_model	.168
228	mobilenet_v2_0.75_160-tf	.164
229	convnext_tiny_sup	.161
230	alexnet_early_checkpoint	.161
231	mobilenet_v1_0.25_192	.160
232	0.5x_resnet-18_LC_w_sh_1_iter	.160
233	mobilenet_v2_1_4_224	.159
234	mobilenet_v2_1-4_224_pytorch	.159
235	mobilenet_v2_0.5_224-tf	.156
236	artResNet18_1	.156
237	mobilenet_v2_0.35_160	.150
238	mobilenet_v1_0.75_128	.148
239	mobilenet_v2_1.0_128-tf	.143
240	mobilenet_v1_0.75_192	.141
241	resnet50_imagenet_100_seed-0	.140
242	AdvProp_efficientnet-b0	.140
243	mobilenet_v1_0.5_128	.138
244	imagenet_l2_10_0	.132
245	nasnet_mobile-tf	.132
246	mobilenet_v2_1.0_96	.131
247	mobilenet_v2_0.35_96	.131
248	texture_shape_alexnet_trained_on_SIN	.129
249	squeezenet1_0	.126
250	mobilenet_v1_0.5_192	.126
251	mobilenet_v1_0.25_160	.125
252	AlexNet_SIN	.122
253	mobilenet_v2_0.5_160	.120
254	mobilenet_v2_0_75_192	.120
255	mobilenet_v1_0.25_224	.113
256	mobilenet_v2_0.5_128	.113
257	mobilenet_v2_0.75_192-tf	.111
258	mobilenet_v1_0.75_160	.107
259	mobilenet_v1_0.5_224	.097
260	mobilenet_v2_0_75_224	.097
261	resnet50-meshes-lt-100-original-scratch	.094
262	deit_small_imagenet_1_seed-0	.094
263	deit_small_imagenet_10_seed-0	.094
264	resnet50_imagenet_10_seed-0	.093
265	alexnet_training_seed_06	.089
266	alexnet_training_seed_02	.085
267	alexnet_training_seed_08	.085
268	alexnet_training_seed_04	.085
269	alexnet_training_seed_10	.085
270	convnext_small_imagenet_10_seed-0	.084
271	alexnet_training_seed_07	.084
272	mobilenet_v2_0.5_96	.077
273	mobilenet_v2_0.35_128	.070
274	barlow-twins-resnet50	.063
275	resnet50-vicregl0p9	.059
276	mobilenet_v2_0.75_224-tf	.058
277	resnet50-meshes-lt-100-original-pretrained	.058
278	resnet50_FractalDB	.057
279	mobilenet_v2_0.35_224	.056
280	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.055
281	resnet50-vicregl0p75	.051
282	resnet50-vicreg	.044
283	alexnet_training_seed_09	.042
284	resnet50-barlow	.041
285	mobilenet_v1_0.5_160	.038
286	mobilenet_v2_0.35_192	.036
287	my-model	.029
288	convnext_small_imagenet_1_seed-0	.015
289	mobilenet_v1_0.25_128	.013
290	resnet50-simclr-vissl	.009
291	mobilevit_small	.008
292	resnet50_imagenet_1_seed-0	.008
293	alexnet_training_seed_03	.002
294	resnet-50_untrained	.002
295	yudixie_resnet18_translation_reg_0_240719	X
296	yudixie_resnet18_distance_reg_0_240719	X
297	yudixie_resnet18_rotation_reg_0_240719	X
298	ReAlnet03_cornet	X
299	ViT_L_32	X
300	my-model	X
301	ReAlnet02_cornet	X
302	resnet18_ecoset_full	X
303	resnet34_ecoset_full	X
304	resnet-18-LC_w_sh_1_iter_conv_init_m	X
305	resnet18-simclr	X
306	VOneCORnet-S	X
307	resnet50_ecoset_full	X
308	resnet101_ecoset_full	X
309	yudixie_resnet50_distance_rotation_0_240908	X
310	yudixie_resnet50_rotation_reg_0_240908	X
311	yudixie_resnet50_distance_translation_0_240908	X
312	resnet18-autoencoder	X
313	resnet152_ecoset_full	X
314	yudixie_resnet18_distance_translation_0_240719	X
315	ReAlnet04_cornet	X
316	yudixie_resnet18_translation_rotation_0_240719	X
317	yudixie_resnet18_distance_rotation_0_240719	X
318	BiT-M-R152x2	X
319	yudixie_resnet18_category_class_0_240719	X
320	ReAlnet05_cornet	X
321	yudixie_resnet18_random_0_240719	X
322	resnet18-relative_position	X
323	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
324	resnet-18_test_m	X
325	yudixie_resnet18_distance_translation_rotation_0_240719	X
326	yudixie_resnet18_object_class_0_240719	X
327	yudixie_resnet18_imagenet1kpret_0_240719	X
328	ReAlnet06_cornet	X
329	yudixie_resnet50_translation_reg_0_240908	X
330	yudixie_resnet50_translation_rotation_0_240908	X
331	ReAlnet07_cornet	X
332	RN50	X
333	0.5x_resnet-18_LC_w_sh_100_iter	X
334	ReAlnet08_cornet	X
335	RN50	X
336	omnivore_swinB	X
337	ReAlnet09_cornet	X
338	0.5x_resnet-18	X
339	BiT-M-R50x1	X
340	resnet18-contrastive_multiview	X
341	BiT-M-R50x3	X
342	resnet-18-LC_conv_init_m	X
343	resnet-18-LC_1st_conv_m	X
344	omnivore_swinS	X
345	resnet18-supervised	X
346	resnet-18-LC_m	X
347	r3m_resnet50_nocrop	X
348	r3m_resnet34	X
349	ReAlnet10_cornet	X
350	vggface	X
351	yudixie_resnet50_distance_translation_rotation_0_240908	X
352	r3m_resnet50	X
353	ViT-B/32	X
354	0.5x_resnet-18_LC_w_sh_10_iter	X
355	resnet-18-LC_w_sh_100_iter_conv_init_m	X
356	resnet18-local_aggregation	X
357	resnet18-colorization	X
358	alexnet_ks_torevert	X
359	yudixie_resnet50_random_0_240908	X
360	yudixie_resnet50_imagenet1kpret_0_240908	X
361	yudixie_resnet50_imagenet1kpret_0_240312	X
362	0.5x_resnet-18_LC	X
363	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
364	yudixie_resnet50_object_class_0_240908	X
365	CORnetZ_CIFAR10	X
366	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
367	FrankRobWobv0	X
368	resnet18-deepcluster	X
369	resnet50_ImageNet	X
370	resnet50_pretrained_with_retinal_waves	X
371	resnet50-cifar	X
372	resnet-18-LC_w_sh_10_iter_conv_init_m	X
373	r3m_resnet18	X
374	CORnet-Z	X
375	yudixie_resnet50_category_class_0_240908	X
376	resnet-18-LC_w_sh_100_iter_m	X
377	openclip	X
378	resnet50	X
379	resnet18_imagenet21kP	X
380	dorinet_cornet_z	X
381	pixels	X
382	yudixie_resnet50_distance_reg_0_240908	X
383	ConvLSTM	X
384	CORnetZ_CIFAR10_bs32_20_04	X
385	PredRNN	X
386	SimVP	X
387	TAU	X
388	MIM	X
389	hmax	X
390	unet_entire	X
391	v1-pyr-nodown	X
392	dcgan	X
393	resnet-50x2_untrained	X
394	pnasnet_large_pytorch	X
395	ReAlnet01_cornet	X
396	resnet-18_untrained	X
397	resnet-18-LC_w_sh_10_iter_m	X
398	resnet-18-LC_w_sh_1_iter_m	X
399	resnet-18-LC_untrained	X
400	omnivore_swinT	X
401	resnet18-contrastive_predictive	X
402	r3m_resnet34_nocrop	X
403	ViT_B_16	X
404	resnet18-instance_recognition	X
405	resnet18-depth_prediction	X
406	ViT_B_32	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.39.

Note that scores are relative to this ceiling.

Data: Geirhos2021eidolonI

Metric: error_consistency