Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021cueconflict-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.943
2	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.938
3	vit_large_patch14_clip_224:laion2b_ft_in1k	.855
4	vit_large_patch14_clip_224:openai_ft_in1k	.840
5	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.804
6	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.784
7	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.780
8	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.752
9	vit_large_patch14_clip_336:laion2b_ft_in1k	.748
10	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.740
11	convnext_large:fb_in22k_ft_in1k	.698
12	convnext_base:clip_laiona_augreg_ft_in1k_384	.686
13	vision_transformer_vit_large_patch16_224	.656
14	convnext_xlarge:fb_in22k_ft_in1k	.651
15	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.633
16	fixres_resnext101_32x48d_wsl	.611
17	resnext101_32x32d_wsl	.596
18	resnext101_32x16d_wsl	.595
19	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.571
20	resnext101_32x48d_wsl	.560
21	vit_base_patch16_clip_224:openai_ft_in1k	.560
22	SWSL_resnext101_32x8d	.546
23	resnext101_32x8d_wsl	.540
24	CLIP_ViT-B_32	.482
25	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.477
26	resnet50-VITO-8deg-cc	.451
27	effnetv2m_custom384	.448
28	cvt_cvt-13-384-in1k_4_LucyV4	.441
29	resnet-152_v2-tf	.405
30	resnet_152_v2	.402
31	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.394
32	resnet_101_v2	.393
33	SWSL_resnet50	.390
34	resnet_50_v2	.383
35	vit_relpos_base_patch16_clsgap_224:sw_in1k	.380
36	ViT_B_16_imagenet1k	.376
37	cvt_cvt-21-384-in1k_4_LucyV4	.372
38	CLIP_resnet50	.371
39	CLIP_resnet50_float32	.371
40	effnetb1_272x240	.357
41	pnasnet_large	.355
42	efficientnet_b0	.346
43	inception_v3	.345
44	ViT_L_32_imagenet1k	.344
45	xception-keras	.343
46	nasnet_large	.332
47	cv_18_dagger_408_pretrained	.328
48	pnasnet_large-tf	.327
49	convnext_femto_ols:d1_in1k	.325
50	convnext_tiny_imagenet_full_seed-0	.321
51	CLIP-RN50	.317
52	pnasnet_large_pytorch	.316
53	resnet101	.316
54	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.316
55	focalnet_tiny_lrf_in1k	.313
56	inception_v3-tf	.311
57	ViT_B_32_imagenet1k	.309
58	resnet152_imagenet_full	.309
59	cvt_cvt-21-224-in1k_4_LucyV4	.309
60	inception_v3_pytorch	.300
61	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.299
62	convnext_tiny:in12k_ft_in1k	.297
63	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.294
64	xception	.292
65	resnet-50_v2-tf	.292
66	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.284
67	convnext_large_imagenet_full_seed-0	.284
68	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.282
69	convnext_base_imagenet_full_seed-0	.278
70	resnet50-SIN_IN	.272
71	resnet_101_v1	.271
72	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.271
73	nasnet_large-tf	.266
74	resnet-152_v2_pytorch	.260
75	resnet_152_v1	.255
76	inception_v4	.254
77	resnet50_byol	.254
78	antialias-resnet152	.253
79	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.250
80	resnet50_imagenet_full	.244
81	resnet101_imagenet_full	.242
82	antialiased-rnext101_32x8d	.240
83	AdvProp_efficientnet-b7	.238
84	densenet_201_pytorch	.236
85	densenet-201	.236
86	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.235
87	custom_model_cv_18_dagger_408	.235
88	cvt_cvt-13-224-in1k_4_LucyV4	.233
89	resnet50	.233
90	densenet-201-keras	.233
91	resnet34_imagenet_full	.233
92	AdvProp_efficientnet-b2	.232
93	AT_efficientnet-b2	.232
94	resnet50-moclr8deg	.229
95	efficientnet-b7	.228
96	resnet_SIN_IN_FT_IN	.228
97	resnet50-SIN_IN_IN	.228
98	resnet50_tutorial	.226
99	resnet_50_v1	.226
100	resnet-50-pytorch	.226
101	resnet50-sup	.226
102	resnet50_random_l2_perturb	.220
103	AdvProp_efficientnet-b6	.219
104	Res2Net50_26w_4s	.218
105	densenet-121-keras	.213
106	AT_efficientnet-b4	.213
107	AdvProp_efficientnet-b4	.213
108	densenet-121	.212
109	mobilenet_v2_0_5_224	.211
110	mobilenet_v2_1-4_224_pytorch	.210
111	mobilenet_v2_1_4_224	.210
112	inception_v4-tf	.210
113	grcnn_109	.208
114	convnext_small_imagenet_full_seed-0	.206
115	voneresnet-50-non_stochastic	.204
116	ReAlnet04	.204
117	texture_shape_resnet50_trained_on_SIN	.202
118	resnet-152_v1-tf	.196
119	resnet50_moco_v2	.191
120	cornet_s	.191
121	nasnet_mobile	.190
122	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.189
123	mobilenet_v2_1.4_224-tf	.189
124	resnet-101_v1-tf	.189
125	BiT-S-R152x4	.189
126	vgg_16	.188
127	ReAlnet03	.187
128	resnet18_imagenet_full	.186
129	ReAlnet09	.184
130	antialiased-r50	.182
131	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.181
132	ReAlnet_incorrect	.181
133	ReAlnet01	.181
134	deit_large_imagenet_full_seed-0	.181
135	resnet-50_v1-tf	.180
136	ReAlnet10	.179
137	grcnn	.179
138	resnet50_random_linf8_perturb	.177
139	BiT-S-R101x1	.177
140	blt_vs	.177
141	CORnet-S	.175
142	resnet50_linf_4_robust	.175
143	ReAlnet08	.173
144	resnet50_linf_8_robust	.173
145	BiT-S-R50x1	.173
146	mobilenet_v2_1_0_224	.171
147	inception_v1	.167
148	densenet-169	.166
149	ReAlnet06	.166
150	mobilenet_v1_0.75_224	.165
151	texture_shape_alexnet_trained_on_SIN	.164
152	mobilenet_v2_1_0_160	.164
153	BiT-S-R50x3	.163
154	resnet-18	.163
155	resnet50-vitoimagevidnet8	.162
156	voneresnet-50-robust	.162
157	inception_v2-tf	.160
158	deit_base_imagenet_full_seed-0	.160
159	AdvProp_efficientnet-b8	.159
160	ReAlnet05	.157
161	regnet_y_400mf	.157
162	mobilenet_v1_0.5_192	.157
163	vonealexnet_gaussian_noise_std4_fixed	.157
164	BiT-S-R152x2	.156
165	mobilenet_v1_1.0_224	.156
166	resnet50_l2_3_robust	.155
167	alexnet_reduced_aliasing_early_checkpoint	.155
168	inception_v1-tf	.154
169	alexnet_training_seed_03	.153
170	mobilenet_v2_1.0_224-tf	.153
171	ReAlnet07	.153
172	mobilenet_v2_1_3_224	.153
173	alexnet_random_linf8_perturb	.153
174	ReAlnet02	.153
175	mobilenet_v2_0.5_224-tf	.153
176	vonegrcnn_52e_full	.153
177	mobilenet_v2_1.3_224-tf	.152
178	resnet50_simclr	.152
179	densenet-169-keras	.149
180	mobilenet_v1_1.0_192	.147
181	vgg-16-keras	.146
182	0.5x_resnet-18_LC_w_sh_1_iter	.146
183	squeezenet1_1	.146
184	efficientnet_b1_imagenet_full	.145
185	vonegrcnn_47e	.144
186	resnet50_robust_l2_eps1	.144
187	resnet50-SIN	.144
188	efficientnet_b2_imagenet_full	.144
189	BiT-S-R101x3	.144
190	vgg_19	.142
191	alexnet_random_l2_3_perturb	.142
192	mobilenet_v2_0.5_96	.141
193	mobilenet_v1_1.0_128	.140
194	mobilenet_v2_1.0_160-tf	.137
195	vonegrcnn_62e_nobn	.137
196	mobilenet_v1_0.75_192	.136
197	resnet-34	.136
198	mobilenet_v2_1_0_128	.135
199	AdvProp_efficientnet-b0	.135
200	alexnet_early_checkpoint	.135
201	efficientnet_b0_imagenet_full	.134
202	mobilenet_v2_0.35_128	.134
203	convnext_small_imagenet_100_seed-0	.133
204	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.133
205	mobilenet_v2_0.35_96	.133
206	konkle_alexnetgn_ipcl_ref01_primary_model	.132
207	mobilenet_v2_1.0_128-tf	.132
208	mobilenet_v1_0.25_224	.132
209	mobilenet_v2_0.75_224-tf	.132
210	AlexNet_SIN	.132
211	vonegrcnn_52e_full	.132
212	imagenet_l2_3_0	.131
213	resnet-50-robust	.131
214	resnet50_robust_l2_eps3	.131
215	mobilenet_v2_1_0_192	.130
216	alexnet_ks_torevert	.128
217	alexnet-baseline	.128
218	alexnet	.128
219	alexnet	.128
220	alexnet	.128
221	alexnet	.128
222	alexnet	.128
223	alexnet	.128
224	alexnet	.128
225	alexnet	.128
226	alexnet	.128
227	mobilenet_v2_0.5_192-tf	.128
228	resnet50_imagenet_100_seed-0	.127
229	vgg-19-keras	.127
230	vonegrcnn_62e_nobn	.127
231	mobilenet_v2_0_75_224	.125
232	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.125
233	mobilenet_v2_1.0_192-tf	.125
234	shufflenet_v2_x1_0	.123
235	mobilenet_v1_0.75_128	.122
236	alexnet	.121
237	voneresnet-50	.121
238	vonegrcnn_47e	.121
239	mobilenet_v2_0.75_96	.119
240	mobilenet_v2_0.5_160	.117
241	mobilenet_v1_1.0_160	.114
242	mobilenet_v1_0.25_192	.112
243	convnext_tiny_sup	.112
244	alexnet_l2_3_robust	.110
245	squeezenet1_0	.110
246	mobilenet_v2_0.5_128	.109
247	mobilenet_v2_0_75_192	.107
248	mobilenet_v2_0_5_192	.106
249	mobilenet_v2_0.35_160	.102
250	mobilenet_v2_0.35_192	.102
251	mobilenet_v1_0.5_128	.102
252	mobilenet_v2_0.75_160-tf	.101
253	nasnet_mobile-tf	.101
254	mobilenet_v2_0_75_160	.098
255	mobilenet_v1_0.25_160	.098
256	mobilenet_v1_0.75_160	.097
257	deit_small_imagenet_full_seed-0	.096
258	mobilenet_v2_0.75_128	.096
259	convnext_small_imagenet_1_seed-0	.096
260	mobilenet_v2_0.35_224	.093
261	convnext_small_imagenet_10_seed-0	.091
262	deit_small_imagenet_10_seed-0	.090
263	mobilenet_v2_0.75_192-tf	.090
264	deit_small_imagenet_1_seed-0	.087
265	mobilenet_v1_0.5_224	.083
266	mobilevit_small	.079
267	mobilenet_v2_1.0_96	.078
268	alexnet_training_seed_08	.068
269	alexnet_training_seed_04	.068
270	alexnet_training_seed_02	.068
271	alexnet_training_seed_10	.068
272	alexnet_training_seed_09	.067
273	alexnet_training_seed_07	.066
274	resnet50-meshes-lt-100-original-pretrained	.066
275	alexnet_robust_correct	.063
276	mobilenet_v1_0.25_128	.059
277	mobilenet_v1_0.5_160	.055
278	barlow-twins-resnet50	.053
279	resnet50-meshes-lt-100-original-scratch	.052
280	alexnet_training_seed_06	.050
281	resnet50-vicreg	.049
282	resnet50-barlow	.046
283	alexnet_linf_8_robust	.045
284	imagenet_l2_10_0	.034
285	resnet50_FractalDB	.032
286	resnet50-vicregl0p9	.032
287	resnet-50_untrained	.031
288	deit_small_imagenet_100_seed-0	.028
289	artResNet18_1	.013
290	resnet50_imagenet_1_seed-0	.011
291	resnet50-simclr-vissl	.011
292	resnet50-vicregl0p75	.011
293	resnet50_imagenet_10_seed-0	.003
294	alexnet_training_seed_01	.003
295	alexnet_training_seed_05	.003
296	resnet18-colorization	X
297	resnet101_ecoset_full	X
298	yudixie_resnet50_distance_rotation_0_240908	X
299	yudixie_resnet50_rotation_reg_0_240908	X
300	resnet152_ecoset_full	X
301	yudixie_resnet18_distance_translation_0_240719	X
302	ReAlnet04_cornet	X
303	yudixie_resnet50_distance_translation_0_240908	X
304	r3m_resnet18	X
305	yudixie_resnet18_translation_rotation_0_240719	X
306	yudixie_resnet18_distance_rotation_0_240719	X
307	yudixie_resnet18_category_class_0_240719	X
308	ReAlnet05_cornet	X
309	yudixie_resnet18_random_0_240719	X
310	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
311	resnet-18_test_m	X
312	yudixie_resnet18_distance_translation_rotation_0_240719	X
313	yudixie_resnet18_object_class_0_240719	X
314	resnet18-deepcluster	X
315	ReAlnet06_cornet	X
316	yudixie_resnet50_translation_reg_0_240908	X
317	ReAlnet07_cornet	X
318	yudixie_resnet50_translation_rotation_0_240908	X
319	CORnetZ_CIFAR10	X
320	0.5x_resnet-18_LC	X
321	ReAlnet08_cornet	X
322	resnet18-local_aggregation	X
323	VOneCORnet-S	X
324	FrankRobWobv0	X
325	ReAlnet09_cornet	X
326	omnivore_swinB	X
327	resnet-18-LC_w_sh_10_iter_conv_init_m	X
328	resnet-18-LC_w_sh_100_iter_m	X
329	yudixie_resnet50_random_0_240908	X
330	omnivore_swinS	X
331	resnet-18-LC_w_sh_10_iter_m	X
332	ReAlnet10_cornet	X
333	resnet-18-LC_w_sh_1_iter_m	X
334	resnet-18-LC_w_sh_1_iter_conv_init_m	X
335	yudixie_resnet50_distance_translation_rotation_0_240908	X
336	BiT-M-R152x2	X
337	BiT-M-R50x1	X
338	BiT-M-R50x3	X
339	ViT-B/32	X
340	resnet-18-LC_w_sh_100_iter_conv_init_m	X
341	resnet50_ImageNet	X
342	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
343	resnet50_pretrained_with_retinal_waves	X
344	resnet50-cifar	X
345	yudixie_resnet50_imagenet1kpret_0_240908	X
346	openclip	X
347	yudixie_resnet50_imagenet1kpret_0_240312	X
348	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
349	resnet50	X
350	my-model	X
351	dorinet_cornet_z	X
352	pixels	X
353	yudixie_resnet50_object_class_0_240908	X
354	CORnetZ_CIFAR10_bs32_20_04	X
355	unet_entire	X
356	v1-pyr-nodown	X
357	dcgan	X
358	resnet-50x2_untrained	X
359	resnet-18_untrained	X
360	resnet-18-LC_untrained	X
361	omnivore_swinT	X
362	resnet18-contrastive_predictive	X
363	r3m_resnet34_nocrop	X
364	ViT_B_16	X
365	resnet18-depth_prediction	X
366	ViT_B_32	X
367	ViT_L_16_imagenet1k	X
368	ViT_L_32	X
369	resnet18-simclr	X
370	resnet18-autoencoder	X
371	yudixie_resnet50_category_class_0_240908	X
372	resnet18-relative_position	X
373	RN50	X
374	0.5x_resnet-18_LC_w_sh_100_iter	X
375	RN50	X
376	resnet18_imagenet21kP	X
377	ConvLSTM	X
378	PredRNN	X
379	yudixie_resnet50_distance_reg_0_240908	X
380	SimVP	X
381	briaai_rmbg_1_4	X
382	TAU	X
383	MIM	X
384	hmax	X
385	0.5x_resnet-18	X
386	ReAlnet01_cornet	X
387	resnet18-contrastive_multiview	X
388	resnet-18-LC_conv_init_m	X
389	resnet-18-LC_1st_conv_m	X
390	resnet18-supervised	X
391	r3m_resnet50_nocrop	X
392	yolos_tiny	X
393	resnet-18-LC_m	X
394	r3m_resnet34	X
395	r3m_resnet50	X
396	0.5x_resnet-18_LC_w_sh_10_iter	X
397	yudixie_resnet18_translation_reg_0_240719	X
398	vggface	X
399	ReAlnet02_cornet	X
400	my-model	X
401	ReAlnet03_cornet	X
402	yudixie_resnet18_distance_reg_0_240719	X
403	yudixie_resnet18_rotation_reg_0_240719	X
404	yudixie_resnet18_imagenet1kpret_0_240719	X
405	resnet18-instance_recognition	X
406	CORnet-Z	X
407	resnet18_ecoset_full	X
408	resnet34_ecoset_full	X
409	resnet50_ecoset_full	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.33.

Note that scores are relative to this ceiling.

Data: Geirhos2021cueconflict

Metric: error_consistency