Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021falsecolour-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	cvt_cvt-13-384-in1k_4_LucyV4	.818
2	SWSL_resnext101_32x8d	.804
3	nasnet_large	.780
4	resnext101_32x16d_wsl	.757
5	vision_transformer_vit_large_patch16_224	.756
6	resnext101_32x48d_wsl	.755
7	CLIP-RN50	.754
8	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.752
9	SWSL_resnet50	.752
10	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.739
11	inception_v4	.738
12	resnext101_32x32d_wsl	.735
13	cvt_cvt-21-384-in1k_4_LucyV4	.726
14	nasnet_large-tf	.708
15	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.706
16	pnasnet_large	.702
17	cvt_cvt-13-224-in1k_4_LucyV4	.699
18	CLIP_resnet50_float32	.688
19	CLIP_resnet50	.688
20	pnasnet_large-tf	.682
21	resnext101_32x8d_wsl	.668
22	effnetb1_272x240	.667
23	vit_relpos_base_patch16_clsgap_224:sw_in1k	.648
24	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.640
25	convnext_femto_ols:d1_in1k	.633
26	convnext_large_imagenet_full_seed-0	.622
27	resnet_50_v2	.622
28	effnetv2m_custom384	.617
29	focalnet_tiny_lrf_in1k	.610
30	fixres_resnext101_32x48d_wsl	.610
31	CLIP_ViT-B_32	.605
32	xception-keras	.599
33	ViT_B_16_imagenet1k	.588
34	vit_base_patch16_clip_224:openai_ft_in1k	.581
35	resnet50-VITO-8deg-cc	.580
36	resnet101	.580
37	convnext_xlarge:fb_in22k_ft_in1k	.579
38	convnext_large:fb_in22k_ft_in1k	.572
39	resnet_101_v2	.568
40	antialias-resnet152	.566
41	resnet-152_v2_pytorch	.564
42	convnext_tiny_imagenet_full_seed-0	.564
43	convnext_tiny:in12k_ft_in1k	.560
44	resnet152_imagenet_full	.559
45	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.559
46	inception_v3	.558
47	cvt_cvt-21-224-in1k_4_LucyV4	.554
48	inception_v4-tf	.547
49	resnet_152_v1	.546
50	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.542
51	vit_large_patch14_clip_224:laion2b_ft_in1k	.539
52	resnet_152_v2	.537
53	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.536
54	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.532
55	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.532
56	resnet50_imagenet_full	.531
57	resnet101_imagenet_full	.528
58	convnext_base_imagenet_full_seed-0	.520
59	inception_v3-tf	.519
60	antialiased-rnext101_32x8d	.511
61	efficientnet_b0	.507
62	inception_v3_pytorch	.505
63	resnet-152_v2-tf	.504
64	resnet50_random_l2_perturb	.497
65	resnet50_moco_v2	.495
66	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.493
67	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.493
68	cv_18_dagger_408_pretrained	.492
69	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.487
70	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.486
71	convnext_small_imagenet_full_seed-0	.483
72	resnet50	.475
73	vgg_19	.471
74	xception	.470
75	grcnn_109	.462
76	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.461
77	deit_large_imagenet_full_seed-0	.453
78	resnet-50_v2-tf	.449
79	deit_base_imagenet_full_seed-0	.447
80	resnet_101_v1	.440
81	AdvProp_efficientnet-b7	.434
82	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.429
83	ViT_L_32_imagenet1k	.419
84	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.416
85	resnet-152_v1-tf	.409
86	resnet50_byol	.407
87	nasnet_mobile-tf	.401
88	pnasnet_large_pytorch	.392
89	resnet34_imagenet_full	.390
90	deit_small_imagenet_full_seed-0	.371
91	ViT_B_32_imagenet1k	.368
92	resnet50-SIN_IN_IN	.367
93	resnet_SIN_IN_FT_IN	.367
94	cornet_s	.364
95	densenet-201-keras	.362
96	resnet-101_v1-tf	.359
97	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.356
98	AT_efficientnet-b4	.353
99	AdvProp_efficientnet-b4	.353
100	Res2Net50_26w_4s	.353
101	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.351
102	vit_large_patch14_clip_336:laion2b_ft_in1k	.348
103	densenet-121-keras	.346
104	AdvProp_efficientnet-b2	.338
105	AT_efficientnet-b2	.338
106	vgg_16	.337
107	vit_large_patch14_clip_224:openai_ft_in1k	.337
108	antialiased-r50	.336
109	AdvProp_efficientnet-b6	.336
110	mobilenet_v2_1.4_224-tf	.336
111	efficientnet-b7	.336
112	densenet-169-keras	.335
113	densenet-201	.333
114	resnet18_imagenet_full	.333
115	resnet50-moclr8deg	.332
116	resnet50_random_linf8_perturb	.326
117	densenet-121	.324
118	inception_v1	.312
119	resnet_50_v1	.309
120	resnet50_tutorial	.309
121	resnet50-sup	.309
122	resnet-50-pytorch	.309
123	nasnet_mobile	.306
124	BiT-S-R50x3	.300
125	BiT-S-R101x3	.299
126	densenet-169	.299
127	mobilenet_v2_1_3_224	.297
128	inception_v1-tf	.296
129	mobilenet_v2_1.3_224-tf	.295
130	CORnet-S	.292
131	resnet50-SIN_IN	.291
132	BiT-S-R101x1	.289
133	resnet-34	.282
134	grcnn	.280
135	convnext_tiny_sup	.269
136	custom_model_cv_18_dagger_408	.269
137	resnet50_simclr	.268
138	resnet-50_v1-tf	.267
139	resnet50-vitoimagevidnet8	.263
140	inception_v2-tf	.262
141	ReAlnet09	.257
142	ReAlnet01	.257
143	ReAlnet_incorrect	.257
144	BiT-S-R152x2	.256
145	convnext_base:clip_laiona_augreg_ft_in1k_384	.256
146	convnext_small_imagenet_100_seed-0	.255
147	ReAlnet06	.253
148	vonegrcnn_62e_nobn	.252
149	mobilenet_v2_1_0_224	.251
150	ReAlnet03	.251
151	efficientnet_b2_imagenet_full	.249
152	ReAlnet04	.246
153	efficientnet_b0_imagenet_full	.240
154	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.238
155	BiT-S-R152x4	.237
156	mobilenet_v2_1_0_192	.233
157	voneresnet-50-non_stochastic	.231
158	AdvProp_efficientnet-b8	.230
159	vonegrcnn_47e	.228
160	vonegrcnn_52e_full	.226
161	mobilenet_v2_1.0_224-tf	.225
162	BiT-S-R50x1	.225
163	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.223
164	vonegrcnn_62e_nobn	.221
165	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.220
166	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.219
167	ReAlnet02	.219
168	vonegrcnn_47e	.218
169	mobilenet_v2_0_75_224	.216
170	vonegrcnn_52e_full	.215
171	resnet-18	.212
172	ReAlnet07	.212
173	mobilenet_v2_0.75_224-tf	.212
174	ReAlnet10	.209
175	ReAlnet08	.195
176	mobilenet_v2_1-4_224_pytorch	.193
177	mobilenet_v2_1_4_224	.193
178	blt_vs	.188
179	mobilenet_v2_1.0_192-tf	.187
180	ReAlnet05	.183
181	efficientnet_b1_imagenet_full	.181
182	regnet_y_400mf	.179
183	mobilenet_v2_1_0_160	.175
184	mobilenet_v2_0_5_224	.174
185	mobilenet_v1_0.75_224	.164
186	mobilenet_v2_0_75_192	.163
187	mobilenet_v2_0_5_192	.157
188	mobilenet_v2_1.0_160-tf	.157
189	vgg-16-keras	.155
190	voneresnet-50	.152
191	mobilenet_v2_0.5_224-tf	.145
192	mobilenet_v1_1.0_224	.143
193	mobilenet_v2_0_75_160	.139
194	mobilenet_v1_1.0_192	.137
195	resnet50_linf_4_robust	.136
196	mobilenet_v2_0.5_192-tf	.123
197	texture_shape_resnet50_trained_on_SIN	.119
198	konkle_alexnetgn_ipcl_ref01_primary_model	.116
199	mobilenet_v1_0.75_192	.114
200	resnet50_imagenet_100_seed-0	.113
201	shufflenet_v2_x1_0	.107
202	vgg-19-keras	.107
203	squeezenet1_1	.106
204	mobilenet_v1_0.5_192	.104
205	mobilenet_v1_1.0_160	.103
206	mobilenet_v2_0.75_160-tf	.101
207	mobilenet_v2_0.75_128	.101
208	mobilenet_v2_0.75_192-tf	.100
209	mobilenet_v2_0.35_192	.092
210	resnet50_robust_l2_eps1	.088
211	squeezenet1_0	.087
212	alexnet_random_linf8_perturb	.086
213	AdvProp_efficientnet-b0	.085
214	alexnet	.082
215	mobilenet_v1_0.5_224	.073
216	alexnet_early_checkpoint	.073
217	mobilenet_v1_0.5_160	.070
218	mobilenet_v2_0.35_224	.069
219	mobilenet_v2_0.5_128	.068
220	mobilenet_v2_1.0_96	.067
221	mobilenet_v2_0.75_96	.066
222	resnet50_l2_3_robust	.065
223	resnet50_linf_8_robust	.064
224	voneresnet-50-robust	.064
225	mobilenet_v2_0.35_160	.063
226	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.056
227	mobilenet_v2_0.5_160	.055
228	alexnet-baseline	.055
229	alexnet	.055
230	alexnet	.055
231	alexnet	.055
232	alexnet	.055
233	alexnet	.055
234	alexnet	.055
235	alexnet	.055
236	alexnet	.055
237	alexnet	.055
238	alexnet_ks_torevert	.055
239	mobilenet_v1_0.75_160	.054
240	mobilenet_v1_0.25_128	.051
241	alexnet_random_l2_3_perturb	.049
242	resnet50-SIN	.048
243	mobilenet_v2_1_0_128	.045
244	vonealexnet_gaussian_noise_std4_fixed	.045
245	mobilenet_v2_1.0_128-tf	.043
246	alexnet_training_seed_01	.043
247	alexnet_training_seed_05	.043
248	convnext_small_imagenet_1_seed-0	.039
249	mobilenet_v1_0.25_192	.038
250	resnet50-vicregl0p9	.037
251	alexnet_reduced_aliasing_early_checkpoint	.036
252	mobilenet_v1_0.25_160	.035
253	mobilenet_v1_1.0_128	.035
254	resnet50_robust_l2_eps3	.034
255	resnet-50-robust	.034
256	imagenet_l2_3_0	.034
257	alexnet_training_seed_06	.034
258	deit_small_imagenet_100_seed-0	.033
259	mobilenet_v1_0.75_128	.031
260	resnet50_imagenet_10_seed-0	.029
261	alexnet_training_seed_09	.027
262	texture_shape_alexnet_trained_on_SIN	.027
263	my-model	.019
264	resnet50-vicregl0p75	.019
265	resnet50_FractalDB	.019
266	imagenet_l2_10_0	.017
267	resnet50-vicreg	.016
268	deit_small_imagenet_1_seed-0	.016
269	deit_small_imagenet_10_seed-0	.016
270	mobilenet_v2_0.5_96	.016
271	resnet50_imagenet_1_seed-0	.015
272	mobilenet_v2_0.35_96	.015
273	mobilenet_v1_0.25_224	.014
274	alexnet_training_seed_07	.013
275	convnext_small_imagenet_10_seed-0	.012
276	mobilenet_v1_0.5_128	.011
277	mobilenet_v2_0.35_128	.011
278	alexnet_training_seed_10	.010
279	alexnet_training_seed_02	.010
280	alexnet_training_seed_04	.010
281	alexnet_training_seed_08	.010
282	resnet50-meshes-lt-100-original-scratch	.010
283	mobilevit_small	.010
284	alexnet_linf_8_robust	.010
285	alexnet_training_seed_03	.009
286	0.5x_resnet-18_LC_w_sh_1_iter	.009
287	resnet50-barlow	.009
288	resnet50-meshes-lt-100-original-pretrained	.008
289	barlow-twins-resnet50	.004
290	artResNet18_1	.004
291	alexnet_robust_correct	.003
292	resnet-50_untrained	.003
293	alexnet_l2_3_robust	.003
294	resnet50-simclr-vissl	.001
295	MIM	X
296	r3m_resnet50_nocrop	X
297	resnet18-supervised	X
298	densenet_201_pytorch	X
299	vggface	X
300	resnet-18-LC_1st_conv_m	X
301	resnet-18-LC_conv_init_m	X
302	my-model	X
303	yudixie_resnet50_object_class_0_240908	X
304	ReAlnet01_cornet	X
305	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
306	resnet18-contrastive_multiview	X
307	yudixie_resnet50_imagenet1kpret_0_240908	X
308	resnet18-instance_recognition	X
309	yudixie_resnet50_random_0_240908	X
310	yudixie_resnet50_imagenet1kpret_0_240312	X
311	0.5x_resnet-18	X
312	RN50	X
313	0.5x_resnet-18_LC_w_sh_100_iter	X
314	yudixie_resnet18_translation_reg_0_240719	X
315	RN50	X
316	yudixie_resnet18_distance_reg_0_240719	X
317	CORnetZ_CIFAR10_bs32_20_04	X
318	yudixie_resnet18_rotation_reg_0_240719	X
319	CORnet-Z	X
320	yudixie_resnet18_imagenet1kpret_0_240719	X
321	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
322	ReAlnet02_cornet	X
323	resnet18-relative_position	X
324	r3m_resnet18	X
325	resnet18-autoencoder	X
326	ReAlnet03_cornet	X
327	resnet18-simclr	X
328	resnet18_ecoset_full	X
329	resnet50_ImageNet	X
330	resnet34_ecoset_full	X
331	VOneCORnet-S	X
332	resnet50_pretrained_with_retinal_waves	X
333	resnet50_ecoset_full	X
334	resnet18-deepcluster	X
335	resnet50-cifar	X
336	yudixie_resnet50_rotation_reg_0_240908	X
337	resnet101_ecoset_full	X
338	resnet18-local_aggregation	X
339	yudixie_resnet50_distance_translation_0_240908	X
340	resnet152_ecoset_full	X
341	ViT_L_32	X
342	ViT_B_32	X
343	yudixie_resnet18_distance_translation_0_240719	X
344	CORnetZ_CIFAR10	X
345	yudixie_resnet18_translation_rotation_0_240719	X
346	ReAlnet04_cornet	X
347	resnet18-depth_prediction	X
348	0.5x_resnet-18_LC	X
349	ViT_B_16	X
350	yudixie_resnet18_distance_rotation_0_240719	X
351	yudixie_resnet18_category_class_0_240719	X
352	resnet-18_test_m	X
353	yudixie_resnet18_random_0_240719	X
354	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
355	ReAlnet05_cornet	X
356	yudixie_resnet50_distance_rotation_0_240908	X
357	yudixie_resnet18_distance_translation_rotation_0_240719	X
358	r3m_resnet34_nocrop	X
359	yudixie_resnet18_object_class_0_240719	X
360	hmax	X
361	yudixie_resnet50_translation_reg_0_240908	X
362	resnet18-contrastive_predictive	X
363	ReAlnet06_cornet	X
364	yudixie_resnet50_translation_rotation_0_240908	X
365	FrankRobWobv0	X
366	omnivore_swinT	X
367	ReAlnet07_cornet	X
368	resnet-18-LC_untrained	X
369	resnet-18-LC_w_sh_10_iter_conv_init_m	X
370	ReAlnet08_cornet	X
371	resnet-18_untrained	X
372	omnivore_swinB	X
373	resnet-18-LC_w_sh_100_iter_m	X
374	resnet-18-LC_w_sh_1_iter_m	X
375	ReAlnet09_cornet	X
376	resnet-18-LC_w_sh_10_iter_m	X
377	resnet-50x2_untrained	X
378	omnivore_swinS	X
379	dcgan	X
380	v1-pyr-nodown	X
381	resnet-18-LC_w_sh_1_iter_conv_init_m	X
382	BiT-M-R152x2	X
383	ReAlnet10_cornet	X
384	BiT-M-R50x1	X
385	BiT-M-R50x3	X
386	unet_entire	X
387	openclip	X
388	yudixie_resnet50_distance_translation_rotation_0_240908	X
389	ViT-B/32	X
390	resnet50	X
391	dorinet_cornet_z	X
392	resnet-18-LC_w_sh_100_iter_conv_init_m	X
393	r3m_resnet50	X
394	yudixie_resnet50_category_class_0_240908	X
395	resnet18-colorization	X
396	pixels	X
397	AlexNet_SIN	X
398	resnet18_imagenet21kP	X
399	r3m_resnet34	X
400	ConvLSTM	X
401	yudixie_resnet50_distance_reg_0_240908	X
402	PredRNN	X
403	SimVP	X
404	0.5x_resnet-18_LC_w_sh_10_iter	X
405	TAU	X
406	resnet-18-LC_m	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021falsecolour

Metric: error_consistency