Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021lowpass-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	SWSL_resnext101_32x8d	.730
2	resnext101_32x32d_wsl	.695
3	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.690
4	convnext_large:fb_in22k_ft_in1k	.667
5	vit_large_patch14_clip_224:openai_ft_in1k	.663
6	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.648
7	fixres_resnext101_32x48d_wsl	.630
8	resnet50-VITO-8deg-cc	.625
9	vit_large_patch14_clip_336:laion2b_ft_in1k	.623
10	SWSL_resnet50	.613
11	vit_large_patch14_clip_224:laion2b_ft_in1k	.603
12	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.602
13	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.600
14	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.589
15	resnext101_32x48d_wsl	.579
16	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.578
17	resnext101_32x8d_wsl	.563
18	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.554
19	CLIP_resnet50_float32	.544
20	CLIP_resnet50	.544
21	deit_large_imagenet_full_seed-0	.543
22	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.537
23	resnext101_32x16d_wsl	.535
24	vit_base_patch16_clip_224:openai_ft_in1k	.535
25	convnext_xlarge:fb_in22k_ft_in1k	.527
26	effnetv2m_custom384	.527
27	convnext_base:clip_laiona_augreg_ft_in1k_384	.516
28	vision_transformer_vit_large_patch16_224	.499
29	resnet_152_v2	.498
30	effnetb1_272x240	.497
31	CLIP_ViT-B_32	.496
32	cvt_cvt-13-384-in1k_4_LucyV4	.492
33	convnext_large_imagenet_full_seed-0	.471
34	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.463
35	vit_relpos_base_patch16_clsgap_224:sw_in1k	.463
36	deit_small_imagenet_full_seed-0	.460
37	deit_base_imagenet_full_seed-0	.460
38	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.454
39	focalnet_tiny_lrf_in1k	.436
40	convnext_tiny:in12k_ft_in1k	.435
41	CLIP-RN50	.419
42	convnext_tiny_imagenet_full_seed-0	.419
43	resnet_50_v2	.415
44	pnasnet_large	.410
45	ViT_B_16_imagenet1k	.409
46	ViT_L_32_imagenet1k	.406
47	inception_v4	.400
48	resnet_101_v2	.397
49	nasnet_large	.396
50	cvt_cvt-21-384-in1k_4_LucyV4	.391
51	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.385
52	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.382
53	convnext_small_imagenet_full_seed-0	.379
54	convnext_femto_ols:d1_in1k	.379
55	resnet-152_v2_pytorch	.371
56	cvt_cvt-21-224-in1k_4_LucyV4	.368
57	cvt_cvt-13-224-in1k_4_LucyV4	.361
58	convnext_base_imagenet_full_seed-0	.354
59	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.352
60	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.338
61	pnasnet_large-tf	.338
62	antialias-resnet152	.326
63	xception-keras	.317
64	cv_18_dagger_408_pretrained	.314
65	resnet101	.313
66	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.311
67	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.306
68	nasnet_large-tf	.305
69	inception_v3-tf	.299
70	inception_v4-tf	.295
71	resnet152_imagenet_full	.290
72	resnet-152_v2-tf	.285
73	resnet50_byol	.274
74	resnet_152_v1	.270
75	resnet_101_v1	.268
76	antialiased-rnext101_32x8d	.268
77	inception_v3	.260
78	convnext_tiny_sup	.260
79	AdvProp_efficientnet-b4	.258
80	AT_efficientnet-b4	.258
81	grcnn_109	.252
82	inception_v3_pytorch	.252
83	resnet50	.251
84	ViT_B_32_imagenet1k	.244
85	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.243
86	resnet50_moco_v2	.232
87	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.225
88	AdvProp_efficientnet-b6	.221
89	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.218
90	densenet-121-keras	.214
91	antialiased-r50	.214
92	resnet50_imagenet_full	.213
93	voneresnet-50-non_stochastic	.209
94	convnext_small_imagenet_100_seed-0	.207
95	densenet-169-keras	.203
96	AdvProp_efficientnet-b7	.203
97	resnet34_imagenet_full	.203
98	resnet-50_v2-tf	.202
99	densenet-121	.201
100	densenet-201-keras	.197
101	grcnn	.194
102	resnet50-moclr8deg	.187
103	densenet-169	.186
104	mobilenet_v2_1_0_224	.186
105	resnet50_tutorial	.185
106	resnet_50_v1	.185
107	resnet50-sup	.185
108	resnet-50-pytorch	.185
109	efficientnet-b7	.185
110	vgg_19	.184
111	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.180
112	BiT-S-R152x4	.175
113	pnasnet_large_pytorch	.175
114	Res2Net50_26w_4s	.174
115	xception	.168
116	resnet-50_v1-tf	.167
117	resnet101_imagenet_full	.165
118	AT_efficientnet-b2	.165
119	AdvProp_efficientnet-b2	.165
120	resnet18_imagenet_full	.163
121	densenet-201	.161
122	densenet_201_pytorch	.161
123	ReAlnet04	.156
124	custom_model_cv_18_dagger_408	.156
125	resnet-101_v1-tf	.155
126	vonegrcnn_62e_nobn	.153
127	efficientnet_b1_imagenet_full	.153
128	efficientnet_b0	.152
129	inception_v1-tf	.144
130	deit_small_imagenet_100_seed-0	.143
131	vonegrcnn_47e	.141
132	efficientnet_b2_imagenet_full	.140
133	resnet50_random_linf8_perturb	.140
134	vonegrcnn_52e_full	.139
135	resnet-152_v1-tf	.138
136	mobilenet_v2_1.0_224-tf	.135
137	ReAlnet03	.135
138	nasnet_mobile	.133
139	mobilenet_v2_1.4_224-tf	.133
140	voneresnet-50	.132
141	ReAlnet09	.132
142	vonegrcnn_62e_nobn	.130
143	resnet50-SIN_IN	.129
144	BiT-S-R50x3	.129
145	blt_vs	.129
146	mobilenet_v2_1_0_192	.127
147	BiT-S-R101x3	.127
148	ReAlnet07	.127
149	vonegrcnn_52e_full	.126
150	vonegrcnn_47e	.126
151	ReAlnet08	.126
152	resnet50_random_l2_perturb	.126
153	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.125
154	inception_v2-tf	.124
155	cornet_s	.124
156	AdvProp_efficientnet-b8	.123
157	mobilenet_v2_1_4_224	.122
158	mobilenet_v2_1-4_224_pytorch	.122
159	ReAlnet02	.121
160	voneresnet-50-robust	.119
161	mobilenet_v2_0_5_192	.116
162	ReAlnet06	.115
163	nasnet_mobile-tf	.113
164	resnet50-vitoimagevidnet8	.113
165	mobilenet_v1_0.75_224	.112
166	ReAlnet10	.112
167	mobilenet_v2_0_5_224	.111
168	vgg-19-keras	.111
169	resnet50_l2_3_robust	.110
170	resnet50_linf_4_robust	.110
171	inception_v1	.109
172	resnet_SIN_IN_FT_IN	.109
173	resnet50-SIN_IN_IN	.109
174	mobilenet_v2_1.0_160-tf	.109
175	mobilenet_v2_1_0_160	.108
176	resnet50_linf_8_robust	.107
177	resnet50_robust_l2_eps3	.103
178	resnet-50-robust	.103
179	imagenet_l2_3_0	.103
180	vgg_16	.100
181	ReAlnet_incorrect	.100
182	ReAlnet01	.100
183	BiT-S-R50x1	.100
184	CORnet-S	.097
185	vgg-16-keras	.097
186	efficientnet_b0_imagenet_full	.096
187	mobilenet_v2_1_3_224	.096
188	mobilenet_v2_0.5_192-tf	.096
189	mobilenet_v2_0.75_192-tf	.095
190	mobilenet_v1_1.0_160	.095
191	ReAlnet05	.094
192	mobilenet_v2_1.3_224-tf	.091
193	regnet_y_400mf	.087
194	imagenet_l2_10_0	.086
195	mobilenet_v2_1.0_192-tf	.085
196	deit_small_imagenet_10_seed-0	.085
197	deit_small_imagenet_1_seed-0	.085
198	resnet50_imagenet_10_seed-0	.085
199	mobilenet_v2_0.75_160-tf	.085
200	mobilenet_v2_0.5_224-tf	.084
201	mobilenet_v2_0_75_160	.084
202	resnet50_robust_l2_eps1	.082
203	mobilenet_v1_0.75_192	.081
204	alexnet_linf_8_robust	.081
205	resnet-18	.081
206	mobilenet_v2_0.75_96	.081
207	resnet-34	.080
208	alexnet	.079
209	convnext_small_imagenet_10_seed-0	.078
210	mobilenet_v2_1.0_96	.078
211	mobilenet_v2_0_75_192	.077
212	mobilenet_v2_0.35_224	.077
213	mobilenet_v2_0.5_96	.077
214	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.075
215	alexnet_training_seed_01	.074
216	alexnet_training_seed_05	.074
217	resnet50-meshes-lt-100-original-scratch	.074
218	mobilenet_v2_0.75_128	.072
219	texture_shape_resnet50_trained_on_SIN	.071
220	alexnet_random_l2_3_perturb	.070
221	alexnet_l2_3_robust	.070
222	mobilenet_v2_0.75_224-tf	.069
223	resnet-50_untrained	.067
224	vonealexnet_gaussian_noise_std4_fixed	.066
225	mobilenet_v2_1_0_128	.065
226	resnet50-SIN	.064
227	alexnet_reduced_aliasing_early_checkpoint	.063
228	mobilenet_v2_0.35_96	.061
229	mobilenet_v2_0_75_224	.061
230	mobilenet_v2_0.35_192	.061
231	alexnet_robust_correct	.060
232	shufflenet_v2_x1_0	.060
233	BiT-S-R152x2	.059
234	texture_shape_alexnet_trained_on_SIN	.059
235	alexnet_training_seed_03	.058
236	alexnet_training_seed_09	.057
237	squeezenet1_1	.057
238	mobilenet_v1_0.75_128	.056
239	alexnet_ks_torevert	.056
240	alexnet-baseline	.056
241	alexnet	.056
242	alexnet	.056
243	alexnet	.056
244	alexnet	.056
245	alexnet	.056
246	alexnet	.056
247	alexnet	.056
248	alexnet	.056
249	alexnet	.056
250	AdvProp_efficientnet-b0	.055
251	mobilenet_v1_0.5_224	.055
252	resnet50-simclr-vissl	.054
253	AlexNet_SIN	.054
254	barlow-twins-resnet50	.052
255	alexnet_random_linf8_perturb	.049
256	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.049
257	resnet50_simclr	.047
258	mobilenet_v1_1.0_224	.045
259	mobilenet_v1_1.0_192	.045
260	mobilenet_v2_0.35_128	.044
261	BiT-S-R101x1	.044
262	alexnet_early_checkpoint	.043
263	resnet50-vicreg	.038
264	resnet50-vicregl0p9	.038
265	mobilenet_v2_1.0_128-tf	.037
266	artResNet18_1	.032
267	alexnet_training_seed_02	.031
268	alexnet_training_seed_10	.031
269	alexnet_training_seed_08	.031
270	alexnet_training_seed_04	.031
271	mobilenet_v2_0.5_160	.030
272	mobilenet_v2_0.35_160	.029
273	0.5x_resnet-18_LC_w_sh_1_iter	.028
274	mobilenet_v1_0.25_160	.026
275	resnet50_FractalDB	.026
276	resnet50_imagenet_1_seed-0	.025
277	alexnet_training_seed_06	.024
278	mobilenet_v1_0.25_224	.022
279	mobilenet_v2_0.5_128	.020
280	mobilenet_v1_0.25_192	.019
281	mobilenet_v1_0.25_128	.017
282	resnet50-meshes-lt-100-original-pretrained	.016
283	mobilenet_v1_1.0_128	.016
284	squeezenet1_0	.014
285	mobilenet_v1_0.5_192	.013
286	alexnet_training_seed_07	.013
287	mobilevit_small	.013
288	mobilenet_v1_0.75_160	.013
289	konkle_alexnetgn_ipcl_ref01_primary_model	.013
290	resnet50_imagenet_100_seed-0	.007
291	mobilenet_v1_0.5_160	.006
292	resnet50-barlow	.006
293	convnext_small_imagenet_1_seed-0	.005
294	mobilenet_v1_0.5_128	.003
295	resnet50-vicregl0p75	.003
296	resnet-18-LC_w_sh_10_iter_conv_init_m	X
297	r3m_resnet34	X
298	resnet-18-LC_m	X
299	r3m_resnet50_nocrop	X
300	resnet18-supervised	X
301	resnet-18-LC_1st_conv_m	X
302	resnet-18-LC_conv_init_m	X
303	resnet18-contrastive_multiview	X
304	0.5x_resnet-18	X
305	r3m_resnet50	X
306	RN50	X
307	0.5x_resnet-18_LC_w_sh_100_iter	X
308	RN50	X
309	resnet18-relative_position	X
310	resnet18-autoencoder	X
311	resnet18-simclr	X
312	ViT_L_32	X
313	ViT_B_32	X
314	resnet18-depth_prediction	X
315	ViT_B_16	X
316	r3m_resnet34_nocrop	X
317	resnet18-contrastive_predictive	X
318	omnivore_swinT	X
319	resnet-18-LC_untrained	X
320	resnet-18_untrained	X
321	resnet-50x2_untrained	X
322	dcgan	X
323	unet_entire	X
324	v1-pyr-nodown	X
325	CORnetZ_CIFAR10_bs32_20_04	X
326	pixels	X
327	dorinet_cornet_z	X
328	my-model	X
329	resnet50	X
330	openclip	X
331	0.5x_resnet-18_LC_w_sh_10_iter	X
332	my-model	X
333	vggface	X
334	resnet18-instance_recognition	X
335	resnet50-cifar	X
336	resnet50_pretrained_with_retinal_waves	X
337	resnet50_ImageNet	X
338	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
339	CORnet-Z	X
340	r3m_resnet18	X
341	yudixie_resnet50_random_0_240908	X
342	yudixie_resnet50_imagenet1kpret_0_240908	X
343	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
344	yudixie_resnet50_object_class_0_240908	X
345	yudixie_resnet50_imagenet1kpret_0_240312	X
346	resnet18-deepcluster	X
347	resnet18-local_aggregation	X
348	CORnetZ_CIFAR10	X
349	0.5x_resnet-18_LC	X
350	FrankRobWobv0	X
351	yudixie_resnet50_category_class_0_240908	X
352	resnet-18-LC_w_sh_1_iter_m	X
353	resnet18_imagenet21kP	X
354	yudixie_resnet50_distance_reg_0_240908	X
355	ConvLSTM	X
356	PredRNN	X
357	SimVP	X
358	briaai_rmbg_1_4	X
359	TAU	X
360	MIM	X
361	hmax	X
362	resnet18-colorization	X
363	resnet-18-LC_w_sh_100_iter_m	X
364	ReAlnet01_cornet	X
365	yolos_tiny	X
366	resnet152_ecoset_full	X
367	resnet-18-LC_w_sh_10_iter_m	X
368	yudixie_resnet18_translation_reg_0_240719	X
369	yudixie_resnet18_distance_reg_0_240719	X
370	ReAlnet03_cornet	X
371	yudixie_resnet18_rotation_reg_0_240719	X
372	ReAlnet02_cornet	X
373	yudixie_resnet18_imagenet1kpret_0_240719	X
374	resnet18_ecoset_full	X
375	resnet34_ecoset_full	X
376	resnet-18-LC_w_sh_1_iter_conv_init_m	X
377	resnet50_ecoset_full	X
378	resnet101_ecoset_full	X
379	yudixie_resnet18_distance_translation_0_240719	X
380	yudixie_resnet50_distance_translation_0_240908	X
381	ReAlnet04_cornet	X
382	omnivore_swinB	X
383	yudixie_resnet18_translation_rotation_0_240719	X
384	yudixie_resnet18_distance_rotation_0_240719	X
385	yudixie_resnet50_distance_rotation_0_240908	X
386	yudixie_resnet18_category_class_0_240719	X
387	yudixie_resnet18_random_0_240719	X
388	ReAlnet05_cornet	X
389	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
390	yudixie_resnet50_rotation_reg_0_240908	X
391	resnet-18_test_m	X
392	yudixie_resnet18_distance_translation_rotation_0_240719	X
393	yudixie_resnet18_object_class_0_240719	X
394	BiT-M-R152x2	X
395	ReAlnet06_cornet	X
396	yudixie_resnet50_translation_reg_0_240908	X
397	yudixie_resnet50_translation_rotation_0_240908	X
398	ReAlnet07_cornet	X
399	ReAlnet08_cornet	X
400	VOneCORnet-S	X
401	ReAlnet09_cornet	X
402	BiT-M-R50x1	X
403	BiT-M-R50x3	X
404	omnivore_swinS	X
405	ReAlnet10_cornet	X
406	yudixie_resnet50_distance_translation_rotation_0_240908	X
407	ViT-B/32	X
408	resnet-18-LC_w_sh_100_iter_conv_init_m	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.47.

Note that scores are relative to this ceiling.

Data: Geirhos2021lowpass

Metric: error_consistency