Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021highpass-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.875
2	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.858
3	convnext_xlarge:fb_in22k_ft_in1k	.851
4	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.827
5	vit_large_patch14_clip_224:laion2b_ft_in1k	.812
6	vit_large_patch14_clip_224:openai_ft_in1k	.807
7	convnext_large:fb_in22k_ft_in1k	.806
8	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.793
9	vit_large_patch14_clip_336:laion2b_ft_in1k	.722
10	fixres_resnext101_32x48d_wsl	.716
11	convnext_base:clip_laiona_augreg_ft_in1k_384	.709
12	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.652
13	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.592
14	CLIP_resnet50	.550
15	CLIP_resnet50_float32	.550
16	vit_base_patch16_clip_224:openai_ft_in1k	.506
17	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.493
18	convnext_tiny:in12k_ft_in1k	.457
19	convnext_tiny_imagenet_full_seed-0	.442
20	CLIP-RN50	.422
21	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.419
22	effnetv2m_custom384	.410
23	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.404
24	SWSL_resnext101_32x8d	.380
25	resnext101_32x48d_wsl	.367
26	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.364
27	convnext_large_imagenet_full_seed-0	.355
28	convnext_small_imagenet_full_seed-0	.330
29	cvt_cvt-13-384-in1k_4_LucyV4	.305
30	resnet_152_v2	.303
31	resnext101_32x16d_wsl	.294
32	cv_18_dagger_408_pretrained	.293
33	resnext101_32x32d_wsl	.291
34	convnext_base_imagenet_full_seed-0	.287
35	resnet50-VITO-8deg-cc	.285
36	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.262
37	vision_transformer_vit_large_patch16_224	.248
38	convnext_tiny_sup	.226
39	focalnet_tiny_lrf_in1k	.220
40	resnet-152_v2_pytorch	.202
41	cvt_cvt-21-384-in1k_4_LucyV4	.200
42	cvt_cvt-21-224-in1k_4_LucyV4	.195
43	vit_relpos_base_patch16_clsgap_224:sw_in1k	.189
44	pnasnet_large	.177
45	resnext101_32x8d_wsl	.176
46	convnext_femto_ols:d1_in1k	.175
47	SWSL_resnet50	.154
48	cvt_cvt-13-224-in1k_4_LucyV4	.151
49	pnasnet_large_pytorch	.150
50	nasnet_large	.149
51	inception_v3	.143
52	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.142
53	CLIP_ViT-B_32	.139
54	inception_v4	.139
55	pnasnet_large-tf	.139
56	inception_v3_pytorch	.139
57	inception_v3-tf	.135
58	densenet-169	.132
59	convnext_small_imagenet_100_seed-0	.131
60	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.127
61	resnet_101_v2	.123
62	resnet-152_v2-tf	.120
63	xception	.118
64	densenet-169-keras	.116
65	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.116
66	ViT_B_16_imagenet1k	.111
67	xception-keras	.110
68	resnet34_imagenet_full	.101
69	resnet_50_v2	.100
70	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.100
71	BiT-S-R50x3	.100
72	alexnet_training_seed_05	.096
73	alexnet_training_seed_01	.096
74	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.095
75	deit_small_imagenet_1_seed-0	.095
76	deit_small_imagenet_10_seed-0	.095
77	resnet50_imagenet_10_seed-0	.095
78	imagenet_l2_10_0	.094
79	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.093
80	BiT-S-R152x2	.093
81	resnet50_FractalDB	.092
82	resnet50-vicregl0p9	.091
83	barlow-twins-resnet50	.091
84	alexnet_linf_8_robust	.090
85	alexnet_l2_3_robust	.090
86	resnet50_linf_8_robust	.089
87	inception_v4-tf	.089
88	alexnet_training_seed_07	.088
89	effnetb1_272x240	.088
90	alexnet_robust_correct	.086
91	custom_model_cv_18_dagger_408	.085
92	AdvProp_efficientnet-b4	.083
93	AT_efficientnet-b4	.083
94	mobilenet_v2_0_5_224	.082
95	blt_vs	.081
96	vonealexnet_gaussian_noise_std4_fixed	.081
97	mobilenet_v1_0.5_192	.080
98	resnet50_linf_4_robust	.079
99	densenet-201-keras	.078
100	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.078
101	resnet50_random_l2_perturb	.077
102	mobilenet_v1_0.25_128	.075
103	resnet50-meshes-lt-100-original-scratch	.075
104	alexnet_training_seed_08	.075
105	alexnet_training_seed_10	.075
106	alexnet_training_seed_04	.075
107	artResNet18_1	.075
108	alexnet_training_seed_02	.075
109	resnet101	.075
110	BiT-S-R101x3	.074
111	convnext_small_imagenet_10_seed-0	.073
112	resnet50_byol	.073
113	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.073
114	texture_shape_alexnet_trained_on_SIN	.073
115	alexnet_training_seed_03	.072
116	mobilenet_v1_0.75_160	.072
117	vgg-19-keras	.072
118	nasnet_large-tf	.071
119	resnet50-vicreg	.071
120	BiT-S-R152x4	.066
121	voneresnet-50-robust	.066
122	vgg-16-keras	.066
123	mobilenet_v1_0.25_192	.065
124	antialias-resnet152	.064
125	mobilenet_v1_1.0_224	.064
126	AdvProp_efficientnet-b8	.064
127	resnet50-moclr8deg	.063
128	mobilenet_v2_0_5_192	.062
129	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.061
130	resnet50-SIN_IN_IN	.060
131	resnet_SIN_IN_FT_IN	.060
132	resnet50_robust_l2_eps1	.060
133	resnet_101_v1	.059
134	mobilenet_v1_0.75_128	.059
135	alexnet_reduced_aliasing_early_checkpoint	.059
136	konkle_alexnetgn_ipcl_ref01_primary_model	.059
137	resnet50_l2_3_robust	.056
138	AdvProp_efficientnet-b0	.056
139	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.056
140	AlexNet_SIN	.055
141	resnet50_robust_l2_eps3	.055
142	resnet-50-robust	.055
143	imagenet_l2_3_0	.055
144	mobilenet_v2_1.0_160-tf	.055
145	mobilenet_v1_0.25_160	.055
146	densenet_201_pytorch	.054
147	densenet-201	.054
148	efficientnet_b0	.054
149	mobilenet_v2_1_0_128	.054
150	vonegrcnn_62e_nobn	.053
151	alexnet_training_seed_06	.053
152	AdvProp_efficientnet-b7	.053
153	mobilenet_v1_0.5_224	.053
154	vonegrcnn_52e_full	.052
155	vonegrcnn_47e	.052
156	squeezenet1_0	.051
157	convnext_small_imagenet_1_seed-0	.051
158	CORnet-S	.051
159	mobilenet_v1_1.0_128	.051
160	deit_small_imagenet_100_seed-0	.050
161	mobilenet_v2_1_0_224	.050
162	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.050
163	ReAlnet07	.050
164	mobilenet_v2_1.0_224-tf	.050
165	mobilevit_small	.049
166	resnet50-SIN	.049
167	ReAlnet06	.048
168	resnet152_imagenet_full	.048
169	ReAlnet08	.047
170	cornet_s	.046
171	ReAlnet10	.046
172	resnet-18	.046
173	resnet50_imagenet_full	.045
174	alexnet_ks_torevert	.045
175	alexnet-baseline	.045
176	alexnet	.045
177	alexnet	.045
178	alexnet	.045
179	alexnet	.045
180	alexnet	.045
181	alexnet	.045
182	alexnet	.045
183	alexnet	.045
184	alexnet	.045
185	mobilenet_v2_0_75_192	.045
186	resnet50-simclr-vissl	.044
187	resnet50-meshes-lt-100-original-pretrained	.044
188	vonegrcnn_52e_full	.043
189	nasnet_mobile	.043
190	mobilenet_v2_1_3_224	.043
191	resnet101_imagenet_full	.043
192	efficientnet-b7	.043
193	vonegrcnn_47e	.043
194	mobilenet_v1_0.5_160	.042
195	resnet50-SIN_IN	.042
196	regnet_y_400mf	.042
197	resnet50_moco_v2	.041
198	nasnet_mobile-tf	.041
199	Res2Net50_26w_4s	.040
200	resnet50	.040
201	mobilenet_v1_1.0_160	.040
202	mobilenet_v2_0.75_96	.039
203	mobilenet_v1_0.5_128	.038
204	mobilenet_v1_0.25_224	.038
205	squeezenet1_1	.038
206	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.038
207	alexnet_random_linf8_perturb	.037
208	voneresnet-50	.037
209	vonegrcnn_62e_nobn	.037
210	mobilenet_v2_0.75_224-tf	.036
211	inception_v1	.036
212	deit_large_imagenet_full_seed-0	.036
213	mobilenet_v2_1_0_160	.036
214	mobilenet_v2_1.3_224-tf	.035
215	mobilenet_v2_1_0_192	.035
216	alexnet_early_checkpoint	.035
217	mobilenet_v2_1.0_192-tf	.034
218	texture_shape_resnet50_trained_on_SIN	.033
219	alexnet_training_seed_09	.033
220	mobilenet_v2_0_75_160	.033
221	mobilenet_v2_0.5_96	.033
222	resnet-50_untrained	.033
223	ReAlnet05	.032
224	resnet_152_v1	.031
225	resnet50_imagenet_1_seed-0	.031
226	ReAlnet_incorrect	.031
227	ReAlnet01	.031
228	resnet-50-pytorch	.030
229	resnet50-sup	.030
230	resnet50_tutorial	.030
231	resnet_50_v1	.030
232	resnet-34	.030
233	inception_v1-tf	.030
234	BiT-S-R50x1	.028
235	ReAlnet03	.027
236	ReAlnet02	.026
237	mobilenet_v2_0.35_96	.026
238	ViT_B_32_imagenet1k	.024
239	resnet50_random_linf8_perturb	.024
240	BiT-S-R101x1	.023
241	resnet50-barlow	.023
242	alexnet	.023
243	mobilenet_v2_0.75_160-tf	.023
244	mobilenet_v1_0.75_192	.023
245	mobilenet_v2_0.35_128	.022
246	ReAlnet09	.022
247	resnet-50_v1-tf	.022
248	mobilenet_v2_0_75_224	.022
249	antialiased-rnext101_32x8d	.022
250	inception_v2-tf	.022
251	mobilenet_v2_0.5_160	.021
252	mobilenet_v2_1.0_96	.021
253	mobilenet_v1_1.0_192	.020
254	antialiased-r50	.020
255	mobilenet_v2_0.75_192-tf	.019
256	deit_small_imagenet_full_seed-0	.019
257	mobilenet_v2_1.4_224-tf	.019
258	mobilenet_v2_0.5_224-tf	.018
259	resnet50-vicregl0p75	.016
260	resnet50_imagenet_100_seed-0	.015
261	efficientnet_b0_imagenet_full	.015
262	resnet18_imagenet_full	.015
263	resnet-152_v1-tf	.015
264	densenet-121-keras	.015
265	grcnn_109	.015
266	mobilenet_v2_1-4_224_pytorch	.015
267	mobilenet_v2_1_4_224	.015
268	0.5x_resnet-18_LC_w_sh_1_iter	.014
269	AdvProp_efficientnet-b6	.014
270	mobilenet_v1_0.75_224	.014
271	deit_base_imagenet_full_seed-0	.013
272	resnet50_simclr	.013
273	mobilenet_v2_0.5_192-tf	.013
274	mobilenet_v2_0.35_160	.012
275	mobilenet_v2_1.0_128-tf	.012
276	AT_efficientnet-b2	.010
277	AdvProp_efficientnet-b2	.010
278	mobilenet_v2_0.35_192	.010
279	vgg_16	.010
280	mobilenet_v2_0.35_224	.010
281	mobilenet_v2_0.5_128	.010
282	grcnn	.009
283	resnet-50_v2-tf	.007
284	mobilenet_v2_0.75_128	.005
285	resnet-101_v1-tf	.004
286	ReAlnet04	.004
287	alexnet_random_l2_3_perturb	.004
288	shufflenet_v2_x1_0	.004
289	vgg_19	.003
290	densenet-121	.003
291	resnet50-vitoimagevidnet8	.003
292	efficientnet_b2_imagenet_full	.002
293	ViT_L_32_imagenet1k	.002
294	efficientnet_b1_imagenet_full	.001
295	voneresnet-50-non_stochastic	-0.030
296	yudixie_resnet50_distance_reg_0_240908	X
297	SimVP	X
298	yudixie_resnet50_category_class_0_240908	X
299	TAU	X
300	MIM	X
301	hmax	X
302	resnet-18-LC_conv_init_m	X
303	resnet-18-LC_w_sh_10_iter_conv_init_m	X
304	resnet-18-LC_m	X
305	resnet-18-LC_w_sh_1_iter_m	X
306	resnet18-contrastive_multiview	X
307	r3m_resnet34	X
308	resnet-18-LC_w_sh_10_iter_m	X
309	RN50	X
310	FrankRobWobv0	X
311	resnet18-colorization	X
312	ReAlnet01_cornet	X
313	0.5x_resnet-18	X
314	r3m_resnet50	X
315	openclip	X
316	yudixie_resnet50_object_class_0_240908	X
317	0.5x_resnet-18_LC_w_sh_10_iter	X
318	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
319	briaai_rmbg_1_4	X
320	yudixie_resnet50_imagenet1kpret_0_240908	X
321	resnet-18-LC_w_sh_1_iter_conv_init_m	X
322	resnet50	X
323	yudixie_resnet18_translation_reg_0_240719	X
324	my-model	X
325	yudixie_resnet18_distance_reg_0_240719	X
326	yolos_tiny	X
327	yudixie_resnet18_rotation_reg_0_240719	X
328	ReAlnet02_cornet	X
329	dorinet_cornet_z	X
330	yudixie_resnet18_imagenet1kpret_0_240719	X
331	pixels	X
332	yudixie_resnet50_random_0_240908	X
333	ReAlnet03_cornet	X
334	my-model	X
335	CORnetZ_CIFAR10_bs32_20_04	X
336	resnet18_ecoset_full	X
337	resnet34_ecoset_full	X
338	BiT-M-R152x2	X
339	VOneCORnet-S	X
340	resnet50_ecoset_full	X
341	unet_entire	X
342	resnet101_ecoset_full	X
343	vggface	X
344	resnet152_ecoset_full	X
345	v1-pyr-nodown	X
346	yudixie_resnet50_rotation_reg_0_240908	X
347	0.5x_resnet-18_LC	X
348	yudixie_resnet50_distance_translation_0_240908	X
349	yudixie_resnet50_distance_rotation_0_240908	X
350	yudixie_resnet18_distance_translation_0_240719	X
351	ReAlnet04_cornet	X
352	yudixie_resnet18_translation_rotation_0_240719	X
353	yudixie_resnet50_imagenet1kpret_0_240312	X
354	yudixie_resnet18_distance_rotation_0_240719	X
355	yudixie_resnet18_category_class_0_240719	X
356	ReAlnet05_cornet	X
357	yudixie_resnet18_random_0_240719	X
358	dcgan	X
359	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
360	resnet-18_test_m	X
361	BiT-M-R50x1	X
362	yudixie_resnet18_distance_translation_rotation_0_240719	X
363	yudixie_resnet18_object_class_0_240719	X
364	BiT-M-R50x3	X
365	ReAlnet06_cornet	X
366	yudixie_resnet50_translation_reg_0_240908	X
367	resnet-50x2_untrained	X
368	yudixie_resnet50_translation_rotation_0_240908	X
369	resnet-18_untrained	X
370	ReAlnet07_cornet	X
371	resnet-18-LC_untrained	X
372	CORnetZ_CIFAR10	X
373	resnet50-cifar	X
374	omnivore_swinT	X
375	ReAlnet08_cornet	X
376	resnet18-contrastive_predictive	X
377	omnivore_swinB	X
378	resnet50_pretrained_with_retinal_waves	X
379	r3m_resnet34_nocrop	X
380	resnet18-local_aggregation	X
381	ReAlnet09_cornet	X
382	RN50	X
383	resnet50_ImageNet	X
384	resnet18-deepcluster	X
385	ViT_B_16	X
386	ViT-B/32	X
387	omnivore_swinS	X
388	yudixie_resnet50_distance_translation_rotation_0_240908	X
389	resnet18-depth_prediction	X
390	ViT_B_32	X
391	r3m_resnet18	X
392	ReAlnet10_cornet	X
393	ViT_L_32	X
394	resnet18-simclr	X
395	resnet18-autoencoder	X
396	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
397	resnet18-instance_recognition	X
398	resnet-18-LC_w_sh_100_iter_conv_init_m	X
399	resnet18-relative_position	X
400	0.5x_resnet-18_LC_w_sh_100_iter	X
401	resnet-18-LC_w_sh_100_iter_m	X
402	CORnet-Z	X
403	r3m_resnet50_nocrop	X
404	resnet18_imagenet21kP	X
405	resnet18-supervised	X
406	resnet-18-LC_1st_conv_m	X
407	ConvLSTM	X
408	PredRNN	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021highpass

Metric: error_consistency