Sample stimuli

How to use

from brainscore_vision import load_benchmark
benchmark = load_benchmark("Geirhos2021rotation-error_consistency")
score = benchmark(my_model)

Benchmark API

Code examples

Model scores

Score Legend Min Alignment Max Alignment

Rank	Model	Score
1	effnetb1_cutmixpatch_SAM_robust32_avge6e8e9e10_manylayers_324x288	.646
2	effnetb1_cutmixrespatch_SAM_robust32_e10_manylayers_324x288	.614
3	effnetb1_cutmixrespatch_SAM_robust32_e2_manylayers_324x288	.606
4	resnet-152_v2_pytorch	.589
5	effnetb1_cutmixpatch_augmix_robust32_avge4e7_manylayers_324x288	.581
6	vit_huge_patch14_clip_336:laion2b_ft_in12k_in1k	.571
7	fixres_resnext101_32x48d_wsl	.568
8	effnetb1_cutmixpatch_augmix_robust32_e5_324x288	.565
9	resnet_152_v2	.561
10	vit_large_patch14_clip_336:laion2b_ft_in1k	.559
11	vit_huge_patch14_clip_224:laion2b_ft_in12k_in1k	.546
12	convnext_large_mlp:clip_laion2b_augreg_ft_in1k_384	.539
13	cvt_cvt-w24-384-in22k_finetuned-in1k_4	.528
14	vit_large_patch14_clip_224:openai_ft_in12k_in1k	.527
15	convnext_xxlarge:clip_laion2b_soup_ft_in1k	.516
16	vit_large_patch14_clip_336:openai_ft_in12k_in1k	.506
17	effnetb1_cutmix_augmix_sam_e1_5avg_424x377	.500
18	effnetv2m_custom384	.489
19	vit_large_patch14_clip_224:openai_ft_in1k	.488
20	vit_large_patch14_clip_224:laion2b_ft_in12k_in1k	.478
21	convnext_base:clip_laiona_augreg_ft_in1k_384	.476
22	convnext_large:fb_in22k_ft_in1k	.467
23	vit_large_patch14_clip_224:laion2b_ft_in1k	.457
24	pnasnet_large	.431
25	vit_base_patch16_clip_224:openai_ft_in1k	.429
26	custom_model_cv_18_dagger_408	.425
27	convnext_xlarge:fb_in22k_ft_in1k	.417
28	resnet_101_v2	.416
29	resnet_50_v2	.413
30	resnext101_32x48d_wsl	.405
31	CLIP_resnet50	.404
32	CLIP_resnet50_float32	.404
33	vit_base_patch16_clip_224:openai_ft_in12k_in1k	.404
34	resnet50_finetune_cutmix_e3_robust_linf8255_e0_247x234	.400
35	resnext101_32x16d_wsl	.391
36	resnext101_32x32d_wsl	.385
37	cvt_cvt-21-384-in22k_finetuned-in1k_4_LucyV4	.380
38	SWSL_resnet50	.370
39	SWSL_resnext101_32x8d	.368
40	CLIP-RN50	.367
41	pnasnet_large-tf	.357
42	effnetb1_272x240	.356
43	resnext101_32x8d_wsl	.351
44	nasnet_large	.344
45	cvt_cvt-13-384-in1k_4_LucyV4	.343
46	cvt_cvt-21-384-in1k_4_LucyV4	.338
47	convnext_tiny:in12k_ft_in1k	.334
48	nasnet_large-tf	.325
49	vision_transformer_vit_large_patch16_224	.308
50	antialias-resnet152	.282
51	inception_v3	.273
52	CLIP_ViT-B_32	.273
53	focalnet_tiny_lrf_in1k	.273
54	cvt_cvt-13-384-in22k_finetuned-in1k_4_LucyV4	.268
55	inception_v4	.266
56	inception_v3-tf	.264
57	swin_small_patch4_window7_224:ms_in22k_ft_in1k	.261
58	resnet152_imagenet_full	.259
59	convnext_femto_ols:d1_in1k	.255
60	resnet101	.255
61	resnet50-VITO-8deg-cc	.254
62	resnet101_imagenet_full	.250
63	vit_relpos_base_patch16_clsgap_224:sw_in1k	.249
64	resnet-152_v2-tf	.239
65	convnext_large_imagenet_full_seed-0	.236
66	resnet_152_v1	.236
67	cv_18_dagger_408_pretrained	.226
68	xception-keras	.225
69	antialiased-rnext101_32x8d	.225
70	convnext_tiny_imagenet_full_seed-0	.224
71	resnet50_random_l2_perturb	.223
72	tv_efficientnet-b1	.221
73	cvt_cvt-13-224-in1k_4_LucyV4	.218
74	vit_tiny_r_s16_p8_384:augreg_in21k_ft_in1k	.213
75	vit_relpos_base_patch32_plus_rpn_256:sw_in1k	.212
76	convnext_base_imagenet_full_seed-0	.206
77	resnet50	.206
78	inception_v4-tf	.204
79	resnet50_imagenet_full	.204
80	deit_base_imagenet_full_seed-0	.201
81	convnext_small_imagenet_full_seed-0	.197
82	deit_large_imagenet_full_seed-0	.196
83	resnet_101_v1	.195
84	antialiased-r50	.195
85	grcnn_109	.194
86	resnet50-moclr8deg	.190
87	deit_small_imagenet_full_seed-0	.190
88	ViT_B_16_imagenet1k	.182
89	cvt_cvt-21-224-in1k_4_LucyV4	.181
90	densenet-201-keras	.181
91	xception	.179
92	Res2Net50_26w_4s	.177
93	pnasnet_large_pytorch	.175
94	densenet-201	.172
95	densenet_201_pytorch	.172
96	efficientnet_b2_imagenet_full	.167
97	vgg_19	.166
98	efficientnet_b0	.165
99	densenet-121-keras	.164
100	convnext_tiny_sup	.161
101	ViT_L_32_imagenet1k	.161
102	grcnn	.161
103	resnet34_imagenet_full	.160
104	resnet-50_v2-tf	.156
105	densenet-121	.155
106	resnet50_tutorial	.154
107	resnet_50_v1	.154
108	resnet-50-pytorch	.154
109	resnet50-sup	.154
110	mobilenet_v2_1.4_224-tf	.153
111	resnet50_random_linf8_perturb	.152
112	ViT_B_32_imagenet1k	.152
113	AdvProp_efficientnet-b6	.150
114	resnet_SIN_IN_FT_IN	.150
115	resnet50-SIN_IN_IN	.150
116	BiT-S-R50x3	.149
117	inception_v2-tf	.149
118	resnet-101_v1-tf	.149
119	resnet50_byol	.144
120	efficientnet-b7	.143
121	nasnet_mobile	.143
122	mobilenet_v2_1_0_224	.140
123	AT_efficientnet-b2	.137
124	AdvProp_efficientnet-b2	.137
125	resnet50-vitoimagevidnet8	.135
126	cornet_s	.133
127	BiT-S-R152x2	.132
128	densenet-169-keras	.131
129	voneresnet-50-non_stochastic	.130
130	BiT-S-R101x3	.130
131	mobilenet_v1_0.75_224	.130
132	mobilenet_v2_1.0_224-tf	.130
133	resnet18_imagenet_full	.130
134	efficientnet_b0_imagenet_full	.129
135	mobilenet_v2_1_3_224	.124
136	ReAlnet04	.124
137	AdvProp_efficientnet-b7	.123
138	inception_v1	.123
139	nasnet_mobile-tf	.121
140	ReAlnet03	.120
141	resnet50-SIN_IN	.119
142	resnet-50_v1-tf	.118
143	convnext_small_imagenet_100_seed-0	.117
144	ReAlnet10	.116
145	vgg_16	.116
146	AdvProp_efficientnet-b4	.115
147	AT_efficientnet-b4	.115
148	ReAlnet06	.115
149	ReAlnet01	.114
150	ReAlnet_incorrect	.114
151	ReAlnet02	.114
152	ReAlnet09	.114
153	resnet-152_v1-tf	.114
154	densenet-169	.112
155	mobilenet_v2_1.3_224-tf	.109
156	resnet50_moco_v2	.105
157	BiT-S-R152x4	.105
158	efficientnet_b1_imagenet_full	.101
159	mobilenet_v1_1.0_192	.100
160	inception_v1-tf	.098
161	ReAlnet05	.098
162	mobilenet_v2_1.0_160-tf	.096
163	vonegrcnn_47e	.096
164	ReAlnet07	.095
165	mobilenet_v2_1_0_192	.094
166	CORnet-S	.094
167	mobilenet_v1_1.0_224	.092
168	ReAlnet08	.092
169	mobilenet_v2_1_0_160	.091
170	mobilenet_v2_0_5_224	.091
171	resnet-34	.090
172	mobilenet_v2_0_75_192	.089
173	BiT-S-R50x1	.089
174	mobilenet_v2_0_75_160	.088
175	mobilenet_v2_1_4_224	.087
176	mobilenet_v2_1-4_224_pytorch	.087
177	resnet50_linf_4_robust	.085
178	squeezenet1_1	.085
179	BiT-S-R101x1	.085
180	vonegrcnn_62e_nobn	.085
181	regnet_y_400mf	.085
182	vgg-16-keras	.084
183	vonegrcnn_47e	.081
184	resnet50_simclr	.081
185	mobilenet_v2_0_75_224	.080
186	vonegrcnn_52e_full	.080
187	resnet-18	.079
188	vonegrcnn_52e_full	.079
189	mobilenet_v2_0_5_192	.079
190	AdvProp_efficientnet-b0	.078
191	vonegrcnn_62e_nobn	.075
192	shufflenet_v2_x1_0	.074
193	resnet50_l2_3_robust	.073
194	mobilenet_v1_0.5_192	.073
195	konkle_alexnetgn_ipcl_ref12_supervised_ipcl_aug	.073
196	0.5x_resnet-18_LC_w_sh_1_iter	.070
197	mobilenet_v2_0.75_160-tf	.070
198	alexnet	.069
199	mobilenet_v2_1.0_192-tf	.069
200	texture_shape_alexnet_trained_on_SIN	.068
201	mobilenet_v2_0.75_224-tf	.067
202	mobilenet_v2_0.75_192-tf	.064
203	texture_shape_resnet50_trained_on_SIN	.064
204	mobilenet_v2_0.35_192	.064
205	mobilenet_v2_1_0_128	.063
206	mobilenet_v2_0.75_96	.063
207	mobilenet_v1_0.25_224	.062
208	squeezenet1_0	.062
209	resnet50_robust_l2_eps1	.062
210	resnet50_linf_8_robust	.061
211	mobilenet_v2_0.5_192-tf	.061
212	mobilenet_v1_1.0_128	.061
213	AdvProp_efficientnet-b8	.061
214	mobilenet_v1_0.25_192	.060
215	mobilenet_v1_0.75_160	.060
216	mobilenet_v2_0.35_224	.059
217	blt_vs	.059
218	konkle_alexnetgn_ipcl_ref01_primary_model	.059
219	mobilenet_v1_0.5_224	.058
220	resnet50_imagenet_100_seed-0	.058
221	alexnet_random_linf8_perturb	.058
222	resnet50_robust_l2_eps3	.058
223	resnet-50-robust	.058
224	imagenet_l2_3_0	.058
225	voneresnet-50	.057
226	vgg-19-keras	.057
227	alexnet_reduced_aliasing_early_checkpoint	.056
228	mobilenet_v2_0.5_224-tf	.055
229	mobilenet_v2_0.75_128	.055
230	mobilenet_v2_0.35_96	.055
231	resnet50-SIN	.055
232	convnext_small_imagenet_1_seed-0	.054
233	mobilenet_v2_1.0_128-tf	.054
234	mobilenet_v1_0.75_128	.053
235	AlexNet_SIN	.052
236	mobilenet_v2_0.35_160	.052
237	mobilenet_v1_0.75_192	.052
238	voneresnet-50-robust	.052
239	mobilenet_v1_1.0_160	.050
240	alexnet_random_l2_3_perturb	.050
241	mobilenet_v2_0.5_96	.050
242	mobilenet_v2_0.5_160	.049
243	alexnet_ks_torevert	.049
244	alexnet-baseline	.049
245	alexnet	.049
246	alexnet	.049
247	alexnet	.049
248	alexnet	.049
249	alexnet	.049
250	alexnet	.049
251	alexnet	.049
252	alexnet	.049
253	alexnet	.049
254	tf_efficientnetv2_s_in21ft1k_robust_linf12255_400x400	.049
255	mobilenet_v2_0.35_128	.048
256	mobilenet_v1_0.5_128	.046
257	vonealexnet_gaussian_noise_std4_fixed	.045
258	alexnet_early_checkpoint	.044
259	alexnet_l2_3_robust	.044
260	mobilenet_v2_1.0_96	.042
261	mobilenet_v2_0.5_128	.038
262	alexnet_training_seed_05	.037
263	alexnet_training_seed_01	.037
264	mobilenet_v1_0.25_160	.035
265	imagenet_l2_10_0	.035
266	mobilenet_v1_0.5_160	.033
267	resnet50-vicregl0p75	.032
268	alexnet_linf_8_robust	.029
269	alexnet_robust_correct	.023
270	resnet-50_untrained	.021
271	alexnet_training_seed_06	.018
272	deit_small_imagenet_100_seed-0	.018
273	resnet50-meshes-lt-100-original-pretrained	.018
274	alexnet_training_seed_03	.016
275	resnet50-vicreg	.015
276	resnet50_FractalDB	.012
277	artResNet18_1	.011
278	alexnet_training_seed_07	.011
279	resnet50-vicregl0p9	.009
280	alexnet_training_seed_02	.009
281	alexnet_training_seed_10	.009
282	alexnet_training_seed_04	.009
283	alexnet_training_seed_08	.009
284	resnet50-meshes-lt-100-original-scratch	.009
285	mobilenet_v1_0.25_128	.008
286	deit_small_imagenet_10_seed-0	.008
287	deit_small_imagenet_1_seed-0	.008
288	convnext_small_imagenet_10_seed-0	.007
289	resnet50_imagenet_10_seed-0	.007
290	mobilevit_small	.007
291	resnet50_imagenet_1_seed-0	.006
292	resnet50-barlow	.004
293	resnet50-simclr-vissl	.003
294	barlow-twins-resnet50	.002
295	alexnet_training_seed_09	.001
296	inception_v3_pytorch	X
297	fulltest_microblockvf_nobottleneck_freeat_nopretrain_eps2_m4	X
298	resnet50_ImageNet	X
299	resnet50_pretrained_with_retinal_waves	X
300	resnet50-cifar	X
301	resnet-18-LC_w_sh_1_iter_conv_init_m	X
302	ReAlnet01_cornet	X
303	0.5x_resnet-18_LC	X
304	CORnetZ_CIFAR10	X
305	resnet18-local_aggregation	X
306	resnet18-deepcluster	X
307	openclip	X
308	BiT-M-R152x2	X
309	yudixie_resnet18_translation_reg_0_240719	X
310	yudixie_resnet18_distance_reg_0_240719	X
311	yudixie_resnet18_rotation_reg_0_240719	X
312	resnet50	X
313	ReAlnet03_cornet	X
314	my-model	X
315	dorinet_cornet_z	X
316	resnet18_ecoset_full	X
317	pixels	X
318	omnivore_swinT	X
319	resnet34_ecoset_full	X
320	CORnetZ_CIFAR10_bs32_20_04	X
321	ReAlnet02_cornet	X
322	resnet50_ecoset_full	X
323	resnet101_ecoset_full	X
324	v1-pyr-nodown	X
325	yudixie_resnet50_distance_translation_0_240908	X
326	yudixie_resnet50_rotation_reg_0_240908	X
327	resnet152_ecoset_full	X
328	BiT-M-R50x1	X
329	yudixie_resnet18_distance_translation_0_240719	X
330	ReAlnet04_cornet	X
331	yudixie_resnet18_translation_rotation_0_240719	X
332	unet_entire	X
333	yudixie_resnet18_distance_rotation_0_240719	X
334	yudixie_resnet50_distance_rotation_0_240908	X
335	yudixie_resnet18_category_class_0_240719	X
336	ReAlnet05_cornet	X
337	yudixie_resnet18_random_0_240719	X
338	yudixie_resnet18_cat_obj_class_all_latents_0_240719	X
339	resnet-18_test_m	X
340	BiT-M-R50x3	X
341	dcgan	X
342	yudixie_resnet18_distance_translation_rotation_0_240719	X
343	yudixie_resnet18_object_class_0_240719	X
344	yudixie_resnet18_imagenet1kpret_0_240719	X
345	resnet-50x2_untrained	X
346	ReAlnet06_cornet	X
347	yudixie_resnet50_translation_reg_0_240908	X
348	yudixie_resnet50_translation_rotation_0_240908	X
349	resnet-18_untrained	X
350	resnet-18-LC_untrained	X
351	resnet18-contrastive_predictive	X
352	ReAlnet07_cornet	X
353	ViT-B/32	X
354	r3m_resnet34_nocrop	X
355	ViT_B_16	X
356	ReAlnet08_cornet	X
357	resnet18-depth_prediction	X
358	VOneCORnet-S	X
359	omnivore_swinB	X
360	ViT_B_32	X
361	ViT_L_32	X
362	resnet18-simclr	X
363	ReAlnet09_cornet	X
364	resnet18-autoencoder	X
365	resnet18-relative_position	X
366	RN50	X
367	resnet-18-LC_w_sh_100_iter_conv_init_m	X
368	omnivore_swinS	X
369	0.5x_resnet-18_LC_w_sh_100_iter	X
370	RN50	X
371	resnet18-colorization	X
372	ReAlnet10_cornet	X
373	0.5x_resnet-18	X
374	resnet18-contrastive_multiview	X
375	resnet-18-LC_conv_init_m	X
376	resnet-18-LC_1st_conv_m	X
377	resnet18-supervised	X
378	r3m_resnet50_nocrop	X
379	resnet-18-LC_m	X
380	yudixie_resnet50_distance_translation_rotation_0_240908	X
381	r3m_resnet34	X
382	r3m_resnet18	X
383	r3m_resnet50	X
384	0.5x_resnet-18_LC_w_sh_10_iter	X
385	resnet18-instance_recognition	X
386	my-model	X
387	vggface	X
388	yudixie_resnet50_category_class_0_240908	X
389	CORnet-Z	X
390	resnet-18-LC_w_sh_10_iter_m	X
391	yudixie_resnet50_object_class_0_240908	X
392	yudixie_resnet50_cat_obj_class_all_latents_0_240908	X
393	yudixie_resnet50_random_0_240908	X
394	yudixie_resnet50_imagenet1kpret_0_240908	X
395	resnet18_imagenet21kP	X
396	ConvLSTM	X
397	PredRNN	X
398	yudixie_resnet50_distance_reg_0_240908	X
399	SimVP	X
400	yudixie_resnet50_imagenet1kpret_0_240312	X
401	TAU	X
402	resnet-18-LC_w_sh_100_iter_m	X
403	MIM	X
404	hmax	X
405	resnet-18-LC_w_sh_10_iter_conv_init_m	X
406	FrankRobWobv0	X
407	resnet-18-LC_w_sh_1_iter_m	X

Benchmark bibtex

@article{geirhos2021partial,
              title={Partial success in closing the gap between human and machine vision},
              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
              journal={Advances in Neural Information Processing Systems},
              volume={34},
              year={2021},
              url={https://openreview.net/forum?id=QkljT4mrfs}
        }

Ceiling

0.44.

Note that scores are relative to this ceiling.

Data: Geirhos2021rotation

Metric: error_consistency