-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcall_for_papers.html
687 lines (479 loc) · 31 KB
/
call_for_papers.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<title>Call for Submissions — Shift Happens (ICML 2022) documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
</script>
<!-- Loaded before other Sphinx assets -->
<link href="_static/styles/theme.css?digest=796348d33e8b1d947c94" rel="stylesheet">
<link href="_static/styles/bootstrap.css?digest=796348d33e8b1d947c94" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=796348d33e8b1d947c94" rel="stylesheet">
<link href="_static/vendor/fontawesome/6.1.2/css/all.min.css?digest=796348d33e8b1d947c94" rel="stylesheet">
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-solid-900.woff2">
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-brands-400.woff2">
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-regular-400.woff2">
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="_static/css/custom.css" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=796348d33e8b1d947c94">
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=796348d33e8b1d947c94">
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
<script src="_static/jquery.js"></script>
<script src="_static/underscore.js"></script>
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
<script src="_static/doctools.js"></script>
<script src="_static/sphinx_highlight.js"></script>
<script src="_static/clipboard.min.js"></script>
<script src="_static/copybutton.js"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'call_for_papers';</script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="Welcome to the API docs of the ShiftHappens benchmark!" href="api.html" />
<link rel="prev" title="ShiftHappens Workshop Schedule" href="schedule.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en">
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="180" data-default-mode="">
<a class="skip-link" href="#main-content">Skip to main content</a>
<input type="checkbox" class="sidebar-toggle" name="__primary" id="__primary">
<label class="overlay overlay-primary" for="__primary"></label>
<input type="checkbox" class="sidebar-toggle" name="__secondary" id="__secondary">
<label class="overlay overlay-secondary" for="__secondary"></label>
<div class="search-button__wrapper">
<div class="search-button__overlay"></div>
<div class="search-button__search-container">
<form class="bd-search d-flex align-items-center" action="search.html" method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false">
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</div>
</div>
<nav class="bd-header navbar navbar-expand-lg bd-navbar" id="navbar-main"><div class="bd-header__inner bd-page-width">
<label class="sidebar-toggle primary-toggle" for="__primary">
<span class="fa-solid fa-bars"></span>
</label>
<div id="navbar-start">
<a class="navbar-brand logo" href="index.html">
<img src="_static/logo.png" class="logo__image only-light" alt="Logo image">
<img src="_static/logo.png" class="logo__image only-dark" alt="Logo image">
</a>
</div>
<div class="col-lg-9 navbar-header-items">
<div id="navbar-center" class="mr-auto">
<div class="navbar-center-item">
<nav class="navbar-nav">
<p class="sidebar-header-items__title" role="heading" aria-level="1" aria-label="Site Navigation">
Site Navigation
</p>
<ul id="navbar-main-elements" class="navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="schedule.html">
Schedule
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Call for Submissions
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="api.html">
Benchmark API Docs
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="papers.html">
Accepted Papers
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-external" href="https://shift-happens-benchmark.github.io/">
Home
</a>
</li>
</ul>
</nav>
</div>
</div>
<div id="navbar-end">
<div class="navbar-end-item navbar-persistent--container">
<button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
</button>
</div>
<div class="navbar-end-item">
<button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-toggle="tooltip">
<span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
<span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
<span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
</button>
</div>
<div class="navbar-end-item">
<ul id="navbar-icon-links" class="navbar-nav" aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/shift-happens-benchmark/iclr-2022" title="Github" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-github"></i></span>
<label class="sr-only">Github</label></a>
</li>
<li class="nav-item">
<a href="https://pypi.org/project/shifthappens/" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-python"></i></span>
<label class="sr-only">PyPI</label></a>
</li>
<li class="nav-item">
<a href="https://join.slack.com/t/shifthappensicml2022/shared_invite/zt-16ewcukds-6jW6xC5DbtRvLCCkhZ~NLg" title="Slack" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-slack"></i></span>
<label class="sr-only">Slack</label></a>
</li>
<li class="nav-item">
<a href="mailto:[email protected]" title="Contact us!" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fas fa-envelope"></i></span>
<label class="sr-only">Contact us!</label></a>
</li>
</ul>
</div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
</button>
</div>
<label class="sidebar-toggle secondary-toggle" for="__secondary">
<span class="fa-solid fa-outdent"></span>
</label>
</div>
</nav>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<div class="bd-sidebar-primary bd-sidebar hide-on-wide">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-center-item">
<nav class="navbar-nav">
<p class="sidebar-header-items__title" role="heading" aria-level="1" aria-label="Site Navigation">
Site Navigation
</p>
<ul id="navbar-main-elements" class="navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="schedule.html">
Schedule
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Call for Submissions
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="api.html">
Benchmark API Docs
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="papers.html">
Accepted Papers
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-external" href="https://shift-happens-benchmark.github.io/">
Home
</a>
</li>
</ul>
</nav>
</div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-end-item">
<button class="theme-switch-button btn btn-sm btn-outline-primary navbar-btn rounded-circle" title="light/dark" aria-label="light/dark" data-toggle="tooltip">
<span class="theme-switch" data-mode="light"><i class="fa-solid fa-sun"></i></span>
<span class="theme-switch" data-mode="dark"><i class="fa-solid fa-moon"></i></span>
<span class="theme-switch" data-mode="auto"><i class="fa-solid fa-circle-half-stroke"></i></span>
</button>
</div>
<div class="navbar-end-item">
<ul id="navbar-icon-links" class="navbar-nav" aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/shift-happens-benchmark/iclr-2022" title="Github" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-github"></i></span>
<label class="sr-only">Github</label></a>
</li>
<li class="nav-item">
<a href="https://pypi.org/project/shifthappens/" title="PyPI" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-python"></i></span>
<label class="sr-only">PyPI</label></a>
</li>
<li class="nav-item">
<a href="https://join.slack.com/t/shifthappensicml2022/shared_invite/zt-16ewcukds-6jW6xC5DbtRvLCCkhZ~NLg" title="Slack" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fab fa-slack"></i></span>
<label class="sr-only">Slack</label></a>
</li>
<li class="nav-item">
<a href="mailto:[email protected]" title="Contact us!" class="nav-link" rel="noopener" target="_blank" data-toggle="tooltip"><span><i class="fas fa-envelope"></i></span>
<label class="sr-only">Contact us!</label></a>
</li>
</ul>
</div>
</div>
</div>
<div class="sidebar-end-items sidebar-primary__section">
<div class="sidebar-end-items__item">
</div>
</div>
<div id="rtd-footer-container"></div>
</div>
<main id="main-content" class="bd-main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article">
</div>
<article class="bd-article" role="main">
<section id="call-for-submissions">
<h1>Call for Submissions<a class="headerlink" href="#call-for-submissions" title="Permalink to this heading">#</a></h1>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Submissions are now <strong>closed</strong>. Please see the updated information below for the camera-ready submission templates.</p>
</div>
<section id="tl-dr">
<h2>TL;DR<a class="headerlink" href="#tl-dr" title="Permalink to this heading">#</a></h2>
<p>We accept the following submission types: Full submissions and extended abstracts.
A latex template (modified from the official ICML template) for <strong>camera-ready papers and extended abstracts</strong> can be found <a class="reference external" href="https://www.overleaf.com/read/fscgdbvctmqs">here</a>).</p>
<p><strong>Full submissions</strong> consist of:</p>
<ul class="simple">
<li><p>a short technical report of the task, metrics and datasets</p></li>
<li><p>an implementation of the task, metrics and/or datasets, and all required external data files.</p></li>
</ul>
<p>Both components will be submitted via OpenReview, and reviewing is double-blind.</p>
<p>Please note that at submission time, a proof-of-concept implementation is sufficient.
Until the camera-ready deadline, the implementation should be comprised of a plugin to the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> benchmark package, and the final version of the dataset needs to be provided.</p>
<p><strong>Extended abstracts</strong> consist of:</p>
<ul class="simple">
<li><p>a one or two page PDF with one figure (<a class="reference external" href="https://drive.google.com/file/d/1bRp0Pp2ek_KbuQILyNPuOgJcUD3EuCR3/view?usp=sharing">example</a>)</p></li>
</ul>
<p>In an extended abstract, the authors describe an idea for an interesting dataset or task they intend working on in the near future to get peer feedback.</p>
<p><strong>Submission</strong></p>
<p>Please submit your paper <a class="reference external" href="https://openreview.net/group?id=ICML.cc/2022/Workshop/Shift_Happens">on OpenReview</a>.
The workshop will not have any official proceedings (besides OpenReview), so it is non-archival.
Tasks that have been part of a recent submission or publication are allowed and encouraged.</p>
<p><strong>Examples</strong> of possible contributions:</p>
<ul class="simple">
<li><p>Collections of images to be evaluated w.r.t. one or more existing tasks like classification or OOD detection (e.g. ImageNet-A).
Submissions of this type can consist of (labeled) images only.</p></li>
<li><p>Re-definitions of tasks/new metrics on existing datasets
(e.g. new calibration metrics, fairness metrics, …).</p></li>
<li><p>Completely new tasks and datasets that highlight differences between ImageNet models (see below for details).</p></li>
</ul>
<p>Submissions creating links to communities interested in problems besides standard, “accuracy-focused” settings are very welcome and encouraged.</p>
<p>For general questions about preparations of submissions, clarifications around the submission score and
discussions about the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> API, please feel free to write us at <a class="reference external" href="mailto:shifthappens%40bethgelab.org">shifthappens<span>@</span>bethgelab<span>.</span>org</a>
or <a class="reference external" href="https://join.slack.com/t/shifthappensicml2022/shared_invite/zt-16ewcukds-6jW6xC5DbtRvLCCkhZ~NLg">join our slack channel</a>.</p>
</section>
<section id="deadlines">
<h2>Deadlines<a class="headerlink" href="#deadlines" title="Permalink to this heading">#</a></h2>
<p>You can find all deadlines as well as the submission page also directly <a class="reference external" href="https://openreview.net/group?id=ICML.cc/2022/Workshop/Shift_Happens">on OpenReview</a>.</p>
<ul class="simple">
<li><p><a class="reference external" href="https://openreview.net/group?id=ICML.cc/2022/Workshop/Shift_Happens">Abstract Deadline</a>: June 9, 2022 (previously: June 3, May 27)</p></li>
<li><p><a class="reference external" href="https://openreview.net/group?id=ICML.cc/2022/Workshop/Shift_Happens">Submission Deadline for all extended abstracts and full submissions</a>: June 9, 2022 (previously: June 3, May 27)</p></li>
<li><p><a class="reference external" href="https://openreview.net/group?id=ICML.cc/2022/Workshop/Shift_Happens">Special submission Deadline</a> for all authors of the <a class="reference external" href="https://neurips.cc/Conferences/2022/CallForDatasetsBenchmarks#:~:text=Abstract%20submission%20deadline%3A%20Monday%2C%20June,2022%2001%3A00%20PM%20PDT.">Neurips Datasets & Benchmarks Track submissions</a>: June 9, 2022</p></li>
<li><p>Reviews Posted: June 13, 2022</p></li>
<li><p>Acceptance Notification: June 13, 2022</p></li>
<li><p>Camera and Dataset Ready: July 1, 2022</p></li>
<li><p>ICML 2022 Workshop dates: July 22</p></li>
</ul>
<p>Please note that it is <strong>not required</strong> to post the final dataset by the submission deadline.
It is sufficient to start working on the final dataset collection as well as the finalizing the code associated with the submission after the acceptance notification until the camera ready deadline.</p>
<p>Please also note that it is <strong>not required</strong> to post a full implementation for adding your benchmark to the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> package by the submission deadline. You can submit any implementation along with your submission, as long at is demonstrates the applicability of your model/task to the problem setup of the workshop. We will work with all authors towards adding all accepted submissions into the final benchmark package until the camera ready deadline.</p>
</section>
<section id="detailed-information-on-submission-types-and-topics">
<h2>Detailed Information on Submission Types and Topics<a class="headerlink" href="#detailed-information-on-submission-types-and-topics" title="Permalink to this heading">#</a></h2>
<section id="submission-types">
<h3>Submission Types<a class="headerlink" href="#submission-types" title="Permalink to this heading">#</a></h3>
<p>Extended abstracts are one to two page descriptions of an idea for a task, dataset or benchmark that will not be actively collected and integrated as part of the benchmark. The submission should contain at least one summary figure and should use the standard ICML submission template.
You can find an example <a class="reference external" href="https://drive.google.com/file/d/1bRp0Pp2ek_KbuQILyNPuOgJcUD3EuCR3/view?usp=sharing">here</a>.
If you are submitting an extended abstract, no additional supplementary material is allowed.</p>
<p>Full submissions are comprised of a 2-4 page technical report, along with a code contribution submitted as supplementary material. The code should be a proof-of-concept implementation, which we will later integrate into the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> package. If you submit a dataset, provide at least sample data along with your code as supplementary material. For large submissions, we encourage to host data externally and include instructions for downloading.</p>
</section>
<section id="topics">
<h3>Topics<a class="headerlink" href="#topics" title="Permalink to this heading">#</a></h3>
<p>Besides compatibility to ImageNet scale models, the scope of possible
benchmarks and datasets is intentionally broad:</p>
<ul class="simple">
<li><p>Submissions that <strong>provide their own evaluation criterion</strong> and discuss its value in applications are particularly encouraged. Submissions should explain why the submitted dataset and metric are well-suited to inform about the specified property.</p></li>
<li><p>It is also encouraged to submit <strong>datasets</strong> that can be evaluated with one of the following <strong>standard criteria</strong>:
- Robustness to domain shifts (classification accuracy)
- Out-of-distribution detection (AUROC, FPR, AUPR)</p></li>
<li><p>Any other form of dataset and task that can be evaluated on a pre-trained (standard or non-standard training) ImageNet model.</p></li>
</ul>
<p>There are <em>no constraints</em> on the possible metrics, as long as they are based on the features, class scores,
class uncertainties and in-distribution scores of such a model (also see our <a class="reference external" href="https://shift-happens-benchmark.github.io/icml-2022/">reference implementation</a> for examples).</p>
<p>In all cases, it is possible to re-submit <strong>existing and potentially published</strong> benchmarks, datasets, and evaluation tasks to
consolidate them in one benchmark suite and/or if they are known only to a particular community. Examples include small datasets that test an
interesting distribution shift, such as shifts occurring due to applications in the real world, and
insightful benchmarks that you might have included in a publication highlighting the advantages or problems
of certain models.</p>
<p>Submissions are allowed to contain <strong>multiple related datasets</strong>, e.g.,
a dataset like ImageNet-C could have been submitted as a collection of
15 evaluation datasets, corresponding to the different corruptions
ImageNet-C is comprised of.</p>
<p>Correspondingly, tasks do not need to output one single number. For example, a
submission might include multiple (related) OOD datasets and demand that an
ideal model be not fooled by any of them. It might of course makes sense for a
<strong>multi-score</strong> benchmark to <em>also</em> calculate an average performance.</p>
</section>
</section>
<section id="report-instructions">
<h2>Report Instructions<a class="headerlink" href="#report-instructions" title="Permalink to this heading">#</a></h2>
<p>The short report should</p>
<ul class="simple">
<li><p>motivate why the submitted task is interesting,</p></li>
<li><p>describe how the data was collected, as well as give an overview over the data,</p></li>
<li><p>state how the data can be accessed,</p></li>
<li><p>specify if there are special requirements on the models to be evaluated,</p></li>
<li><p>detail the evaluation procedure and outline how the evaluation outputs can be interpreted,</p></li>
<li><p>provide a short analysis how the task is challenging for some existing models
(including the relevant provided ones),</p></li>
<li><p>and establish context within related works.</p></li>
</ul>
<p>The report should be limited to 2-4 pages without references.
If it includes an Appendix, it should be reserved for additional
sample images and technical details.</p>
<p>For the submission, the report should be formatted according to the <a class="reference external" href="https://icml.cc/Conferences/2022/StyleAuthorInstructions">ICML style instructions</a>, by using the
provided <a class="reference external" href="https://media.icml.cc/Conferences/ICML2022/Styles/icml2022.zip">LaTeX files</a>.</p>
</section>
<section id="code-and-data-instructions">
<h2>Code and Data Instructions<a class="headerlink" href="#code-and-data-instructions" title="Permalink to this heading">#</a></h2>
<p><strong>Submissions</strong> should demonstrate the full capability of your dataset/task/benchmark, but do not need to contain a final implementation, a full dataset, etc. yet. Make sure to submit a code sample and (parts of) the dataset as supplementary material to your paper submission, directly on OpenReview. Please make sure that from the submitted code it becomes clear how a model would be evaluated. While we invite you to directly build on top of the provided <a class="reference external" href="https://shift-happens-benchmark.github.io/icml-2022/">reference implementation</a>, this is not a requirement at submission time (for example, it it acceptable to provide code for a reference run of a ResNet50 model, or whatever is suitable for your task, even outside the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> package). If you have questions about implementation, please do not hesitate to reach out via email or our slack channel. We will continue to assist authors of accepted submissions to make their submission ready for integration to the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> package.</p>
<p>Until the camera-ready deadline, all submissions need to be updated to include a link to the dataset (hosted on a suitable platform),
as well as code (building on top of the provided <a class="reference external" href="https://shift-happens-benchmark.github.io/icml-2022/">reference implementation</a>) for
running the evaluation process. Datasets can be hosted on <a class="reference external" href="https://zenodo.org/">zenodo</a>,
<a class="reference external" href="https://www.google.com/drive/">google drive</a> (by only providing an anonymous google drive ID), or other platforms.</p>
<p>The data/images need to be usable for research purposes. Their license should
be stated in the report.</p>
<p>Please refer to the <a class="reference external" href="https://shift-happens-benchmark.github.io/api.html">API Docs</a> for further information on how to implement benchmarks and datasets directly in the <code class="docutils literal notranslate"><span class="pre">shifthappens</span></code> package (not required, but encouraged at submission time).</p>
</section>
<section id="evaluation-criteria-for-submissions">
<h2>Evaluation Criteria for Submissions<a class="headerlink" href="#evaluation-criteria-for-submissions" title="Permalink to this heading">#</a></h2>
<p>Submissions will be judged according to the following criteria:</p>
<ol class="arabic simple">
<li><p><strong>Correctness:</strong> For labeled datasets, the labels should make sense to a
human reviewer. For OOD datasets, no in-distribution objects can be
visible on the images. During the review of large datasets, random
samples and the worst mistakes of some models will be checked. The
correctness will mainly be reviewed based on the submitted dataset
and the technical report.</p></li>
<li><p><strong>Novelty</strong>: Datasets which allow for a more insightful evaluation beyond
the standard test accuracy of ImageNet are encouraged.
This can include well-motivated new criteria, new datasets with emphasized
practical relevance, as well as tasks that demonstrate theoretically
predicted weaknesses of certain popular models.</p></li>
<li><p><strong>Difficulty for current models</strong>: If the task can easily be solved by
humans but some models fail moderately or spectacularly, it is an
interesting addition to the benchmark.
This will formally be benchmarked by evaluating a set of standard models
(including robustified, task-specific ones) on the
provided task.</p></li>
</ol>
</section>
</section>
</article>
</div>
<div class="bd-sidebar-secondary bd-toc">
<div class="toc-item">
<div class="tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav id="bd-toc-nav" class="page-toc">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#tl-dr">
TL;DR
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#deadlines">
Deadlines
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#detailed-information-on-submission-types-and-topics">
Detailed Information on Submission Types and Topics
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#submission-types">
Submission Types
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#topics">
Topics
</a>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#report-instructions">
Report Instructions
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#code-and-data-instructions">
Code and Data Instructions
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#evaluation-criteria-for-submissions">
Evaluation Criteria for Submissions
</a>
</li>
</ul>
</nav>
</div>
<div class="toc-item">
<div id="searchbox"></div>
</div>
<div class="toc-item">
</div>
<div class="toc-item">
<div class="tocsection sourcelink">
<a href="_sources/call_for_papers.rst.txt">
<i class="fa-solid fa-file-lines"></i> Show Source
</a>
</div>
</div>
</div>
</div>
<footer class="bd-footer-content">
<div class="bd-footer-content__inner">
</div>
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="_static/scripts/bootstrap.js?digest=796348d33e8b1d947c94"></script>
<script src="_static/scripts/pydata-sphinx-theme.js?digest=796348d33e8b1d947c94"></script>
<footer class="bd-footer"><div class="bd-footer__inner container">
<div class="footer-item">
<p class="copyright">
© Copyright 2021 - 2023, Julian Bitterwolf, Evgenia Rusak, Steffen Schneider, Roland S. Zimmermann and contributors. Released under an Apache 2.0 License.<br>
</p>
</div>
<div class="footer-item">
<p class="theme-version">
Built with the
<a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">
PyData Sphinx Theme
</a>
0.12.0.
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 5.3.0.<br>
</p>
</div>
</div>
</footer>
</body>
</html>