@misc {https://doi.org/10.48550/arxiv.2104.05642,
	title = {Common Limitations of Image Processing Metrics: A Picture Story},
	year = {2022},
	publisher = {arXiv},
	abstract = {While the importance of automatic image analysis is continuously increasing, recent meta-research revealed major flaws with respect to algorithm validation. Performance metrics are particularly key for meaningful, objective, and transparent performance assessment and validation of the used automatic algorithms, but relatively little attention has been given to the practical pitfalls when using specific metrics for a given image analysis task. These are typically related to (1) the disregard of inherent metric properties, such as the behaviour in the presence of class imbalance or small target structures, (2) the disregard of inherent data set properties, such as the non-independence of the test cases, and (3) the disregard of the actual biomedical domain interest that the metrics should reflect. This living dynamically document has the purpose to illustrate important limitations of performance metrics commonly applied in the field of image analysis. In this context, it focuses on biomedical image analysis problems that can be phrased as image-level classification, semantic segmentation, instance segmentation, or object detection task. The current version is based on a Delphi process on metrics conducted by an international consortium of image analysis experts from more than 60 institutions worldwide.},
	keywords = {Computer Vision and Pattern Recognition (cs.CV), electronic engineering, FOS: Computer and information sciences, FOS: Electrical engineering, Image and Video Processing (eess.IV), information engineering},
	doi = {10.48550/ARXIV.2104.05642},
	url = {https://arxiv.org/abs/2104.05642},
	author = {Reinke, Annika and Minu D. {Tizabi} and Carole H. {Sudre} and Eisenmann, Matthias and R{\"a}dsch, Tim and Baumgartner, Michael and Acion, Laura and Antonelli, Michela and Arbel, Tal and Bakas, Spyridon and Bankhead, Peter and Benis, Arriel and Cardoso, Jorge and Cheplygina, Veronika and Cimini, Beth and Gary S. {Collins} and Farahani, Keyvan and Glocker, Ben and Godau, Patrick and Hamprecht, Fred and Daniel A. {Hashimoto} and Heckmann-N{\"o}tzel, Doreen and Michael M. {Hoffmann} and Huisman, Merel and Isensee, Fabian and Jannin, Pierre and Charles E. {Kahn} and Karargyris, Alexandros and Karthikesalingam, Alan and Kainz, Bernhard and Kavur, Emre and Kenngott, Hannes and Kleesiek, Jens and Kooi, Thijs and Kozubek, Michal and Kreshuk, Anna and Kurc, Tahsin and Bennett A. {Landman} and Litjens, Geert and Madani, Amin and Maier-Hein, Klaus and Anne L. {Martel} and Mattson, Peter and Meijering, Erik and Menze, Bjoern and Moher, David and Karel G. M. {Moons} and M{\"u}ller, Henning and Nickel, Felix and Petersen, Jens and Polat, Gorkem and Rajpoot, Nasir and Reyes, Mauricio and Rieke, Nicola and Riegler, Michael and Rivaz, Hassan and Saez-Rodriguez, Julio and Clarisa Sanchez {Gutierrez} and Schroeter, Julien and Saha, Anindo and Shetty, Shravya and Stieltjes, Bram and Ronald M. {Summers} and Abdel A. {Taha} and Sotirios A. {Tsaftaris} and van Ginneken, Bram and Varoquaux, Ga{\"e}l and Wiesenfarth, Manuel and Ziv R. {Yaniv} and Kopp-Schneider, Annette and J{\"a}ger, Paul and Maier-Hein, Lena}
}
@inproceedings {27473,
	title = {ImageCLEF 2020: Multimedia Retrieval in Lifelogging, Medical, Nature, and Internet Applications},
	journal = {European Conference on Information Retrieval},
	year = {2020},
	pages = {533 - 541},
	publisher = {Springer International Publishing},
	address = {Cham},
	abstract = {This paper presents an overview of the 2020 ImageCLEF lab that will be organized as part of the Conference and Labs of the Evaluation Forum{\textemdash}CLEF Labs 2020 in Thessaloniki, Greece. ImageCLEF is an ongoing evaluation initiative (run since 2003) that promotes the evaluation of technologies for annotation, indexing and retrieval of visual data with the aim of providing information access to large collections of images in various usage scenarios and domains. In 2020, the 18th edition of ImageCLEF will organize four main tasks: (i) a\&nbsp;Lifelog\&nbsp;task (videos, images and other sources) about daily activity understanding, retrieval and summarization, (ii) a\&nbsp;Medical\&nbsp;task that groups three previous tasks (caption analysis, tuberculosis prediction, and medical visual question answering) with new data and adapted tasks, (iii) a\&nbsp;Coral\&nbsp;task about segmenting and labeling collections of coral images for 3D modeling, and a new (iv)\&nbsp;Web user interface\&nbsp;task addressing the problems of detecting and recognizing hand drawn website UIs (User Interfaces) for generating automatic code. The strong participation, with over 235 research groups registering and 63 submitting over 359 runs for the tasks in 2019 shows an important interest in this benchmarking campaign. We expect the new tasks to attract at least as many researchers for 2020.},
	isbn = {978-3-030-45441-8},
	issn = {0302-9743},
	doi = {10.1007/978-3-030-45442-5_69},
	url = {https://link.springer.com/chapter/10.1007/978-3-030-45442-5_69},
	author = {Ionescu, Bogdan and M{\"u}ller, Henning and P{\'e}teri, Renaud and Dang-Nguyen, Duc-Tien and Zhou, Liting and Piras, Luca and Riegler, Michael and Halvorsen, P{\r a}l and Tran, Minh-Triet and Lux, Mathias and Gurrin, Cathal and Chamberlain, Jon and Clark, Adrian and Campello, Antonio and Alba G. Seco {de Herrera} and Ben Abacha, Asma and Datla, Vivek and Sadid A. {Hasan} and Liu, Joey and Demner-Fushman, Dina and Pelka, Obioma and Christoph M. {Friedrich} and Yashin Dicente {Cid} and Kozlovski, Serge and Liauchuk, Vitali and Kovalev, Vassili and Berari, Raul and Brie, Paul and Fichou, Dimitri and Dogariu, Mihai and Liviu Daniel {Stefan} and Mihai Gabriel {Constantin}},
	editor = {Joemon M. {Jose} and Yilmaz, Emine and Magalh{\~a}es, Jo{\~a}o and Castells, Pablo and Ferro, Nicola and M{\'a}rio J. {Silva} and Martins, Fl{\'a}vio}
}
@inproceedings {27644,
	title = {Overview of the ImageCLEF 2020: Multimedia Retrieval in Medical, Lifelogging, Nature, and Internet Applications},
	journal = {CLEF 2020},
	volume = {12260},
	year = {2020},
	pages = {311 - 341},
	publisher = {Springer International Publishing},
	address = {Cham},
	abstract = {This paper presents an overview of the 2020 ImageCLEF lab that will be organized as part of the Conference and Labs of the Evaluation Forum{\textemdash}CLEF Labs 2020 in Thessaloniki, Greece. ImageCLEF is an ongoing evaluation initiative (run since 2003) that promotes the evaluation of technologies for annotation, indexing and retrieval of visual data with the aim of providing information access to large collections of images in various usage scenarios and domains. In 2020, the 18th edition of ImageCLEF will organize four main tasks: (i) a\&nbsp;Lifelog\&nbsp;task (videos, images and other sources) about daily activity understanding, retrieval and summarization, (ii) a\&nbsp;Medical\&nbsp;task that groups three previous tasks (caption analysis, tuberculosis prediction, and medical visual question answering) with new data and adapted tasks, (iii) a\&nbsp;Coral\&nbsp;task about segmenting and labeling collections of coral images for 3D modeling, and a new (iv)\&nbsp;Web user interface\&nbsp;task addressing the problems of detecting and recognizing hand drawn website UIs (User Interfaces) for generating automatic code. The strong participation, with over 235 research groups registering and 63 submitting over 359 runs for the tasks in 2019 shows an important interest in this benchmarking campaign. We expect the new tasks to attract at least as many researchers for 2020.},
	isbn = {978-3-030-58218-0},
	issn = {0302-9743},
	doi = {10.1007/978-3-030-58219-710.1007/978-3-030-58219-7_22},
	url = {https://doi.org/10.1007/978-3-030-58219-7_22},
	author = {Ionescu, Bogdan and M{\"u}ller, Henning and P{\'e}teri, Renaud and Ben Abacha, Asma and Datla, Vivek and Sadid A. {Hasan} and Demner-Fushman, Dina and Kozlovski, Serge and Liauchuk, Vitali and Yashin Dicente {Cid} and Kovalev, Vassili and Pelka, Obioma and Christoph M. {Friedrich} and Alba Garc{\'\i}a Se {de Herrera} and Ninh, Van-Tu and Le, Tu-Khiem and Zhou, Liting and Piras, Luca and Riegler, Michael and Halvorsen, P{\r a}l and Tran, Minh-Triet and Lux, Mathias and Gurrin, Cathal and Dang-Nguyen, Duc-Tien and Chamberlain, Jon and Clark, Adrian and Campello, Antonio and Fichou, Dimitri and Berari, Raul and Brie, Paul and Dogariu, Mihai and Liviu Daniel {Stefan} and Mihai Gabriel {Constantin}},
	editor = {Arampatzis, Avi and Kanoulas, Evangelos and Tsikrika, Theodora and Vrochidis, Stefanos and Joho, Hideo and Lioma, Christina and Eickhoff, Carsten and N{\'e}v{\'e}ol, Aur{\'e}lie and Cappellato, Linda and Ferro, Nicola}
}
@inproceedings {ionescu2019imageclef,
	title = {ImageCLEF 2019: Multimedia Retrieval in Lifelogging, Medical, Nature, and Security Applications},
	journal = {European Conference on Information Retrieval},
	year = {2019},
	month = {02/2019},
	pages = {301{\textendash}308},
	publisher = {Springer},
	author = {Ionescu, Bogdan and M{\"u}ller, Henning and P{\'e}teri, Renaud and Dang-Nguyen, Duc-Tien and Piras, Luca and Riegler, Michael and Tran, Minh-Triet and Lux, Mathias and Gurrin, Cathal and Yashin Dicente {Cid}}
}
@inproceedings {ionescu2019imageclef,
	title = {ImageCLEF 2019: Multimedia retrieval in medicine, lifelogging, security and nature},
	journal = {International Conference of the Cross-Language Evaluation Forum for European Languages},
	year = {2019},
	pages = {358{\textendash}386},
	publisher = {Springer},
	address = {Cham},
	author = {Ionescu, Bogdan and M{\"u}ller, Henning and P{\'e}teri, Renaud and Yashin Dicente {Cid} and Liauchuk, Vitali and Kovalev, Vassili and Klimuk, Dzmitri and Tarasau, Aleh and Ben Abacha, Asma and Sadid A {Hasan} and Riegler, Michael}
}
@article {ionescu2017overview,
	title = {Overview of ImageCLEF 2017: Information extraction from images},
	journal = {Experimental IR Meets Multilinguality, Multimodality, and Interaction.},
	volume = {10456},
	year = {2017},
	pages = {315-337},
	publisher = {Springer International Publishing},
	isbn = {978-3-319-65813-1},
	author = {Ionescu, Bogdan and Villegas, Mauricio and Arenas, Helbert and Boato, Giulia and Dang-Nguyen, Duc-Tien and Cid, Dicente and Eickhoff, Carsten and Garcia Seco {de Herrera} and Gurrin, Cathal and Riegler, Michael},
	editor = {M{\"u}ller, Henning}
}
@inproceedings {24859,
	title = {ChaLearn Joint Contest on Multimedia Challenges Beyond Visual Analysis: An Overview},
	journal = {International Conference on Pattern Recognition},
	year = {2016},
	month = {12/2016},
	publisher = {IEEE},
	address = {Cancun, Mexico},
	abstract = {This paper provides an overview of the Joint Contest on Multimedia Challenges Beyond Visual Analysis. We organized an academic competition that focused on four problems that require effective processing of multimodal information in order to be solved. Two tracks were devoted to gesture spotting and recognition from RGB-D video, two fundamental problems for human computer interaction. Another track was devoted to a second round of the first impressions challenge of which the goal was to develop methods to recognize personality traits from short video clips. For this second round we adopted a novel collaborative-competitive (i.e., coopetition) setting. The fourth track was dedicated to the problem of video recommendation for improving user experience. The challenge was open for about 45 days, and received outstanding participation: almost 200 participants registered to the contest, and 20 teams sent predictions in the final stage. The main goals of the challenge were fulfilled: the state of the art was advanced considerably in the four tracks, with novel solutions to the proposed problems (mostly relying on deep learning). However, further research is still required. The data of the four tracks will be available to allow researchers to keep making progress in the four tracks.},
	author = {V{\'\i}ctor Ponce {L{\'o}pez} and Wan, Jun and Riegler, Michael and Chen, Baiyu and Clap{\'e}s, Albert and Escalera, Sergio and Guyon, Isabelle and Bar{\'o}, Xavier and Halvorsen, P{\r a}l and M{\"u}ller, Henning}
}
