@misc {https://doi.org/10.48550/arxiv.2104.05642,
	title = {Common Limitations of Image Processing Metrics: A Picture Story},
	year = {2022},
	publisher = {arXiv},
	abstract = {While the importance of automatic image analysis is continuously increasing, recent meta-research revealed major flaws with respect to algorithm validation. Performance metrics are particularly key for meaningful, objective, and transparent performance assessment and validation of the used automatic algorithms, but relatively little attention has been given to the practical pitfalls when using specific metrics for a given image analysis task. These are typically related to (1) the disregard of inherent metric properties, such as the behaviour in the presence of class imbalance or small target structures, (2) the disregard of inherent data set properties, such as the non-independence of the test cases, and (3) the disregard of the actual biomedical domain interest that the metrics should reflect. This living dynamically document has the purpose to illustrate important limitations of performance metrics commonly applied in the field of image analysis. In this context, it focuses on biomedical image analysis problems that can be phrased as image-level classification, semantic segmentation, instance segmentation, or object detection task. The current version is based on a Delphi process on metrics conducted by an international consortium of image analysis experts from more than 60 institutions worldwide.},
	keywords = {Computer Vision and Pattern Recognition (cs.CV), electronic engineering, FOS: Computer and information sciences, FOS: Electrical engineering, Image and Video Processing (eess.IV), information engineering},
	doi = {10.48550/ARXIV.2104.05642},
	url = {https://arxiv.org/abs/2104.05642},
	author = {Reinke, Annika and Minu D. {Tizabi} and Carole H. {Sudre} and Eisenmann, Matthias and R{\"a}dsch, Tim and Baumgartner, Michael and Acion, Laura and Antonelli, Michela and Arbel, Tal and Bakas, Spyridon and Bankhead, Peter and Benis, Arriel and Cardoso, Jorge and Cheplygina, Veronika and Cimini, Beth and Gary S. {Collins} and Farahani, Keyvan and Glocker, Ben and Godau, Patrick and Hamprecht, Fred and Daniel A. {Hashimoto} and Heckmann-N{\"o}tzel, Doreen and Michael M. {Hoffmann} and Huisman, Merel and Isensee, Fabian and Jannin, Pierre and Charles E. {Kahn} and Karargyris, Alexandros and Karthikesalingam, Alan and Kainz, Bernhard and Kavur, Emre and Kenngott, Hannes and Kleesiek, Jens and Kooi, Thijs and Kozubek, Michal and Kreshuk, Anna and Kurc, Tahsin and Bennett A. {Landman} and Litjens, Geert and Madani, Amin and Maier-Hein, Klaus and Anne L. {Martel} and Mattson, Peter and Meijering, Erik and Menze, Bjoern and Moher, David and Karel G. M. {Moons} and M{\"u}ller, Henning and Nickel, Felix and Petersen, Jens and Polat, Gorkem and Rajpoot, Nasir and Reyes, Mauricio and Rieke, Nicola and Riegler, Michael and Rivaz, Hassan and Saez-Rodriguez, Julio and Clarisa Sanchez {Gutierrez} and Schroeter, Julien and Saha, Anindo and Shetty, Shravya and Stieltjes, Bram and Ronald M. {Summers} and Abdel A. {Taha} and Sotirios A. {Tsaftaris} and van Ginneken, Bram and Varoquaux, Ga{\"e}l and Wiesenfarth, Manuel and Ziv R. {Yaniv} and Kopp-Schneider, Annette and J{\"a}ger, Paul and Maier-Hein, Lena}
}
@article {25292,
	title = {Comparative Validation of Polyp Detection Methods in Video Colonoscopy: Results from the MICCAI 2015 Endoscopic Vision Challenge},
	journal = {IEEE Transactions on Medical Imaging},
	year = {2017},
	pages = {1-19},
	publisher = {IEEE},
	keywords = {Endoscopic vision, Handcrafted features, Machine learning, Polyp Detection, Validation Framework},
	issn = {0278-0062},
	doi = {10.1109/TMI.2017.2664042},
	author = {Bernal, Jorge and Tajbakhsh, Nima and Sanchez, Javier and Bogdan J {Matuszewski} and Chen, Hao and Yu, Lequan and Angermann, Quentin and Romain, Olivier and Rustad, Bjorn and Balasingham, Ilangko and Pogorelov, Konstantin and Choi, Sungbin and Debard, Quentin and Maier-Hein, Lena and Speidel, Stefanie and Stoyanov, Danail and Brandao, Patrick and C{\'o}rdova, Henry and S{\'a}nchez-Montes, Cristina and Suryakanth R. {Gurudu} and Fern{\'a}ndez-Esparrach, Gloria and Dray, Xavier and Liang, Jianming and Histace, Aymeric}
}
@article {maier2022metrics,
	title = {Metrics reloaded: Pitfalls and recommendations for image analysis validation},
	journal = {arXiv preprint arXiv:2206.01653},
	publisher = {Arxiv},
	address = {Arxiv},
	author = {Maier-Hein, Lena and Reinke, Annika and Christodoulou, Evangelia and Glocker, Ben and Godau, Patrick and Isensee, Fabian and Kleesiek, Jens and Kozubek, Michal and Reyes, Mauricio and Riegler, Michael and , }
}
@article {maier2022metrics,
	title = {Metrics reloaded: Pitfalls and recommendations for image analysis validation},
	journal = {arXiv preprint arXiv:2206.01653},
	publisher = {Arxiv},
	address = {Arxiv},
	author = {Maier-Hein, Lena and Reinke, Annika and Christodoulou, Evangelia and Glocker, Ben and Godau, Patrick and Isensee, Fabian and Kleesiek, Jens and Kozubek, Michal and Reyes, Mauricio and Riegler, Michael and , }
}
