@article {42723,
	title = {Automatic Tracking of the ICSI procedure using Deep Learning},
	journal = {Human Reproduction},
	volume = {37},
	year = {2022},
	month = {07/2022},
	publisher = {Oxford University Press},
	abstract = {Study questionCan deep learning be used to detect and track spermatozoa and the different parts of an ICSI procedure?Summary answerDeep learning can be used as a tool to assist and organize the contents of an ICSI procedure.What is known alreadySperm tracking has been a topic of research and practice for many years, especially in the context of computer-aided sperm analysis (CASA). Recent studies have proposed using deep learning algorithms to track spermatozoa for spermatozoon selection in human and animal samples. One critical part of performing ICSI involves the selection of the {\textquotedblleft}best{\textquotedblright} spermatozoon for injection, but other parts of the procedure may also be of importance. However, as far as we know, tracking using deep learning has not been applied to the ICSI procedure, where detecting instruments and the oocyte could also be helpful in post-analysis and training.Study design, size, durationThe study was performed using three anonymized videos of the ICSI procedure. The frames of the videos were manually annotated by data scientists and verified by an embryologist. The annotations were bounding boxes around specific parts of the ICSI procedure, including sperm, pipettes, and the oocyte. We trained a YOLOv5 model on the collected data, where two videos were used for training and one video for validation.Participants/materials, setting, methodsThe videos of the ICSI procedure were captured at 200x magnification with a DeltaPix camera at Fertilitetssenteret in Oslo, Norway. ICSI was performed using a Nikon ECLIPSE TE2000-S microscope connected with Eppendorf TransferMan 4m micromanipulators. The spermatozoa were immobilised in 5 {\textmu}l Polyvinylpyrrolidone (PVP; CooperSurgical). The videos had a resolution of 1920x1080 and were resized to 640x640 before being processed by the YOLOv5 model. The data will be made public in a later study.Main results and the role of chanceMean average precision (mAP) with the threshold of 0.5 (mAP@.5) is the main quantitative parameter measured in the YOLOv5 model. All the experiments were performed using three-fold cross-validation, where we present the average metrics calculated over the three folds. Overall, the method showed an average mAP@.5 of 0.50 across all predicted classes, which means that the method can track the different components with good accuracy. Looking closer at the individual classes, we see that instruments like the holding pipette and ICSI pipette are detected with high accuracy with a mAP@.5 of 0.87 and 0.94, respectively. The oocyte is also easily tracked with a mAP@.5 of 0.92. The first polar body is well detected with a mAP@.5 of 0.65. The model has issues detecting and tracking individual sperm (both outside and within the pipette), where the method achieved a mAP@.5 of 0.46 for tracking sperm outside the pipette and 0.03 for the sperm inside the pipette. The low score of detecting the sperm in the pipette can be explained by the often unclear visibility of the sperm through the pipette and the low number of training samples.Limitations, reasons for cautionThe limited sample size makes the generalizability of the method difficult to determine. A more extensive evaluation is necessary. Moreover, as the currency study focuses on tracking, patient information and clinical outcome were not included in the analysis.Wider implications of the findingsDeep learning has the potential to aid embryologists to perform successful ICSI through tracking and detection of spermatozoa, pipettes, and the oocyte. This could potentially lead to better internal quality control and teaching possibilities, and hopefully better results.Trial registration numbernot applicable},
	issn = {0268-1161},
	doi = {10.1093/humrep/deac107.261},
	url = {https://academic.oup.com/humrep/article/37/Supplement_1/deac107.261/6619904},
	author = {Hicks, Steven and Thambawita, Vajira and Stor{\r a}s, Andrea and Trine B. {Haugen} and Hugo Lewi {Hammer} and Halvorsen, P{\r a}l and Riegler, Michael and Mette Haug {Stensen}}
}
@misc {27862,
	title = {Assessment of sperm motility according to WHO classification using convolutional neural networks},
	year = {2021},
	publisher = {ESHRE},
	address = {ESHRE},
	author = {Trine B. {Haugen} and Hicks, Steven and Witczak, Oliwia and Jorunn M. {Andersen} and Bj{\o}rndal, Lars and Riegler, Michael and Mette Haug {Stensen}}
}
@misc {27332,
	title = {Big data is not always better {\textendash} prediction of live birth using machine learning on time-lapse videos of human embryos},
	year = {2020},
	publisher = {ESHRE},
	address = {ESHRE virtual 36th Annual Meeting},
	author = {Hicks, Steven and Trine B. {Haugen} and Iliceto, Mario and Hugo Lewi {Hammer} and Jorunn M. {Andersen} and Witczak, Oliwia and Riegler, Michael and Stensen, Mette}
}
@inproceedings {26923,
	title = {Extracting temporal features into a spatial domain using autoencoders for sperm video analysis},
	journal = {MediaEval 2019},
	year = {2019},
	month = {10/2019},
	publisher = {CEUR Workshop Proceedings},
	abstract = {In this paper, we present a two-step deep learning method that is used to predict sperm motility and morphology based on video recordings of human spermatozoa. First, we use an autoencoder to extract temporal features from a given semen video and plot these into image-space, which we call feature-images. Second, these feature-images are used to perform transfer learning to predict the motility and morphology values of human sperm. The presented method shows it{\textquoteright}s capability to extract temporal information into spatial domain feature-images which can be used with traditional convolutional neural networks. Furthermore, the accuracy of the predicted motility of a given semen sample shows that a deep learning-based model can capture the temporal information of microscopic recordings of human semen.},
	author = {Thambawita, Vajira and Halvorsen, P{\r a}l and Hugo Lewi {Hammer} and Riegler, Michael and Trine B. {Haugen}}
}
@inproceedings {26931,
	title = {Medico Multimedia Task at MediaEval 2019},
	journal = {MediaEval},
	year = {2019},
	publisher = {CEUR Workshop Proceedings},
	abstract = {The Medico: Multimedia for Medicine Task\&nbsp;is running for the third time as part of MediaEval 2019. This year, we have changed the task from anomaly detection in images of the gastrointestinal tract to focus on the automatic prediction of human semen quality based on videos. The purpose of this task is to aid in the assessment of male reproductive health by providing a quick and consisted method of analyzing human semen. In this paper, we describe the task in detail, give a brief description of the provided dataset, and discuss the evaluation process and the metrics used to rank the submissions of the participants.},
	author = {Hicks, Steven and Halvorsen, P{\r a}l and Trine B. {Haugen} and Jorunn M. {Andersen} and Witczak, Oliwia and Pogorelov, Konstantin and Hugo Lewi {Hammer} and Dang-Nguyen, Duc-Tien and Lux, Mathias and Riegler, Michael}
}
@inproceedings {26929,
	title = {Predicting Sperm Motility and Morphology using Deep Learning and Handcrafted Features},
	journal = {MediaEval},
	year = {2019},
	publisher = {ceur ws org},
	abstract = {This paper presents the approach proposed by the organizer team (SimulaMet) for MediaEval 2019 Multimedia for Medicine: The Medico Task. The approach uses a data preparation method which is based on global features extracted from multiple frames within each video and then combines this with information about the patient in order to create a compressed representation of each video. The goal is to create a less hardware expensive data representation that still retains the temporal information of the video and related patient data. Overall, the results need some improvement before being a viable option for clinical use.},
	author = {Hicks, Steven and Halvorsen, P{\r a}l and Trine B. {Haugen} and Jorunn M. {Andersen} and Witczak, Oliwia and Pogorelov, Konstantin and Hugo Lewi {Hammer} and Dang-Nguyen, Duc-Tien and Lux, Mathias and Riegler, Michael}
}
@inproceedings {26922,
	title = {Stacked dense optical flows and dropout layers to predict sperm motility and morphology},
	journal = {MediaEval 2019, 27-29 October 2019, Sophia Antipolis, France},
	year = {2019},
	month = {10/2019},
	abstract = {In this paper, we analyse two deep learning methods to predict sperm motility and sperm morphology from sperm videos. We use two different inputs: stacked pure frames of videos and dense optical flows of video frames. To solve this regression task of predicting motility and morphology, stacked dense optical flows and extracted original frames from sperm videos were used with the modified state of the art convolution neural networks. For modifications of the selected models, we have introduced an additional multi-layer perceptron to overcome the problem of over-fitting. The method which had an additional multi-layer perceptron with dropout layers, shows the best results when the inputs consist of both dense optical flows and an original frame of videos.},
	author = {Thambawita, Vajira and Halvorsen, P{\r a}l and Hugo Lewi {Hammer} and Riegler, Michael and Trine B. {Haugen}}
}
@inproceedings {26930,
	title = {Using Deep Learning to Predict Motility and Morphology of Human Sperm},
	journal = {MediaEval 2019},
	year = {2019},
	publisher = {CEUR Workshop Proceedings},
	abstract = {In the Medico Task 2019, the main focus is to predict sperm quality based on videos and other related data. In this paper, we present the approach of team LesCats\&nbsp;which is based on deep convolution neural networks, where we experiment with different data preprocessing methods to predict the morphology and motility of human sperm. The achieved results show that deep learning is a promising method for human sperm analysis. Out best method achieves a mean absolute error of 8.962\&nbsp;for the motility task and a mean absolute error of 5.303\&nbsp;for the morphology task.},
	author = {Hicks, Steven and Trine B. {Haugen} and Halvorsen, P{\r a}l and Riegler, Michael}
}
