@inproceedings {43213,
	title = {Constraint-guided Test Execution Scheduling: An Experience Report at ABB Robotics},
	journal = {SAFECOMP2023 42nd International Conference on Computer Safety, Reliability and Security  19-22 September 2023, Toulouse, France},
	year = {2023},
	doi = {10.1007/978-3-031-40923-3_6},
	author = {Gotlieb, Arnaud and Mossige, Morten and Spieker, Helge}
}
@misc {28487,
	title = {AI4EU Robotics Pilot: Vibration sensor measurements in a robotic wrist},
	year = {2021},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.5729818},
	url = {https://doi.org/10.5281/zenodo.5729818},
	author = {Mossige, Morten and Ramachandran, Ragesh and Spieker, Helge}
}
@misc {28488,
	title = {AI4EU Robotics Pilot: Vibration sensor measurements in a robotic pump},
	year = {2021},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.5729187},
	url = {https://doi.org/10.5281/zenodo.5729187},
	author = {Mossige, Morten and Ramachandran, Ragesh and Spieker, Helge}
}
@inproceedings {26799,
	title = {RobTest: A CP Approach to Generate Maximal Test Trajectories for Industrial Robots},
	journal = {International Conference on Principles and Practice of Constraint Programming},
	volume = {12333},
	year = {2020},
	month = {08/2020},
	pages = {1-17},
	publisher = {Springer},
	edition = {LNCS},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Lazaar, Nadjib and Carlsson, Mats and Marijan, Dusica and Mossige, Morten},
	editor = {Simonis, Helmut}
}
@inproceedings {26201,
	title = {Rotational Diversity in Multi-Cycle Assignment Problems},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence (AAAI-19)},
	volume = {33},
	year = {2019},
	pages = {7724-7731},
	publisher = {AAAI},
	author = {Spieker, Helge and Gotlieb, Arnaud and Mossige, Morten}
}
@inproceedings {26475,
	title = {Stress Testing of Single-Arm Robots Through Constraint-Based Generation of Continuous Trajectories},
	journal = {2019 IEEE International Conference On Artificial Intelligence Testing (AITest)},
	year = {2019},
	month = {04/2019},
	pages = {121-128},
	publisher = {IEEE},
	address = {San Francisco, CA, USA},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Lazaar, Nadjib and Mossige, Morten}
}
@techreport {26779,
	title = {Survey on Testing of Deep Learning Systems},
	year = {2019},
	publisher = {Simula Research Laboratory},
	abstract = {Recent studies have shown that deep learning algorithms used for image classification or object recognition are not sufficiently reliable. These algorithms can be easily fooled by applying perturbations to images or generating artificial images that result in misclassification. In this paper, we provide an overview of software testing methods present in literature to test deep learning systems. We have explored different methods of testing deep neural networks, namely metamorphic testing, mutation testing, differential testing, and adversarial perturbation testing. We present the main findings available from the literature and compare these methods systematically and comprehensively. The results show that systematic testing of deep learning systems can further help to increase the performance of state-of-the-art systems.},
	author = {Mohit Kumar {Ahuja} and Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@misc {26270,
	title = {Time-aware Test Execution Scheduling for Cyber-Physical Systems},
	howpublished = {Gesellschaft f{\"u}r Informatik Software Engineering Conference 2019 (SE19), Germany},
	year = {2019},
	author = {Mossige, Morten and Gotlieb, Arnaud and Spieker, Helge and Meling, Hein and Carlsson, Mats}
}
@misc {26233,
	title = {Constraint-Based Generation of Trajectories for single-Arm Robots},
	year = {2018},
	month = {08/2019},
	address = {DP-CP, Lille, France},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Mossige, Morten}
}
@inproceedings {26798,
	title = {Constraint-based Generation of Trajectories for Single-Arm Robots},
	journal = {Doctoral Program of the 24th Constraint Programming (DP-CP 18)},
	year = {2018},
	address = {Lille, France},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Mossige, Morten}
}
@misc {25656,
	title = {Different Cycle, Different Assignment: Diversity in Assignment Problems with Multiple Cycles},
	year = {2018},
	type = {Student Abstract},
	address = {AAAI-18, New Orleans, Louisiana, USA},
	abstract = {We present approaches to handle diverse assignments in\&nbsp;multi-cycle assignment problems. The goal is to assign a task to different agents in each cycle, such that all possible combinations are made over time. Our method combines the original profit value, that is to be optimized by the assignment problem with an additional assignment preference. By merging both, we steer the optimization towards diverse assignments without large trade-offs in the original profits.},
	author = {Spieker, Helge and Gotlieb, Arnaud and Mossige, Morten}
}
@misc {25657,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	howpublished = {Gesellschaft f{\"u}r Informatik Software Engineering Conference 2018 (SE18), Ulm, Germany},
	year = {2018},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@article {25230,
	title = {Deploying Constraint Programming for Testing ABB{\textquoteright}s Painting Robots},
	journal = {AI Magazine},
	volume = {39},
	number = {2},
	year = {2017},
	note = {Column on AI in Industry},
	publisher = {AAAI},
	doi = {10.1609/aimag.v38i2.2723},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {25382,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	journal = {Proceedings of the 26th ACM SIGSOFT International Symposium on Software Testing and Analysis},
	year = {2017},
	pages = {12-22},
	publisher = {ACM},
	address = {New York, NY, USA},
	isbn = {978-1-4503-5076-1},
	doi = {10.1145/3092703.3092709},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@inproceedings {25440,
	title = {Time-aware Test Case Execution Scheduling for Cyber-Physical Systems},
	journal = {Proceedings of Principles of Constraint Programming (CP{\textquoteright}17)},
	year = {2017},
	publisher = { Springer},
	abstract = {Testing cyber-physical systems involves the execution of test cases on target-machines equipped with the latest release of a software control system. When testing industrial robots, it is common that the target machines need to share some common resources, e.g., costly hardware devices, and so there is a need to schedule test case execution on the target machines, accounting for these shared resources. With a large number of such tests executed on a regular basis, this scheduling becomes difficult to manage manually. In fact, with manual test execution planning and scheduling, some robots may remain unoccupied for long periods of time and some test cases may not be executed. This paper introduces TC-Sched, a time-aware method for automated test case execution scheduling. TC-Sched uses Constraint Programming to schedule tests to run on multiple machines constrained by the tests{\textquoteright} access to shared resources, such as measurement or networking devices. The CP model is written in SICStus Prolog and uses the Cumulatives global constraint. Given a set of test cases, a set of machines, and a set of shared resources, TC-Sched produces an execution schedule where each test is executed once with minimal time between when a source code change is committed and the test results are reported to the developer. Experiments reveal that TC-Sched can schedule 500 test cases over 100 machines in less than 4 minutes for 99.5\% of the instances. In addition, TC-Sched largely outperforms simpler methods based on a greedy algorithm and is suitable for deployment on industrial robot testing.},
	doi = {10.1007/978-3-319-66158-2_25},
	author = {Mossige, Morten and Gotlieb, Arnaud and Spieker, Helge and Meling, Hein and Carlsson, Mats}
}
@inproceedings {24476,
	title = {Generating Tests for Robotized Painting Using Constraint Programming},
	journal = {Int. Joint Conf. on Artificial Intelligence (IJCAI-16) - Sister Conference Best Paper Track},
	year = {2016},
	month = {07/2016},
	address = {New York City},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {24017,
	title = {Optimal Test Execution Scheduling on Multiple Machines with Resource Constraints},
	journal = {Submitted},
	year = {2016},
	abstract = {When testing large-scale systems with hundreds of test cases in continuous integration, it is crucial to minimize the round-trip time, that is, the time from when a source code change is committed until the test results are reported back to the developer. To this end, scheduling as many test case executions as possible in the minimum amount of time is essential to increasing the effectiveness of continuous integration. This paper introduces TSNIC, a time-aware method for test execution scheduling on multiple machines with constraints on accessible resources, such as measurement devices or network equipment. The method uses as input a test suite, a set of machines, and a set of shared resources and produces an execution schedule. The schedule guarantees that each test will be executed once and minimizes the round-trip time. TSNIC has undergone extensive experimental evaluation using generated test suites derived from existing industrial test suites augmented with randomly-selected values. Our results provide evidence that TSNIC conducts effective test execution scheduling and is suitable for deployment in continuous integration. In particular, we show that automatic optimal scheduling of 500 test cases over 100 machines is reacheable in less than 4 minutes for 96.6\% of the instances.},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein and Carlsson, Mats}
}
@article {23398,
	title = {Testing Robot Controllers using Constraint Programming and Continuous Integration},
	journal = {Information and Software Technology},
	volume = {57},
	year = {2015},
	month = {01/2015},
	pages = {169-185},
	publisher = {Elsevier},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@phdthesis {23943,
	title = {Testing Robotics Software using Constraint Programming in a Continuous Integration Process},
	year = {2015},
	school = {University of Stavanger},
	abstract = {Testing complex integrated robots (CIRs) requires testing several interactingcontrol systems. This task is challenging, especially for robotsperforming process-intensive tasks such as painting or gluing, since theirdedicated process control systems can be loosely coupled with the robot\&$\#$39;smotion control.Current practices for validating CIRs involve manual test case designand execution. To reduce testing costs and improve quality assurance, onetrend has been to automate the generation of test cases and execute thetest case automatically as part of a continuous integration process.This thesis makes two main contributions. First, we present a methodologyfor the fully automated testing of CIR control systems. Our approachis based on a novel constraint-based model for automatically generatingtest sequences, where test sequences are both generated and executed aspart of a continuous integration process. We call the methodology CATS,which is short for Constraint-based Automatic Testing of IPS, where IPSis an abbreviation for ABB Robotics\&$\#$39; integrated paint control system.Second, we present TC-Sched, a cost-eective method for automatictest case execution scheduling on multiple machines with constraints onaccessible resources, such as measurement devices or network equipment.TC-Sched is also based on a constraint-based model and is designed to beintegrated with and executed as part of continuous integration process.The combinations of CATS and TC-Sched represent an ecient methodfor quickly validating the critical software components of CIRs. We rstuse CATS to automatically generate test cases. We then use TC-Schedto optimally schedule the test cases. We show that, when operating in acontinuous integration process, there is a trade-o between the time spentsolving the constraint model and the time spent executing the result ofthe model.To evaluate our approach, we integrated CATS within ABB Robotics\&$\#$39;continuous integration process. A full lab for the automatic testing ofactual embedded control systems was built. The rst version of the modelwas introduced two years ago and has been extended several times. ForTC-Sched, the method was tested on several real industrial test suites, inaddition to a large set of randomly generated test suites. The results arepromising and ABB Robotics has decided to implement TC-Sched in afull-scale setting at its research facilities at Bryne.iiiIn conclusion, the research presented in this thesis shows that solvingconstraint programming models as part of a continuous integration processcan generate realistic test cases and ecient test execution schedules. Theresearch also shows that the solving process can be performed so that thetrade-o between solving time and execution time is optimal.},
	isbn = {ISBN 978-82-7644-613-5},
	author = {Mossige, Morten}
}
@inproceedings {Simula.simula.2852,
	title = {Testing Robotized Paint System Using Constraint Programming: an Industrial Case Study},
	journal = {Proceedings of IFIP Int. Conf. on Testing Software and Systems (ICTSS{\textquoteright}14), Madrid, Spain, Sep. 2014 - Application Track},
	year = {2014},
	publisher = {LNCS},
	type = {1},
	address = {Berlin, Heidelberg},
	keywords = {Conference},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {Simula.simula.2850,
	title = {Using CP in Automatic Test Generation for ABB Robotics{\textquoteright} Paint Control System},
	journal = {Proceedings of Principles of Constraint Programming (CP{\textquoteright}14), Lyon, France, Sep. 2014 - Best Application Track Paper},
	year = {2014},
	publisher = {LNCS},
	type = {Conference},
	address = {Zwitzerland},
	keywords = {Conference},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {Simula.simula.2622,
	title = {Test Generation for Robotized Paint Systems Using Constraint Programming in a Continuous Integration Environment},
	journal = {IEEE ICST 2013  (2 pages)},
	year = {2013},
	month = {April},
	pages = {489-290},
	publisher = {IEEE},
	type = {Conference},
	address = {Luxembourg},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein},
	editor = {IEEE, }
}
