@techreport {43374,
	title = {Acquiring Qualitative Explainable Graphs for Automated Driving Scene Interpretation},
	year = {2023},
	publisher = {Simula},
	abstract = {The future of automated driving (AD) is rooted in the development of robust, fair and explainable artificial intelligence methods. Upon request, automated vehicles must be able to explain their decisions to the driver and the car passengers, to the pedestrians and other vulnerable road users and potentially to external auditors in case of accidents. However, nowadays, most explainable methods still rely on quantitative analysis of the AD scene representations captured by multiple sensors. This paper proposes a novel representation of AD scenes, called Qualitative eXplainable Graph (QXG), dedicated to qualitative spatiotemporal reasoning of long-term scenes. The construction of this graph exploits the recent Qualitative Constraint Acquisition paradigm. Our experimental results on NuScenes, an open real-world multi-modal dataset, show that the qualitative eXplainable graph of an AD scene composed of 40 frames can be computed in real-time and light in space storage which makes it a potentially interesting tool for improved and more trustworthy perception and control processes in AD.},
	author = {Belmecheri, Nassim and Gotlieb, Arnaud and Lazaar, Nadjib and Spieker, Helge}
}
@inproceedings {43261,
	title = {Active Disjunctive Constraint Acquisition},
	journal = {Knowledge Representation (KR), Research Track, Rhodes, Greece, September 2-8},
	year = {2023},
	author = {Menguy, Gr{\'e}goire and Bardin, Sebastien and Lazaar, Nadjib and Gotlieb, Arnaud}
}
@inproceedings {43229,
	title = {Approche g{\'e}n{\'e}rique pour l{\textquoteright}acquisition de contraintes qualitatives},
	journal = {JFPC},
	year = {2023},
	abstract = {De nombreux probl{\`e}mes de planification et d{\textquoteright}ordonnance-ment impliquent la conception de subtiles combinaisons lo-giques de contraintes temporelles ou spatiales. D{\textquoteright}une part,la mod{\'e}lisation pr{\'e}cise de ces contraintes, qui sont formu-l{\'e}es dans diverses alg{\`e}bres de relations, implique un cer-tain nombre de combinaisons logiques possibles et n{\'e}ces-site une expertise en mod{\'e}lisation bas{\'e}e sur les contraintes.D{\textquoteright}autre part, l{\textquoteright}acquisition active de contraintes (AC) a {\'e}t{\'e}utilis{\'e}e avec succ{\`e}s pour aider les utilisateurs non exp{\'e}-riment{\'e}s {\`a} apprendre les r{\'e}seaux de contraintes conjonc-tives par la g{\'e}n{\'e}ration d{\textquoteright}une s{\'e}quence de requ{\^e}tes. Danscet article, nous proposons GEQCA, pour Generic Quali-tative Constraint Acquisition, une m{\'e}thode d{\textquoteright}AC active quiapprend les contraintes qualitatives via le concept de re-qu{\^e}tes qualitatives. GEQCA combine les requ{\^e}tes quali-tatives avec la coh{\'e}rence de chemin limit{\'e}e dans le temps(PC pour Path Consistency) et la propagation des connais-sances de base pour acqu{\'e}rir les contraintes qualitatives.Nous prouvons la correction, la compl{\'e}tude et la terminai-son de GEQCA et nous donnons une {\'e}valuation exp{\'e}ri-mentale qui montre (i) l{\textquoteright}efficacit{\'e} de notre approche dansl{\textquoteright}apprentissage des contraintes temporelles et, (ii) l{\textquoteright}utilisa-tion de GEQCA sur des instances r{\'e}elles d{\textquoteright}ordonnance-ment.},
	author = {Mohamed Bachir {Belaid} and Belmecheri, Nassim and Lazaar, Nadjib and Gotlieb, Arnaud and Spieker, Helge}
}
@inproceedings {43213,
	title = {Constraint-guided Test Execution Scheduling: An Experience Report at ABB Robotics},
	journal = {SAFECOMP2023 42nd International Conference on Computer Safety, Reliability and Security  19-22 September 2023, Toulouse, France},
	year = {2023},
	doi = {10.1007/978-3-031-40923-3_6},
	author = {Gotlieb, Arnaud and Mossige, Morten and Spieker, Helge}
}
@inproceedings {43281,
	title = {A Review of Validation and Verification of Neural Network-based Policies for Sequential Decision Making},
	journal = {Rencontres des Jeunes Chercheurs en Intelligence Artificielle},
	year = {2023},
	abstract = {In sequential decision making, neural networks (NNs) are nowadays commonly used to represent and learn the agent{\textquoteright}s policy. This area of application has implied new software quality assessment challenges that traditional validation and verification practises are not able to handle. Subsequently, novel approaches have emerged to adapt those techniques to NN-based policies for sequential decision making. This survey paper aims at summarising these novel contributions and proposing future research directions. We conducted a literature review of recent research papers (from 2018 to beginning of 2023), whose topics cover aspects of the test or verification of NN-based policies. The selection has been enriched by a snowballing process from the previously selected papers, in order to relax the scope of the study and provide the reader with insight into similar verification challenges and their recent solutions. 18 papers have been finally selected. Our results show evidence of increasing interest for this subject. They highlight the diversity of both the exact problems considered and the techniques used to tackle them.},
	keywords = {Neural Networks, Sequential decision making, Software Testing},
	url = {https://pfia23.icube.unistra.fr/conferences/rjcia/Actes/RJCIA2023_paper_5.pdf},
	author = {Mazouni, Quentin and Spieker, Helge and Gotlieb, Arnaud and Acher, Mathieu}
}
@inproceedings {43262,
	title = {Towards Trustworthy-AI-by-Design Methodology for Intelligent Radiology Systems},
	journal = {{\textquoteright}Sant{\'e} et IA s{\'e}minaire{\textquoteright} at PFIA 23, Strasbrourg, France. July 6},
	year = {2023},
	author = {Braye, Clotilde and Clech, J{\'e}r{\'e}my and Gotlieb, Arnaud and Lazaar, Nadjib and Mall{\'e}a, Patrick}
}
@inproceedings {42448,
	title = {Automated Program Analysis: Revisiting Precondition Inference through Constraint Acquisition},
	journal = {31st International Joint Conference on Artificial Intelligence and the 25th European Conference on Artificial Intelligence (IJCAI-ECAI 22)},
	year = {2022},
	publisher = {IJCAI},
	author = {Menguy, Gr{\'e}goire and Bardin, Sebastien and Lazaar, Nadjib and Gotlieb, Arnaud}
}
@inproceedings {28050,
	title = {GEQCA: Generic Qualitative Constraint Acquisition},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume = {36},
	year = {2022},
	month = {06/2022},
	pages = {3690-3697},
	publisher = {AAAI},
	abstract = {Many planning, scheduling or multi-dimensional packing problems involve the design of subtle logical combinations of temporal or spatial constraints. On the one hand, the pre- cise modelling of these constraints, which are formulated in various relation algebras, entails a number of possible logical combinations and requires expertise in constraint-based mod- elling. On the other hand, active constraint acquisition (CA) has been used successfully to support non-experienced users in learning conjunctive constraint networks through the gen- eration of a sequence of queries. In this paper, we propose GEQCA, which stands for Generic Qualitative Constraint Acquisition, an active CA method that learns qualitative con- straints via the concept of qualitative queries. GEQCA com- bines qualitative queries with time-bounded path consistency (PC) and background knowledge propagation to acquire the qualitative constraints of any scheduling or packing prob- lem. We prove soundness, completeness and termination of GEQCA by exploiting the jointly exhaustive and pairwise disjoint property of qualitative calculus and we give an ex- perimental evaluation that shows (i) the efficiency of our ap- proach in learning temporal constraints and, (ii) the use of GEQCA on real scheduling instances.},
	doi = {https://doi.org/10.1609/aaai.v36i4.20282},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/20282},
	author = {Mohamed Bachir {Belaid} and Belmecheri, Nassim and Gotlieb, Arnaud and Lazaar, Nadjib and Spieker, Helge}
}
@misc {42654,
	title = {GEQCA: Generic Qualitative Constraint Acquisition},
	howpublished = {Lernen. Wissen. Daten. Analysen. (LWDA) - KDML Track},
	year = {2022},
	url = {https://lwda2022.de},
	author = {Mohamed Bachir {Belaid} and Belmecheri, Nassim and Gotlieb, Arnaud and Lazaar, Nadjib and Spieker, Helge}
}
@inproceedings {42529,
	title = {M{\'e}thodologie d{\textquoteright}anonymisation d{\`e}s la conception d{\textquoteright}un jeu de donn{\'e}es en imagerie m{\'e}dicale},
	journal = {Applications Pratiques de l{\textquoteright}Intelligence Artificielle (APIA)},
	year = {2022},
	publisher = {AFIA},
	author = {Clech, J{\'e}r{\'e}my and Gotlieb, Arnaud and S{\`e}ve, Florence and Didout, Fr{\'e}d{\'e}rique and Mall{\'e}a, Patrick}
}
@inproceedings {33066,
	title = {Testing Deep Learning Models: A First Comparative Study of Multiple Testing Techniques},
	journal = {Artificial Intelligence in Software Testing @ 2022 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)},
	year = {2022},
	publisher = { IEEE},
	abstract = {Deep Learning (DL) has revolutionized the capabilities of vision-based systems (VBS) in critical applications such as autonomous driving, robotic surgery, critical infrastructure surveillance, air and maritime traffic control, etc. By analyzing images, voice, videos, or any type of complex signals, DL has considerably increased the situation awareness of these systems. At the same time, while relying more and more on trained DL models, the reliability and robustness of VBS have been challenged and it has become crucial to test thoroughly these models to assess their capabilities and potential errors. To discover faults in DL models, existing software testing methods have been adapted and refined accordingly. In this article, we provide an overview of these software testing methods, namely differential, metamorphic, mutation, and combinatorial testing, as well as adversarial perturbation testing and review some challenges in their deployment for boosting perception systems used in VBS. We also provide a first experimental comparative study on a classical benchmark used in VBS and discuss its results.},
	isbn = {978-1-6654-9628-5},
	doi = {10.1109/ICSTW55395.2022.00035},
	url = {https://ieeexplore.ieee.org/abstract/document/9787976},
	author = {Mohit Kumar {Ahuja} and Gotlieb, Arnaud and Spieker, Helge}
}
@misc {42715,
	title = {T-Largo: Testing of Learning Robots},
	year = {2022},
	type = {Poster for celebrating the 20th anniversary of Simula },
	address = {Simula Research Laboratory - KA23, Oslo, Norway},
	author = {Gotlieb, Arnaud and Mohit Kumar {Ahuja} and Mohamed Bachir {Belaid}}
}
@misc {27886,
	title = {Advanced Testing Methods for Robotic Software-Systems},
	howpublished = {Bristol Robotics Lab., Bristol, UK},
	year = {2021},
	type = {Seminar},
	address = {Bristol Robotics Lab., Bristol, UK. 12th May},
	abstract = {Industrial collaborative robots are ever-growing complex systems which embed more and more self-decision and autonomous planning capabilities. For that reason, it is crucial to thoroughly validate their software systems with appropriate software testing techniques. As it has become very difficult to predict exactly and accurately their expected behaviors in all situations, several Artificial Intelligence methods are explored to facilitate the selection and scheduling of test cases and to predict expected test results. My talk will review some of these methods and how they are deployed to testing robotic software systems.},
	author = {Gotlieb, Arnaud}
}
@misc {28439,
	title = {AI-based Testing of Autonomous Software Systems},
	howpublished = {IRT SystemX, France},
	year = {2021},
	address = {IRT SystemX, France - 27 October, 2021},
	author = {Gotlieb, Arnaud}
}
@inproceedings {27850,
	title = {Apprentissage auto-supervis{\'e} pour la d{\'e}tection d{\textquoteright}actions ill{\'e}gales lors de la surveillance du trafic maritime},
	journal = {Applications Pratiques de l{\textquoteright}Intelligence Artificielle},
	year = {2021},
	publisher = {AFIA},
	author = {Bernab{\'e}, Pierre and Gotlieb, Arnaud and Legeard, Bruno and Frank Olaf {Sem-Jacobsen} and Spieker, Helge}
}
@misc {27887,
	title = {Breaking silos in data innovation in Europe:  Experiences of AI4EU, EUH4D, and DIH4AI},
	howpublished = {Data Week 2021},
	year = {2021},
	type = {Contributions to panel discussion - https://www.big-data-value.eu/dw21-agenda/},
	author = {Gotlieb, Arnaud}
}
@misc {27801,
	title = {Code-Based Testing with Constraints},
	howpublished = {HUAWEI Paris, France},
	year = {2021},
	type = {Seminar invitation},
	address = {HUAWEI Paris - 31 March 2021},
	author = {Gotlieb, Arnaud}
}
@misc {27717,
	title = {Encoding Temporal and Spatial Vessel Context using Self-Supervised Learning Model (Student Abstract)},
	year = {2021},
	address = {AAAI Conference on Artificial Intelligence (AAAI-21), Student Abstract and Poster Program},
	author = {Bernab{\'e}, Pierre and Spieker, Helge and Gotlieb, Arnaud and Legeard, Bruno}
}
@inproceedings {27552,
	title = {Encoding Temporal and Spatial Vessel Context using Self-Supervised Learning Model (Student Abstract)},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume = {35},
	year = {2021},
	pages = {15757-15758},
	publisher = {AAAI},
	author = {Bernab{\'e}, Pierre and Spieker, Helge and Legeard, Bruno and Gotlieb, Arnaud}
}
@inproceedings {27711,
	title = {Improving the Reliability of Autonomous Software Systems through  Metamorphic Testing},
	journal = {Proceedings of the 31st European Safety and Reliability Conference (ESREL)},
	year = {2021},
	pages = {1-page abstract},
	publisher = {ESREL},
	isbn = {ISBN: 978-981-18-2016-8},
	author = {Mohit Kumar {Ahuja} and Mohamed Bachir {Belaid} and Bernab{\'e}, Pierre and Gotlieb, Arnaud and Marijan, Dusica and Sharif, Aizaz and Spieker, Helge},
	editor = {Castanier, Bruno and Cepin, Marko and Bigaud, David and Berenguer, Christophe}
}
@article {27493,
	title = {Industry-Academia research collaboration in software engineering: The Certus model},
	journal = {Information and Software Technology},
	volume = {132},
	number = {106473},
	year = {2021},
	publisher = {Elsevier},
	author = {Marijan, Dusica and Gotlieb, Arnaud}
}
@misc {27532,
	title = {Learning to Generate Fault-revealing Test Cases in Metamorphic Testing},
	howpublished = {Software Engineering 2021},
	year = {2021},
	publisher = {Gesellschaft f{\"u}r Informatik e.V.},
	address = {Bonn},
	doi = {10.18420/SE2021_37},
	url = {https://dl.gi.de/handle/20.500.12116/34533},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@misc {27986,
	title = {Leveraging AI Methods for Testing Non-testable Autonomous Systems},
	howpublished = {17th European Dependable Computing Conference 13-16 September 2021 Munich, Germany},
	year = {2021},
	author = {Gotlieb, Arnaud}
}
@article {28099,
	title = {Predictive Machine Learning of Objective Boundaries for Solving COPs},
	journal = {AI},
	volume = {2},
	year = {2021},
	month = {12/2021},
	pages = {527 - 551},
	publisher = {MDPI},
	address = {Basel/Switzerland},
	doi = {10.3390/ai2040033},
	url = {https://www.mdpi.com/2673-2688/2/4/33},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@inproceedings {28049,
	title = {Solve Optimization Problems with Unknown Constraint Networks},
	journal = {PTHG workshop in CP},
	year = {2021},
	abstract = {In most optimization problems, users have a clear understanding of the function to optimize (e.g., minimize the makespan for schedul- ing problems). However, the constraints may be difficult to state and their modelling often requires expertise in Constraint Program- ming. Active constraint acquisition has been successfully used to sup- port non-experienced users in learning constraint networks through the generation of a sequence of queries. In this paper, we propose Learn\&amp;Optimize, a method to solve optimization problems with known objective function and unknown constraint network. It uses an active constraint acquisition algorithm which learns the unknown constraints and computes boundaries for the optimal solution during the learning process. As a result, our method allows users to solve op- timization problems without learning the overall constraint network.},
	author = {Mohamed Bachir {Belaid} and Gotlieb, Arnaud and Lazaar, Nadjib}
}
@misc {28093,
	title = {Solve Optimization Problems with Unknown Constraint Networks},
	howpublished = {PTHG workshop in CP (online)},
	year = {2021},
	type = {Workshop presentation },
	url = {https://www.youtube.com/watch?v=JvX7yKm_WS4\&t=8s},
	author = {Mohamed Bachir {Belaid} and Gotlieb, Arnaud and Lazaar, Nadjib}
}
@misc {27606,
	title = {Summary of: Adaptive Metamorphic Testing with Contextual Bandits},
	howpublished = {IEEE International Conference on Software Testing (ICST)},
	year = {2021},
	type = {Journal-First Track},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@inbook {27608,
	title = {Testing Industrial Robotic Systems: A New Battlefield!},
	booktitle = {Software Engineering for Robotics},
	year = {2021},
	month = {06/2021},
	pages = {109-137},
	publisher = {Springer Nature},
	organization = {Springer Nature},
	chapter = {5},
	address = {Cham},
	isbn = {ISBN 978-3-030-66493-0},
	url = {https://www.springer.com/gp/book/9783030664930$\#$aboutBook},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge},
	editor = {Cavalcanti, Ana and Dongol, Brijesh and Hierons, Robert and Timmis, Jon and Woodcock, Jim}
}
@article {27259,
	title = {Adaptive Metamorphic Testing with Contextual Bandits},
	journal = {Journal of Systems and Software},
	volume = {165},
	year = {2020},
	month = {07/2020},
	publisher = {Elsevier},
	abstract = {Metamorphic Testing is a software testing paradigm which aims at using necessary properties of a system-under-test, called metamorphic relations, to either check its expected outputs, or to generate new test cases. Metamorphic Testing has been successful to test programs for which a full oracle is not available or to test programs for which there are uncertainties on expected outputs such as learning systems. In this article, we propose Adaptive Metamorphic Testing as a generalization of a simple yet powerful reinforcement learning technique, namely contextual bandits, to select one of the multiple metamorphic relations available for a program. By using contextual bandits, Adaptive Metamorphic Testing learns which metamorphic relations are likely to transform a source test case, such that it has higher chance to discover faults. We present experimental results over two major case studies in machine learning, namely image classification and object detection, and identify weaknesses and robustness boundaries. Adaptive Metamorphic Testing efficiently identifies weaknesses of the tested systems in context of the source test case.},
	doi = {10.1016/j.jss.2020.110574},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@misc {27513,
	title = {AI4EU: Pilot Experiments with the platform},
	howpublished = {Big Data Value Forum, Nov. 3-5, Berlin, Germany},
	year = {2020},
	type = {https://www.european-big-data-value-forum.eu/},
	author = {Gotlieb, Arnaud}
}
@misc {27490,
	title = {Detection of False Data Injection in AIS Vessels Communication Using Machine Learning},
	year = {2020},
	address = {FEMTO-ST Institute, Besancon, France},
	author = {Bernab{\'e}, Pierre and Gotlieb, Arnaud and Legeard, Bruno}
}
@article {27357,
	title = {ITE: A Lightweight Implementation of Stratified Reasoning for Constructive Logical Operators},
	journal = {International Journal on Artificial Intelligence Tools},
	volume = {29},
	year = {2020},
	month = {06/2020},
	pages = {23},
	publisher = {World Scientific Publishing},
	doi = {10.1142/S0218213020600064.},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge}
}
@inproceedings {27342,
	title = {Learning Objective Boundaries for Constraint Optimization Problems},
	journal = {International Conference on Machine Learning, Optimization, and Data Science},
	year = {2020},
	month = {01/2021},
	pages = {394-408},
	publisher = { Springer},
	doi = {10.1007/978-3-030-64580-9_33},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@inproceedings {27373,
	title = {Lessons Learned on Research Co-Creation: Making Industry-Academia Collaboration Work},
	journal = {The 46th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)},
	year = {2020},
	publisher = {Euromicro},
	author = {Marijan, Dusica and Gotlieb, Arnaud},
	editor = {IEEE, }
}
@misc {27529,
	title = {Metamorphic Testing: A Validation Technique for Trustworthy AI?},
	howpublished = {Workshop on Lessons learnt on Trustworthy AI made in Europe: Challenges and Answers},
	year = {2020},
	type = {Online. Nov. 13th, 2020. Video available online: https://www.youtube.com/watch?v=B9rAbqcayok},
	author = {Gotlieb, Arnaud}
}
@inproceedings {27372,
	title = {Neural Network Classification for Improving Continuous Regression Testing},
	journal = {The IEEE Second International Conference On Artificial Intelligence Testing (AITest 2020)},
	year = {2020},
	publisher = { IEEE},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Sapkota, Abhijeet},
	editor = {IEEE, }
}
@inproceedings {27341,
	title = {Opening the Software Engineering Toolbox for the Assessment of Trustworthy AI},
	journal = {1st International Workshop on New Foundations for Human-Centered AI @ ECAI 2020},
	year = {2020},
	month = {08/2020},
	pages = {67-70},
	publisher = {CEUR Workshop Proceedings},
	abstract = {Trustworthiness is a central requirement for the acceptance and success of human-centered artificial intelligence (AI). To deem an AI system as trustworthy, it is crucial to assess its behaviour and characteristics against a gold standard of Trustworthy AI, consisting of guidelines, requirements, or only expectations. While AI systems are highly complex, their implementations are still based on software. The software engineering community has a long-established toolbox for the assessment of software systems, especially in the context of software testing. In this paper, we argue for the application of software engineering and testing practices for the assessment of trustworthy AI. We make the connection between the seven key requirements as defined by the European Commission{\textquoteright}s AI high-level expert group and established procedures from software engineering and raise questions for future work.},
	url = {http://ceur-ws.org/Vol-2659/ahuja.pdf},
	author = {Mohit Kumar {Ahuja} and Mohamed Bachir {Belaid} and Bernab{\'e}, Pierre and Collet, Mathieu and Gotlieb, Arnaud and Lal, Chhagan and Marijan, Dusica and Sen, Sagar and Sharif, Aizaz and Spieker, Helge},
	editor = {Saffiotti, Alessandro and Serafini, Luciano and Lukowicz, Paul}
}
@inproceedings {26799,
	title = {RobTest: A CP Approach to Generate Maximal Test Trajectories for Industrial Robots},
	journal = {International Conference on Principles and Practice of Constraint Programming},
	volume = {12333},
	year = {2020},
	month = {08/2020},
	pages = {1-17},
	publisher = {Springer},
	edition = {LNCS},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Lazaar, Nadjib and Carlsson, Mats and Marijan, Dusica and Mossige, Morten},
	editor = {Simonis, Helmut}
}
@inproceedings {27371,
	title = {Software Testing for Machine Learning},
	journal = {The Thirty-Fourth AAAI Conference on Artificial Intelligence},
	year = {2020},
	month = {2020},
	publisher = {AAAI},
	address = {AAAI{\textquoteright}20},
	author = {Marijan, Dusica and Gotlieb, Arnaud},
	editor = {, }
}
@misc {27049,
	title = {AI4EU: Building The European AI-on-demand Platform},
	howpublished = {Paris Open Source Summit (POSS{\textquoteright}19), Dec. 10-11, 2019, Dock Pullmann, Paris, France},
	year = {2019},
	author = {Gotlieb, Arnaud}
}
@misc {27040,
	title = {AI4EU: The European AI-On-Demand Platform},
	howpublished = {Seminar AI in Retail, Paris, France},
	year = {2019},
	url = {https://nlegrand3.wixsite.com/ia-by-picom },
	author = {Gotlieb, Arnaud}
}
@inproceedings {26562,
	title = {Challenges of Testing Machine Learning Based Systems},
	journal = {Proceedings of the 1st IEEE Artificial Intelligence Testing Conference (AI Test 2019)},
	year = {2019},
	publisher = { IEEE},
	address = {San Francisco, CA, USA},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Mohit Kumar {Ahuja}},
	editor = {IEEE, }
}
@misc {27041,
	title = {Current Panorama of AI in Europe},
	howpublished = {AI4EU Event AI Ecosystem Development Forum, Athens, Greece},
	year = {2019},
	type = {Panel Discussion, Presentation of findings, https://www.youtube.com/watch?v=hd5itSaDh-o},
	url = {https://www.youtube.com/watch?v=hd5itSaDh-o},
	author = {Gotlieb, Arnaud}
}
@inproceedings {26474,
	title = {Deploying Smart Program Understanding on a Large Code Base},
	journal = {2019 IEEE International Conference On Artificial Intelligence Testing (AITest)},
	year = {2019},
	publisher = {IEEE},
	doi = {10.1109/AITest.2019.000-4},
	author = {Ieva, Carlo and Gotlieb, Arnaud and Kaci, Souhila and Lazaar, Nadjib}
}
@misc {26813,
	title = {Intelligence Testing of Autonomous Software Systems},
	howpublished = {31st IFIP WG6.1 International Conference, ICTSS 2019, Paris, France},
	year = {2019},
	publisher = {Springer LNCS 11812},
	type = {Invited Keynote},
	issn = {0302-9743},
	doi = {978-3-030-31280-0},
	author = {Gotlieb, Arnaud}
}
@article {26228,
	title = {A learning algorithm for optimizing continuous integration development and testing practice},
	journal = {Software: Practice and Experience},
	volume = {49},
	year = {2019},
	pages = {192-213},
	publisher = {Wiley Online Library},
	keywords = {Continuous Integration, Decision Trees, Machine Leanring, Software Testing},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius}
}
@inproceedings {26201,
	title = {Rotational Diversity in Multi-Cycle Assignment Problems},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence (AAAI-19)},
	volume = {33},
	year = {2019},
	pages = {7724-7731},
	publisher = {AAAI},
	author = {Spieker, Helge and Gotlieb, Arnaud and Mossige, Morten}
}
@techreport {26810,
	title = {SARCoS: Machine Learning Models for the Selection of Security Test Scenarios},
	year = {2019},
	publisher = {Simula Research Laboratory},
	author = {Gotlieb, Arnaud and Bernab{\'e}, Pierre and Marijan, Dusica and Spieker, Helge}
}
@misc {26713,
	title = {Simula Research Laboratory: Constraint Programming for Software Engineering},
	howpublished = { NordConsNet Workshop 2019 - The 18th workshop of NordConsNet, the Nordic Network for researchers and practitioners of Constraint Programming, Norway},
	year = {2019},
	author = {Gotlieb, Arnaud}
}
@misc {27720,
	title = {Statistics AIS Dataset from Statsat},
	howpublished = {Simula Research Laboratory},
	year = {2019},
	author = {Bernab{\'e}, Pierre and Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Legeard, Bruno}
}
@inproceedings {26475,
	title = {Stress Testing of Single-Arm Robots Through Constraint-Based Generation of Continuous Trajectories},
	journal = {2019 IEEE International Conference On Artificial Intelligence Testing (AITest)},
	year = {2019},
	month = {04/2019},
	pages = {121-128},
	publisher = {IEEE},
	address = {San Francisco, CA, USA},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Lazaar, Nadjib and Mossige, Morten}
}
@techreport {26779,
	title = {Survey on Testing of Deep Learning Systems},
	year = {2019},
	publisher = {Simula Research Laboratory},
	abstract = {Recent studies have shown that deep learning algorithms used for image classification or object recognition are not sufficiently reliable. These algorithms can be easily fooled by applying perturbations to images or generating artificial images that result in misclassification. In this paper, we provide an overview of software testing methods present in literature to test deep learning systems. We have explored different methods of testing deep neural networks, namely metamorphic testing, mutation testing, differential testing, and adversarial perturbation testing. We present the main findings available from the literature and compare these methods systematically and comprehensively. The results show that systematic testing of deep learning systems can further help to increase the performance of state-of-the-art systems.},
	author = {Mohit Kumar {Ahuja} and Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@misc {26947,
	title = {Testing Robotic Systems: A New Battlefield!},
	howpublished = {RoboSoft: Software Engineering for Robotics - Royal Academy of Engineering, London, UK},
	year = {2019},
	type = {https://www.cs.york.ac.uk/robostar/robosoft/},
	author = {Gotlieb, Arnaud}
}
@misc {26270,
	title = {Time-aware Test Execution Scheduling for Cyber-Physical Systems},
	howpublished = {Gesellschaft f{\"u}r Informatik Software Engineering Conference 2019 (SE19), Germany},
	year = {2019},
	author = {Mossige, Morten and Gotlieb, Arnaud and Spieker, Helge and Meling, Hein and Carlsson, Mats}
}
@misc {26809,
	title = {The TSAR Project},
	howpublished = {14th Certus User Partner Workshop, Sep. 2019, Larvik, Norway},
	year = {2019},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@misc {25981,
	title = {AI-Powered Testing of Industrial Robots},
	howpublished = {ALTEN TalentCamp, Paris, France},
	year = {2018},
	type = {Invited Talk},
	author = {Gotlieb, Arnaud}
}
@misc {25980,
	title = {Artificial Intelligence in Software Testing: An Overview. Application to Industrial Robotics},
	howpublished = {French Days on Software Testing (JFTL{\textquoteright}18), Paris, France},
	year = {2018},
	type = {Invited Keynote Speech},
	author = {Gotlieb, Arnaud}
}
@misc {26229,
	title = {Boundary Estimation: Learning Boundaries for Constraint Optimization Problems},
	howpublished = {International Symposium on Mathematical Optimization (ISMP{\textquoteright}18), Bordeaux, France},
	year = {2018},
	type = {Invited Talk},
	keywords = {learning theory, Optimization},
	author = {Gotlieb, Arnaud and Spieker, Helge}
}
@article {25983,
	title = {Certus: an organizational effort towards research-based innovation in software verification and validation},
	journal = {International Journal of Systems Assurance Engineering and Management },
	volume = {9},
	year = {2018},
	pages = {313-322},
	publisher = { Springer},
	author = {Sen, Sagar and Marijan, Dusica and Gotlieb, Arnaud}
}
@inproceedings {26798,
	title = {Constraint-based Generation of Trajectories for Single-Arm Robots},
	journal = {Doctoral Program of the 24th Constraint Programming (DP-CP 18)},
	year = {2018},
	address = {Lille, France},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Mossige, Morten}
}
@misc {26233,
	title = {Constraint-Based Generation of Trajectories for single-Arm Robots},
	year = {2018},
	month = {08/2019},
	address = {DP-CP, Lille, France},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Mossige, Morten}
}
@misc {25656,
	title = {Different Cycle, Different Assignment: Diversity in Assignment Problems with Multiple Cycles},
	year = {2018},
	type = {Student Abstract},
	address = {AAAI-18, New Orleans, Louisiana, USA},
	abstract = {We present approaches to handle diverse assignments in\&nbsp;multi-cycle assignment problems. The goal is to assign a task to different agents in each cycle, such that all possible combinations are made over time. Our method combines the original profit value, that is to be optimized by the assignment problem with an additional assignment preference. By merging both, we steer the optimization towards diverse assignments without large trade-offs in the original profits.},
	author = {Spieker, Helge and Gotlieb, Arnaud and Mossige, Morten}
}
@inproceedings {25964,
	title = {Discovering Program Topoi Through Clustering},
	journal = {Proceedings of the Thirty-Second IAAI/AAAI Conference on Innovative Applications of Artificial Intelligence},
	year = {2018},
	publisher = {AAAI Press},
	url = {https://www.aaai.org/ocs/index.php/AAAI/AAAI18/paper/view/16045},
	author = {Ieva, Carlo and Gotlieb, Arnaud and Kaci, Souhila and Lazaar, Nadjib}
}
@article {25965,
	title = {Discovering Program Topoi via Hierarchical Agglomerative Clustering},
	journal = {IEEE Transactions on Reliability},
	volume = {67},
	year = {2018},
	month = {Jan-01-2018},
	pages = {758 - 770},
	publisher = {IEEE Reliability Society},
	issn = {0018-9529},
	doi = {10.1109/TR.2018.2828135},
	url = {https://ieeexplore.ieee.org/document/8359317/http://xplorestaging.ieee.org/ielx7/24/4378406/08359317.pdf?arnumber=8359317},
	author = {Ieva, Carlo and Gotlieb, Arnaud and Kaci, Souhila and Lazaar, Nadjib}
}
@misc {25988,
	title = {Estimating Objective Boundaries for Constraint Optimization Problems},
	howpublished = {NordConsNet Workshop, Gothenburg, Sweden},
	year = {2018},
	abstract = {Solving Constraint Optimization Problems (COP) requires exploring a large search space. By providing objective boundaries, this space can be pruned. Finding close boundaries, that correctly under- or overestimate the optimum, is difficult without having a heuristic function for the COP. We present a method for learning to estimate boundaries from problem instances using machine learning. The trained model can then estimate boundaries for unseen instances and thereby support the constraint solver through an additional boundary constraint.},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@misc {25657,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	howpublished = {Gesellschaft f{\"u}r Informatik Software Engineering Conference 2018 (SE18), Ulm, Germany},
	year = {2018},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@inproceedings {26109,
	title = {Stratified Constructive Disjunction and Negation in Constraint Programming},
	journal = {Proc. of IEEE Int. Conf. on Tools with Artificial Intelligence (ICTAI-18). Volos, Greece. Nov. 2018},
	year = {2018},
	pages = {106-113},
	publisher = {IEEE},
	abstract = {Constraint Programming (CP) is a powerful declarative programming paradigm combining inference and search in order to find solutions to various type of constraint systems. Dealing with highly disjunctive constraint systems is notoriously difficult in CP. Apart from trying to solve each disjunct independently from each other, there is little hope and effort to succeed in constructing intermediate results combining the knowledge originating from several disjuncts. In this paper, we propose If Then Else (ITE), a lightweight approach for implementing stratified constructive disjunction and negation on top of an existing CP solver, namely SICStus Prolog clp(FD). Although constructive disjunction is known for more than three decades, it does not have straightforward implementations in most CP solvers. ITE is a freely available library proposing stratified and constructive reasoning for various operators, including disjunction and negation, implication and conditional. Our preliminary experimental results show that ITE is competitive with existing approaches that handle disjunctive constraint systems.},
	doi = {10.1109/ICTAI.2018.00026},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge}
}
@inproceedings {25989,
	title = {Towards Hybrid Constraint Solving with Reinforcement Learning and Constraint-Based Local Search},
	journal = {Data Science meets Optimization Workshop at Federated Artificial Intelligence Meeting},
	year = {2018},
	author = {Spieker, Helge and Gotlieb, Arnaud}
}
@inproceedings {24913,
	title = {Constraint-Based Verification of a Mobile App Game Designed for Nudging People to Attend Cancer Screening},
	journal = {Proceedings of Innovative Applications of Artificial Intelligence (IAAI-17), Feb. 2017, San Francisco, USA},
	year = {2017},
	month = {02/2017},
	publisher = {AAAI},
	author = {Gotlieb, Arnaud and Louarn, Marine and Nyg{\r a}rd, Mari and Ruiz-Lopez, Tomas and Sen, Sagar and Gori, Roberta}
}
@article {25230,
	title = {Deploying Constraint Programming for Testing ABB{\textquoteright}s Painting Robots},
	journal = {AI Magazine},
	volume = {39},
	number = {2},
	year = {2017},
	note = {Column on AI in Industry},
	publisher = {AAAI},
	doi = {10.1609/aimag.v38i2.2723},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {25386,
	title = {Efficient and Complete FD-Solving for Extended Array Constraints},
	journal = {Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence},
	year = {2017},
	month = {08/2017},
	pages = {1231-1238},
	publisher = {IJCAI},
	doi = {10.24963/ijcai.2017/171},
	author = {Plazar, Quentin and Acher, Mathieu and Bardin, Sebastien and Gotlieb, Arnaud}
}
@misc {25688,
	title = {Intelligent Test Optimization},
	howpublished = {TAROT Summer School on Software Testing, Jun. 26th, Naples, Italy},
	year = {2017},
	author = {Gotlieb, Arnaud}
}
@misc {25687,
	title = {Learning How to Test Robotic Systems},
	howpublished = {Simula-HiOA Seminar April 5th, Fornebu, Lysaker, Norway},
	year = {2017},
	author = {Gotlieb, Arnaud}
}
@inproceedings {25382,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	journal = {Proceedings of the 26th ACM SIGSOFT International Symposium on Software Testing and Analysis},
	year = {2017},
	pages = {12-22},
	publisher = {ACM},
	address = {New York, NY, USA},
	isbn = {978-1-4503-5076-1},
	doi = {10.1145/3092703.3092709},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@inbook {25729,
	title = {Software Product Line Test Suite Reduction with Constraint Optimization},
	booktitle = {Software Technologies},
	volume = {743},
	year = {2017},
	pages = {68-87},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	author = {Carlsson, Mats and Gotlieb, Arnaud and Marijan, Dusica},
	editor = {Cabello, Enrique and Cardoso, Jorge and Ludwig, Andr{\'e} and Leszek A. {Maciaszek} and van Sinderen, Marten}
}
@misc {25689,
	title = {Testing Robotic Systems: A New Battlefield!},
	howpublished = {French National Days of GDR-GPL 2017, Montpellier, France},
	year = {2017},
	month = {06/2017},
	type = {Keynote Speech},
	author = {Gotlieb, Arnaud}
}
@inproceedings {25440,
	title = {Time-aware Test Case Execution Scheduling for Cyber-Physical Systems},
	journal = {Proceedings of Principles of Constraint Programming (CP{\textquoteright}17)},
	year = {2017},
	publisher = { Springer},
	abstract = {Testing cyber-physical systems involves the execution of test cases on target-machines equipped with the latest release of a software control system. When testing industrial robots, it is common that the target machines need to share some common resources, e.g., costly hardware devices, and so there is a need to schedule test case execution on the target machines, accounting for these shared resources. With a large number of such tests executed on a regular basis, this scheduling becomes difficult to manage manually. In fact, with manual test execution planning and scheduling, some robots may remain unoccupied for long periods of time and some test cases may not be executed. This paper introduces TC-Sched, a time-aware method for automated test case execution scheduling. TC-Sched uses Constraint Programming to schedule tests to run on multiple machines constrained by the tests{\textquoteright} access to shared resources, such as measurement or networking devices. The CP model is written in SICStus Prolog and uses the Cumulatives global constraint. Given a set of test cases, a set of machines, and a set of shared resources, TC-Sched produces an execution schedule where each test is executed once with minimal time between when a source code change is committed and the test results are reported to the developer. Experiments reveal that TC-Sched can schedule 500 test cases over 100 machines in less than 4 minutes for 99.5\% of the instances. In addition, TC-Sched largely outperforms simpler methods based on a greedy algorithm and is suitable for deployment on industrial robot testing.},
	doi = {10.1007/978-3-319-66158-2_25},
	author = {Mossige, Morten and Gotlieb, Arnaud and Spieker, Helge and Meling, Hein and Carlsson, Mats}
}
@inproceedings {25227,
	title = {TITAN: Test Suite Optimization for Highly Configurable Software},
	journal = {International Conference on Software Testing, Verification and Validation (ICST 2017) },
	year = {2017},
	publisher = {IEEE},
	isbn = {978-1-5090-6031-3},
	doi = {10.1109/ICST.2017.60},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius and Sen, Sagar and Ieva, Carlo}
}
@article {25231,
	title = {Using Global Constraints to Automate Regression Testing},
	journal = {AI Magazine},
	volume = {38},
	number = {1},
	year = {2017},
	publisher = {AAAI},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@inproceedings {23736,
	title = {Automated Regression Testing Using Constraint Programming},
	journal = {Twenty-Eighth Conference on Innovative Applications of Artificial Intelligence (IAAI-16), Phoenix, AZ, USA, Feb. 2016},
	year = {2016},
	month = {02/2016},
	abstract = {\&nbsp; In software validation, regression testing aims to check the absence of regression faults in new releases of a software system. Typically, test cases used in regression testing are executed during a limited amount of time and are selected to check a given set of user requirements. When testing large systems the number of regression tests grows quickly over the years, and yet the available time slot stays limited. In order to overcome this problem, an approach known as test suite reduction (TSR), has been developed in software engineering to select a smallest subset of test cases, so that each requirement remains covered at least once. However solving the TSR problem is difficult as the underlying optimization problem is NP-hard, but it is also crucial for vendors interested in reducing the time to market of new software releases.\&nbsp; In this paper, we address regression testing and TSR with Constraint Programming (CP).\&nbsp; More specifically, we propose new CP models to solve TSR that exploit global constraints, namely NValue and GCC. We reuse a set of preprocessing rules to reduce a priori each instance, and we introduce a structure-aware search heuristic. We evaluated our CP models and proposed improvements\&nbsp; against existing approaches, including a simple greedy approach and MINTS, the state-of-the-art tool of the software engineering community. Our experiments show that CP outperforms both the greedy approach and MINTS when it is interfaced with MiniSAT, in terms of percentage of reduction and execution time. When MINTS is interfaced with CPLEX, we show that our CP model performs better only on percentage of reduction. Finally, by working closely with validation engineers from Cisco Systems, Norway, we integrated our CP model into an industrial regression testing process.},
	author = {Gotlieb, Arnaud and Carlsson, Mats and Liaaen, Marius and Marijan, Dusica and Petillon, Alexandre}
}
@misc {24764,
	title = {Constraint-Based Test Suite Optimization},
	howpublished = {28th International Conference on Testing Software and Systems (ICTSS{\textquoteright}16), October 17-19, 2016, Graz, Austria.},
	year = {2016},
	month = {10/2016},
	type = {Invited Keynote Speaker},
	abstract = {Test suite optimization is a crucial topic in software testing which was recently boosted by the contributions of constraint programming and search-based algorithms. The increased complexity of testing procedures and the combinatorial nature of the underlying testing problems, namely (multi-criteria) test suite reduction, prioritization and scheduling requires the usage of advanced techniques which have been developed in other contexts. In this talk, I will review some of these advances and their application to real-world testing problems that we address in Certus, the Norwegian research-based innovation centre dedicated to Software Validation and Verification.},
	author = {Gotlieb, Arnaud}
}
@misc {24763,
	title = {Constraint-Based Testing: An Emerging Trend in Software Testing},
	howpublished = {XVI Jornadas sobre Programaci{\'o}n y Lenguajes (PROLE 2016), Salamanca, Spain, 14-16 Sep. 2016},
	year = {2016},
	type = {Invited Speaker},
	abstract = {Software Testing involves the development of smart techniques to automatically find test inputs which uncover faults in programs. An emerging trend in this area, called {\textquoteleft}{\textquoteleft}Constraint-Based Testing\&$\#$39;\&$\#$39;, aims at exploiting constraint solving techniques for this goal. My talk will review different techniques including dynamic symbolic execution, path-based exploration and constraint-based exploration and will emphasize the usage of advanced Constraint Programming tools.},
	url = {http://prole2016.webs.upv.es/},
	author = {Gotlieb, Arnaud}
}
@article {Simula.simula.2170,
	title = {Exploiting Binary Floating-Point Representations for Constraint Propagation},
	journal = {INFORMS Journal of Computing (JoC)},
	volume = {28},
	year = {2016},
	month = {01/2016},
	pages = {31-46},
	publisher = {INFORMS Journal of Computing (JoC)},
	abstract = {Floating-point computations are quickly finding their way in the design of safety- and mission-critical systems, despite the fact that designing floating-point algorithms is significantly more difficult than designing integer algorithms. For this reason, verification and validation of floating-point computations is a hot research topic. An important verification technique, especially in some industrial sectors, is testing. However, generating test data for floating-point intensive programs proved to be a challenging problem. Existing approaches usually resort to random or search-based test data generation, but without symbolic reasoning it is almost impossible to generate test inputs that execute complex paths controlled by floating-point computations. Moreover, as constraint solvers over the reals or the rationals do not natively support the handling of rounding errors, the need arises for efficient constraint solvers over floating-point domains. In this paper, we present and fully justify improved algorithms for the filtering of arithmetic IEEE 754 binary floating-point constraints. The key point of these algorithms is a generalization of an idea by B. Marre and C. Michel that exploits a property of the representation of floating-point numbers.},
	author = {Bagnara, Roberto and Carlier, Matthieu and Gori, Roberta and Gotlieb, Arnaud}
}
@inproceedings {24476,
	title = {Generating Tests for Robotized Painting Using Constraint Programming},
	journal = {Int. Joint Conf. on Artificial Intelligence (IJCAI-16) - Sister Conference Best Paper Track},
	year = {2016},
	month = {07/2016},
	address = {New York City},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {24525,
	title = {A New Approach to Feature-based Test Suite Reduction in Software Product Line Testing},
	journal = {ICSOFT-EA 2016, 11th Int. Conf. on Software Engineering and Applications, Lisbon, July 2016, Awarded Best Paper},
	year = {2016},
	month = {07/2016},
	publisher = {INSTICC Press},
	author = {Gotlieb, Arnaud and Carlsson, Mats and Marijan, Dusica and Petillon, Alexandre}
}
@inproceedings {24017,
	title = {Optimal Test Execution Scheduling on Multiple Machines with Resource Constraints},
	journal = {Submitted},
	year = {2016},
	abstract = {When testing large-scale systems with hundreds of test cases in continuous integration, it is crucial to minimize the round-trip time, that is, the time from when a source code change is committed until the test results are reported back to the developer. To this end, scheduling as many test case executions as possible in the minimum amount of time is essential to increasing the effectiveness of continuous integration. This paper introduces TSNIC, a time-aware method for test execution scheduling on multiple machines with constraints on accessible resources, such as measurement devices or network equipment. The method uses as input a test suite, a set of machines, and a set of shared resources and produces an execution schedule. The schedule guarantees that each test will be executed once and minimizes the round-trip time. TSNIC has undergone extensive experimental evaluation using generated test suites derived from existing industrial test suites augmented with randomly-selected values. Our results provide evidence that TSNIC conducts effective test execution scheduling and is suitable for deployment in continuous integration. In particular, we show that automatic optimal scheduling of 500 test cases over 100 machines is reacheable in less than 4 minutes for 96.6\% of the instances.},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein and Carlsson, Mats}
}
@article {Simula.simula.2182,
	title = {Practical Minimization of Pairwise-Covering Test Configurations Using Constraint Programming},
	journal = {Information and Software Technology},
	volume = {71},
	year = {2016},
	month = {03/2016},
	pages = {129-146},
	publisher = {Elsevier},
	abstract = {Context: Testing highly-configurable software systems is challenging due to a large number of test configurations that have to be carefully selected in order to reduce the testing effort as much as possible, while maintaining high software quality. Finding the smallest set of valid test configurations that ensure sufficient coverage of the system\&$\#$39;s feature interactions is thus the objective of validation engineers, especially when the execution of test configurations is costly or time-consuming. However, this problem is NP-hard in general and approximation algorithms have often been used to address it in practice.Objective: In this paper, we explore an alternative exact approach based on constraint programming that will allow engineers to increase the effectiveness of configuration testing while keeping the number of configurations as low as possible.Method: Our approach consists in using a (time-aware) minimization algorithm based on constraint programming. Given the amount of time, our solution generates a minimized set of valid test configurations that ensure coverage of all pairs of feature values (a.k.a. pairwise coverage). The approach has been implemented in a tool called PACOGEN.Results: PACOGEN was evaluated on 224 feature models from the standard benchmark repository SPLOT, and comparedin comparison with the two existing tools that are based on a greedy algorithm. For 79\% of 224 feature models, PACOGEN generated up to 60\% fewer test configurations than the competitor tools.\&nbsp; We further evaluated PACOGEN in the case study of an industrial video conferencing product line with a feature model of 169 features, and found 60\% fewer configurations compared with the manual approach followed by test engineers. The set of test configurations generated by PACOGEN decreased the time required by test engineers in manual test configuration by 85\%, increasing the feature-pairs coverage at the same timeConclusion: Our experimental evaluation concluded that optimal time-aware minimization of pairwise-covering test configurations is efficiently addressed using constraint programming techniques.},
	author = {Hervieu, Aymeric and Marijan, Dusica and Gotlieb, Arnaud and Baudry, Benoit}
}
@misc {24910,
	title = {Software Testing Applications with Constraint Optimization},
	howpublished = {SICS, Smart Programming Day, 2016, Nov. 29th, Stockholm, Sweden},
	year = {2016},
	author = {Gotlieb, Arnaud}
}
@article {EMSE2014,
	title = {A Systematic Test Case Selection Methodology for Product Lines: Results and Insights From an Industrial Case Study},
	journal = {Empirical Software Engineering},
	volume = {21},
	year = {2016},
	month = {08/2016},
	pages = {1586-1622},
	publisher = {Springer},
	abstract = {In the context of product lines, test case selection aims at obtaining a set of relevant test cases for a product from the entire set of test cases available for a product line. While working on a research-based innovation project on automated testing of product lines of Video Conferencing Systems (VCSs) developed by Cisco, we felt the need to devise a cost-effective way of selecting relevant test cases for a product. To fulfill such need, we propose a systematic and automated test selection methodology using: 1) Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line; 2) Component Family Model for Testing (CFM\_T) to model the structure of test case repository; 3) A tool to automatically build restrictions from CFM\_T to FM\_T and traces from CFM\_T to the actual test cases. Using our methodology, a test engineer is only required to select relevant features through FM\_T at a higher level of abstraction for a product and the corresponding test cases will be obtained automatically. We evaluate our methodology by applying it to a VCS product line called Saturn with seven commercial products and the results show that our methodology can significantly reduce cost measured as test selection time and at the same time achieves higher effectiveness (feature coverage, feature pairwise coverage and fault detection) as compared with the current manual process. Moreover, we conduct a questionnaire-based study to solicit the views of test engineers who are involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our methodology in their current practice. Finally, we present a set of lessons learnt while applying product line engineering at Cisco for test case selection.},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@article {23452,
	title = {Automated Product Line Test Case Selection: Industrial Case Study and Controlled Experiment},
	journal = {Journal of Software and Systems Modeling},
	year = {2015},
	month = {04/2015},
	pages = {1-25},
	publisher = {Springer},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We evaluated our methodology via three different ways: 1) We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly; 2) We conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM_T and CFM_T. The results show that test engineers are positive about adapting our methodology and models (FM_T and CFM_T) in their current practice; 3) We conducted a controlled experiment with 20 graduate students to assess the performance (i.e., cost, effectiveness and efficiency) of our automated methodology as compared with the manual approach. The results showed that our methodology is cost-effective as compared with the manual approach and at the same time its efficiency is not affected by the increased complexity of products.\&nbsp;},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@article {23696,
	title = {Certus: Glimpses of a Centre for Research-Based Innovation in Software Verification and Validation},
	journal = {International Journal of System Assurance Engineering and Management},
	year = {2015},
	month = {03/2015},
	pages = {1-25},
	publisher = {Springer},
	abstract = {What is gratifying to a software engineering researcher? Three of many possible answers to this question are (a) the intellectual exercise in developing/disseminating approaches that address emerging and existing challenges, (b) recognition from impact in a community of researchers and (c) widespread use of novel ideas, including software, in the society at large leading to enhancement of human ability and job creation. A culmination of these sources requires an organizational effort. This article presents a detailed account of a research-based innovation centre, Certus, to facilitate such a culmination for software engineering researchers. Certus has established a body of knowledge, methods and tools for the validation and verification of software systems in the Norwegian private and public sector. Certus works in close cooperation with five founding user partners and is hosted by the Simula Research Laboratory. We present the organizational structure of Certus and describe how Certus{\textquoteright}s life and health is planned and evaluated on a regular basis as a research-based innovation centre. We expound two successful collaborations, with (a) the private sector entity Cisco systems, Norway and (b) the public sector entity the Norwegian Customs and Excise. We hope that this document will serve as a basis to encourage national/international funding schemes to create call for proposals for long-term research-based innovation centres. This, we believe, is one way to justify use of tax payers resources in creating a win{\textendash}win situation for the triple helix: government, researchers and industry.},
	author = {Sen, Sagar and Marijan, Dusica and Gotlieb, Arnaud}
}
@article {dialesio2015combining,
	title = {Combining Genetic Algorithms and Constraint Programming to Support Stress Testing of Task Deadlines},
	journal = {ACM Transactions on Software Engineering and Methodology (TOSEM)},
	volume = {25},
	year = {2015},
	month = {12/2015},
	pages = {1-37},
	publisher = {ACM},
	abstract = {Tasks in Real Time Embedded Systems (RTES) are often subject to hard deadlines, that constrain how quickly the system must react to external inputs. These inputs and their timing vary in a large domain depending on the environment state, and can never be fully predicted prior to system execution. Therefore, approaches for stress testing must be developed to uncover possible deadline misses of tasks for different input arrival times. In this paper, we describe stress test case generation as a search problem over the space of task arrival times. Specifically, we search for worst case scenarios maximizing deadline misses where each scenario characterizes a test case. In order to scale our search to large industrial-size problems, we combine two state-of-the-art search strategies, namely Genetic Algorithms (GA) and Constraint Programming (CP). Our experimental results show that, in comparison with GA and CP in isolation, GA+CP achieves nearly the same effectiveness as CP and the same efficiency and solution diversity as GA, thus combining the advantages of the two strategies. In light of these results, we conclude that a combined GA+CP approach to stress testing is more likely to scale to large and complex systems.},
	doi = {10.1145/2818640},
	author = {Di Alesio, Stefano and Nejati, Shiva and Briand, Lionel and Gotlieb, Arnaud}
}
@inbook {23636,
	title = {Constraint-Based Testing: An Emerging Trend in Software Testing},
	booktitle = {Advances in Computers},
	volume = {99},
	year = {2015},
	month = {10/2015},
	pages = {67-101},
	publisher = {Elsevier},
	organization = {Elsevier},
	edition = {ADCOM, UK: Academic Press},
	chapter = {2},
	abstract = {Constraint-based testing is an emerging trend in software testing research and innovation. The general idea beneath this testing paradigm is to exploit a constraint solver to calculate the test inputs to be used for testing a software-system, so that the fault-revealing capabilities of the testing process can be improved. The incredible progresses achieved these last years on the development of powerful constraint solvers have fostered the emergence of effective and efficient constraint-based testing techniques. This chapter reviews in detail the main achievements reached in the field by illustrating them on simple examples and mentioning the main bibliographic references.},
	isbn = {978-0-12-802131-6},
	url = {http://elsev.spi-bpo.com/books/EComp},
	author = {Gotlieb, Arnaud},
	editor = {Memon, Atif}
}
@article {jss2014,
	title = {Cost-Effective Test Suite Minimization in Product Lines Using Search Techniques},
	journal = {Journal of Systems and Software},
	volume = {103},
	year = {2015},
	month = {05/2015},
	pages = {370{\textendash}391},
	publisher = {Elsevier},
	abstract = {Cost-effective testing of a product in a product line requires obtaining a set of relevant test cases from the entire test suite via test selection and minimization techniques. In this paper, we particularly focus on test minimization for product lines, which identifies and eliminates redundant test cases from test suites in order to reduce the total number of test cases to execute, thereby improving the efficiency of testing. However, such minimization may result in the minimized test suite with low test coverage, low fault revealing capability, low priority test cases, and require more time than the allowed testing budget (e.g., time) as compared to the original test suite. To deal with the above issues, we formulated the minimization problem as a search problem and defined a fitness function considering various optimization objectives based on the above issues. To assess the performance of our fitness function, we conducted an extensive empirical evaluation by investigating the fitness function with three weight-based Genetic Algorithms (GAs) and seven multi-objective search algorithms using an industrial case study and 500 artificial problems inspired from the industrial case study. The results show that Random-Weighted Genetic Algorithm (RWGA) significantly outperforms the other algorithms since RWGA can balance all the objectives together by dynamically updating weights during each generation. Based on the results of our empirical evaluation, we also implemented a tool called TEst Minimization using Search Algorithms (TEMSA) to support test minimization using various search algorithms in the context of product lines.},
	doi = {10.1016/j.jss.2014.08.024},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@misc {24014,
	title = {A CP approach of the variability testing of software product lines},
	howpublished = {Universit{\'e} Paris 1 Panth{\'e}on - La Sorbonne},
	year = {2015},
	type = {Invited talk},
	author = {Gotlieb, Arnaud}
}
@article {23735,
	title = {Focus section on quality software},
	journal = {Software, Practice and Experience},
	volume = {45},
	year = {2015},
	month = {07/2015},
	pages = {873-874},
	publisher = {John Wiley \& Sons},
	author = {Tse, T.H. and Gotlieb, Arnaud and chen, zhenyu}
}
@misc {24015,
	title = {Global Constraints in Software Testing Applications},
	howpublished = {Universit{\'e} de Montpellier, France - LIRMM},
	year = {2015},
	type = {Invited talk},
	author = {Gotlieb, Arnaud}
}
@article {23399,
	title = {Infeasible Path Generalization in Dynamic Symbolic Execution},
	journal = {Information and Software Technology},
	volume = {58},
	year = {2015},
	month = {02/2015},
	pages = {403{\textendash}418},
	publisher = {Elsevier},
	author = {Delahaye, Mickael and Botella, Bernard and Gotlieb, Arnaud}
}
@misc {24012,
	title = {An Overview of Constraint-Based Testing},
	howpublished = {Centre de Recherche en Informatique de Montreal (CRIM)},
	year = {2015},
	type = {Invited},
	author = {Gotlieb, Arnaud}
}
@inproceedings {23639,
	title = {Synthesis of attributed feature models from product descriptions},
	journal = {Proceedings of the 19th International Conference on Software Product Line, SPLC{\textquoteright}15, Nashville, USA, July 20-24.},
	year = {2015},
	month = {07/2015},
	pages = {1-10},
	isbn = {978-1-4503-3613-0},
	author = {B{\'e}can, Guillaume and Behjati, Razieh and Gotlieb, Arnaud and Acher, Mathieu}
}
@techreport {AFM_Synthesis,
	title = {Synthesis of Attributed Feature Models From Product Descriptions: Foundations},
	number = {RR-8680},
	year = {2015},
	publisher = {INRIA},
	abstract = {Feature modeling is a widely used formalism to characterize a set of products (also called configurations). As a manual elaboration is a long and arduous task, numerous techniques have been proposed to reverse engineer feature models from various kinds of artefacts. But none of them synthesize feature attributes (or constraints over attributes) despite the practical relevance of attributes for documenting the different values across a range of products. In this report, we develop an algorithm for synthesizing attributed feature models given a set of product descriptions. We present sound, complete, and parametrizable techniques for computing all possible hierarchies, feature groups, placements of feature attributes, domain values, and constraints. We perform a complexity analysis w.r.t. number of features, attributes, configurations, and domain size. We also evaluate the scalability of our synthesis procedure using randomized configuration matrices. This report is a first step that aims to describe the foundations for synthesizing attributed feature models.},
	url = {https://hal.inria.fr/hal-01116663},
	author = {B{\'e}can, Guillaume and Behjati, Razieh and Gotlieb, Arnaud and Acher, Mathieu}
}
@article {23398,
	title = {Testing Robot Controllers using Constraint Programming and Continuous Integration},
	journal = {Information and Software Technology},
	volume = {57},
	year = {2015},
	month = {01/2015},
	pages = {169-185},
	publisher = {Elsevier},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {23638,
	title = {Towards More Relational Feature Models},
	journal = {ICSOFT-EA 2015 - Proceedings of the 10th International Conference on Software Engineering and Applications, Colmar, Alsace, France, 20-22 July.},
	year = {2015},
	month = {07/2015},
	pages = {381-386},
	publisher = {SciTePress},
	abstract = {Feature modeling is of paramount importance to capture variabilities and commonalities within a software product line. Nevertheless, current feature modeling notations are limited, representing only propositional formulae over attributed variables. This position paper advocates the extension of feature modeling formalisms with richer computational domains and relational operations. In particular, it proposes to extend feature modeling with finite and continuous domain variables, with first-order logic quantifiers, and with N-ary relations between features attributes, and with so-called global constraints. In order to extend the expressiveness while preserving automated analysis facilities, feature models could be semantically interpreted as first-order logic formulae (instead of propositional logic formulae), including global and continuous dependency between features. In simpler words, this paper emphasizes the importance of having more relational feature models and presents next generation applications.},
	isbn = {978-989-758-114-4},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Sen, Sagar}
}
@misc {Simula.simula.2854,
	title = {Challenges in Constraint-Based Testing},
	howpublished = {Selected talk at {\textquoteright}Verification meets CP{\textquoteright} workshop 2014, Lyon, France},
	year = {2014},
	publisher = {.},
	address = {.},
	abstract = {Constraint-Based Testing emerged ten years ago as an important software testing topic. Many developments in symbolic execution, a technique used to symbolically explore the execution paths of a program under test, revealed the pressing needs for having at hand powerful constraint solving methods.},
	keywords = {Workshop},
	author = {Gotlieb, Arnaud and Ganesh, Vijay}
}
@article {Simula.simula.2180,
	title = {A Combined Approach for Constraints Over Finite Domains and Arrays},
	journal = {journal},
	year = {2014},
	publisher = {Springer Berlin Heidelberg},
	author = {Bardin, Sebastien and Gotlieb, Arnaud and Flener, Pierre}
}
@inproceedings {Simula.simula.2710,
	title = {FLOWER: Optimal Test Suite Reduction As a Network Maximum Flow},
	journal = {Proceedings of Int. Symp. on Soft. Testing and Analysis (ISSTA{\textquoteright}14), San Jos{\'e}, CA, USA, Jul. 2014},
	year = {2014},
	month = {07/2014},
	pages = {171-180},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	keywords = {Conference},
	isbn = {978-1-4503-2645-2},
	doi = {10.1145/2610384.2610416},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@inproceedings {splc2014,
	title = {Multi-Objective Test Prioritization in Software Product Line Testing: an Industrial Case Study},
	journal = {Software Product Line Conference (SPLC14), Florence, Italy},
	year = {2014},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	abstract = {Test prioritization is crucial for testing products in a product line considering limited budget in terms of available time and resources. In general, it is not practically feasible to execute all the possible test cases and so, ordering test case execution permits test engineers to discover faults earlier in the testing process. An efficient prioritization of test cases for one or more products requires a clear consideration of the tradeoff among various costs (e.g., time, required resources) and effectiveness (e.g., feature coverage) objectives. As an integral part of the future Cisco{\textquoteright}s test scheduling system for validating video conferencing products, we introduce a search-based multi-objective test prioritization technique, considering multiple cost and effectiveness measures. In particular, our multi-objective optimization setup includes the minimization of execution cost (e.g., time), and the maximization of number of prioritized test cases, feature pairwise coverage and fault detection capability. Based on cost-effectiveness measures, a novel fitness function is defined for such test prioritization problem. The fitness function is empirically evaluated together with three commonly used search algorithms (e.g., (1+1) Evolutionary algorithm (EA)) and Random Search as a comparison baseline based on the Cisco{\textquoteright}s industrial case study and 500 artificial designed problems. The results show that (1+1) EA achieves the best performance for solving the test prioritization problem and it scales up to solve the problems of varying complexity.},
	keywords = {Conference},
	author = {Wang, Shuai and Buchmann, David and Ali, Shaukat and Gotlieb, Arnaud and Pradhan, Dipesh and Liaaen, Marius}
}
@book {Simula.simula.3052,
	title = {Proceedings of the Software Engineering in Practice (SEIP) Track of ICSE{\textquoteright}14, June 1-7, Hyderabad, India},
	year = {2014},
	publisher = {IEEE},
	author = {Gotlieb, Arnaud and Shroff, Gautam}
}
@inproceedings {ssbse2014,
	title = {Random-Weighted Search-Based Multi-Objective Optimization Revisited},
	journal = {International Symposium on Search-Based Software Engineering (SSBSE{\textquoteright}14), Fortaleza, Brazil},
	year = {2014},
	month = {August},
	publisher = {Springer International Publishing},
	type = {Conference},
	address = {Switzerland },
	abstract = {Weight-based multi-objective optimization requires assigning appro- priate weights using a weight strategy to each of the objectives such that an overall optimal solution can be obtained with a search algorithm. Choosing weights using an appropriate weight strategy has a huge impact on the obtained solutions and thus warrants the need to seek the best weight strategy. In this pa- per, we propose a new weight strategy called Uniformly Distributed Weights (UDW), which generates weights from uniform distribution, while satisfying a set of user-defined constraints among various cost and effectiveness measures. We compare UDW with two commonly used weight strategies, i.e., Fixed Weights (FW) and Randomly-Assigned Weights (RAW), based on five cost/effectiveness measures for an industrial problem of test minimization de- fined in the context of Video Conferencing System Product Line developed by Cisco Systems. We empirically evaluate the performance of UDW, FW, and RAW in conjunction with four search algorithms ((1+1) Evolutionary Algorithm (EA), Genetic Algorithm, Alternating Variable Method, and Random Search) using the industrial case study and 500 artificial problems of varying complexi- ty. Results show that UDW along with (1+1) EA achieves the best performance among the other combinations of weight strategies and algorithms.},
	keywords = {Conference},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@misc {24022,
	title = {Symbolic Path-Oriented Test Data Generation for Floating-Point Programs},
	howpublished = {Dagsthul Seminar (Symbolic Methods and Constraint Solving) },
	year = {2014},
	type = {Contrinuted talk},
	author = {Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.2852,
	title = {Testing Robotized Paint System Using Constraint Programming: an Industrial Case Study},
	journal = {Proceedings of IFIP Int. Conf. on Testing Software and Systems (ICTSS{\textquoteright}14), Madrid, Spain, Sep. 2014 - Application Track},
	year = {2014},
	publisher = {LNCS},
	type = {1},
	address = {Berlin, Heidelberg},
	keywords = {Conference},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@misc {Simula.simula.2856,
	title = {Towards an Effective Formally Certified Constraint Solver},
	howpublished = {Selected talk at the {\textquoteright}Verification meets CP{\textquoteright} 2014 workshop, Lyon, France},
	year = {2014},
	publisher = {.},
	address = {.},
	abstract = {When used to verify safety-critical properties of a software system, the answers provided by a CP-based constraint solver must be formally certied through trusted and faithful methodologies. In such cases, one has often to formally verify that a constraint system is unsatisable, as it is required to prove that no reachable states can violate the property to be ensured. Unfortunately, proving that a constraint system is unsatisable is much more complex for a CP-solver than checking a possible solution.},
	keywords = {Workshop},
	author = {Dubois, Catherine and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.2850,
	title = {Using CP in Automatic Test Generation for ABB Robotics{\textquoteright} Paint Control System},
	journal = {Proceedings of Principles of Constraint Programming (CP{\textquoteright}14), Lyon, France, Sep. 2014 - Best Application Track Paper},
	year = {2014},
	publisher = {LNCS},
	type = {Conference},
	address = {Zwitzerland},
	keywords = {Conference},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein}
}
@inproceedings {dialesio2014worstcase,
	title = {Worst-case Scheduling of Software Tasks {\textendash} A Constraint Optimization Model to Support Performance Testing},
	journal = {The 20th International Conference on Principles and Practice of Constraint Programming (CP 2014)},
	year = {2014},
	type = {Conference},
	abstract = {Real-Time Embedded Systems (RTES) in safety-critical domains, such as maritime and energy, must satisfy strict performance requirements to be deemed safe. Therefore, such systems have to be thoroughly tested to ensure their correct behavior even under the worst operating conditions. In this paper, we address the need of deriving worst case scenarios with respect to three common performance requirements, namely task deadlines, response time, and CPU usage. Specifically, we investigate whether this worst-case analysis can be effectively re-expressed as a Constrained Optimization Problem (COP) over the space of possible inputs to the system. Solving this problem means finding the sets of inputs that maximize the chance to violate performance requirements at runtime. Such inputs can in turn be used to test if the target RTES meets the expected performance even in the worst case. We develop an OPL model for IBM ILOG CP Optimizer that implements a task priority-based preemptive scheduling, and apply it to a case study from the maritime and energy domain. Our validation shows that (1) the input to our model can be provided with reasonable effort in an industrial setting, and (2) the COP effectively identifies test cases that maximize deadline misses, response time, and CPU usage.},
	doi = {10.1007/978-3-319-10428-7_58},
	author = {Di Alesio, Stefano and Nejati, Shiva and Briand, Lionel Claude and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.2074,
	title = {Automated Product Line Methodologies to Support Model-Based Testing},
	journal = {The ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS), online publication at CEUR},
	year = {2013},
	publisher = {IEEE},
	type = {Annual},
	address = {.},
	abstract = {Testing products in a cost-efficient way remains an attractive topic for Model-Based Testing (MBT) of product lines in both academia and industry, which can be addressed by employing systematic and automated approaches based on models (such as feature models and UML models). Cost-effective testing products can be divided into three main problems, i.e., test selection, test generation, and test minimization. Driven by the needs of our industrial problems for testing Video Conferencing Systems (VCSs) product line developed by Cisco, Norway, this paper presents Product Line Model-based Testing Methodologies (PL-MTM) to tackle the above-mentioned three problems for cost- effective testing a product in product line, which includes: 1) an systematic and automated test selection methodology; 2) an automated test minimization approach; and 3) an automated and systematic test generation methodology},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud},
	editor = {Tse, T.H.}
}
@techreport {27065,
	title = {Automated Search-Based Test Suite Minimization in Product Lines: An Empirical Study},
	year = {2013},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Gotlieb, Arnaud and Wang, Shuai}
}
@inproceedings {Simula.simula.2018,
	title = {Automated Test Case Selection Using Feature Model: an Industrial Case Study},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS), Best Paper Award},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM\_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM\_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly. Moreover, we conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our method- ology and models (FM\_T and CFM\_T) in their current practice.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius},
	editor = {Grey, Jeff}
}
@inproceedings {QUAMES2013,
	title = {Dealing With Software Model Quality in Practice: Experience in a Research Project},
	journal = {13th International Conference on Quality Software (QSIC 2013)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Najing},
	abstract = {Although past research has resulted in different means to deal with software model quality, creation of adequate software models remains challenging. Any modelling effort must be carefully analysed and planned before it starts, and definition or adoption of modelling guidelines is usually necessary. In addition, the amount of publications addressing model quality in practice is low, and the knowledge about others{\textquoteright} experience regarding model quality is limited. This paper reports on our experience in dealing with software model quality in the context of a project between industry and academia. Such a project corresponds to a large-scale research project in which modelling has been used both as part of the necessary work for executing the project and for creating project results. We present how we have dealt with model quality in requirements modelling and in conceptual model specification, as well as a set of lessons learned. The insights provided can help both researchers and practitioners when having to deal with software model quality.},
	author = {Jose Luis {de la Vara} and Espinoza, Huascar},
	editor = {Gotlieb, Arnaud}
}
@article {Simula.simula.2276,
	title = {FocalTest: a Constraint Programming Approach for Property-Based Testing},
	journal = {Communications in Computer and Information Science},
	number = {170},
	year = {2013},
	note = {Revised selected papers from ICSOFT 2010 - Edited by Cordeiro Virvou Shishkov (Eds), SPRINGER},
	pages = {140-155},
	publisher = {Springer Berlin Heidelberg},
	author = {Carlier, Matthieu and Dubois, Catherine and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.1960,
	title = {Minimizing Test Suites in Software Product Lines Using Weight-Based Genetic Algorithms},
	journal = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2013},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Test minimization techniques aim at identifying and eliminating redundant test cases from test suites in order to reduce the total number of test cases to execute, thereby improving the efficiency of testing. In the context of software product line, we can save efforts and cost in the selection and minimization of test cases for testing a specific product by modeling the product line. However, minimizing the test suite for a product requires addressing two potential issues: 1) the reduced test suite may not cover all test requirements compared with the original suite; 2) the reduced test suite may have less fault revealing capability than the original suite. In this paper, we apply weight-based Genetic Algorithms (GAs) to minimize the test suite for testing a product, while preserving fault detection capability and testing coverage of the original test suite. The challenge behind concerns the definition of an appropriate fitness function, which is able to preserve the coverage of complex testing criteria (e.g., Combinatorial Interaction Testing criterion). Based on the defined fitness function, we have empirically evaluated three different weight- based GAs on an industrial case study provided by Cisco Systems, Inc. Norway. We also presented our results of applying the three weight-based GAs on five existing case studies from the literature. Based on these case studies, we conclude that among the three weight-based GAs, Random-Weighted GA (RWGA) achieved significantly better performance than the other ones.},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud},
	editor = {Harman, Mark}
}
@inproceedings {MODRE2013,
	title = {Modelling Data Interaction Requirements: a Position Paper},
	journal = {2013 International Workshop on Model-Driven Requirements Engineering (MoDRE 2013)},
	year = {2013},
	publisher = {IEEE},
	type = {Workshop},
	address = {Rio de Janeiro },
	abstract = {Data-intensive information systems constitute the backbone of e-commerce and e-governance services running worldwide. Structured data is a central artefact in these information systems. Requirements for structure in data are typically modelled in a database schema. However, information system behaviour is often a function of interactions that cross-cut database features such as field values in different tables. For instance, consultants at the Norwegian Customs and Excise reveal that taxation rules are triggered due to data interactions between 10,000 items, 88 country groups, and 934 tax codes. There are about 12.9 trillion possible three-wise interactions of which only about 220,000 interactions are used in reality as customs rules. Therefore, we ask, how can we model data interaction requirements to further bound the input domain of an information system? In this position paper, we address this question by modelling data interaction requirements using classification tree models. We also present different applications of data interaction requirements in the development of information systems.},
	author = {Sen, Sagar and Jose Luis {de la Vara} and Sarkar, Arnab and Gotlieb, Arnaud},
	editor = {Moreira, A. and Mussbacher, G. and Araujo, J. and Bencomo, N. and Sanchez, P.}
}
@inproceedings {Simula.simula.2172,
	title = {Practical Pairwise Testing for Software Product Lines},
	journal = {Proceedings of the International Software Product Line Conference (SPLC)},
	year = {2013},
	pages = {227-235},
	publisher = {ACM},
	type = {1},
	address = {New York, NY, USA},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Hervieu, Aymeric and Sen, Sagar}
}
@book {Simula.simula.2642,
	title = {Proceedings of 13th International Conference on Quality Software 2013 (QSIC 2013), Nanjing, China, Jul. 2013},
	year = {2013},
	month = {July},
	publisher = {Conference Publishing Services},
	address = {Nanjing, China},
	keywords = {Conference},
	author = {Gotlieb, Arnaud and chen, zhenyu}
}
@inproceedings {Simula.simula.1750,
	title = {Scenario Realizability With Constraint Optimization},
	journal = {Proc. of. Fundamental Aspects of Software Engineering (FASE{\textquoteright}2013)},
	year = {2013},
	month = {March},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin, Heidelberg},
	keywords = {Conference},
	author = {Abdallah, Rouwaida and Gotlieb, Arnaud and Hellouet, Loic and Jard, Claude}
}
@inproceedings {27057,
	title = {Scenario realizability with constraint optimization, In: Proc. Of Fundamental Aspects of Software Engineering },
	journal = {Proc. Of Fundamental Aspects of Software Engineering (FASE 2013), },
	year = {2013},
	publisher = { Springer},
	author = {Gotlieb, Arnaud and Hellouet, Loic and Jard, Claude},
	editor = {Abdallah, Rouwaida}
}
@inproceedings {19550,
	title = {Stress Testing of Task Deadlines: A Constraint Programming Approach},
	journal = {The 24th IEEE International Symposium on Software Reliability Engineering (ISSRE 2013)},
	year = {2013},
	type = {Conference},
	abstract = {Safety-critical Real Time Embedded Systems (RTESs) are usually subject to strict timing and performance requirements that must be satisfied for the system to be deemed safe. In this paper, we use effective search strategies whose goal is finding worst case scenarios with respect to deadline misses. Such scenarios can in turn be used to test the target RTES and ensure that it satisfies its timing requirements even under worst case conditions. Specifically, we develop an approach based on Constraint Programming (CP) to automate the generation of test cases that reveal, or are likely to, task deadline misses. We evaluate it through a comparison with a state-of-the-art approach based on Genetic Algorithms (GA). In particular, we compare CP and GA in five case studies for efficiency, effectiveness, and scalability. Our experimental results show that, on the largest and more complex case studies, CP performs significantly better than GA. Furthermore, CP offers some advantages over GA, such as it guarantees a complete search when there is sufficient time, and, being deterministic, it doesn\&$\#$39;t rely on parameters that potentially have a significant effect on the search and therefore need to be tuned. Hence, we conclude that our results are encouraging and suggest this is an advantageous approach for stress testing of RTESs with respect to timing constraints.},
	doi = {10.1109/ISSRE.2013.6698915},
	author = {Di Alesio, Stefano and Nejati, Shiva and Briand, Lionel and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.1748,
	title = {Symbolic Path-Oriented Test Data Generation for Floating-Point Programs},
	journal = {Proc. of Int. Conf. on Soft. Testing, Validation and Verification (ICST{\textquoteright}2013) - 10 pages},
	year = {2013},
	month = {March},
	publisher = {IEEE},
	type = {Conference},
	address = {Luxembourg},
	author = {Bagnara, Roberto and Carlier, Matthieu and Gori, Roberta and Gotlieb, Arnaud},
	editor = {IEEE, }
}
@inproceedings {Simula.simula.2178,
	title = {Test Case Prioritization for Continuous Regression Testing: an Industrial Case Study},
	journal = {Proceedings of the International Conference on Software Maintenance (ICSM)},
	year = {2013},
	publisher = {IEEE},
	type = {1},
	address = {Eindhoven},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Sen, Sagar},
	editor = {IEEE, }
}
@inproceedings {Simula.simula.2622,
	title = {Test Generation for Robotized Paint Systems Using Constraint Programming in a Continuous Integration Environment},
	journal = {IEEE ICST 2013  (2 pages)},
	year = {2013},
	month = {April},
	pages = {489-290},
	publisher = {IEEE},
	type = {Conference},
	address = {Luxembourg},
	author = {Mossige, Morten and Gotlieb, Arnaud and Meling, Hein},
	editor = {IEEE, }
}
@techreport {Simula.simula.2280,
	title = {Test Selection Based on Data Interactions in Data-Intensive Systems},
	number = {2013-03},
	year = {2013},
	month = {10/2013},
	publisher = {Simula Research Laboratory},
	abstract = {Testing data-intensive systems is paramount to increase our reliance on information in e-governance, scientific/ medical research, and social networks. Common practice to test these systems is by using a live production database. This testing approach is space and time inefficient and lacks clarity about what test cases or scenarios are covered. In this paper, we leverage classification tree modelling to specify desired test cases as data interactions between a set of fields across multiple tables of an existing database. Our methodology and tool, DEPICT, uses test case specifications in classification tree models to (a) automatically derive a spanning tree representing a relationship between any set of fields for any given database schema (b) generates queries to create an efficient inner join between related tables in the spanning tree (c) extract records from various tables that satisfy data interactions in the classification tree model (d) discovers holes or unsatisfied test cases in the test databases. We perform experiments to show that our approach is fast and scalable to extract test databases. Our experiments are based on selecting test databases from 8000 declarations for 60,000 items from the Norwegian Customs and Excise information system TVINN.},
	keywords = {Conference},
	author = {Sen, Sagar and Ieva, Carlo and Marijan, Dusica and Sarkar, Arnab and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.1880,
	title = {Testing a Data-Intensive System With Generated Data Interactions: the Norwegian Customs and Excise Case Study},
	journal = {Proceedings of 25th International Conference on Advanced Information Systems Engineering (CAiSE{\textquoteright}13),June 17-21 2013, Valencia, Spain},
	year = {2013},
	month = {June},
	publisher = {Springer},
	type = {Conference},
	author = {Sen, Sagar and Gotlieb, Arnaud}
}
@techreport {25300,
	title = {Automated Selection of Test Cases using Feature Model for Product Lines: An Industrial Case Study},
	number = {2012-20},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version\&nbsp;can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@techreport {25305,
	title = {Automated Test Case Selection using Feature Model: an Industrial Case Study},
	number = {2012-28},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@techreport {25312,
	title = {Automatic Derivation of Test Execution Plans From a Video Conferencing System Product Line Modelling},
	number = {2012-02},
	year = {2012},
	note = {This technical report is an early version of a published poster.https://www.simula.no/publications/automatic-derivation-test-execution-plans-video-conferencing-system-product-line},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Lionel C. {Briand}}
}
@misc {Simula.simula.2038,
	title = {Automatic Derivation of Test Execution Plans From a Video Conferencing System Product Line Modelling},
	year = {2012},
	address = {.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Briand, Lionel Claude}
}
@inproceedings {23895,
	title = {Automatic Selection of Test Execution Plans from a Video Conferencing System Product Line},
	journal = {Proceedings of the VARiability for You Workshop: Variability Modeling Made Useful for Everyone},
	year = {2012},
	pages = {32 - 37},
	publisher = {ACM},
	isbn = {978-1-4503-1809-9},
	doi = {10.1145/2425415.2425422},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Briand, Lionel}
}
@inproceedings {Simula.simula.1322,
	title = {A Certified Constraint Solver Over Finite Domains},
	journal = {Proceedings of Formal Methods (FM{\textquoteright}12), Paris, Aug. 2012},
	volume = {7436},
	year = {2012},
	month = {August},
	pages = {116-131},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Constraint programs such as those written in modern Constraint Programming languages and platforms aim at solving problems coming from optimization, scheduling, planning, etc. Recently CP programs have been used in business-critical or safety-critical areas as well, e.g., e-Commerce, air-traffic control applications, or software verification. This implies a more skeptical regard on the implementation of constraint solvers, especially when the result is that a constraint problem has no solution, i.e., unsatisfiability. For example, in software model checking, using an unsafe constraint solver may result in a dramatic wrong answer saying that a safety property is satisfied while there exist counterexamples. In this paper, we present a Coq formalization of a constraint filtering algorithm - AC3 and one of its variant AC2001 - and a simple labeling procedure. The proof of their soundness and completeness has been completed using Coq. As a result, a formally verified constraint solver written in OCaml has been automatically extracted from the Coq specification of the filtering and labeling algorithms. The solver, yet not as efficient as specialized existing (unsafe) implementations, can be used to formally certify that a constraint system is unsatisfiable.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-32759-9\_12},
	author = {Dubois, Catherine and Gotlieb, Arnaud},
	editor = {Dominique M{\'e}ry Dimi {Giannakopoulou}}
}
@misc {Simula.simula.2278,
	title = {Constraint-Based Reacheability},
	howpublished = {INFINITY Workshop, co-located with FM 2012, Paris},
	year = {2012},
	note = {INFINITY Workshop, co-located with FM 2012, Paris},
	month = {August},
	publisher = {.},
	address = {.},
	author = {Gotlieb, Arnaud}
}
@article {Simula.simula.1072,
	title = {A CP Framework for Testing CP},
	journal = {Constraints},
	volume = {17},
	number = {2},
	year = {2012},
	month = {February},
	pages = {123-147},
	publisher = {Springer US},
	abstract = {The success of several constraint-based modeling languages such as OPL, ZINC or COMET, appeals for better software engineering practices, particularly in the testing phase. This paper introduces a testing framework enabling automated test case generation for constraint programming. We propose a general framework of constraint program development which supposes that a first declarative and simple constraint model is available from the problem specifications analysis. Then, this model is refined using classical techniques such as constraint reformulation, surrogate, redundant, implied, global constraint and symmetry-breaking to form an improved constraint model that must be thoroughly tested before being used to address real-sized problems. We think that most of the faults are introduced in this refinement step and propose a process which takes the first declarative model as an oracle for detecting non-conformities and derive practical test purposes from this process. We implemented this approach in a new tool called {\sf CPTEST} that was used to automatically detect non-conformities on classical benchmark programs, such as the Golomb rulers, n-queens, social golfer and the car-sequencing problems.},
	author = {Lazaar, Nadjib and Gotlieb, Arnaud and Lebbah, Yahia}
}
@inproceedings {Simula.simula.1220,
	title = {fdcc: a Combined Approach for Solving Constraints Over Finite Domains and Arrays},
	journal = {Proceedings of CPAIOR 2012 (Constraint Programming-Artificial Intelligence-Operations Research), Nantes, France},
	year = {2012},
	pages = {17-33},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Arrays are ubiquitous in the context of software verification. However, effective reasoning over arrays is still rare in CP, as local reasoning is dramatically ill-conditioned for constraints over arrays. In this paper, we propose an approach combining both global symbolic reasoning and local filtering in order to solve constraint systems involving arrays (with accesses, updates and size constraints) and finite-domain constraints over their elements and indexes. Our approach, named FDCC, is based on a combination of a congruence closure algorithm for the standard theory of arrays and a CP solver over finite domains. The tricky part of the work lies in the bi-directional communication mechanism between both solvers. We identify the significant information to share, and design ways to master the communication overhead. Experiments on random instances show that FDCC solves more formulas than any portfolio combination of the two solvers taken in isolation, while overhead is kept reasonable.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-29828-8\_2},
	author = {Bardin, Sebastien and Gotlieb, Arnaud},
	editor = {Beldiceanu, Nicolas and Jussien, Narendra and Pinson, {\'E}ric}
}
@inproceedings {Simula.simula.1222,
	title = {A First Step in the Design of a Formally Verified Constraint-Based Testing Tool: FocalTest},
	journal = {Proceedings of the Tests And Proofs Conference (TAP 2012), Prague, Czech Republic},
	volume = {7305},
	year = {2012},
	pages = {35-50},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Constraint-based test data generators rely on SMT or constraint solvers to automatically generate test data (e.g., Pex, Sage, Gatel, Path\-Crawler, Euclide, etc.). However, for some test data generation requests corresponding to particular testing objectives, these tools may fail to deliver the expected test data because they rather focus on efficiency rather than correction and completeness. In our work, we adopt an opposite view in the development of FocalTest, a test data generation tool for Focalize programs. The goal of the tool is to generate an MC/DC-compliant set of test data over the precondition of user-defined program properties. The development of such a correct-by-construction test data generator requires 1) to provide a formally verified translation of Focalize programs and properties into constraint systems; 2) to introduce a formally verified constraint solver able to solve those constraint systems. This paper is concerned with the first step only where we formally demonstrate with Coq the correctness of the translation of an intermediate functional language into a constraint system. This objective requires to formally define the operational semantics of the source language that features the manipulation of concrete data types via pattern-matching and function calls, constructions that are mirrored in the constraint language. Although such a semantics-oriented formalization is only a first step of a larger goal which is to provide a formally verified constraint-based testing tool, we argue that it is an important contribution to the building of more robust software testing tools.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-30473-6\_5},
	author = {Carlier, Matthieu and Dubois, Catherine and Gotlieb, Arnaud},
	editor = {Jacques Julliand A {Brucker}}
}
@techreport {Simula.simula.1168,
	title = {Guided Interactive Configuration of Embedded Software Systems Using Constraint Satisfaction Over Finite Domains},
	number = {2012-05},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Modern society is increasingly dependent on highly-configurable software systems, in particular, architecturally configurable software such as that embedded in Integrated Control Systems (ICSs). Configuring ICSs is expensive, time-consuming and error-prone. This is due, in large part, to the fact that the hardware and software configuration processes are, typically, rather isolated from one another, resulting in many configuration errors to be detected only after the integration of software and hardware. In this work, we propose a model-based configuration approach that allows us to configure software in a stepwise manner, to automate some configuration decisions, and to iteratively validate software and hardware configuration decisions. Our approach has two major steps. In the first step (modeling), a generic model describing an ICS family is built. In the second step (configuration), we interactively guide a user to derive a particular product specification complying with the generic model of its ICS family. We use a constraint solver, SICStus Prolog, to evaluate user decisions at each round, to automatically infer configuration decisions and to ensure that software and hardware configurations are consistent. We evaluated our approach by applying it to a real subsea production system. Specifically, we rebuilt a number of existing verified product configurations of our industry partner. Our experience shows that our approach successfully enforces consistency of configurations, can automatically infer up to 50\% of the configuration decisions, and reduces the complexity of making configuration decisions by guiding the user during the configuration process.},
	keywords = {Workshop},
	author = {Behjati, Razieh and Nejati, Shiva and Gotlieb, Arnaud and Yue, Tao and Briand, Lionel}
}
@inproceedings {Simula.simula.1558,
	title = {Managing Execution Environment Variability During Software Testing: an Industrial Experience},
	journal = {Proceedings of the 24th IFIP Int. Conference on Testing Software and Systems (ICTSS{\textquoteright}12)},
	year = {2012},
	month = {November},
	publisher = {LNCS},
	type = {Conference},
	address = {.},
	abstract = {Nowadays, telecom software applications are expected to run on a tremendous variety of execution environments. For example, network connection software must deliver the same functionalities on distinct physical platforms, which themselves run several distinct operating systems, with various applications and physical devices. Testing those applications is challenging as it is simply impossible to consider every possible environment configuration. This paper reports on an industrial case study called BIEW (Business and Internet EveryWhere) where the combinatorial explosion of environment configurations has been tackled with a dedicated and original methodology devised by KEREVAL, a french SME focusing on software testing services. The proposed solution samples a subset of configurations to be tested, based on environment modelling, requirement analysis and systematic traceability. From the experience on this case study, we outline the challenges to develop means to select relevant environment configurations from variability modelling and requirement analysis in the testing processes of telecom software.},
	author = {Hervieu, Aymeric and Gotlieb, Arnaud},
	editor = {Verlag, Springer}
}
@techreport {25303,
	title = {Minimizing Test Suites in Software Product Lines Using Weight-Based Genetic Algorithms},
	number = {2012-25},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published paper can be found here:https://www.simula.no/publications/minimizing-test-suites-software-product-lines-using-weight-based-genetic-algorithms},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@inproceedings {23894,
	title = {Minimum Pairwise Coverage Using Constraint Programming Techniques},
	journal = {2012 IEEE Fifth International Conference},
	year = {2012},
	month = {04/2012},
	pages = {773 - 774},
	publisher = {IEEE},
	isbn = {978-1-4577-1906-6},
	doi = {10.1109/ICST.2012.174},
	author = {Gotlieb, Arnaud and Hervieu, Aymeric and Baudry, B.}
}
@inproceedings {Simula.simula.1280,
	title = {Model-Based Automated and Guided Configuration of Embedded Software Systems},
	journal = {ECMFA 2012},
	volume = {7349},
	year = {2012},
	pages = {226-243},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Configuring Integrated Control Systems (ICSs) is largely manual, time-consuming and error-prone. In this paper, we propose a model-based configuration approach that interactively guides engineers to configure software embedded in ICSs. Our approach verifies engineers{\textquoteright} decisions at each configuration iteration, and further, automates some of the decisions. We use a constraint solver, SICStus Prolog, to automatically infer configuration decisions and to ensure the consistency of configuration data. We evaluated our approach by applying it to a real subsea oil production system. Specifically, we rebuilt a number of existing verified product configurations of our industry partner. Our experience shows that our approach successfully enforces consistency of configurations, can automatically infer up to 50\% of the configuration decisions, and reduces the complexity of making configuration decisions.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-31491-9\_18},
	author = {Behjati, Razieh and Nejati, Shiva and Yue, Tao and Gotlieb, Arnaud and Briand, Lionel Claude},
	editor = {Vallecillo, Antonio and Tolvanen, Juha-Pekka and Kindler, Ekkart and St{\"o}rrle, Harald and Kolovos, Dimitris}
}
@inproceedings {Simula.simula.1228,
	title = {MuTIL: Mutation-Based Statistical Test Inputs Generation for Automatic Fault Localization},
	journal = {Proceedings of the Sixth International Conference on Software Security and Reliability (SERE-12), Washington, USA},
	year = {2012},
	month = {06/2012},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {Automatic Fault Localization (AFL) is a process to locate faults automatically in software programs. Essentially, an AFL method takes as input a set of test cases including failed test cases, and ranks the statements of a program from the most likely to the least likely to contain a fault. As a result, the efficiency of an AFL method depends on the "quality" of the test cases used to rank statements. More specifically, in order to improve the accuracy of their ranking within test budget constraints, we have to ensure that program statements are executed by a reasonably large number of test cases which provide a coverage as uniform as possible of the input domain. This paper proposes μTIL, a new statistical test inputs generation method dedicated to AFL, based on constraint solving and mutation testing. Using mutants where the locations of injected faults are known, μTIL is able to significantly reduce the length of an AFL test suite while retaining its accuracy (i.e., the code size to examine before spotting the fault). In order to address the motivations stated above, the statistical generator objectives are two-fold: 1) each feasible path of the program is activated with the same probability; 2) the subdomain associated to each feasible path is uniformly covered. Using several widely used ranking techniques (i.e., Tarantula, Jaccard, Ochiai), we show on a small but realistic program that a proof-of-concept implementation of μTIL can generate test sets with significantly better fault localization accuracy than both random testing and adaptive random testing. We also show on the same program that using mutation testing enables a 75\% length reduction of the AFL test suite without decrease in accuracy.},
	keywords = {Conference},
	author = {Delahaye, Mickael and Briand, Lionel Claude and Gotlieb, Arnaud and Petit, Matthieu}
}
@techreport {27056,
	title = {Research report on test configuration generation},
	year = {2012},
	publisher = {Simula Research Laboratory},
	author = {Marijan, Dusica and Gotlieb, Arnaud}
}
@article {Simula.simula.1434,
	title = {Tcas Software Verification Using Constraint Programming},
	journal = {The Knowledge Engineering Review},
	volume = {27},
	number = {3},
	year = {2012},
	month = {September},
	publisher = {Cambridge University Press New York},
	author = {Gotlieb, Arnaud}
}
@inproceedings {14931,
	title = {Testing Deadline Misses for Real-Time Systems Using Constraint Optimization Techniques},
	journal = {The 4th Workshop on Constraints in Software Testing, Verification, and Analysis (CSTVA 2012)},
	year = {2012},
	type = {Workshop},
	abstract = {Safety-critical real-time applications are typically subject to stringent timing constraints which are dictated by the surrounding physical environments. Specifically, tasks in these applications need to finish their execution before given deadlines, otherwise the system is deemed unsafe. It is therefore important to test real-time systems for deadline misses. In this paper, we present a strategy for testing real-time applications that aims at finding test scenarios in which deadline misses become more likely. We identify such test scenarios by searching the possible ways that a set of real-time tasks can be executed according to the scheduling policy of the operating system on which they are running. We formulate this search problem using a constraint optimization model that includes (1) a set of constraints capturing how a given set of tasks with real-time constraints are executed according to a particular scheduling policy, and (2) a cost function that estimates how likely the given tasks are to miss their deadlines. We implement our constraint optimization model in ILOG, apply our model to several examples, and report on the performance results.},
	doi = {10.1109/ICST.2012.171},
	author = {Di Alesio, Stefano and Gotlieb, Arnaud and Nejati, Shiva and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.880,
	title = {Filtering by ULP Maximum},
	journal = {Proceedings of the 23rd IEEE International Conference on Tools with Artificial Intelligence (ICTAI{\textquoteright}2011), Nov. 7-9, 2011, Boca Raton, Florida, USA},
	year = {2011},
	month = {11/2011},
	publisher = {IEEE},
	type = {Annual},
	abstract = {Constraint solving over floating-point numbers is an emerging topic that found interesting applications in software analysis and testing. Even for IEEE-754 compliant programs, correct reasoning over floating-point computations is challenging and requires dedicated constraint solving approaches to be developed. Recent advances indicate that numerical properties of floating-point numbers can be used to efficiently prune the search space. In this paper, we reformulate the Marre and Michel property over floating-point addition/subtraction constraint to ease its implementation in real-world floating-point constraint solvers. We also generalize the property to the case of multiplication/division in order to benefit from its improvements in more cases.},
	author = {Carlier, Matthieu and Gotlieb, Arnaud}
}
@inproceedings {27052,
	title = {Filtering by ULP maximum, In: Proceedings of the 23rd IEEE International Conference on Tools with Artificial Intelligence (ICTAI{\textquoteright}2011), },
	journal = {23rd IEEE International Conference on Tools with Artificial Intelligence (ICTAI{\textquoteright}2011)},
	year = {2011},
	publisher = { IEEE},
	author = {Gotlieb, Arnaud},
	editor = {Carlier, Matthieu}
}
@inproceedings {Simula.simula.876,
	title = {PACOGEN: Automatic Generation of Pairwise Test Configurations From Feature Models},
	journal = {Proceedings of the IEEE 22nd International Symposium on Software Reliability Engineering (ISSRE 2011)},
	year = {2011},
	month = {November},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Hiroshima},
	abstract = {Feature models are commonly used to specify variability in software product lines. Several tools support feature models for variability management at different steps in the development process. However, tool support for test configuration generation is currently limited. This test generation task consists in systematically selecting a set of configurations that represent a relevant sample of the variability space and that can be used to test the product line. In this paper we propose PACOGEN to analyze feature models and automatically generate a set of configurations that cover all pairwise interactions between features. PACOGEN relies on constraint programming to generate configurations that satisfy all constraints imposed by the feature model and to minimize the set of the tests configurations. This work also proposes an extensive experiment, based on the state-of-the art SPLOT feature models repository, showing that PACOGEN scales over variability spaces with millions of configurations and covers pairwise with less configurations than other available tools.},
	isbn = {978-0-7695-4568-4},
	author = {Hervieu, Aymeric and Gotlieb, Arnaud},
	editor = {Cukic, Bojan}
}
