@inproceedings {28131,
	title = {DeepOrder: Deep Learning for Test Case Prioritization in Continuous Integration Testing},
	journal = {2021 IEEE International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2021},
	pages = {525-534},
	publisher = { IEEE},
	abstract = {Continuous integration testing is an important step in the modern software engineering life cycle. Test prioritization is a method that can improve the efficiency of continuous integration testing by selecting test cases that can detect faults in the early stage of each cycle. As continuous integration testing produces voluminous test execution data, test history is a commonly used artifact in test prioritization. However, existing test prioritization techniques for continuous integration either cannot handle large test history or are optimized for using a limited number of historical test cycles. We show that such a limitation can decrease fault detection effectiveness of prioritized test suites.This work introduces DeepOrder, a deep learning-based model that works on the basis of regression machine learning. DeepOrder ranks test cases based on the historical record of test executions from any number of previous test cycles. DeepOrder learns failed test cases based on multiple factors including the duration and execution status of test cases. We experimentally show that deep neural networks, as a simple regression model, can be efficiently used for test case prioritization in continuous integration testing. DeepOrder is evaluated with respect to time-effectiveness and fault detection effectiveness in comparison with an industry practice and the state of the art approaches. The results show that DeepOrder outperforms the industry practice and state-of-the-art test prioritization approaches in terms of these two metrics.},
	doi = {10.1109/ICSME52107.2021.00053},
	author = {Sharif, Aizaz and Marijan, Dusica and Liaaen, Marius},
	editor = {, }
}
@misc {27006,
	title = {Cisco and Certus after 8 years: Current projects and reflections on what has been learned},
	howpublished = {The 14th Certus User Partner Workshop},
	year = {2019},
	author = {Carl Martin {Rosenberg} and Liaaen, Marius}
}
@article {26506,
	title = {Employing Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	journal = {Journal of Systems and Software},
	volume = {153},
	year = {2019},
	month = {07/2019},
	pages = {86-104},
	publisher = {Elsevier},
	abstract = {Test case prioritization (TP) is widely used in regression testing for optimal reordering of test cases to achieve specific criteria (e.g., higher fault detection capability) as early as possible. In our earlier work, we proposed an approach for black-box dynamic TP using rule mining and multi-objective search (named as REMAP) by defining two objectives (fault detection capability and test case reliance score) and considering test case execution results at runtime. In this paper, we conduct an extensive empirical evaluation of REMAP by employing three different rule mining algorithms and three different multi-objective search algorithms, and we also evaluate REMAP with one additional objective (estimated execution time) for a total of 18 different configurations (i.e., 3 rule mining algorithms {\texttimes}  3 search algorithms {\texttimes}  2 different set of objectives) of REMAP. Specifically, we empirically evaluated the 18 variants of REMAP with 1) two variants of random search while using two objectives and three objectives, 2) three variants of greedy algorithm based on one objective, two objectives, and three objectives, 3) 18 variants of static search-based prioritization approaches, and 4) six variants of rule-based prioritization approaches using two industrial and three open source case studies. Results showed that the two best variants of REMAP with two objectives and three objectives significantly outperformed the best variants of competing approaches by 84.4\% and 88.9\%, and managed to achieve on average 14.2\% and 18.8\% higher Average Percentage of Faults Detected per Cost (APFDc) scores.},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@article {26228,
	title = {A learning algorithm for optimizing continuous integration development and testing practice},
	journal = {Software: Practice and Experience},
	volume = {49},
	year = {2019},
	pages = {192-213},
	publisher = {Wiley Online Library},
	keywords = {Continuous Integration, Decision Trees, Machine Leanring, Software Testing},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius}
}
@misc {26941,
	title = {Search-based test case implantation for testing untested configurations},
	howpublished = {34th IEEE/ACM International Conference on Automated Software Engineering (ASE 2019), San Diego (CA), USA},
	year = {2019},
	publisher = {ACM},
	type = {Journal First Presentation},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@article {26488,
	title = {Search-Based Test Case Implantation for Testing Untested Configurations},
	journal = {Information and Software Technology},
	volume = {111},
	year = {2019},
	month = {07/2019},
	pages = {22-36},
	publisher = {Elsevier},
	abstract = {Context: Modern large-scale software systems are highly configurable, and thus require a large number of test cases to be implemented and revised for testing a variety of system configurations. This makes testing highly configurable systems very expensive and time-consuming.Objective: Driven by our industrial collaboration with a video conferencing company, we aim to automatically analyze and implant existing test cases (i.e., an original test suite) to test the untested configurations.Method: We propose a search-based test case implantation approach (named as SBI) consisting of two key components: 1) Test case analyzer that statically analyzes each test case in the original test suite to obtain the program dependence graph for test case statements and 2) Test case implanter that uses multi-objective search to select suitable test cases for implantation using three operators, i.e., selection, crossover, and mutation (at the test suite level) and implants the selected test cases using a mutation operator at the test case level including three operations (i.e., addition, modification, and deletion).Results: We empirically evaluated SBI with an industrial case study and an open source case study by comparing the implanted test suites produced by three variants of SBI with the original test suite using evaluation metrics such as statement coverage (SC), branch coverage (BC), and mutation score (MS). Results show that for both the case studies, the test suites implanted by the three variants of SBI performed significantly better than the original test suites. The best variant of SBI achieved on average 19.3\% higher coverage of configuration variable values for both the case studies. Moreover, for the open source case study, the best variant of SBI managed to improve SC, BC, and MS with 5.0\%, 7.9\%, and 3.2\%, respectively.Conclusion: SBI can be applied to automatically implant a test suite with the aim of testing untested configurations and thus achieving higher configuration coverage.\&nbsp;},
	keywords = {genetic algorithms, Multi-objective optimization, Search, test case implantation},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@techreport {26017,
	title = {Automated Test Case Implantation to Test Untested Configurations: A Cost-Effective Search-Based Approach},
	year = {2018},
	note = {This research was supported by the Research Council of Norway (RCN) funded Certus SFI. Shuai Wang is also supported by RFF Hovedstaden funded MBE-CR project and COST Action CA15140 (ImAppNIO). Tao Yue and Shaukat Ali are supported by RCN funded Zen-Configurator project, RCN funded MBTCPS project, and COST Action IC1404.\&nbsp;},
	publisher = {Simula Research Laboratory},
	abstract = {Modern large-scale software systems are highly configurable, and thus require a large number of test cases to be implemented and revised for testing a variety of system configurations. This makes testing highly configurable systems very expensive and time-consuming. Driven by our industrial collaboration with a video conferencing company, we aim to automatically analyze and implant existing test cases (i.e., an original test suite) to test the untested configurations. We propose a search-based test case implantation approach (named as SBI) consisting of two key components: 1) Test case analyzer that statically analyzes each test case in the original test suite to obtain the program dependence graph for test case statements and 2) Test case implanter that uses multi-objective search to select suitable test cases for implantation using three operators, i.e., selection, crossover, and mutation (at the test suite level) and implants the selected test cases using a mutation operator at the test case level including three operations (i.e., addition, modification, and deletion). We empirically evaluated SBI with an industrial case study and an open source case study by comparing the implanted test suites produced by SBI with the original test suite using evaluation metrics such as statement coverage (SC), branch coverage (BC), mutation score (MS). Results show that for both the case studies, the implanted test suites performed significantly better than the original test suites with on average 21.9\% higher coverage of configuration variable values. For the open source case study, SBI managed to improve SC, BC, and MS with 4.8\%, 7.5\%, and 2.6\%, respectively. SBI can be applied to automatically implant an existing test suite with the aim of testing untested configurations and thus achieving higher configuration coverage.\&nbsp;},
	keywords = {Genetic Algorithm, Multi-objective optimization, Search, test case implantation},
	url = {https://www.simula.no/sites/default/files/publications/files/automated_test_case_implantation_to_test_untested_configurations-_a_cost-effective_search-based_approach_tr.pdf},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@article {26214,
	title = {CBGA-ES+: A Cluster-Based Genetic Algorithm with Non-Dominated Elitist Selection for Supporting Multi-Objective Test Optimization},
	journal = {IEEE Transactions on Software Engineering},
	year = {2018},
	publisher = {IEEE},
	abstract = {Many real-world test optimization problems (e.g., test case prioritization) are multi-objective intrinsically and can be tackled using various multi-objective search algorithms (e.g., Non-dominated Sorting Genetic Algorithm (NSGA-II)). However, existing multi-objective search algorithms have certain randomness when selecting parent solutions for producing offspring solutions. In a worse case, suboptimal parent solutions may result in offspring solutions with bad quality, and thus affect the overall quality of the solutions in the next generation. To address such a challenge, we propose CBGA-ES+, a novel cluster-based genetic algorithm with non-dominated elitist selection to reduce the randomness when selecting the parent solutions to support multi-objective test optimization. We empirically compared CBGA-ES+ with random search and greedy (as baselines), four commonly used multi-objective search algorithms (i.e., Multi-objective Cellular genetic algorithm (MOCell), NSGA-II, Pareto Archived Evolution Strategy (PAES), and Strength Pareto Evolutionary Algorithm (SPEA2)), and the predecessor of CBGA-ES+ (named CBGA-ES) using five multi-objective test optimization problems with eight subjects (two industrial, one real world, and five open source). The results showed that CBGA-ES+ managed to significantly outperform the selected search algorithms for a majority of the experiments. Moreover, for the solutions in the same search space, CBGA-ES+ managed to perform better than CBGA-ES, MOCell, NSGA-II, PAES, and SPEA2 for 2.2\%, 13.6\%, 14.5\%, 17.4\%, and 9.9\%, respectively. Regarding the running time of the algorithm, CBGA-ES+ was faster than CBGA-ES for all the experiments.\&nbsp;},
	issn = {0098-5589},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {25976,
	title = {DevOps Improvements for Reduced Cycle Times with Integrated Test Optimizations for Continuous Integration},
	journal = {2018 IEEE 42nd Annual Computer Software and Applications Conference (COMPSAC)},
	year = {2018},
	pages = {22-27},
	publisher = {IEEE},
	doi = {10.1109/COMPSAC.2018.00012},
	author = {Marijan, Dusica and Sen, Sagar and Liaaen, Marius}
}
@techreport {25958,
	title = {Employing Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	year = {2018},
	note = {This research was supported by the Research Council of Norway (RCN) funded Certus SFI (grant no. 203461/O30). Tao Yue and Shaukat Ali are also supported by RCN funded Zen-Configurator project (grant no. 240024/F20) and RCN funded MBT4CPS project.\&nbsp;},
	publisher = {Simula Research Laboratory},
	abstract = {Test case prioritization (TP) is widely used in regression testing for optimal reordering of test cases to achieve specific criteria (e.g., higher fault detection capability) as early as possible. In our earlier work, we proposed an approach for black-box dynamic TP using rule mining and multi-objective search (named as REMAP) by defining two objectives (fault detection capability and test case reliance score) by considering test case execution results at runtime. In this paper, we conduct an extensive empirical evaluation of REMAP by employing three different rule mining algorithms and three different multi-objective search algorithms, and we also evaluate REMAP with one additional objective (estimated execution time) for a total of 18 different configurations\&nbsp;of REMAP. Specifically, we empirically evaluated the 18 variants of REMAP with 1) two variants of random search while using two objectives and three objectives, 2) three variants of greedy algorithm based on one objective, two objectives, and three objectives, 3) 18 variants of static search-based prioritization approaches, and 4) six variants of rule-based prioritization approaches using two industrial and three open source case studies. Results showed that the two best variants of REMAP with two objectives and three objectives significantly outperformed the best variants of competing approaches by 84.4\% and 88.9\%, and managed to achieve on average 14.2\% and 18.8\% higher Average Percentage of Faults Detected per Cost (APFDc) scores.},
	keywords = {Black-box regression testing, Dynamic test case prioritization, Multi-objective optimization, Rule Mining, Search},
	url = {https://www.simula.no/sites/default/files/publications/files/employing_rule_mining_and_multi-objective_search_for_dynamic_test_case_prioritization_tr.pdf},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {25975,
	title = {Practical Selective Regression Testing with Effective Redundancy in Interleaved Tests},
	journal = {Proceedings of the 40th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP{\textquoteright}18)},
	year = {2018},
	pages = {153-162},
	publisher = {ACM},
	isbn = {978-1-4503-5659-6},
	doi = {10.1145/3183519.3183532},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {25710,
	title = {REMAP: Using Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	journal = {11th IEEE Conference on Software Testing, Validation and Verification (ICST)},
	year = {2018},
	publisher = {IEEE},
	abstract = {Test case prioritization (TP) prioritizes test cases into an optimal order for achieving specific criteria (e.g., higher fault detection capability) as early as possible. However, the existing TP techniques usually only produce a static test case order before the execution without taking runtime test case execution results into account. In this paper, we propose an approach for dynamic TP using rule mining and multi-objective search (named as REMAP). REMAP has three key components: 1) Rule Miner, which mines execution relations among test cases from historical execution data; 2) Static Prioritizer, which defines two objectives (i.e., fault detection capability (FDC) and test case reliance score (TRS)) and applies multi-objective search to prioritize test cases statically; and 3) Dynamic Executor and Prioritizer, which executes statically-prioritized test cases and dynamically updates the test case order based on the runtime test case execution results. We empirically evaluated REMAP with random search, greedy based on FDC, greedy based on FDC and TRS, static search-based prioritization, and rule-based prioritization using two industrial and three open source case studies. Results showed that REMAP significantly outperformed the other approaches for 96\% of the case studies and managed to achieve on average 18\% higher Average Percentage of Faults Detected (APFD).\&nbsp;},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@misc {25962,
	title = {REMAP: Using Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	howpublished = {IEEE Conference on Software Testing, Validation and Verification (ICST), V{\"a}ster{\r a}s, Sweden},
	year = {2018},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@misc {26074,
	title = {SW Testing: Can ML save us?},
	howpublished = {NDC TechTown 2018, Kongsberg, Norway},
	year = {2018},
	abstract = {Nothing has happened in testing for a long time. While products are getting ever more complex and the pressure to ship grows every day, we are still hand-crafting test cases and spending immense manual efforts, especially when testing GUIs and other complex aspects of the system.In the past decade, however, there has been a massive Machine Learning renaissance, showing a lot of promise in all fields. How is it affecting software testing? We have been studying the trends for a while, and we will discuss the promising aspects as well as some fundamental limitations.},
	author = {Carl Martin {Rosenberg} and Liaaen, Marius}
}
@misc {25690,
	title = {Anomaly Detection and Data Clustering},
	howpublished = {12th Certus User Partner Workshop, Norway},
	year = {2017},
	author = {Carl Martin {Rosenberg} and Liaaen, Marius}
}
@inproceedings {24925,
	title = {CBGA-ES: A Cluster-Based Genetic Algorithm with Elitist Selection for Supporting Multi-objective Test Optimization},
	journal = {10th IEEE International Conference on Software Testing, Verification and Validation (ICST 2017)},
	year = {2017},
	publisher = {IEEE},
	abstract = {Multi-objective search algorithms (e.g., non- dominated sorting genetic algorithm II (NSGA-II)) have been frequently applied to address various testing problems requiring multi-objective optimization such as test case selection. However, existing multi-objective search algorithms have certain randomness when selecting parent solutions for producing offspring solutions. In the worse case, suboptimal parent solutions may result in offspring solutions with bad quality, and thus affect the overall quality of the next generation. To address such a challenge, we propose a cluster-based genetic algorithm with elitist selection (CBGA-ES) with the aim to reduce such randomness for supporting multi-objective test optimization. We empirically compared CBGA-ES with random search, greedy (as baselines) and four commonly used multi-objective search algorithms (e.g., NSGA-II) using three industrial test optimization problems, i.e., test suite minimization, test case prioritization, and test case selection. The results showed that CBGA-ES significantly outperformed the baseline algorithms (e.g., greedy), and the four selected search algorithms for all the three test optimization problems. CBGA-ES managed to outperform more than 75\% of the objectives for all the four algorithms in each test optimization problem. Moreover, CBGA- ES was able to improve the quality of the solutions for an average of 32.5\% for each objective as compared to the four algorithms for the three test optimization problems.\&nbsp;},
	doi = {10.1109/ICST.2017.40},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius},
	editor = {Memon, Atif and Nishi, Yasuharu}
}
@misc {25677,
	title = {Effective test scrubbing with machine learning and Python},
	howpublished = {NDC TechTown, Kongsberg, Norway},
	year = {2017},
	type = {Technical talk},
	abstract = {Fast feedback on every change you do to your software codebase is essential for a rapid and safe development cycle. Failed test runs need attention from human experts that can quickly analyze and take necessary action. Our problem: the number of failed tests results is higher than our little group of domain experts can handle. We need a more efficient and effective way to remove the risks of having un-scrubbed failed test results lying around. We developed a machine learning tool based on test results clustering and this allows us to open up for even more automation for handling test results.This presentation is about what we needed and how we solved it with only an entry-level machine-learning course and some Python skills. Clustering techniques can be useful in any field and the path from an idea to get something of value running can be short with the right approach and tools.},
	author = {Carl Martin {Rosenberg} and Liaaen, Marius and Thomas Hanssen {Nordnes}}
}
@inproceedings {24960,
	title = {Empowering Testing Activities with Modeling: Achievements and Insights from Nine Years of Collaboration with Cisco},
	journal = {The International Conference on Model-Driven Engineering and Software Development (MODELSWARD)},
	year = {2017},
	pages = {581-589},
	publisher = {SCITEPRESS {\textendash} Science and Technology Publications},
	abstract = {Research-Based Innovation (RBI) aims at bringing research-driven innovative solutions to the industrial problems identified from the industry in close collaboration with researchers. This paper focuses on presenting one such instance of RBI between Cisco Systems, Norway and the Software Engineering department of Simula Research Laboratory, Norway over the period of last nine years. The main topic of the collaboration was related to improving the current testing practice at Cisco with the use of models. We present a brief overview of various model-driven testing projects, and lessons learned from such RBI collaboration.},
	author = {Ali, Shaukat and Liaaen, Marius and Wang, Shuai and Yue, Tao},
	editor = {Luis Ferreira {Pires} and Hammoudi, Slimane and Selic, Bran}
}
@article {25652,
	title = {Integrating Weight Assignment Strategies with NSGA-II for Supporting User Preference Multi-Objective Optimization},
	journal = {IEEE Transactions on Evolutionary Computation (TEVC)},
	year = {2017},
	publisher = {IEEE},
	abstract = {Driven by the needs of several industrial projects on the applications of multi-objective search algorithms, we observed that user preferences must be properly incorporated into optimization objectives. However, existing algorithms usually treat all the objectives with equal priorities and do not provide a mechanism to reflect user preferences. To address this, we propose an extension{\textemdash}User-Preference Multi-Objective Optimization Algorithm (UPMOA), to the most commonly applied, non-dominated sorting genetic algorithm II (NSGA-II) by introducing a user preference indicator !, based on existing weight assignment strategies (e.g., Uniformly Distributed Weights (UDW)). We empirically evaluated UPMOA using four industrial problems from three diverse domains (i.e., Communication, Maritime and Subsea Oil\&amp;Gas). We also performed a sensitivity analysis for UPMOA with 625 algorithm parameter settings. To further assess the performance and scalability, 103500 artificial problems were created and evaluated representing 207 sets of user preferences. Results show that the UDW strategy with UPMOA achieves the best performance and UPMOA significantly outperformed other three multi-objective search algorithms, and has the ability to solve problems with a wide range of complexity. We also observed that different parameter settings led to the varied performance of UPMOA, thus suggesting that configuring proper parameters is highly problem-specific.\&nbsp;},
	doi = {10.1109/TEVC.2017.2778560},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {25064,
	title = {Test Prioritization with Optimally Balanced Configuration Coverage},
	journal = {IEEE International Symposium on High Assurance Systems Engineering},
	year = {2017},
	publisher = {ACM/IEEE},
	isbn = {978-1-5090-4636-2},
	doi = {10.1109/HASE.2017.26},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {25227,
	title = {TITAN: Test Suite Optimization for Highly Configurable Software},
	journal = {International Conference on Software Testing, Verification and Validation (ICST 2017) },
	year = {2017},
	publisher = {IEEE},
	isbn = {978-1-5090-6031-3},
	doi = {10.1109/ICST.2017.60},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius and Sen, Sagar and Ieva, Carlo}
}
@inproceedings {23736,
	title = {Automated Regression Testing Using Constraint Programming},
	journal = {Twenty-Eighth Conference on Innovative Applications of Artificial Intelligence (IAAI-16), Phoenix, AZ, USA, Feb. 2016},
	year = {2016},
	month = {02/2016},
	abstract = {\&nbsp; In software validation, regression testing aims to check the absence of regression faults in new releases of a software system. Typically, test cases used in regression testing are executed during a limited amount of time and are selected to check a given set of user requirements. When testing large systems the number of regression tests grows quickly over the years, and yet the available time slot stays limited. In order to overcome this problem, an approach known as test suite reduction (TSR), has been developed in software engineering to select a smallest subset of test cases, so that each requirement remains covered at least once. However solving the TSR problem is difficult as the underlying optimization problem is NP-hard, but it is also crucial for vendors interested in reducing the time to market of new software releases.\&nbsp; In this paper, we address regression testing and TSR with Constraint Programming (CP).\&nbsp; More specifically, we propose new CP models to solve TSR that exploit global constraints, namely NValue and GCC. We reuse a set of preprocessing rules to reduce a priori each instance, and we introduce a structure-aware search heuristic. We evaluated our CP models and proposed improvements\&nbsp; against existing approaches, including a simple greedy approach and MINTS, the state-of-the-art tool of the software engineering community. Our experiments show that CP outperforms both the greedy approach and MINTS when it is interfaced with MiniSAT, in terms of percentage of reduction and execution time. When MINTS is interfaced with CPLEX, we show that our CP model performs better only on percentage of reduction. Finally, by working closely with validation engineers from Cisco Systems, Norway, we integrated our CP model into an industrial regression testing process.},
	author = {Gotlieb, Arnaud and Carlsson, Mats and Liaaen, Marius and Marijan, Dusica and Petillon, Alexandre}
}
@inproceedings {24939,
	title = {Coverage-based Test Prioritization for Regression Testing of Configurable Software},
	journal = {IEEE 27th International Symposium on Software Reliability Engineering (ISSRE)},
	year = {2016},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {24868,
	title = {Effect of Time Window on the Performance of Continuous Regression Testing},
	journal = {32nd IEEE International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2016},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {24043,
	title = {Enhancing Test Case Prioritization in an Industrial Setting with Resource Awareness and Multi-Objective Search},
	journal = {The 38th International Conference on Software Engineering (ICSE), Software Engineering in Practice (SEIP) track },
	year = {2016},
	pages = {182-191},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Bakkeli, {\O}yvind and Liaaen, Marius}
}
@inproceedings {24438,
	title = {A Practical Guide to Select Quality Indicators for Assessing Pareto-Based Search Algorithms in Search-Based Software Engineering},
	journal = {the 38th International Conference on Software Engineering (ICSE)},
	year = {2016},
	pages = {631-642},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Li, Yan and Liaaen, Marius}
}
@inproceedings {24632,
	title = {STIPI: Using Search to Prioritize Test Cases based on Multi-Objectives Derived from Industrial Practice},
	journal = {The 28th International Conference on Testing Software and Systems (ICTSS) },
	year = {2016},
	month = {10/2016},
	pages = {172-190},
	publisher = {Lecture Notes in Computer Science, Springer Verlag},
	address = {Graz, Austria},
	abstract = {The importance of cost-effective test case prioritization is undeniable in automated testing practice in industry. Such prioritization typically relies on various cost and effective objectives. This paper focuses on prioritizing test cases developed to test product lines of Video Conferencing Systems (VCSs) at Cisco Systems, Norway. Each test case requires setting up configurations of a set of VCSs, invoking a set of test APIs with specific inputs, and checking the status of the VCSs. Based on these characteristics and information available about the execution of test cases (e.g., number of faults detected), we identified that the test case prioritization problem in our particular context should focus on achieving high coverage of configurations, test APIs, statuses, and high fault detection capability as fast as possible. We propose a search-based test case prioritization approach (named STIPI) to solve this problem by defining a fitness function with four objectives and integrating it with the widely applied multi-objective Non-dominated Sorting Genetic Algorithm II. We compared STIPI with random search (RS), Greedy algorithm, and three approaches adapted from literature, using three real sets of test cases from Cisco with four time budgets (25\%, 50\%, 75\% and 100\%). Results show that STIPI significantly outperformed the selected approaches and managed to achieve better performance than RS for on average 39.9\%, 18.6\%, 32.7\% and 43.9\% for the coverage of configurations, test APIs, statuses and fault detection capability, respectively.},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@techreport {24571,
	title = {STIPI: Using Search to Prioritize Test Cases based on Multi-Objectives Derived from Industrial Practice},
	year = {2016},
	note = {This research is supported by the Research Council of Norway (RCN) funded Certus SFI. Shuai Wang is also supported by the RFF Hovedstaden funded MBE-CR project. Shaukat Ali and Tao Yue are also supported by the RCN funded Zen-Configurator project, the EU Horizon 2020 project funded U-Test, the RFF Hovedstaden funded MBE-CR project and the RCN funded MBT4CPS project.\&nbsp;},
	publisher = {Simula Research Laboratory},
	keywords = {Configurations, Search, Test APIs, Test Case Prioritization},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@article {EMSE2014,
	title = {A Systematic Test Case Selection Methodology for Product Lines: Results and Insights From an Industrial Case Study},
	journal = {Empirical Software Engineering},
	volume = {21},
	year = {2016},
	month = {08/2016},
	pages = {1586-1622},
	publisher = {Springer},
	abstract = {In the context of product lines, test case selection aims at obtaining a set of relevant test cases for a product from the entire set of test cases available for a product line. While working on a research-based innovation project on automated testing of product lines of Video Conferencing Systems (VCSs) developed by Cisco, we felt the need to devise a cost-effective way of selecting relevant test cases for a product. To fulfill such need, we propose a systematic and automated test selection methodology using: 1) Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line; 2) Component Family Model for Testing (CFM\_T) to model the structure of test case repository; 3) A tool to automatically build restrictions from CFM\_T to FM\_T and traces from CFM\_T to the actual test cases. Using our methodology, a test engineer is only required to select relevant features through FM\_T at a higher level of abstraction for a product and the corresponding test cases will be obtained automatically. We evaluate our methodology by applying it to a VCS product line called Saturn with seven commercial products and the results show that our methodology can significantly reduce cost measured as test selection time and at the same time achieves higher effectiveness (feature coverage, feature pairwise coverage and fault detection) as compared with the current manual process. Moreover, we conduct a questionnaire-based study to solicit the views of test engineers who are involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our methodology in their current practice. Finally, we present a set of lessons learnt while applying product line engineering at Cisco for test case selection.},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@article {23452,
	title = {Automated Product Line Test Case Selection: Industrial Case Study and Controlled Experiment},
	journal = {Journal of Software and Systems Modeling},
	year = {2015},
	month = {04/2015},
	pages = {1-25},
	publisher = {Springer},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We evaluated our methodology via three different ways: 1) We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly; 2) We conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM_T and CFM_T. The results show that test engineers are positive about adapting our methodology and models (FM_T and CFM_T) in their current practice; 3) We conducted a controlled experiment with 20 graduate students to assess the performance (i.e., cost, effectiveness and efficiency) of our automated methodology as compared with the manual approach. The results showed that our methodology is cost-effective as compared with the manual approach and at the same time its efficiency is not affected by the increased complexity of products.\&nbsp;},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@inproceedings {23561,
	title = {UPMOA: An Improved Search Algorithm to Support User- Preference Multi-Objective Optimization},
	journal = {The 26th IEEE International Symposium on Software Reliability Engineering (ISSRE). Gaithersburg, USA, November 2-5.},
	year = {2015},
	pages = {393-404},
	abstract = {Multi-objective search algorithms (e.g., non- dominated sorting genetic algorithm II (NSGA-II)) have been applied extensively to solve various multi-objective optimization problems in software engineering such as problems in testing. However, existing multi-objective algorithms usually treat all the objectives with equivalent priorities and do not provide a mechanism to reflect various user preferences when guiding search. The need to have such a mechanism was observed in one of our industrial projects on applying search algorithms for test optimization of a product line of Videoconferencing Systems (VCSs) called Saturn, where user preferences must be incorporated into optimization objectives, based on domain knowledge of test engineers for VCS testing. To address this, we propose an extension to the most commonly-used multi-objective search algorithm NSGA-II, which has shown promising results with user preferences. We name the extension as User-Preference Multi-Objective Optimization Algorithm (UPMOA), which includes a user preference indicator p\&nbsp;and is based on existing weight assignment strategies. We empirically evaluated UPMOA with two industrial problems focusing on optimizing the test execution system for Saturn in Cisco. To assess the performance and scalability of UPMOA, inspired by the two industrial problems, in total we created 64000 artificial problems with 128 different sets of user preferences. The evaluation includes two aspects: 1) Three weight assignment strategies together with UPMOA were empirically evaluated to identify a best weight assignment strategy for p. Results show that the Uniformly Distributed Weights (UDW) strategy can assist UPMOA in achieving the best performance; 2) UPMOA was compared with three representative multi-objective search algorithms (including NSGA-II) and results show that UPMOA significantly outperformed the others and has the ability to solve problems with a wide range of complexity.\&nbsp;},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {splc2014,
	title = {Multi-Objective Test Prioritization in Software Product Line Testing: an Industrial Case Study},
	journal = {Software Product Line Conference (SPLC14), Florence, Italy},
	year = {2014},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	abstract = {Test prioritization is crucial for testing products in a product line considering limited budget in terms of available time and resources. In general, it is not practically feasible to execute all the possible test cases and so, ordering test case execution permits test engineers to discover faults earlier in the testing process. An efficient prioritization of test cases for one or more products requires a clear consideration of the tradeoff among various costs (e.g., time, required resources) and effectiveness (e.g., feature coverage) objectives. As an integral part of the future Cisco{\textquoteright}s test scheduling system for validating video conferencing products, we introduce a search-based multi-objective test prioritization technique, considering multiple cost and effectiveness measures. In particular, our multi-objective optimization setup includes the minimization of execution cost (e.g., time), and the maximization of number of prioritized test cases, feature pairwise coverage and fault detection capability. Based on cost-effectiveness measures, a novel fitness function is defined for such test prioritization problem. The fitness function is empirically evaluated together with three commonly used search algorithms (e.g., (1+1) Evolutionary algorithm (EA)) and Random Search as a comparison baseline based on the Cisco{\textquoteright}s industrial case study and 500 artificial designed problems. The results show that (1+1) EA achieves the best performance for solving the test prioritization problem and it scales up to solve the problems of varying complexity.},
	keywords = {Conference},
	author = {Wang, Shuai and Buchmann, David and Ali, Shaukat and Gotlieb, Arnaud and Pradhan, Dipesh and Liaaen, Marius}
}
@inproceedings {Simula.simula.2018,
	title = {Automated Test Case Selection Using Feature Model: an Industrial Case Study},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS), Best Paper Award},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM\_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM\_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly. Moreover, we conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our method- ology and models (FM\_T and CFM\_T) in their current practice.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius},
	editor = {Grey, Jeff}
}
@inproceedings {Simula.simula.2012,
	title = {Using Feature Model to Support Model-Based Testing of Product Lines: an Industrial Case Study},
	journal = {The 13th International Conference on Quality Software (QSIC 2013)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Najing},
	abstract = {In the context of Model-Based Testing (MBT) of product lines, effort required to develop models can be significantly reduced by applying systematic product line modeling and configuration methodologies. In our previous work, we developed such a methodology to capture variability in configurable UML state machines and aspect state machines. For each product, these state machines are to be configured for generating executable test cases. In this paper, we extended this methodology using Feature Model for Testing (FM\_T) and Component Family Model for Behaviors (CFM\_B). FM\_T captures variable testing functionalities of a product line, whereas CFM\_B provides an abstraction layer on top of the configurable state machines. With our current methodology, a test engineer doesn{\textquoteright}t need to acquire expertise on behavioral modeling and can simply configure models for a product by selecting features in FM\_T and configuring provided attributes in CFM\_B. The configured models are then given input to our model-based testing tool, TRansformation-based tool for Uml-baSed Testing (TRUST) for executable test case generation. We applied our extended methodology to a product line of video conferencing system developed by Cisco Systems, Norway. Results show that the methodology significantly reduces the complexity of configuration; thereby significantly reducing required effort and cost (e.g., in terms of training). In addition, it does not require test engineers to have expertise in UML modeling, aspect-oriented modeling, and OCL specification and therefore eases the adoption of MBT in industry.},
	keywords = {Conference},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius},
	editor = {Tse, T.H.}
}
@techreport {25300,
	title = {Automated Selection of Test Cases using Feature Model for Product Lines: An Industrial Case Study},
	number = {2012-20},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version\&nbsp;can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@techreport {25305,
	title = {Automated Test Case Selection using Feature Model: an Industrial Case Study},
	number = {2012-28},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@misc {Simula.simula.2038,
	title = {Automatic Derivation of Test Execution Plans From a Video Conferencing System Product Line Modelling},
	year = {2012},
	address = {.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Briand, Lionel Claude}
}
@techreport {25312,
	title = {Automatic Derivation of Test Execution Plans From a Video Conferencing System Product Line Modelling},
	number = {2012-02},
	year = {2012},
	note = {This technical report is an early version of a published poster.https://www.simula.no/publications/automatic-derivation-test-execution-plans-video-conferencing-system-product-line},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Lionel C. {Briand}}
}
@inproceedings {23895,
	title = {Automatic Selection of Test Execution Plans from a Video Conferencing System Product Line},
	journal = {Proceedings of the VARiability for You Workshop: Variability Modeling Made Useful for Everyone},
	year = {2012},
	pages = {32 - 37},
	publisher = {ACM},
	isbn = {978-1-4503-1809-9},
	doi = {10.1145/2425415.2425422},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Briand, Lionel}
}
@inproceedings {ND.5.Ryan.1996,
	title = {Cluster Communication Using a PCI to SCI Interface},
	journal = {Proceedings IASTED International Conference on Parallel and Distributed Computing and Systems},
	year = {1996},
	month = {Oktober,},
	pages = {283-289},
	type = {1},
	address = {Chicago, Illinois, USA},
	author = {S. J. {Ryan} and Gjessing, Stein and Liaaen, Marius}
}
