@inproceedings {43377,
	title = {Automated Test Generation for Medical Rules Web Services: A Case Study at the Cancer Registry of Norway},
	journal = {31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE 2023)},
	year = {2023},
	publisher = {ACM},
	doi = {10.1145/3611643.3613882},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat and Schwitalla, Thomas and Jan F. {Nyg{\r a}rd}}
}
@inproceedings {43324,
	title = {Challenges of Testing an Evolving Cancer Registration Support System in Practice},
	journal = {45th IEEE/ACM International Conference on Software Engineering: Companion Proceedings (ICSE-Companion 2023)},
	year = {2023},
	publisher = {IEEE},
	doi = {10.1109/ICSE-Companion58688.2023.00102},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat and Schwitalla, Thomas and Jan F. {Nyg{\r a}rd}}
}
@inproceedings {43359,
	title = {DeepScenario: An Open Driving Scenario Dataset for Autonomous Driving System Testing},
	journal = {2023 IEEE/ACM 20th International Conference on Mining Software Repositories (MSR)},
	year = {2023},
	month = {05/2023},
	publisher = {IEEE},
	address = {Melbourne, Australia},
	abstract = {With the rapid development of autonomous driving systems (ADSs), testing ADSs under various environmental conditions has become a key method to ensure the successful deployment of ADS in the real world. However, it is impossible to test all the scenarios due to the inherent complexity and uncertainty of ADSs and the driving tasks. Further, testing of ADSs is expensive regarding time and computational resources. Therefore, a large-scale driving scenario dataset consisting of various driving conditions is needed. To this end, we present an open driving scenario dataset DeepScenario, containing over 30K executable driving scenarios, which are collected by 2880 test executions of three driving scenario generation strategies. Each scenario in the dataset is labeled with six attributes characterizing test results. We further show the attribute statistics and distribution of driving scenarios. For example, there are 1050 collision scenarios, in 917 scenarios there were collisions with other vehicles, 105 and 28 with pedestrians and static obstacles, respectively. Target users include ADS developers who need to validate their systems under various environmental conditions.},
	keywords = {autonomous driving system testing, dataset, driving scenario, Open Source},
	doi = {10.1109/MSR59073.2023.00020},
	url = {https://ieeexplore.ieee.org/document/10174023/http://xplorestaging.ieee.org/ielx7/10173934/10173935/10174023.pdf?arnumber=10174023},
	author = {Lu, Chengjie and Yue, Tao and Ali, Shaukat}
}
@article {43107,
	title = {Digital Twin-based Anomaly Detection with Curriculum Learning in Cyber-physical Systems},
	journal = {ACM Transactions on Software Engineering and Methodology},
	year = {2023},
	publisher = {ACM},
	address = {TOSEM},
	abstract = {Anomaly detection is critical to ensure the security of cyber-physical systems (CPS). However, due to the increasing complexity of attacks and CPS themselves, anomaly detection in CPS is becoming more and more challenging. In our previous work, we proposed a digital twin-based anomaly detection method, called ATTAIN, which takes advantage of both historical and real-time data of CPS. However, such data vary significantly in terms of difficulty. Therefore, similar to human learning processes, deep learning models (e.g., ATTAIN) can benefit from an easy-to-difficult curriculum. To this end, in this paper, we present a novel approach, nameddigitaL twin-based Anomaly deTecTion wIth Curriculum lEarning (LATTICE), which extends ATTAIN by introducing curriculum learning to optimize its learning paradigm. LATTICE attributes each sample with a difficulty score, before being fed into a training scheduler. The training scheduler samples batches of trainingdata based on these difficulty scores such that learning from easy to difficult data can be performed. To evaluate LATTICE, we use five publicly available datasets collected from five real-world CPS testbeds. We compare LATTICE with ATTAIN and two other state-of-the-art anomaly detectors. Evaluation results show that LATTICE outperforms the three baselines and ATTAIN by 0.906\%-2.367\% in terms of the F1 score. LATTICE also, on average, reduces the training time of ATTAIN by 4.2\% on the five datasets and is on par with the baselines in terms of detection delay time},
	author = {Xu, Qinghua and Ali, Shaukat and Yue, Tao}
}
@misc {43378,
	title = {QCNorway: Contributions Towards a Norwegian Quantum Computing Strategy},
	year = {2023},
	month = {06/2023},
	publisher = {Simula},
	type = {Position Paper},
	abstract = {Norway lacks a national strategy and corresponding investments for the development of its own expertise on quantum computing through educational programmes, active research, business creation, and support from technical infrastructures.Given the enormous momentum in quantum technology and quantum computing R\&amp;D world-wide and in our neighbouring countries, there is an immediate threat that Norway will arrive too late at a technology race that has the potential of redefining the digital society and digital business development.This position paper presents the most important takeaway messages from the open workshop QCNorway: Towards a Norwegian Quantum Computing Strategy, which took place in Oslo on November 7{\textendash}8, 2022 with more than 150 participants. The discussion herein is limited to quantum computing and does not claim to address quantum technology in general. Hopefully, this paper and complementary inputs from other stakeholders can inspire the development of a national quantum strategy, in which quantum computing is a principal part.},
	keywords = {national strategy, quantum computing},
	url = {https://www.qcnorway.no/finalpaper},
	author = {Are Magnus {Bruaset} and Ali, Shaukat and Brodtkorb, Andr{\'e} and B{\o}e, Gunnar and Denysov, Sergiy and Eide, Hans and Selst{\o}, S{\o}lve}
}
@misc {43241,
	title = {Quantum Software Testing: A Brief Introduction},
	howpublished = {2023 International Conference on Software Engineering},
	year = {2023},
	publisher = {IEEE/ACM},
	abstract = {Quantum software testing is an emerging software engineering field that focuses on testing quantum programs to find quantum faults in the programs cost-effectively. Given the foundations in quantum mechanics, the way quantum programs perform computations is significantly different than the classical programs. Therefore, quantum software testing also differs than classical software testing. There has been quite an interest in building quantum software testing techniques since 2019 in the software engineering community. Thus, we aim to provide an introduction to quantum software testing to the community. In particular, we will present the basic foundations of quantum computing and quantum programming as circuits, followed by the current state of the art on quantum software testing. Next, we will present some basic quantum software testing techniques and finally give the research directions that deserve attention from the software engineering community.},
	author = {Ali, Shaukat and Yue, Tao}
}
@misc {42891,
	title = {AI-enabled Digital Twins for Cyber-Physical Systems},
	howpublished = {EDT Community: Engineering Digital Twins {\textendash} Seminar Series},
	year = {2022},
	author = {Ali, Shaukat}
}
@inproceedings {42667,
	title = {Are Elevator Software Robust Against Uncertainties? Results and Experiences from an Industrial Case Study},
	journal = {The ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE)},
	year = {2022},
	month = {11/2022},
	pages = {1331-1342},
	publisher = {ACM},
	author = {Han, Liping and Yue, Tao and Ali, Shaukat and Arrieta, Aitor and Arratibel, Maite}
}
@misc {42888,
	title = {Assuring the Quality of Quantum Programs with Automated Testing},
	howpublished = {University of Lisbon, Portugal},
	year = {2022},
	author = {Ali, Shaukat}
}
@misc {42886,
	title = {Automating Test Oracle Generation in DevOps for Industrial Elevators},
	howpublished = {19TH IEEE INTERNATIONAL CONFERENCE ON SOFTWARE ARCHITECTURE (ICSA 2022)},
	year = {2022},
	publisher = { IEEE},
	author = {Ali, Shaukat}
}
@inproceedings {28419,
	title = {Automating Test Oracle Generation in DevOps for Industrial Elevators},
	journal = {29th IEEE International Conference on Software Analysis, Evolution and Reengineering},
	year = {2022},
	pages = {284-288},
	publisher = { IEEE},
	doi = {10.1109/SANER53432.2022.00044},
	author = {Arrieta, Aitor and Otaegi, Maialen and Han, Liping and Sagardui, Goiuria and Ali, Shaukat and Arratibel, Maite},
	editor = {Ciolkowski, Marcus and Palomba, Fabio}
}
@misc {42867,
	title = {Building Complex Software Systems in Classical and Quantum Computing Domains },
	howpublished = {Connecting Education and Research Communities for an Innovative Resource Aware Society, Meeting Denmark},
	year = {2022},
	author = {Ali, Shaukat}
}
@techreport {28492,
	title = {Classifying Uncertainties in an Industrial Elevator with the Cynefin Framework},
	year = {2022},
	publisher = {Simula Research Laboratory},
	author = {Han, Liping and Yue, Tao and Ali, Shaukat and Arrieta, Aitor}
}
@misc {42979,
	title = {Dependable and Noise-Aware Quantum Software Engineering},
	howpublished = {CREST Center, The University of Adelaide, Australia},
	year = {2022},
	author = {Ali, Shaukat}
}
@misc {42885,
	title = {Digital Twin for Elevators Use Case},
	howpublished = {ETSI Event, Berlin, Germany},
	year = {2022},
	author = {Ali, Shaukat}
}
@techreport {28494,
	title = {Digital Twin-based Anomaly Detection with Curriculum Learning in Cyber-physical Systems},
	year = {2022},
	publisher = {Simula Research Laboratory},
	author = {Xu, Qinghua and Ali, Shaukat and Yue, Tao}
}
@misc {42894,
	title = {Digital Twins for Cyber-Physical Systems: From AI to Quantum Computing},
	howpublished = {COEMS Forsterk Seminar},
	year = {2022},
	author = {Ali, Shaukat}
}
@inproceedings {42862,
	title = {Enhancing the realism of autonomous driving simulation with real-time co-simulation},
	journal = {4th International Workshop on Multi-Paradigm Modelling for Cyber-Physical Systems (MPM4CPS{\textquoteright}22) },
	year = {2022},
	publisher = { IEEE/ACM},
	author = {Chen, Qiwei and Wang, Tiexin and Lu, Chengjie and Yue, Tao and Ali, Shaukat},
	editor = {Amrani, Moussa and Blouin, Dominique and Challenger, Moharram and Exelmans, Joeri and Heinrich, Robert and Paredis, Randy}
}
@inproceedings {42903,
	title = {Generating Failing Test Suites for Quantum Programs with Search (hot off the press track at GECCO 2022)},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {47-48},
	publisher = {ACM},
	abstract = {The inherent complexity of quantum programs, due to features such as superposition and entanglement, makes their testing particularly challenging. To tackle these challenges, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size that possibly expose failures of the quantum program under test. QuSBT encodes a test suite as a search individual, and tries to maximize the objective function that counts the number of failing tests in the test suite. Due to non-deterministic nature of quantum programs, the approach repeats the execution of each test multiple times, and uses suitable statistical tests to assess if a test passes or fails. QuSBT employs a genetic algorithm to perform the search. Experiments on 30 faulty quantum programs show that QuSBT is statistically better than random search, and is able to efficiently generate maximal failing test suites.

This is an extended abstract of the paper [1]: X. Wang, P. Arcaini, T. Yue, and S. Ali "Generating Failing Test Suites for Quantum Programs With Search", 13th International Symposium on Search-Based Software Engineering (SSBSE 2021).},
	doi = {10.1145/3520304.3534067},
	url = {https://dl.acm.org/doi/10.1145/3520304.3534067},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42476,
	title = {Investigating Quantum Cause-Effect Graphs},
	journal = {2022 IEEE/ACM 3rd International Workshop on Quantum Software Engineering (Q-SE)},
	year = {2022},
	pages = {8-15},
	publisher = {IEEE},
	abstract = {Cause-effect graphs have shown promising results in identifying relations among causes and effects of classical software systems, followed by designing effective test cases from them. Towards this end, we investigate the use of cause-effect graphs for quantum programs. Classical cause-effect graphs apply classical logic (e.g., AND, OR) to express these relations, which might not be practical for describing similar relations in quantum programs due to quantum superposition and entanglement. Thus, we propose an extension of cause-effect graphs, where quantum logic inspired functions (e.g., Hadamard) and their generalizations are defined and applied. Moreover, we present a metamodel describing various forms of cause-effect graphs. Finally, we demonstrate a possible method for generating test cases from a quantum cause-effect graph applied to a Bell state quantum program. Lastly, the design and utility of the resulting testing method is discussed, along with future prospects for general quantum cause-effect graphs.\&nbsp;\&nbsp;},
	author = {Oldfield, Noah and Yue, Tao and Ali, Shaukat}
}
@article {28452,
	title = {Learning Configurations of Operating Environment of Autonomous Vehicles to Maximize their Collisions},
	journal = {IEEE Transactions on Software Engineering},
	volume = {49},
	year = {2022},
	pages = {384-402},
	publisher = { IEEE},
	author = {Lu, Chengjie and Shi, Yize and Zhang, Huihui and Zhang, Man and Wang, Tiexin and Yue, Tao and Ali, Shaukat}
}
@misc {42665,
	title = {Learning Digital Twin Models},
	howpublished = {Model-Driven Engineering of Digital Twins Seminar at Dagstuhl, Germany},
	year = {2022},
	publisher = {Schloss Dagstuhl, Germany},
	abstract = {Given that operational cyber-physical systems (CPS) produce continuous data, a complementary approach to model-based engineering is to learn digital twins models with machine learning techniques and providing functionalities such as predictions and anomaly detection.This talk will start with presenting an opinion on the next generation of digital twins (Quantum Digital Twins), where some aspects of digital twins will be implemented as quantum software and executed on quantum computers, e.g., for simulating the physical environment that can be realistically simulated with quantum-mechanical principles.Followed by this opinion, the talk will present some recent works on learning digital twins from historical data and continuous updates of digital twins with continuous data from operational CPS. Various machine learning techniques were applied, such as generative adversarial networks, curriculum learning, and transfer learning to learn digital twins. The digital twins were built for use cases from the transportation domain and water distribution/treatment plants. These digital twins were focused on anomaly detection and waiting time predictions.},
	author = {Ali, Shaukat}
}
@misc {42892,
	title = {Modeling robustness behavior using aspect-oriented modeling to support robustness testing of industrial systems},
	howpublished = {ACM / IEEE 25th International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	year = {2022},
	type = {10-year SOSYM Most Influential Paper Award},
	author = {Ali, Shaukat}
}
@article {laaber2022mosmbprio,
	title = {Multi-Objective Search-Based Software Microbenchmark Prioritization},
	journal = {arXiv},
	year = {2022},
	publisher = {arXiv},
	abstract = {Ensuring that software performance does not degrade after a code change is paramount. A potential solution, particularly for libraries and frameworks, is regularly executing software microbenchmarks, a performance testing technique similar to (functional) unit tests. This often becomes infeasible due to the extensive runtimes of microbenchmark suites, however. To address that challenge, research has investigated regression testing techniques, such as test case prioritization (TCP), which reorder the execution within a microbenchmark suite to detect larger performance changes sooner. Such techniques are either designed for unit tests and perform sub-par on microbenchmarks or require complex performance models, reducing their potential application drastically. In this paper, we propose a search-based technique based on multi-objective evolutionary algorithms (MOEAs) to improve the current state of microbenchmark prioritization. The technique utilizes three objectives, i.e., coverage to maximize, coverage overlap to minimize, and historical performance change detection to maximize. We find that our technique improves over the best coverage-based, greedy baselines in terms of average percentage of fault-detection on performance (APFD-P) and Top-3 effectiveness by 26 percentage points (pp) and 43 pp (for Additional) and 17 pp and 32 pp (for Total) to 0.77 and 0.24, respectively. Employing the Indicator-Based Evolutionary Algorithm (IBEA) as MOEA leads to the best effectiveness among six MOEAs. Finally, the technique{\textquoteright}s runtime overhead is acceptable at 19\% of the overall benchmark suite runtime, if we consider the enormous runtimes often spanning multiple hours. The added overhead compared to the greedy baselines is miniscule at 1\%.These results mark a step forward for universally applicable performance regression testing techniques.},
	keywords = {JMH, Multi-objective optimization, Performance Testing, Regression testing, Search-Based Software Engineering, software microbenchmarking, Test Case Prioritization},
	doi = {10.48550/ARXIV.2211.13525},
	url = {https://arxiv.org/abs/2211.13525},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42902,
	title = {Mutation-Based Test Generation for Quantum Programs with Multi-Objective Search},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {1345-1353},
	publisher = {ACM},
	abstract = {Mutation testing is often used for designing new tests, and involves changing a program in minor ways, which results in mutated versions of the program, i.e., mutants. An effective test suite should find faults (or kill mutants) with a minimum number of test cases, to save resources required for executing test cases. In this paper, in the context of mutation testing for quantum programs, we present a multi-objective and search-based approach (MutTG) to generate the minimum number of test cases killing as many mutants as possible. MutTG tries to estimate the likelihood that a mutant is equivalent, and uses this as a\&nbsp;discount factor\&nbsp;in the fitness definition to avoid keeping on trying to kill mutants that cannot be killed. We employed NSGA-II as the multi-objective search algorithm. Then, we compared MutTG with another version of the approach that does not use the discount factor in its fitness definition, and with random search (RS), over a set of open-source quantum programs and their mutants of varying complexity. Results show that the discount factor does indeed help in guiding the test generation, as the approach with the discount factor performs better than the one without it.},
	doi = {10.1145/3512290.3528869},
	url = {https://dl.acm.org/doi/abs/10.1145/3512290.3528869},
	author = {Wang, Xinyi and Yu, Tongxuan and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@article {28417,
	title = {On the Preferences of Quality Indicators for Multi-Objective Search Algorithms in Search-Based Software Engineering},
	journal = {The Empirical Software Engineering Journal (EMSE)},
	volume = {27},
	year = {2022},
	pages = {144},
	publisher = { Springer},
	author = {Wu, Jiahui and Arcaini, Paolo and Yue, Tao and Ali, Shaukat and Zhang, Huihui}
}
@book {42863,
	title = {Quantum Computing - Introduction to the special theme},
	volume = {128},
	year = {2022},
	publisher = {ERCIM},
	author = {Ali, Shaukat and Selst{\o}, S{\o}lve}
}
@misc {42884,
	title = {Quantum Software Engineering},
	howpublished = {Simula Research Laboratory, Norway},
	year = {2022},
	type = {Seminar with Lionel Briand},
	author = {Ali, Shaukat}
}
@misc {42981,
	title = {Quantum Software Engineering: What, Why, and Where are we now?},
	howpublished = {Quantum Leap and Mathematics, Soria Moria, Oslo, Norway},
	year = {2022},
	author = {Ali, Shaukat}
}
@article {28248,
	title = {Quantum Software Testing: Challenges, Early Achievements, and Opportunities},
	journal = {the European Research Consortium for Informatics and Mathematics News},
	year = {2022},
	publisher = {ERCIM},
	author = {Yue, Tao and Arcaini, Paolo and Ali, Shaukat}
}
@misc {42895,
	title = {Quantum Software Testing Tutorial},
	howpublished = {NordIQuEst-ENCCS online HPC-QC workshop},
	year = {2022},
	author = {Ali, Shaukat and Wang, Xinyi}
}
@inproceedings {42901,
	title = {QuSBT: Search-Based Testing of Quantum Programs},
	journal = {2022 IEEE/ACM 44th International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)},
	year = {2022},
	publisher = {IEEE},
	abstract = {Generating a test suite for a quantum program such that it has the maximum number of failing tests is an optimization problem. For such optimization, search-based testing has shown promising results in the context of classical programs. To this end, we present a test generation tool for quantum programs based on a genetic algorithm, called QuSBT (Search-based Testing of Quantum Programs). QuSBT automates the testing of quantum programs, with the aim of finding a test suite having the maximum number of failing test cases. QuSBT utilizes IBM{\textquoteright}s Qiskit as the simulation framework for quantum programs. We present the tool architecture in addition to the implemented methodology (i.e., the encoding of the search individual, the definition of the fitness function expressing the search problem, and the test assessment w.r.t. two types of failures). Finally, we report results of the experiments in which we tested a set of faulty quantum programs with QuSBT to assess its effectiveness. 
Repository (code and experimental results): https://github.com/Simula-COMPLEX/qusbt-tool 
Video: https://youtu.be/3apRCtluAn4},
	doi = {10. 1145/3510454.3516839},
	url = {https://ieeexplore.ieee.org/document/9793826},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@misc {42893,
	title = {Time for new Simula!},
	howpublished = {Towards a Norwegian Quantum Computing Strategy, Simula Research Laboratory, Norway},
	year = {2022},
	author = {Ali, Shaukat}
}
@misc {42890,
	title = { Towards Requirements Engineering for Digital Twins of Cyber-Physical Systems},
	howpublished = {In 11th International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	year = {2022},
	publisher = {LNCS},
	author = {Ali, Shaukat}
}
@inproceedings {42861,
	title = {Towards Requirements Engineering for Digital Twins of Cyber-Physical Systems},
	journal = {11th International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	year = {2022},
	publisher = {LNCS},
	author = {Yue, Tao and Ali, Shaukat and Arcaini, Paolo and Ishikawa, Fuyuki and Cl{\'a}udio {\^A}ngelo Go {Gomes}},
	editor = {Fitzgerald, John and Margaria, Tiziana and Peter Gorm {Larsen} and Woodcock, Jim}
}
@misc {42881,
	title = {Uncertainty in Deep Learning},
	howpublished = {Simula Research Laboratory, Norway},
	year = {2022},
	author = {Ali, Shaukat}
}
@article {28416,
	title = {Uncertainty-Aware Prediction Validator in Deep Learning Models for Cyber-Physical System Data},
	journal = {ACM Transactions on Software Engineering and Methodology (TOSEM)},
	volume = {31},
	number = {4},
	year = {2022},
	pages = {1-31},
	publisher = {ACM},
	author = {Ferhat Ozgur {Catak} and Yue, Tao and Ali, Shaukat}
}
@article {42860,
	title = {Uncertainty-aware Robustness Assessment of Industrial Elevator Systems},
	journal = {ACM Transactions on Software Engineering and Methodology},
	year = {2022},
	publisher = {ACM},
	doi = {10.1145/3576041},
	url = {https://doi.org/10.1145/3576041},
	author = {Han, Liping and Ali, Shaukat and Yue, Tao and Arrieta, Aitor and Arratibel, Maite},
	editor = {Pezze, Mauro}
}
@techreport {28493,
	title = {Uncertainty-aware Robustness Assessment of Industrial Elevator Systems},
	year = {2022},
	publisher = {Simula Research Laboratory},
	author = {Han, Liping and Ali, Shaukat and Yue, Tao and Arrieta, Aitor and Arratibel, Maite}
}
@inproceedings {42666,
	title = {Uncertainty-Aware Transfer Learning to Evolve Digital Twins for Industrial Elevators},
	journal = {The ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE)},
	year = {2022},
	publisher = {ACM},
	abstract = {Digital twins are increasingly developed to support the development, operation, and maintenance of cyber-physical systems such as industrial elevators. However, industrial elevators continuously evolve due to changes in physical installations, introducing new software features, updating existing ones, and making changes due to regulations (e.g., enforcing restricted elevator capacity due to COVID-19), etc. Thus, digital twin functionalities (often built on neural network-based models) need to evolve themselves constantly to be synchronized with the industrial elevators. Such an evolution is preferred to be automated, as manual evolution is time-consuming and error-prone. Moreover, collecting sufficient data to re-train neural network models of digital twins could be expensive or even infeasible. To this end, we propose unceRtaInty-aware tranSfer lEarning enriched Digital Twins LATTICE, a\&nbsp;transfer learning\&nbsp;based approach capable of transferring knowledge about the waiting time prediction capability of a digital twin of an industrial elevator across different scenarios. LATTICE also leverages\&nbsp;uncertainty quantification\&nbsp;to further improve its effectiveness. To evaluate LATTICE, we conducted experiments with 10 versions of an elevator dispatching software from Orona, Spain, which are deployed in a Software in the Loop (SiL) environment. Experiment results show that LATTICE, on average, improves the Mean Squared Error by 13.131\% and the utilization of uncertainty quantification further improves it by 2.71\%.},
	doi = {https://doi.org/10.1145/3540250.3558957},
	author = {Xu, Qinghua and Ali, Shaukat and Yue, Tao and Arratibel, Maite}
}
@article {28247,
	title = {When Software Engineering Meets Quantum Computing},
	journal = {Communications of ACM},
	volume = {65},
	year = {2022},
	pages = {84-88},
	publisher = {ACM},
	author = {Ali, Shaukat and Yue, Tao and Abreu, Rui}
}
@misc {42879,
	title = {WTT4Oslo: Improving Quality of IoT-based Welfare Technology Solutions in the City of Oslo},
	howpublished = {Simula KA23, Oslo, Norway},
	year = {2022},
	author = {Kousias, Konstantinos},
	editor = {Ali, Shaukat and Yue, Tao}
}
@book {28291,
	title = {2021 IEEE/ACM 2nd International Workshop on Quantum Software Engineering (Q-SE)},
	year = {2021},
	publisher = {IEEE/ACM},
	edition = {Second},
	isbn = {978-1-6654-4462-0},
	editor = {Ali, Shaukat and Abreu, Rui and Arrieta, Aitor}
}
@inproceedings {27879,
	title = {Analyzing the Impact of Product Configuration Variations on Advanced Driver Assistance Systems with Search},
	journal = {The Genetic and Evolutionary Computation Conference},
	year = {2021},
	pages = {1106{\textendash}1114},
	publisher = {ACM},
	author = {Yin, Kaiou and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@misc {27878,
	title = {Anomaly Detection with Digital Twin in Cyber-Physical Systems},
	howpublished = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	type = {Paper Presentation},
	author = {Xu, Qinghua and Ali, Shaukat and Yue, Tao}
}
@inproceedings {27571,
	title = {Anomaly Detection with Digital Twin in Cyber-Physical Systems},
	journal = {2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Cyber-Physical Systems (CPSs) are susceptible to various anomalies during their operations. Thus, it is important to detect such anomalies. Detecting such anomalies is challenging since it is uncertain when and where anomalies can happen. To this end, we present a novel approach called Anomaly deTection with digiTAl twIN (ATTAIN), which continuously and automatically builds a digital twin with live data obtained from a CPS for anomaly detection. ATTAIN builds a Timed Automaton Machine (TAM) as the digital representation of the CPS, and implements a Generative Adversarial Network (GAN) to detect anomalies. GAN uses a GCN-LSTM-based module as a generator, which can capture temporal and spatial characteristics of the input data and learn to produce realistic unlabeled fake samples. TAM labels these fake samples, which are then fed into a discriminator along with real labeled samples. After training, the discriminator is capable of distinguishing anomalous data from normal data with a high F1 score. To evaluate our approach, we used three publicly available datasets collected from three CPS testbeds. Evaluation results show that ATTAIN improved the performance of two state-of-art anomaly detection methods by 2.413\%, 8.487\% and 5.438\% on average on the three datasets, respectively. Moreover, ATTAIN achieved on average 8.39\% increase in the anomaly detection capability with digital twins as compared with an approach of not using digital twins.\&nbsp; \&nbsp; \&nbsp;\&nbsp;},
	doi = {10.1109/ICST49551.2021.00031},
	author = {Xu, Qinghua and Ali, Shaukat and Yue, Tao}
}
@misc {28261,
	title = {Application of Artificial Intelligence Techniques for Requirements Optimization, Learning, and Evolution for Cyber-Physical Systems},
	howpublished = {An RE{\textquoteright}21 Workshop on Environment-Driven Requirements Engineering},
	year = {2021},
	author = {Ali, Shaukat}
}
@inproceedings {28244,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {IEEE International Conference on Software Security and Reliability},
	year = {2021},
	publisher = { IEEE},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat},
	editor = {Wotawa, Franz and Wang, Xianpeng}
}
@inproceedings {42899,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)},
	year = {2021},
	publisher = {IEEE},
	abstract = {The capability of Quantum Computing (QC) in solving complex problems has been increasingly recognized. However, similar to classical computing, to fully exploit QC{\textquoteright}s potential, it is important to ensure the correctness of quantum programs. Doing so via software testing is, however, very challenging because of QC{\textquoteright}s inherent properties: superposition and entanglement. Towards the direction of ensuring the correctness of quantum programs, we propose an approach called QuCAT (QUantum CombinAtorial Testing) for systematic and automated testing of quantum programs by benefiting from combinatorial testing, which has been proven to be cost-effective in testing classical programs. QuCAT supports two combinatorial test suite generation scenarios, i.e., generating combinatorial test suites of a given strength, and incrementally generating and executing combinatorial test suites of increasing strength until a fault is found. The approach employs two types of test oracles to assess test results. We performed an empirical study with 18 faulty versions of quantum programs to evaluate QuCAT with strengths of two, three, and four in the two test generation scenarios. We compare the cost-effectiveness of combinatorial testing of various strengths and random testing (taken as baseline approach). Results show that combinatorial testing always performs better than random testing with the same cost and finds faults more quickly (in terms of required number of test cases). In addition, in most cases, combinatorial testing with a higher strength outperforms the lower strength in terms of effectiveness.},
	doi = {10.1109/QRS54544.2021.00029},
	url = {https://ieeexplore.ieee.org/document/9724888},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27579,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	pages = {13-23},
	publisher = { IEEE},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao},
	editor = {Hierons, Robert and Ferrari, Fabiano}
}
@inproceedings {42889,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Quantum programs implement quantum algorithms solving complex computational problems. Testing such programs is challenging due to the inherent characteristics of Quantum Computing (QC), such as the probabilistic nature and computations in superposition. However, automated and systematic testing is needed to ensure the correct behavior of quantum programs. To this end, we present an approach called Quito (QUantum InpuT Output coverage) consisting of three coverage criteria defined on the inputs and outputs of a quantum program, together with their test generation strategies. Moreover, we define two types of test oracles, together with a procedure to determine the passing and failing of test suites with statistical analyses. To evaluate the cost-effectiveness of the three coverage criteria, we conducted experiments with five quantum programs. We used mutation analysis to determine the coverage criteria{\textquoteright} effectiveness and cost in terms of the number of test cases. Based on the results of mutation analysis, we also identified equivalent mutants for quantum programs.},
	doi = {10.1109/ICST49551.2021.00014},
	url = {https://ieeexplore.ieee.org/document/9438603},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao}
}
@misc {28287,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	howpublished = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	publisher = { IEEE},
	type = {Conference Paper},
	author = {Ali, Shaukat}
}
@misc {27882,
	title = {Automated, Systematic, and Optimized Testing of Quantum Programs},
	howpublished = {Gemini Center},
	year = {2021},
	type = {Invited Talk},
	author = {Ali, Shaukat}
}
@misc {27883,
	title = {Black Box Testing of Quantum Programs},
	howpublished = {QC Talks, University of Porto, Portugal},
	year = {2021},
	author = {Ali, Shaukat}
}
@misc {27880,
	title = {Do Quality Indicators Prefer Particular Multi-Objective Search Algorithms in Search-Based Software Engineering?	(Hot Off the Press)},
	howpublished = {The Genetic and Evolutionary Computation Conference},
	year = {2021},
	type = {Hot Off the Press},
	author = {Ali, Shaukat and Arcaini, Paolo and Yue, Tao}
}
@article {27940,
	title = {First International Workshop on Quantum Software Engineering (Q-SE 2020)},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {46},
	year = {2021},
	pages = {30-32},
	publisher = {ACM},
	author = {Abreu, Rui and Ali, Shaukat and Yue, Tao}
}
@inproceedings {27924,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {13th Symposium on Search-Based Software Engineering},
	year = {2021},
	publisher = {LNCS},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42898,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {Symposium on Search Base Software Engineering 2021},
	year = {2021},
	abstract = {Testing quantum programs requires systematic, automated, and intelligent methods due to their inherent complexity, such as their superposition and entanglement. To this end, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size depending on available testing budget, with the aim of maximizing the number of failing test cases in the test suite. QuSBT consists of definitions of the problem encoding, failure types, test assessment with statistical tests, fitness function, and test case generation with a Genetic Algorithm (GA). To empirically evaluate QuSBT, we compared it with Random Search (RS) by testing six quantum programs. We assessed the effectiveness of QuSBT and RS with 30 carefully designed faulty versions of the six quantum programs. Results show that QuSBT provides a viable solution for testing quantum programs, and achieved a significant improvement over RS in 87\% of the faulty programs, and no significant difference in the rest of 13\% of the faulty programs.},
	doi = {10.1007/978-3-030-88106-1_2},
	url = {https://link.springer.com/chapter/10.1007/978-3-030-88106-1_2},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27929,
	title = {Genetic Algorithm-based Testing of Industrial Elevators under Passenger Uncertainty},
	journal = {The 32nd International Symposium on Software Reliability Engineering (ISSRE 2021)},
	year = {2021},
	publisher = { IEEE},
	author = {Galarraga, Joritz and Arrieta, Aitor and Ali, Shaukat and Sagardui, Goiuria and Arratibel, Maite}
}
@misc {28281,
	title = {Genetic Algorithm-based Testing of Industrial Elevators under Passenger Uncertainty},
	howpublished = {The 32nd International Symposium on Software Reliability Engineering (ISSRE 2021)},
	year = {2021},
	publisher = { IEEE},
	type = {Conference Paper},
	author = {Ali, Shaukat}
}
@inproceedings {27888,
	title = {Microservices for Continuous Deployment, Monitoring and Validation in Cyber-Physical Systems: an Industrial Case Study for Elevators Systems},
	journal = {ICSA Software Architecture in Practice (SAIP) },
	year = {2021},
	pages = {46-53},
	publisher = { IEEE},
	author = {Gartziandia, Aitor and Arrieta, Aitor and Ali, Shaukat and Yue, Tao and Agirre, Aitor and Sagardui, Goiuria and Arratibel, Maite}
}
@inproceedings {27927,
	title = {Muskit: A Mutation Analysis Tool for Quantum Software Testing},
	journal = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = { IEEE/ACM},
	author = {Mendiluze, E{\~n}aut and Ali, Shaukat and Arcaini, Paolo and Yue, Tao}
}
@techreport {27892,
	title = {Muskit: A Mutation Analysis Tool for Quantum Software Testing},
	year = {2021},
	publisher = {Technical Report, Simula Research Laboratory},
	abstract = {Given that quantum software testing is a new area of research, there is a lack of benchmark programs and bug repositories to assess the effectiveness of testing techniques. To this end, quantum mutation analysis focuses on systematically generating faulty versions of Quantum Programs (QPs), called mutants, using mutation operators. Such mutants can be used as benchmarks to assess the quality of test cases in a test suite. Thus, we present muskit -- a quantum mutation analysis tool for QPs coded in IBM{\textquoteright}s Qiskit language. muskit defines mutation operators on gates of QPs and selection criteria to reduce the number of mutants to generate. Moreover, it allows for the execution of test cases on mutants and the generation of results for test analyses. muskit is provided as a command-line interface, GUI, and web application. We validated muskit by using it to generate and execute mutants for four QPs.},
	author = {Mendiluze, E{\~n}aut and Ali, Shaukat and Arcaini, Paolo and Yue, Tao}
}
@inproceedings {27881,
	title = {Parameter-Based Testing and Debugging of Autonomous Driving Systems},
	journal = {2021 IEEE Intelligent Vehicles Symposium (IV)},
	year = {2021},
	publisher = {IEEE},
	author = {Arcaini, Paolo and Cal{\v n}, Alessandro and Ishikawa, Fuyuki and Laurent, Thomas and Zhang, Xiao-Yi and Ali, Shaukat and Hauer, Florian and Ventresque, Anthony}
}
@inproceedings {27911,
	title = {Prediction Surface Uncertainty Quantification in Object Detection Models for Autonomous Driving},
	journal = {2021 IEEE International Conference on Artificial Intelligence Testing (AITest)},
	year = {2021},
	pages = {93-100},
	publisher = { IEEE},
	address = {Oxford, England},
	abstract = {Object detection in autonomous cars is commonly based on camera images and Lidar inputs, which are often used to train prediction models such as deep artificial neural networks for decision making for object recognition, adjusting speed, etc. A mistake in such decision making can be damaging; thus, it is vital to measure the reliability of decisions made by such prediction models via uncertainty measurement. Uncertainty, in deep learning models, is often measured for classification problems. However, deep learning models in autonomous driving are often multi-output regression models. Hence, we propose a novel method called PURE (Prediction sURface uncErtainty) for measuring prediction uncertainty of such regression models. We formulate the object recognition problem as a regression model with more than one outputs for finding object locations in a 2-dimensional camera view. For evaluation, we modified three widely-applied object recognition models (i.e., YoLo, SSD300 and SSD512) and used the KITTI, Stanford Cars, Berkeley DeepDrive, and NEXET datasets. Results showed the statistically significant negative correlation between prediction surface uncertainty and prediction accuracy suggesting that uncertainty significantly impacts the decisions made by autonomous driving.},
	doi = {10.1109/AITEST52744.2021.00027},
	author = {Ferhat Ozgur {Catak} and Yue, Tao and Ali, Shaukat}
}
@misc {27876,
	title = {Quality Indicators in Search-Based Software Engineering: An Empirical Evaluation},
	howpublished = {43rd International Conference on Software Engineering},
	year = {2021},
	publisher = { IEEE},
	type = {Journal First Presentation},
	author = {Ali, Shaukat and Arcaini, Paolo and Pradhan, Dipesh and Safdar Aqeel {Safdar} and Yue, Tao}
}
@article {28249,
	title = {Quantum Software: Model-driven or Search-driven? A Q-SE 2021 Workshop Report},
	journal = { ACM SIGSOFT Software Engineering Notes},
	volume = {46},
	year = {2021},
	pages = {23-25},
	publisher = {ACM},
	author = {Abreu, Rui and Ali, Shaukat and Yue, Tao and Felderer, Michael and Exman:, Iaakov}
}
@inproceedings {42900,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Automation in quantum software testing is essential to support systematic and cost-effective testing. Towards this direction, we present a quantum software testing tool called Quito that can automatically generate test suites covering three coverage criteria defined on inputs and outputs of a quantum program coded in Qiskit, i.e., input coverage, output coverage, and input-output coverage. Quito also implements two types of test oracles based on program specifications, i.e., checking whether a quantum program produced a wrong output or checking a probabilistic test oracle with statistical test. We describe the architecture and methodology of the tool. We also validated the tool with one quantum program and one faulty version of it. Results indicate that Quito can generate test suites and perform test assessments that detect faults, and produce test results with a good time performance.
Quito{\textquoteright}s code: https://github.com/Simula-COMPLEX/quito
Quito{\textquoteright}s video: https://youtu.be/kuI9QaCo8A8Artifact Available: https://doi.org/10.5281/zenodo.5288665},
	doi = {0.1109/ASE51524.2021.9678798},
	url = {https://ieeexplore.ieee.org/document/9678798},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27928,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = {IEEE/ACM},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@article {27884,
	title = {Recommending Faulty Configurations for Interacting Systems Under Test Using Multi-Objective Search},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {30},
	year = {2021},
	pages = {1-36},
	publisher = {ACM},
	doi = {https://doi.org/10.1145/3464939},
	author = {Safdar Aqeel {Safdar} and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27926,
	title = {Restricted Natural Language and Model-based Adaptive Test Generation for Autonomous Driving},
	journal = {ACM/IEEE 24th International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	year = {2021},
	publisher = {ACM/IEEE},
	author = {Shi, Yize and Lu, Chengjie and Zhang, Man and Zhang, Huihui and Yue, Tao and Ali, Shaukat}
}
@misc {27944,
	title = {The Role of Quantum Software Engineering in the Second Quantum Revolution },
	howpublished = {Communications of the ACM Europe Region Special Section Virtual Workshop},
	year = {2021},
	author = {Ali, Shaukat and Yue, Tao and Abreu, Rui}
}
@inproceedings {27925,
	title = {Search-based Selection and Prioritization of Test Scenarios for Autonomous Driving Systems},
	journal = {13th Symposium on Search-Based Software Engineering},
	year = {2021},
	publisher = {LNCS Springer},
	author = {Lu, Chengjie and Zhang, Huihui and Yue, Tao and Ali, Shaukat}
}
@article {27668,
	title = {Testing Self-Healing Cyber-Physical Systems under Uncertainty with Reinforcement Learning: An Empirical Study},
	journal = {Empirical Software Engineering},
	volume = {26},
	year = {2021},
	pages = {52},
	publisher = { Springer},
	doi = {https://doi.org/10.1007/s10664-021-09941-z},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao}
}
@misc {28288,
	title = {Understanding Digital Twins for Cyber-Physical Systems: A Conceptual Model},
	howpublished = {International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	year = {2021},
	publisher = {LNCS},
	type = {Conference Paper},
	author = {Ali, Shaukat}
}
@inproceedings {27524,
	title = {Understanding Digital Twins for Cyber-Physical Systems: A Conceptual Model},
	journal = {International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	volume = {12479},
	year = {2021},
	pages = {54-71},
	publisher = {LNCS, Springer, Cham},
	doi = {https://doi.org/10.1007/978-3-030-83723-5_5},
	author = {Yue, Tao and Arcaini, Paolo and Ali, Shaukat},
	editor = {Fitzgerald, John and Peter Gorm {Larsen}}
}
@inproceedings {27406,
	title = {Do Quality Indicators Prefer Particular Multi-Objective Search Algorithms in Search-Based Software Engineering?},
	journal = {12th Symposium on Search-Based Software Engineering},
	year = {2020},
	publisher = {LNCS},
	abstract = {In Search-Based Software Engineering (SBSE), users typically select a set of Multi-Objective Search Algorithms (MOSAs) for their experiments without any justification, or they simply choose an MOSA because of its popularity (e.g., NSGA-II). On the other hand, users know certain characteristics of solutions they are interested in. Such characteristics are typically measured with Quality Indicators (QIs) that are commonly used to evaluate the quality of solutions produced by an MOSA. Consequently, these QIs are often employed to empirically evaluate a set of MOSAs for a particular search problem to find the best MOSA. Thus, to guide SBSE users in choosing an MOSA that represents the solutions measured by a specific QI they are interested in, we present an empirical evaluation with a set of SBSE problems to study the relationships among commonly used QIs and MOSAs in SBSE. Our aim, by studying such relationships, is to identify whether there are certain characteristics of a QI because of which it prefers a certain MOSA. Such preferences are then used to provide insights and suggestions to SBSE users in selecting an MOSA, given that they know which quality aspects of solutions they are looking for.},
	author = {Ali, Shaukat and Arcaini, Paolo and Yue, Tao},
	editor = {Aleti, Aldeida and Panichella, Annibale}
}
@misc {27409,
	title = {Empowering Model-based Engineering with the CynefinFramework for Systematic Uncertainty Thinking},
	howpublished = {1st Uncertainty in Modeling Workshop 2020, Co-located with the ACM/IEEE 213rd International Conference on Model Driven Engineering Languages and Systems},
	year = {2020},
	author = {Yue, Tao and Ali, Shaukat}
}
@article {27522,
	title = {A framework for automated multi-stage and multi-step product confguration of cyber-physical systems},
	journal = {Software and Systems Modeling (SoSym)},
	volume = {19},
	year = {2020},
	month = {06/2020},
	pages = {1-55},
	publisher = {Springer},
	abstract = {Product line engineering (PLE) has been employed to large-scale cyber-physical systems (CPSs) to provide customization based on users{\textquoteright} needs. A PLE methodology can be characterized by its support for capturing and managing the abstractions as commonalities and variabilities and the automation of the confguration process for efective selection and customization of reusable artifacts. The automation of a confguration process heavily relies on the captured abstractions and formally specifed constraints using a well-defned modeling methodology. Based on the results of our previous work and a thorough literature review, in this paper, we propose a conceptual framework to support multi-stage and multi-step automated product confguration of CPSs, including a comprehensive classifcation of constraints and a list of automated functionalities of a CPS confguration solution. Such a framework can serve as a guide for researchers and practitioners to evaluate an existing CPS PLE solution or devise a novel CPS PLE solution. To validate the framework, we conducted three real-world case studies. Results show that the framework fulflls all the requirements of the case studies in terms of capturing and managing variabilities and constraints. Results of the literature review indicate that the framework covers all the functionalities concerned by the literature, suggesting that the framework is complete for enabling the maximum automation of confguration in CPS PLE.},
	keywords = {Automated configuration, Constraint classification, Cyber-Physical Systems, Multi-stage and multi-step configuration process, Product Line Engineering, Real-world case studies, Variability Modeling},
	doi = {https://doi.org/10.1007/s10270-020-00803-8},
	author = {Safdar Aqeel {Safdar} and Lu, Hong and Yue, Tao and Ali, Shaukat and Nie, Kunming}
}
@inproceedings {27145,
	title = {Generating Avoidable Collision Scenarios for Testing Autonomous Driving Systems},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST) 2020},
	year = {2020},
	pages = {375-386},
	publisher = { IEEE},
	author = {Cal{\`o}, Alessandro and Arcaini, Paolo and Ali, Shaukat and Hauer, Florian and Ishikawa, Fuyuki},
	editor = {Micco, John and Pistoia, Marco}
}
@misc {27351,
	title = {Handling Uncertainties in Cyber-Physical Systems during TheirOperations with Digital Twins},
	howpublished = {5th Workshop on Monitoring and Testing of Cyber-Physical Systems (MT-CPS)},
	year = {2020},
	author = {Ali, Shaukat and Yue, Tao}
}
@techreport {27306,
	title = {Handling Uncertainties in Cyber-Physical Systems during TheirOperations with Digital Twins},
	year = {2020},
	publisher = {Simula Research Laboratory},
	abstract = {It is a well-recognized fact that a Cyber-Physical System (CPS) experiences uncertain (including unknown) situations during their operations. Some of such uncertainties could potentially lead to failures of CPS operations. Factors contribute to such uncertainties include 1) the intrinsically unpredictable physical environment of a CPS, 2) the use of communication networks continuously experiencing problems (e.g., slower connection than expected), and 3) the increasing use of machine learning algorithms in CPSs which introduce inherent uncertainties to these CPSs. No matter how meticulously a CPS is designed and developed, it is impossible to predict all possible uncertain situations it will experience during its operation. Thus, there is a need for new methods for discovering and handling uncertain situations during the CPS operation to prevent it from failure. In this paper, we present our ideas on how digital twins, i.e., "live models" of CPSs can help in discovering and handling potentially unsafe situations during its operation. We present the research challenges and potential solutions to develop, deploy, and operate such digital twins. The presented work is planned to be performed in a recently accepted European research-based innovation project, ADEPTNESS.\&nbsp;},
	author = {Ali, Shaukat and Yue, Tao}
}
@misc {27408,
	title = {Modeling Foundations for Executable Model-Based Testing of Self-Healing Cyber-Physical Systems},
	howpublished = {IEEE International Conference on Software Testing, Verification and Validation (ICST 2020)},
	year = {2020},
	type = { Journal First Presentation},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao}
}
@inproceedings {27523,
	title = {Modeling Quantum Programs: Challenges, Initial Results, and Research Directions},
	journal = {1st International Workshop on Architectures and Paradigms for Engineering Quantum Software (APEQS)},
	year = {2020},
	publisher = {ACM},
	author = {Ali, Shaukat and Yue, Tao},
	editor = {Mauerer, Wolfgang and Feld, Sebastian and Linnhoff-Popien, Claudia}
}
@article {27209,
	title = {Pattern-based Interactive Configuration Derivation for Cyber-Physical System Product Lines},
	journal = {ACM Transactions on Cyber-Physical Systems},
	volume = {4},
	year = {2020},
	pages = {24},
	publisher = {ACM},
	address = {New York, NY, USA},
	issn = {2378-962X},
	doi = {10.1145/3389397},
	url = {https://doi.org/10.1145/3389397},
	author = {Lu, Hong and Yue, Tao and Ali, Shaukat}
}
@article {27132,
	title = {Quality Indicators in Search-based Software Engineering},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {29},
	year = {2020},
	month = {May-04-2020},
	pages = {1 - 29},
	publisher = {ACM},
	address = {New York, NY, USA},
	issn = {1049-331X},
	doi = {10.1145/3375636},
	url = {https://dl.acm.org/doi/10.1145/3375636},
	author = {Ali, Shaukat and Arcaini, Paolo and Pradhan, Dipesh and Safdar Aqeel {Safdar} and Yue, Tao}
}
@article {27812,
	title = {Quality indicators in search-based software engineering: an empirical evaluation},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {29},
	number = {2},
	year = {2020},
	month = {04/2020},
	publisher = {ACM},
	abstract = {Search-Based Software Engineering (SBSE) researchers who apply multi-objective search algorithms (MOSAs) often assess the quality of solutions produced by MOSAs with one or more quality indicators (QIs). However, SBSE lacks evidence providing insights on commonly used QIs, especially about agreements among them and their relations with SBSE problems and applied MOSAs. Such evidence about QIs agreements is essential to understand relationships among QIs, identify redundant QIs, and consequently devise guidelines for SBSE researchers to select appropriate QIs for their specific contexts. To this end, we conducted an extensive empirical evaluation to provide insights on commonly used QIs in the context of SBSE, by studying agreements among QIs with and without considering differences of SBSE problems and MOSAs. In addition, by defining a systematic process based on three common ways of comparing MOSAs in SBSE, we present additional observations that were automatically produced based on the results of our empirical evaluation. These observations can be used by SBSE researchers to gain a better understanding of the commonly used QIs in SBSE, in particular, regarding their agreements. Finally, based on the results, we also provide a set of guidelines for SBSE researchers to select appropriate QIs for their particular context.},
	author = {Ali, Shaukat and Arcaini, Paolo and Pradhan, Dipesh and Safdar Aqeel {Safdar} and Yue, Tao}
}
@inproceedings {27344,
	title = {Simultaneously Searching and Solving Multiple Avoidable Collisions for Testing Autonomous Driving Systems},
	journal = {The Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2020},
	publisher = {ACM},
	author = {Cal{\`o}, Alessandro and Arcaini, Paolo and Ali, Shaukat and Hauer, Florian and Ishikawa, Fuyuki}
}
@misc {27350,
	title = {Specifying Uncertainty in Use Case Models},
	howpublished = {International Conference on Model-Driven Engineering and Software Development (MODELSWARD 2020)},
	year = {2020},
	publisher = {INSTICC },
	author = {Ali, Shaukat and Yue, Tao}
}
@misc {27429,
	title = {Testing Quantum Programs},
	howpublished = {Oslo Metropolitan University, Oslo, Norway},
	year = {2020},
	author = {Ali, Shaukat}
}
@article {27362,
	title = {Uncertainty-wise Requirements Prioritization with Search},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {30},
	year = {2020},
	month = {December 2020},
	pages = {1-54},
	publisher = {ACM},
	address = {New York, NY, USA},
	issn = {1049-331X},
	doi = {10.1145/3408301},
	url = {https://doi.org/10.1145/3408301},
	author = {Zhang, Huihui and Zhang, Man and Yue, Tao and Ali, Shaukat and Yan, Li},
	editor = {Breaux, Travis}
}
@book {26942,
	title = {The 15th European Conference on Modelling Foundations and Applications (ECMFA)},
	volume = {18},
	year = {2019},
	month = {09/2019},
	publisher = {The Journal of Object Technology},
	edition = {2},
	doi = {10.5381/jot.2019.18.2.e1},
	url = {http://www.jot.fm/contents/issue_2019_02.html},
	author = {Combemale, Benoit and Ali, Shaukat}
}
@inproceedings {26920,
	title = {Big data from the cloud to the edge: the aggregate computing solution},
	journal = {Proceedings of the 13th European Conference on Software Architecture},
	volume = {2},
	year = {2019},
	pages = {177-180},
	publisher = {ACM},
	address = {New York, NY, USA},
	doi = {https://doi.org/10.1145/3344948.3344988},
	url = {https://dl.acm.org/citation.cfm?id=3344988\&dl=ACM\&coll=DL},
	author = {Ali, Shaukat and Damiani, Ferruccio and Dustdar, Schahram and Sanseverino, Marialuisa and Viroli, Mirko and Weyns, Danny},
	editor = {Duchien, Laurence and Trubiani, Catia}
}
@misc {27430,
	title = {Digital Twins for Cyber-Physical Systems, and Quantum Software Engineering: Research Agenda},
	howpublished = {National Institute of Informatics, Tokyo, Japan},
	year = {2019},
	type = {Colloquium },
	author = {Ali, Shaukat}
}
@article {26506,
	title = {Employing Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	journal = {Journal of Systems and Software},
	volume = {153},
	year = {2019},
	month = {07/2019},
	pages = {86-104},
	publisher = {Elsevier},
	abstract = {Test case prioritization (TP) is widely used in regression testing for optimal reordering of test cases to achieve specific criteria (e.g., higher fault detection capability) as early as possible. In our earlier work, we proposed an approach for black-box dynamic TP using rule mining and multi-objective search (named as REMAP) by defining two objectives (fault detection capability and test case reliance score) and considering test case execution results at runtime. In this paper, we conduct an extensive empirical evaluation of REMAP by employing three different rule mining algorithms and three different multi-objective search algorithms, and we also evaluate REMAP with one additional objective (estimated execution time) for a total of 18 different configurations (i.e., 3 rule mining algorithms {\texttimes}  3 search algorithms {\texttimes}  2 different set of objectives) of REMAP. Specifically, we empirically evaluated the 18 variants of REMAP with 1) two variants of random search while using two objectives and three objectives, 2) three variants of greedy algorithm based on one objective, two objectives, and three objectives, 3) 18 variants of static search-based prioritization approaches, and 4) six variants of rule-based prioritization approaches using two industrial and three open source case studies. Results showed that the two best variants of REMAP with two objectives and three objectives significantly outperformed the best variants of competing approaches by 84.4\% and 88.9\%, and managed to achieve on average 14.2\% and 18.8\% higher Average Percentage of Faults Detected per Cost (APFDc) scores.},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@misc {26955,
	title = {Enabling automated requirements reuse and configuration},
	howpublished = {23rd International Systems and Software Product Line Conference, Paris, France},
	year = {2019},
	publisher = {ACM},
	type = {Paris, France},
	doi = {https://doi.org/10.1145/3336294.3342370},
	author = {Li, Yan and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@book {26560,
	title = {Evaluation and Assessment in Software Engineering 2019},
	year = {2019},
	publisher = {ACM},
	author = {Garousi, Vahid and Ali, Shaukat}
}
@inproceedings {26646,
	title = {Experiences of studying Attention through EEG in the Context of Review Tasks},
	journal = {EASE {\textquoteright}19: Proceedings of the Evaluation and Assessment on Software Engineering},
	year = {2019},
	pages = {313-318},
	publisher = {ACM Press},
	address = {New York, NY, USA},
	abstract = {Context: Electroencephalograms (EEG) have been used in a few cases in the context of software engineering (SE). EEGs allow capturing emotions and cognitive functioning. Such human factors have already shown to be important to understand software engineering tasks. Therefore, it is essential to gain experience in the community to utilize EEG as a research tool. Objective: To report experiences of using EEG in the context of a software engineering education (review of master theses proposals). We provide our reflections and lessons learned of (1) how to plan an EEG study, (2) how to conduct and execute (e.g., tools), (3) how to analyze. Method: We carried out an experiment using an EEG headset to measure the participants{\textquoteright} attention rate. The experiment task includes reviewing three master thesis project plans. Results: We describe how we evolved our understanding of experimentation practices to collect and analyze psychological and cognitive data. We also provide a set of lessons learned regarding the application of EEG technology for research. Conclusions: We believe that that EEG could benefit software engineering research to collect cognitive information under certain conditions. The lessons learned reported here should be used as inputs for future experiments in software engineering, where human aspects are of interest.},
	keywords = {attention, experiment, human subjects, lectroencephalogram},
	isbn = {9781450371452},
	doi = {10.1145/331900810.1145/3319008.3319357},
	url = {http://dl.acm.org/citation.cfm?doid=3319008http://dl.acm.org/citation.cfm?doid=3319008.3319357http://dl.acm.org/ft_gateway.cfm?id=3319357\&ftid=2047753\&dwn=1},
	author = {Jefferson Seide {Moll{\'e}ri} and Nurdiani, Indira and Fotrousi, Farnaz and Petersen, Kai},
	editor = {Garousi, Vahid and Ali, Shaukat}
}
@misc {26557,
	title = {Practical Cyber-Physical Systems Testing with Artificial Intelligence Techniques},
	howpublished = {Artificial Intelligence Lab, Oslo Metropolitan University, Norway},
	year = {2019},
	author = {Ali, Shaukat}
}
@article {26488,
	title = {Search-Based Test Case Implantation for Testing Untested Configurations},
	journal = {Information and Software Technology},
	volume = {111},
	year = {2019},
	month = {07/2019},
	pages = {22-36},
	publisher = {Elsevier},
	abstract = {Context: Modern large-scale software systems are highly configurable, and thus require a large number of test cases to be implemented and revised for testing a variety of system configurations. This makes testing highly configurable systems very expensive and time-consuming.Objective: Driven by our industrial collaboration with a video conferencing company, we aim to automatically analyze and implant existing test cases (i.e., an original test suite) to test the untested configurations.Method: We propose a search-based test case implantation approach (named as SBI) consisting of two key components: 1) Test case analyzer that statically analyzes each test case in the original test suite to obtain the program dependence graph for test case statements and 2) Test case implanter that uses multi-objective search to select suitable test cases for implantation using three operators, i.e., selection, crossover, and mutation (at the test suite level) and implants the selected test cases using a mutation operator at the test case level including three operations (i.e., addition, modification, and deletion).Results: We empirically evaluated SBI with an industrial case study and an open source case study by comparing the implanted test suites produced by three variants of SBI with the original test suite using evaluation metrics such as statement coverage (SC), branch coverage (BC), and mutation score (MS). Results show that for both the case studies, the test suites implanted by the three variants of SBI performed significantly better than the original test suites. The best variant of SBI achieved on average 19.3\% higher coverage of configuration variable values for both the case studies. Moreover, for the open source case study, the best variant of SBI managed to improve SC, BC, and MS with 5.0\%, 7.9\%, and 3.2\%, respectively.Conclusion: SBI can be applied to automatically implant a test suite with the aim of testing untested configurations and thus achieving higher configuration coverage.\&nbsp;},
	keywords = {genetic algorithms, Multi-objective optimization, Search, test case implantation},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@misc {26941,
	title = {Search-based test case implantation for testing untested configurations},
	howpublished = {34th IEEE/ACM International Conference on Automated Software Engineering (ASE 2019), San Diego (CA), USA},
	year = {2019},
	publisher = {ACM},
	type = {Journal First Presentation},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@book {26559,
	title = {Second International Workshop on Verification and Validation of Internet of Things},
	year = {2019},
	publisher = { IEEE},
	author = {Ali, Shaukat and Yue, Tao and Abreu, Rui}
}
@inproceedings {26493,
	title = {Stability Analysis for Safety of Automotive Multi-Product Lines: A Search-Based Approach},
	journal = {The Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2019},
	pages = {1241-1249},
	publisher = {ACM},
	author = {Lee, Nian-Ze and Arcaini, Paolo and Ali, Shaukat and Ishikawa, Fuyuki}
}
@article {25979,
	title = {Testing Self-Healing Cyber-Physical Systems under Uncertainty: A Fragility-Oriented Approach},
	journal = {Software Quality Journal},
	volume = {27},
	year = {2019},
	month = {03/2019},
	pages = {615{\textendash}649},
	publisher = { Springer},
	url = {https://link.springer.com/article/10.1007/s11219-018-9437-3},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao and Elaasar, Maged}
}
@inproceedings {26314,
	title = {Towards a Framework for the Analysis of Multi-Product Lines in the Automotive Domain},
	journal = {Proceedings of the 13th International Workshop on Variability Modelling of Software-Intensive Systems},
	year = {2019},
	publisher = {ACM},
	address = {New York, NY, USA},
	author = {Ali, Shaukat and Arcaini, Paolo and Hasuo, Ichiro and Ishikawa, Fuyuki and Lee, Nian-Ze},
	editor = {Perrouin, Gilles}
}
@misc {26558,
	title = {Uncertainty in Requirements},
	howpublished = {Nanjing University of Aeronautics and Astronautics, Nanjing, China},
	year = {2019},
	author = {Ali, Shaukat}
}
@misc {26940,
	title = {Uncertainty-wise Test Case Generation and Minimization for Cyber-Physical Systems},
	howpublished = {34th IEEE/ACM International Conference on Automated Software Engineering (ASE 2019), San Diego (CA), USA},
	year = {2019},
	publisher = {ACM},
	type = {Journal First Presentation},
	address = {San Diego (CA), USA},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao}
}
@article {26487,
	title = {Uncertainty-wise Test Case Generation and Minimization for CyberPhysical Systems},
	journal = {Journal of Systems and Software},
	volume = {153},
	year = {2019},
	month = {07/2019},
	publisher = {Elsevier},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao}
}
@article {26659,
	title = {Using multi-objective search and machine learning to infer rules constraining product configurations},
	journal = {Automated Software Engineering},
	year = {2019},
	pages = {1-62},
	publisher = { Springer},
	doi = {https://doi.org/10.1007/s10515-019-00266-2},
	author = {Safdar Aqeel {Safdar} and Yue, Tao and Ali, Shaukat and Lu, Hong}
}
@techreport {26017,
	title = {Automated Test Case Implantation to Test Untested Configurations: A Cost-Effective Search-Based Approach},
	year = {2018},
	note = {This research was supported by the Research Council of Norway (RCN) funded Certus SFI. Shuai Wang is also supported by RFF Hovedstaden funded MBE-CR project and COST Action CA15140 (ImAppNIO). Tao Yue and Shaukat Ali are supported by RCN funded Zen-Configurator project, RCN funded MBTCPS project, and COST Action IC1404.\&nbsp;},
	publisher = {Simula Research Laboratory},
	abstract = {Modern large-scale software systems are highly configurable, and thus require a large number of test cases to be implemented and revised for testing a variety of system configurations. This makes testing highly configurable systems very expensive and time-consuming. Driven by our industrial collaboration with a video conferencing company, we aim to automatically analyze and implant existing test cases (i.e., an original test suite) to test the untested configurations. We propose a search-based test case implantation approach (named as SBI) consisting of two key components: 1) Test case analyzer that statically analyzes each test case in the original test suite to obtain the program dependence graph for test case statements and 2) Test case implanter that uses multi-objective search to select suitable test cases for implantation using three operators, i.e., selection, crossover, and mutation (at the test suite level) and implants the selected test cases using a mutation operator at the test case level including three operations (i.e., addition, modification, and deletion). We empirically evaluated SBI with an industrial case study and an open source case study by comparing the implanted test suites produced by SBI with the original test suite using evaluation metrics such as statement coverage (SC), branch coverage (BC), mutation score (MS). Results show that for both the case studies, the implanted test suites performed significantly better than the original test suites with on average 21.9\% higher coverage of configuration variable values. For the open source case study, SBI managed to improve SC, BC, and MS with 4.8\%, 7.5\%, and 2.6\%, respectively. SBI can be applied to automatically implant an existing test suite with the aim of testing untested configurations and thus achieving higher configuration coverage.\&nbsp;},
	keywords = {Genetic Algorithm, Multi-objective optimization, Search, test case implantation},
	url = {https://www.simula.no/sites/default/files/publications/files/automated_test_case_implantation_to_test_untested_configurations-_a_cost-effective_search-based_approach_tr.pdf},
	author = {Pradhan, Dipesh and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liaaen, Marius}
}
@article {26214,
	title = {CBGA-ES+: A Cluster-Based Genetic Algorithm with Non-Dominated Elitist Selection for Supporting Multi-Objective Test Optimization},
	journal = {IEEE Transactions on Software Engineering},
	year = {2018},
	publisher = {IEEE},
	abstract = {Many real-world test optimization problems (e.g., test case prioritization) are multi-objective intrinsically and can be tackled using various multi-objective search algorithms (e.g., Non-dominated Sorting Genetic Algorithm (NSGA-II)). However, existing multi-objective search algorithms have certain randomness when selecting parent solutions for producing offspring solutions. In a worse case, suboptimal parent solutions may result in offspring solutions with bad quality, and thus affect the overall quality of the solutions in the next generation. To address such a challenge, we propose CBGA-ES+, a novel cluster-based genetic algorithm with non-dominated elitist selection to reduce the randomness when selecting the parent solutions to support multi-objective test optimization. We empirically compared CBGA-ES+ with random search and greedy (as baselines), four commonly used multi-objective search algorithms (i.e., Multi-objective Cellular genetic algorithm (MOCell), NSGA-II, Pareto Archived Evolution Strategy (PAES), and Strength Pareto Evolutionary Algorithm (SPEA2)), and the predecessor of CBGA-ES+ (named CBGA-ES) using five multi-objective test optimization problems with eight subjects (two industrial, one real world, and five open source). The results showed that CBGA-ES+ managed to significantly outperform the selected search algorithms for a majority of the experiments. Moreover, for the solutions in the same search space, CBGA-ES+ managed to perform better than CBGA-ES, MOCell, NSGA-II, PAES, and SPEA2 for 2.2\%, 13.6\%, 14.5\%, 17.4\%, and 9.9\%, respectively. Regarding the running time of the algorithm, CBGA-ES+ was faster than CBGA-ES for all the experiments.\&nbsp;},
	issn = {0098-5589},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@book {26166,
	title = {Editorial to the Theme Issue on Model-based Testing},
	year = {2018},
	publisher = { Springer},
	address = {Software \& Systems Modeling},
	author = {Papadakis, Mike and Ali, Shaukat and Perrouin, Gilles}
}
@techreport {26111,
	title = {Employing Multi-Objective Search and Machine Learning to Mine Cross Product Line Rules: A Technical Report},
	year = {2018},
	note = {This work was supported by the Zen-Configurator project funded by the Research Council of Norway (Grant No. 240024/F20) under the category of Young Research Talents of the FRIPO funding scheme. Tao Yue and Shaukat Ali are also supported by the Co-evolver project funded by the Research Council of Norway (grant no. 286898/LIS) under the category of Researcher Projects of the FRIPO funding scheme.},
	month = {05/2018},
	pages = {1-59},
	publisher = {Simula Research Laboratory},
	address = {Oslo, Norway},
	abstract = {Modern systems are being developed by integrating multiple products within/across product lines that communicate with each other through information networks. Runtime behaviors of such systems are related to product configurations and information networks. Cost-effectively supporting Product Line Engineering (PLE) of such systems is challenging mainly because of lacking the support of automation of the configuration process. Capturing rules is the key for automating the configuration process in PLE. However, there does not exist explicitly-specified rules constraining configurable parameter values of such products and product lines. Manually specifying such rules is tedious and time-consuming. To address this challenge, in this paper, we present an improved version (named as SBRM+) of our previously proposed Search-based Rule Mining (SBRM) approach. SBRM+ incorporates two machine learning algorithms (i.e., C4.5 and PART) and two multi-objective search algorithms (i.e., NSGA-II and NSGA-III), employs a clustering algorithm (i.e., k-means) for classifying rules as high or low confidence rules, which are used for defining three objectives to guide the search. To evaluate SBRM+ (i.e., SBRM+NSGA-II-C45, SBRM+NSGA-III-C45, SBRM+NSGA-II-PART, and SBRM+NSGA-III-PART), we performed two case studies (Cisco and Jitsi) and conducted three types of analyses of results: difference analysis, correlation analysis, and trend analysis. Results of the analyses show that all the SBRM+ approaches performed significantly better than two Random Search-based approaches (RBRM+-C45 and RBRM+-PART) in terms of fitness values, six quality indicators, and 17 machine learning quality measurements (MLQMs). As compared to RBRM+ approaches, SBRM+ approaches have improved the quality of rules based on MLQMs up to 27\% for the Cisco case study and 28\% for the Jitsi case study.},
	keywords = {Configuration, Interacting Products, Machine learning, Multi-Objective Search, Product Line, Rule Mining},
	url = {https://www.simula.no/sites/default/files/publications/files/employing_multi-objective_search_and_machine_learning_to_mine_cross_product_line_rules.pdf},
	author = {Safdar Aqeel {Safdar} and Yue, Tao and Ali, Shaukat and Lu, Hong}
}
@techreport {25958,
	title = {Employing Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	year = {2018},
	note = {This research was supported by the Research Council of Norway (RCN) funded Certus SFI (grant no. 203461/O30). Tao Yue and Shaukat Ali are also supported by RCN funded Zen-Configurator project (grant no. 240024/F20) and RCN funded MBT4CPS project.\&nbsp;},
	publisher = {Simula Research Laboratory},
	abstract = {Test case prioritization (TP) is widely used in regression testing for optimal reordering of test cases to achieve specific criteria (e.g., higher fault detection capability) as early as possible. In our earlier work, we proposed an approach for black-box dynamic TP using rule mining and multi-objective search (named as REMAP) by defining two objectives (fault detection capability and test case reliance score) by considering test case execution results at runtime. In this paper, we conduct an extensive empirical evaluation of REMAP by employing three different rule mining algorithms and three different multi-objective search algorithms, and we also evaluate REMAP with one additional objective (estimated execution time) for a total of 18 different configurations\&nbsp;of REMAP. Specifically, we empirically evaluated the 18 variants of REMAP with 1) two variants of random search while using two objectives and three objectives, 2) three variants of greedy algorithm based on one objective, two objectives, and three objectives, 3) 18 variants of static search-based prioritization approaches, and 4) six variants of rule-based prioritization approaches using two industrial and three open source case studies. Results showed that the two best variants of REMAP with two objectives and three objectives significantly outperformed the best variants of competing approaches by 84.4\% and 88.9\%, and managed to achieve on average 14.2\% and 18.8\% higher Average Percentage of Faults Detected per Cost (APFDc) scores.},
	keywords = {Black-box regression testing, Dynamic test case prioritization, Multi-objective optimization, Rule Mining, Search},
	url = {https://www.simula.no/sites/default/files/publications/files/employing_rule_mining_and_multi-objective_search_for_dynamic_test_case_prioritization_tr.pdf},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@book {26082,
	title = {First International Workshop on Verification and Validation of Internet of Things},
	year = {2018},
	publisher = {IEEE},
	author = {Ali, Shaukat and Yue, Tao and Abreu, Rui}
}
@inproceedings {26076,
	title = {Model- Based Personalized Visualization System for Monitoring Evolving Industrial Cyber-Physical System},
	journal = {The 25th Asia-Pacific Software Engineering Conference (APSEC 2018) },
	year = {2018},
	publisher = {IEEE},
	author = {Iglesias, Aitziber and Yue, Tao and Arellano, Cristobal and Ali, Shaukat and Sagardui, Goiuria}
}
@misc {26080,
	title = {Model-Based Testing of Cyber-Physical Systems with Machine Learning and Search Algorithms},
	howpublished = {DNV-GL, Trondheim, Norway},
	year = {2018},
	author = {Ali, Shaukat}
}
@misc {25728,
	title = {Model-Based Testing of Cyber-Physical Systems with Machine Learning and Search Algorithms},
	howpublished = {International Conference on Model-Driven Engineering and Software Development, MODELSWARD 2018},
	year = {2018},
	publisher = {SCITEPRESS Digital Library},
	abstract = {Testing of Cyber-Physical Systems (CPS) is manifesting unprecedented challenges due to the tremendous complexity and interdisciplinary nature of CPS. Thus, CPS testing must be driven by automated, scalable, optimised, and intelligent solutions. This keynote will focus on the key results from such CPS testing solutions that we developed in the last several years. These solutions rely on model-based testing, machine learning, and search algorithms. The results from the application of these testing solutions in industrial case studies and real-world systems will be presented together with the future research directions.},
	author = {Ali, Shaukat}
}
@article {26183,
	title = {Modeling Foundations for Executable Model-Based Testing of Self-Healing Cyber-Physical Systems},
	journal = {Software and Systems Modeling},
	year = {2018},
	month = {11/2018},
	pages = {1-31},
	publisher = { Springer},
	address = {Berlin Heidelberg},
	abstract = {Self-healing Cyber-Physical Systems (SH-CPSs) detect and recover from faults by themselves at runtime. Testing such systems is challenging due to the complex implementation of self-healing behaviors and their interaction with the physical environment, both of which are uncertain. To this end, we propose an executable model-based approach to test self-healing behaviors under environmental uncertainties. The approach consists of a Modeling Framework of SH-CPSs (MoSH) and an accompanying Test Model Executor (TM-Executor). MoSH provides a set of modeling constructs and a methodology to specify executable test models, which capture expected system behaviors and environmental uncertainties. TM-Executor executes the test models together with the systems under test, to dynamically test their self-healing behaviors under uncertainties. We demonstrated the successful application of MoSH to specify 11 self-healing behaviors and 17 uncertainties for three SH-CPSs. The time spent by TM-Executor to perform testing activities was in the order of milliseconds, though the time spent was strongly correlated with the complexity of test models.\&nbsp;},
	issn = {1619-1374},
	doi = {10.1007/s10270-018-00703-y},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao}
}
@misc {26135,
	title = {Practical Cyber-Physical Systems Testing: Results and Future Directions},
	howpublished = {The 7th Workshop of Advanced Software Engineering, Gold Coast, Australia},
	year = {2018},
	author = {Ali, Shaukat}
}
@misc {26081,
	title = {Presentation on Symposium on Search-Based Software Engineering (SSBSE 2019)},
	howpublished = {Symposium on Search-Based Software Engineering (SSBSE 2018), Montpellier, France},
	year = {2018},
	author = {Ali, Shaukat}
}
@misc {25962,
	title = {REMAP: Using Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	howpublished = {IEEE Conference on Software Testing, Validation and Verification (ICST), V{\"a}ster{\r a}s, Sweden},
	year = {2018},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {25710,
	title = {REMAP: Using Rule Mining and Multi-Objective Search for Dynamic Test Case Prioritization},
	journal = {11th IEEE Conference on Software Testing, Validation and Verification (ICST)},
	year = {2018},
	publisher = {IEEE},
	abstract = {Test case prioritization (TP) prioritizes test cases into an optimal order for achieving specific criteria (e.g., higher fault detection capability) as early as possible. However, the existing TP techniques usually only produce a static test case order before the execution without taking runtime test case execution results into account. In this paper, we propose an approach for dynamic TP using rule mining and multi-objective search (named as REMAP). REMAP has three key components: 1) Rule Miner, which mines execution relations among test cases from historical execution data; 2) Static Prioritizer, which defines two objectives (i.e., fault detection capability (FDC) and test case reliance score (TRS)) and applies multi-objective search to prioritize test cases statically; and 3) Dynamic Executor and Prioritizer, which executes statically-prioritized test cases and dynamically updates the test case order based on the runtime test case execution results. We empirically evaluated REMAP with random search, greedy based on FDC, greedy based on FDC and TRS, static search-based prioritization, and rule-based prioritization using two industrial and three open source case studies. Results showed that REMAP significantly outperformed the other approaches for 96\% of the case studies and managed to achieve on average 18\% higher Average Percentage of Faults Detected (APFD).\&nbsp;},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@misc {26110,
	title = {Search-based Test Optimization: A Very Short Introduction},
	howpublished = {ERATO MMSD Summer Camp, National Institute of Informatics, Japan},
	year = {2018},
	author = {Ali, Shaukat}
}
@misc {26034,
	title = {Search-based Test Optimization for Software Systems},
	howpublished = {GECCO 2018, Kyoto, Japan},
	year = {2018},
	abstract = {Test optimization is a crucial activity to select, generate, minimize, or prioritize test cases to test software systems cost-effectively. Test optimization is a complex problem that requires often solving contradictory relationships among cost (e.g., test execution time), effectiveness (e.g., fault detection ability), and efficiency (fault detection per execution time) objectives. Search algorithms including both single objective (e.g., Genetic Algorithms) and multi-objective (e.g., NSGA-II) have been extensively used for solving a variety of test optimization problems in diverse contexts. This tutorial will provide an introduction to test optimization with both single and multi-objective search algorithms. Mainly, it will focus on a variety of real industrial test selection, test minimization, and test prioritization problems. The tutorial will focus on the following aspects:1) How to optimally encoding test optimization problems for search algorithms? 2) How to define effective fitness functions that can guide search algorithms to find optimal solutions cost-effectively? 3) How to pick a suitable search algorithm to solve test optimization problem? 4) How to select and configure proper parameter settings for the selected search algorithms? 5) How to choose appropriate quality indicators to assess the quality of solutions generated by multi-objective algorithms? 6) How to choose appropriate statistical tests to evaluate search algorithms for a particular test optimization problem empirically?\&nbsp;},
	author = {Ali, Shaukat and Yue, Tao}
}
@article {26027,
	title = {Specifying Uncertainty in Use Case Models},
	journal = {Journal of Systems and Software},
	volume = {144},
	year = {2018},
	pages = {573-603},
	publisher = {Elsevier},
	abstract = {Context: Latent uncertainty in the context of software-intensive systems (e.g., Cyber-Physical Systems (CPSs)) demands explicit attention right from the start of development. Use case modeling{\textemdash}a commonly used method for specifying requirements in practice, should also be extended for explicitly specifying uncertainty.Objective: Since uncertainty is a common phenomenon in requirements engineering, it is best to address it explicitly by identifying, qualifying, and, where possible, quantifying uncertainty at the beginning stage. The ultimate aim, though not within the scope of this paper, was to use these use cases as the starting point to create test-ready models to support automated testing of CPSs under uncertainty.Method: We extend the Restricted Use Case Modeling (RUCM) methodology and its supporting tool to specify uncertainty as part of system requirements. Such uncertainties include those caused by insufficient domain expertise of stakeholders, disagreements among them, and known uncertainties about assumptions about the environment of the system. The extended RUCM, called U-RUCM, inherits the features of RUCM, such as automated analyses and generation of models, to mention but a few. Consequently, U-RUCM provides all the key benefits offered by RUCM (i.e., reducing ambiguities in requirements), but also, it allows specification of uncertainties with the possibilities of reasoning and refining existing ones and even uncovering unknown ones.Results: We evaluated U-RUCM with two industrial CPS case studies. After refining RUCM models (specifying initial requirements), by applying the U-RUCM methodology, we successfully identified and specified additional 306\% and 512\% (previously unknown) uncertainty requirements, as compared to the initial requirements specified in RUCM. This showed that, with U-RUCM, we were able to get a significantly better and more precise characterization of uncertainties in requirement engineering.Conclusion: Evaluation results show that U-RUCM is an effective methodology (with tool support) for dealing with uncertainty in requirements engineering. We present our experience, lessons learned, and future challenges, based on the two industrial case studies.},
	keywords = {Belief, Uncertainty, Use Case Modeling},
	doi = {10.1016/j.jss.2018.06.075},
	author = {Zhang, Man and Yue, Tao and Ali, Shaukat and Selic, Bran and Okariz, Oscar and Norgren, Roland and Intxausti, Karmele}
}
@misc {25961,
	title = {Testing Cyber-Physical Systems with Machine Learning and Search Algorithms},
	howpublished = {National Institute of Informatics, Tokyo, Japan},
	year = {2018},
	author = {Ali, Shaukat}
}
@inproceedings {26075,
	title = {Tool Support for Restricted Use Case Specification: Findings from a Controlled Experiment},
	journal = {The 25th Asia-Pacific Software Engineering Conference (APSEC 2018) },
	year = {2018},
	publisher = {IEEE},
	author = {Weninger, Markus and Gr{\"u}nbacher, Paul and Zhang, Huihui and Yue, Tao and Ali, Shaukat}
}
@inproceedings {25727,
	title = {Uncovering Unknown System Behaviors in Uncertain Networks with Model and Search-based Testing},
	journal = {11th IEEE Conference on Software Testing, Validation and Verification},
	year = {2018},
	publisher = {IEEE},
	abstract = {Modern software systems rely on information networks for communication. Such information networks are inherently unpredictable and unreliable. Consequently, software systems behave in an unstipulated manner in uncertain network conditions. Discovering unknown behaviors of these software systems in uncertain network conditions is essential to ensure their correct behaviors. Such discovery requires the development of systematic and automated methods. We propose an online and iterative Model-Based Testing approach to evolve test models with search algorithms. Our ultimate aim is to discover unknown expected behaviors that can only be observed in uncertain network conditions. Also, we implement an adaptive search-based test case generation strategy to generate test cases that are executed on the system under test. We evaluated our approach with an open source video conference application{\textemdash}Jitsi with four search algorithms. Results show that our approach was effective in discovering unknown system behaviors. In particular, (1+1) Evolutionary Algorithm outperformed the other three search algorithms including Random Search.\&nbsp;},
	author = {Ji, Ruihua and Li, Zhong and Chen, Shouyu and Pan, Minxue and Zhang, Tian and Yue, Tao and Ali, Shaukat and Li, Xuandong}
}
@misc {25726,
	title = {Advancing Testing Methods with the Explicit Consideration of Environmental Uncertainty},
	howpublished = {MBSE Seminar on Uncertainty, Nanjing University, China},
	year = {2017},
	abstract = {As compared with classical software/system testing, uncertainty-wise testing explicitly addresses known uncertainty about the behavior of a System Under Test (SUT), its operating environment, and interactions between the SUT and its operational environment, across all testing phases, including test design, test generation, test optimization, and test execution, with the aim to mainly achieve the following two goals. First, uncertainty-wise testing aims to ensure that the SUT deals with known uncertainty adequately. Second, uncertainty-wise testing should also be capable of learning new (previously unknown) uncertainties such that the SUT{\textquoteright}s implementation can be improved to guard against newly learned uncertainties during its operation. The necessity to integrate uncertainty in testing is becoming imperative because of the emergence of new types of intelligent and communicating software-based systems such as Cyber-Physical Systems (CPSs). Intrinsically, such systems are exposed to uncertainty because of their interactions with highly indeterminate physical environments. In this talk, I will provide the experience of applying uncertainty-wise testing techniques that rely on modeling and search algorithms in several industrial contexts. Furthermore, the vision about this new testing paradigm and its plausible future research directions will be presented.},
	author = {Ali, Shaukat}
}
@article {25644,
	title = {Automated Refactoring of OCL Constraints with Search},
	journal = {IEEE Transactions on Software Engineering (TSE)},
	year = {2017},
	publisher = {IEEE},
	abstract = {Object Constraint Language (OCL) constraints are typically used to provide precise semantics to models developed with the Unified Modeling Language (UML). When OCL constraints evolve regularly, it is essential that they are easy to understand and maintain. For instance, in cancer registries, to ensure the quality of cancer data, more than one thousand medical rules are defined and evolve regularly. Such rules can be specified with OCL. It is, therefore, important to ensure the understandability and maintainability of medical rules specified with OCL. To tackle such a challenge, we propose an automated search-based OCL constraint refactoring approach (SBORA) by defining and applying four semantics-preserving refactoring operators (i.e., Context Change, Swap, Split and Merge) and three OCL quality metrics (Complexity, Coupling, and Cohesion) to measure the understandability and maintainability of OCL constraints. We evaluate SBORA along with six commonly used multi-objective search algorithms (e.g., Indicator-Based Evolutionary Algorithm (IBEA)) by employing four case studies from different domains: healthcare (i.e., cancer registry system from Cancer Registry of Norway (CRN)), Oil\&amp;Gas (i.e., subsea production systems), warehouse (i.e., handling systems), and an open source case study named SEPA. Results show: 1) IBEA achieves the best performance among all the search algorithms and 2) the refactoring approach along with IBEA can manage to reduce on average 29.25\% Complexity and 39\% Coupling and improve 47.75\% Cohesion, as compared to the original OCL constraint set from CRN. Furthermore, we conducted a controlled experiment with 96 subjects and results show that the understandability and maintainability of the original constraint set can be improved significantly from the perspectives of the 96 participants of the controlled experiment.},
	doi = {10.1109/TSE.2017.2774829},
	url = {http://ieeexplore.ieee.org/document/8114267/},
	author = {Lu, Hong and Wang, Shuai and Yue, Tao and Ali, Shaukat and Nyg{\r a}rd, Jan}
}
@inproceedings {24925,
	title = {CBGA-ES: A Cluster-Based Genetic Algorithm with Elitist Selection for Supporting Multi-objective Test Optimization},
	journal = {10th IEEE International Conference on Software Testing, Verification and Validation (ICST 2017)},
	year = {2017},
	publisher = {IEEE},
	abstract = {Multi-objective search algorithms (e.g., non- dominated sorting genetic algorithm II (NSGA-II)) have been frequently applied to address various testing problems requiring multi-objective optimization such as test case selection. However, existing multi-objective search algorithms have certain randomness when selecting parent solutions for producing offspring solutions. In the worse case, suboptimal parent solutions may result in offspring solutions with bad quality, and thus affect the overall quality of the next generation. To address such a challenge, we propose a cluster-based genetic algorithm with elitist selection (CBGA-ES) with the aim to reduce such randomness for supporting multi-objective test optimization. We empirically compared CBGA-ES with random search, greedy (as baselines) and four commonly used multi-objective search algorithms (e.g., NSGA-II) using three industrial test optimization problems, i.e., test suite minimization, test case prioritization, and test case selection. The results showed that CBGA-ES significantly outperformed the baseline algorithms (e.g., greedy), and the four selected search algorithms for all the three test optimization problems. CBGA-ES managed to outperform more than 75\% of the objectives for all the four algorithms in each test optimization problem. Moreover, CBGA- ES was able to improve the quality of the solutions for an average of 32.5\% for each objective as compared to the four algorithms for the three test optimization problems.\&nbsp;},
	doi = {10.1109/ICST.2017.40},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius},
	editor = {Memon, Atif and Nishi, Yasuharu}
}
@misc {25413,
	title = {An Empirical Evaluation of Mutation and Crossover Operators for Multi-Objective Uncertainty-Wise Test Minimization},
	howpublished = {Proceedings of the IEEE/ACM 10th International Workshop on Search-Based Software Testing under the 2017 IEEE/ACM 39th International Conference on Software Engineering, Buenos Aires, Argentina},
	year = {2017},
	publisher = {ACM IEEE},
	address = {Buenos Aires, Argentina},
	author = {Ali, Shaukat}
}
@inproceedings {25132,
	title = {An Empirical Evaluation of Mutation and Crossover Operators for Multi-Objective Uncertainty-Wise Test Minimization},
	journal = {10th International Workshop on Search-based Software Testing},
	year = {2017},
	publisher = {IEEE},
	abstract = {Multi-objective uncertainty-wise test case minimization focuses on selecting a minimum number of test cases to execute out of all available ones while maximizing effectiveness (e.g., coverage), minimizing cost (e.g., time to execute test cases), and at the same time optimizing uncertainty-related objectives. In our previous work, we developed four uncertainty-wise test case minimization strategies relying on Uncertainty Theory and multi-objective search (NSGA-II with default settings), which were evaluated with one real Cyber-Physical System (CPS) with inherent uncertainty. However, a fundamental question to answer is whether these default settings of NSGA-II are good enough to provide optimized solutions. In this direction, we report one of the preliminary empirical evaluations, where we performed an experiment with three different mutation operators and three crossover operators, i.e., in total nine combinations with NSGA-II for the four uncertainty-wise test case minimization strategies using a real CPS case study. Results show that the Blend Alpha crossover operator together with the polynomial mutation operator permits NSGA-II achieving the best performance in terms of solving our uncertainty-wise test minimization problems.},
	keywords = {Cyber-Physical Systems, Multi-Objective Search, Test Case Minimization, Uncertainty-Wise Testing},
	doi = {10.1109/SBST.2017.9},
	author = {Ali, Shaukat and Li, Yan and Yue, Tao and Zhang, Man},
	editor = {Petke, Justyna and Galeotti, Juan}
}
@techreport {25014,
	title = {An Empirical Evaluation of Mutation and Crossover Operators for Multi-Objective Uncertainty-Wise Test Minimization},
	year = {2017},
	publisher = {Simula Research Laboratory},
	abstract = {Multi-objective uncertainty-wise test case minimization focuses on selecting a minimum number of test cases to execute out of all available ones while maximizing effectiveness (e.g., coverage), minimizing cost (e.g., time to execute test cases), and at the same time optimizing uncertainty-related objectives. In our previous work, we developed four uncertainty-wise test case minimization strategies relying on Uncertainty Theory and multi-objective search (NSGA-II with default settings), which were evaluated with one real Cyber-Physical System (CPS) with inherent uncertainty. However, a fundamental question to answer is whether these default settings\&nbsp;of NSGA-II are good enough to provide optimized solutions. In this direction, we report one of the preliminary empirical evaluations, where we performed an experiment with three different mutation operators and three crossover operators, i.e., in total nine combinations with NSGA-II for the four uncertainty-wise test case minimization strategies using a real CPS case study. Results show that the Blend Alpha crossover operator together with the polynomial mutation operator permits NSGA-II achieving the best performance in terms of solving our uncertainty-wise test minimization problems.},
	keywords = {Cyber-Physical Systems, Multi-Objective Search, Test Case Minimization, Uncertainty-Wise Testing},
	author = {Ali, Shaukat and Li, Yan and Yue, Tao and Zhang, Man}
}
@misc {25449,
	title = {Empowering Testing Activities with Modeling},
	howpublished = {The International Conference on Model-Driven Engineering and Software Development (MODELSWARD), Porto, Portugal},
	year = {2017},
	author = {Ali, Shaukat}
}
@inproceedings {24960,
	title = {Empowering Testing Activities with Modeling: Achievements and Insights from Nine Years of Collaboration with Cisco},
	journal = {The International Conference on Model-Driven Engineering and Software Development (MODELSWARD)},
	year = {2017},
	pages = {581-589},
	publisher = {SCITEPRESS {\textendash} Science and Technology Publications},
	abstract = {Research-Based Innovation (RBI) aims at bringing research-driven innovative solutions to the industrial problems identified from the industry in close collaboration with researchers. This paper focuses on presenting one such instance of RBI between Cisco Systems, Norway and the Software Engineering department of Simula Research Laboratory, Norway over the period of last nine years. The main topic of the collaboration was related to improving the current testing practice at Cisco with the use of models. We present a brief overview of various model-driven testing projects, and lessons learned from such RBI collaboration.},
	author = {Ali, Shaukat and Liaaen, Marius and Wang, Shuai and Yue, Tao},
	editor = {Luis Ferreira {Pires} and Hammoudi, Slimane and Selic, Bran}
}
@article {25646,
	title = {Enabling Automated Requirements Reuse and Configuration},
	journal = {Software and Systems Modeling},
	year = {2017},
	publisher = {Springer},
	address = {Berlin Heidelberg},
	issn = {1619-1374},
	doi = {10.1007/s10270-017-0641-6},
	author = {Li, Yan and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@inproceedings {25461,
	title = {Fragility-Oriented Testing with Model Execution and Reinforcement Learning},
	journal = {The 29th International Conference on Testing Software and Systems},
	year = {2017},
	publisher = {LNCS},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao and Elaasar, Maged}
}
@techreport {25201,
	title = {Fragility-Oriented Testing with Model Execution and Reinforcement Learning},
	year = {2017},
	publisher = {Simula Research Laboratory},
	abstract = {Self-healing is becoming an intrinsic feature of Cyber-Physical Systems (CPSs), which allows them to recover from faults in an autonomous manner during their operation. It becomes even more challenging when testing self-healing behaviors of CPSs in the presence of environment uncertainty. Such uncertainty makes system behaviors even more unpredictable in addition to the autonomous nature of self-healing behaviors. To this end, we propose Fragility-Oriented Testing (FOT) relying on model execution and reinforcement learning to efficiently test self-healing behaviors of a CPS in the presence of environment uncertainty. To evaluate the efficiency of FOT, we compared it with random testing (RT) and coverage-based testing (CBT). Results show that FOT significantly outperformed the two baselines for 7 out of 10 experiments in terms of fault revelation. Compared with RT and CBT, FOT respectively reduces 50\% and 35\% test execution time to find a fault.},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao and Elaasar, Maged}
}
@article {25652,
	title = {Integrating Weight Assignment Strategies with NSGA-II for Supporting User Preference Multi-Objective Optimization},
	journal = {IEEE Transactions on Evolutionary Computation (TEVC)},
	year = {2017},
	publisher = {IEEE},
	abstract = {Driven by the needs of several industrial projects on the applications of multi-objective search algorithms, we observed that user preferences must be properly incorporated into optimization objectives. However, existing algorithms usually treat all the objectives with equal priorities and do not provide a mechanism to reflect user preferences. To address this, we propose an extension{\textemdash}User-Preference Multi-Objective Optimization Algorithm (UPMOA), to the most commonly applied, non-dominated sorting genetic algorithm II (NSGA-II) by introducing a user preference indicator !, based on existing weight assignment strategies (e.g., Uniformly Distributed Weights (UDW)). We empirically evaluated UPMOA using four industrial problems from three diverse domains (i.e., Communication, Maritime and Subsea Oil\&amp;Gas). We also performed a sensitivity analysis for UPMOA with 625 algorithm parameter settings. To further assess the performance and scalability, 103500 artificial problems were created and evaluated representing 207 sets of user preferences. Results show that the UDW strategy with UPMOA achieves the best performance and UPMOA significantly outperformed other three multi-objective search algorithms, and has the ability to solve problems with a wide range of complexity. We also observed that different parameter settings led to the varied performance of UPMOA, thus suggesting that configuring proper parameters is highly problem-specific.\&nbsp;},
	doi = {10.1109/TEVC.2017.2778560},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@misc {25448,
	title = {Introduction to U-Test: Uncertain CPS behaviour and reliability},
	howpublished = {Exploitation Event ULMA Handling Systems, Spain},
	year = {2017},
	author = {Ali, Shaukat}
}
@article {25460,
	title = {IOCL: An Interactive Tool for Specifying, Validating and Evaluating OCL Constraints},
	journal = {Science of Computer Programming (SCP)},
	volume = {149},
	year = {2017},
	month = {08/2017},
	pages = {3-8},
	publisher = {Elsevier},
	abstract = {The Object Constraint Language (OCL) is commonly used for specifying additional constraints on models, in addition, to the ones enforced by the semantics of the models. However, a lot of practitioners and even researchers are reluctant in using OCL to some extent due to the lack of sufficient familiarity with OCL. To facilitate practitioners and researchers in specifying OCL constraints, we designed and developed a web-based tool called interactive OCL (iOCL) for interactively specifying constraints on a given model. The core idea behind iOCL is to present and display only relevant details (e.g., operations) of OCL to users at a given step of constraint specification process, in addition to helping modelers with its syntax. We evaluated iOCL using a real-world case study from Cancer Registry of Norway and the results showed that iOCL can significantly reduce the time required to specify OCL constraints and decrease the possibility of making syntactic errors during the specification process. Thus, we conclude that iOCL can facilitate the process of OCL constraint specification.\&nbsp;},
	author = {Muhammad, Hammad and Yue, Tao and Wang, Shuai and Ali, Shaukat and Nyg{\r a}rd, Jan}
}
@inproceedings {25221,
	title = {Mining Cross Product Line Rules with Multi-Objective Search and Machine Learning},
	journal = { Genetic and Evolutionary Computation Conference (GECCO) },
	year = {2017},
	publisher = {ACM},
	abstract = {Nowadays, an increasing number of systems are being developed by integrating products (belonging to different products lines) that communicate with each other through information networks. Cost-effectively supporting Product Line Engineering (PLE) and in particular enabling automation of configuration in PLE is a challenge. Capturing rules is the key for enabling automation of configuration. Product configuration has a direct impact on runtime interactions of communicating products. Such products might be within or across product lines and there usually don{\textquoteright}t exist explicitly specified rules constraining configurable parameter values of such products. Manually specifying such rules is tedious, time-consuming, and requires expert{\textquoteright}s knowledge of the domain and the product lines. To address this challenge, we propose an approach named as SBRM that combines multi- objective search with machine learning to mine rules. To evaluate the proposed approach, we performed a real case study of two communicating Video Conferencing Systems belonging to two different product lines. Results show that SBRM performed significantly better than Random Search in terms of fitness values, Hyper-Volume, and machine learning quality measurements. When comparing with rules mined with real data, SBRM performed significantly better particularly in terms of Failed Precision (18\%), Failed Recall (72\%), and Failed F-measure (59\%).},
	keywords = {Rule Mining; Multi-Objective Search; Configuration; Machine Learning; Product Line.},
	doi = {http://dx.doi.org/10.1145/3071178.307126},
	author = {Safdar Aqeel {Safdar} and Lu, Hong and Yue, Tao and Ali, Shaukat}
}
@inproceedings {25340,
	title = {A Multi-objective and Cost-Aware Optimization of Requirements Assignment},
	journal = {EEE Congress on Evolutionary Computation 2017 (CEC)},
	year = {2017},
	publisher = {IEEE},
	abstract = {A typical way to improve the quality of requirements is to assign them to suitable stakeholders for reviewing. Due to different characteristics of requirements and diverse background of stakeholders, it is needed to find an optimal solution for requirements assignment. Existing search-based requirements assignment solutions focus on maximizing stakeholders{\textquoteright} familiarities to assigned requirements and balancing the overall workload of each stakeholder. However, a cost-effective requirements assignment solution should also take into account another two optimization objectives: 1) minimizing required time for re-viewing requirements, and 2) minimizing the monetary cost required for performing reviewing tasks. We formulated the requirements assignment problem as a search problem and defined a fitness function considering all the five optimization objectives. We conducted an empirical evaluation to assess the fitness function together with six search algorithms using a real-world case study and 120 artificial problems to assess the scalability of the proposed fitness function. Results show that overall, our optimization problem is complex and further justifies the use for multi-objectives search algorithms, and the Speed-constrained Multi-Objective Particle Swarm Optimization (SMPSO) algorithm performed the best among all the search algorithms.},
	author = {Li, Yan and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@techreport {25013,
	title = {A Pilot Experiment to Assess Interactive OCL Specification in a Real Setting},
	year = {2017},
	publisher = {Simula Research Laboratory},
	abstract = {The Object Constraint Language (OCL) is a formal, declarative, and side-effect free language, standardized by the Object Management Group, for specifying constraints or queries on models specified in the Unified Modeling Language (UML). OCL was designed with the aim to bridge the gap between natural language and traditional formal languages requiring a strong mathematical background to understand and apply. OCL, along with UML, have been applied in practice for various purposes such as facilitating automated model-based testing. In most of such contexts of OCL, engineers with software engineering backgrounds specify OCL constraints. However, it is still a challenge for constraint authors (e.g., medical coders) who have no such background to apply OCL for other purposes (e.g., specifying medical rules). In this direction, in our previous work, we proposed a user-interactive specification framework, named iOCL, for facilitating OCL constraint specification and validation. The aim was to ease its adoption in practice in a wider application scope. In this paper, we present a pilot experiment that was conducted to assess the practical applicability of iOCL in Cancer Registry of Norway with real users of iOCL in terms of specifying medical cancer coding rules with iOCL. Results of the pilot experiment showed that, with iOCL, time to specify OCL constraints can be significantly reduced as compared to directly specifying OCL constraints without the tool support. In addition, participants of the experiment found that iOCL is easy to use.\&nbsp;\&nbsp;\&nbsp;\&nbsp;},
	author = {Ali, Shaukat and Muhammad, Hammad and Lu, Hong and Nyg{\r a}rd, Jan and Wang, Shuai and Yue, Tao}
}
@inproceedings {25414,
	title = {Product Line Engineering of Monitoring Functionality in Industrial Cyber-Physical Systems: A Domain Analysis},
	journal = {The 21st International Systems and Software Product Line Conference},
	year = {2017},
	publisher = {ACM},
	author = {Iglesias, Aitziber and Lu, Hong and Arellano, Cristobal and Yue, Tao and Ali, Shaukat},
	editor = {Sagardui, Goiuria}
}
@inproceedings {25464,
	title = {RCIA: Automated Change Impact Analysis to Facilitate a Practical Cancer Registry System},
	journal = {The International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2017},
	pages = {603-612},
	publisher = {IEEE},
	abstract = {The Cancer Registry of Norway (CRN) employs a cancer registry system to collect cancer patient data (e.g., diagnosis and treatments) from various medical entities (e.g., clinic hospitals). The collected data are then checked for validity (i.e., validation) and assembled as cancer cases (i.e., aggregation) based on more than 1000 cancer coding rules in the system. However, it is frequent in practice that the collected cancer data changes due to various reasons (e.g., different treatments) and the cancer coding rules can also change/evolve due to new medical knowledge. Thus, such a cancer registry system requires an efficient means to automatically analyze these changes and provide consequent impacts to medical experts for further actions. This paper proposes an automated Rule-based Change Impact Analysis (CIA) approach named RCIA that includes: 1) a change classification to capture the potential changes that can occur at CRN; 2) in total 80 change impact analysis rules including 50 dependency rules and 30 impact rules; and 3) an efficient algorithm to analyze changes and produce consequent impacts. We evaluate RCIA via a case study with 12 real change sets from CRN and a conducted interview. The results showed that RCIA managed to produce 100\% actual change impacts and the medical expert at CRN is quite positive to apply RCIA to facilitate their cancer registry system. We also shared a set of lessons learned based on the collaboration with CRN.\&nbsp;},
	author = {Wang, Shuai and Schwitalla, Thomas and Yue, Tao and Ali, Shaukat and Nyg{\r a}rd, Jan},
	editor = {IEEE, }
}
@article {24849,
	title = {Reliability-Redundancy-Location Allocation with Maximum Reliability and Minimum Cost Using Search Technique},
	journal = {Information and Software Techonology},
	volume = {82},
	year = {2017},
	pages = {36-54},
	publisher = {Elsevier},
	issn = {0950-5849},
	doi = {http://dx.doi.org/10.1016/j.infsof.2016.09.010},
	author = {Qiu, Xiang and Ali, Shaukat and Yue, Tao and Zhang, Li}
}
@inproceedings {25544,
	title = {A Restricted Natural Language based Use Case Modeling Methodology for Real-time Systems},
	journal = { 9th Workshop on Modelling in Software Engineering (MiSE{\textquoteright}2017)},
	year = {2017},
	publisher = {IEEE},
	author = {Zhang, Huihui and Yue, Tao and Ali, Shaukat and Wu, Ji and Liu, Chao}
}
@misc {25410,
	title = {Reusable Use Case and Test Case Specification Modeling},
	howpublished = {The 16th International Conference on Software Reuse, Salvador, Brazil},
	year = {2017},
	type = {Tutorial},
	author = {Yue, Tao and Ali, Shaukat}
}
@article {25010,
	title = {Search and similarity based selection of use case scenarios: An empirical study},
	journal = {Empirical Software Engineering},
	year = {2017},
	month = {04/2017},
	pages = {1-78},
	publisher = { Springer},
	abstract = {Use case modeling is a well-known requirements specification method and has been widely applied in practice. Use case scenarios of use case models are input elements for requirements inspection and analysis, requirements-based testing, and other downstream activities. It is, however, a practical challenge to inspect all use case scenarios that can be obtained from any non-trivial use case model, as such an inspection activity is often performed manually by domain experts. Therefore, it is needed to propose an automated solution for selecting a subset of use case scenarios with the ultimate aim of enabling cost-effective requirements (use case) inspection, analysis, and other relevant activities. Our solution is built on a natural language based, restricted use case modeling methodology (named as RUCM), in the sense that requirements specifications are specified as RUCM use case models. Use case scenarios can be automatically derived from RUCM use case models with the already established Zen-RUCM framework. In this paper, we propose a search-based and similarity-based approach called S3RCUM, through an empirical study, to select most diverse use case scenarios to enable cost-effective use case inspections. The empirical study was designed to evaluate the performance of three search algorithms together with eight similarity functions, through one real-world case study and six case studies from literature. Results show that (1+1) Evolutionary Algorithm together with Needleman-Wunsch similarity function significantly outperformed the other 31 combinations of the search algorithms and similarity functions. The combination managed to select 50\% of all the generated RUCM use case scenarios for all the case studies to detect all the seeded defects.},
	author = {Zhang, Huihui and Wang, Shuai and Yue, Tao and Ali, Shaukat and Liu, Chao}
}
@techreport {25444,
	title = {Search-based Uncertainty-wise Requirements Prioritization},
	year = {2017},
	publisher = {Simula Research Laboratory},
	abstract = {To ensure the quality of requirements, a common practice, especially in critical domains, is to review requirements within a limited time and monetary budgets. A requirement with higher importance, larger number of dependencies with other requirements, and higher implementation cost should be reviewed with the highest priority. However, requirements are inherently uncertain in terms of their impact on the requirements implementation cost. Such cost is typically estimated by stakeholders as an interval, though an exact value is often used in the literature for requirements optimization (e.g., prioritization). Such a practice, therefore, ignores uncertainty inherent in the estimation of requirements implementation cost. This paper explicitly taken into account such uncertainty for requirement prioritization and formulates four objectives for uncertainty-wise requirements prioritization with the aim of maximizing 1) the importance of requirements, 2) requirements dependencies, 3) the implementation cost of requirements, and 4) cost overrun probability. We evaluated the multi-objective search algorithm NSGA-II together with Random Search (RS) using the RALIC dataset and 19 artificial problems. Results show that NSGA-II can solve the requirements prioritization problem with a significantly better performance than RS. Moreover, NSGA-II can prioritize requirements with higher priority earlier in the prioritization sequence. For example, in the case of the RALIC dataset, the first 10\% of prioritized requirements in the prioritization sequence are on average 50\% better than RS in terms of prioritization effectiveness.},
	author = {Li, Yan and Zhang, Man and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@inproceedings {25519,
	title = {Search-based Uncertainty-wise Requirements Prioritization},
	journal = {The 22nd International Conference on Engineering of Complex Computer Systems},
	year = {2017},
	publisher = {IEEE},
	abstract = {To ensure the quality of requirements, a common practice, especially in critical domains, is to review requirements within a limited time and monetary budgets. A requirement with higher importance, larger number of dependencies with other requirements, and higher implementation cost should be re-viewed with the highest priority. However, requirements are inherently uncertain in terms of their impact on the requirements implementation cost. Such cost is typically estimated by stakeholders as an interval, though an exact value is often used in the literature for requirements optimization (e.g., prioritization). Such a practice, therefore, ignores uncertainty inherent in the estimation of requirements implementation cost. This paper explicitly taken into account such uncertainty for requirement prioritization and formulates four objectives for uncertainty-wise requirements prioritization with the aim of maximizing 1) the importance of requirements, 2) requirements dependencies, 3) the implementation cost of requirements, and 4) cost over-run probability. We evaluated the multi-objective search algorithm NSGA-II together with Random Search (RS) using the RALIC dataset and 19 artificial problems. Results show that NSGA-II can solve the requirements prioritization problem with a significantly better performance than RS. Moreover, NSGA-II can prioritize requirements with higher priority earlier in the prioritization sequence. For example, in the case of the RALIC dataset, the first 10\% of prioritized requirements in the prioritization sequence are on average 50\% better than RS in terms of prioritization effectiveness.},
	author = {Li, Yan and Zhang, Man and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@misc {25383,
	title = {SUnCPS: A Taxonomy of Security-related Uncertainty in Cyber-Physical Systems},
	year = {2017},
	address = {International Symposium on Engineering Secure Software and Systems (ESSoS{\textquoteright}17), Bonn, Germany},
	abstract = {Cyber-Physical Systems (CPS) are driving the fourth industrial revolution. In CPS, uncertainty is inherent that may cause security issue either directly or indirectly. To understand security together with uncertainty, we propose SUnCPS, a taxonomy of security-related uncertainty in the context of CPS. SUnCPS can provide a structured representation of security-related uncertainty in CPS as the basis for different (early) security engineering activities for CPS such as security risk analysis and management, vulnerability/attack analysis, and security testing for CPS.},
	author = {Phu Hong {Nguyen} and Ali, Shaukat and Yue, Tao}
}
@misc {25133,
	title = {Testing Cyber-Physical Systems under Uncertainty},
	howpublished = {CPS Concertation Event, Brussels, Belgium},
	year = {2017},
	author = {Ali, Shaukat}
}
@misc {25659,
	title = {Uncertainty Modeling Framework Version 2},
	year = {2017},
	publisher = {None},
	abstract = {This a U-Test public deliverable.},
	author = {Ali, Shaukat}
}
@misc {25409,
	title = {Uncertainty modeling (UM) {\textendash} Progress Summary},
	howpublished = {OMG Technical Meeting at Brussels, Belgium},
	year = {2017},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {25660,
	title = {Uncertainty Testing Framework V.2},
	year = {2017},
	publisher = {None},
	abstract = {This is a public deliverable for the U-Test project.},
	author = {Ali, Shaukat}
}
@misc {25661,
	title = {Uncertainty Testing Framework V.3},
	year = {2017},
	publisher = {None},
	abstract = {This is a public deliverable for U-Test project.},
	author = {Ali, Shaukat}
}
@techreport {24840,
	title = {Uncertainty-based Test Case Generation and Minimization for Cyber-Physical Systems: A Multi-Objective Search-based Approach},
	year = {2017},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Cyber-Physical Systems (CPSs) typically operate in highly indeterminate environmental conditions, which require the development of testing methods that must explicitly consider uncertainty in test design, test generation, and test optimization. Towards this direction, we propose uncertainty-wise test case generation and test case minimization strategies that rely on test ready models explicitly specifying subjective uncertainty. We propose two test case generation strategies and four test case minimization strategies based on Uncertainty Theory and multi-objective search. These strategies include a novel methodology for designing and introducing indeterminacy sources in the environment during test execution and a novel set of uncertainty-wise test verdicts. We performed an extensive empirical study to select the best algorithm out of eight commonly used multi-objective search algorithms, for each of the four minimization strategies, with five use cases of two industrial CPS case studies. The minimized set of test cases obtained with the best algorithm for each minimization strategy were executed on the two real CPSs. The results showed that our best test strategy managed to observe 51\% more uncertainties due to unknown indeterminate behaviors of the physical environment of the CPSs as compared to the rest of the test strategies. In addition, the same test strategy managed to observed 118\% more unknown uncertainties as compared to the unique number of known uncertainties},
	keywords = {Cyber-Physical Systems, Multi-Objective Search, Test Case Generation, Test Case Minimization, Uncertainty},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Malin Hedman, }
}
@misc {25428,
	title = {Uncertainty-wise and Model-based Testing of Industrial Cyber-Physical Systems},
	year = {2017},
	address = {5th User Conference and Advanced Automated Testing (UCAAT), Berlin, Germany},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Phu Hong {Nguyen}}
}
@techreport {25015,
	title = {Uncertainty-Wise and Time-Aware Test Case Prioritization for Cyber-Physical Systems},
	year = {2017},
	publisher = {Simula Research Laboratory},
	abstract = {Context: Complex systems (e.g., Cyber-Physical Systems) that interact with the real world, behave in an unstipulated manner while operating in uncertain environments. Testing such systems in uncertainty is a big challenge. Devising uncertainty-wise testing solutions can be considered as a mandate for dealing with this challenge. Though uncertainty-wise testing is gaining attention in the last few years, industry-strengthening solutions are still missing.Objective: Our objective is to propose uncertainty-wise test case prioritization approaches that can significantly improve the cost-effectiveness of test case execution to maximize the occurrence of uncertainty.Method: In this paper, we identified and defined twenty uncertainty-wise, search-based, multi-objective test case prioritization problems. To solve these problems with multi-objective search algorithms, we defined twenty corresponding fitness functions based on various cost, effectiveness, and uncertainty measures.Results: We performed a large-scale empirical evaluation to evaluate the well-known multi-objective search algorithms, i.e., NSGA-II, MOCell, SPEA2 and CellDE, and Random Search (RS), with five different use cases from two industrial CPS case studies to solve the twenty prioritization problems. Results showed that all the selected search algorithms significantly outperformed RS signifying that our identified test prioritization problems are complex. When comparing search algorithms, we observed that different search algorithms performed best regarding providing the best solutions for different uncertainty-wise prioritization problems. Based on the results, we provide recommendations to select search algorithms for different prioritization problems under different time budgets.\&nbsp;Conclusion: This paper presented uncertainty-wise and time-aware test case prioritization approaches, which were specially developed to improve the cost and effectiveness of test case execution and at the same time maximizing the occurrence of uncertainties during CPS testing},
	keywords = {Uncertainty-wise Testing; Test Case Prioritization; Multi-objective Search},
	author = {Zhang, Man and Li, Yan and Ali, Shaukat and Yue, Tao}
}
@article {25456,
	title = {Uncertainty-Wise Cyber-Physical System Test Modeling},
	journal = {Software \& Systems Modeling},
	year = {2017},
	publisher = { Springer},
	abstract = {It is important that a Cyber-Physical System (CPS) deals with uncertainty in its behavior caused by its unpredictable operating environment, to ensure its reliable operation. One method to ensure that the CPS will handle such uncertainty during its operation is by testing the CPS with Model-based Testing (MBT) techniques. However, existing MBT techniques do not explicitly capture uncertainty in test ready models i.e., capturing the uncertain expected behavior of a CPS in the presence of environment uncertainty. To fill this gap, we present an Uncertainty-Wise test-modeling framework, named as Uncertum, to create test ready models to support MBT of CPSs facing uncertainty. Uncertum relies on the definition of a UML profile (the UML Uncertainty Profile (UUP)) and a set of UML model libraries extending the UML profile for Modeling and Analysis of Real-Time and Embedded Systems (MARTE). Uncertum also benefits from the UML Testing Profile (UTP) V.2 to support standard-based MBT. Uncertum was evaluated with two industrial CPS case studies, one real-world case study, and one open source CPS case study from the following four perspectives: 1) Completeness and Coverage of the profiles and model libraries in terms of concepts defined in their underlying uncertainty conceptual model for CPSs (i.e., U-Model and MARTE, 2) Effort required to model uncertainty with Uncertum, and 3) Correctness of the developed test ready models, which was assessed via model execution. Based on the evaluation, we can conclude that we were successful in modeling all the uncertainties identified in the four case studies, which gives us an indication that Uncertum is sufficiently complete. In terms of modeling effort, we concluded that on average Uncertum requires18.5\% more time to apply stereotypes from UUP on test ready models.},
	keywords = {Cyber-Physical System; UML, Model-based Testing, Uncertainty},
	issn = {1619-1374},
	doi = {10.1007/s10270-017-0609-6},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland and Okariz, Oscar}
}
@article {25187,
	title = {Uncertainty-Wise Evolution of Test Ready Models},
	journal = {Information and Software Technology (IST)},
	volume = {87},
	year = {2017},
	pages = {140-159},
	publisher = {Elsevier},
	abstract = {Context: Cyber-Physical Systems (CPSs), when deployed for operation, are inherently prone to uncertainty. Considering their applications in critical domains (e.g., healthcare), it is important that such CPSs are tested sufficiently, with the explicit consideration of uncertainty. Model-based testing (MBT) involves creating test ready models capturing the expected behavior of a CPS and its operating environment. These test ready models are then used for generating executable test cases. It is, therefore, necessary to develop methods that can continuously evolve, based on real operational data collected during the operation of CPSs, test ready models and uncertainty captured in them, all together termed as Belief Test Ready Models (BMs)Objective: Our objective is to propose a model evolution framework that can interactively improve the quality of BMs, based on operational data. Such BMs are developed by one or more test modelers (belief agents) with their assumptions about the expected behavior of a CPS, its expected physical environment, and potential future deployments. Thus, these models explicitly contain subjective uncertainty of the test modelers.Method: We propose a framework (named as UncerTolve) for interactively evolving BMs (specified with extended UML notations) of CPSs with subjective uncertainty developed by test modelers. The key inputs of UncerTolve include initial BMs of CPSs with known subjective uncertainty and real data collected from the operation of CPSs. UncerTolve has three key features: 1) Validating the syntactic correctness and conformance of BMs against real operational data via model execution, 2) Evolving objective uncertainty measurements of BMs via model execution, and 3) Evolving state invariants (modeling test oracles) and guards of transitions (modeling constraints for test data generation) of BMs with a machine learning technique.Results: As a proof-of-concept, we evaluated UncerTolve with one industrial CPS case study, i.e., GeoSports from the healthcare domain. Using UncerTolve, we managed to evolve 51\% of belief elements, 18\% of states, and 21\% of transitions as compared to the initial BM developed in an industrial setting.Conclusion: UncerTolve can successfully evolve model elements of the initial BM, in addition to objective uncertainty measurements using real operational data. The evolved model can be used to generate additional test cases covering evolved model elements and objective uncertainty. These additional test cases can be used to test the current and future deployments of a CPS to ensure that it will handle uncertainty gracefully during its operations.\&nbsp;\&nbsp;\&nbsp;\&nbsp;\&nbsp;},
	keywords = {Belief Model, Belief Test Ready Model, Model Evolution, Model-based Testing, Uncertainty},
	issn = {0950-5849},
	doi = {10.1016/j.infsof.2017.03.003},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland}
}
@misc {25564,
	title = {Uncertainty-wise Test Case Generation and Minimization for Cyber-Physical Systems: A Multi-Objective Search-based Approach},
	howpublished = {National Institute of Informatics, Tokyo, Japan},
	year = {2017},
	author = {Ali, Shaukat}
}
@misc {25220,
	title = {Uncertainty-Wise Testing},
	howpublished = {Advances in Model-Based Testing (A-MOST), Tokyo, Japan},
	year = {2017},
	abstract = {Uncertainty-Wise testing explicitly integrates known uncertainty about the behavior of the System Under Test (SUT), its operating environment, and interactions between them in test design, generation, optimization, and execution with the following two key objectives. First, to ensure that the SUT deals with known uncertainty appropriately. Second, to learn new uncertainties such that the SUT{\textquoteright}s implementation can be improved to guard against these uncertainties, when it is operational. The necessity to integrate uncertainty in testing is becoming imperative because of the emergence of new types of intelligent and communicating software-based systems, e.g., Cyber-Physical Systems (CPSs). Intrinsically, such systems are exposed to uncertainty because of their interactions with highly indeterminate physical environment including human. On top of that, these systems are becoming more and more autonomous, i.e., making decisions themselves at runtime (e.g., self-healing and self-configuration) and thus introduce an extra layer of complexity to test these systems. This keynote first focuses on novel challenges posed by uncertainty-wise testing of CPSs. Second, it presents some research results from the applications of novel approaches for uncertainty-wise testing of CPSs that were devised based on model-based testing, search-based testing, and machine learning techniques.},
	author = {Ali, Shaukat}
}
@inbook {25233,
	title = {Uncertainty-wise Testing of Cyber-Physical Systems},
	booktitle = {Advances in Computers},
	volume = {107},
	year = {2017},
	pages = {23-94},
	publisher = {Elsevier},
	organization = {Elsevier},
	chapter = {2},
	abstract = {As compared with classical software/system testing, uncertainty-wise testing explicitly addresses known uncertainty about the behavior of a System Under Test (SUT), its operating environment, and interactions between the SUT and its operational environment, across all testing phases, including test design, test generation, test optimization, and test execution, with the aim to mainly achieve the following two goals. First, uncertainty-wise testing aims to ensure that the SUT deals with known uncertainty adequately. Second, uncertainty-wise testing should be also capable of learning new (previously unknown) uncertainties such that the SUT{\textquoteright}s implementation can be improved to guard against newly learned uncertainties during its operation. The necessity to integrate uncertainty in testing is becoming imperative because of the emergence of new types of intelligent and communicating software-based systems such as Cyber-Physical Systems (CPSs). Intrinsically, such systems are exposed to uncertainty because of their interactions with highly indeterminate physical environments. In this chapter, we provide our understanding and experience of uncertainty-wise testing from the aspects of uncertainty-wise model-based testing, uncertainty-wise modeling and evolution of test ready models, and uncertainty-wise multi-objective test optimization, in the context of testing CPSs under uncertainty. Furthermore, we present our vision about this new testing paradigm and its plausible future research directions.},
	author = {Ali, Shaukat and Lu, Hong and Wang, Shuai and Yue, Tao and Zhang, Man},
	editor = {Memon, Atif}
}
@misc {25619,
	title = {Uncertainty-wise Testing of Cyber-Physical Systems},
	howpublished = {2017 IEEE International Symposium on Systems Engineering, Vienna, Austria},
	year = {2017},
	type = {Tutorial},
	author = {Ali, Shaukat and Yue, Tao and Zhang, Man}
}
@article {24520,
	title = {Assessing the Quality of Industrial Avionics Software: An Extensive Empirical Evaluation},
	journal = {Empirical Software Engineering},
	year = {2016},
	publisher = {Springer},
	author = {Wu, Ji and Ali, Shaukat and Yue, Tao and Tian, Jie and Liu, Chao}
}
@techreport {24569,
	title = {Conceptually Understanding Uncertainty in Self-Healing Cyber-Physical Systems},
	year = {2016},
	publisher = {Simula Research Laboratory},
	abstract = {Smart Cyber-Physical Systems (CPSs) are autonomic systems capable of making decisions at runtime by themselves. These systems are complex in nature and typically operate in highly unpredictable and uncertain environments. One key autonomic capability of such systems is to recover from failures in an autonomic manner, referred to as self-healing. Due to the wide range of applications of smart CPSs in our daily life, self-healing behaviors of smart CPSs must be reliable even under uncertainty. Uncertainty and Self-Healing in CPS, in general, are understudied areas of research and thus as the\&nbsp;first step towards understanding self-healing and uncertainty, we propose a conceptual model to understand key concepts including self-healing behaviors, uncertainties, and their relationships. Our ultimate objective is to provide a unified understanding of these concepts, which forms a foundation for additional analyses in the future such as testing. We validated the conceptual model with six case studies having self-healing behaviors from the literature and industry.},
	keywords = {Cyber-Physical Systems, Search-Based Software Engineering, Self-Healing, Testing, Uncertainty},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao}
}
@inproceedings {24043,
	title = {Enhancing Test Case Prioritization in an Industrial Setting with Resource Awareness and Multi-Objective Search},
	journal = {The 38th International Conference on Software Engineering (ICSE), Software Engineering in Practice (SEIP) track },
	year = {2016},
	pages = {182-191},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Bakkeli, {\O}yvind and Liaaen, Marius}
}
@inproceedings {24638,
	title = {Evaluating Variability Modeling Techniques for Supporting Cyber-Physical System Product Line Engineering},
	journal = {System Analysis and Modelling (SAM) Conference},
	volume = {9959},
	year = {2016},
	month = {10/2016},
	pages = {1-19},
	publisher = {Springer International Publishing},
	edition = {9},
	address = {Saint Malo, France},
	abstract = {Modern society is increasingly dependent on Cyber-Physical Systems (CPSs) in diverse domains such as aerospace, energy and healthcare. Employing Product Line Engineering (PLE) in CPSs is cost-effective in terms of reducing production cost, and achieving high productivity of a CPS development process as well as higher quality of produced CPSs. To apply CPS PLE in practice, one needs to first select an appropriate variability modeling technique (VMT), with which variabilities of a CPS Product Line (PL) can be specified. In this paper, we proposed a set of basic and CPS-specific variation point (VP) types and modeling requirements for proposing CPS-specific VMTs. Based on the proposed set of VP types (basic and CPS-specific) and modeling requirements, we evaluated four VMTs: Feature Modeling, Cardinality Based Feature Modeling, Common Variability Language, and SimPL (a variability modeling technique dedicated to CPS PLE), with a real-world case study. Evaluation results show that none of the selected VMTs can capture all the basic and CPS-specific VP and meet all the modeling requirements. Therefore, there is a need to extend existing techniques or propose new ones to satisfy all the requirements.},
	keywords = {and Cyber- Physical Systems, Product Line Engineering, Variability Modeling},
	author = {Safdar Aqeel {Safdar} and Yue, Tao and Ali, Shaukat and Lu, Hong}
}
@misc {24742,
	title = {Generating Boundary Values from OCL Constraints using Constraints Rewriting and Search Algorithms},
	howpublished = {IEEE World Congress on Computational Intelligence},
	year = {2016},
	publisher = {IEEE},
	author = {Ali, Shaukat}
}
@inproceedings {24467,
	title = {Generating Boundary Values from OCL Constraints using Constraints Rewriting and Search Algorithms},
	journal = {IEEE World Congress on Computational Intelligence},
	year = {2016},
	author = {Ali, Shaukat and Yue, Tao and Qiu, Xiang and Lu, Hong}
}
@techreport {24609,
	title = {An Integrated Modeling Framework to Facilitate Model-Based Testing of Cyber-Physical Systems under Uncertainty},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Undertaking innate uncertainty during the operation of Cyber-Physical Systems (CPSs) is crucial for their intact behavior. One method to handle such uncertainty is using Model-based Testing (MBT); however, existing MBT techniques do not explicitly capture uncertainty in test ready models that capture expected behavior of a CPS and its operating environment. To fill this gap, we present an Uncertainty-Wise test-modeling framework, named as Uncertum, to create test ready models to support MBT of CPSs facing uncertainty. Uncertum relies on the definition of a UML profile (the UML Uncertainty Profile (UUP)) and a set of UML model libraries extending the UML profile for Modeling and Analysis of Real-Time and Embedded Systems (MARTE). Uncertum also benefits from the UML Testing Profile (UTP) V.2 to support standard-based MBT. Uncertum was evaluated with two industrial CPS case studies, one real world and one open source CPS case study from the following four perspectives: 1) Completeness and Coverage of the profiles and model libraries in terms of concepts defined in their underlying uncertainty conceptual model for CPSs (i.e., U-Model and MARTE, 2) Effort required to model uncertainty with Uncertum, and 3) Correctness of the developed test ready models, which was assessed via model execution.},
	keywords = {Uncertainty; Cyber-Physical System; UML Profile; Model-based Testing},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland and Okariz, Oscar}
}
@techreport {24660,
	title = {Interactively Evolving Test Ready Models with Uncertainty Developed for Testing Cyber-Physical Systems},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Context: Cyber-Physical Systems (CPSs), when deployed for operation, are inherently prone to uncertainty. Considering their applications in critical domains (e.g., healthcare), it is important that such CPSs are tested sufficiently, with the explicit consideration of uncertainty. Model-based testing (MBT) involves creating test ready models capturing the expected behavior of a CPS and its operating environment. These test ready models are then used for generating executable test cases. It is, therefore, necessary to develop methods that can continuously evolve, based on real operational data collected during the operation of CPSs, test ready models and uncertainty captured in them, all together termed as Belief Test Ready Models (BMs)Objective: Our objective is to propose a model evolution framework that can interactively improve the quality of BMs, based on operational data. Such BMs are developed by one or more test modelers (belief agents) with their assumptions about the expected behavior of a CPS, its expected physical environment, and potential future deployments. Thus, these models explicitly contain subjective uncertainty of the test modelers.Method: We propose a framework (named as UncerTolve) for interactively evolving BMs (specified with extended UML notations) of CPSs with subjective uncertainty developed by test modelers. The key inputs of UncerTolve include initial BMs of CPSs with known subjective uncertainty and real data collected from the operation of CPSs. UncerTolve has three key features: 1) Validating the syntactic correctness and conformance of BMs against real operational data via model execution, 2) Evolving objective uncertainty measurements of BMs via model execution, and 3) Evolving state invariants (modeling test oracles) and guards of transitions (modeling constraints for test data generation) of BMs with a machine learning technique.Results: As a proof-of-concept, we evaluated UncerTolve with one industrial CPS case study, i.e., GeoSports from the healthcare domain. Using UncerTolve, we managed to evolve 51\% of belief elements, 18\% of states, and 21\% of transitions as compared to the initial BM developed in an industrial setting.Conclusion: UncerTolve can successfully evolve model elements of the initial BM, in addition to objective uncertainty measurements using real operational data. The evolved model can be used to generate additional test cases covering evolved model elements and objective uncertainty. These additional test cases can be used to test the current and future deployments of a CPS to ensure that it will handle uncertainty gracefully during its operations.\&nbsp;\&nbsp;\&nbsp;\&nbsp;\&nbsp;},
	keywords = {Belief State Machine, Cyber-Physical System, Model Evolution, Model-based Testing, Uncertainty},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland}
}
@inproceedings {24651,
	title = {iOCL: A Interactive Tool for Specifying, Validating and Evaluating OCL Constraints},
	journal = {Tool Demonstrations Track, ACM/IEEE 19th International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	year = {2016},
	month = {09/2016},
	pages = {1-7},
	abstract = {The Object Constraint Language (OCL) is frequently used to specify additional constraints on models, in addition, to the ones enforced by semantics of the models. It is a well- known fact that due to the lack of familiarity with OCL, practitioners and even researcher to some extent are reluctant in using OCL. To help practitioners and researchers in writing OCL constraints for their specific problem at hand, we developed a tool called interactive OCL (iOCL) for interactively specifying constraints on a given model. The basic philosophy behind the tool is to present only those details (e.g., operations) of OCL to modelers that are valid at a given step of constraint specification process, in addition to helping modelers with its syntax. Our ultimate aim is to reduce the effort required to specify constraints, subsequently lowering down training cost and increasing the correctness of the constraints. iOCL is a web-based ap- plication that integrates other tools including Eclipse OCL for validation and evaluation of OCL constraints, and EsOCL for automatically generating valid instances of models that satisfy the specified constraints.\&nbsp;},
	author = {Muhammad, Hammad and Yue, Tao and Ali, Shaukat and Wang, Shuai}
}
@inproceedings {24479,
	title = {MBF4CR: A Model-Based Framework for Supporting An Automated Cancer Registry System},
	journal = {12th European Conference on Modelling Foundations and Applications (ECMFA 2016)},
	year = {2016},
	pages = {191-204},
	publisher = {Lecture Notes in Computer Science, Springer Verlag},
	abstract = {The Cancer Registry of Norway (CRN) collects medical information (e.g., laboratory results, clinical procedures and treatment) of cancer patients from different medical entities, for all cancer patients in Norway. The collected data are checked for validity and correctness (i.e., validation) and is the basis for the registration of cancer cases (i.e., aggregation) by employing more than a thousand of medical rules. However, the current practice of CRN lacks of a systematic way to capture the domain knowledge and maintain medical rules at a proper level of abstraction. To tackle these challenges, this paper proposes a model-based framework (named as MBF4CR) for capturing the domain knowledge, formalizing medi- cal rules, automating rule selection, and enabling data (cancer messages and cancer cases) validation and aggregation using Unified Modeling Language (UML) and Object Constraint Language (OCL). MBF4CR systematically cap- tures domain knowledge (e.g., cancer messages) as a UML class diagram and formally specifies medical rules as OCL constraints. By associating tags to OCL constraints, MBF4CR enables an automated rule selection process with tool support. We employed a case study from CRN that consists of 187 medical rules to evaluate MBF4CR from two aspects: Performance in terms of se- lecting and executing rules, and Correctness in terms of producing correct validation and aggregation results. Results show that MBF4CR can facilitate the current practice by complying with the medical domain knowledge with an acceptable performance, while reducing the maintenance effort.\&nbsp;},
	author = {Wang, Shuai and Lu, Hong and Yue, Tao and Ali, Shaukat and Nyg{\r a}rd, Jan}
}
@inproceedings {24936,
	title = {A Model-Based Approach with Tool Support to Facilitate the Cancer Registration Process in Cancer Registry of Norway},
	journal = {European Telemedicine Conference (ETC)},
	year = {2016},
	author = {Wang, Shuai and Lu, Hong and Yue, Tao and Ali, Shaukat and Nyg{\r a}rd, Jan}
}
@article {24588,
	title = {Model-based Incremental Conformance Checking to Enable Interactive Product Configuration},
	journal = {Information and Software Technology},
	volume = {72},
	year = {2016},
	month = {04/2016},
	pages = {68-89},
	publisher = {Elsevier},
	abstract = {Context: Model-based product line engineering (PLE) is a paradigm that can enable automated product configuration of large- scale software systems, in which models are used as an abstract specification of commonalities and variabilities of products of a product line. Objective: In the context of PLE, providing immediate feedback on the correctness of a manual configuration step to users has a practical impact on whether a configuration process with tool support can be successfully adopted in practice. Method: In an existing work, an UML-based variability modeling methodology named as SimPL and an interactive configuration process was proposed. Based on the existing work, we propose an automated, incremental and efficient conformance checking approach to ensure that the manual configuration of a variation point conforms to a set of pre-defined conformance rules specified in the Object Constraint Language (OCL). The proposed approach, named as Zen-CC, has been implemented as an integrated part of our product configuration and derivation tool: Zen-Configurator. Results: The performance and scalability of Zen-CC have been evaluated with a real-world case study. Results show that Zen- CC significantly outperformed two baseline engines in terms of performance. Besides, the performance of Zen-CC remains stable during the configuration of all the 10 products of the product line and its efficiency also remains un-impacted even with the growing product complexity, which is not the case for both of the baseline engines. Conclusion: The results suggest that Zen-CC performs practically well and is much more scalable than the two baseline engines and is scalable for configuring products with a larger number of variation points.\&nbsp;},
	author = {Lu, Hong and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@techreport {25289,
	title = {Model-Based Security Engineering for Cyber-Physical Systems: A Systematic Mapping Study},
	number = {2016-11},
	year = {2016},
	note = {This technical report is an earlier version of a published article. The published article can be found here:https://www.simula.no/publications/model-based-security-engineering-cyber-physical-systems-systematic-mapping-study},
	publisher = {Simula Research Laboratory},
	author = {Phu Hong {Nguyen} and Ali, Shaukat and Yue, Tao}
}
@article {24648,
	title = {Model-Based Security Engineering for Cyber-Physical Systems: A Systematic Mapping Study},
	journal = {Information and Software Technology},
	volume = {83},
	year = {2016},
	month = {11/2016},
	pages = {116{\textendash}135},
	publisher = {Elsevier},
	abstract = {Context: Cyber-physical systems (CPSs) have emerged to be the next generation of engineered systems driving the so-called fourth industrial revolution. CPSs are becoming more complex, open and more prone to security threats, which urges security to be engineered systematically into CPSs. Model-Based Security Engineering (MBSE) could be a key means to tackle this challenge via security by design, abstraction and automation.Objective: We aim at providing an initial assessment on the state of the art in MBSE for CPSs (MBSE4CPS). Specifically, this work focuses on finding out 1) the publication statistics of MBSE4CPS studies; 2) the characteristics of MBSE4CPS studies; and 3) the open issues of MBSE4CPS research.Method: We conducted a systematic mapping study (SMS) following a rigorous protocol that was developed based on the state-of-the-art SMS and systematic review guidelines. From thousands of relevant publications, we systematically identified 34 primary MBSE4CPS studies for data extraction and synthesis to answer predefined research questions.Results: SMS results show that for two recent years (2014-2015) the number of primary MBSE4CPS studies has increased significantly. Within the primary studies, the popularity of using Domain-Specific Languages (DSLs) is comparable with the use of the standardized UML modeling notation. Most primary studies do not explicitly address specific security concerns (e.g., confidentiality, integrity) but rather focus on security analyses in general on threats, attacks or vulnerabilities. Few primary studies propose to engineer security solutions for CPSs. Many focus on the early stages of development lifecycle such as security requirement engineering or analysis.Conclusion: The SMS does not only provide the state of the art in MBSE4CPS, but also points out several open issues that would deserve more investigation, e.g., the lack of engineering security solutions for CPSs, limited tool support, too few industrial case studies, and the challenge of bridging DSLs in engineering secure CPSs.\&nbsp;},
	keywords = {Cyber-Physical Systems, Security Testing, Testing, Uncertainty},
	url = {http://dx.doi.org/10.1016/j.infsof.2016.11.004},
	author = {Phu Hong {Nguyen} and Ali, Shaukat and Yue, Tao}
}
@misc {24941,
	title = {Model-Driven Testing of Cyber-Physical Systems with the Explicit Consideration of Uncertainty},
	howpublished = {MPM4CPS WG meetings in Malaga, Spain, 24-25 November 2016},
	year = {2016},
	abstract = {It is a well-recognized fact that Cyber-Physical Systems (CPSs) face both known and unknown uncertainty during their operation. This demands the development of testing techniques that must take into account known uncertainty both in the CPS and its environment with the final goal of discovering unknown uncertainty against which the CPS can be tested. Eventually, the implementation of the CPS can be improved to shield it against the newly discovered uncertainty. In this presentation, we will present some of the results that we have achieved in an EU Horizon2020 project (U-Test) in this regard. We, first, demonstrate the modeling of test ready models of CPSs together with subjective uncertainty using the Uncertainty Modeling Framework. Second, we present two uncertainty-based test case generation and four test case minimization techniques relying on the test ready models founded on Uncertainty theory and multi-objective search. Third, we present the evaluation of the test case generation and minimization techniques that was conducted to select the best strategy to be used in the practice to test a real CPS. Fourth, we present the results of testing a real CPS with the selected best strategy.},
	keywords = {Cyber-Physical System, Model-based Testing, Search-Based Testing, Uncertainty},
	author = {Ali, Shaukat}
}
@techreport {24846,
	title = {Modeling Healing Behaviors of Cyber-Physical Systems with Uncertainty to Support Automated Testing},
	year = {2016},
	publisher = {Simula Research Laboratory},
	author = {Ma, Tao and Ali, Shaukat and Yue, Tao}
}
@misc {24895,
	title = {Modeling Uncertainty in Complex Software Systems to Support Testing},
	howpublished = {Systems Engineering Domain Special Interest Group (SE DSIG), OMG Technical Meeting, Orlando, USA},
	year = {2016},
	author = {Ali, Shaukat and Yue, Tao}
}
@inproceedings {23904,
	title = {Nonconformity Resolving Recommendations for Product Line Configuration},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2016},
	month = {04/2016},
	abstract = {In the context of large-scale system product line engineering, manual configuration is often mandatory and therefore inevitably introduces nonconformities: violating pre-defined constraints for conformance checking. Resolving nonconformities without proper tool support is more or less random, as there are usually hundreds and thousands of configurable parameters and conformance constraints, in the context of configuring a large-scale and directly deployable system. Moreover, inter-connections among constraints and configurable parameters worsen the feasibility of manual resolving nonconformities without proper tool support. In this paper, we present an automatic approach (named as Zen-FIX) to optimally recommend solutions to resolve nonconformities by combining multi-objective search and constraint solving techniques. Solutions recommended by Zen-FIX conform to all pre-defined constraints and are optimal in terms of maximizing the overall efficiency of an interactive product configuration process. We evaluated Zen-FIX with a real- world case study containing 52454 optimization problems, with which we evaluated seven multi-objective search algorithms. Results show that MoCell significantly outperformed all the others: CellDE, IBEA, NSGA-II, PESA2, Random, SPEA2, for most of the problems, in terms of Efficiency (a combined metric of finding optimal solutions and time performance).\&nbsp;},
	author = {Lu, Hong and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@inproceedings {24438,
	title = {A Practical Guide to Select Quality Indicators for Assessing Pareto-Based Search Algorithms in Search-Based Software Engineering},
	journal = {the 38th International Conference on Software Engineering (ICSE)},
	year = {2016},
	pages = {631-642},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Li, Yan and Liaaen, Marius}
}
@inproceedings {24434,
	title = {A Practical Use Case Modeling Approach to Specify Crosscutting Concerns: Industrial Applications},
	journal = {International Conference on Software Reuse (ICSR)},
	year = {2016},
	author = {Yue, Tao and Zhang, Huihui and Ali, Shaukat and Liu, Chao}
}
@misc {25131,
	title = {Report on Dissemination and Communication Activities (Year 2)},
	year = {2016},
	author = {Ali, Shaukat}
}
@misc {24666,
	title = {Report on Dissemination and Communication plan},
	year = {2016},
	note = {This is a public deliverable for the U-Test project (Deliverable D6.1)},
	publisher = {None},
	abstract = {The objective of this deliverable is to define the dissemination and communication strategy and as well present record of every activity performed during the first year of the project; e-newsletter, brochures, events attended and promoted, community building activities, publications of technical and specialized articles in high impact journals. U-Test is promoted through various channels. This includes communication of project concepts and results to a wider audience beyond academically and commercially interested stakeholders. This report plans in detail dissemination, communication and awareness activities that aim to:1-Inform the research community of the state-of-the-art developments taking place in the project and how the project may affect its research field2-Foster community building and to realize impact on industry and research in Europe and worldwide3-Inform user groups and the public of the state-of-the-art developments taking place in the project and its socio-economic impactsThis deliverable reports dissemination and communication activities took place during the first reporting period.},
	author = {Ali, Shaukat}
}
@misc {24667,
	title = {Report on Selection of Standardization Bodies},
	year = {2016},
	note = {This is a public deliverable of the U-Test project (Deliverable D6.2)},
	abstract = {This deliverable (D.6.2) describes the work (Task 6.1) for selecting relevant standardization bodies and standards, and summarizing the standardization activities of the U-Test consortium in 2015. The deliverable serves as a reference for developing the U-Test frameworks and provides an overview of the standards that are relevant to the U-Test projects, which can be: used as references when developing the U-Test frameworks, considered as potential candidates where the U-Test outcomes can be promoted, and the targets where the U-Test consortium can potentially contribute within the scope of the U-Test project.},
	author = {Ali, Shaukat}
}
@inproceedings {24633,
	title = {Search-Based Cost-Effective Test Case Selection within a Time Budget: An Empirical Study},
	journal = {Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2016},
	month = {07/2016},
	pages = {1085-1092},
	address = {Denver, Colorado, USA},
	abstract = {Due to limited time and resources available for execution, test case selection always remains crucial for cost-effective testing. It is even more prominent when test cases require manual steps, e.g., operating physical equipment. Thus, test case selection must consider complicated trade-offs between cost (e.g., execution time) and effectiveness (e.g., fault detection capability). Based on our industrial collaboration within the Maritime domain, we identified a real-world and multi-objective test case selection problem in the context of robustness testing, where test case execution requires human involvement in certain steps, such as turning on the power supply to a device. The high-level goal is to select test cases for execution within a given time budget, where test engineers provide weights for a set of objectives, depending on testing requirements, standards, and regulations. To address the identified test case selection problem, we defined a fitness function including one cost measure, i.e., Time Difference (TD) and three effectiveness measures, i.e., Mean Priority (MPR), Mean Probability (MPO) and Mean Consequence (MC) that were identified together with test engineers. We further empirically evaluated eight multi-objective search algorithms, which include three weight-based search algorithms (e.g., Alternating Variable Method) and five Pareto-based search algorithms (e.g., Strength Pareto Evolutionary Algorithm 2 (SPEA2)) using two weight assignment strategies (WASs). Notice that Random Search (RS) was used as a comparison baseline. We conducted two sets of empirical evaluations: 1) Using a real world case study that was developed based on our industrial collaboration; 2) Simulating the real world case study to a larger scale to assess the scalability of the search algorithms. Results show that SPEA2 with either of the WASs performed the best for both the studies. Overall, SPEA2 managed to improve on average 32.7\%, 39\% and 33\% in terms of MPR, MPO and MC respectively as compared to RS.},
	keywords = {Search-Based Testing},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao}
}
@inproceedings {24074,
	title = {Search-based Decision Ordering to Facilitate Product Line Engineering of Cyber-Physical System},
	journal = {International Conference on Model-Driven Engineering and Software Development},
	year = {2016},
	publisher = {IEEE},
	author = {Yue, Tao and Ali, Shaukat and Lu, Hong and Nie, Kunming}
}
@techreport {24867,
	title = {Specifying Uncertainty in Use Case Models in Industrial Settings},
	year = {2016},
	publisher = {Simula Research Laboratory},
	abstract = {Latent uncertainty in the context of software-intensive systems (e.g., Cyber-Physical Systems (CPSs)) demands explicit attention right from the start of development. Use case modeling\&mdash;a commonly used method for specifying requirements in practice, should also be extended for explicitly specifying uncertainty. To this end, we extend the Restricted Use Case Modeling (RUCM) methodology and its supporting tool to specify uncertainty as part of system requirements. Such uncertainties include those caused by insufficient domain expertise of stakeholders, disagreements among them, and known uncertainties pertaining to assumptions about the environment of the system. The extended RUCM, called U-RUCM, inherits the features of RUCM, such as automated analyses and generation of models, to mention but a few. Consequently, U-RUCM provides all the key benefits offered by RUCM (i.e., reducing ambiguities in requirements), but in addition, it allows specification of uncertainties with the possibilities of reasoning and refining existing ones and even uncovering unknown ones.We evaluated U-RUCM in the context of the U-Test project (www.u-test.eu) with two industrial CPS case studies. Evaluation results showed that, with U-RUCM, we were able to get a significantly better and more precise characterization of the uncertainties involved compared to RUCM. This suggests that U-RUCM is an effective tool for dealing with uncertainty in requirements engineering. We present our experience, lessons learned, and future challenges based on the two industrial case studies.},
	keywords = {Belief, Uncertainty, Use Case Modeling},
	author = {Zhang, Man and Yue, Tao and Ali, Shaukat and Selic, Bran and Okariz, Oscar and Norgren, Roland and Intxausti, Karmele}
}
@techreport {24565,
	title = {Standardization Bodies and Standards Relevant for Uncertainty Modelling},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	keywords = {Cyber-Physical Systems, Object Management Group, Standards, Testing, Uncertainty},
	author = {Yue, Tao and Ali, Shaukat and Zhang, Man and Pradhan, Dipesh}
}
@inproceedings {24632,
	title = {STIPI: Using Search to Prioritize Test Cases based on Multi-Objectives Derived from Industrial Practice},
	journal = {The 28th International Conference on Testing Software and Systems (ICTSS) },
	year = {2016},
	month = {10/2016},
	pages = {172-190},
	publisher = {Lecture Notes in Computer Science, Springer Verlag},
	address = {Graz, Austria},
	abstract = {The importance of cost-effective test case prioritization is undeniable in automated testing practice in industry. Such prioritization typically relies on various cost and effective objectives. This paper focuses on prioritizing test cases developed to test product lines of Video Conferencing Systems (VCSs) at Cisco Systems, Norway. Each test case requires setting up configurations of a set of VCSs, invoking a set of test APIs with specific inputs, and checking the status of the VCSs. Based on these characteristics and information available about the execution of test cases (e.g., number of faults detected), we identified that the test case prioritization problem in our particular context should focus on achieving high coverage of configurations, test APIs, statuses, and high fault detection capability as fast as possible. We propose a search-based test case prioritization approach (named STIPI) to solve this problem by defining a fitness function with four objectives and integrating it with the widely applied multi-objective Non-dominated Sorting Genetic Algorithm II. We compared STIPI with random search (RS), Greedy algorithm, and three approaches adapted from literature, using three real sets of test cases from Cisco with four time budgets (25\%, 50\%, 75\% and 100\%). Results show that STIPI significantly outperformed the selected approaches and managed to achieve better performance than RS for on average 39.9\%, 18.6\%, 32.7\% and 43.9\% for the coverage of configurations, test APIs, statuses and fault detection capability, respectively.},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@techreport {24571,
	title = {STIPI: Using Search to Prioritize Test Cases based on Multi-Objectives Derived from Industrial Practice},
	year = {2016},
	note = {This research is supported by the Research Council of Norway (RCN) funded Certus SFI. Shuai Wang is also supported by the RFF Hovedstaden funded MBE-CR project. Shaukat Ali and Tao Yue are also supported by the RCN funded Zen-Configurator project, the EU Horizon 2020 project funded U-Test, the RFF Hovedstaden funded MBE-CR project and the RCN funded MBT4CPS project.\&nbsp;},
	publisher = {Simula Research Laboratory},
	keywords = {Configurations, Search, Test APIs, Test Case Prioritization},
	author = {Pradhan, Dipesh and Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@article {EMSE2014,
	title = {A Systematic Test Case Selection Methodology for Product Lines: Results and Insights From an Industrial Case Study},
	journal = {Empirical Software Engineering},
	volume = {21},
	year = {2016},
	month = {08/2016},
	pages = {1586-1622},
	publisher = {Springer},
	abstract = {In the context of product lines, test case selection aims at obtaining a set of relevant test cases for a product from the entire set of test cases available for a product line. While working on a research-based innovation project on automated testing of product lines of Video Conferencing Systems (VCSs) developed by Cisco, we felt the need to devise a cost-effective way of selecting relevant test cases for a product. To fulfill such need, we propose a systematic and automated test selection methodology using: 1) Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line; 2) Component Family Model for Testing (CFM\_T) to model the structure of test case repository; 3) A tool to automatically build restrictions from CFM\_T to FM\_T and traces from CFM\_T to the actual test cases. Using our methodology, a test engineer is only required to select relevant features through FM\_T at a higher level of abstraction for a product and the corresponding test cases will be obtained automatically. We evaluate our methodology by applying it to a VCS product line called Saturn with seven commercial products and the results show that our methodology can significantly reduce cost measured as test selection time and at the same time achieves higher effectiveness (feature coverage, feature pairwise coverage and fault detection) as compared with the current manual process. Moreover, we conduct a questionnaire-based study to solicit the views of test engineers who are involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our methodology in their current practice. Finally, we present a set of lessons learnt while applying product line engineering at Cisco for test case selection.},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@article {24649,
	title = {Tackling Uncertainty in Cyber-Physical Systems with Automated Testing},
	journal = {ADA User Journal},
	volume = {37},
	year = {2016},
	note = {\&nbsp;This work is supported by EU project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {ADA},
	abstract = {The U-Test-EU project aims at developing new methods and techniques for testing Cyber-Physical Systems (CPSs) under uncertainty. This paper aims to provide the current status of the results achieved in the project during the first one and half years. Our ultimate aim is to enable collaboration among several Horizon2020 projects focusing on CPSs. This paper focuses on the research results from the following four perspectives in the context of the project: 1) Understanding uncertainty in CPSs, 2) Modeling uncertainty in CPSs to support automated testing, 3) Discovering unspecified uncertainties, 4) Testing CPSs under the specified and discovered uncertainties. In addition to the research results, we also present a set of standardization activities that are planned in the project with the final goal of bringing results to a wider audience than the targeted projects and consortium of the project.},
	keywords = {Cyber-Physical Systems, Search-Based Software Engineering, Testing, Uncertainty},
	author = {Ali, Shaukat and Yue, Tao and Zhang, Man}
}
@misc {24661,
	title = {Tackling Uncertainty in Cyber-Physical Systems with Automated Testing},
	howpublished = {DE-CPS Workshop, Pisa, Italy},
	year = {2016},
	abstract = {The U-Test project (www.u-test.eu) aims at developing new methods and techniques for testing Cyber-Physical Systems (CPSs) under uncertainty. This position paper aims to provide the current status of the results achieved in the project during the first year. Our ultimate aim is to facilitate collaboration among various Horizon2020 projects focusing on CPSs. This position paper focuses on the research results from the following four perspectives: 1) Understanding Uncertainty in CPSs, 2) Modeling Uncertainty in CPSs, 3) Discovering Uncertainties, and 4) Testing Uncertainty. In addition to the research results, we also present a set of standardization activities that are planned in the project.},
	keywords = {Cyber-Physical Systems, Model-based Testing, Testing, Uncertainty, Uncertainty Modeling},
	author = {Ali, Shaukat and Yue, Tao and Zhang, Man}
}
@misc {24972,
	title = {Testing Cyber-Physical Systems in Uncertainty},
	howpublished = {IEEE 10th International Conference on Open Source Systems \& Technologies (ICOSST), Lahore, Pakistan and 14th International Conference on Frontiers of Information Technology (FIT) Islamabad Pakistan},
	year = {2016},
	author = {Ali, Shaukat}
}
@inproceedings {24645,
	title = {Towards Mutation Analysis for Use Cases},
	journal = {ACM/IEEE 19th International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	year = {2016},
	publisher = {ACM/IEEE},
	author = {Zhang, Huihui and Yue, Tao and Ali, Shaukat and Liu, Chao}
}
@techreport {24547,
	title = {Towards Mutation Analysis for Use Cases},
	year = {2016},
	publisher = {Simula Research Laboratory},
	abstract = {Requirements inspection is a well-known method for detecting defects. Various defect detection techniques for requirements inspection have been widely applied in practice such as checklist and defect-based techniques. Use case modeling is a widely accepted requirements specification method in practice; therefore, inspecting defects in use case models in a cost-effective manner is an important challenge. However, there does not exist a systematic mutation analysis approach for evaluating inspection techniques for use case models. As the first step towards a fully-fledged mutation analysis for use case models, in this paper we present the methodology we followed to systematically derive mutation operators for use case models. More specifically, we first propose a defect taxonomy defining 94 defect types, based on the IEEE Std. 830-1998 standard. Second, we systematically applied the basic guide words of the standardized Hazard and Operability Study (HAZOP) methodology to define 189 mutation operators, together with the defect taxonomy. Last, we define a set of guidelines for devising defect seeding strategies. The proposed methodology was evaluated by a real world case study and six case studies taken from the literature. Results show that all the RUCM mutation operators are feasible to apply and the defect taxonomy is the most comprehensive one to compare with the literature.},
	author = {Zhang, Huihui and Yue, Tao and Ali, Shaukat and Liu, Chao}
}
@techreport {24435,
	title = {Uncertainty Modeling Framework for the Integration Level V.1},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	keywords = {Cyber-Physical Systems, Search-Based Software Engineering, Testing, Uncertainty},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Phu Hong {Nguyen}}
}
@misc {25208,
	title = {Uncertainty Modeling Framework V.2},
	year = {2016},
	publisher = {None},
	abstract = {This is a public deliverable for the project.},
	author = {Ali, Shaukat}
}
@misc {25207,
	title = {Uncertainty Modeling Framework Version 1},
	year = {2016},
	publisher = {None},
	abstract = {This is a public deliverable for the project.},
	author = {Ali, Shaukat}
}
@misc {24713,
	title = {Uncertainty Modeling (Request For Information)},
	year = {2016},
	publisher = {OMG},
	author = {Yue, Tao and Ali, Shaukat and Selic, Bran}
}
@misc {24687,
	title = {Uncertainty modeling (UM) - RFI},
	howpublished = {OMG ADTF Technical Meeting, Orlando, USA},
	year = {2016},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {24896,
	title = {Uncertainty modeling (UM) - RFI Presentation and voting for RFI issuance},
	howpublished = {ADTF at OMG. OMG Technical Meeting, Chicago, September},
	year = {2016},
	author = {Ali, Shaukat}
}
@misc {24665,
	title = {Uncertainty Taxonomy for Cyber-Physical Systems},
	year = {2016},
	note = {This is a public deliverable of the U-Test project (Deliverable D1.2).},
	publisher = {None},
	abstract = {Uncertainty is inherent in Cyber-Physical Systems (CPS). Dealing with uncertainty in CPS in a cost-effective manner is imperative for their reliable operations. Since designing, developing, and testing modern and highly sophisticated CPS is an expanding field, a step towards supporting handling uncertainty is to identify, define, and classify uncertainties at various levels of CPS. This will help develop a systematic and comprehensive understanding of uncertainty. To that end, we propose a taxonomy of uncertainty specifically designed for CPS. Since the study of uncertainty in CPS development and testing is still irrelatively unexplored, this taxonomy was derived in a large part by reviewing existing work on uncertainty in other fields, including philosophy, physics, statistics, and healthcare. The taxonomy is mapped to the three logical levels of CPS: Application, Infrastructure, and Integration.\&nbsp;},
	author = {Ali, Shaukat}
}
@misc {25209,
	title = {Uncertainty Testing Framework V.1},
	year = {2016},
	abstract = {This is a public deliverable for the project.},
	author = {Ali, Shaukat}
}
@inproceedings {24517,
	title = {Understanding Uncertainty in Cyber-Physical Systems: A Conceptual Model},
	journal = {European Conference on Modelling Foundations and Applications(ECMFA)},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	abstract = {Uncertainty is intrinsic in most technical systems, including Cyber-Physical Systems (CPS). Therefore, handling uncertainty in a graceful manner during the real operation of CPS is critical. Since designing, developing, and testing modern and highly sophisticated CPS is an expanding field, a step towards dealing with uncertainty is to identify, define, and classify uncertainties at various levels of CPS. This will help develop a systematic and comprehensive understanding of uncertainty. To that end, we propose a conceptual model for uncertainty specifically designed for CPS. Since the study of uncertainty in CPS development and testing is still irrelatively unexplored, this conceptual model was derived in a large part by reviewing existing work on uncertainty in other fields, including philosophy, physics, statistics, and healthcare. The conceptual model is mapped to the three logical levels of CPS: Application, Infrastructure, and Integration. It is captured using UML class diagrams, including relevant OCL constraints. To validate the conceptual model, we identified, classified, and specified uncertainties in two distinct industrial case studies},
	keywords = {Cyber-Physical Systems, Testing, Uncertainty},
	author = {Zhang, Man and Selic, Bran and Ali, Shaukat and Yue, Tao and Okariz, Oscar and Norgren, Roland}
}
@article {24478,
	title = {Zen-ReqOptimizer: A Search-based Approach for Requirements Assignment Optimization},
	journal = {Empirical Software Engineering},
	volume = {22},
	year = {2016},
	month = {03/2016},
	pages = {175-234},
	publisher = {Springer},
	author = {Li, Yan and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@inproceedings {23472,
	title = {Applying A Restricted Natural Language Based Test Case Generation Approach in An Industrial Context},
	journal = {International Symposium on Software Testing and Analysis (ISSTA), Maryland, USA, July 14-17 },
	year = {2015},
	abstract = {Based on our experience of collaborating with industry, we observed that test case generation usually relies on test case specifications (TCSs), commonly written in natural language (NL), specifying test cases of a System Under Test at a high level of abstraction. In practice, TCSs are commonly used by test engineers as reference documents to perform any of the following activities: 1) Manually executing test cases documented as TCSs; 2) Manually coding test cases in a test scripting language for automated test case execution. In the latter case, the gap between TCSs and executable test cases has to be filled by test engineers, requiring a significant amount of coding effort and domain knowledge. In addition, both cases prefer an easy-to-apply and a NL based TCS language.Motivated by the above observations from the industry, we first propose, in this paper, a TCS language, named as Restricted Test Case Modeling (RTCM), which is based on NL and composed of an easy-to-use template, a set of carefully selected restriction rules and keywords. Second, we propose a test case generation tool (aToucan4Test), which takes TCSs in RTCM as input and generates either manual test cases or automatically executable test cases, based on various coverage criteria defined on RTCM. To assess the applicability of RTCM, we manually modeled two industrial case studies and examined 30 automatically generated TCSs. To evaluate aToucan4Test, we modeled three subsystems of a Video Conferencing System developed by Cisco Systems, Norway (our industrial partner) and automatically generated executable test cases. These test cases were successfully executed on two commercial software versions. In the paper, we also discuss our experience of applying RTCM and aToucan4Test in an industrial context and compare our proposed approach with other model-based testing methodologies.},
	author = {Yue, Tao and Ali, Shaukat and Zhang, Man}
}
@misc {24744,
	title = {Applying A Restricted Natural Language Based Test Case Generation Approach in An Industrial Context},
	howpublished = {International Symposium on Software Testing and Analysis (ISSTA)},
	year = {2015},
	publisher = {ACM},
	address = {Baltimore, USA},
	author = {Ali, Shaukat}
}
@article {23452,
	title = {Automated Product Line Test Case Selection: Industrial Case Study and Controlled Experiment},
	journal = {Journal of Software and Systems Modeling},
	year = {2015},
	month = {04/2015},
	pages = {1-25},
	publisher = {Springer},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We evaluated our methodology via three different ways: 1) We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly; 2) We conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM_T and CFM_T. The results show that test engineers are positive about adapting our methodology and models (FM_T and CFM_T) in their current practice; 3) We conducted a controlled experiment with 20 graduate students to assess the performance (i.e., cost, effectiveness and efficiency) of our automated methodology as compared with the manual approach. The results showed that our methodology is cost-effective as compared with the manual approach and at the same time its efficiency is not affected by the increased complexity of products.\&nbsp;},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud and Liaaen, Marius}
}
@article {jss2014,
	title = {Cost-Effective Test Suite Minimization in Product Lines Using Search Techniques},
	journal = {Journal of Systems and Software},
	volume = {103},
	year = {2015},
	month = {05/2015},
	pages = {370{\textendash}391},
	publisher = {Elsevier},
	abstract = {Cost-effective testing of a product in a product line requires obtaining a set of relevant test cases from the entire test suite via test selection and minimization techniques. In this paper, we particularly focus on test minimization for product lines, which identifies and eliminates redundant test cases from test suites in order to reduce the total number of test cases to execute, thereby improving the efficiency of testing. However, such minimization may result in the minimized test suite with low test coverage, low fault revealing capability, low priority test cases, and require more time than the allowed testing budget (e.g., time) as compared to the original test suite. To deal with the above issues, we formulated the minimization problem as a search problem and defined a fitness function considering various optimization objectives based on the above issues. To assess the performance of our fitness function, we conducted an extensive empirical evaluation by investigating the fitness function with three weight-based Genetic Algorithms (GAs) and seven multi-objective search algorithms using an industrial case study and 500 artificial problems inspired from the industrial case study. The results show that Random-Weighted Genetic Algorithm (RWGA) significantly outperforms the other algorithms since RWGA can balance all the objectives together by dynamically updating weights during each generation. Based on the results of our empirical evaluation, we also implemented a tool called TEst Minimization using Search Algorithms (TEMSA) to support test minimization using various search algorithms in the context of product lines.},
	doi = {10.1016/j.jss.2014.08.024},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@inproceedings {23529,
	title = {Cyber-Physical System Product Line Engineering: Comprehensive Domain Analysis and Experience Report},
	journal = {The 19th International Software Product Line Conference (SPLC). Nashville, USA, July 20-24},
	year = {2015},
	abstract = {Cyber-Physical Systems (CPSs) are the future generation of highly connected embedded systems having applications in diverse domains including Oil and Gas. Employing Product Line Engineering (PLE) is believed to bring potential benefits with respect to reduced cost, higher productivity, higher quality, and faster time-to-market. However, relatively few industrial field studies are reported regarding the application of PLE to develop large-scale systems, and more specifically CPSs. In this paper, we report about our experiences and insights gained from investigating the application of model-based PLE at a large international organization developing subsea production systems (typical CPSs) to manage the exploitation of oil and gas production fields. We report in this paper 1) how two systematic domain analyses (on requirements engineering and product configuration/derivation) were conducted to elicit CPS PLE requirements and challenges, 2) key results of the domain analysis (commonly observed in other domains), and 3) our initial experience of developing and applying two Model Based System Engineering (MBSE) PLE solution to address some of the requirements and challenges elicited during the domain analyses.\&nbsp;},
	author = {Yue, Tao and Ali, Shaukat and Selic, Bran}
}
@misc {24745,
	title = {Cyber-Physical System Product Line Engineering: Comprehensive Domain Analysis and Experience Report},
	howpublished = {The 19th International Software Product Line Conference (SPLC)},
	year = {2015},
	publisher = {ACM},
	address = {Nashville, USA},
	author = {Ali, Shaukat}
}
@misc {24743,
	title = {Evolving, Modelling and Testing Realistic Uncertain Behaviours of Cyber-Physical Systems},
	howpublished = {Testing in Practice track of International Conference on Software Testing (ICST)},
	year = {2015},
	publisher = {IEEE},
	author = {Ali, Shaukat}
}
@inproceedings {23733,
	title = {Facilitating Requirements Inspection with Search-Based Selection of Diverse Use Case Scenarios},
	journal = {9th EAI International Conference on Bio-inspired Information and Communications Technologies (formerly BIONETICS)},
	year = {2015},
	author = {Zhang, Huihui and Yue, Tao and Ali, Shaukat and Liu, Chao}
}
@article {23546,
	title = {Improving the Performance of OCL Constraint Solving with Novel Heuristics for Logical Operations: A Search-Based Approach},
	journal = {The Empirical Software Engineering Journal (EMSE)},
	year = {2015},
	pages = {1-44},
	publisher = {Springer},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Khalid, Maham and Arcuri, Andrea}
}
@misc {25447,
	title = {Model-based engineering (MBE) Of Cyber-Physical Systems},
	howpublished = {Aker Solutions, Oslo},
	year = {2015},
	author = {Ali, Shaukat}
}
@misc {24762,
	title = {Model-based testing of product lines in an industrial context},
	howpublished = {University of Waterloo Seminar},
	year = {2015},
	address = {Waterloo, Canada},
	author = {Ali, Shaukat}
}
@misc {24678,
	title = {Modelling and Testing Cyber-Physical Systems: Robustness, Product Line and Self * Properties},
	howpublished = {Ericsson, Sweden},
	year = {2015},
	author = {Ali, Shaukat and Yue, Tao}
}
@techreport {25290,
	title = {Modelling and Validating Real-Time Properties at the Requirements Level},
	number = {2015-2},
	year = {2015},
	note = {This technical report is an earlier version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Zhang, Huihui and Yue, Tao and Ali, Shaukat and Wu, Ji}
}
@techreport {23934,
	title = {A Practical Use Case Modeling Approach to Specify Crosscutting Concerns: Industrial Applications},
	number = {2012-04},
	year = {2015},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Zhang, Huihui and Ali, Shaukat and Liu, Chao}
}
@misc {24683,
	title = {Research Projects on Modeling Cyber-Physical Systems},
	howpublished = {OMG Systems Engineering Domain Special Interest Group Meeting, Berlin, Germany},
	year = {2015},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {24761,
	title = {Systematic Model and Search-Based Testing of Cyber-Physical Systems},
	howpublished = {User Conference on Advanced Automated Testing (UCAAT)},
	year = {2015},
	publisher = {ETSI},
	address = {Sophia Antipolis, France},
	author = {Ali, Shaukat}
}
@misc {24685,
	title = {Testing Cyber-Physical Systems under Realistic and Unknown Uncertainty by Combining Model and Search-Based Approaches},
	howpublished = {DE-CPS 2015, Madrid, Spain},
	year = {2015},
	author = {Ali, Shaukat and Yue, Tao}
}
@misc {24679,
	title = {Uncertainty in Complex CPS},
	howpublished = {NASA Jet Propulsion Laboratory (JPL), Pasadena, USA},
	year = {2015},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {24688,
	title = {An Uncertainty Taxonomy},
	howpublished = {OMG ADTF Technical Meeting, La Jolla, USA},
	year = {2015},
	author = {Yue, Tao and Ali, Shaukat and Selic, Bran}
}
@techreport {23476,
	title = {Understanding Uncertainty in Cyber-Physical Systems: A Conceptual Model},
	year = {2015},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Uncertainty is intrinsic in most technical systems, including Cyber-Physical Systems (CPS). Therefore, handling uncertainty in a graceful manner during the real operation of CPS is critical. Since designing, developing, and testing modern and highly sophisticated CPS is an expanding field, a step to-wards dealing with uncertainty is to identify, define, and classify uncertainties at various levels of CPS. This will help develop a systematic and comprehen-sive understanding of uncertainty. To that end, we propose a conceptual model for uncertainty specifically designed for CPS. Since the study of uncertainty in CPS development and testing is still irrelatively unexplored, this conceptual model was derived in a large part by reviewing existing work on uncertainty in other fields, including philosophy, physics, statistics, and healthcare. The con-ceptual model is mapped to the three logical levels of CPS: Application, Infra-structure, and Integration. It is captured using UML class diagrams, including relevant OCL constraints. To validate the conceptual model, we identified, clas-sified, and specified uncertainties in two distinct industrial case studies.},
	keywords = {Cyber-Physical Systems, Model-based Testing, Search-Based Testing, UML, Uncertainty},
	author = {Zhang, Man and Selic, Bran and Ali, Shaukat and Yue, Tao and Okariz, Oscar and Norgren, Roland}
}
@inproceedings {23561,
	title = {UPMOA: An Improved Search Algorithm to Support User- Preference Multi-Objective Optimization},
	journal = {The 26th IEEE International Symposium on Software Reliability Engineering (ISSRE). Gaithersburg, USA, November 2-5.},
	year = {2015},
	pages = {393-404},
	abstract = {Multi-objective search algorithms (e.g., non- dominated sorting genetic algorithm II (NSGA-II)) have been applied extensively to solve various multi-objective optimization problems in software engineering such as problems in testing. However, existing multi-objective algorithms usually treat all the objectives with equivalent priorities and do not provide a mechanism to reflect various user preferences when guiding search. The need to have such a mechanism was observed in one of our industrial projects on applying search algorithms for test optimization of a product line of Videoconferencing Systems (VCSs) called Saturn, where user preferences must be incorporated into optimization objectives, based on domain knowledge of test engineers for VCS testing. To address this, we propose an extension to the most commonly-used multi-objective search algorithm NSGA-II, which has shown promising results with user preferences. We name the extension as User-Preference Multi-Objective Optimization Algorithm (UPMOA), which includes a user preference indicator p\&nbsp;and is based on existing weight assignment strategies. We empirically evaluated UPMOA with two industrial problems focusing on optimizing the test execution system for Saturn in Cisco. To assess the performance and scalability of UPMOA, inspired by the two industrial problems, in total we created 64000 artificial problems with 128 different sets of user preferences. The evaluation includes two aspects: 1) Three weight assignment strategies together with UPMOA were empirically evaluated to identify a best weight assignment strategy for p. Results show that the Uniformly Distributed Weights (UDW) strategy can assist UPMOA in achieving the best performance; 2) UPMOA was compared with three representative multi-objective search algorithms (including NSGA-II) and results show that UPMOA significantly outperformed the others and has the ability to solve problems with a wide range of complexity.\&nbsp;},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius}
}
@inproceedings {23473,
	title = {U-Test: Evolving, Modelling and Testing Realistic Uncertain Behaviours of Cyber-Physical Systems},
	journal = {Abstract for the Testing in Practice track of International Conference on Software Testing (ICST)},
	year = {2015},
	note = {Extended abstract for industry\&nbsp;track presentation\&nbsp;},
	publisher = {IEEE},
	address = {Graz, Austria},
	abstract = {Short Abstract:The goal of the U-Test project (a recently funded project under the EU Horizon2020 program (http://ec.europa.eu/programmes/horizon2020/) is to improve the dependability of Cyber-Physical Systems (CPSs), via cost-effective, model-based and search-based testing of CPSs under unknown uncertainty. Unknown uncertainty is the state of a CPS that can only be determined at the runtime as opposed to known uncertainty that is known at the design time and outcome from uncertainty is undesirable. To achieve our goal, we will advance the current state-of-art of testing CPSs by developing a novel- solution based on sound theoretical foundation for uncertainty testing in the following steps: 1) Developing a light-weight modelling solution with rich formalism to support minimal modelling of known uncertainty with risk information; 2) Intelligently evolving known uncertainty models towards realistic and risky unknown uncertainty models (evolved models) using search algorithms (e.g., genetic algorithms mimicking natural selection); and 3) Automatically generating test cases from the evolved models to test a CPS under unknown uncertainty to ensure that the CPS continues to operate properly and possibly at a reduced quality of operation, rather than failing completely.\&nbsp;},
	author = {Ali, Shaukat and Yue, Tao}
}
@techreport {25291,
	title = {Zen-CI: Combining Automated Configuration Inference and Incremental Conformance Checking to Support Interactive Configuration of Product Lines},
	number = {2015-05},
	year = {2015},
	publisher = {Simula Research Laboratory},
	author = {Lu, Hong and Yue, Tao and Ali, Shaukat and Zhang, Li}
}
@misc {24696,
	title = {Zen-RUCM},
	howpublished = {MODELS 2015, Ottawa, Canada},
	year = {2015},
	type = {Tutorial},
	author = {Yue, Tao and Ali, Shaukat}
}
@inproceedings {Simula.simula.2606,
	title = {Applying Search Algorithms for Optimizing Stakeholders Familiarity and Balancing Workload in Requirements Assignment},
	journal = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2014},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	abstract = {During the early phase of project development lifecycle of large scale cyber-physical systems, a large number of requirements are needed to be assigned to different stakeholders from different organizations or different departments of the same organization for reviewing, clarifying and checking their conformance to industry standards and government or other regulations. These requirements have different characteristics such as various extents of importance to the organization, complexity, and dependencies between each other, thereby requiring different effort (workload) to review and clarify. While working with our industrial partners in the domain of cyber-physical systems, we discovered an optimization problem, where an optimal solution is required for assigning requirements to different stakeholders by maximizing their familiarities to the assigned requirements and at the same time balancing the overall workload of each stakeholder. In this direction, we propose a fitness function which takes into the account all the above-mentioned factors. The proposed fitness function is investigated with four search algorithms: (1+1) Evolutionary Algorithm (EA), Genetic Algorithm, and Alternating Variable Method, whereas Random Search is used as a comparison base line. We empirically evaluated their performance in terms of finding an optimal solution using a large-scale industrial case study and 120 artificial problems with varying complexity. Our results show that (1+1) EA gives the best results together with our proposed fitness function as compared to the other three algorithms.},
	keywords = {Conference},
	author = {Yue, Tao and Ali, Shaukat}
}
@article {Simula.simula.2514,
	title = {Applying UML/MARTE on Industrial Projects: Challenges, Experiences, and Guidelines},
	journal = {Software and Systems Modeling},
	number = {1619-1366},
	year = {2014},
	month = {March 2014},
	pages = {1-19},
	publisher = {Springer Publishing Company},
	abstract = {MARTE (Modeling and Analysis of Real-Time and Embedded Systems) is a UML profile, which has been developed to model concepts specific to Real-Time and Embedded Systems (RTES). In the last five years, we have applied UML/MARTE to three distinct industrial problems in three industry sectors: architecture modeling and configuration of large-scale and highly configurable integrated control systems, model-based robustness testing of communication-intensive systems, and model-based environment simulator generation of large-scale RTES for testing. In this paper, we report on our experience of solving these problems by applying UML/MARTE on four industrial case studies. We highlight the challenges we faced with respect to the industrial adoption of MARTE. Based on our combined experience, we derive a framework to guide practitioners for future applications of UML/MARTE in an industrial context. The framework provides a set of detailed guidelines that help reduce the gap between the modeling notations and real-world industrial application needs.},
	author = {Muhammad Zohaib {Iqbal} and Ali, Shaukat and Yue, Tao and Briand, Lionel Claude}
}
@techreport {Simula.simula.2864,
	title = {Assessing the Modeling of Aspect State Machines for Testing From the Perspective of Modelers},
	number = {2014-16},
	year = {2014},
	publisher = {Simula Research Laboratory},
	address = {.},
	abstract = {Aspect state machines (ASMs) are extended UML state machines that use stereotypes from a UML profile called AspectSM. ASMs were specifically developed to support model-based robustness testing at Cisco Systems Norway, Inc. In our previous experiments, we empirically evaluated ASMs from the perspectives of readability, comprehensibility, understandability, modeling errors, and modeling effort and the results showed that ASMs are significantly better than the standard UML state machines for modeling robustness behavior for testing. However, a fundamental question still remained to be answered about how modelers/testers modeling ASMs feel about their use. With this in mind, we report results from a series of controlled experiments that were conducted to evaluate subjective opinions of modelers/testers from various perspectives using several questionnaires. The results of the experiment showed that the participants found it difficult to apply AspectSM and weren\&$\#$39;t confident about their solutions. We further observed that the participants\&$\#$39; understandability and experience of applying AspectSM improved after performing various modeling activities. Although, our results seem very generic, but notice that these results provide preliminary evidence about these observations, which is missing in the Aspect-Oriented Modeling literature.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao}
}
@inproceedings {Simula.simula.2846,
	title = {Assessing the Modeling of Aspect State Machines for Testing From the Perspective of Modelers},
	journal = {The 14th International Conference on Quality Software (QSIC)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Allen, TX, USA},
	abstract = {Aspect state machines (ASMs) are extended UML state machines that use stereotypes from a UML profile called AspectSM. ASMs were specifically developed to support model-based robustness testing at Cisco Systems Norway, Inc. In our previous experiments, we empirically evaluated ASMs from the perspectives of readability, comprehensibility, understandability, modeling errors, and modeling effort and the results showed that ASMs are significantly better than the standard UML state machines for modeling robustness behavior for testing. However, a fundamental question still remained to be answered about how modelers/testers modeling ASMs feel about their use. With this in mind, we report results from a series of controlled experiments that were conducted to evaluate subjective opinions of modelers/testers from various perspectives using several questionnaires. The results of the experiment showed that the participants found it difficult to apply AspectSM and weren{\textquoteright}t confident about their solutions. We further observed that the participants{\textquoteright} understandability and experience of applying AspectSM improved after performing various modeling activities. Although, our results seem very generic, but notice that these results provide preliminary evidence about these observations, which is missing in the Aspect-Oriented Modeling literature.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao},
	editor = {T. H. {Tse} and Wong, Eric and McMillin, Bruce}
}
@inproceedings {Simula.simula.2862,
	title = {Automated and Incremental Conformance Checking to Support Interactive Product Configuration},
	journal = {The 25th IEEE International Symposium on Software Reliability Engineering (ISSRE)},
	year = {2014},
	type = {Conference},
	abstract = {In the context of product line engineering (PLE), providing immediate feedback on the correctness of a manual configuration step to users has a practical impact on whether a configuration process with tool support can be successfully adopted in practice. Model-based PLE has brought opportunities to enable automated product configuration and derivation for large-scale systems/software, in which models are used as the abstract specification of commonalities and variabilities of products of a product line. In our previous work, we have proposed a UML-based variability modeling methodology and an interactive configuration process. Based on these work, in this paper, we propose an automated and incremental conformance checking approach to ensure that the manual configuration to each variation point conforms to a set of pre-defined conformance rules specified in OCL. The proposed approach, called Zen-CC is implemented as a component of our product configuration and derivation tool, named as Zen-Configurator. The proposed approach is evaluated with two real-world case studies and results showed that the performance of Zen-CC is significantly better than a baseline algorithm checking all the conformance rules at each configuration step. Moreover, the performance of Zen-CC rarely varies during the configuration process, suggesting that our approach is scalable for configuring products with a large number of configuration points.},
	keywords = {Conference},
	author = {Lu, Hong and Yue, Tao and Ali, Shaukat and Nie, Kunming and Zhang, Li}
}
@article {Simula.simula.2904,
	title = {Empirically Evaluating OCL and Java for Specifying Constraints on UML Models},
	journal = {Software and Systems Modeling},
	year = {2014},
	publisher = {Springer International Publishing},
	abstract = {The Object Constraint Language (OCL) has been applied, along with UML models, for various purposes such as supporting model-based testing, code generation, and automated consistency checking of UML models. However, a lot of challenges have been raised in the literature regarding its applicability in industry such as extensive training, slow learning curve, and significant effort to use OCL due to lack of familiarity of practitioners. To confirm these challenges, empirical evidence is needed, which is severely lacking in the literature. To build such preliminary evidence, we report a controlled experiment that was designed to evaluate OCL by comparing it with Java, a programming language that has also been used to specify constraints on UML models. Results show that the participants using OCL perform as good as the participants working with Java in terms of three objective quality metrics (i.e., Completeness, Conformance and Redundancy) and two subjective metrics (i.e., Applicability and Confidence Level). In addition, the participants using OCL performed consistently well for all the constraints of varying complexity, while fluctuating results were obtained for the participants using Java for the same constraints. Based on the empirical evidence, we can conclude that it does not make much difference to use OCL or Java for specifying constraints on UML models. However, the participants working with OCL performed consistently well on specifying constraints of varying complexity suggesting that OCL can be used to model complicated constraints (commonly observed in industrial applications) with the same quality as for simpler constraints. Moreover, additional analyses on the constraints when using Java and OCL tools revealed that tools are needed to specify fully correct constraints that can be used to support automation.},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {24747,
	title = {Evaluating Normalization Functions With Search Algorithms for Solving OCL Constraints},
	howpublished = {26th IFIP International Conference on Testing Software and Systems, ICTSS},
	year = {2014},
	publisher = {IFIP},
	address = {Madrid, Spain},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.2818,
	title = {Evaluating Normalization Functions With Search Algorithms for Solving OCL Constraints},
	journal = {26th IFIP International Conference on Testing Software and Systems, ICTSS 2014},
	year = {2014},
	publisher = {LNCS},
	type = {Conference},
	address = {Berlin, Heidelberg},
	abstract = {The use of search algorithms requires the definition of a fitness function that guides the algorithms to find an optimal solution. The definition of a fitness function may require the use of a normalization function for various purposes such as assigning equal importance to various factors constituting a fitness function and normalizing only one factor of a fitness function to give it less/more importance than the others. In our previous work, we defined various branch distance functions (a commonly used heuristic in the literature at the code-level) corresponding to the constructs defined in the Object Constraint Language (OCL) to solve OCL constraints to generate test data for supporting automated Model-Based Testing (MBT). The definition of several of these dis-tance functions required the use of a normalization function. In this paper, we extend the empirical evaluation reported in one of the works in the literature that compares the impact of using various normalization functions for calculat-ing branch distances at the code-level on the performance of search algorithms. The empirical evaluation reported in this paper assesses the impact of the commonly used normalization functions for the branch distance calculation of OCL constraints at the model-level. Results show that for one of the newly studied algorithms Harmony Search (HS) and Random Search (RS), the use of the normalization functions has no impact on the performance of the search. However, HS achieved 100\% success rates for all the problems, where RS ob-tained very poor success rates (less than 38\%). Based on the results, we conclude that the randomness in creating a solution in search algorithms may mask the impact of using a normalization function.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao},
	editor = {Mercedes G. {Merayo} and Edgardo Montes {de Oca}}
}
@misc {24748,
	title = {Evaluating Normalization Functions With Search Algorithms for Solving OCL Constraints},
	howpublished = {26th IFIP International Conference on Testing Software and Systems, ICTSS},
	year = {2014},
	publisher = {IFIP},
	address = {Madrid, Spain},
	author = {Ali, Shaukat}
}
@inbook {Simula.simula.2780,
	title = {An Evolutionary and Automated Virtual Team Making Approach for Crowdsourcing Platforms},
	booktitle = {Cloud-based Software Crowdsourcing"},
	year = {2014},
	pages = {1-17},
	publisher = {Springer},
	organization = {Springer},
	chapter = {2},
	address = {.},
	abstract = {Crowdsourcing has demonstrated its capability of supporting various software development activities including development and testing as it can be seen by several successful crowdsourcing platforms such as TopCoder and uTest. However, to crowd source large-scale and complex software development and testing tasks, there are several optimization challenges to be addressed such as division of tasks, searching a set of registrants, and assignment of tasks to registrants. Since in crowdsourcing a task can be assigned to registrants geo-graphically distributed with various background, the quality of final task deliverables is a key issue. As the first step to improve the quality, we propose a systematic and automated approach to optimize the assignment of registrants in a crowdsourcing platform to a crowdsourcing task. The objective is to find the best fit of a group of registrants to the defined task. A few examples of factors forming the optimization problem include budget defined by the task submitter and pay expectation from a registrant, skills required by a task, skills of a registrant, task delivering deadline, and availability of a registrant. We first collected a set of commonly seen factors that have impact on the {\textquotedblleft}perfect matching{\textquotedblright} between tasks submitted and a virtual team that consists of a selected set of registrants. We then formulated the optimization objective as a fitness function-the heuristics used by search algorithms (e.g., Genetic Algorithms) to find an optimal solution. We empirically evaluated a set of well-known search algorithms in software engineering, along with the proposed fit-ness function, to identify the best solution for our optimization problem. Results of our experiments are very positive in terms of solving optimization problems in a crowdsourcing context.},
	author = {Yue, Tao and Ali, Shaukat and Wang, Shuai},
	editor = {Michael N. {Huhns} and Li, Wei and Tsai, Wei-Tek}
}
@inproceedings {Simula.simula.2844,
	title = {Exploring Model-Based Repositories for a Broad Range of Industrial Applications and Challenges},
	journal = {The 14th International Conference on Quality Software (QSIC)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Allen, TX, USA},
	abstract = {Nowadays, systems are becoming increasingly complex and large and the process of developing such large-scale systems is becoming complicated with high cost and enormous effort required. Such a complicated process has a prominent challenge to ensure the quality of delivered artifacts. Therefore there is clearly a need to facilitate reuse of developed artifacts (e.g., requirements, architecture, tests) and enable automated analyses such as risk analyses, prioritizing test cases, change impact analysis, with the objective to reduce cost, effort and improve quality. Model-based engineering provides a promising mechanism to facilitate reuse and enable automation. The key idea is to use models as the backbone of structuring repositories that contain reusable artifacts (e.g., test cases, requirements). Such a backbone model is subsequently used to enable various types of automation such as model-based testing and automated rule verification. In this paper, we report 12 industrial projects from five differ-ent industry domains that all require the construction of model-based reposito-ries to enable various types of automation. We believe using models as the backbone to structure repositories for the purpose of enabling different types of automation in different contexts is a new and non-conventional model-based development research approach. This exploratory paper will serve the basis for future research to derive a generic model-based repository.},
	author = {Yue, Tao and Ali, Shaukat},
	editor = {T. H. {Tse} and Wong, Eric and McMillin, Bruce}
}
@inproceedings {Simula.simula.2860,
	title = {A Heuristic-Based Approach to Refactor Crosscutting Behaviors in UML State Machines},
	journal = {30th International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2014},
	publisher = {IEEE Computer Society},
	type = {1},
	abstract = {UML state machines are commonly used to model the state-based behavior of communication and control systems to support various activities such as test cases and code generation. Standard UML state machines are well suited to model functional behavior; however extra-functional behavior such as robustness and security can also be directly modeled on them, but this often results in cluttered models since extra-functional behaviors are often crosscutting. Such modeling crosscutting behavior directly on UML state machines is a common practice. Aspect-Oriented Modeling (AOM) allows systematically modeling of crosscutting behavior and has shown to provide a scalable solution in the recent years. However, due to lack of familiarity of AOM in both academic and industry, extra-functional behavior is often modeled directly on UML state machines and as results those UML state machines are difficult to read and maintain. To improve the readability of already developed UML state machines and ease maintenance, we propose a set of heuristics, derived from two industrial cases studies, implemented in a tool to automatically identify commonly observed crosscutting behaviors in UML state machines and refactor them as Aspect State Machines. Such refactoring makes the state machines easier to maintain and comprehend. We present the results of applying our proposed heuristics to the existing UML state machines of two industrial case studies developed for model-based testing.},
	keywords = {Conference},
	author = {Muhammad Uzair {Khan} and Muhammad Zohaib {Iqbal} and Ali, Shaukat},
	editor = {M{\"u}ller, Hausi}
}
@inproceedings {Simula.simula.3040,
	title = {How Does the UML Testing Profile Support Risk-Based Testing?},
	journal = {2nd International Workshop on Risk Assessment and Risk-driven Testing (RISK 2014)},
	year = {2014},
	publisher = {IEEE},
	type = {Workshop},
	address = {.},
	author = {Ali, Shaukat and Yue, Tao and Alessandra Brosse {Bagnato} and Brosse, Etienne and Zhen Ru {Dai} and Hoffmann, Andreas and Wendland, Marc-Florian and Schacher, Markus},
	editor = {Schieferdecker, Ina}
}
@misc {24746,
	title = {Improved Heuristics for Solving OCL Constraints Using Search Algorithms},
	howpublished = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2014},
	publisher = {ACM},
	address = {Vancouver, Canada},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.2604,
	title = {Improved Heuristics for Solving OCL Constraints Using Search Algorithms},
	journal = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2014},
	publisher = {ACM},
	type = {Conference},
	address = {.},
	abstract = {The Object Constraint Language (OCL) is the de-facto standard for specifying constraints on Unified Modeling Language (UML) models. The specified constraints can be used for various purposes including verification, and model-based testing (e.g., test data generation). Efficiently solving OCL constraints is one of the key requirements for the practical use of OCL. Keeping this in mind, we propose an improvement in existing heuristics to solve OCL constraints using search algorithms. We evaluate our improved heuristics using two empirical studies with three search algorithms: Alternating Variable Method (AVM), (1+1) Evolutionary Algorithm (EA), and a Genetic Algorithm (GA) and used Random Search (RS) as a comparison baseline. The first empirical study was conducted using carefully designed artificial problems (constraints) to assess each individual heuristics. The second empirical study is based on an industrial case study provided by Cisco about model-based testing of Videoconferencing Systems. The results of both empirical evaluations reveal that the effectiveness of the search algorithms, measured in terms of time to solve the OCL constraints to generate data, is significantly improved when using the novel heuristics presented in this paper. In particular, our experiments show that (1+1) EA with the novel heuristics has the highest success rate among all the analyzed algorithms, as it requires the least number of iterations to solve constraints.},
	keywords = {Conference},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea}
}
@inproceedings {Simula.simula.2820,
	title = {Insights on the Use of OCL in Diverse Industrial Applications},
	journal = {8th System Analysis and Modelling Conference (SAM{\textquoteright}14)},
	year = {2014},
	publisher = {Springer},
	type = {Conference},
	address = {Switzerland},
	abstract = {The Object Constraint Language (OCL) is a widely accepted language, standardized by OMG, for specifying constraints at various meta levels (e.g., meta-models and models). Despite its wide acceptance, there is a lack of understanding about terminology and purposes for which OCL can be used. In this paper, we aim to reduce this gap and provide guidance for applying OCL in practical contexts and we report our experience of applying OCL for different industrial projects in diverse domains: Communications and Control, Oil and Gas production, Energy Equipment and Services, and Recycling. Based on our experience, first, we unify the commonly used terminology in the literature for applying OCL in different ways for addressing diverse industrial problems. Second, we report the key results of the industrial application of OCL. Finally, we provide guidance to researchers and practitioners for choosing an appropriate meta-level and purpose for their specific industrial problem at hand.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao and Muhammad Zohaib {Iqbal} and Rajwinder Kaur {Panesar-Walawege}},
	editor = {Amyot, Daniel and Pau Fonseca i. {Casas} and Mussbacher, Gunter}
}
@misc {24749,
	title = {Insights on the Use of OCL in Diverse Industrial Applications},
	howpublished = {8th System Analysis and Modelling Conference (SAM)},
	year = {2014},
	publisher = {ACM/IEEE},
	address = {Valencia, Spain},
	author = {Ali, Shaukat}
}
@techreport {25247,
	title = {A Keyword and Restricted Natural Language Based Test Case Specification Language for Automated Testing},
	year = {2014},
	note = {This technical report is an earlier version of published articles.},
	publisher = {Simula Research Laboratory},
	author = {Zhang, Man and Yue, Tao and Ali, Shaukat}
}
@misc {24758,
	title = {Mimicking Natural Phenomena to Solve Complex Industrial Problems},
	howpublished = {Seminar at Beihang University},
	year = {2014},
	address = {Beijing, China},
	author = {Ali, Shaukat}
}
@misc {24677,
	title = {Mimicking Natural Phenomena to Solve Complex Industrial Problems},
	howpublished = {JavaZone, Oslo, Norway},
	year = {2014},
	author = {Ali, Shaukat and Yue, Tao}
}
@inproceedings {Simula.simula.2848,
	title = {Model-Based Testing of Obligations},
	journal = {The 14th International Conference on Quality Software (QSIC)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Allen, TX, USA},
	abstract = {Obligations are mandatory actions that users must perform, addressing access control requirements. To ensure that such obligations are implemented correctly, an automated and systematic testing approach is often recommended. One such approach is Model-Based Testing (MBT) that allows defining cost-effective testing strategies to support rigorous testing via automation. In this paper, we present MBT for obligations by extending the Unified Modeling Language (UML) via a profile called the Obligations Profile. Based on the profile, we define a modeling methodology based on the concepts of Obligations Class Diagrams (OCDs) and Obligations State Machines (OSMs), which are standard UML Class Diagrams and UML State Machines with stereotypes from the Obligations Profile. Our methodology, using OCDs and OSMs, is automatically enforced by the validation of constraints defined in the profile. To assess the completeness and applicability of the profile and methodology, we modeled 47 obligations from four different systems. The results of our experiment show that we successfully modeled all the obligations and used 75\% of the stereotypes that we defined in the profile. In addition, using OCDs and OSMs, we automatically generate executable test cases using a standard state machine structural coverage criterion and common test data generation strategies. The effectiveness of generated test cases is assessed using mutation analysis on two systems, using mutation operators specifically designed for obligation faults. Test case executions killed 75\% of the mutants and a careful analysis further suggests that more sophisticated testing strategies must be defined to further improve testing effectiveness.},
	keywords = {Conference},
	author = {Rubab, Iram and Ali, Shaukat and Briand, Lionel Claude and Le Traon, Yves},
	editor = {T. H. {Tse} and Wong, Eric and McMillin, Bruce}
}
@inproceedings {Simula.simula.2370,
	title = {Model-Based Testing of Video Conferencing Systems: Challenges, Lessons Learnt, and Results},
	journal = {IEEE International Conference on Software Testing, Verification, and Validation (ICST)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Cleveland, OH},
	keywords = {Conference},
	author = {Ali, Shaukat and Hemmati, Hadi}
}
@article {Simula.simula.2764,
	title = {A Modeling Methodology to Facilitate Safety-Oriented Architecture Design of Industrial Avionics Software},
	journal = {Software: Practice and Experience},
	year = {2014},
	publisher = {John Wiley and Sons},
	abstract = {Ensuring that avionics software meets safety requirements at each development stage is very important to warrant the safe operation of an avionics system. Many safety requirements are imposed by various standards and industrial regulations that must be met by avionics software. One of such standards is DO-178B/C, which provides guidelines (e.g., development process and objectives to satisfy in development activities) for meeting safety requirements. This paper presents a modeling methodology including a UML profile for specifying safety requirements on a component-based architecture model and a set of design guidelines on avionics software. These safety requirements were identified from both standards (mainly DO-178B/C) and current engineering practices in the domain of avionics systems. The methodology automatically enforces these safety requirements. We have applied the methodology on an industrial autopilot system and several previously uncaught faults were revealed.},
	author = {Wu, Ji and Yue, Tao and Ali, Shaukat and Zhang, Huihui}
}
@inproceedings {Simula.simula.2600,
	title = {A MOF-Based Framework for Defining Metrics to Measure the Quality of Models},
	journal = {10th European Conference on Modelling Foundations and Applications (ECMFA)},
	year = {2014},
	publisher = {Springer International Publishing},
	type = {Conference},
	address = {Switzerland},
	abstract = {Controlled experiments in model-based software engineering, especially those involving human subjects performing modeling tasks, often require comparing models produced by experiment subjects with reference models, which are considered to be correct and complete. The purpose of such comparison is to assess the quality of models produced by experiment subjects so that experiment hypotheses can be accepted or rejected. The quality of models is typically measured quantitatively based on metrics. Manually defining such metrics for a rich modeling language is often cumbersome and error-prone. It can also result in metrics that do not systematically consider relevant details and in turn may produce biased results. In this paper, we present a framework to au-tomatically generate quality metrics for MOF-based metamodels, which in turn can be used to measure the quality of models (instances of the MOF-based met-amodels). This framework was evaluated by comparing its results with manual-ly derived quality metrics for UML class and sequence diagrams and it has been used to derive metrics for measuring the quality of UML state machine dia-grams. Results show that it is more efficient and systematic to define quality metrics with the framework than doing it manually.},
	keywords = {Conference},
	author = {Yue, Tao and Ali, Shaukat}
}
@inproceedings {splc2014,
	title = {Multi-Objective Test Prioritization in Software Product Line Testing: an Industrial Case Study},
	journal = {Software Product Line Conference (SPLC14), Florence, Italy},
	year = {2014},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	abstract = {Test prioritization is crucial for testing products in a product line considering limited budget in terms of available time and resources. In general, it is not practically feasible to execute all the possible test cases and so, ordering test case execution permits test engineers to discover faults earlier in the testing process. An efficient prioritization of test cases for one or more products requires a clear consideration of the tradeoff among various costs (e.g., time, required resources) and effectiveness (e.g., feature coverage) objectives. As an integral part of the future Cisco{\textquoteright}s test scheduling system for validating video conferencing products, we introduce a search-based multi-objective test prioritization technique, considering multiple cost and effectiveness measures. In particular, our multi-objective optimization setup includes the minimization of execution cost (e.g., time), and the maximization of number of prioritized test cases, feature pairwise coverage and fault detection capability. Based on cost-effectiveness measures, a novel fitness function is defined for such test prioritization problem. The fitness function is empirically evaluated together with three commonly used search algorithms (e.g., (1+1) Evolutionary algorithm (EA)) and Random Search as a comparison baseline based on the Cisco{\textquoteright}s industrial case study and 500 artificial designed problems. The results show that (1+1) EA achieves the best performance for solving the test prioritization problem and it scales up to solve the problems of varying complexity.},
	keywords = {Conference},
	author = {Wang, Shuai and Buchmann, David and Ali, Shaukat and Gotlieb, Arnaud and Pradhan, Dipesh and Liaaen, Marius}
}
@inproceedings {ssbse2014,
	title = {Random-Weighted Search-Based Multi-Objective Optimization Revisited},
	journal = {International Symposium on Search-Based Software Engineering (SSBSE{\textquoteright}14), Fortaleza, Brazil},
	year = {2014},
	month = {August},
	publisher = {Springer International Publishing},
	type = {Conference},
	address = {Switzerland },
	abstract = {Weight-based multi-objective optimization requires assigning appro- priate weights using a weight strategy to each of the objectives such that an overall optimal solution can be obtained with a search algorithm. Choosing weights using an appropriate weight strategy has a huge impact on the obtained solutions and thus warrants the need to seek the best weight strategy. In this pa- per, we propose a new weight strategy called Uniformly Distributed Weights (UDW), which generates weights from uniform distribution, while satisfying a set of user-defined constraints among various cost and effectiveness measures. We compare UDW with two commonly used weight strategies, i.e., Fixed Weights (FW) and Randomly-Assigned Weights (RAW), based on five cost/effectiveness measures for an industrial problem of test minimization de- fined in the context of Video Conferencing System Product Line developed by Cisco Systems. We empirically evaluate the performance of UDW, FW, and RAW in conjunction with four search algorithms ((1+1) Evolutionary Algorithm (EA), Genetic Algorithm, Alternating Variable Method, and Random Search) using the industrial case study and 500 artificial problems of varying complexi- ty. Results show that UDW along with (1+1) EA achieves the best performance among the other combinations of weight strategies and algorithms.},
	keywords = {Conference},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.2822,
	title = {A Systematic Approach to Automatically Derive Test Cases From Use Cases Specified in Restricted Natural Languages},
	journal = {8th System Analysis and Modelling Conference (SAM{\textquoteright}14)},
	year = {2014},
	publisher = {Springer},
	type = {Conference},
	address = {Switzerland},
	abstract = {In many domains, such as avionics, oil and gas, and maritime, a common practice is to derive and execute test cases manually from requirements, where both requirements and test cases are specified in NL by domain experts. The manual execution of test cases is largely dependent on the domain experts who wrote the test cases. The process of manual writing of requirements and test cases introduce ambiguity in their description and in addition test cases may not be effective since they may not be derived by systematic applying coverage criteria. In this paper, we report a systematic approach to support automatic derivation of manually executable test cases from use cases. Both use cases and test cases are specified in restricted NLs along with carefully defined templates implemented in a tool. We evaluate our approach with four case studies (in total having 30 use cases and 579 steps of flow of events), two of which are industrial case studies from the oil and gas and avionics domains. Results show that our tool was able to correctly process all the case studies and systematically (by following the carefully defined structure coverage criterion) generate 30 TCSs and 389 test cases. Moreover our approach allows defining different test coverage criteria on requirements other than the one already implemented in our tool.},
	keywords = {Conference},
	author = {Zhang, Man and Yue, Tao and Ali, Shaukat and Zhang, Huihui and Wu, Ji},
	editor = {Amyot, Daniel and Pau Fonseca i. {Casas} and Mussbacher, Gunter}
}
@inproceedings {Simula.simula.1654,
	title = {Assessing Quality and Effort of Applying Aspect State Machines for Robustness Testing: a Controlled Experiment},
	journal = {International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2013},
	month = {March},
	publisher = {IEEE},
	type = {Conference},
	address = {Luembourg},
	abstract = {Aspect-Oriented Modeling (AOM) has been the subject of intense research over the last decade and aims to provide numerous benefits to modeling, such as enhanced modularization, easier evolution, higher quality as well as reduced modeling effort. However, these benefits can only be obtained at the cost of learning and applying new modeling approaches. Studying their applicability is therefore important to assess whether they are worth using in practice. In this paper, we report a controlled experiment to assess the applicability of AOM, focusing on a recently published UML profile (AspectSM). This profile was originally designed to support model-based robustness testing in an industrial context but is applicable to the behavioral modeling of other crosscutting concerns. This experiment assesses the applicability of AspectSM from two aspects: the quality of derived state machines and the effort required to build them. With AspectSM, a crosscutting behavior is modeled using an {\textquotedblleft}aspect state machine{\textquotedblright}. The applicability of aspect state machines is evaluated by comparing them with standard UML state machines that directly model the entire system behavior, including crosscutting concerns. The quality of both aspect and standard UML state machines derived by subjects is measured by comparing them against predefined reference state machines. Results show that aspect state machines derived with AspectSM are significantly more complete and correct though AspectSM took significantly more time than the standard approach.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude},
	editor = {Le Traon, Yves}
}
@misc {24751,
	title = {Assessing Quality and Effort of Applying Aspect State Machines for Robustness Testing: a Controlled Experiment},
	howpublished = {International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2013},
	publisher = {IEEE},
	address = {Luxembourg, Luxembourg},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.2066,
	title = {Assessing the Reliability of an Industrial Avionics Software: Results, Insights and Recommendations},
	journal = {The 24th IEEE International Symposium on Software Reliability Engineering},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Pasadena, CA},
	abstract = {Real Time Operating System for Avionics (RTOS4A) is responsible for providing an operating environment for avionics application software. Avionics software being safety-critical in nature poses several safety and reliability requirements on RTOS4A in addition to the requirements imposed by standards, for instance, DO-178B. Due to this reason, reliability assessment of RTOS4A is very critical to demonstrate confidence about its reliability to its relevant stakeholders. One common way of assessing reliability is by systematic analyses of testing data such as number of tests, number of failures, and coverage using appropriate statistical tests. In this paper, we report our experience of assessing the reliability of an industrial RTOS4A based on testing data collected for 17 months on eight continuous releases. We studied correlation among various measures including: Testing Effort Measures (e.g., complexity of test cases), Testing Effectiveness Measures (e.g., number of failures), and Complexity Measures (e.g., number of functions in a release) and provide a set of recommendations to assess the reliability of RTOS4A that serve as guidelines to practitioners in the domain of RTOS4A.},
	author = {Wu, Ji and Ali, Shaukat and Yue, Tao and Tian, Jie}
}
@inproceedings {Simula.simula.2074,
	title = {Automated Product Line Methodologies to Support Model-Based Testing},
	journal = {The ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS), online publication at CEUR},
	year = {2013},
	publisher = {IEEE},
	type = {Annual},
	address = {.},
	abstract = {Testing products in a cost-efficient way remains an attractive topic for Model-Based Testing (MBT) of product lines in both academia and industry, which can be addressed by employing systematic and automated approaches based on models (such as feature models and UML models). Cost-effective testing products can be divided into three main problems, i.e., test selection, test generation, and test minimization. Driven by the needs of our industrial problems for testing Video Conferencing Systems (VCSs) product line developed by Cisco, Norway, this paper presents Product Line Model-based Testing Methodologies (PL-MTM) to tackle the above-mentioned three problems for cost- effective testing a product in product line, which includes: 1) an systematic and automated test selection methodology; 2) an automated test minimization approach; and 3) an automated and systematic test generation methodology},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud},
	editor = {Tse, T.H.}
}
@techreport {27065,
	title = {Automated Search-Based Test Suite Minimization in Product Lines: An Empirical Study},
	year = {2013},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Gotlieb, Arnaud and Wang, Shuai}
}
@inproceedings {Simula.simula.2018,
	title = {Automated Test Case Selection Using Feature Model: an Industrial Case Study},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS), Best Paper Award},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {Automated test case selection for a new product in a product line is challenging due to several reasons. First, the variability within the product line needs to be captured in a systematic way; second, the reusable test cases from the repository are required to be identified for testing a new product. The objective of such automated process is to reduce the overall effort for selection (e.g., selection time), while achieving an acceptable level of the coverage of testing functionalities. In this paper, we propose a systematic and automated methodology using a Feature Model for Testing (FM\_T) to capture commonalities and variabilities of a product line and a Component Family Model for Testing (CFM\_T) to capture the overall structure of test cases in the repository. With our methodology, a test engineer does not need to manually go through the repository to select a relevant set of test cases for a new product. Instead, a test engineer only needs to select a set of relevant features using FM\_T at a higher level of abstraction for a product and a set of relevant test cases will be selected automatically. We applied our methodology to a product line of video conferencing systems called Saturn developed by Cisco and the results show that our methodology can reduce the selection effort significantly. Moreover, we conducted a questionnaire-based study to solicit the views of test engineers who were involved in developing FM\_T and CFM\_T. The results show that test engineers are positive about adapting our method- ology and models (FM\_T and CFM\_T) in their current practice.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius},
	editor = {Grey, Jeff}
}
@inproceedings {Simula.simula.2016,
	title = {Constraints: the Core of Supporting Automated Product Configuration of Cyber-Physical Systems},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {In the context of product line engineering of cyber-physical systems, there exists a large number of constraints to support for example consistency checking of design decisions made in hardware and software components during configuration. Manual configuration is not feasible in this context considering that managing and manipulating all these constraints in a real industrial context is way too complicated. Therefore, automation is the only way to address such a complexity. However, to this end, relevant constraints have to be well-specified and characterized in the way such that automated configuration can be enabled. Such automation includes automated configuration value inference, optimizing configuration steps and consistency checking. In this paper, we classify and characterize constraints that are required to be specified to support most of the key functionalities of any automated product configuration solution, based on our experience of studying three industrial product line families.},
	keywords = {Conference},
	author = {Nie, Kunming and Yue, Tao and Ali, Shaukat and Zhang, Li and Fan, Zhiqiang},
	editor = {Grey, Jeff}
}
@misc {24759,
	title = {Empirical Software Engineering in Collaboration with BUAA},
	howpublished = {Seminar at Beihang University},
	year = {2013},
	address = {Beijing, China},
	author = {Ali, Shaukat}
}
@inproceedings {27059,
	title = {Ensuring Safety of Avionics Software at the Architecture Design Level, An Industrial Case Study},
	journal = {The 13th International Conference on Quality Software (QSIC 2013)},
	year = {2013},
	author = {Wu, Ji and Yue, Tao and Ali, Shaukat}
}
@inproceedings {Simula.simula.2014,
	title = {Ensuring Safety of Avionics Software at the Architecture Design Level: an Industrial Case Study},
	journal = {The 13th International Conference on Quality Software (QSIC 2013)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {.},
	abstract = {Ensuring that avionics software meets safety requirements at each development stage is very important to warrant the safe operation of an avionics system. Many safety requirements are imposed by various standards and industrial regulations that must be met by avionics software. One of such standards is DO-178B/C, which provides guidelines (e.g. development process and the objectives to satisfy in development activities) for meeting the safety requirements. This paper presents a modeling methodology including a UML profile for specifying safety requirements on a component-based architecture model and a set of design guidelines on avionics software. These safety requirements were identified from both standards (mainly DO-178B/C) and current engineering practices in the domain of avionics system. The methodology enforces safety requirements automatically. We have applied the methodology on an industrial autopilot system and several previously uncaught faults were revealed.},
	keywords = {Conference},
	author = {Wu, Ji and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27060,
	title = {An Extensible Use Case Modeling Approach for Cyber-Physical Systems},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS 2013)},
	year = {2013},
	author = {Yue, Tao and Ali, Shaukat}
}
@article {TestData,
	title = {Generating Test Data From OCL Constraints With Search Techniques},
	journal = {IEEE Transactions on Software Engineering},
	volume = {39},
	number = {10},
	year = {2013},
	publisher = {IEEE},
	abstract = {Model-based testing (MBT) aims at automated, scalable, and systematic testing solutions for complex industrial software systems. To increase chances of adoption in industrial contexts, software systems can be modeled using well-established standards such as the Unified Modeling Language (UML) and the Object Constraint Language (OCL). Given that test data generation is one of the major challenges to automate MBT, we focus on test data generation from OCL constraints in this paper. This endeavor is all the more challenging given the numerous OCL constructs and operations that are designed to facilite the definition of constraints. Though search-based software testing has been applied to test data generation for white-box testing (e.g., branch coverage), its application to the MBT of industrial software systems has been limited. In this paper, we propose a set of search heuristics targeted to OCL constraints to guide test data generation and automate MBT in industrial applications. We evaluate these heuristics for three search algorithms: Genetic Algorithm, (1+1) Evolutionary Algorithm, and Alternating Variable Method. We empirically evaluate our heuristics using complex artificial problems, followed by empirical analyses of the feasibility of our approach on one industrial system in the context of robustness testing. Our approach is also compared with the most widely referenced OCL solver (UMLtoCSP) in the literature and shows to be significantly more efficient.},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1960,
	title = {Minimizing Test Suites in Software Product Lines Using Weight-Based Genetic Algorithms},
	journal = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2013},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Test minimization techniques aim at identifying and eliminating redundant test cases from test suites in order to reduce the total number of test cases to execute, thereby improving the efficiency of testing. In the context of software product line, we can save efforts and cost in the selection and minimization of test cases for testing a specific product by modeling the product line. However, minimizing the test suite for a product requires addressing two potential issues: 1) the reduced test suite may not cover all test requirements compared with the original suite; 2) the reduced test suite may have less fault revealing capability than the original suite. In this paper, we apply weight-based Genetic Algorithms (GAs) to minimize the test suite for testing a product, while preserving fault detection capability and testing coverage of the original test suite. The challenge behind concerns the definition of an appropriate fitness function, which is able to preserve the coverage of complex testing criteria (e.g., Combinatorial Interaction Testing criterion). Based on the defined fitness function, we have empirically evaluated three different weight- based GAs on an industrial case study provided by Cisco Systems, Inc. Norway. We also presented our results of applying the three weight-based GAs on five existing case studies from the literature. Based on these case studies, we conclude that among the three weight-based GAs, Random-Weighted GA (RWGA) achieved significantly better performance than the other ones.},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud},
	editor = {Harman, Mark}
}
@inproceedings {Simula.simula.2072,
	title = {Modeling BCMS Product Line Using Feature Model, Component Family Model, and UML},
	journal = {Comparing Modeling Approaches Workshop (MODELS 2013)},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Workshop},
	address = {.},
	abstract = {In the context of Model-Based Engineering (MBE) of product lines, effort required to develop models can be significantly reduced by applying systematic product line modeling and configuration methodologies. Our previous work presented models of bCMS developed using AspectSM, a UML profile for Aspect-Oriented Modeling (AOM), which was defined to model crosscutting behaviors using extended UML state machines, with the objectives of minimizing modeling effort and the learning curve for modeling crosscutting behavior. However, such approach still requires users to be familiar with specific expertise and concepts on various UML behavior models. In this paper, we extend our previous work using Feature Model (FM) and Component Family Model (CFM) to model bCMS product line. More specifically, a FM is designed and developed to capture all variations points for bCMS product line and a CFM is built to provide an abstraction layer on top of the configurable state machines. With our current methodology, a user doesn{\textquoteright}t need to acquire expertise on behavioral modeling and can simply configure models for a product by selecting features in FM and configuring provided attributes in CFM.},
	author = {Wang, Shuai and Ali, Shaukat},
	editor = {Grey, Jeff}
}
@inproceedings {Simula.simula.2082,
	title = {Modeling Crisis Management System With the Restricted Use Case Modeling Approach},
	journal = {Comparing Modeling Approaches (CMA) Workshop at ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {1},
	address = {.},
	abstract = {Use case modeling is commonly applied to document requirements. Use case specifications (UCSs) are usually structured, unrestricted textual documents complying with a certain template. However, because they remain essentially textual, ambiguities are inevitable. In our previous work, we proposed a new use case modeling approach, named as Restricted Use Case Modeling (RUCM), which is composed of a set of well-defined restriction rules and a new template. The goal was to reduce ambiguity and facilitate automated analysis. In our works, RUCM has been systematically and empirically evaluated through case studies to be easy to apply and leads to higher quality of UML analysis models. In this paper, we modeled the Crisis Management System (CMS) case study using RUCM and our experience proved that RUCM is easy to apply and sufficient to capture the requirements provided for the case study.},
	keywords = {Workshop},
	author = {Zhang, Gong and Yue, Tao and Ali, Shaukat},
	editor = {Grey, Jeff}
}
@misc {24750,
	title = {A Rigorous and Comprehensive Analysis of Effort for Modeling Aspect State Machines: Results From a Series of Controlled Experiments},
	howpublished = {Annual International Computer Software \& Applications Conference (COMPSAC)},
	year = {2013},
	publisher = {IEEE},
	address = {Kyoto, Japan},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.1962,
	title = {A Rigorous and Comprehensive Analysis of Effort for Modeling Aspect State Machines: Results From a Series of Controlled Experiments},
	journal = {Annual International Computer Software \& Applications Conference (COMPSAC)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	abstract = {In this paper, we report a comprehensive analysis of Effort (measured in time) required for performing various modeling activities, based on the data we collected from a series of controlled experiments. These experiments were conducted to evaluate the overall applicability of a UML profile for Aspect-oriented Modeling (AOM): AspectSM. AspectSM was developed to model crosscutting behaviors on UML state machines and has been successfully applied for supporting model-based robustness testing at Cisco Systems, Norway. Our analysis results show that Effort required for applying AspectSM does not deviate significantly from Effort required for applying non-AOM modeling approaches for various modeling activities related to crosscutting behaviors. In addition, our previous studies show that AspectSM helps achieving improved readability of state machines, improved modeling quality, and increased understandability, reduced modeling errors, and require lesser number of modeling elements. Therefore, we conclude that modeling crosscutting behaviors using AspectSM can bring significant benefits in practice at no significant additional Effort as compared to using standard UML state machines.},
	author = {Ali, Shaukat and Yue, Tao},
	editor = {Sedighsarvestani, Sahra and Ohsaki, Hiroyuki and Kuepper, Axel}
}
@inproceedings {Simula.simula.2076,
	title = {Towards a Search-Based Interactive Configuration of Cyber Physical System Product Lines},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems, Poster},
	year = {2013},
	publisher = {ACM/IEEE},
	type = {Conference},
	abstract = {Product Line Enginnering (PLE) is a technique to improve the quality and productivity of developing (via configuration) Cyber Phsical Systems (CPSs). A CPS often contains many heterogeneous components with complex constraints relevant to product configuration in the context of PLE. Manual configuration is error-prone and has low productivity since managing and manipulating such constraints in a real industrial context is very complicated and thus warrants an automated solution. However, fully automated solution is often impossible for CPSs since some decisions must be made manually by users, thus requiring an interactive configuration solution. Therefore, we propose a semi-automated and interactive configuration solution for CPSs. We started our research by analyzing the characteristics of three industrial CPS product lines and constraints required for supporting such a configuration solution. Then we conducted some pilot experiments on applying search algorithms to find optimal decision orders for configuring a product. In this poster, we describe the whole research idea and, discussion on the work we have completed, the initial results, and the future plan.},
	keywords = {Conference},
	author = {Nie, Kunming and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27062,
	title = {Towards a Search-based Interactive Configuration of Cyber Physical System Product Lines_Submmit},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systesm},
	year = {2013},
	author = {Yue, Tao and Ali, Shaukat}
}
@techreport {25248,
	title = {Towards A Search-based Interactive Configuration of Cyber-Physical System Product Lines},
	year = {2013},
	note = {This technical report is an early version of a published conference paper. The published paper can be found here:https://www.simula.no/publications/towards-search-based-interactive-configuration-cyber-physical-system-product-lines},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Ali, Shaukat and Nie, Kunming}
}
@techreport {27066,
	title = {Towards Systematic Requirements Engineering Practices in Product Line Engineering: A Comprehensive Domain Analysis in an Industrial Setting},
	year = {2013},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Hesari, Shokoofeh and Ali, Shaukat and Selic, Bran}
}
@inproceedings {Simula.simula.2012,
	title = {Using Feature Model to Support Model-Based Testing of Product Lines: an Industrial Case Study},
	journal = {The 13th International Conference on Quality Software (QSIC 2013)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Najing},
	abstract = {In the context of Model-Based Testing (MBT) of product lines, effort required to develop models can be significantly reduced by applying systematic product line modeling and configuration methodologies. In our previous work, we developed such a methodology to capture variability in configurable UML state machines and aspect state machines. For each product, these state machines are to be configured for generating executable test cases. In this paper, we extended this methodology using Feature Model for Testing (FM\_T) and Component Family Model for Behaviors (CFM\_B). FM\_T captures variable testing functionalities of a product line, whereas CFM\_B provides an abstraction layer on top of the configurable state machines. With our current methodology, a test engineer doesn{\textquoteright}t need to acquire expertise on behavioral modeling and can simply configure models for a product by selecting features in FM\_T and configuring provided attributes in CFM\_B. The configured models are then given input to our model-based testing tool, TRansformation-based tool for Uml-baSed Testing (TRUST) for executable test case generation. We applied our extended methodology to a product line of video conferencing system developed by Cisco Systems, Norway. Results show that the methodology significantly reduces the complexity of configuration; thereby significantly reducing required effort and cost (e.g., in terms of training). In addition, it does not require test engineers to have expertise in UML modeling, aspect-oriented modeling, and OCL specification and therefore eases the adoption of MBT in industry.},
	keywords = {Conference},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao and Liaaen, Marius},
	editor = {Tse, T.H.}
}
@inproceedings {27061,
	title = {Zen-RUCM: A Tool for Supporting a Comprehensive and Extensible Use Case Modeling Framework},
	journal = {ACM/IEEE 16th International Conference on Model Driven Engineering Languages and Systems (MODELS 2013)},
	year = {2013},
	author = {Yue, Tao and Ali, Shaukat}
}
@inproceedings {Simula.simula.1958,
	title = {Assessing Composition in Modeling Approaches},
	journal = {Proceedings of the CMA 2012 Workshop},
	year = {2012},
	publisher = {ACM},
	type = {Workshop},
	address = {.},
	abstract = {Modeling approaches are based on various paradigms, e.g., aspect-oriented, feature-oriented, object-oriented, and logic-based. Modeling approaches may cover requirements models to low-level design models, are developed for various purposes, use various means of composition, and thus are difficult to compare. However, such comparisons are critical to help practitioners know under which conditions approaches are most applicable, and how they might be successfully generalized and combined to achieve end-to-end methods. This paper reports on work done at the 2nd International Comparing Modeling Approaches (CMA) workshop towards the goal of identifying potential comprehensive modeling methodologies with a particular emphasis on composition: (i) an improved set of comparison criteria; (ii) 19 assessments of modeling approaches based on the comparison criteria and a common, focused case study.},
	author = {Mussbacher, Gunter and Alam, Omar and Alhaj, Mohammad and Ali, Shaukat},
	editor = {Mussbacher, Gunter}
}
@techreport {25300,
	title = {Automated Selection of Test Cases using Feature Model for Product Lines: An Industrial Case Study},
	number = {2012-20},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version\&nbsp;can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@techreport {25305,
	title = {Automated Test Case Selection using Feature Model: an Industrial Case Study},
	number = {2012-28},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/automated-test-case-selection-using-feature-model-industrial-case-study},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Gotlieb, Arnaud and Ali, Shaukat and Liaaen, Marius}
}
@inproceedings {Simula.simula.1262,
	title = {Bridging the Gap Between Requirements and Aspect State Machines to Support Non-Functional Testing: Industrial Case Studies},
	journal = {8th European Conference on Modelling Foundations and Applications (ECMFA)},
	year = {2012},
	month = {July},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Requirements are often structured and documented as use cases while UML state machine diagrams often describe the behavior of a system. State machines capture rich and detailed behavior of a system, which can serve as a basis for many automated activities such as automated test case and code generation. The former is of interest in this paper. Non-functional behavior can be modeled using standard UML state machines, but usually results in complex state machines. To cope with such complexity, Aspect-Oriented Modeling (AOM) is often recommended. AspectSM is a UML profile defined to model crosscutting behavior on UML state machines called as aspect state machines with the focus of supporting model-based test case generation for non-functional behavior. Hence, an automatic transition from use cases to aspect state machines would provide significant, practical help for testing system requirements. In this paper, we propose an approach to automatically generate aspect state machines from use cases for the purpose of non-functional testing. Our approach is implemented in a tool, which we used for two industrial case studies. Results show that high quality aspect state machines can be generated, which can be manually refined at a reasonable cost to support testing.},
	doi = {10.1007/978-3-642-31491-9\_12},
	author = {Yue, Tao and Ali, Shaukat},
	editor = {Vallecillo, Antonio and Tolvanen, Juha-Pekka and Kindler, Ekkart and St{\"o}rrle, Harald and Kolovos, Dimitris}
}
@techreport {Simula.simula.1506,
	title = {Categorizing and Assessing Empirical Investigations in Aspect-Oriented Modeling: a Systematic Review},
	number = {2012-15},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Empirical investigations in software modeling are extremely essential to provide evidence regarding worth of modeling approaches. As is the case with any modeling approach, it is crucial to demonstrate that an Aspect-oriented Modeling (AOM) approach fulfills the benefits offered by the approach via empirical investigations. Unfortunately, no attempt has been made in AOM to systematically investigate the quality of reported empirical investigations. To fill this gap, we report a systematic review of empirical investigations that have been carried out in AOM. We identify and classify the published AOM approaches based on empirical investigations such as case studies, controlled experiments, and surveys. Our research revealed that empirical investigations reported in AOM are limited since we found only a few controlled experiments and we couldn\&$\#$39;t find any case study or survey based research reported in AOM. However, we found many case studies (not case study research) and examples, which were used to simply demonstrate the applicability of respective approaches. We further comment on strengths and limitations of the reported empirical investigations and provide future directions.},
	author = {Hajra, Iram and Andleeb, Saba and Ali, Shaukat and Malik, Zafar}
}
@misc {24752,
	title = {Comprehensively Evaluating Conformance Error Rates of Applying Aspect State Machines for Robustness Testing},
	howpublished = {International Conference on Aspect-Oriented Software Development (AOSD)},
	year = {2012},
	publisher = {ACM},
	address = {Potsdam, Germany},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.1086,
	title = {Comprehensively Evaluating Conformance Error Rates of Applying Aspect State Machines for Robustness Testing},
	journal = {International Conference on Aspect-Oriented Software Development (AOSD 2012)},
	year = {2012},
	pages = {155-166},
	publisher = {ACM},
	type = {1},
	address = {Potsdam, Germany},
	abstract = {Aspect Oriented Modeling (AOM) aims to provide enhanced separation of concerns during the design phase and proclaims many benefits (e.g., easier model evolution, reduced modeling effort, and reduced modeling errors) over traditional modeling paradigms such as object-oriented modeling. However, empirical evaluations of these benefits is severely lacking in the AOM community. In this paper, we empirically evaluate one of the AOM profiles: AspectSM, via a controlled experiment to assess if it can help in reducing modeling errors (referred as conformance errors in this paper), which one of the benefits offered by AOM. AspectSM is a UML profile, which is developed to support automated state-based robustness testing. With AspectSM, crosscutting behaviors are modeled as aspect state machines using the stereotypes defined in AspectSM. We evaluate the conformance error rates of applying AspectSM from various perspectives by conducting four activities: 1) identifying modeling defects, 2) comprehending state machines, 3) modeling state machines, and 4) weaving aspect state machines into base state machines. For most of these activities, experimental results show that the error rates while performing these four activities using AspectSM are significantly lower than standard UML state machine modeling approaches.},
	isbn = {978-1-4503-1092-5},
	doi = {10.1145/2162049.2162068},
	author = {Ali, Shaukat and Yue, Tao}
}
@misc {Simula.simula.1432,
	title = {Comprehensively Evaluating Conformance Error Rates of Applying Aspect State Machines for Robustness Testing},
	howpublished = {Aspect Oriented Software Development Conference, Potsdam Germany},
	year = {2012},
	publisher = {.},
	address = {.},
	abstract = {Aspect Oriented Modeling (AOM) aims to provide enhanced separation of concerns during the design phase and proclaims many benefits (e.g., easier model evolution, reduced modeling effort, and reduced modeling errors) over traditional modeling paradigms such as object-oriented modeling. However, empirical evaluations of these benefits is severely lacking in the AOM community. In this paper, we empirically evaluate one of the AOM profiles: AspectSM, via a controlled experiment to assess if it can help in reducing modeling errors (referred as conformance errors in this paper), which is one of the benefits offered by AOM. AspectSM is a UML profile, which is developed to support automated state-based robustness testing. With AspectSM, crosscutting behaviors are modeled as aspect state machines using the stereotypes defined in AspectSM. We evaluate the conformance error rates of applying AspectSM from various perspectives by conducting four activities: 1) identifying modeling defects, 2) comprehending state machines, 3) modeling state machines, and 4) weaving aspect state machines into base state machines. For most of these activities, experimental results show that the error rates while performing these four activities using AspectSM are significantly lower than standard UML state machine modeling approaches.},
	keywords = {Conference},
	author = {Ali, Shaukat}
}
@techreport {25310,
	title = {The Core of Supporting Automated Product Configuration of Cyber-Physical Systems},
	number = {2012-33},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published paper can be found here:https://www.simula.no/publications/constraints-core-supporting-automated-product-configuration-cyber-physical-systems},
	publisher = {Simula Research Laboratory},
	author = {Nie, Kunming and Yue, Tao and Ali, Shaukat and Zhang, Li and Fan, Zhiqiang}
}
@article {Simula.simula.760,
	title = {Does Aspect-Oriented Modeling Help Improve the Readability of UML State Machines?},
	journal = {Software and Systems Modeling},
	number = {DOI 10.1007/s10270-012-0293-5},
	year = {2012},
	publisher = {Springer Berlin Heidelberg},
	abstract = {Aspect-oriented Modeling (AOM) is a relatively recent and very active field of research, which application has however been limited in practice. AOM is assumed to yield several potential benefits such as enhanced modularization, easier evolution, increased reusability, and improved readability of models, as well as reduced modeling effort. However, credible, solid empirical evidence of such benefits is lacking. We evaluate the {\textquotedblleft}readability{\textquotedblright} of state machines when modeling crosscutting behavior using AOM and more specifically AspectSM, a UML profile extending the UML state machine notation to provide mechanisms to define aspects using state machines. With AspectSM, crosscutting behavior is modeled using so-called {\textquotedblleft}aspect state machines{\textquotedblright}. Their readability is compared with that of system state machines directly modeling crosscutting and standard behavior together. A controlled experiment was conducted with 25 trained, graduate students to achieve the above objective. Readability is defined based on the identification of defects seeded in state machines and the score obtained when answering a comprehension questionnaire about the system behavior. We use two baselines of comparisons-standard UML state machines without hierarchical features (flat state machines) and standard state machines with hierarchical features (hierarchical state machines)-in order to assess the role of hierarchies in the benefits brought by AspectSM. The results showed that defect identification with AspectSM is significantly better than that with both flat and hierarchical state machines. In terms of the comprehension questionnaire, the AspectSM scores are better than flat state machines, but worse than hierarchical state machines. Regarding the effort for inspecting the state machines, no significant difference could be observed. Conclusion: Results of the experiment suggest using hierarchical features of UML state machines for crosscutting concerns where this is possible and resort otherwise to aspect state machines or use them together for better readability.},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude}
}
@techreport {Simula.simula.1546,
	title = {Empirically Evaluating Improved Heuristics for Test Data Generation From OCL Constraints Using Search Algorithms},
	number = {2012-17},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Efficiently generating test data is one of the key testing requirements of automated model-based test case generation. Keeping this in mind and driven by the needs of our industrial partners, we propose an improvement in heuristics that we previously defined to generate test data from OCL constraints using search algorithms. We evaluate our improved heuristics using two empirical studies with three search algorithms: Alternating Variable Method (AVM), (1+1) Evolutionary Algorithm (EA), and a Genetic Algorithm (GA). Furthermore, we used Random Search (RS) as a comparison baseline. The first empirical study was conducted using carefully designed artificial problems (constraints) to assess each individual heuristics. The second empirical study is based on an industrial case study provided by Cisco. The results of both empirical evaluations reveal that the effectiveness of the search algorithms, measured in terms of time to solve the OCL constraints to generate data, is significantly improved when using the novel heuristics presented in this paper. In particular, our experiments show that (1+1) EA with the novel heuristics has the highest success rate among all the analyzed algorithms, as it requires the least number of iterations to solve constraints to generate test data.},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea}
}
@techreport {25297,
	title = {Experiences of Applying UML/MARTE on Three Industrial Projects},
	number = {2012-08},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/experiences-applying-umlmarte-three-industrial-projects},
	publisher = {Simula Research Laboratory},
	author = {Muhammad Zohaib {Iqbal} and Ali, Shaukat and Yue, Tao and Briand, Lionel}
}
@inproceedings {Simula.simula.1472,
	title = {Experiences of Applying UML/MARTE on Three Industrial Projects},
	journal = {ACM/IEEE International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	volume = {7590},
	year = {2012},
	pages = {642-658},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {MARTE (Modeling and Analysis of Real-Time and Embedded Systems) is a UML profile, which has been developed to model concepts specific to Real-Time and Embedded Systems (RTES). In previous years, we have applied UML/MARTE to three distinct industrial problems in various industry sectors: architecture modeling and configuration of large-scale and highly configurable integrated control systems, model-based robustness testing of communication-intensive systems, and model-based environment simulator generation of large-scale RTES for testing. In this paper, we report on our experiences of solving these problems by applying UML/MARTE on four industrial case studies. Based on our common experiences, we derive a framework to help practitioners for future applications of UML/MARTE. The framework provides a set of detailed guidelines on how to apply MARTE in industrial contexts and will help reduce the gap between the modeling standards and industrial needs.},
	doi = {10.1007/978-3-642-33666-9\_41},
	author = {Muhammad Zohaib {Iqbal} and Ali, Shaukat and Yue, Tao and Briand, Lionel Claude},
	editor = {Robert B. {France} and Kazmeier, J{\"u}rgen and Breu, Ruth and Atkinson, Colin}
}
@techreport {Simula.simula.14,
	title = {Generating Test Data From OCL Constraints With Search Techniques},
	number = {2010-16},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Model-based testing (MBT) aims at automated, scalable, and systematic testing solutions for complex industrial software systems. To increase chances of adoption in industrial contexts, software systems should be modeled using well-established standards such as the Unified Modeling Language (UML) and Object Constraint Language (OCL). Given that test data generation is one of the major challenges to automate MBT, we focus on test data generation from OCL constraints in this paper. Though search-based software testing has been applied to test data generation for white-box testing (e.g., branch coverage), its application to the MBT of industrial software systems has been limited. In this paper, we propose a set of search heuristics targeted to OCL constraints to guide test data generation and automate MBT in industrial applications. These heuristics are used to develop an efficient OCL solver exclusively based on search. We evaluate these heuristics for search algorithms such as Genetic Algorithms, (1+1) Evolutionary Algorithm and Alternating Variable Method. We empirically evaluate our heuristics using complex artificial problems followed by empirical analyses to evaluate the feasibility of our approach on one industrial system. Our approach is compared with a downloadable, standard OCL solver and shows to be significantly more efficient and more widely applicable.},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@misc {Simula.simula.1452,
	title = {Industry-Driven Testing: Past, Present, and Future Activities at Simula},
	howpublished = {CIO Forum, Oslo Norway},
	year = {2012},
	month = {June},
	publisher = {.},
	address = {.},
	abstract = {his talk will provide an overview of various Industry-driven testing projects taking place at Certus Software V\&amp;V Center, Simula Research Laboratory, Norway. More specifically, the talk will focus on model-based testing projects that we conducted with several industrial partners such as Cisco Systems, WesternGeco, and Tomra in the span of last several years. The talk will present an overview of problems we solved, challenges we faced, key results, and future works.},
	author = {Ali, Shaukat}
}
@techreport {25303,
	title = {Minimizing Test Suites in Software Product Lines Using Weight-Based Genetic Algorithms},
	number = {2012-25},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published paper can be found here:https://www.simula.no/publications/minimizing-test-suites-software-product-lines-using-weight-based-genetic-algorithms},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Ali, Shaukat and Gotlieb, Arnaud}
}
@misc {24756,
	title = {Modeling Crosscutting Behaviors (Robustness)  using AspectSM Profile to Support  Model-based Testing},
	howpublished = {Bellairs Workshop on (Aspect-Oriented) Model-Driven Software Development},
	year = {2012},
	author = {Ali, Shaukat}
}
@article {Simula.simula.118,
	title = {Modeling Robustness Behavior Using Aspect-Oriented Modeling to Support Robustness Testing of Industrial Systems},
	journal = {Software and Systems Modeling},
	volume = {11},
	number = {4},
	year = {2012},
	pages = {633-670},
	publisher = {Springer-Verlag},
	abstract = {Model-based robustness testing requires, precise and complete behavioral, robustness modeling, for example using state machines, to support automated test case generation. Robustness behavior is a crosscutting behavior and, if modeled directly, often results in large, complex state machines. These in practice tend to be error-prone and difficult to read and understand. As a result, modeling robustness behavior in this way is not scalable for complex industrial systems. To overcome these problems, Aspect Oriented Modeling (AOM) can be employed to model robustness behavior as aspects in the form of state machines specifically designed to model robustness behavior. In this paper, we present a RobUstness Modeling Methodology (RUMM) that allows modeling robustness behavior as aspects. At the core of RUMM is a UML profile (AspectSM) that allows modeling UML state machine aspects as UML state machines (aspect state machines). Such an approach, relying on a standard and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of aspect modeling easier, and thus in our context support robustness modeling for the purpose of automated model-based testing in industry. We discuss the benefits, such as reduced modeling effort and improved readability, achieved when applying RUMM to an industrial case study where we have used the profile to model the crosscutting robustness behavior of a videoconferencing system.},
	author = {Ali, Shaukat and Briand, Lionel Claude and Hemmati, Hadi}
}
@techreport {25304,
	title = {Myths, Facts, and Misconceptions of OCL: Experiences from Diverse Industrial Applications},
	number = {2012-26},
	year = {2012},
	note = {This technical report is an early version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao and Muhammad Zohaib {Iqbal} and Rajwinder Kaur {Panesar-Walawege}}
}
@techreport {27055,
	title = {A Practical and Scalable Use Case Modeling Approach to Specify Crosscutting Concerns: Industrial Applications},
	year = {2012},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Ali, Shaukat}
}
@misc {24753,
	title = {A Product Line Modeling and Configuration Methodology to Support Model-Based Testing: an Industrial Case Study},
	howpublished = {ACM/IEEE 15th International Conference on Model Driven Engineering Languages \& Systems (MODELS)},
	year = {2012},
	publisher = {ACM/IEEE},
	address = {Innsbruck, Austria},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.1476,
	title = {A Product Line Modeling and Configuration Methodology to Support Model-Based Testing: an Industrial Case Study},
	journal = {ACM/IEEE 15th International Conference on Model Driven Engineering Languages \& Systems (MODELS)},
	year = {2012},
	pages = {726-742},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Product Line Engineering (PLE) is expected to enhance quality and productivity, speed up time-to-market and decrease development effort, through reuse-the key mechanism of PLE. In addition, one can also apply PLE to support systematic testing and more specifically model-based testing (MBT) of product lines-the original motivation behind this work. MBT has shown to be cost-effective in many industry sectors but at the expense of building models of the system under test (SUT). However, the modeling effort to support MBT can significantly be reduced if an adequate product line modeling and configuration methodology is followed, which is the main motivation of this paper. The initial motivation for this work emerged while working with MBT for a Video Con-ferencing product line at Cisco Systems, Norway. In this paper, we report on our experience in modeling product family models and various types of behavioral variability in the Saturn product line. We focus on behavioral variability in UML state machines since the Video Conferencing Systems (VCSs) exhibit strong state-based behavior and these models are the main drivers for MBT; however, the approach can be also tailored to other UML diagrams. We also provide a mechanism to specify and configure various types of variability using stereotypes and Aspect-Oriented Modeling (AOM). Results of applying our methodology to the Saturn product line modeling and configuration process show that the effort required for modeling and configuring products of the product line family can be significantly reduced.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-33666-9\_46},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude and Walawege, Suneth}
}
@techreport {25298,
	title = {A Product Line Modeling and Configuration Methodology to Support Model-Based Testing: an Industrial Case Study},
	number = {2012-07},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/product-line-modeling-and-configuration-methodology-support-model-based-testing},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel and Walawege, Suneth}
}
@techreport {25302,
	title = {Product Line Modeling and Configuration Methodology using Feature Model for Supporting Model-Based Testing},
	number = {2012-24},
	year = {2012},
	note = {This technical report is an early version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Wang, Shuai and Ali, Shaukat and Yue, Tao}
}
@techreport {25299,
	title = {A Rigorous and Comprehensive Analysis of Effort for Modeling Aspect State Machines},
	number = {2012-16},
	year = {2012},
	note = {This technical report is an early version of a published conference paper. The published\&nbsp;version can be found here:https://www.simula.no/publications/rigorous-and-comprehensive-analysis-effort-modeling-aspect-state-machines-results},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao}
}
@phdthesis {Simula.simula.1124,
	title = {Scalable Model-Based Robustness Testing: Novel Methodologies and Industrial Application},
	year = {2012},
	school = {University of Oslo},
	type = {phd},
	address = {Oslo, Norway},
	abstract = {Embedded systems, as for example communication and control systems, are being increasingly used in our daily lives and hence require thorough and systematic testing before their actual use. Many of these systems interact with their environment and, therefore, their functionality is largely dependent on this environment whose behavior can be unpredictable. Robustness testing aims at testing the behavior of a system in the presence of faulty situations in its operating environment (e.g., sensors and actuators). In such situations, the system should gracefully degrade its performance instead of abruptly stopping execution. To systematically perform robustness testing, one option is to resort to Model-Based Robustness Testing (MBRT), which is a systematic, rigorous, and automated way of conducting robustness testing. However, to successfully apply MBRT in industrial contexts, new technologies need to be developed to scale to the complexity of real industrial systems. This thesis presents a solution for MBRT on industrial systems, including scalable robustness modeling and executable test case generation. One important contribution of this thesis is a scalable RobUstness Modeling Methodology (RUMM), which is achieved using Aspect-Oriented Modeling (AOM). It is a complete, automated, and practical methodology that covers all features of state machines and aspect concepts necessary for MBRT. Such methodology, relying on a standard (Unified Modeling Language or UML) and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of robustness modeling easier in industrial contexts. The applicability of the methodology is demonstrated using an industrial case study. Results showed that the approach significantly reduced modeling effort (98\% on average), improved separation of concerns, and eased model evolution. The approach is further empirically evaluated using two controlled experiments involving human subjects and results showed that the proposed methodology significantly improves the readability of models as compared to modeling using standard UML notations. Another important contribution of this thesis is an efficient approach for solving constraints (written in Objects Constraint Language (OCL)) on the operating environment of a system, which is mandatory for emulating faulty situation in the environment for the purpose of MBRT. A set of novel heuristics is devised for various OCL constructs, which are required for the application of search algorithms. The heuristics have been empirically evaluated on an industrial case study for robustness testing and the results showed to be very promising and significantly better than the existing works in the literature on OCL constraint solvers. A final contribution of the thesis is robustness test case generation from the models developed using RUMM. Test case generation also includes scripts generation for environment emulation, which is mandatory for automated robustness testing again using an industrial case study. In preliminary experiments, the execution of test cases found one critical, robustness fault in a deployed industrial system.},
	isbn = {ISSN 1501-7710},
	author = {Ali, Shaukat}
}
@techreport {25301,
	title = {A Study on Industrial Requirements Engineering Practice of Highly Configurable Systems},
	number = {2012-21},
	year = {2012},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Hesari, Shokoofeh and Ali, Shaukat and Selic, Bran}
}
@inproceedings {Simula.simula.1536,
	title = {Studying the Understandability of Aspect State Machines Through the Weaving Activity},
	journal = {The 19th Asia-Pacific Software Engineering Conference (APSEC 2012)},
	year = {2012},
	publisher = {IEEE},
	type = {Conference},
	address = {Hong Kong},
	abstract = {Aspect-oriented Modeling (AOM) is a relatively recent, very active field of research and is assumed to yield several potential benefits such as enhanced modularization, easier evolution, increased reusability, and improved readability and understandability of models, as well as reduced modeling effort. However, credible, solid empirical evidence of such benefits is very rare. In this paper, we evaluate the understandability of state machines, when modeling crosscutting behavior using AOM and more specifically AspectSM, a UML profile extending the UML state machine notation to provide mechanisms to define aspects using state machines. With AspectSM, crosscutting behavior is modeled using so-called aspect state machines, which are woven into a base state machine (modeling core functionality) to produce a woven state machine. Modeling aspect state machines separately from the base state machine no doubt offers several abovementioned benefits but on the other hand poses extra challenges for a modeler to understand them because of implicit interactions between both aspect and base state machines. This paper reports a study, which was specifically designed to evaluate the understandability of aspect state machines. The understandability of aspect state machines in conjunction with a base state machine is evaluated by comparing woven state machines produced by subjects (subject woven state machines) with the woven state machines automatically generated by our weaver (reference woven state machines). Understandability is measured from the aspects of Completeness and Redundancy of a subject{\textquoteright}s woven state machine when compared with the corresponding reference woven state machine. Results of the study show that on average, we observed completeness of 71\%, whereas we observed approximately 6\% of redundancy. We also observed that subjects took significantly more time to weave more complex aspect state machines (with more states, transitions, and pointcuts).},
	author = {Ali, Shaukat and Yue, Tao}
}
@techreport {25319,
	title = {Automated Transition from Use Cases to UML State Machines for State-based Testing},
	number = {2011-05},
	year = {2011},
	note = {This technical report is an early version of a published paper. The published version can be found here:https://www.simula.no/publications/automated-transition-use-cases-uml-state-machines-support-state-based-testing},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao}
}
@inproceedings {Simula.simula.502,
	title = {Automated Transition From Use Cases to UML State Machines to Support State-Based Testing},
	journal = {In Seventh European Conference on Modelling Foundations and Applications (ECMFA) 2011},
	year = {2011},
	pages = {115-131},
	publisher = {Springer-Verlag Berlin, Heidelberg},
	type = {Conference},
	abstract = {Use cases are commonly used to structure and document requirements while UML state machine diagrams often describe the behavior of a system and serve as a basis to automate test case generation in many model-based testing (MBT) tools. Therefore, automated support for the transition from use cases to state machines would provide significant, practical help for testing system requirements. Additionally, traceability could be established through automated transformations, which could then be used for instance to link requirements to design decisions and test cases, and assess the impact of requirements changes. In this paper, we propose an approach to automatically generate state machine diagrams from use cases while establishing traceability links. Our approach is implemented in a tool, which we used to perform three case studies, including an industrial case study. The results show that high quality state machine diagrams can be generated, which can be manually refined at reasonable cost to support MBT. Automatically generated state machines showed to largely conform to the actual system behavior as evaluated by a domain expert.},
	author = {Yue, Tao and Ali, Shaukat and Briand, Lionel}
}
@inproceedings {Simula.simula.1538,
	title = {Comparing Six Modeling Approaches},
	journal = {Models in Software Engineering},
	volume = {7167/2012},
	number = {217-243},
	year = {2011},
	publisher = {Lecture Notes in Computer Science},
	type = {Workshop},
	abstract = {While there are many aspect-oriented modeling (AOM) approaches, from requirements to low-level design, it is still difficult to compare them and know under which conditions different approaches are most applicable. This comparison, however, is crucially important to unify existing AOM and more traditional object-oriented modeling (OOM) approaches and to generalize individual approaches into a comprehensive end-to-end method. Such a method does not yet exist. This paper reports on work done at the inaugural Comparing Modeling Approaches (CMA) workshop towards the goal of identifying potential comprehensive methodologies: (i) a common, focused case study for six modeling approaches, (ii) a set of criteria applied to each of the six approaches, and (iii) the assessment results.},
	author = {Mussbacher, Gunter and Wisam Al {Abed} and Alam, Omar and Ali, Shaukat and Beugnard, Antoine and Bonnet, Valentin},
	editor = {Kienzle, J{\"o}rg}
}
@techreport {25317,
	title = {Comprehensively Evaluating Conformance Error Rates of Applying Aspect State Machines for Robustness Testing},
	number = {2011-08},
	year = {2011},
	note = {This technical report is an early version of a published conference paper. The published version can be found here:https://www.simula.no/publications/comprehensively-evaluating-conformance-error-rates-applying-aspect-state-machines},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao and Malik, Zafar}
}
@techreport {Simula.approve.70,
	title = {Does Aspect-Oriented Modeling Help Improve the Readability of UML State Machines?},
	number = {2010-11},
	year = {2011},
	publisher = {Simula Research Laboratory},
	abstract = {Aspect-oriented Modeling (AOM) is a relatively recent and very active field of research, which application has however been limited in practice. AOM is assumed to yield several potential benefits such as enhanced modularization, easier evolution, increased reusability, and improved readability of models, as well as reduced modeling effort. However, credible, solid empirical evidence of such benefits is lacking. We evaluate the \&ldquo;readability\&rdquo; of state machines when modeling crosscutting behavior using AOM and more specifically AspectSM, a UML profile extending the UML state machine notation to provide mechanisms to define aspects using state machines. With AspectSM, crosscutting behavior is modeled using so-called \&ldquo;aspect state machines\&rdquo;. Their readability is compared with that of system state machines directly modeling crosscutting and standard behavior together. A controlled experiment was conducted with 25 trained, graduate students to achieve the above objective. Readability is defined based on the identification of defects seeded in state machines and the score obtained when answering a comprehension questionnaire about the system behavior. We use two baselines of comparisons-standard UML state machines without hierarchical features (flat state machines) and standard state machines with hierarchical features (hierarchical state machines)-in order to assess the role of hierarchies in the benefits brought by AspectSM. The results showed that defect identification with AspectSM is significantly better than that with both flat and hierarchical state machines. In terms of the comprehension questionnaire, the AspectSM scores are better than flat state machines, but worse than hierarchical state machines. Regarding the effort for inspecting the state machines, no significant difference could be observed. Conclusion: Results of the experiment suggest using hierarchical features of UML state machines for crosscutting concerns where this is possible and resort otherwise to aspect state machines or use them together for better readability.},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel}
}
@techreport {Simula.simula.372,
	title = {Empirically Evaluating the Impact of Applying Aspect State Machines on Modeling Quality and Effort},
	number = {2011-06},
	year = {2011},
	publisher = {Simula Research Laboratory},
	abstract = {Aspect-Oriented Modeling (AOM) has been the subject of intense research over the last decade and aims to provide numerous benefits to modeling, such as enhanced modularization, easier evolution, higher applicability as well as reduced modeling effort. However, these benefits can only be obtained at the cost of learning and applying new modeling approaches. Studying their applicability is therefore important to assess whether they are worth using in practice. In this paper, we report the first controlled experiment to assess the applicability of AOM, focusing on a recently published UML profile (AspectSM). This profile was originally designed to support model-based robustness testing in an industrial context but is applicable to the behavioral modeling of other crosscutting concerns. This experiment assesses the applicability of AspectSM from two aspects: the quality of derived state machines and the effort required to build them. With AspectSM, a crosscutting behavior is modeled using so-called \&ldquo;aspect state machine\&rdquo;. The applicability of aspect state machines is evaluated by comparing them with standard UML state machines that directly model the entire system behavior, including crosscutting concerns. The quality of both aspect and standard UML state machines derived by subjects is measured by comparing them against their corresponding reference state machines. Results show that aspect state machines derived with AspectSM are significantly more complete and correct though AspectSM took significantly more time than the standard approach, probably due to a lack of familiarity of the subjects.},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel}
}
@misc {24757,
	title = {An Industrial Application of AOM  for Robustness Testing},
	howpublished = {Bellairs Workshop on Developing End-to-End AOSD Artifacts for REMODD},
	year = {2011},
	address = {Barbados},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.694,
	title = {An Industrial Application of Robustness Testing Using Aspect-Oriented Modeling, UML/MARTE, and Search Algorithms},
	journal = {ACM/IEEE 14th International Conference on Model Driven Engineering Languages and Systems (Models 2011)},
	year = {2011},
	pages = {108-122},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {Systematic and rigorous robustness testing is very critical for embedded systems, as for example communication and control systems. Robustness testing aims at testing the behavior of a system in the presence of faulty situations in its operating environment (e.g., sensors and actuators). In such situations, the system should gracefully degrade its performance instead of abruptly stopping execution. To systematically perform robustness testing, one option is to resort to model-based robustness testing (MBRT), based for example on UML/MARTE models. However, to successfully apply MBRT in industrial contexts, new technology needs to be developed to scale to the complexity of real industrial systems. In this paper, we report on our experience of performing MBRT on video conferencing systems developed by Cisco Systems, Norway. We discuss how we developed and integrated various techniques and tools to achieve a fully automated MBRT that is able to detect previously uncaught software faults in those systems. We provide an overview of how we achieved scalable modeling of robustness behavior using aspect-oriented modeling, test case generation using search algorithms, and environment emulation for test case execution. Our experience and lessons learned identify challenges and open research questions for the industrial application of MBRT.},
	author = {Ali, Shaukat and Briand, Lionel Claude and Arcuri, Andrea and Walawege, Suneth},
	editor = {Whittle, Jon and Clark, Tony and K{\"u}hne, Thomas}
}
@misc {24754,
	title = {An Industrial Application of Robustness Testing Using Aspect-Oriented Modeling, UML/MARTE, and Search Algorithms},
	howpublished = { ACM/IEEE 14th International Conference on Model Driven Engineering Languages and Systems (Models 2011)},
	year = {2011},
	publisher = {ACM/IEEE},
	address = {Wellington, New Zealand},
	author = {Ali, Shaukat}
}
@misc {24755,
	title = {A Search-Based OCL Constraint Solver for Model-Based Test Data Generation},
	howpublished = {11th International Conference On Quality Software (QSIC 2011)},
	year = {2011},
	publisher = {IEEE},
	address = {Madrid, Spain},
	author = {Ali, Shaukat}
}
@inproceedings {Simula.simula.500,
	title = {A Search-Based OCL Constraint Solver for Model-Based Test Data Generation},
	journal = {Proceedings of the 11th International Conference On Quality Software (QSIC 2011)},
	year = {2011},
	pages = {41-50},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Model-based testing (MBT) aims at automated, scalable, and systematic testing solutions for complex industrial software systems. To increase chances of adoption in industrial contexts, software systems should be modeled using well-established standards such as the Unified Modeling Language (UML) and Object Constraint Language (OCL). Given that test data generation is one of the major challenges to automate MBT, this is the topic of this paper with a specific focus on test data generation from OCL constraints. Though search-based software testing (SBST) has been applied to test data generation for white-box testing (e.g., branch coverage), its application to the MBT of industrial software systems has been limited. In this paper, we propose a set of search heuristics based on OCL constraints to guide test data generation and automate MBT in industrial applications. These heuristics are used to develop an OCL solver exclusively based on search, in this particular case genetic algorithm and (1+1) EA. Empirical analyses to evaluate the feasibility of our approach are carried out on one industrial system.},
	isbn = {978-0-7695-4468-7},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel}
}
@techreport {25259,
	title = {On Understandability of Aspect State Machines for Robustness Testing: A Controlled Experiment},
	number = {2011-02},
	year = {2011},
	note = {This technical report is an early version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Ali, Shaukat and Yue, Tao and Malik, Zafar}
}
@techreport {25318,
	title = {A Use Case Modeling Approach for Industrial, Network-based, Distributed, Real-Time Embedded Systems},
	number = {2011-07},
	year = {2011},
	note = {This technical report is an early version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Ali, Shaukat}
}
@techreport {27054,
	title = {A Use Case Modeling Approach for Large-scale, Industrial, Network-based, Distributed, Real-Time Embedded Systems},
	year = {2011},
	publisher = {Simula Research Laboratory },
	author = {Yue, Tao and Ali, Shaukat}
}
@inproceedings {Simula.approve.6,
	title = {An Enhanced Test Case Selection Approach for Model-Based Testing: an Industrial Case Study},
	journal = {18th ACM SIGSOFT international symposium on Foundations of Software Engineering (FSE)},
	year = {2010},
	publisher = {ACM},
	type = {1},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. MBT allows automatic generation of a large and comprehensive set of test cases from system models (e.g., state machines), which leads to the systematic testing of the system. However, even when using simple test strategies, applying MBT in large industrial systems often leads to generating large sets of test cases that cannot possibly be executed within time and cost constraints. In this situation, test case selection techniques are employed to select a subset from the entire test suite such that the selected subset conforms to available resources while maximizing fault detection. In this paper, we propose a new similarity-based selection technique for state machine-based test case selection, which includes a new similarity function using triggers and guards on transitions of state machines and a Genetic algorithm-based selection algorithm. Applying this technique on an industrial case study, we show that our proposed approach is more effective in detecting real faults than existing alternatives. We also assess the overall benefits of model-based test case selection in our case study by comparing the fault detection rate of the selected subset with the maximum possible fault detection rate of the original test suite.},
	isbn = {978-1-60558-791-2},
	author = {Hemmati, Hadi and Briand, Lionel and Arcuri, Andrea and Ali, Shaukat},
	editor = {Roman, Gruia-Catalin and van der Hoek, Andr{\'e}}
}
@techreport {25324,
	title = {A Framework for Measuring Quality of Models:~Experiences from a Series of Controlled~Experiments},
	year = {2010},
	note = {This technical report is an early version of a published paper.},
	publisher = {Simula Research Laboratory},
	author = {Yue, Tao and Ali, Shaukat}
}
@techreport {Simula.approve.2,
	title = {Model Transformations As a Strategy to Automate Model-Based Testing - a Tool and Industrial Case Studies},
	number = {2010-01},
	year = {2010},
	month = {March},
	publisher = {Simula Research Laboratory},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. The beneficial use of MBT, however, requires tools that not only automate the testing process, but that also rely in an extensible and configurable architecture that make them adaptable to various contexts of application. Though a number of tools have been developed to support MBT, this technical report introduces a new approach for designing and developing MBT tools that is based on model transformation technology. We report on the experimental development of a novel MBT tool, TRansformation-based tool for Uml-baSed Testing (TRUST), which software architecture and implementation strategy supports configurable and extensible features such as input models, test models, coverage criteria, test data generation strategies, and test script languages. Based on two industrial case studies, we demonstrate the configurability and extensibility of TRUST. We also investigate the challenges and likely cost savings when compared to manual test generation.},
	author = {Ali, Shaukat and Hemmati, Hadi and Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@techreport {Simula.approve.14,
	title = {Modeling Robustness Behavior Using Aspect-Oriented Modeling to Support Robustness Testing of Industrial Systems},
	number = {2010-03},
	year = {2010},
	month = {May 2010},
	publisher = {Simula Research Laboratory},
	abstract = {Model-based robustness testing requires, precise and complete behavioral, robustness modeling. For example state machines can be used to model software behavior when hardware (e.g., sensors) breaks down and be fed to a tool to automate test case generation. But robustness behavior is a crosscutting behavior and, if modeled directly, often results in large, complex state machines. These in practice tend to be error-prone and difficult to read and understand. As a result, modeling robustness behavior in this way is not scalable for complex industrial systems. To overcome these problems, Aspect Oriented Modeling (AOM) can be employed to model robustness behavior as aspects in the form of state machines specifically designed to model robustness behavior. In this paper, we present a RobUstness Modeling Methodology (RUMM) that allows modeling robustness behavior as aspects. Our goal is to have a complete and practical methodology that covers all features of state machines and aspect concepts necessary for model-based robustness testing. At the core of RUMM is a UML profile (AspectSM) that allows modeling UML state machine aspects as UML state machines (aspect state machines). Such an approach, relying on a standard and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of aspect modeling easier in industrial contexts. We have used AspectSM to model the crosscutting robustness behavior of a videoconferencing system and discuss the benefits of doing so in terms of reduced modeling effort and improved readability.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi}
}
@article {Simula.SE.661,
	title = {A Systematic Review of the Application and Empirical Investigation of Search-Based Test-Case Generation},
	journal = {IEEE Transactions on Software Engineering},
	volume = {36},
	number = {6},
	year = {2010},
	pages = {742-762},
	publisher = {IEEE},
	abstract = {Metaheuristic search techniques have been extensively used to automate the process of generating test cases and thus providing solutions for a more cost-effective testing process. This approach to test automation, often coined as \&ldquo;Search-based Software Testing\&rdquo; (SBST), has been used for a wide variety of test case generation purposes, differing in terms of test objectives, test levels, and other characteristics [1]. Since SBST techniques are heuristic by nature, they must be empirically investigated in terms of how costly and effective they are at reaching their test objectives and whether they scale up to realistic development artifacts. However, approaches to empirically study SBST techniques have shown wide variation in the literature. This paper presents the results of a systematic, comprehensive review that aims at characterizing how empirical studies have been designed to investigate SBST cost-effectiveness and what empirical evidence is available in the literature regarding SBST cost-effectiveness and scalability. We also provide a framework that drives the data collection process of this systematic review and can be the starting point of guidelines on how SBST techniques can be empirically assessed. The intent is to aid future researchers doing empirical studies in SBST by providing an unbiased view of the body of empirical evidence and by guiding them in performing well designed and executed empirical studies.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi and Rajwinder Kaur {Panesar-Walawege}}
}
@misc {24760,
	title = {Model-based Testing of Video Conferencing Systems: Automation Solution},
	howpublished = {D-Mint Project Meeting},
	year = {2009},
	address = {Tallin, Estonia},
	author = {Ali, Shaukat}
}
@techreport {Simula.SE.293,
	title = {A Systematic Review of the Application and Empirical Investigation of Search-Based Test-Case Generation},
	year = {2008},
	month = {September},
	publisher = {Simula Research Laboratory},
	abstract = {Metaheuristic search techniques have been extensively used to automate the process of generating test cases and thus providing solutions for a more cost-effective testing process. This approach to test automation, often coined as {\textquotedblleft}Search-based Software Testing{\textquotedblright} (SBST), has been used for a wide variety of test case generation purposes, differing in terms of test objectives, test levels, and other characteristics. Since SBST techniques are heuristic by nature, they must be empirically investigated in terms of how costly and effective they are at reaching their test objectives and whether they scale up to realistic development artifacts. However, approaches to empirically study SBST techniques have shown wide variation in the literature. This paper presents the results of a systematic, comprehensive review that aims at characterizing how empirical studies have been designed to investigate SBST cost-effectiveness and what empirical evidence is available in the literature regarding SBST cost-effectiveness and scalability. We also provide a framework that drives the data collection process of this systematic review and can be the starting point of guidelines on how SBST techniques can be empirically assessed. The intent is to aid future researchers doing empirical studies in SBST by providing an unbiased view of the body of empirical evidence and by guiding them in performing well designed and executed empirical studies.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi and Rajwinder Kaur {Panesar-Walawege}}
}
@article {Simula.approve.74,
	title = {Automated Discovery of State Transitions and Their Functions in Source Code},
	journal = {Software Testing, Verification and Reliability},
	volume = {18},
	number = {2},
	year = {2007},
	pages = {99-121},
	publisher = {John Wiley \& Sons},
	abstract = {Finite-state machine specifications form the basis for a number of rigorous state-based testing techniques and can help to understand program behaviour. Unfortunately they are rarely maintained during software development, which means that these benefits can rarely be fully exploited. This paper describes a technique that, given a set of states that are of interest to a developer, uses symbolic execution to reverse-engineer state transitions from source code. A particularly novel aspect of our approach is that, besides determining whether or not a state transition can take place, it also identifies the paths through the source code that govern a transition. The technique has been implemented as a prototype, enabling its preliminary evaluation with respect to real software systems.},
	author = {Walkinshaw, Neil and Bogdanov, Kirill and Ali, Shaukat and Holcombe, Mike}
}
@techreport {Simula.approve.72,
	title = {A Comparative Study of Methods for Dynamic Reverse-Engineering of State Models},
	number = {CS-07-16},
	year = {2007},
	publisher = {Deptartment of Computer Science, The University of Sheffield},
	abstract = {This paper provides a comparison of dynamic analysis techniques to reverse engineer state models of software. Since running a program generates a lot of data, the utility of deferent techniques is related to their ability to construct models which are both reflecting the relevant characteristics of a program and of manageable size. The classification of techniques in this paper aims to highlight the deferent approaches to abstraction utilized in the surveyed papers, so that one can mix-and-match deferent techniques depending on the kind of models which need to be built.},
	author = {Ali, Shaukat and Bogdanov, Kirill and Walkinshaw, Neil}
}
@article {Simula.SE.174,
	title = {A State-Based Approach to Integration Testing Based on UML Models},
	journal = {Information and Software Technology},
	volume = {49},
	number = {11-12},
	year = {2007},
	pages = {1087-1106},
	abstract = {Correct functioning of object-oriented software depends upon the successful integration of classes. While individual classes may function correctly, several new faults can arise when these classes are integrated together. In this paper, we present a technique to enhance testing of interactions among modal classes. The technique combines UML collaboration diagrams and statecharts to automatically generate an intermediate test model, called SCOTEM (State COllaboration TEst Model). The SCOTEM is then used to generate valid test paths. We also define various coverage criteria to generate test paths from the SCOTEM model. In order to assess our technique, we have developed a tool and applied it to a case study to investigate its fault detection capability. The results show that the proposed technique effectively detects all the seeded integration faults when complying with the most demanding adequacy criterion and still achieves reasonably good results for less expensive adequacy criteria.},
	author = {Ali, Shaukat and Briand, Lionel and Muhammad Jaffar-ur {Rehman} and Asghar, Hajra and Muhammad Zohaib {Iqbal} and Nadeem, Aamer}
}
@inproceedings {Mejia.2006.1,
	title = {Segment-Based Routing: an Efficient Fault-Tolerant Routing Algorithm for Meshes and Tori},
	journal = {20th IEEE International Parallel \& Distributed Processing Symposium},
	year = {2006},
	month = {April},
	pages = {1-10},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Washington, USA},
	abstract = {Computers get faster every year, but the demand for computing resources seems to grow at an even faster rate. Science keeps demanding more processing power for calculations and simulations, growth in E-commerce requires powerful servers to offer seamless online shopping, and massive multiplayer online games requires powerful and stable systems to keep their virtual worlds running 24 hours a day. Depending on the problem domain, this demand for more power can be satisfied by either, massively parallel computers, or clusters of computer. Common for both approaches is the dependence on high performance interconnect networks such as Myrinet, Infiniband, or 10 Gigabit Ethernet. While high throughput and low latency are key features of interconnection networks, the issue of fault-tolerance is now becoming increasingly important. As the number of network components grows so does the probability for failure, thus it becomes important to also consider the fault-tolerance mechanism of interconnection networks. The main challenge then lies in combining performance and fault-tolerance, while still keeping cost and complexity low. This paper proposes a new deterministic routing methodology for tori and meshes, which achieves high performance without the use of virtual channels. Furthermore, it is topology agnostic in nature, meaning it can handle any topology derived from any combination of faults when combined with static reconfiguration. The algorithm, referred to as Segment-based Routing (SR), works by partitioning a topology into subnets, and subnets into segments. This allows us to place bidirectional turn restrictions locally within a segment. As segments are independent, we gain the freedom to place turn restrictions within a segment independently from other segments. This results in a larger degree of freedom when placing turn restrictions compared to other routing strategies. In this paper a way to compute segment-based routing tables is presented and applied to meshes and tori. Preliminary evaluation results show that the concept of segments leads to an increase in performance by a factor of 1.8 over FX and up*/down* routing.},
	isbn = {1-4244-0054-6},
	author = {Mej{\'\i}a, Andres and Flich, Jos{\'e} and Duato, Jose and Reinemo, Sven-Arne and Skeie, Tor},
	editor = {Ali, Shaukat}
}
