@inproceedings {42903,
	title = {Generating Failing Test Suites for Quantum Programs with Search (hot off the press track at GECCO 2022)},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {47-48},
	publisher = {ACM},
	abstract = {The inherent complexity of quantum programs, due to features such as superposition and entanglement, makes their testing particularly challenging. To tackle these challenges, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size that possibly expose failures of the quantum program under test. QuSBT encodes a test suite as a search individual, and tries to maximize the objective function that counts the number of failing tests in the test suite. Due to non-deterministic nature of quantum programs, the approach repeats the execution of each test multiple times, and uses suitable statistical tests to assess if a test passes or fails. QuSBT employs a genetic algorithm to perform the search. Experiments on 30 faulty quantum programs show that QuSBT is statistically better than random search, and is able to efficiently generate maximal failing test suites.

This is an extended abstract of the paper [1]: X. Wang, P. Arcaini, T. Yue, and S. Ali "Generating Failing Test Suites for Quantum Programs With Search", 13th International Symposium on Search-Based Software Engineering (SSBSE 2021).},
	doi = {10.1145/3520304.3534067},
	url = {https://dl.acm.org/doi/10.1145/3520304.3534067},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42902,
	title = {Mutation-Based Test Generation for Quantum Programs with Multi-Objective Search},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {1345-1353},
	publisher = {ACM},
	abstract = {Mutation testing is often used for designing new tests, and involves changing a program in minor ways, which results in mutated versions of the program, i.e., mutants. An effective test suite should find faults (or kill mutants) with a minimum number of test cases, to save resources required for executing test cases. In this paper, in the context of mutation testing for quantum programs, we present a multi-objective and search-based approach (MutTG) to generate the minimum number of test cases killing as many mutants as possible. MutTG tries to estimate the likelihood that a mutant is equivalent, and uses this as a\&nbsp;discount factor\&nbsp;in the fitness definition to avoid keeping on trying to kill mutants that cannot be killed. We employed NSGA-II as the multi-objective search algorithm. Then, we compared MutTG with another version of the approach that does not use the discount factor in its fitness definition, and with random search (RS), over a set of open-source quantum programs and their mutants of varying complexity. Results show that the discount factor does indeed help in guiding the test generation, as the approach with the discount factor performs better than the one without it.},
	doi = {10.1145/3512290.3528869},
	url = {https://dl.acm.org/doi/abs/10.1145/3512290.3528869},
	author = {Wang, Xinyi and Yu, Tongxuan and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@article {28417,
	title = {On the Preferences of Quality Indicators for Multi-Objective Search Algorithms in Search-Based Software Engineering},
	journal = {The Empirical Software Engineering Journal (EMSE)},
	volume = {27},
	year = {2022},
	pages = {144},
	publisher = { Springer},
	author = {Wu, Jiahui and Arcaini, Paolo and Yue, Tao and Ali, Shaukat and Zhang, Huihui}
}
@article {28248,
	title = {Quantum Software Testing: Challenges, Early Achievements, and Opportunities},
	journal = {the European Research Consortium for Informatics and Mathematics News},
	year = {2022},
	publisher = {ERCIM},
	author = {Yue, Tao and Arcaini, Paolo and Ali, Shaukat}
}
@inproceedings {42901,
	title = {QuSBT: Search-Based Testing of Quantum Programs},
	journal = {2022 IEEE/ACM 44th International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)},
	year = {2022},
	publisher = {IEEE},
	abstract = {Generating a test suite for a quantum program such that it has the maximum number of failing tests is an optimization problem. For such optimization, search-based testing has shown promising results in the context of classical programs. To this end, we present a test generation tool for quantum programs based on a genetic algorithm, called QuSBT (Search-based Testing of Quantum Programs). QuSBT automates the testing of quantum programs, with the aim of finding a test suite having the maximum number of failing test cases. QuSBT utilizes IBM{\textquoteright}s Qiskit as the simulation framework for quantum programs. We present the tool architecture in addition to the implemented methodology (i.e., the encoding of the search individual, the definition of the fitness function expressing the search problem, and the test assessment w.r.t. two types of failures). Finally, we report results of the experiments in which we tested a set of faulty quantum programs with QuSBT to assess its effectiveness. 
Repository (code and experimental results): https://github.com/Simula-COMPLEX/qusbt-tool 
Video: https://youtu.be/3apRCtluAn4},
	doi = {10. 1145/3510454.3516839},
	url = {https://ieeexplore.ieee.org/document/9793826},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42861,
	title = {Towards Requirements Engineering for Digital Twins of Cyber-Physical Systems},
	journal = {11th International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	year = {2022},
	publisher = {LNCS},
	author = {Yue, Tao and Ali, Shaukat and Arcaini, Paolo and Ishikawa, Fuyuki and Cl{\'a}udio {\^A}ngelo Go {Gomes}},
	editor = {Fitzgerald, John and Margaria, Tiziana and Peter Gorm {Larsen} and Woodcock, Jim}
}
@inproceedings {42899,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)},
	year = {2021},
	publisher = {IEEE},
	abstract = {The capability of Quantum Computing (QC) in solving complex problems has been increasingly recognized. However, similar to classical computing, to fully exploit QC{\textquoteright}s potential, it is important to ensure the correctness of quantum programs. Doing so via software testing is, however, very challenging because of QC{\textquoteright}s inherent properties: superposition and entanglement. Towards the direction of ensuring the correctness of quantum programs, we propose an approach called QuCAT (QUantum CombinAtorial Testing) for systematic and automated testing of quantum programs by benefiting from combinatorial testing, which has been proven to be cost-effective in testing classical programs. QuCAT supports two combinatorial test suite generation scenarios, i.e., generating combinatorial test suites of a given strength, and incrementally generating and executing combinatorial test suites of increasing strength until a fault is found. The approach employs two types of test oracles to assess test results. We performed an empirical study with 18 faulty versions of quantum programs to evaluate QuCAT with strengths of two, three, and four in the two test generation scenarios. We compare the cost-effectiveness of combinatorial testing of various strengths and random testing (taken as baseline approach). Results show that combinatorial testing always performs better than random testing with the same cost and finds faults more quickly (in terms of required number of test cases). In addition, in most cases, combinatorial testing with a higher strength outperforms the lower strength in terms of effectiveness.},
	doi = {10.1109/QRS54544.2021.00029},
	url = {https://ieeexplore.ieee.org/document/9724888},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {28244,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {IEEE International Conference on Software Security and Reliability},
	year = {2021},
	publisher = { IEEE},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat},
	editor = {Wotawa, Franz and Wang, Xianpeng}
}
@inproceedings {27579,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	pages = {13-23},
	publisher = { IEEE},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao},
	editor = {Hierons, Robert and Ferrari, Fabiano}
}
@inproceedings {42889,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Quantum programs implement quantum algorithms solving complex computational problems. Testing such programs is challenging due to the inherent characteristics of Quantum Computing (QC), such as the probabilistic nature and computations in superposition. However, automated and systematic testing is needed to ensure the correct behavior of quantum programs. To this end, we present an approach called Quito (QUantum InpuT Output coverage) consisting of three coverage criteria defined on the inputs and outputs of a quantum program, together with their test generation strategies. Moreover, we define two types of test oracles, together with a procedure to determine the passing and failing of test suites with statistical analyses. To evaluate the cost-effectiveness of the three coverage criteria, we conducted experiments with five quantum programs. We used mutation analysis to determine the coverage criteria{\textquoteright} effectiveness and cost in terms of the number of test cases. Based on the results of mutation analysis, we also identified equivalent mutants for quantum programs.},
	doi = {10.1109/ICST49551.2021.00014},
	url = {https://ieeexplore.ieee.org/document/9438603},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao}
}
@inproceedings {27924,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {13th Symposium on Search-Based Software Engineering},
	year = {2021},
	publisher = {LNCS},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42898,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {Symposium on Search Base Software Engineering 2021},
	year = {2021},
	abstract = {Testing quantum programs requires systematic, automated, and intelligent methods due to their inherent complexity, such as their superposition and entanglement. To this end, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size depending on available testing budget, with the aim of maximizing the number of failing test cases in the test suite. QuSBT consists of definitions of the problem encoding, failure types, test assessment with statistical tests, fitness function, and test case generation with a Genetic Algorithm (GA). To empirically evaluate QuSBT, we compared it with Random Search (RS) by testing six quantum programs. We assessed the effectiveness of QuSBT and RS with 30 carefully designed faulty versions of the six quantum programs. Results show that QuSBT provides a viable solution for testing quantum programs, and achieved a significant improvement over RS in 87\% of the faulty programs, and no significant difference in the rest of 13\% of the faulty programs.},
	doi = {10.1007/978-3-030-88106-1_2},
	url = {https://link.springer.com/chapter/10.1007/978-3-030-88106-1_2},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27927,
	title = {Muskit: A Mutation Analysis Tool for Quantum Software Testing},
	journal = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = { IEEE/ACM},
	author = {Mendiluze, E{\~n}aut and Ali, Shaukat and Arcaini, Paolo and Yue, Tao}
}
@techreport {27892,
	title = {Muskit: A Mutation Analysis Tool for Quantum Software Testing},
	year = {2021},
	publisher = {Technical Report, Simula Research Laboratory},
	abstract = {Given that quantum software testing is a new area of research, there is a lack of benchmark programs and bug repositories to assess the effectiveness of testing techniques. To this end, quantum mutation analysis focuses on systematically generating faulty versions of Quantum Programs (QPs), called mutants, using mutation operators. Such mutants can be used as benchmarks to assess the quality of test cases in a test suite. Thus, we present muskit -- a quantum mutation analysis tool for QPs coded in IBM{\textquoteright}s Qiskit language. muskit defines mutation operators on gates of QPs and selection criteria to reduce the number of mutants to generate. Moreover, it allows for the execution of test cases on mutants and the generation of results for test analyses. muskit is provided as a command-line interface, GUI, and web application. We validated muskit by using it to generate and execute mutants for four QPs.},
	author = {Mendiluze, E{\~n}aut and Ali, Shaukat and Arcaini, Paolo and Yue, Tao}
}
@inproceedings {42900,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Automation in quantum software testing is essential to support systematic and cost-effective testing. Towards this direction, we present a quantum software testing tool called Quito that can automatically generate test suites covering three coverage criteria defined on inputs and outputs of a quantum program coded in Qiskit, i.e., input coverage, output coverage, and input-output coverage. Quito also implements two types of test oracles based on program specifications, i.e., checking whether a quantum program produced a wrong output or checking a probabilistic test oracle with statistical test. We describe the architecture and methodology of the tool. We also validated the tool with one quantum program and one faulty version of it. Results indicate that Quito can generate test suites and perform test assessments that detect faults, and produce test results with a good time performance.
Quito{\textquoteright}s code: https://github.com/Simula-COMPLEX/quito
Quito{\textquoteright}s video: https://youtu.be/kuI9QaCo8A8Artifact Available: https://doi.org/10.5281/zenodo.5288665},
	doi = {0.1109/ASE51524.2021.9678798},
	url = {https://ieeexplore.ieee.org/document/9678798},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27928,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = {IEEE/ACM},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27524,
	title = {Understanding Digital Twins for Cyber-Physical Systems: A Conceptual Model},
	journal = {International Symposium On Leveraging Applications of Formal Methods, Verification and Validation},
	volume = {12479},
	year = {2021},
	pages = {54-71},
	publisher = {LNCS, Springer, Cham},
	doi = {https://doi.org/10.1007/978-3-030-83723-5_5},
	author = {Yue, Tao and Arcaini, Paolo and Ali, Shaukat},
	editor = {Fitzgerald, John and Peter Gorm {Larsen}}
}
@inproceedings {27406,
	title = {Do Quality Indicators Prefer Particular Multi-Objective Search Algorithms in Search-Based Software Engineering?},
	journal = {12th Symposium on Search-Based Software Engineering},
	year = {2020},
	publisher = {LNCS},
	abstract = {In Search-Based Software Engineering (SBSE), users typically select a set of Multi-Objective Search Algorithms (MOSAs) for their experiments without any justification, or they simply choose an MOSA because of its popularity (e.g., NSGA-II). On the other hand, users know certain characteristics of solutions they are interested in. Such characteristics are typically measured with Quality Indicators (QIs) that are commonly used to evaluate the quality of solutions produced by an MOSA. Consequently, these QIs are often employed to empirically evaluate a set of MOSAs for a particular search problem to find the best MOSA. Thus, to guide SBSE users in choosing an MOSA that represents the solutions measured by a specific QI they are interested in, we present an empirical evaluation with a set of SBSE problems to study the relationships among commonly used QIs and MOSAs in SBSE. Our aim, by studying such relationships, is to identify whether there are certain characteristics of a QI because of which it prefers a certain MOSA. Such preferences are then used to provide insights and suggestions to SBSE users in selecting an MOSA, given that they know which quality aspects of solutions they are looking for.},
	author = {Ali, Shaukat and Arcaini, Paolo and Yue, Tao},
	editor = {Aleti, Aldeida and Panichella, Annibale}
}
@inproceedings {27145,
	title = {Generating Avoidable Collision Scenarios for Testing Autonomous Driving Systems},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST) 2020},
	year = {2020},
	pages = {375-386},
	publisher = { IEEE},
	author = {Cal{\`o}, Alessandro and Arcaini, Paolo and Ali, Shaukat and Hauer, Florian and Ishikawa, Fuyuki},
	editor = {Micco, John and Pistoia, Marco}
}
@article {27812,
	title = {Quality indicators in search-based software engineering: an empirical evaluation},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {29},
	number = {2},
	year = {2020},
	month = {04/2020},
	publisher = {ACM},
	abstract = {Search-Based Software Engineering (SBSE) researchers who apply multi-objective search algorithms (MOSAs) often assess the quality of solutions produced by MOSAs with one or more quality indicators (QIs). However, SBSE lacks evidence providing insights on commonly used QIs, especially about agreements among them and their relations with SBSE problems and applied MOSAs. Such evidence about QIs agreements is essential to understand relationships among QIs, identify redundant QIs, and consequently devise guidelines for SBSE researchers to select appropriate QIs for their specific contexts. To this end, we conducted an extensive empirical evaluation to provide insights on commonly used QIs in the context of SBSE, by studying agreements among QIs with and without considering differences of SBSE problems and MOSAs. In addition, by defining a systematic process based on three common ways of comparing MOSAs in SBSE, we present additional observations that were automatically produced based on the results of our empirical evaluation. These observations can be used by SBSE researchers to gain a better understanding of the commonly used QIs in SBSE, in particular, regarding their agreements. Finally, based on the results, we also provide a set of guidelines for SBSE researchers to select appropriate QIs for their particular context.},
	author = {Ali, Shaukat and Arcaini, Paolo and Pradhan, Dipesh and Safdar Aqeel {Safdar} and Yue, Tao}
}
@inproceedings {27344,
	title = {Simultaneously Searching and Solving Multiple Avoidable Collisions for Testing Autonomous Driving Systems},
	journal = {The Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2020},
	publisher = {ACM},
	author = {Cal{\`o}, Alessandro and Arcaini, Paolo and Ali, Shaukat and Hauer, Florian and Ishikawa, Fuyuki}
}
@inproceedings {26493,
	title = {Stability Analysis for Safety of Automotive Multi-Product Lines: A Search-Based Approach},
	journal = {The Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2019},
	pages = {1241-1249},
	publisher = {ACM},
	author = {Lee, Nian-Ze and Arcaini, Paolo and Ali, Shaukat and Ishikawa, Fuyuki}
}
@inproceedings {26314,
	title = {Towards a Framework for the Analysis of Multi-Product Lines in the Automotive Domain},
	journal = {Proceedings of the 13th International Workshop on Variability Modelling of Software-Intensive Systems},
	year = {2019},
	publisher = {ACM},
	address = {New York, NY, USA},
	author = {Ali, Shaukat and Arcaini, Paolo and Hasuo, Ichiro and Ishikawa, Fuyuki and Lee, Nian-Ze},
	editor = {Perrouin, Gilles}
}
@book {25143,
	title = {The Proceeding of the 13th Workshop on Advances in Model Based Testing (A-MOST)},
	volume = {13},
	year = {2017},
	month = {03/2017},
	publisher = {IEEE},
	address = {Tokyo, Japan},
	abstract = {The increasing complexity of software poses new challenges for testing. Model Based Testing (MBT) is an important research area, where new approaches, methods and tools make MBT techniques more useful and applicable also in industry. Models and different abstractions can ease comprehension of a complex system and allow the automation of test generation. A-MOST 2017 will bring together researchers and practitioners interested in the topic of Model Based Testing.Using models for designing and testing software is currently one of the most salient industrial trends with significant impact on the development and testing processes. Model-based tools and methods have been successfully applied and continue to converge into comprehensive approaches to software and system engineering. The area encompasses models derived from object-oriented software engineering, formal methods, and other mathematical and engineering disciplines. The execution of software using test cases or sequences derived in a manual or automatic manner from models is an encouraging scientific and industrial trend to cope with growing software system complexity. Modeling requires substantial investment, and practical and scalable MBT solutions can help leverage this investment. The testing models may have been adapted from system design models or might have been devised specifically to support MBT. Naturally, the greatest benefits are often obtained when test generation is automated, but many practitioners report that the modeling process itself is of value, often highlighting requirements issues. The use of industrial scale software demands the model-based construction of software and systems as compositions of independent and reusable actors. In this engineering paradigm, complex system functionality arises out of the composition of many component services. For these systems, model based testing may significantly improve component acceptance and move component integration testing towards a canonical validation and certification of complete systems. Automation of software development and software testing on the basis of executable models and simulation promises significant reductions in fault-removal cost and development time. As a consequence of automating MBT, changes in requirements analysis, development and testing processes are needed that demand combined efforts from research and industry towards a broadly accepted solution.A-MOST 2017 will focus on three main areas: the models used in MBT; the processes, techniques, and tools that support MBT; and evaluation. Here, evaluation includes the evaluation of software using MBT and the evaluation of MBT. These areas can be further broken down into the following topics.},
	author = {Wang, Shuai and Arcaini, Paolo and Devroey, Xavier}
}
