@inproceedings {42903,
	title = {Generating Failing Test Suites for Quantum Programs with Search (hot off the press track at GECCO 2022)},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {47-48},
	publisher = {ACM},
	abstract = {The inherent complexity of quantum programs, due to features such as superposition and entanglement, makes their testing particularly challenging. To tackle these challenges, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size that possibly expose failures of the quantum program under test. QuSBT encodes a test suite as a search individual, and tries to maximize the objective function that counts the number of failing tests in the test suite. Due to non-deterministic nature of quantum programs, the approach repeats the execution of each test multiple times, and uses suitable statistical tests to assess if a test passes or fails. QuSBT employs a genetic algorithm to perform the search. Experiments on 30 faulty quantum programs show that QuSBT is statistically better than random search, and is able to efficiently generate maximal failing test suites.

This is an extended abstract of the paper [1]: X. Wang, P. Arcaini, T. Yue, and S. Ali "Generating Failing Test Suites for Quantum Programs With Search", 13th International Symposium on Search-Based Software Engineering (SSBSE 2021).},
	doi = {10.1145/3520304.3534067},
	url = {https://dl.acm.org/doi/10.1145/3520304.3534067},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42902,
	title = {Mutation-Based Test Generation for Quantum Programs with Multi-Objective Search},
	journal = {GECCO {\textquoteright}22: Proceedings of the Genetic and Evolutionary Computation Conference},
	year = {2022},
	pages = {1345-1353},
	publisher = {ACM},
	abstract = {Mutation testing is often used for designing new tests, and involves changing a program in minor ways, which results in mutated versions of the program, i.e., mutants. An effective test suite should find faults (or kill mutants) with a minimum number of test cases, to save resources required for executing test cases. In this paper, in the context of mutation testing for quantum programs, we present a multi-objective and search-based approach (MutTG) to generate the minimum number of test cases killing as many mutants as possible. MutTG tries to estimate the likelihood that a mutant is equivalent, and uses this as a\&nbsp;discount factor\&nbsp;in the fitness definition to avoid keeping on trying to kill mutants that cannot be killed. We employed NSGA-II as the multi-objective search algorithm. Then, we compared MutTG with another version of the approach that does not use the discount factor in its fitness definition, and with random search (RS), over a set of open-source quantum programs and their mutants of varying complexity. Results show that the discount factor does indeed help in guiding the test generation, as the approach with the discount factor performs better than the one without it.},
	doi = {10.1145/3512290.3528869},
	url = {https://dl.acm.org/doi/abs/10.1145/3512290.3528869},
	author = {Wang, Xinyi and Yu, Tongxuan and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@misc {42895,
	title = {Quantum Software Testing Tutorial},
	howpublished = {NordIQuEst-ENCCS online HPC-QC workshop},
	year = {2022},
	author = {Ali, Shaukat and Wang, Xinyi}
}
@misc {42904,
	title = {Quantum Software Testing Work in ComplexSE},
	howpublished = {Simula Research Laboratory, Norway},
	year = {2022},
	author = {Wang, Xinyi and Oldfield, Noah and Mendiluze, E{\~n}aut}
}
@inproceedings {42901,
	title = {QuSBT: Search-Based Testing of Quantum Programs},
	journal = {2022 IEEE/ACM 44th International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)},
	year = {2022},
	publisher = {IEEE},
	abstract = {Generating a test suite for a quantum program such that it has the maximum number of failing tests is an optimization problem. For such optimization, search-based testing has shown promising results in the context of classical programs. To this end, we present a test generation tool for quantum programs based on a genetic algorithm, called QuSBT (Search-based Testing of Quantum Programs). QuSBT automates the testing of quantum programs, with the aim of finding a test suite having the maximum number of failing test cases. QuSBT utilizes IBM{\textquoteright}s Qiskit as the simulation framework for quantum programs. We present the tool architecture in addition to the implemented methodology (i.e., the encoding of the search individual, the definition of the fitness function expressing the search problem, and the test assessment w.r.t. two types of failures). Finally, we report results of the experiments in which we tested a set of faulty quantum programs with QuSBT to assess its effectiveness. 
Repository (code and experimental results): https://github.com/Simula-COMPLEX/qusbt-tool 
Video: https://youtu.be/3apRCtluAn4},
	doi = {10. 1145/3510454.3516839},
	url = {https://ieeexplore.ieee.org/document/9793826},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@misc {28250,
	title = {Application of Combinatorial Testing to Quantum Programs},
	howpublished = {21st IEEE International Conference on Software Quality, Reliability, and Security (QRS 2021)},
	year = {2021},
	publisher = {IEEE},
	type = {Conference Paper},
	author = {Wang, Xinyi}
}
@inproceedings {28244,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {IEEE International Conference on Software Security and Reliability},
	year = {2021},
	publisher = { IEEE},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat},
	editor = {Wotawa, Franz and Wang, Xianpeng}
}
@inproceedings {42899,
	title = {Application of Combinatorial Testing to Quantum Programs},
	journal = {2021 IEEE 21st International Conference on Software Quality, Reliability and Security (QRS)},
	year = {2021},
	publisher = {IEEE},
	abstract = {The capability of Quantum Computing (QC) in solving complex problems has been increasingly recognized. However, similar to classical computing, to fully exploit QC{\textquoteright}s potential, it is important to ensure the correctness of quantum programs. Doing so via software testing is, however, very challenging because of QC{\textquoteright}s inherent properties: superposition and entanglement. Towards the direction of ensuring the correctness of quantum programs, we propose an approach called QuCAT (QUantum CombinAtorial Testing) for systematic and automated testing of quantum programs by benefiting from combinatorial testing, which has been proven to be cost-effective in testing classical programs. QuCAT supports two combinatorial test suite generation scenarios, i.e., generating combinatorial test suites of a given strength, and incrementally generating and executing combinatorial test suites of increasing strength until a fault is found. The approach employs two types of test oracles to assess test results. We performed an empirical study with 18 faulty versions of quantum programs to evaluate QuCAT with strengths of two, three, and four in the two test generation scenarios. We compare the cost-effectiveness of combinatorial testing of various strengths and random testing (taken as baseline approach). Results show that combinatorial testing always performs better than random testing with the same cost and finds faults more quickly (in terms of required number of test cases). In addition, in most cases, combinatorial testing with a higher strength outperforms the lower strength in terms of effectiveness.},
	doi = {10.1109/QRS54544.2021.00029},
	url = {https://ieeexplore.ieee.org/document/9724888},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42889,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Quantum programs implement quantum algorithms solving complex computational problems. Testing such programs is challenging due to the inherent characteristics of Quantum Computing (QC), such as the probabilistic nature and computations in superposition. However, automated and systematic testing is needed to ensure the correct behavior of quantum programs. To this end, we present an approach called Quito (QUantum InpuT Output coverage) consisting of three coverage criteria defined on the inputs and outputs of a quantum program, together with their test generation strategies. Moreover, we define two types of test oracles, together with a procedure to determine the passing and failing of test suites with statistical analyses. To evaluate the cost-effectiveness of the three coverage criteria, we conducted experiments with five quantum programs. We used mutation analysis to determine the coverage criteria{\textquoteright} effectiveness and cost in terms of the number of test cases. Based on the results of mutation analysis, we also identified equivalent mutants for quantum programs.},
	doi = {10.1109/ICST49551.2021.00014},
	url = {https://ieeexplore.ieee.org/document/9438603},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao}
}
@inproceedings {27579,
	title = {Assessing the Effectiveness of Input and Output Coverage Criteria for Testing Quantum Programs},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2021},
	pages = {13-23},
	publisher = { IEEE},
	author = {Ali, Shaukat and Arcaini, Paolo and Wang, Xinyi and Yue, Tao},
	editor = {Hierons, Robert and Ferrari, Fabiano}
}
@inproceedings {27924,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {13th Symposium on Search-Based Software Engineering},
	year = {2021},
	publisher = {LNCS},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {42898,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	journal = {Symposium on Search Base Software Engineering 2021},
	year = {2021},
	abstract = {Testing quantum programs requires systematic, automated, and intelligent methods due to their inherent complexity, such as their superposition and entanglement. To this end, we present a search-based approach, called Quantum Search-Based Testing (QuSBT), for automatically generating test suites of a given size depending on available testing budget, with the aim of maximizing the number of failing test cases in the test suite. QuSBT consists of definitions of the problem encoding, failure types, test assessment with statistical tests, fitness function, and test case generation with a Genetic Algorithm (GA). To empirically evaluate QuSBT, we compared it with Random Search (RS) by testing six quantum programs. We assessed the effectiveness of QuSBT and RS with 30 carefully designed faulty versions of the six quantum programs. Results show that QuSBT provides a viable solution for testing quantum programs, and achieved a significant improvement over RS in 87\% of the faulty programs, and no significant difference in the rest of 13\% of the faulty programs.},
	doi = {10.1007/978-3-030-88106-1_2},
	url = {https://link.springer.com/chapter/10.1007/978-3-030-88106-1_2},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@misc {28284,
	title = {Generating Failing Test Suites for Quantum Programs with Search},
	howpublished = {13th Symposium on Search-Based Software Engineering},
	year = {2021},
	type = {Conference Paper},
	author = {Wang, Xinyi}
}
@misc {28246,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	howpublished = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = {IEEE/ACM},
	type = {Lighting Talk and Tool Demonstration},
	author = {Wang, Xinyi}
}
@inproceedings {42900,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)},
	year = {2021},
	publisher = {IEEE},
	abstract = {Automation in quantum software testing is essential to support systematic and cost-effective testing. Towards this direction, we present a quantum software testing tool called Quito that can automatically generate test suites covering three coverage criteria defined on inputs and outputs of a quantum program coded in Qiskit, i.e., input coverage, output coverage, and input-output coverage. Quito also implements two types of test oracles based on program specifications, i.e., checking whether a quantum program produced a wrong output or checking a probabilistic test oracle with statistical test. We describe the architecture and methodology of the tool. We also validated the tool with one quantum program and one faulty version of it. Results indicate that Quito can generate test suites and perform test assessments that detect faults, and produce test results with a good time performance.
Quito{\textquoteright}s code: https://github.com/Simula-COMPLEX/quito
Quito{\textquoteright}s video: https://youtu.be/kuI9QaCo8A8Artifact Available: https://doi.org/10.5281/zenodo.5288665},
	doi = {0.1109/ASE51524.2021.9678798},
	url = {https://ieeexplore.ieee.org/document/9678798},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
@inproceedings {27928,
	title = {Quito: a Coverage-Guided Test Generator for Quantum Programs},
	journal = {The 36th IEEE/ACM International Conference on Automated Software Engineering},
	year = {2021},
	publisher = {IEEE/ACM},
	author = {Wang, Xinyi and Arcaini, Paolo and Yue, Tao and Ali, Shaukat}
}
