@article {24507,
	title = {Empirical evaluations on the cost-effectiveness of state-based testing: An industrial case study},
	journal = {Information and Software Technology},
	volume = {56},
	year = {2014},
	pages = {890-910},
	publisher = {Elsevier},
	author = {Nina Elisabeth {Holt} and Lionel C. {Briand} and Torkar, Richard}
}
@phdthesis {Simula.simula.1870,
	title = {Empirical Evaluations on the Cost-Effectiveness of State-Based Testing: Industrial Case Studies and Extensible Tool},
	year = {2012},
	month = {September},
	school = {University of Oslo},
	type = {phd},
	address = {Oslo, Norway},
	abstract = {Software testing is often conducted as a manual, ad hoc task, as compared to following an automated and more systematic procedure. Consequently, testing is likely to be incomplete and costly to ensure the required level of dependability. Safety-critical software systems must be tested so as to ensure its safe behavior. Despite the importance of being systematic while testing, all testing activities take place, even for safety-critical software, under resource constraints. In order for industry to make the right choices when deciding on how to test their software, more knowledge about how various testing strategies compare in terms of cost effectiveness is necessary. As thorough software testing is an expensive task, reducing the cost of testing while ensuring sufficient fault-detection effectiveness should be of common interest to industry. Enabling automated testing to check the compliance of implementations against their specifications, model-based testing has become a popular area of research and practice. Test models, for example expressed as UML state machines, describe the expected behavior of the software and provide the basis for systematic and automated generation of test suites. One specific area of research is related to how different coverage criteria of the test models affect the cost-effectiveness of the resulting test suites. This thesis assesses six state-based coverage criteria and evaluates their cost and fault-detection effectiveness based on 26 real faults collected in a field study at ABB. Eleven of the faults were sneak paths - thus, only 15 of the faults could be killed by the six conformance coverage criteria. Two different test oracles have been applied to compare their cost effectiveness. Moreover, this thesis also investigates the effect of increasing the test-model abstraction level on the cost-effectiveness of the testing strategies. The coverage criteria were complemented with sneak-path testing. To enable evaluation of the state-based testing techniques, a model-based testing approach, TRUST, was developed and used to automatically generate the studied test suites. Four industrial case studies evaluate each of the testing aspects: coverage criteria, oracles, test models, and sneak paths. The case studies are based on a research project at ABB where a safety-monitoring component in a control system was developed using state machines and implemented according to the extended state-design pattern. The findings of this thesis include: (1) Development and demonstration of a model-based testing approach based on model transformations. (2) An empirical investigation of the cost effectiveness of six systematic coverage criteria applied in an industrial project and evaluated by using real faults: all transitions (AT), all round-trip paths (RTP), all transition pairs (ATP), paths of length 2 (LN2), paths of length 3 (LN3), and paths of length 4 (LN4). (3) A comparison of two oracles: Oracle O1 checks the state invariant of the resulting state in addition to that the current state pointer of the system corresponds to the expected state after the test. Oracle O2 only checks that the state pointer to the current state of the system corresponds to the expected state after the test. (4) An evaluation of the cost-effectiveness when varying the level of details in the test model. (5) A demonstration of the importance of sneak-path testing. The results show that test suites generated from a precise model according to coverage criteria AT, RTP, ATP, and LN4 when utilizing oracle O1, yields high-quality test suites powerful enough to detect the seeded faults (except from sneak paths). The average cost measured as preparation and execution time were as follows: AT: 5,603 seconds; RTP: 2,731 seconds; ATP: 32,160 seconds; and LN4: 6,145 seconds. LN3 killed 93 percent (14/15) of the mutants at the cost of 645 seconds. Across all six coverage criteria, 88 percent of the seeded faults were detected at 7,905 seconds in preparation and execution time. Applying the weaker oracle O2 at an average decrease in cost around 13 percent, 67 percent of the mutants were killed. By removing details from the test model, the cost of testing was significantly decreased with 85 percent (for both oracles O1 and O2), while only reducing the fault-detection ability by 24 percent for oracle O1 and 37 percent for oracle O2. Note that these results were obtained in spite of a high number of infeasible test cases in test suites generated from the less detailed model as a consequence of wrong test data. Moreover, the sneak-path test suite detected the eleven remaining mutants that could not be killed by any of the conformance test suites. Thus, the results indicate quite strongly that sneak-path testing is a necessary step in state-based testing as the presence of sneak paths is undetectable by conformance testing. Finally, this thesis has demonstrated how model transformations can enable state-based testing. The tool was demonstrated to be extensible to support different types of state based coverage criteria and oracles.},
	author = {Nina Elisabeth {Holt}}
}
@techreport {Simula.approve.2,
	title = {Model Transformations As a Strategy to Automate Model-Based Testing - a Tool and Industrial Case Studies},
	number = {2010-01},
	year = {2010},
	month = {March},
	publisher = {Simula Research Laboratory},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. The beneficial use of MBT, however, requires tools that not only automate the testing process, but that also rely in an extensible and configurable architecture that make them adaptable to various contexts of application. Though a number of tools have been developed to support MBT, this technical report introduces a new approach for designing and developing MBT tools that is based on model transformation technology. We report on the experimental development of a novel MBT tool, TRansformation-based tool for Uml-baSed Testing (TRUST), which software architecture and implementation strategy supports configurable and extensible features such as input models, test models, coverage criteria, test data generation strategies, and test script languages. Based on two industrial case studies, we demonstrate the configurability and extensibility of TRUST. We also investigate the challenges and likely cost savings when compared to manual test generation.},
	author = {Ali, Shaukat and Hemmati, Hadi and Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@techreport {Simula.se.731,
	title = {An Eclipse Plug-In for the Flattening of Concurrency and Hierarchy in UML State Machines},
	number = {2009-06},
	year = {2009},
	publisher = {Simula Research Laboratory},
	abstract = {In this technical report, we present an algorithm that can be used to flatten simple-composite states (or-states), orthogonal states (and-states), and submachine states in complex UML state-machines [2]. The algorithm is appli-cable to state machines that include guarded transitions. The objective of this research is to automate the flattening of complex state machines. Orthogonal states and submachine states are transformed to simple-composite states. Simple-composite states are transformed to simple states by redirecting incoming and outgoing transitions. The result is a state machine where all state components are simple states. The flattening aspects are exemplified with isolated illustrations. Furthermore, we present STATE MACHINE FLATTENER - an Eclipse plug-in that implements this algorithm using Kermeta. Interesting aspect of the flattening are exemplified and discussed. Thus, the contribution of this study is threefold: 1) a flattening algorithm, 2) a plug-in that implements this algorithm to provide automatic flattening of complex UML state machines, and 2) a demonstration of the flattening approach in a number of example models.},
	author = {Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@inproceedings {Holt.2006.2,
	title = {Experiences With Precise State Modeling in an Industrial Safety Critical System},
	journal = {Critical Systems Development Using Modeling Lanuguages, CSDUML{\textquoteright}06},
	year = {2006},
	month = {October},
	pages = {68-77},
	publisher = {Springer},
	type = {Workshop},
	edition = {9th edition},
	chapter = {6},
	abstract = {The development of safety critical systems is a complex and challenging task. There are many claims regarding how precise modeling, either with the UML or other modeling notations, can yield benefits in terms of more rigorous specifications and designs. This paper reports on experiences from applying statechart-driven UML modeling in the development of a safety-critical system at ABB. The primary lessons learned from the study are related to the impact of precise modeling on the ease of transitioning to design.},
	author = {Nina Elisabeth {Holt} and Anda, Bente Cecilie Dahlum and Asskildt, Knut and Briand, Lionel and Endresen, Jan and Fr{\o}ystein, Sverre},
	editor = {Siv Hilde {Houmb} and Georg, Geri and France, Robert and Dorina C. {Petriu} and J{\"u}rjens, Jan}
}
@mastersthesis {Holt.2006.1,
	title = {A Systematic Review of Case Studies in Software Engineering},
	year = {2006},
	month = {May},
	publisher = {University of Oslo},
	type = {masters},
	author = {Nina Elisabeth {Holt}}
}
