@inproceedings {Simula.simula.2370,
	title = {Model-Based Testing of Video Conferencing Systems: Challenges, Lessons Learnt, and Results},
	journal = {IEEE International Conference on Software Testing, Verification, and Validation (ICST)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Cleveland, OH},
	keywords = {Conference},
	author = {Ali, Shaukat and Hemmati, Hadi}
}
@article {Simula.simula.116,
	title = {Achieving Scalable Model-Based Testing Through Test Case Diversity},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {22},
	number = {1},
	year = {2013},
	publisher = {ACM},
	abstract = {The increase in size and complexity of modern software systems entails scalable, systematic, and automated testing approaches. Model-based testing (MBT), as a systematic and automated test case generation technique, is being successfully applied to verify industrial-scale systems and is supported by commercial tools. However, scalability is still an open issue for large systems as in practice there are limits to the amount of testing that can be performed in industrial contexts. Even with standard coverage criteria, the resulting test suites generated by MBT techniques can be very large and expensive to execute, especially for system level testing on real deployment platforms and network facilities. Therefore, a scalable MBT technique should be flexible regarding the size of the generated test suites and should be easily accommodated to fit resource and time constraints. Our approach is to select a subset of the generated test suite in such a way that it can be realistically executed and analyzed within the time and resource constraints, while preserving the fault revealing power of the original test suite to a maximum extent. In this paper, to address this problem, we introduce a family of similarity-based test case selection techniques. We evaluate 320 different similarity-based selection techniques and then compare the effectiveness of the best similarity-based selection technique with other common selection techniques in the literature. The results based on two industrial case studies show significant benefits and a large improvement in performance when using a similarity-based approach. We complement these analyses with further studies on the scalability of the technique and the effects of failure rate on its effectiveness. We also propose a method to identify optimal tradeoffs between the number of test cases to run and fault detection.},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel Claude}
}
@article {Simula.simula.118,
	title = {Modeling Robustness Behavior Using Aspect-Oriented Modeling to Support Robustness Testing of Industrial Systems},
	journal = {Software and Systems Modeling},
	volume = {11},
	number = {4},
	year = {2012},
	pages = {633-670},
	publisher = {Springer-Verlag},
	abstract = {Model-based robustness testing requires, precise and complete behavioral, robustness modeling, for example using state machines, to support automated test case generation. Robustness behavior is a crosscutting behavior and, if modeled directly, often results in large, complex state machines. These in practice tend to be error-prone and difficult to read and understand. As a result, modeling robustness behavior in this way is not scalable for complex industrial systems. To overcome these problems, Aspect Oriented Modeling (AOM) can be employed to model robustness behavior as aspects in the form of state machines specifically designed to model robustness behavior. In this paper, we present a RobUstness Modeling Methodology (RUMM) that allows modeling robustness behavior as aspects. At the core of RUMM is a UML profile (AspectSM) that allows modeling UML state machine aspects as UML state machines (aspect state machines). Such an approach, relying on a standard and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of aspect modeling easier, and thus in our context support robustness modeling for the purpose of automated model-based testing in industry. We discuss the benefits, such as reduced modeling effort and improved readability, achieved when applying RUMM to an industrial case study where we have used the profile to model the crosscutting robustness behavior of a videoconferencing system.},
	author = {Ali, Shaukat and Briand, Lionel Claude and Hemmati, Hadi}
}
@inproceedings {Simula.simula.8,
	title = {Empirical Investigation of the Effects of Test Suite Properties on Similarity-Based Test Case Selection},
	journal = {4th International Conference on Software Testing, Verification and Validation (ICST2011)},
	year = {2011},
	pages = {327-336},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Berlin},
	abstract = {Our experience with applying model-based testing on industrial systems showed that the generated test suites are often too large and costly to execute given project deadlines and the limited resources for system testing on real platforms. In such industrial contexts, it is often the case that only a small subset of test cases can be run. In previous work, we proposed novel test case selection techniques that minimize the similarities among selected test cases and outperforms other selection alternatives. In this paper, our goal is to gain insights into why and under which conditions similarity-based selection techniques, and in particular our approach, can be expected to work. We investigate the properties of test suites with respect to similarities among fault revealing test cases. We thus identify the ideal situation in which a similarity-based selection works best, which is useful for devising more effective similarity functions. We also address the specific situation in which a test suite contains outliers, that is a small group of very different test cases, and show that it decreases the effectiveness of similarity-based selection. We then propose, and successfully evaluate based on two industrial systems, a solution based on rank scaling to alleviate this problem.},
	isbn = {978-1-61284-174-8},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel Claude}
}
@phdthesis {Simula.simula.366,
	title = {Similarity-Based Test Case Selection: Toward Scalable and Practical Model-Based Testing},
	year = {2011},
	school = {University of Oslo},
	type = {phd},
	abstract = {The growing complexity and size of software systems, along with the increasing role of software in everyday life, makes verification and validation, and testing in particular, essential in software engineering. High fault revealing power, with minimum cost, is the ultimate goal in software testing. Model-based testing (MBT) targets this goal by automating test generation in a systematic way from abstract models of the software under test. Automation reduces the test generation cost dramatically, but the total testing cost includes the cost of test execution and evaluation as well. In practice, there are limitations in testing budget both in terms of time and testing resources. A testing approach cannot be scalable and practical in large industrial systems, unless it addresses all dimensions of the testing cost. But the systematic test generation nature of MBT potentially results in large test suites with execution costs that far exceed the testing budget. Therefore, a mechanism for adjusting the size of the output test suite in MBT is an absolute necessity to ensure success in industry. This thesis proposes techniques that minimize the test suite size while preserving (to the maximum extent) its fault detection rate. The proposed family of techniques, called similarity-based test case selection, hypothesizes that the more diverse the test cases, the higher the fault detection rate. The thesis initiates with a systematic review on search-based techniques for test case generation, which is a starting point for identifying the potential approaches for search-based test case selection being used in similarity-based test case selection. Finding the most effective ways of defining similarity measures and selection algorithms constitutes the core of the thesis. The best selection techniques among different variants of the proposed similarity-based techniques are identified through rigorous empirical analyses on two industrial case studies. The cost-effectiveness of the approach is also compared to the existing selection techniques in the literature. Then, different influential factors on the effectiveness of the technique are examined through controlled experiments in order to gain insight on the analyzed problem, and to gain confidence in the reliability of the results. The main contribution of this thesis includes the proposal and evaluation of highly effective similarity-based test case selection techniques, which turns out to be extremely beneficial in two industrial contexts (up to 50\% reduction in the number of test cases required for detecting the same number of faults as to the current, best alternatives). Furthermore, the technique showed to be scalable with test suite size and also robust to variations in the fault detection rate of the test suite. Another contribution is a complementary study on estimating the best size for a test suite based on similarity comparisons among test cases. From a practical standpoint, this is a significant contribution to increase the usability of the proposed techniques, since testers are no longer required to select an arbitrary test suite size. In conclusion, similarity-based test case selection showed promising results on two industrial case studies with respect to minimizing the testing cost in MBT. The proposed technique is far more effective than existing techniques. It helps make MBT applicable on larger systems by adjusting the output test suite according to test budget. This research has therefore the potential to significantly impact how MBT is performed today.},
	author = {Hemmati, Hadi}
}
@techreport {Simula.simula.120,
	title = {Achieving Scalable Model-Based Testing Through Test Case Diversity},
	number = {2010-18},
	year = {2010},
	publisher = {Simula Research Laboratory},
	abstract = {The increase in size and complexity of modern software systems entails scalable, systematic, and automated testing approaches. Model-based testing (MBT), as a systematic and automated test case generation technique, is being successfully applied to verify industrial-scale systems and is supported by commercial tools. However, scalability is still an open issue for large systems as in practice there are limits to the amount of testing that can be performed in industrial contexts. Even with standard coverage criteria, the resulting test suites generated by MBT techniques can be very large and expensive to execute, especially for system level testing on real deployment platforms and network facilities. Therefore, a scalable MBT technique should be flexible regarding the size of the generated test suites and should be easily accommodated to fit resource and time constraints. Our approach is to select a subset of the generated test suite in such a way that it can be realistically executed and analyzed within the time and resource constraints, while preserving the fault revealing power of the original test suite to a maximum extent. In this paper, to address this problem, we introduce a family of similarity-based test case selection techniques. We evaluate 320 different similarity-based selection techniques and then compare the effectiveness of the best similarity-based selection technique with other common selection techniques in the literature. The results based on two industrial case studies show significant benefits and a large improvement in performance when using a similarity-based approach. We complement these analyses with further studies on the scalability of the technique and the effects of failure rate on its effectiveness. We also propose a method to identify optimal tradeoffs between the number of test cases to run and fault detection.},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel}
}
@techreport {Simula.simula.10,
	title = {Empirical Investigation of the Effects of Test Suite Properties on Similarity-Based Test Case Selection},
	number = {2010-14},
	year = {2010},
	publisher = {Simula Research Laboratory},
	abstract = {Our experience with applying model-based testing on industrial systems showed that the generated test suites are often too large and costly to execute given project deadlines and the limited resources for system testing on real platforms. In such industrial contexts, it is often the case that only a small subset of test cases can be run. In previous work, we proposed novel test case selection techniques that minimize the similarities among selected test cases and outperforms other selection alternatives. In this report, our goal is to gain insights into why and under which conditions similarity-based selection techniques, and in particular our approach, can be expected to work. We investigate the properties of test suites with respect to similarities among fault revealing test cases. We thus identify the ideal situation in which a similarity-based selection works best, which is useful for devising more effective similarity functions. We also address the specific situation in which a test suite contains outliers, that is a small group of very different test cases, and show that it decreases the effectiveness of similarity-based selection. We then propose, and successfully evaluate based on two industrial systems, a solution based on rank scaling to alleviate this problem.},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel}
}
@inproceedings {Simula.approve.6,
	title = {An Enhanced Test Case Selection Approach for Model-Based Testing: an Industrial Case Study},
	journal = {18th ACM SIGSOFT international symposium on Foundations of Software Engineering (FSE)},
	year = {2010},
	publisher = {ACM},
	type = {1},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. MBT allows automatic generation of a large and comprehensive set of test cases from system models (e.g., state machines), which leads to the systematic testing of the system. However, even when using simple test strategies, applying MBT in large industrial systems often leads to generating large sets of test cases that cannot possibly be executed within time and cost constraints. In this situation, test case selection techniques are employed to select a subset from the entire test suite such that the selected subset conforms to available resources while maximizing fault detection. In this paper, we propose a new similarity-based selection technique for state machine-based test case selection, which includes a new similarity function using triggers and guards on transitions of state machines and a Genetic algorithm-based selection algorithm. Applying this technique on an industrial case study, we show that our proposed approach is more effective in detecting real faults than existing alternatives. We also assess the overall benefits of model-based test case selection in our case study by comparing the fault detection rate of the selected subset with the maximum possible fault detection rate of the original test suite.},
	isbn = {978-1-60558-791-2},
	author = {Hemmati, Hadi and Briand, Lionel and Arcuri, Andrea and Ali, Shaukat},
	editor = {Roman, Gruia-Catalin and van der Hoek, Andr{\'e}}
}
@inproceedings {Simula.approve.32,
	title = {An Industrial Investigation of Similarity Measures for Model-Based Test Case Selection},
	journal = {21st International Symposium on Software Reliability Engineering, ISSRE},
	year = {2010},
	publisher = {IEEE},
	type = {Conference},
	abstract = {Applying model-based testing (MBT) in practice requires practical solutions for scaling up to large industrial systems. One challenge that we have faced while applying MBT was the generation of test suites that were too large to be practical, even for simple coverage criteria. The goal of test case selection techniques is to select a subset of the generated test suite that satisfies resource constraints while yielding a maximum fault detection rate. One interesting heuristic is to choose the most diverse test cases based on a pre-defined similarity measure. In this paper, we investigate and compare possible similarity functions to support similarity-based test selection in the context of state machine testing, which is the most common form of MBT. We apply the proposed similarity measures and a selection strategy based on genetic algorithms to an industrial software system. We compare their fault detection rate based on actual faults. The results show that applying Jaccard Index on test cases represented as a set of trigger-guards is the most cost-effective similarity measure. We also discuss the overall benefits of our test selection approach in terms of test execution savings.},
	isbn = {978-1-4244-9056-1},
	author = {Hemmati, Hadi and Briand, Lionel},
	editor = {Marathe, Mod and Cukier, Michel}
}
@techreport {25327,
	title = {Investigation of Model-based Encodings of Test Cases in Similarity-based Test case Selection},
	year = {2010},
	publisher = {Simula Research Laboratory},
	author = {Hemmati, Hadi}
}
@techreport {Simula.approve.2,
	title = {Model Transformations As a Strategy to Automate Model-Based Testing - a Tool and Industrial Case Studies},
	number = {2010-01},
	year = {2010},
	month = {March},
	publisher = {Simula Research Laboratory},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. The beneficial use of MBT, however, requires tools that not only automate the testing process, but that also rely in an extensible and configurable architecture that make them adaptable to various contexts of application. Though a number of tools have been developed to support MBT, this technical report introduces a new approach for designing and developing MBT tools that is based on model transformation technology. We report on the experimental development of a novel MBT tool, TRansformation-based tool for Uml-baSed Testing (TRUST), which software architecture and implementation strategy supports configurable and extensible features such as input models, test models, coverage criteria, test data generation strategies, and test script languages. Based on two industrial case studies, we demonstrate the configurability and extensibility of TRUST. We also investigate the challenges and likely cost savings when compared to manual test generation.},
	author = {Ali, Shaukat and Hemmati, Hadi and Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@techreport {Simula.approve.14,
	title = {Modeling Robustness Behavior Using Aspect-Oriented Modeling to Support Robustness Testing of Industrial Systems},
	number = {2010-03},
	year = {2010},
	month = {May 2010},
	publisher = {Simula Research Laboratory},
	abstract = {Model-based robustness testing requires, precise and complete behavioral, robustness modeling. For example state machines can be used to model software behavior when hardware (e.g., sensors) breaks down and be fed to a tool to automate test case generation. But robustness behavior is a crosscutting behavior and, if modeled directly, often results in large, complex state machines. These in practice tend to be error-prone and difficult to read and understand. As a result, modeling robustness behavior in this way is not scalable for complex industrial systems. To overcome these problems, Aspect Oriented Modeling (AOM) can be employed to model robustness behavior as aspects in the form of state machines specifically designed to model robustness behavior. In this paper, we present a RobUstness Modeling Methodology (RUMM) that allows modeling robustness behavior as aspects. Our goal is to have a complete and practical methodology that covers all features of state machines and aspect concepts necessary for model-based robustness testing. At the core of RUMM is a UML profile (AspectSM) that allows modeling UML state machine aspects as UML state machines (aspect state machines). Such an approach, relying on a standard and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of aspect modeling easier in industrial contexts. We have used AspectSM to model the crosscutting robustness behavior of a videoconferencing system and discuss the benefits of doing so in terms of reduced modeling effort and improved readability.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi}
}
@inproceedings {Simula.approve.34,
	title = {Reducing the Cost of Model-Based Testing Through Test Case Diversity},
	journal = {22nd IFIP International Conference on Testing Software and Systems (ICTSS) - formerly TestCom/FATES},
	volume = {6435},
	year = {2010},
	publisher = {Springer},
	type = {Conference},
	abstract = {Model-based testing (MBT) suffers from two main problems which in many real world systems make MBT impractical: scalability and automatic oracle generation. When no automated oracle is available, or when testing must be performed on actual hardware or a restricted-access network, for example, only a small set of test cases can be executed and evaluated. However, MBT techniques usually generate large sets of test cases when applied to real systems, regardless of the coverage criteria. Therefore, one needs to select a small enough subset of these test cases that have the highest possible fault revealing power. In this paper, we investigate and compare various techniques for rewarding diversity in the selected test cases as a way to increase the likelihood of fault detection. We use a similarity measure defined on the representation of the test cases and use it in several algorithms that aim at maximizing the diversity of test cases. Using an industrial system with actual faults, we found that rewarding diversity leads to higher fault detection compared to the techniques commonly reported in the literature: coverage-based and random selection. Among the investigated algorithms, diversification using Genetic Algorithms is the most cost-effective technique.},
	isbn = {978-3-642-16572-6},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel},
	editor = {Petrenko, A. and Sim{\~a}o, A. and Maldonado, J.C.}
}
@article {Simula.SE.661,
	title = {A Systematic Review of the Application and Empirical Investigation of Search-Based Test-Case Generation},
	journal = {IEEE Transactions on Software Engineering},
	volume = {36},
	number = {6},
	year = {2010},
	pages = {742-762},
	publisher = {IEEE},
	abstract = {Metaheuristic search techniques have been extensively used to automate the process of generating test cases and thus providing solutions for a more cost-effective testing process. This approach to test automation, often coined as \&ldquo;Search-based Software Testing\&rdquo; (SBST), has been used for a wide variety of test case generation purposes, differing in terms of test objectives, test levels, and other characteristics [1]. Since SBST techniques are heuristic by nature, they must be empirically investigated in terms of how costly and effective they are at reaching their test objectives and whether they scale up to realistic development artifacts. However, approaches to empirically study SBST techniques have shown wide variation in the literature. This paper presents the results of a systematic, comprehensive review that aims at characterizing how empirical studies have been designed to investigate SBST cost-effectiveness and what empirical evidence is available in the literature regarding SBST cost-effectiveness and scalability. We also provide a framework that drives the data collection process of this systematic review and can be the starting point of guidelines on how SBST techniques can be empirically assessed. The intent is to aid future researchers doing empirical studies in SBST by providing an unbiased view of the body of empirical evidence and by guiding them in performing well designed and executed empirical studies.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi and Rajwinder Kaur {Panesar-Walawege}}
}
@inproceedings {Simula.SE.281,
	title = {Self-Reconfiguration in Highly Available Pervasive Computing Systems},
	journal = {Autonomic and Trusted Computing},
	volume = {Volume 5060/2008},
	year = {2008},
	month = {June},
	pages = {289-301},
	publisher = {Springer Berlin / Heidelberg},
	type = {Conference},
	chapter = {Pervasive Systems},
	abstract = {High availability of software systems is an essential requirement for pervasive computing environments. In such systems self-adaptation, using dynamic reconfiguration is also a key feature. However, dynamic reconfiguration potentially decreases the system availability by making parts of the system temporary frozen, especially during incomplete or faulty execution of the reconfiguration process. In this paper, we propose Assured Dynamic Reconfiguration Framework (ADRF), consisting of run-time analysis phases, assuring the desired correctness and completeness of dynamic reconfiguration process. We also specify factors that affect availability of reconfigurable software in pervasive computing systems. Observing the effects of these factors, we present availability improvement of our method in comparison to the other reconfiguration mechanisms.},
	isbn = {978-3-540-69294-2},
	doi = {10.1007/978-3-540-69295-9\_24},
	author = {Hemmati, Hadi and Jalili, Rasool},
	editor = {Rong, Chunming and Martin Gilje {Jaatun} and Frode Eika {Sandnes} and Laurence T. {Yang} and Ma, Jianhua}
}
@techreport {Simula.SE.293,
	title = {A Systematic Review of the Application and Empirical Investigation of Search-Based Test-Case Generation},
	year = {2008},
	month = {September},
	publisher = {Simula Research Laboratory},
	abstract = {Metaheuristic search techniques have been extensively used to automate the process of generating test cases and thus providing solutions for a more cost-effective testing process. This approach to test automation, often coined as {\textquotedblleft}Search-based Software Testing{\textquotedblright} (SBST), has been used for a wide variety of test case generation purposes, differing in terms of test objectives, test levels, and other characteristics. Since SBST techniques are heuristic by nature, they must be empirically investigated in terms of how costly and effective they are at reaching their test objectives and whether they scale up to realistic development artifacts. However, approaches to empirically study SBST techniques have shown wide variation in the literature. This paper presents the results of a systematic, comprehensive review that aims at characterizing how empirical studies have been designed to investigate SBST cost-effectiveness and what empirical evidence is available in the literature regarding SBST cost-effectiveness and scalability. We also provide a framework that drives the data collection process of this systematic review and can be the starting point of guidelines on how SBST techniques can be empirically assessed. The intent is to aid future researchers doing empirical studies in SBST by providing an unbiased view of the body of empirical evidence and by guiding them in performing well designed and executed empirical studies.},
	author = {Ali, Shaukat and Briand, Lionel and Hemmati, Hadi and Rajwinder Kaur {Panesar-Walawege}}
}
