@article {Simula.simula.2292,
	title = {Achieving Scalable Mutation-Based Generation of Whole Test Suites},
	journal = {Empirical Software Engineering},
	volume = {20},
	year = {2014},
	pages = {783-812},
	publisher = {Springer Link},
	abstract = {Without complete formal specification, automatically generated software tests need to be manually checked in order to detect faults. This makes it desirable to produce the strongest possible test set while keeping the number of tests as small as possible. As commonly applied coverage criteria like branch coverage are potentially weak, mutation testing has been proposed as a stronger criterion. However, mutation based test generation is hampered because usually there are simply too many mutants, and too many of these are either trivially killed or equivalent. On such mutants, any effort spent on test generation would per definition be wasted. To overcome this problem, our search-based EVOSUITE test generation tool integrates two novel optimizations: First, we avoid redundant test executions on mutants by monitoring state infection conditions, and second we use whole test suite generation to optimize test suites towards killing the highest number of mutants, rather than selecting individual mutants. These optimizations allowed us to apply EVOSUITE to a random sample of 100 open source projects, consisting of a total of 8,963 classes and more than two million lines of code, leading to a total of 1,380,302 mutants. The experiment demonstrates that our approach scales well, making mutation testing a viable test criterion for automated test case generation tools, and allowing us to analyze the relationship of branch coverage and mutation testing in detail.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1858,
	title = {Automated Unit Test Generation for Classes With Environment Dependencies},
	journal = {IEEE/ACM International Conference on Automated Software Engineering (ASE)},
	year = {2014},
	month = {09/2014},
	pages = {79-90},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {New York, USA},
	abstract = {Automated test generation for object-oriented software typically consists of producing sequences of calls aiming at high code coverage. In practice, the success of this process may be inhibited when classes interact with their environment, such as the file system, network, user-interactions, etc. This leads to two major problems: First, code that depends on the environment can sometimes not be fully covered simply by generating sequences of calls to a class under test. For example, execution of a branch may depend on the contents of a file. Second, even if code that is environmentdependent can be covered, the resulting tests may be unstable, i.e., they would pass when first generated, but then may fail when executed in a different environment. For example, tests on classes that make use of the system time may have failing assertions if the tests are executed at a different time than when they were generated. In this paper, we apply bytecode instrumentation to automatically separate code from its environmental dependencies, and extend the EVOSUITE Java test generation tool such that it can explicitly set the state of the environment as part of the sequences of calls it generates. Using a prototype implementation, which handles a wide range of environmental interactions such as the file system, console inputs and many non-deterministic functions of the JVM, we performed experiments on 100 Java projects randomly selected from SourceForge (the SF100 corpus). The results show significantly improved code coverage - in some cases even in the order of +80\%/+90\%. Furthermore, our techniques reduce the number of unstable tests by more than 50\%.},
	keywords = {Conference},
	author = {Arcuri, Andrea and Fraser, Gordon and Juan Pablo {Galeotti}}
}
@inproceedings {Simula.simula.2732,
	title = {Automated Unit Test Generation for Hadoop Using EvoSuite},
	journal = {conference},
	year = {2014},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {Apache Hadoop is a popular framework for big data analysis. It has a large number of contributors working on large code base-and this naturally implies a certain number of bugs. As part of the SSBSE 2014 Challenge, this paper presents the results of applying the EVOSUITE automated unit test generation tool on a part of Hadoop. We compared the generated tests to those provided by the developers, and found that generated tests are superior in terms of coverage.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.2190,
	title = {Continuous Test Generation: Enhancing Continuous Integration With Automated Test Generation},
	journal = {IEEE/ACM International Conference on Automated Software Engineering (ASE)},
	year = {2014},
	month = {09/2015},
	pages = {55-66},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {New York, USA},
	abstract = {Continuous integration is a common industry practice, where changes on software projects by different developers are integrated, built and tested at regular intervals. Currently, the testing step relies on existing test suites, but continuous integration offers significant potential for automated test generation - which, unlike continuous integration, is not widely adopted by practitioners yet. We argue that a combination of the two techniques, which we call Continuous Test Generation, will help to bridge the gap between academic research and industrial practice in automated test generation. By making automated test generation part of continuous integration, we can leverage the computing power offered by dedicated build servers and cloudinfrastructures, direct testing efforts to where they are needed after changes, and improve test generation by reusing test results across dependencies and versions. In addition, developers do not have to directly interact with a test generation tool, which reduces usability concerns. The notion of Continuous Test Generation leads to many open research questions, of which this paper aims to address the most fundamental ones by empirically studying a prototype implementation based on the EVOSUITE test generation tool on 10 open source projects (from the SF100 corpus) and five industrial case studies, totalling 1,586 Java classes.},
	isbn = {978-1-4503-3013-8},
	doi = {10.1145/2642937.2643002},
	author = {Campos, Jos{\'e} and Arcuri, Andrea and Fraser, Gordon and Abreu, Rui}
}
@article {Simula.simula.2428,
	title = {Does Automated White-Box Unit Test Generation Really Help Software Testers?},
	journal = {journal},
	year = {2014},
	publisher = {ACM},
	abstract = {Work on automated test generation has produced several tools capable of generating test data which achieves high structural coverage over a program. In the absence of a specification, developers are expected to manually construct or verify the test oracle for each test input. Nevertheless, it is assumed that these generated tests ease the task of testing for the developer, as testing is reduced to checking the results of tests. While this assumption has persisted for decades, there has been no conclusive evidence to date confirming it. However, the limited adoption in industry indicates this assumption may not be correct, and calls into question the practice value of test generation tools. To investigate this issue, we performed two controlled experiments comparing a total of 97 subjects split between writing tests manually and writing tests with the aid of an automated unit test generation tool, EVOSUITE. We found that, on one hand, tool support leads to clear improvements in commonly applied quality metrics such as code coverage (up to 300\% increase). However, on the other hand, there was no measurable improvement in the number of bugs actually found by developers. Our results not only cast some doubt on how the research community evaluates test generation tools, but also point to improvements and future work necessary before automated test generation tools will be widely adopted by practitioners.},
	author = {Fraser, Gordon and Staats, Matt and McMinn, Phil and Arcuri, Andrea and Padberg, Frank}
}
@inproceedings {Simula.simula.2754,
	title = {On the Effectiveness of Whole Test Suite Generation},
	journal = {Symposium on Search-Based Software Engineering},
	year = {2014},
	pages = {1-15},
	publisher = {Springer International Publishing},
	type = {Conference},
	address = {.},
	abstract = {A common application of search-based software testing is to generate test cases for all goals defined by a coverage criterion (e.g., statements, branches, mutants). Rather than generating one test case at a time for each of these goals individually, whole test suite generation optimizes entire test suites towards satisfying all goals at the same time. There is evidence that the overall coverage achieved with this approach is superior to that of targeting individual coverage goals. Nevertheless, there remains some uncertainty on whether the whole test suite approach might be inferior to a more focused search in the case of particularly difficult coverage goals. In this paper, we perform an in-depth analysis to study if this is the case. An empirical study on 100 Java classes reveals that indeed there are some testing goals that are easier to cover with the traditional approach. However, their number is not only very small in comparison with those which are only covered by the whole test suite approach, but also those coverage goals appear in small classes for which both approaches already obtain high coverage.},
	isbn = {978-3-319-09940-8},
	doi = {10.1007/978-3-319-09940-8_1},
	author = {Arcuri, Andrea and Fraser, Gordon}
}
@inproceedings {Simula.simula.2272,
	title = {EvoSuite at the Second Unit Testing Tool Competition},
	journal = {Fittest Workshop},
	year = {2014},
	pages = {95-100},
	publisher = {Springer International Publishing},
	type = {Workshop},
	abstract = {EvoSuite is a mature research prototype implementing a search-based approach to unit test generation for Java classes. It has been successfully run on a variety of di erent Java projects in the past, and after winning the rst instance of the unit testing tool competition at SBST{\textquoteright}13, it has also taken part in the second run. This paper reports on the obstacles and challenges encountered during the latter competition.},
	isbn = {978-3-319-07785-7},
	doi = {10.1007/978-3-319-07785-7_6},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.2270,
	title = {Higher Coverage With Combined Search-Based and Constraint-Based Test Suite Generation},
	journal = {ACM International Conference on Software Testing and Analysis (ISSTA)},
	year = {2014},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {Automatic unit test generation aims to support developers by alleviating the burden of test writing. Different techniques have been proposed over the years, each with distinct limitations. To overcome these limitations, we present an extension to the EVOSUITE unit test generator that combines two of the most popular techniques for test case generation: Search-Based Software Testing (SBST) and Dynamic Symbolic Execution (DSE). A novel integration of DSE as a step of local improvement in a genetic algorithm results in an adaptive approach, such that the best test generation technique for the problem at hand is favoured, resulting in overall higher code coverage.},
	author = {Pablo, Juan and Fraser, Gordon and Arcuri, Andrea}
}
@article {Simula.simula.1694,
	title = {A Large Scale Evaluation of Automated Unit Test Generation Using EvoSuite},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {24},
	year = {2014},
	month = {12/2014},
	publisher = {ACM},
	abstract = {Research on software testing produces many innovative automated techniques, but because software testing is by necessity incomplete and approximate, any new technique faces the challenge of an empirical assessment. In the past, we have demonstrated scientific advance in automated unit test generation with the EVOSUITE tool by evaluating it on manually selected open source projects or examples that represent a particular problem addressed by the underlying technique. However, demonstrating scientific advance is not necessarily the same as demonstrating practical value: Even if EVOSUITE worked well on the software projects we selected for evaluation, it might not scale up to the complexity of real systems. Ideally, one would use large {\textquotedblleft}real-world{\textquotedblright} software systems to minimize the threats to external validity when evaluating research tools. However, neither choosing such software systems nor applying research prototypes to them are trivial tasks. In this paper we present the results of a large experiment in unit test generation using the EVOSUITE tool on 100 randomly chosen open source projects, the 10 most popular open source projects according to the SourceForge website, 7 industrial projects, and 11 automatically generated software projects. The study confirms that EVOSUITE can achieve good levels of branch coverage (on average 71\% per class) in practice. However, the study also exemplifies how the choice of software systems for an empirical study can influence the results of the experiments, which can serve to inform researchers to make more conscious choices in the selection of software system subjects. Furthermore, our experiments demonstrate how practical limitations interfere with scientific advances: Branch coverage on an unbiased sample is affected by predominant environmental dependencies. The surprisingly large effect of such practical engineering problems in unit testing will hopefully lead to a larger appreciation of work in this area, thus supporting transfer of knowledge from software testing research to practice.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@article {Simula.simula.2268,
	title = {A Memetic Algorithm for Whole Test Suite Generation},
	journal = {Journal of Systems and Software},
	volume = {103},
	year = {2014},
	month = {05/2015},
	pages = {311{\textendash}327},
	publisher = {University of Sheffield, Department of Computer Science},
	abstract = {The generation of unit-level test cases for structural code coverage is a task well-suited to Genetic Algorithms. Method call sequences must be created that construct objects, put them into the right state and then execute uncovered code. However, the generation of primitive values, such as integers and doubles, characters that appear in strings, and arrays of primitive values, are not so straightforward. Often, small local changes are required to drive the value toward the one needed to execute some target structure. However, global searches like Genetic Algorithms tend to make larger changes that are not concentrated on any particular aspect of a test case. In this paper, we extend the Genetic Algorithm behind the\&nbsp;EvoSuite\&nbsp;test generation tool into a Memetic Algorithm, by equipping it with several local search operators. These operators are designed to efficiently optimize primitive values and other aspects of a test suite that allow the search for test cases to function more effectively. We evaluate our operators using a rigorous experimental methodology on over 12,000 Java classes, comprising open source classes of various different kinds, including numerical applications and text processors. Our study shows that increases in branch coverage of up to 53\% are possible for an individual class in practice.},
	author = {Fraser, Gordon and Arcuri, Andrea and McMinn, Phil}
}
@article {Simula.simula.2222,
	title = {1600 Faults in 100 Projects: Automatically Finding Faults While Achieving High Coverage With EvoSuite.},
	journal = {Empirical Software Engineering},
	number = {online},
	year = {2013},
	publisher = {Springer US},
	abstract = {Automated unit test generation techniques traditionally follow one of two goals: Either they try to find violations of automated oracles (e.g., assertions, contracts, undeclared exceptions), or they aim to produce representative test suites (e.g., satisfying branch coverage) such that a developer can manually add test oracles. Search-based testing (SBST) has delivered promising results when it comes to achieving coverage, yet the use in conjunction with automated oracles has hardly been explored, and is generally hampered by the reliance of SBST techniques on (small) sets of explicit testing targets. In this paper we present a search-based approach to handle both objectives at the same time, implemented in the EVOSUITE tool. An empirical study applying EVOSUITE on 100 randomly selected open source software projects (the SF100 corpus) reveals that SBST has the unique advantage of being well suited to perform both traditional goals at the same time - efficiently triggering faults, while producing representative test sets for any chosen coverage criterion. In our study, EVOSUITE detected twice as many failures in terms of undeclared exceptions as a traditional random testing approach, witnessing thousands of real faults in the 100 open source projects. Two out of every five classes with undeclared exceptions have actual faults, but these are buried within many failures that are caused by implicit preconditions. This {\textquotedblleft}noise{\textquotedblright} can be interpreted as either a call for further research in improving automated oracles - or to make tools like EVOSUITE an integral part of software development to enforce clean program interfaces.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.2034,
	title = {Automated Test Generation for Java Generics},
	journal = {Software Quality Days (SWQD)},
	volume = {166},
	year = {2013},
	pages = {185-198},
	publisher = {Springer International Publishing},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Software testing research has resulted in effective white-box test generation techniques that can produce unit test suites achieving high code coverage. However, research prototypes usually only cover basic programming language features, thus inhibiting practical use and evaluation. One such feature are Java{\textquoteright}s generics, which have been present in the language since 2004. In Java, a generic class has type parameters and can be instantiated for different types; for example, a collection can be parameterized with the type of values it contains. To enable test generation tools to cover generics, two simple changes are required to existing approaches: First, the test generator needs to use Java{\textquoteright}s extended reflection API to retrieve the little information that remains after type erasure. Second, a simple static analysis can identify candidate classes for type parameters of generic classes. The presented techniques are implemented and evaluated in the EvoSuite test data generation tool.},
	keywords = {Conference},
	isbn = {978-3-319-03601-4},
	issn = {1865-1348},
	doi = {10.1007/978-3-319-03602-1_12},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1860,
	title = {Does Automated White-Box Test Generation Really Help Software Testers?},
	journal = {ACM International Conference on Software Testing and Analysis (ISSTA)},
	year = {2013},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Automated test generation techniques can efficiently produce test data that systematically cover structural aspects of a program. In the absence of a specification, a common assumption is that these tests relieve a developer of most of the work, as the act of testing is reduced to checking the results of the tests. Although this assumption has persisted for decades, there has been no conclusive evidence to date confirming it. However, the fact that the approach has only seen a limited uptake in industry suggests the contrary, and calls into question its practical usefulness. To investigate this issue, we performed a controlled experiment comparing a total of 49 subjects split between writing tests manually and writing tests with the aid of an automated unit test generation tool, EVOSUITE. We found that, on one hand, tool support leads to clear improvements in commonly applied quality metrics such as code coverage (up to 300\% increase). However, on the other hand, there was no measurable improvement in the number of bugs actually found by developers. Our results not only cast some doubt on how the research community evaluates test generation tools, but also point to improvements and future work necessary before automated test generation tools will be widely adopted by practitioners.},
	author = {Fraser, Gordon and Staats, Matt and McMinn, Phil and Arcuri, Andrea and Padberg, Frank},
	editor = {Harman, Mark}
}
@inproceedings {27063,
	title = {Efficient Mutation Testing using Whole Test Suite Generation},
	journal = {Conference},
	year = {2013},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1550,
	title = {Efficient Mutation Testing Using Whole Test Suite Generation},
	journal = {conference},
	year = {2013},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {By seeding artificial faults (mutants), mutation testing can tell us how good existing tests are, and it can help to direct test generation efforts. However, mutation based test generation is hampered because there usually are simply too many mutants, and too many of these mutants are either trivially killed or equivalent. Any effort spent on test generation for equivalent mutants is per definition wasted, and misdirects resources from where they could be put to better use - for example to produce tests revealing more non-equivalent mutants. To overcome this problem, our search-based EVOSUITE test generation tool integrates two optimizations: First, we avoid redundant test executions on mutants by monitoring state infection conditions, and second we use whole test suite generation to optimize test suites towards killing the highest number of mutants, rather than selecting individual mutants. These optimizations allowed us to perform one of the largest empirical studies on mutation testing to date, where we applied EVOSUITE to a random sample of 100 open source projects consisting of a total of 8,963 classes, leading to a total of 1,380,302 mutants. The experiment not only demonstrates that our approach scales well, but it also quantifies the relations between weak and strong mutation testing as well as branch coverage, and points out current limitations.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1918,
	title = {EvoSuite at the SBST 2013 Tool Competition},
	journal = {SBST workshop},
	year = {2013},
	publisher = {IEEE},
	type = {Workshop},
	address = {Washington, DC, USA},
	abstract = {EVOSUITE is a mature research prototype that automatically generates unit tests for Java code. This paper summarizes the results and experiences in participating, at the unit testing competition held at SBST 2013, where EVOSUITE ranked first with a score of 156:95.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1552,
	title = {EvoSuite: on the Challenges of Test Case Generation in the Real World},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Luembourg},
	abstract = {Test case generation is an important but tedious task, such that researchers have devised many different prototypes that aim to automate it. As these are research prototypes, they are usually only evaluated on a few hand-selected case studies, such that despite great results there remains the question of usability in the {\textquotedblleft}real world{\textquotedblright}. EVOSUITE is such a research prototype, which automatically generates unit test suites for classes written in the Java programming language. In our ongoing endeavour to achieve real-world usability, we recently passed the milestone success of applying EVOSUITE on hundred projects randomly selected from the SourceForge open source platform. This paper discusses the technical challenges that a testing tool like EVOSUITE needs to address when handling Java classes coming from real-world open source projects, and when producing JUnit test suites intended for real users.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@article {Simula.simula.744,
	title = {Handling Test Length Bloat},
	journal = {Software Testing, Verification and Reliability},
	number = {23},
	year = {2013},
	publisher = {Wiley},
	abstract = {The length of test cases is a little investigated topic in search-based test generation for object-oriented software, where test cases are sequences of method calls. While intuitively longer tests can achieve higher overall code coverage, there is always the threat of \emph{bloat} - a complex phenomenon in evolutionary computation, where the length abnormally grows over time. In this paper, we show that bloat indeed also occurs in the context of test generation for object-oriented software. We present different techniques to overcome the problem of length bloat, and evaluate all possible combinations of these techniques using different search lengths. Experiments on a set of difficult search targets selected from several open source and industrial projects show that controlling bloat in the right way can significantly improve the search performance.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1920,
	title = {Improving Search-Based Test Suite Generation With Dynamic Symbolic Execution},
	journal = {IEEE International Symposium on Software Reliability Engineering},
	year = {2013},
	publisher = {IEEE},
	type = {Conference},
	address = {Pasadena, CA},
	abstract = {Search-based testing can automatically generate unit test suites for object oriented code, but may struggle to generate specific values necessary to cover difficult parts of the code. Dynamic symbolic execution (DSE) efficiently generates such specific values, but may struggle with complex datatypes, in particular those that require sequences of calls for construction. The solution to these problems lies in a hybrid approach that integrates the best of both worlds, but such an integration needs to adapt to the problem at hand to avoid that higher coverage in a few corner cases comes at the price of lower coverage in the general case. We have extended the Genetic Algorithm (GA) in the EVOSUITE unit test generator to integrate DSE in an adaptive approach where feedback from the search determines when a problem is suitable for DSE, and the DSE exploration in turn is driven by a search-based constraint solver that can easily deal with mixed constraints of numerical and string datatypes. In experiments on a set of difficult classes our adaptive hybrid approach achieved an increase in code coverage of up to 63\% (11\% on average); experiments on the SF100 corpus of roughly 9,000 open source classes confirm that the improvement is of practical value, and a comparison with a DSE tool on the Roops set of benchmark classes shows that the hybrid approach improves over both its constituent techniques, GA and DSE.},
	author = {Juan Pablo {Galeotti} and Fraser, Gordon and Arcuri, Andrea},
	editor = {Carol Smidts Myr {Cohen}}
}
@article {Simula.simula.1164,
	title = {Parameter Tuning Or Default Values? an Empirical Investigation in Search-Based Software Engineering},
	journal = {Empirical Software Engineering},
	volume = {18},
	number = {3},
	year = {2013},
	publisher = {Springer US},
	abstract = {Many software engineering problems have been addressed with search algorithms. Search algorithms usually depend on several parameters (e.g., population size and crossover rate in genetic algorithms), and the choice of these parameters can have an impact on the performance of the algorithm. It has been formally proven in the No-Free-Lunch theorem that it is impossible to tune a search algorithm such that it will have optimal settings for all possible problems. So, how to properly set the parameters of a search algorithm for a given software engineering problem? In this paper, we carry out the largest empirical analysis so far on parameter tuning in search-based software engineering. More than one million experiments were carried out and statistically analyzed in the context of test data generation for object-oriented software using the EVOSUITE tool. Results show that tuning does indeed have impact on the performance of a search algorithm. But, at least in the context of test data generation, it does not seem easy to find good settings that outperform by a large extent the {\textquotedblleft}default{\textquotedblright} values suggested in the literature. This has very practical value for both researchers (e.g., when different techniques are compared) and practitioners. Using {\textquotedblleft}default{\textquotedblright} values is a reasonable and justified choice, whereas parameter tuning is a long and expensive process that might or might not pay off in the end.},
	author = {Arcuri, Andrea and Fraser, Gordon}
}
@article {Simula.simula.2044,
	title = {Search-Based Testing: the Swiss Army Knife of Test Generation},
	journal = {journal},
	year = {2013},
	publisher = {.},
	abstract = {Why write tests if one can generate them? This is the promise of searchbased testing, generating and evolving inputs that systematically explore program behavior. The approach can be easily adapted to various kinds of inputs as well as testing goals - whether it is maximizing coverage, locating defects, or reproducing failures in the field.},
	author = {Arcuri, Andrea and Fraser, Gordon and Zeller, Andreas}
}
@inproceedings {Simula.simula.1928,
	title = {Test Suite Generation With Memetic Algorithms},
	journal = {ACM Genetic and Evolutionary Computation Conference (GECCO)},
	year = {2013},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Genetic Algorithms have been successfully applied to the generation of unit tests for classes, and are well suited to create complex objects through sequences of method calls. However, because the neighborhood in the search space for method sequences is huge, even supposedly simple optimiza- tions on primitive variables (e.g., numbers and strings) can be ine ective or unsuccessful. To overcome this problem, we extend the global search applied in the EvoSuite test gen- eration tool with local search on the individual statements of method sequences. In contrast to previous work on local search, we also consider complex datatypes including strings and arrays. A rigorous experimental methodology has been applied to properly evaluate these new local search opera- tors. In our experiments on a set of open source classes of dif- ferent kinds (e.g., numerical applications and text process- ing), the resulting test data generation technique increased branch coverage by up to 32\% on average over the normal Genetic Algorithm.},
	author = {Fraser, Gordon and Arcuri, Andrea and McMinn, Phil},
	editor = {Vanneschi, Leonardo}
}
@article {Simula.simula.718,
	title = {Whole Test Suite Generation},
	journal = {IEEE Transactions on Software Engineering},
	volume = {39},
	number = {2},
	year = {2013},
	publisher = { IEEE},
	abstract = {Recent advances in software testing allow automatic derivation of tests that reach almost any desired point in the source code. There is, however, a fundamental problem with the general idea of targeting one distinct test coverage goal at a time: Coverage goals are neither independent of each other, nor is test generation for any particular coverage goal guaranteed to succeed. We present EVOSUITE, a search-based approach that optimizes whole test suites towards satisfying a coverage criterion, rather than generating distinct test cases directed towards distinct coverage goals. Evaluated on open source libraries and an industrial case study for a total of 1,752 classes, we show that EVOSUITE achieved up to 188 times the coverage of a traditional approach targeting single branches, with up to 62\% smaller test suites.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1474,
	title = {Combining Search-Based and Adaptive Random Testing Strategies for Environment Model-Based Testing of Real-Time Embedded Systems},
	journal = {Symposium on Search Based Software Engineering (SSBSE)},
	year = {2012},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Effective system testing of real-time embedded systems (RTES) requires a fully automated approach. One such black-box system testing approach is to use environment models to automatically generate test cases and test oracles along with an environment simulator to enable early testing of RTES. In this paper, we propose a hybrid strategy, which combines (1+1) Evolutionary Algorithm (EA) and Adaptive Random Testing (ART), to improve the overall performance of system testing that is obtained when using each single strategy in isolation. An empirical study is carried out on a number of artificial problems and one industrial case study. The novel strategy shows significant overall improvement in terms of fault detection compared to individual performances of both (1+1) EA and ART.},
	keywords = {Conference},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude},
	editor = {Fraser, Gordon and Susi, Angelo and Jerffeson Teixeira {de Souza}}
}
@inproceedings {Simula.simula.832,
	title = {The Seed Is Strong: Seeding Strategies in Search-Based Software Testing},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2012},
	publisher = {IEEE},
	type = {Conference},
	address = {Montreal, QC},
	abstract = {Search-based techniques have been proved useful for the task of generating tests, for example in the case of object-oriented software. But, as for any meta-heuristic search, the efficiency is heavily dependent on many different factors; seeding is one such factor that may strongly influence this efficiency. In this paper, we introduce and evaluate different strategies to seed the initial population as well as to seed values introduced during the search when generating tests for object-oriented code. We report the results of a large empirical analysis carried out on 20 Java projects (for a total of 1,752 public classes). Our experiments show with strong statistical confidence that, even for a testing tool that is already able to achieve high coverage, the use of appropriate seeding strategies can further improve performance.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.784,
	title = {Sound Empirical Evidence in Software Testing},
	journal = {ACM/IEEE International Conference on Software Engineering (ICSE)},
	year = {2012},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {Zurich},
	abstract = {Several promising techniques have been proposed to automate different tasks in software testing, such as test data generation for object-oriented software. However, reported studies in the literature only show the feasibility of the proposed techniques, because the choice of the employed artifacts in the case studies (e.g., software applications) is usually done in a nonsystematic way. The chosen case study might be biased, and so it might not be a valid representative of the addressed type of software (e.g., internet applications and embedded systems). The common trend seems to be to accept this fact and get over it by simply discussing it in a threats to validity section. In this paper, we evaluate search-based software testing (in particular the EVOSUITE tool) when applied to test data generation for open source projects. To achieve sound empirical results, we randomly selected 100 Java projects from SourceForge, which is the most popular open source repository (more than 300,000 projects with more than two million registered users). The resulting case study not only is very large (8784 public classes for a total of 291,639 bytecode level branches), but more importantly it is statistically sound and representative for open source projects. Results show that while high coverage on commonly used types of classes is achievable, in practice environmental dependencies prohibit such high coverage, which clearly points out essential future research directions. To support this future research, our SF100 case study can serve as a much needed benchmark for test generation.},
	author = {Fraser, Gordon and Arcuri, Andrea},
	editor = {Martin Glinz Gail {Pezz{\`e}}}
}
@inproceedings {Simula.simula.1146,
	title = {Test Suite Generation With Memetic Algorithms and Dynamic Symbolic Execution},
	journal = {conference},
	year = {2012},
	publisher = {.},
	type = {1},
	address = {.},
	abstract = {Genetic Algorithms have been successfully applied to generate unit tests for classes, and are well suited to create complex objects through sequences of method calls. However, because the neighborhood in the search space for method sequences is huge, even supposedly simple optimizations on primitive variables (e.g., numbers and strings) can be ineffective or unsuccessful. To overcome this problem, we extend the global search applied in the EVOSUITE test generation tool with local search (i.e., Memetic Algorithms, MA) on primitive values contained in method sequences. In contrast to previous work on local search, we also consider string datatypes as primitives, and furthermore study search on path constraints in terms of Dynamic Symbolic Execution (DSE) to address this problem. A rigorous experimental methodology has been applied to properly evaluate the combination of MA with DSE. In our experiments on a set of open source classes of different kinds (e.g., numerical applications and text processing), the resulting MA+DSE technique increased branch coverage by 8\% on average over the normal Genetic Algorithm.},
	author = {Fraser, Gordon and Rusev, Kaloyan and Arcuri, Andrea}
}
@inproceedings {Simula.simula.1510,
	title = {Using Automated Oracles in Search-Based Testing},
	journal = {conference},
	year = {2012},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {Automated unit test generation tech- niques traditionally follow one of two goals: Either they try to find violations of automated oracles (e.g., asser- tions, contracts, undeclared exceptions), or they aim to produce representative test suites (e.g., satisfying branch coverage) such that a developer can manually add test oracles. Search-based testing (SBST) has de- livered promising results when it comes to achieving coverage, yet the use in conjunction with automated oracles has hardly been explored, and is generally ham- pered by the reliance of SBST techniques on (small) sets of explicit testing targets. In this paper we em- pirically investigate the applicability of SBST to the context of automated oracles using a whole test suite optimization approach, and present improvements in terms of testability transformations. Experiments on 100 randomly selected open source software projects (the SF100 corpus) reveal that SBST has the unique ad- vantage of being well suited to perform both traditional goals at the same time efficiently triggering faults, while producing representative test sets for any chosen coverage criterion. In our experiments, our EvoSuite prototype detected on average twice as many failures as a traditional random testing approach.},
	author = {Fraser, Gordon and Arcuri, Andrea}
}
@inproceedings {Simula.simula.404,
	title = {Evolutionary Generation of Whole Test Suites},
	journal = {International Conference On Quality Software (QSIC)},
	year = {2011},
	pages = {31-40},
	publisher = {IEEE},
	type = {Conference},
	abstract = {Recent advances in software testing allow automatic derivation of tests that reach almost any desired point in the source code. There is, however, a fundamental problem with the general idea of targeting one distinct test coverage goal at a time: Coverage goals are neither independent of each other, nor is test generation for any particular coverage goal guaranteed to succeed. We present EVOSUITE, a search-based approach that optimizes whole test suites towards satisfying a coverage criterion, rather than generating distinct test cases directed towards distinct coverage goals. Evaluated on five open source libraries and an industrial case study, we show that EVOSUITE achieves up to 18 times the coverage of a traditional approach targeting single branches, with up to 44\% smaller test suites.},
	isbn = {978-1-4577-0754-4},
	author = {Fraser, Gordon and Arcuri, Andrea},
	editor = {Robert M. {Hierons} and Mercedes G. {Merayo}}
}
@inproceedings {Simula.simula.692,
	title = {EvoSuite: Automatic Test Suite Generation for Object-Oriented Software.},
	journal = {ACM Symposium on the Foundations of Software Engineering (FSE)},
	year = {2011},
	pages = {416-419},
	publisher = {ACM},
	type = {Conference},
	abstract = {To find defects in software, one needs test cases that execute the software systematically, and oracles that assess the correctness of the observed behavior when running these test cases. This paper presents EvoSuite, a tool that automatically generates test cases with assertions for classes written in Java code. To achieve this, EvoSuite applies a novel hybrid approach that generates and optimizes whole test suites towards satisfying a coverage criterion. For the produced test suites, EvoSuite suggests possible oracles by adding small and eff ective sets of assertions that concisely summarize the current behavior; these assertions allow the developer to detect deviations from expected behavior, and to capture the current behavior in order to protect against future defects breaking this behavior.},
	isbn = {978-1-4503-0443-6},
	author = {Fraser, Gordon and Arcuri, Andrea},
	editor = {Zeller, Andreas}
}
@inproceedings {Simula.simula.90,
	title = {It Is Not the Length That Matters, It Is How You Control It},
	journal = {IEEE International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2011},
	pages = {150-159},
	publisher = {IEEE},
	type = {1},
	abstract = {The length of test cases is a little investigated topic in search-based test generation for object oriented software, where test cases are sequences of method calls. While intuitively longer tests can achieve higher overall code coverage, there is always the threat of bloat - a complex phenomenon in evolutionary computation, where the length abnormally grows over time. In this paper, we show that bloat indeed also occurs in the context of test generation for object oriented software. We present different techniques to overcome the problem of length bloat, and evaluate all possible combinations of these techniques using different search lengths. Experiments on a set of difficult search targets selected from several open source and industrial projects show that the important choice in search-based testing is not the length of test cases, but how to make sure that this length does not become bloated.},
	isbn = {978-1-61284-174-8},
	author = {Fraser, Gordon and Arcuri, Andrea},
	editor = {Korel B. {Harman}}
}
@inproceedings {Simula.simula.678,
	title = {On Parameter Tuning in Search Based Software Engineering},
	journal = {nternational Symposium on Search Based Software Engineering (SSBSE)},
	year = {2011},
	pages = {33-47},
	publisher = {Springer},
	type = {Conference},
	abstract = {When applying search-based software engineering (SBSE) techniques one is confronted with a multitude of different parameters that need to be chosen: Which population size for a genetic algorithm? Which selection mechanism to use? What settings to use for dozens of other parameters? This problem not only troubles users who want to apply SBSE tools in practice, but also researchers performing experimentation - how to compare algorithms that can have different parameter settings? To shed light on the problem of parameters, we performed the largest empirical analysis on parameter tuning in SBSE to date, collecting and statistically analysing data from more than a million experiments. As case study, we chose test data generation, one of the most popular problems in SBSE. Our data confirm that tuning does have a critical impact on algorithmic performance, and over-fitting of parameter tuning is a dire threat to external validity of empirical analyses in SBSE. Based on this large empirical evidence, we give guidelines on how to handle parameter tuning.},
	isbn = {10.1007/978-3-642-23716-4},
	author = {Arcuri, Andrea and Fraser, Gordon},
	editor = {Myra B. {Cohen} and Mel {\'O} {Cinn{\'e}ide}}
}
@inproceedings {Simula.simula.180,
	title = {Testing Container Classes: Random Or Systematic?},
	journal = {Fundamental Approaches to Software Engineering (FASE)},
	year = {2011},
	pages = {262-277},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Container classes such as lists, sets, or maps are elementary data structures common to many programming languages. Since they are a part of standard libraries, they are important to test, which led to research on advanced testing techniques targeting such containers and research on comparing testing techniques using such containers. These techniques, however, have not been thoroughly compared to simpler approaches such as random testing. To remedy this situation, we present the results of a larger case study in which we compare random testing with shape abstraction, a systematic approach that showed the best results in a previous study. Our experiments show that random testing is very effective for testing these containers, which raises the question whether containers are well suited as a benchmark for comparing advanced testing techniques.},
	isbn = {10.1007/978-3-642-19811-3\_19},
	author = {Sharma, Rohan and Gligoric, Milos and Arcuri, Andrea and Fraser, Gordon and Marinov, Darko},
	editor = {Giannakopoulou, Dimitra and Orejas, Fernando}
}
