@inproceedings {43377,
	title = {Automated Test Generation for Medical Rules Web Services: A Case Study at the Cancer Registry of Norway},
	journal = {31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE 2023)},
	year = {2023},
	publisher = {ACM},
	doi = {10.1145/3611643.3613882},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat and Schwitalla, Thomas and Jan F. {Nyg{\r a}rd}}
}
@inproceedings {43324,
	title = {Challenges of Testing an Evolving Cancer Registration Support System in Practice},
	journal = {45th IEEE/ACM International Conference on Software Engineering: Companion Proceedings (ICSE-Companion 2023)},
	year = {2023},
	publisher = {IEEE},
	doi = {10.1109/ICSE-Companion58688.2023.00102},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat and Schwitalla, Thomas and Jan F. {Nyg{\r a}rd}}
}
@article {laaber2022mosmbprio,
	title = {Multi-Objective Search-Based Software Microbenchmark Prioritization},
	journal = {arXiv},
	year = {2022},
	publisher = {arXiv},
	abstract = {Ensuring that software performance does not degrade after a code change is paramount. A potential solution, particularly for libraries and frameworks, is regularly executing software microbenchmarks, a performance testing technique similar to (functional) unit tests. This often becomes infeasible due to the extensive runtimes of microbenchmark suites, however. To address that challenge, research has investigated regression testing techniques, such as test case prioritization (TCP), which reorder the execution within a microbenchmark suite to detect larger performance changes sooner. Such techniques are either designed for unit tests and perform sub-par on microbenchmarks or require complex performance models, reducing their potential application drastically. In this paper, we propose a search-based technique based on multi-objective evolutionary algorithms (MOEAs) to improve the current state of microbenchmark prioritization. The technique utilizes three objectives, i.e., coverage to maximize, coverage overlap to minimize, and historical performance change detection to maximize. We find that our technique improves over the best coverage-based, greedy baselines in terms of average percentage of fault-detection on performance (APFD-P) and Top-3 effectiveness by 26 percentage points (pp) and 43 pp (for Additional) and 17 pp and 32 pp (for Total) to 0.77 and 0.24, respectively. Employing the Indicator-Based Evolutionary Algorithm (IBEA) as MOEA leads to the best effectiveness among six MOEAs. Finally, the technique{\textquoteright}s runtime overhead is acceptable at 19\% of the overall benchmark suite runtime, if we consider the enormous runtimes often spanning multiple hours. The added overhead compared to the greedy baselines is miniscule at 1\%.These results mark a step forward for universally applicable performance regression testing techniques.},
	keywords = {JMH, Multi-objective optimization, Performance Testing, Regression testing, Search-Based Software Engineering, software microbenchmarking, Test Case Prioritization},
	doi = {10.48550/ARXIV.2211.13525},
	url = {https://arxiv.org/abs/2211.13525},
	author = {Laaber, Christoph and Yue, Tao and Ali, Shaukat}
}
@misc {42838,
	title = {Predicting Unstable Software Benchmarks using Static Source Code Features},
	howpublished = {International Conference on Software Engineering (ICSE)},
	year = {2022},
	type = {Journal-First Paper Presentation},
	abstract = {Software benchmarks are only as good as the performance measurements they yield. Unstable benchmarks show high variability among repeated measurements, which causes uncertainty about the {\textquotedblleft}true{\textquotedblright} performance of the measured software unit and complicates reliable change assessment. Conversely, if multiple repeated measurements have low variability, i.e., the distribution of the measurement results is narrow, a benchmark is considered stable. However, if a benchmark is stable or unstable only becomes evident after it has been executed, and its results are available.In this paper, we introduce a machine-learning-based approach to predict a benchmark{\textquoteright}s stability without executing it. Our approach statically extracts 58 source code features, for both benchmark code and code called by a benchmark. It parses the abstract syntax trees (ASTs) of all functions, counts the occurrences of each feature for each function, computes the reachable functions from a benchmark with static call graphs (CGs), sums up the feature occurrences for each benchmark, and feeds the features into a binary classifier. Inspired by previous software performance research, the employed features act as proxies for performance variability (and consequently benchmark stability) and are related to: (1) meta information, e.g., lines of code (LOC); (2) programming language elements, e.g., conditionals or loops; (3) potentially performance-impacting standard library calls, e.g., file and network input/output (I/O).To assess our approach{\textquoteright}s effectiveness, we perform a large-scale experiment on 4,461 Go performance benchmarks coming from 230 open-source software (OSS) projects.First, we assess the prediction performance of our machine learning models using 11 binary classification algorithms. We find that Random Forest performs best with good prediction performance from 0.79 to 0.90, and 0.43 to 0.68, in terms of Area Under the Curve (AUC) and Matthews Correlation Coefficient (MCC), respectively.Second, we carry out four sensitivity analyses to investigate the impact on the prediction performance, if the model is trained (1) with different variability thresholds that consider a benchmark as stable or unstable, (2) on benchmark executions with a varying number of measurement repetitions, (3) after applying specific pre-processing steps to remove co-linear and multi-colinear features as well as perform class-rebalancing on the training set, and (4) for different variability measures used as the dependent variable. We find that our model performs best when trained on larger thresholds (10\%) and executions from more repetitions (30). While feature pre-processing does not have an impact across all studied algorithms, removing co-linear and multi-co-linear improves Random Forrest{\textquoteright}s prediction performance by 0.023 MCC and 0.005 AUC. The model shows varying prediction performance, depending on which variability measures is used as dependent variable. However, it performs well across all three studied measures.Third, we perform feature importance analyses for individual features and feature categories. We find that 7 features related to meta-information, slice usage, nested loops, and synchronization application programming interfaces (APIs) are individually important for good predictions; and that the combination of all features of the called source code is paramount for our model, while the combination of features of the benchmark itself is less important.Our results show that although benchmark stability is affected by more than just the source code, we can effectively utilize machine learning models to predict whether a benchmark will be stable or not ahead of execution. This enables spending precious testing time on reliable benchmarks, supporting developers to identify unstable benchmarks during development, allowing unstable benchmarks to be repeated more often, estimating stability in scenarios where repeated benchmark execution is infeasible or impossible, and warning developers if new benchmarks or existing benchmarks executed in new environments will be unstable.},
	author = {Laaber, Christoph}
}
@article {grambow2022,
	title = {Using Microbenchmark Suites to Detect Application Performance Changes},
	journal = {IEEE Transactions on Cloud Computing},
	year = {2022},
	month = {10/2022},
	pages = {1-18},
	publisher = { IEEE},
	abstract = {Software performance changes are costly and often hard to detect pre-release. Similar to software testing frameworks, either application benchmarks or microbenchmarks can be integrated into quality assurance pipelines to detect performance changes before releasing a new application version. Unfortunately, extensive benchmarking studies usually take several hours which is problematic when examining dozens of daily code changes in detail; hence, trade-offs have to be made. Optimized microbenchmark suites, which only include a small subset of the full suite, are a potential solution for this problem, given that they still reliably detect the majority of the application performance changes such as an increased request latency. It is, however, unclear whether microbenchmarks and application benchmarks detect the same performance problems and one can be a proxy for the other. In this paper, we explore whether microbenchmark suites can detect the same application performance changes as an application benchmark. For this, we run extensive benchmark experiments with both the complete and the optimized microbenchmark suites of two time-series database systems, i.e., InfluxDB and VictoriaMetrics, and compare their results to the results of corresponding application benchmarks. We do this for 70 and 110 commits, respectively. Our results show that it is not trivial to detect application performance changes using an optimized microbenchmark suite. The detection (i) is only possible if the optimized microbenchmark suite covers all application-relevant code sections, (ii) is prone to false alarms, and (iii) cannot precisely quantify the impact on application performance. For certain software projects, an optimized microbenchmark suite can, thus, provide fast performance feedback to developers (e.g., as part of a local build process), help estimating the impact of code changes on application performance, and support a detailed analysis while a daily application benchmark detects major performance problems. Thus, although a regular application benchmark cannot be substituted for both studied systems, our results motivate further studies to validate and optimize microbenchmark suites.},
	keywords = {Benchmarking, Microbenchmarks, Performance Change Detection, Performance Testing, Regression Detection},
	issn = {2168-7161},
	doi = {10.1109/TCC.2022.3217947},
	url = {https://ieeexplore.ieee.org/abstract/document/9931939},
	author = {Grambow, Martin and Kovalev, Denis and Laaber, Christoph and Leitner, Philipp and Bermbach, David}
}
@misc {42839,
	title = {Deliberate Microbenchmarking of Software Systems},
	howpublished = {Cancer Registry of Norway},
	year = {2021},
	type = {Seminar Talk},
	abstract = {In this talk, I will summarize the findings of my PhD thesis:Software performance faults have severe consequences for users, developers, and companies. One way to unveil performance faults before they manifest in production is performance testing, which ought to be done on every new version of the software, ideally on every commit. However, performance testing faces multiple challenges that inhibit it from being applied early in the development process, on every new commit, and in an automated fashion.In this dissertation, we investigate three challenges of software microbenchmarks, a performance testing technique on unit granularity which is predominantly used for libraries and frameworks. The studied challenges affect the quality aspects (1) runtime, (2) result variability, and (3) performance change detection of microbenchmark executions. The objective is to understand the extent of these challenges in real-world software and to find solutions to address these.To investigate the challenges{\textquoteright} extent, we perform a series of experiments and analyses. We execute benchmarks in bare-metal as well as multiple cloud environments and conduct a large-scale mining study on benchmark configurations. The results show that all three challenges are common: (1) benchmark suite runtimes are often longer than 3 hours; (2) result variability can be extensive, in some cases up to 100\%; and (3) benchmarks often only reliably detect large performance changes of 60\% or more.To address the challenges, we devise targeted solutions as well as adapt well-known techniques from other domains for software microbenchmarks: (1) a solution that dynamically stops benchmark executions based on statistics to reduce runtime while maintaining low result variability; (2) a solution to identify unstable benchmarks that does not require execution, based on statically-computable source code features and machine learning algorithms; (3) traditional test case prioritization (TCP) techniques to execute benchmarks earlier that detect larger performance changes; and (4) specific execution strategies to detect small performance changes reliably even when executed in unreliable cloud environments.We experimentally evaluate the solutions and techniques on real-world benchmarks and find that they effectively deal with the three challenges. (1) Dynamic reconfiguration enables to drastically reduce runtime by between 48.4\% and 86.0\% without changing the results of 78.8\% to 87.6\% of the benchmarks, depending on the project and statistic used. (2) The instability prediction model allows to effectively identify unstable benchmarks when relying on random forest classifiers, having a prediction performance between 0.79 and 0.90 area under the receiver operating characteristic curve (AUC). (3) TCP applied to benchmarks is effective and efficient, with APFD-P values for the best technique ranging from 0.54 to 0.71 and a computational overhead of 11\%. (4) Batch testing, i.e., executing the benchmarks of two versions on the same instances interleaved and repeated as well as repeated across instances, enables to reliably detect performance changes of 10\% or less, even when using unreliable cloud infrastructure as execution environment.Overall, this dissertation shows that real-world software microbenchmarks are considerably affected by all three challenges (1) runtime, (2) result variability, and (3) performance change detection; however, deliberate planning and execution strategies effectively reduce their impact.},
	author = {Laaber, Christoph}
}
@misc {28162,
	title = {Variability of Microbenchmark Results and How to Deal with It},
	howpublished = {Chalmers, Software Engineering Division, University of Gothenburg, Sweden},
	year = {2021},
	author = {Laaber, Christoph}
}
