@techreport {26318,
	title = {Technical Report: A Semi-Structured Interview on Test Data Need in The Integration Testing with the Norwegian National Registration.},
	year = {2018},
	note = {This report is a work in progress and this is an initial version.\&nbsp;},
	publisher = {Simula Research Laboratory},
	author = {Tan, Chao and Behjati, Razieh and Arisholm, Erik}
}
@inproceedings {Simula.simula.714,
	title = {Industrial Experiences With Automated Regression Testing of a Legacy Database Application},
	journal = {IEEE Int. Conference on Software Maintenance},
	year = {2011},
	pages = {362-371},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {This paper presents a practical approach and tool (DART) for functional black-box regression testing of complex legacy database applications. Such applications are important to many organizations, but are often difficult to change and consequently prone to regression faults during maintenance. They also tend to be built without particular considerations for testability and can be hard to control and observe. We have therefore devised a practical solution for functional regression testing that captures the changes in database state (due to data manipulations) during the execution of a system under test. The differences in changed database states between consecutive executions of the system under test, on different system versions, can help identify potential regression faults. In order to make the regression test approach scalable for large, complex database applications, classification tree models are used to prioritize test cases. The test case prioritization can be applied to reduce test execution costs and analysis effort. We report on how DART was applied and evaluated on business critical batch jobs in a legacy database application in an industrial setting, namely the Norwegian Tax Accounting System (SOFIE) at the Norwegian Tax Department (NTD). DART has shown promising fault detection capabilities and cost-effectiveness and has contributed to identify many critical regression faults for the past eight releases of SOFIE.},
	isbn = {978-1-4577-0663-9},
	author = {Rogstad, Erik and Arisholm, Erik and Briand, Lionel and Dalberg, Ronny and Rynning, Marianne},
	editor = {James R. Cordy A {Marcus} and Tonella, Paolo}
}
@inbook {Simula.SE.643,
	title = {A Method to Measure Productivity Trends During Software Evolution},
	booktitle = {CCIS - Evaluation of Novel Approaches to Software Engineering},
	series = {Communications in Computer and Information Science},
	volume = {69},
	year = {2010},
	pages = {147-162},
	publisher = {Springer},
	organization = {Springer},
	chapter = {2},
	abstract = {Better measures of productivity are needed to support software process improvements. We propose and evaluate indicators of productivity trends that are based on the premise that productivity is closely related to the effort required to complete change tasks. Three indicators use change management data, while a fourth compares effort estimates of benchmarking tasks. We evaluated the indicators using data from 18 months of evolution in two commercial software projects. The productivity trend in the two projects had opposite directions according to the indicators. The evaluation showed that productivity trends can be quantified with little measurement overhead. We expect the methodology to be a step towards making quantitative self-assessment practices feasible even in low ceremony projects.},
	isbn = {978-3-642-14819-4},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik},
	editor = {Leszek A. {Maciaszek} and Gonz{\textasciiacute}lez-P{\'e}rez, C{\'e}sar and Jablonski, Stefan}
}
@techreport {Simula.approve.2,
	title = {Model Transformations As a Strategy to Automate Model-Based Testing - a Tool and Industrial Case Studies},
	number = {2010-01},
	year = {2010},
	month = {March},
	publisher = {Simula Research Laboratory},
	abstract = {In recent years, Model-Based Testing (MBT) has attracted an increasingly wide interest from industry and academia. The beneficial use of MBT, however, requires tools that not only automate the testing process, but that also rely in an extensible and configurable architecture that make them adaptable to various contexts of application. Though a number of tools have been developed to support MBT, this technical report introduces a new approach for designing and developing MBT tools that is based on model transformation technology. We report on the experimental development of a novel MBT tool, TRansformation-based tool for Uml-baSed Testing (TRUST), which software architecture and implementation strategy supports configurable and extensible features such as input models, test models, coverage criteria, test data generation strategies, and test script languages. Based on two industrial case studies, we demonstrate the configurability and extensibility of TRUST. We also investigate the challenges and likely cost savings when compared to manual test generation.},
	author = {Ali, Shaukat and Hemmati, Hadi and Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@article {Simula.SE.313,
	title = {A Systematic and Comprehensive Investigation of Methods to Build and Evaluate Fault Prediction Models},
	journal = {Journal of Systems and Software},
	volume = {83},
	number = {1},
	year = {2010},
	month = {January},
	pages = {2-17},
	abstract = {This paper describes a study performed in an industrial setting that attempts to build predictive models to identify parts of a Java system with a high fault probability. The system under consideration is constantly evolving as several releases a year are shipped to customers. Developers usually have limited resources for their testing and would like to devote extra resources to faulty system parts. The main research focus of this paper is to systematically assess three aspects on how to build and evaluate fault-proneness models in the context of this large Java legacy system development project: (1) compare many data mining and machine learning techniques to build fault-proneness models, (2) assess the impact of using different metric sets entailing different data collection costs, such as source code structural measures and historic change/fault (process) measures, and (3) compare several alternative ways of assessing the performance of the models, in terms of (i) confusion matrix criteria such as accuracy and precision/recall, (ii) ranking ability, using the receiver operating characteristic area (ROC), and (iii) our proposed cost-effectiveness measure (CE). The results of the study indicate that the choice of fault-proneness modeling technique has limited impact on the resulting classification accuracy or cost-effectiveness. There is however large differences between the individual metric sets in terms of cost-effectiveness, and although the process measures are among the most expensive ones to collect, including them as candidate measures significantly improves the prediction models compared with models that only include structural measures and/or their deltas across releases - both in terms of ROC area and cost-effectiveness. Further, we observe that what is considered the best model is highly dependent on the criteria that are used to evaluate and compare the models. The regular confusion matrix criteria, although popular, are not clearly related to what we consider to be a crucial aspect, namely the cost-effectiveness of using fault-proneness prediction models to focus verification effort where it is the most needed.},
	author = {Arisholm, Erik and Briand, Lionel and Eivind B. {Johannessen}}
}
@article {Simula.SE.301,
	title = {Understanding the Cost of Software Change: a Quantitative and Qualitative Investigation of Change Effort During Evolution of Two Software Systems},
	journal = {Empirical Software Engineering},
	volume = {15},
	number = {2},
	year = {2010},
	month = {April},
	pages = {166-203},
	abstract = {Making changes to software systems can prove costly and it remains a challenge to understand the factors that affect the costs of software evolution. This study sought to identify such factors by investigating the effort expended by developers to perform 336 change tasks in two different software organizations. We quantitatively analyzed data from version control systems and change trackers to identify factors that correlated with change effort. In-depth interviews with the developers about a subset of the change tasks further refined the analysis. Two central quantitative results found that volatility of requirements and dispersion of changed code consistently correlated with change effort. The analysis of the qualitative interviews pointed to two important, underlying cost drivers: Difficulties in anticipating side effects of changes and difficulties in comprehending dispersed code. This study demonstrates a novel method for combining qualitative and quantitative analysis to assess cost drivers of software evolution. Given our findings, we propose improvements to design practices and development tools to reduce the costs.},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik}
}
@inproceedings {Simula.SE.323,
	title = {Are We More Productive Now? Analyzing Change Tasks to Assess Productivity Trends During Software Evolution},
	journal = {Evaluation of Novel Approaches to Software Engineering (ENASE)},
	year = {2009},
	publisher = {Elsevier},
	type = {Conference},
	abstract = {Organizations that maintain and evolve software would benefit from measuring productivity in an easy and reliable way. This could allow them to determine if new or improved practices are needed, and to evaluate improvement efforts. We propose and evaluate indicators of productivity trends that are based on the premise that productivity during software evolution is closely related to the effort required to complete change tasks. Three indicators use data about change tasks from change management systems, while a fourth compares effort estimates of benchmarking tasks. We evaluated the indicators using data from 18 months of evolution in two commercial software projects. The productivity trend in the two projects had opposite directions according to the indicators. Possible explanations were that one system became easier to maintain due to a large refactoring effort, while developers in the other project experienced less time pressure due to reduced use of fixed-price contracts. The evaluation of the indicators showed that productivity trends can be quantified with little measurement overhead. The methodology is a step towards making quantitative self-assessment practices feasible even in low ceremony projects.},
	isbn = {978-989811198-2},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik},
	editor = {Jablonski, Stefan and Leszek A. {Maciaszek}}
}
@techreport {Simula.se.731,
	title = {An Eclipse Plug-In for the Flattening of Concurrency and Hierarchy in UML State Machines},
	number = {2009-06},
	year = {2009},
	publisher = {Simula Research Laboratory},
	abstract = {In this technical report, we present an algorithm that can be used to flatten simple-composite states (or-states), orthogonal states (and-states), and submachine states in complex UML state-machines [2]. The algorithm is appli-cable to state machines that include guarded transitions. The objective of this research is to automate the flattening of complex state machines. Orthogonal states and submachine states are transformed to simple-composite states. Simple-composite states are transformed to simple states by redirecting incoming and outgoing transitions. The result is a state machine where all state components are simple states. The flattening aspects are exemplified with isolated illustrations. Furthermore, we present STATE MACHINE FLATTENER - an Eclipse plug-in that implements this algorithm using Kermeta. Interesting aspect of the flattening are exemplified and discussed. Thus, the contribution of this study is threefold: 1) a flattening algorithm, 2) a plug-in that implements this algorithm to provide automatic flattening of complex UML state machines, and 2) a demonstration of the flattening approach in a number of example models.},
	author = {Nina Elisabeth {Holt} and Arisholm, Erik and Briand, Lionel}
}
@article {Simula.SE.116,
	title = {The Effect of Task Order on the Maintainability of Object-Oriented Software},
	journal = {Information and Software Technology},
	volume = {51},
	number = {2},
	year = {2009},
	month = {February},
	pages = {293-305},
	publisher = {Elsevier},
	abstract = {This paper presents results from a quasi-experiment that investigates how the sequence in which maintenance tasks are performed affects the time required to perform them and the functional correctness of the changes made. Specifically, the study compares how time required and correctness are affected by (1) starting with the easiest change task and progressively performing the more difficult tasks (Easy-First), versus (2) starting with the most difficult change task and progressively performing the easier tasks (Hard-First). In both cases, the experimental tasks were performed on two alternative types of design of a Java system to assess whether the choice of the design strategy moderates the effects of task order on effort and correctness. The results show that the time spent on making the changes is not affected significantly by the task order of the maintenance tasks, regardless of the type of design. However, the correctness of the maintainability tasks is significantly higher when the task order of the change tasks is Easy-First compared to Hard-First, again regardless of design. A possible explanation for the results is that a steeper learning curve (Hard-First) causes the programmer to create software that is less maintainable overall.},
	doi = {10.1016/j.infsof.2008.03.005},
	author = {Alf Inge {Wang} and Arisholm, Erik}
}
@article {Simula.SE.309,
	title = {The Effectiveness of Pair-Programming: a Meta-Analysis},
	journal = {Information and Software Technology},
	volume = {55},
	number = {7},
	year = {2009},
	month = {June},
	pages = {1110-1122},
	abstract = {Several experiments on the effects of pair versus solo programming have been reported in the literature. We present a metaanalysis of these studies. The analysis shows a small significant positive overall effect of pair programming on quality, a medium significant positive overall effect on duration, and a medium significant negative overall effect on effort. However, between-study variance is significant, and there are signs of publication bias among published studies on pair programming. A more detailed examination of the evidence suggests that pair programming is faster than solo programming when programming task complexity is low and yields code solutions of higher quality when task complexity is high. The higher quality for complex tasks comes at a price of considerably greater effort, while the reduced completion time for the simpler tasks comes at a price of noticeably lower quality. We conclude that greater attention should be given to moderating factors on the effects of pair programming.},
	author = {Jo Erskine {Hannay} and Dyb{\r a}, Tore and Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Simula.SE.305,
	title = {Effects of Personality on Pair Programming},
	journal = {IEEE Transactions on Software Engineering},
	volume = {36},
	number = {1},
	year = {2009},
	pages = {61-80},
	abstract = {Personality tests in various guises are commonly used in recruitment and career counseling industries. Such tests have also been considered as instruments for predicting programming and team performance in software engineering. However, research suggests that other human-related factors, such as motivation, general mental ability, expertise and task complexity also affect performance in general. This paper reports on a study of the impact of Big-Five personality traits on the performance of pair programmers together with the impact of expertise and task complexity. The study involved 196 software professionals from three countries forming 98 pairs. The analysis consisted of a confirmatory part and an exploratory part, and the results show that (1) our data does not confirm a meta-analysis-based model of the impact of certain personality traits on performance, and (2) personality traits in general have modest predictive value on pair programming performance compared with expertise, task complexity, and even country. Rather than focusing on direct effects of personality on pair programming performance, we conclude that (a) effort should be spent on elaborating on personality{\textquoteright}s (and other factors{\textquoteright}) indirect effects on performancemediated by formalized scales of collaboration, and (b) more effort should be spent on investigating other performance-related predictors such as programming skill, learning, motivation, expertise, and task complexity.},
	author = {Jo Erskine {Hannay} and Arisholm, Erik and Engvik, Harald and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Simula.SE.121,
	title = {Improving the Effectiveness of Root Cause Analysis in Post Mortem Analysis: a Controlled Experiment},
	journal = {Information and Software Technology},
	volume = {51},
	number = {1},
	year = {2009},
	month = {January},
	pages = {150-161},
	publisher = {Elsevier},
	abstract = {Retrospective analysis is a way to share knowledge following the completion of a project or major milestone. However, in the busy workday of a software project, there is rarely time for such reviews and there is a need for effective methods that will yield good results quickly without the need for external consultants or experts. Building on an existing method for retrospective analysis and theories of group involvement, we propose improvements to the root cause analysis phase of a lightweight retrospective analysis method known as post mortem analysis (PMA). In particular, to facilitate brainstorming during the root cause analysis phase of the PMA, we propose certain processual changes to facilitate more active individual participation and the use of less rigidly structured diagrams. We conducted a controlled experiment to compare this new variation of the method with the existing one, and conclude that in our setting of small software teams with no access to an experienced facilitator, the new variation is more effective when it comes to identifying possible root causes of problems and successes. The modified method also produced more specific starting points for improving the software development process.},
	doi = {doi:10.1016/j.infsof.2008.02.003},
	author = {Finn Olav {Bj{\o}rnson} and Alf Inge {Wang} and Arisholm, Erik}
}
@inbook {Simula.SE.433,
	title = {A Series of Controlled Experiments on Software Maintenance},
	booktitle = {Simula Research Laboratory - by thinking constantly about it},
	year = {2009},
	pages = {459-479},
	publisher = {Springer},
	organization = {Springer},
	chapter = {30},
	address = {Heidelberg},
	abstract = {Research background Software maintenance is both difficult and costly, and software organizations have little tangible evidence with regards to how they can improve their software maintenance practices and precisely what elements comprise the tradeoffs, the expected costs, and the benefits of alternative solutions. Scientific challenges The effort required to maintain software systems depends on many interacting factors, such as the software system\&$\#$39;s structural properties; the use of analysis, design principles and patterns; available documentation of requirements; design and code; tool support; and developer skills. Given the complexities of the problems at hand, empirical studies in realistic yet controlled settings are needed to better understand how such factors drive the costs of software maintenance and the quality of the software artefacts. In the Maintenance of Object-Oriented Software (MOOSE) project at Simula Research Laboratory, we have pursued large-scale, controlled experiments that combine a high degree of realism with the necessary degree of control, so as to obtain credible results that are applicable to the problems faced by the software industry. Obtained and expected results The studies reported herein contribute to a significantly improved understanding of software maintenance performance by pushing the boundaries of state-of-the-art empirical software engineering research. The results can help software maintenance organizations improve the efficiency of their maintenance processes and the quality of the products being maintained. Specifically, the results provide empirical evidence on tradeoffs and expected costs and benefits when applying design principles and patterns, simple task prioritization techniques, the Unified Modelling Language (UML), and pair programming in different maintenance contexts.},
	isbn = {978-3-642-01155-9},
	author = {Arisholm, Erik},
	editor = {Tveito, Aslak and Are Magnus {Bruaset} and Lysne, Olav}
}
@techreport {25331,
	title = {A Systematic Review on the Effects of UML during the Maintenance of Object-Oriented Software},
	year = {2009},
	publisher = {Simula Research Laboratory},
	author = {Dzidek, James and Arisholm, Erik and Lionel C. {Briand}}
}
@article {Simula.SE.196,
	title = {Understanding Software Maintenance and Evolution by Analyzing Individual Changes: a Literature Review},
	journal = {Journal of Software Maintenance and Evolution: Research and Practice},
	volume = {21},
	number = {6},
	year = {2009},
	pages = {349-378},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik}
}
@techreport {Simula.SE.317,
	title = {An Investigation of Change Effort in Two Evolving Software Systems},
	year = {2008},
	publisher = {Simula Research Laboratory},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik}
}
@book {Simula.SE.295,
	title = {Proceedings of the First Workshop on Empirical Studies of Model-Driven Engineering (ESMDE)},
	volume = {392},
	year = {2008},
	note = {http://CEUR-WS.org/Vol-392},
	publisher = {CEUR},
	abstract = {It is often difficult to rigorously evaluate Model-Driven Engineering (MDE) technologies. Performing empirical studies require skills, experience and tacit knowledge that are in many ways very different from the {\textquotedblleft}core{\textquotedblright} MDE research. Furthermore, empirical studies often entail large investments in terms of human resources, time and money. Nevertheless, evaluations of MDE technologies are needed in order to demonstrate the soundness, applicability, and cost effectiveness of proposed technologies in various development contexts. The aim of this workshop is to exemplify and discuss ways in which proposed modeldriven engineering (MDE) technologies should be evaluated, with a specific emphasis on how to plan, conduct, analyze and report the results of empirical studies. The workshop will have focus on the challenges of empirical studies involving human users, since MDE technologies are typically expected to be used by software engineers to improve various quality aspects of software systems and the productivity of software development. More detailed topics include: What are the main obstacles and potential remedies when performing empirical studies of MDE? What are the main threats to validity of empirical studies of MDE, and how should they be dealt with? For example, since MDE often represent new and complex technology, the selection and training of human subjects who participate in empirical studies often become critical factors. What are the most important outcome variables of the costs and benefits of MDE? How can quality be measured in the context of MDE? And can we define an unambiguous set of (benchmark) outcome measures to facilitate metaanalyses across subjects, systems, tasks and technologies? The goal of the workshop is to pave the way for the development of a MDE-specific framework for empirical evaluation of MDE technologies, or at least provide a minimum standard for evaluation that published work in the MDE community should abide by.},
	author = {Arisholm, Erik and Briand, Lionel and Anda, Bente Cecilie Dahlum}
}
@article {Simula.SE.120,
	title = {A Realistic Empirical Evaluation of the Costs and Benefits of UML in Software Maintenance},
	journal = {IEEE Transactions on Software Engineering},
	volume = {34},
	number = {3},
	year = {2008},
	month = {May},
	pages = {407-432},
	abstract = {The Unified Modeling Language (UML) is the de facto standard for object-oriented software analysis and design modeling. However, few empirical studies exist that investigate the costs and evaluate the benefits of using UML in realistic contexts. Such studies are needed so that the software industry can make informed decisions regarding the extent to which they should adopt UML in their development practices. This is the first controlled experiment that investigates the costs of maintaining and the benefits of using UML documentation during the maintenance and evolution of a real, non-trivial system, using professional developers as subjects, working with a state-of-the-art UML tool during an extended period of time. The subjects in the control group had no UML documentation. In this experiment, the subjects in the UML group had on average a practically and statistically significant 54\% increase in the functional correctness of changes (p=0.03), an insignificant 7\% overall improvement in design quality (p=0.22) - though a much larger improvement was observed on the first change task (56\%) - at the expense of an insignificant 14\% increase in development time caused by the overhead of updating the UML documentation (p=0.35).},
	doi = {10.1109/TSE.2008.15},
	author = {Dzidek, James and Arisholm, Erik and Briand, Lionel}
}
@techreport {Simula.SE.315,
	title = {A Systematic and Comprehensive Investigation of Methods to Build and Evaluate Fault Prediction Models},
	number = {TR 2008-06},
	year = {2008},
	month = {November},
	publisher = {Simula Research Laboratory},
	abstract = {This paper describes a study performed in an industrial setting that attempts to build predictive models to identify parts of a Java system with a high fault probability. The system under consideration is constantly evolving as several releases a year are shipped to customers. Developers usually have limited resources for their testing and would like to devote extra resources to faulty system parts. The main research focus of this paper is to systematically assess three aspects on how to build and evaluate fault-proneness models in the context of this large Java legacy system development project: (1) compare many data mining and machine learning techniques to build fault-proneness models, (2) assess the impact of using different metric sets entailing different data collection costs, such as source code structural measures and historic change/fault (process) measures, and (3) compare several alternative ways of assessing the performance of the models, in terms of (i) confusion matrix criteria such as accuracy and precision/recall, (ii) ranking ability, using the receiver operating characteristic area (ROC), and (iii) our proposed cost-effectiveness measure (CE). The results of the study indicate that the choice of fault-proneness modeling technique has limited impact on the resulting classification accuracy or cost-effectiveness. There is however large differences between the individual metric sets in terms of cost-effectiveness, and although the process measures are among the most expensive ones to collect, including them as candidate measures significantly improves the prediction models compared with models that only include structural measures and/or their deltas across releases - both in terms of ROC area and cost-effectiveness. Further, we observe that what is considered the best model is highly dependent on the criteria that are used to evaluate and compare the models. The regular confusion matrix criteria, although popular, are not clearly related to what we consider to be a crucial aspect, namely the cost-effectiveness of using fault-proneness prediction models to focus verification effort where it is the most needed.},
	author = {Arisholm, Erik and Briand, Lionel and Eivind B. {Johannessen}}
}
@techreport {Simula.SE.297,
	title = {A Systematic Review of Empirical Software Engineering Studies That Analyze Individual Changes},
	number = {2008-05},
	year = {2008},
	month = {September},
	publisher = {Simula Research Laboratory},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik}
}
@article {Simula.SE.162,
	title = {Are Two Heads Better Than One? on the Effectiveness of Pair Programming},
	journal = {IEEE Software},
	volume = {24},
	number = {6},
	year = {2007},
	month = {November/December},
	pages = {12-15},
	author = {Dyb{\r a}, Tore and Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and Jo Erskine {Hannay} and Shull, Forrest}
}
@inproceedings {Simula.SE.17,
	title = {Data Mining Techniques for Building Fault-Proneness Models in Telecom Java Software},
	journal = {18th International Symposium on Software Reliability Engineering (ISSRE)},
	year = {2007},
	month = {November},
	pages = {215-224},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {This paper describes a study performed in an industrial setting that attempts to build predictive models to identify parts of a Java system with a high probability of fault. The system under consideration is constantly evolving as several releases a year are shipped to customers. Developers usually have limited resources for their testing and inspections and would like to be able to devote extra resources to faulty system parts. The main research focus of this paper is two-fold: (1) use and compare many data mining and machine learning techniques to build fault-proneness models based mostly on source code measures and change/fault history data, and (2) demonstrate that the usual classification evaluation criteria based on confusion matrices may not be fully appropriate to compare and evaluate models.},
	isbn = {0-7695-3024-9},
	author = {Arisholm, Erik and Briand, Lionel and Magnus J. {Fuglerud}}
}
@techreport {25341,
	title = {Data Mining Techniques for Building Fault-Proneness Models in Telecom Java Software},
	year = {2007},
	note = {This technical report is an earlier version of a published conference paper. The published version can be found here:https://www.simula.no/publications/data-mining-techniques-building-fault-proneness-models-telecom-java-software},
	publisher = {Simula Research Laboratory},
	author = {Arisholm, Erik and Lionel C. {Briand} and Magnus J. {Fuglerud}}
}
@inproceedings {Simula.SE.3,
	title = {Educational Approach to an Experiment in a Software Archiecture Course},
	journal = {20th Conference on Software Engineering Education and Training (CSEE\&T 2007)},
	year = {2007},
	pages = {291-300},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {This paper reports experiences from an experiment in a software architecture course where the focus was both on giving students valuable education as well as getting important empirical results. The paper describes how the experiment was integrated in the course, and presents an evaluation of the experiment from an educational point of view. Further, the paper reflects on the costs and the benefits of carrying out an experiment in the context of a software architecture course for the involved stakeholders namely the researchers, the students, and the instructors. The main results show that we managed to integrate the experiment in the course, that the students learned several aspects of software architecture through the experiment, and that the main motivation for putting effort into the experiment was to prove own skills. We also describe some guidelines for planning and executing experiments as a part of a software engineering course.},
	isbn = {0-7695-2893-7},
	author = {Alf Inge {Wang} and Arisholm, Erik and Jaccheri, Letizia}
}
@article {Arisholm.2006.2,
	title = {Evaluating Pair Programming With Respect to System Complexity and Programmer Expertise},
	journal = {IEEE Transactions on Software Engineering},
	volume = {33},
	number = {2},
	year = {2007},
	month = {February},
	pages = {65-86},
	abstract = {A total of 295 junior, intermediate and senior professional Java consultants (99 individuals and 98 pairs) from 29 international consultancy companies in Norway, Sweden and the UK were hired for one day to participate in a controlled experiment on pair programming. The subjects used professional Java tools to perform several change tasks on two alternative Java systems with different degrees of complexity. The results of this experiment do not support the hypotheses that pair programming in general reduces the time required to solve the tasks correctly or increases the proportion of correct solutions. On the other hand, there is a significant 84 percent increase in effort to perform the tasks correctly. However, on the more complex system, the pair programmers had a 48 percent increase in the proportion of correct solutions, but no significant differences in the time taken to solve the tasks correctly. For the simpler system, there was a 20 percent decrease in time taken but no significant differences in correctness. However, the moderating effect of system complexity depends on the expertise of the subjects. The observed benefits of pair programming in terms of correctness on the complex system apply mainly to juniors, whereas the reductions in duration to perform the tasks correctly on the simple system apply mainly to intermediates and seniors.},
	author = {Arisholm, Erik and Hans Enger {Gallis} and Dyb{\r a}, Tore and Sj{\o}berg, Dag Ingar Kondrup}
}
@techreport {Simula.SE.161,
	title = {A Realistic Empirical Evaluation of the Costs and Benefits of UML in Software Maintenance},
	number = {2007 - 04},
	year = {2007},
	month = {September},
	publisher = {Simula Research Laboratory},
	address = {Oslo, Norway},
	abstract = {The Unified Modeling Language (UML) is the de facto standard for object-oriented software analysis and design modeling. However, few empirical studies exist that investigate the costs and evaluate the benefits of using UML in realistic contexts. Such studies are needed so that the software industry can make informed decisions regarding the extent to which they should adopt UML in their development practices. This is the first controlled experiment that investigates the costs of maintaining and the benefits of using UML documentation during the maintenance and evolution of a real, non-trivial system, using professional developers as subjects, working with a state-of-the-art UML tool during an extended period of time. The subjects in the control group had no UML documentation. In this experiment, the subjects in the UML group had on average a practically and statistically significant 54\% increase in the functional correctness of changes (p=0.03), an insignificant 7\% overall improvement in design quality (p=0.22) - though a much larger improvement was observed on the first change task (56\%) - at the expense of an insignificant 14\% increase in development time caused by the overhead of updating the UML documentation (p=0.35).},
	author = {Dzidek, James and Arisholm, Erik and Briand, Lionel}
}
@inproceedings {Benestad.2006.1,
	title = {Assessing Software Product Maintainability Based on Class-Level Structural Measures},
	journal = {7th International Conference on Product-focused Software Process Improvement (PROFES)},
	year = {2006},
	pages = {94-111},
	publisher = {Springer-Verlag},
	type = {Conference},
	abstract = {A number of structural measures have been suggested to support the assessment and prediction of software quality attributes. The aim of our study is to investigate how class-level measures of structural properties can be used to assess the maintainability of a software product as a whole. We survey, structure and discuss current practices on this topic, and apply alternative strategies on four functionally equivalent systems that were constructed as part of a multi-case study. In the absence of historical data needed to build statistically based prediction models, we apply elements of judgment in the assessment. We show how triangulation of alternative strategies as well as sensitivity analysis may increase the confidence in assessments that contain elements of judgment. This paper contributes to more systematic practices in the application of structural measures. Further research is needed to evaluate and improve the accuracy and precision of judgment-based strategies.},
	isbn = {0302-9743},
	author = {Hans Christian {Benestad} and Anda, Bente Cecilie Dahlum and Arisholm, Erik},
	editor = {M{\"u}nch, J{\"u}rgen}
}
@article {Arisholm.2006.5,
	title = {Empirical Assessment of the Impact of Structural Properties on the Changeability of Object-Oriented Software},
	journal = {Information and Software Technology},
	volume = {48},
	number = {11},
	year = {2006},
	month = {November},
	pages = {1046-1055},
	abstract = {Evolutionary software development processes mainly address concerns regarding client requirements and risk management. A side-effect of allowing frequent, and often unanticipated, modifications to the software is that they may result in poor structure, because the designers may not have anticipated the final set of requirements that would be implemented. By monitoring structural properties of the software one may be able to identify and correct design problems before they become unmanageable. Ways in which degradations in structural properties of the software can be measured are described and empirically validated, based on data collected from an industrial Java development project. The measures are validated by using them as candidate variables in a prediction model of the actual effort required to make modifications to the evolving software system. The results suggest that some measures that combine existing structural attribute measures with a weighting factor based on the relative proportion of change in each class are reasonably accurate predictors of change effort. This constitutes initial, empirical evidence that the proposed measures are valid quality indicators. Consequently, they may help designers to identify and correct design problems during the evolutionary development of object-oriented software.},
	author = {Arisholm, Erik}
}
@inbook {Arisholm.2006.1,
	title = {Evaluating the Effect of a Delegated Versus Centralized Control Style on the Maintainability of Object-Oriented Software},
	booktitle = {Software Process Improvement: Results and Experience from the Field},
	year = {2006},
	pages = {379-406},
	publisher = {Springer},
	organization = {Springer},
	chapter = {20},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup},
	editor = {Conradi, Reidar and Dyb{\r a}, Tore and Sj{\o}berg, Dag Ingar Kondrup and Ulsund, Tor}
}
@article {Arisholm.2006.3,
	title = {The Impact of UML Documentation on Software Maintenance: an Experimental Evaluation},
	journal = {IEEE Transactions on Software Engineering},
	volume = {32},
	number = {6},
	year = {2006},
	month = {June},
	pages = {365-381},
	abstract = {The Unified Modeling Language (UML) is becoming the de-facto standard for software analysis and design modeling. However, there is still a significant resistance to model-driven development in many software organizations as it is perceived to be expensive and not necessarily cost-effective. It is therefore important to investigate the benefits obtained through modeling. As a first step in this direction, this paper reports on controlled experiments, spanning across two locations, which investigate the impact of UML documentation on software maintenance. Results show that, for complex tasks and past a certain learning curve, the availability of UML documentation may result in significant improvements of the functional correctness of changes as well as their design quality. On the other hand, there does not seem to be any resulting time saving. For simpler tasks, the time needed to update the UML documentation may be substantial compared with the potential benefits, thus motivating the need for UML tools with better support for software maintenance.},
	author = {Arisholm, Erik and Briand, Lionel and Siw Elisabeth {Hove} and Labiche, Yvan}
}
@inproceedings {Arisholm.2006.4,
	title = {Predicting Fault-Prone Components in a Java Legacy System},
	journal = {5th ACM-IEEE International Symposium on Empirical Software Engineering (ISESE), Rio de Janeiro, Brazil, September 21-22},
	year = {2006},
	month = {September},
	pages = {8-17},
	publisher = {ACM},
	type = {Conference},
	abstract = {This paper reports on the construction and validation of fault-proneness prediction models in the context of an object-oriented, evolving, legacy system. The goal is to help QA engineers focus their limited verification resources on parts of the system likely to contain faults. A number of measures including code quality, class structure, changes in class structure, and the history of class-level changes and faults are included as candidate predictors of class fault-proneness. A cross-validated classification analysis shows that the obtained model has less than 20\% of false positives and false negatives, respectively. However, as shown in this paper, statistics regarding the classification accuracy tend to inflate the potential usefulness of the fault-proneness prediction models. We thus propose a simple and pragmatic methodology for assessing the cost-effectiveness of the predictions to focus verification effort. On the basis of the cost-effectiveness analysis we show that change and fault data from previous releases is paramount to developing a practically useful prediction model. When our model is applied to predict faults in a new release, the estimated potential savings in verification effort is about 29\%. In contrast, the estimated savings in verification effort drops to 0\% when history data is not included.},
	isbn = {1-59593-218-6},
	author = {Arisholm, Erik and Briand, Lionel},
	editor = {Mendes,  and Maldonado, Wohlin}
}
@article {Karahasanovic.2005.2,
	title = {Collecting Feedback During Software Engineering Experiments},
	journal = {Empirical Software Engineering},
	volume = {10},
	number = {2},
	year = {2005},
	pages = {113-147},
	abstract = {Objective: To improve the qualitative data obtained from software engineering experiments by gathering feedback during experiments. Rationale: Existing techniques for collecting quantitative and qualitative data from software engineering experiments do not provide sufficient information to validate or explain all our results. Therefore, we would like a cost-effective and unobtrusive method of collecting feedback from subjects during an experiment to augment other sources of data. Design of study: We formulated a set of qualitative questions that might be answered by collecting feedback during software engineering experiments. We then developed a tool to collect such feedback from experimental subjects. This feedback-collection tool was used in four different experiments and we evaluated the usefulness of the feedback obtained in the context of each experiment. The feedback data was triangulated with other sources of quantitative and qualitative data collected for the experiments. Results: We have demonstrated that the collection of feedback during experiments provides useful additional data to: validate the data obtained from other sources about solution times and quality of solutions; check process conformance; understand problem solving processes; identify problems with experiments; and understand subjects{\textquoteright} perception of experiments. Conclusions: Feedback collection has proved useful in four experiments and we intend to use the feedback-collection tool in a range of other experiments to further explore the cost-effectiveness and limitations of this technique. It is also necessary to carry out a systematic study to more fully understand the impact of the feedback-collecting tool on subjects{\textquoteright} performance in experiments.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Welland, Ray}
}
@inproceedings {Benestad.2005.1,
	title = {How to Recruit Professionals As Subjects in Software Engineering Experiments},
	journal = {IRIS (Information Systems Research in Scandinavia), August 6-9, Kristiansand Norway},
	year = {2005},
	month = {August},
	publisher = {Department of Information Systems, Agder University College},
	type = {Conference},
	abstract = {Controlled experiments are the classical scientific method for identifying cause-effect relationships, and are complementary to case studies and surveys as a means to empirically evaluate information systems development methods and practices. Most controlled experiments that evaluate software development methods and practices use students as subjects. Using students as subjects is convenient. However, a common criticism of controlled experiments in which students are used as subjects is that it is difficult to generalize the results to industrial settings. Consequently, Simula Research Laboratory has included professionals as subjects in software engineering experiments. At present, more than 750 professional software developers from 46 software development organizations have participated in our experiments. From this experience we have identified three important principles for research groups that want to include professional software developers as subjects in controlled experiments. First, practical constraints must be considered when defining the target population of software developers. Second, the participating organizations must be offered flexibility and value using a planned communication strategy, in order to ensure adequately sized representative samples of organizations and individuals. Third, to ensure long-term, relationships with the organizations, high professional and ethical standards must be employed.},
	author = {Hans Christian {Benestad} and Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup},
	editor = {E., Hustad and B.E., Munkvold and K., Rolland and L.S., Flak}
}
@techreport {Arisholm.2005.2,
	title = {The Impact of UML Documentation on Software Maintenance: an Experimental Evaluation},
	number = {2005-14},
	year = {2005},
	month = {August},
	publisher = {Simula Research Laboratory},
	abstract = {The Unified Modeling Language (UML) is becoming the de-facto standard for software analysis and design modeling. However, there is still a significant resistance to model-driven development in many software organizations as it is perceived to be expensive and not necessarily cost-effective. It is therefore important to investigate the benefits obtained through modeling. As a first step in this direction, this paper reports on controlled experiments, spanning across two locations, which investigate the impact of UML documentation on software maintenance. Results show that, for complex tasks and passed a certain learning curve, the availability UML documentation may result into significant improvements of the functional correctness of changes as well as their design quality. On the other hand, there does not seem to be any resulting time saving. For simpler tasks, the time needed to update the UML documentation may be substantial compared with the potential benefits, thus motivating the need for UML tools with better support for performing model changes.},
	author = {Arisholm, Erik and Briand, Lionel and Siw Elisabeth {Hove} and Labiche, Yvan}
}
@inproceedings {Dyba.2005.3,
	title = {Measuring Software Methodology Usage: Challenges of Conceptualization and Operationalization},
	journal = {International Symposium on Empirical Software Engineering (ISESE{\textquoteright}05), Noosa, Australia, November 17-18},
	year = {2005},
	month = {November},
	pages = {447-458},
	publisher = {IEEE Computer Society},
	type = {1},
	abstract = {Most software engineering research implicitly assumes that development methodologies are useful and that there is a direct relationship between software methodologies and their effects on organizational performance. However, a methodology cannot have an impact if it is not used. The purpose of this paper is, thus, to raise a number of challenges related to the conceptualization and operationalization of methodology usage and to report on a study that compared subjective and objective operationalizations of usage. Results of regression analyses show that these operationalizations do not appear to be strongly related. While self-reported usage is related to self-reported measures of the independent variables of methodology acceptance in the study, the objective and computer-recorded measures show different and distinctly weaker links. There are several explanations to these seemingly contradictory results. Most importantly, the results of this study suggest a need for reconceptualization and better validation of methodology usage constructs in future, empirical software engineering research.},
	author = {Dyb{\r a}, Tore and Nils B. {Moe} and Arisholm, Erik}
}
@misc {Gallis.2005.1,
	title = {The Pair Programming Experiment},
	howpublished = {Presentation of the international pair programming experiment, SPIKE seminar, Oslo, June 16th 2005.},
	year = {2005},
	month = {June},
	author = {Hans Enger {Gallis} and Arisholm, Erik}
}
@article {Vokac.2004.2,
	title = {A Controlled Experiment Comparing the Maintainability of Programs Designed With and Without Design Patterns - a Replication in a Real Programming Environment},
	journal = {Empirical Software Engineering},
	volume = {9},
	number = {3},
	year = {2004},
	pages = {149-195},
	abstract = {Software "Design Patterns" seek to package proven solutions to design prob- lems in a form that makes it possible to find, adapt and reuse them. To support the industrial use of Design Patterns, this research investigates when, and how, using patterns is beneficial, and whether some patterns are more di\pmcult to use than others. This paper describes a replication of an earlier controlled experiment on Design Patterns in maintenance, with major extensions. Experimental realism was increased by using a real programming environment instead of pen and paper, and paid professionals from multiple major consultancy companies as subjects. Measurements of elapsed time and correctness were analyzed using regression models and an estimation method that took into account the correlations present in the raw data. Together with on-line logging of the subjects{\textquoteright} work, this made possible a better qualitative understanding of the results. The results indicate quite strongly that some patterns are much easier to understand and use than others. In particular, the Visitor pattern caused much confusion. Conversely, the patterns Observer and, to a certain extent, Decorator were grasped and used intuitively, even by subjects with little or no knowledge of patterns. The implication is that Design Patterns are not universally good or bad, but must be used in a way that matches the problem and the people. When approaching a program with documented Design Patterns, even basic training can improve both the speed and quality of maintenance activities},
	author = {Vok{\'a}c, Marek and Tichy, W. and Sj{\o}berg, Dag Ingar Kondrup and Arisholm, Erik and Aldrin, M.}
}
@article {SE.4.Arisholm.2004.a,
	title = {Dynamic Coupling Measurement for Object-Oriented Software},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {8},
	year = {2004},
	month = {August},
	pages = {491-506},
	abstract = {The relationships between coupling and external quality factors of object-oriented software have been studied extensively for the past few years. For example, several studies have identified clear empirical relationships between class-level coupling and class fault-proneness. A common way to define and measure coupling is through structural properties and static code analysis. However, because of polymorphism, dynamic binding, and the common presence of unused ({\textquotedblleft}dead{\textquotedblright}) code in commercial software, the resulting coupling measures are imprecise as they do not perfectly reflect the actual coupling taking place among classes at run-time. For example, when using static analysis to measure coupling, it is difficult and sometimes impossible to determine what actual methods can be invoked from a client class if those methods are overridden in the subclasses of the server classes. Coupling measurement has traditionally been performed using static code analysis, because most of the existing work was done on non-object oriented code and because dynamic code analysis is more expensive and complex to perform. For modern software systems, however, this focus on static analysis can be problematic, because although dynamic binding existed before the advent of object-orientation, its usage has increased significantly in the last decade. This paper describes how coupling can be defined and precisely measured based on dynamic analysis of systems. We refer to this type of coupling as dynamic coupling. An empirical evaluation of the proposed dynamic coupling measures is reported in which we study the relationship of these measures with the change proneness of classes. Data from maintenance releases of a large Java system are used for this purpose. Preliminary results suggest that some dynamic coupling measures are significant indicators of change proneness and that they complement existing coupling measures based on static analysis.},
	author = {Arisholm, Erik and Briand, Lionel and F{\o}yen, A.}
}
@article {SE.4.Arisholm.2004.b,
	title = {Evaluating the Effect of a Delegated Versus Centralized Control Style on the Maintainability of Object-Oriented Software},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {8},
	year = {2004},
	month = {August},
	pages = {521-534},
	abstract = {A fundamental question in object-oriented design is how to design maintainable software. According to expert opinion, a delegated control style, typically a result of responsibility-driven design, represents object-oriented design at its best, whereas a centralized control style is reminiscent of a procedural solution, or a {\textquotedblleft}bad{\textquotedblright} object-oriented design. This paper presents a controlled experiment that investigates these claims empirically. A total of 99 junior, intermediate and senior professional consultants from several international consultancy companies were hired for one day to participate in the experiment. To compare differences between (categories of) professionals and students, 59 students also participated. The subjects used professional Java tools to perform several change tasks on two alternative Java designs that had a centralized and delegated control style, respectively. The results show that the most skilled developers, in particular the senior consultants, require less time to maintain software with a delegated control style than with a centralized control style. However, more novice developers, in particular the undergraduate students and junior consultants, have serious problems understanding a delegated control style, and perform far better with a centralized control style. Thus, the maintainability of object-oriented software depends, to a large extent, on the skill of the developers who are going to maintain it. These results may have serious implications for object-oriented development in an industrial context: having senior consultants design object-oriented systems may eventually pose difficulties unless they make an effort to keep the designs simple, as the cognitive complexity of {\textquotedblleft}expert{\textquotedblright} designs might be unmanageable for less skilled maintainers.},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@inbook {Sjoberg.2003.1,
	title = {Challenges and Recommendations When Increasing the Realism of Controlled Software Engineering Experiments},
	booktitle = {Empirical Methods and Studies in Software Engineering: Experiences from ESERNET},
	series = {Lecture Notes in Computer Science, Volume 2765},
	year = {2003},
	pages = {24-38},
	publisher = {Springer},
	organization = {Springer},
	chapter = {Part II: Method Chapters},
	address = {Berlin / Heidelberg},
	abstract = {An important goal of most empirical software engineering experiments is the transfer of the research results to industrial applications. To convince industry about the validity and applicability of the results of controlled software engineering experiments, the tasks, subjects and the environments should be as realistic as practically possible. Such experiments are, however, more demanding and expensive than experiments involving students, small tasks and pen-and-paper environments. This chapter describes challenges of increasing the realism of controlled experiments and lessons learned from the experiments that have been conducted at Simula Research Laboratory.},
	isbn = {978-3-540-40672-3},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Vok{\'a}c, Marek},
	editor = {Conradi, Reidar and Alf Inge {Wang}}
}
@techreport {SE.7.Arisholm.2003.b,
	title = {A Controlled Experiment With Professionals to Evaluate the Effect of a Delegated Versus Centralized Control Style on the Maintainability of Object-Oriented Software},
	number = {2003-6},
	year = {2003},
	publisher = {Simula Research Laboratory},
	abstract = {One fundamental question in object-oriented design is how to design maintainable software. According to expert opinion, a delegated control style, typically a result of responsibility-driven design, represents object-oriented design at its best, whereas a centralized control style is reminiscent of a procedural solution, or a {\textquotedblleft}bad{\textquotedblright} object-oriented design. This paper presents a controlled experiment that investigates these claims empirically. A total of 99 junior, intermediate and senior professional consultants from several international consultancy companies were hired for one day to take part in the experiment. To compare differences between (categories of) professionals and students, 59 students also participated. The subjects used professional Java tools to perform several change tasks on two alternative Java designs having a centralized and delegated control style, respectively. The results show that the most skilled developers, in particular the senior consultants, require less time to maintain software with a delegated control style than with a centralized control style. However, more novice developers, in particular the undergraduate students and junior consultants, have serious problems understanding a delegated control style, and perform far better with a centralized control style. Thus, the maintainability of object-oriented software depends to a large extent on the skill of the developers who are going to maintain it. The results may have serious implications for object-oriented development in an industrial context: having senior consultants design object-oriented systems that eventually will be maintained by juniors may be unwise, since the cognitive complexity of such {\textquotedblleft}expert{\textquotedblright} designs might be unmanageable for less skilled maintainers.},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@techreport {SE.7.Arisholm.2003.a,
	title = {Dynamic Coupling Measurement for Object-Oriented Software (Detailed Analyses)},
	number = {2003-5},
	year = {2003},
	note = {Updated April 26, 2004},
	publisher = {Simula Research Laboratory},
	abstract = {The relationships between coupling and external quality factors of object-oriented software have been studied extensively for the past few years. For example, several studies have identified clear empirical relationships between class-level coupling and class fault-proneness. A common way to define and measure coupling is through structural properties and static code analysis. However, because of polymorphism, dynamic binding, and the common presence of unused ({\textquotedblleft}dead{\textquotedblright}) code in commercial software, the resulting coupling measures are imprecise as they do not perfectly reflect the actual coupling taking place among classes at run-time. For example, when using static analysis to measure coupling, it is difficult and sometimes impossible to determine what actual methods can be invoked from a client class if those methods are overridden in the subclasses of the server classes. Similarly, static analysis is not a fully appropriate tool to account for dead code. Coupling measurement has traditionally been performed using static code analysis, because most of the existing work was done on non-object oriented code and because dynamic code analysis is more expensive and complex. This paper describes how coupling can be defined and precisely measured on the basis of dynamic analysis or equivalent dynamic models of the system. We refer to this type of coupling as dynamic coupling. A first empirical evaluation of the proposed dynamic coupling measures is reported in which we study the relationship of these measures with the change proneness of classes. Data from maintenance releases of a large Java system are used for this purpose. Preliminary results suggest that some dynamic coupling measures are strong indicators of change proneness and that they complement existing coupling measures based on static analysis.},
	author = {Arisholm, Erik and Briand, Lionel and F{\o}yen, A.}
}
@techreport {SE.7.Arisholm.2003.c,
	title = {An Initial Controlled Experiment to Evaluate the Effect of UML Design Documentation on the Maintainability of Object-Oriented Software in a Realistic Programming Environment},
	number = {2003-04},
	year = {2003},
	publisher = {Simula Research Laboratory},
	abstract = {The Unified Modeling Language (UML) is becoming the standard notation for expressing object-oriented analysis and design models. However, to date, there are only very few empirical studies that have attempted to evaluate UML, that is, to evaluate the potential costs, benefits and appropriate use of UML. For example, a common claim is that the UML documentation will improve the ease of understanding how to perform changes to the software it represents. However, this claim has not been evaluated empirically. Furthermore, using UML may incur additional costs, because the UML documents need to be developed and maintained. Thus, the tradeoffs between the costs and benefits of using UML need to be investigated. This paper describes a controlled experiment to evaluate the effect of UML documentation (versus no UML documentation) on the maintainability of object-oriented software. In this experiment, 20 students designed and coded several Java change (maintenance) tasks. About half of them received UML documentation, whereas the other half did not. The subjects who received UML documentation also had to update the documentation using a tool, Tau UML. The primary goal of the experiment was to evaluate to which extent the access to UML class and sequence diagrams improves the ease of understanding and changing object-oriented software. To assess the costs and benefits of using UML in object-oriented development projects, the experiment also attempted to assess the additional costs associated with updating the UML documentation. The dependent variables of the study were effort (in minutes spent to solve the tasks) and correctness. The effort data was reported by each subject using a task questionnaire. The correctness of the task solutions was assessed by the researchers. The subjects spent between five and eight hours solving tasks. Five observers were present during the experiment. A secondary goal of the experiment was to evaluate how a so-called {\textquotedblleft}think-aloud screen{\textquotedblright} affected the performance of subjects in controlled software engineering experiments, that is, whether the think-aloud screen decreases performance of the subjects (task solving time and correctness of the task solutions) compared with the silent condition, and whether the think-aloud screen provides valuable information that cannot be collected in the silent condition. This paper focuses on the evaluation of the effect of using UML to understand and change object-oriented software. Further discussion of the effects of using the think-aloud screen is discussed in [5].},
	author = {Arisholm, Erik and S. A. {Ali} and S. E. {Hove}}
}
@inproceedings {SE.5.Gallis.2003.a,
	title = {An Initial Framework for Research on Pair Programming},
	journal = {ISESE{\textquoteright}2003 (Second International Symposium on Empirical Software Engineering)},
	year = {2003},
	month = {September 30 - October 1},
	pages = {132-142},
	publisher = {IEEE Computer Society},
	type = {1},
	address = {Rome, Italy},
	abstract = {In recent years, several claims have been put forward in favour of pair programming, as opposed to individual programming. However, results from existing studies on pair programming contain apparent contradictions. The differences in the context in which the studies were conducted may be one explanation for such results. This paper presents an initial framework for research on pair programming. The aim is to support empirical studies and meta-analysis for developing theories about pair programming. The framework is based on (1) existing studies on pair programming, (2) ongoing studies by the authors, and (3) theories from group dynamics.},
	author = {Gallis, H. and Arisholm, Erik and Dyb{\r a}, Tore}
}
@misc {Gallis.2003.3,
	title = {Pair Programming Versus Individual Programming on Maintenance Tasks With Professional Subjects: a Controlled Experiment},
	howpublished = {Scientific presentation to ISERN, October 4th},
	year = {2003},
	month = {October},
	author = {Hans Enger {Gallis} and Arisholm, Erik and Dyb{\r a}, Tore and Sj{\o}berg, Dag Ingar Kondrup}
}
@techreport {Karahasanovic.2003.3,
	title = {A Think-Aloud Support Tool for Collecting Feedback in Large-Scale Software Engineering Experiments},
	number = {2003-7},
	year = {2003},
	publisher = {Simula Research Laboratory},
	abstract = {The think-aloud method has proven useful for studying how a technology is used. However, it is labour-intensive and makes demands on resources for studies with many subjects. This paper describes a tool, called the think-aloud tool, that we have developed to support the collection of feedback from subjects in large-scale software engineering experiments. This tool instructs the subjects, at regular intervals, to write down their thoughts on a web-based screen and collects their feedback. An empirical evaluation of this tool is reported. The results from four controlled experiments suggest that the think-aloud tool is a valuable means for increasing the quality of large-scale software engineering experiments. The results also indicate that the participants were mostly positive to the tool and had no significant problems in verbalising their thoughts. The tool we proposed adds to an available set of tools for data collection in software engineering experiments. Whereas we have experience with the think-aloud tool in software engineering experiments, there is also potential for using such a tool in other areas in which the think-aloud method is used in studies involving a computer system.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Sjoeberg.2002,
	title = {Conducting Realistic Experiments in Software Engineering},
	journal = {ISESE{\textquoteright}2002 (First International Symposium on Empirical Software Engineering)},
	year = {2002},
	month = {October 3-4},
	pages = {17-26},
	publisher = {IEEE Computer Society},
	type = {1},
	abstract = {An important goal of most empirical software engineering research is the transfer of research results to industrial applications. Two important obstacles for this transfer are the lack of control of variables of case studies, i.e., the lack of explanatory power, and the lack of realism of controlled experiments. While it may be difficult to increase the explanatory power of case studies, there is a large potential for increasing the realism of controlled software engineering experiments. To convince industry about the validity and applicability of the experimental results, the tasks, subjects and the environments of the experiments should be as realistic as practically possible. Such experiments are, however, more expensive than experiments involving students, small tasks and pen-and-paper environments. Consequently, a change towards more realistic experiments requires a change in the amount of resources spent on software engineering experiments. This paper argues that software engineering researchers should apply for resources enabling expensive and realistic software engineering experiments similar to how other researchers apply for resources for expensive software and hardware that are necessary for their research. The paper describes some of the experiences from recent experiments that varied in size from involving one software professional for 5 days to 130 software professionals, from 9 consultancy companies, for one day each.},
	isbn = {0-7695-1796-X},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Koren, E. and Vok{\'a}c, Marek},
	editor = {found, not}
}
@inproceedings {SE.5.Arisholm.2002.a,
	title = {Dynamic Coupling Measures for Object-Oriented Software},
	journal = {8th International Symposium on Software Metrics},
	year = {2002},
	month = {June},
	pages = {33-42},
	publisher = {IEEE Computer Society},
	type = {1},
	address = {Ottawa, Ont., Canada},
	abstract = {The relationships between coupling and external quality factors of object-oriented software have been studied extensively for the past few years. For example, several studies have identified clear empirical relationships between class-level coupling and the fault-proneness of the classes. A common way to quantify the coupling is through static code analysis. However, the resulting static coupling measures only capture certain underlying dimensions of coupling. Other dependencies regarding the dynamic behavior of software can only be inferred from run-time information. For example, due to inheritance and polymorphism, it is not always possible to determine the actual receiver and sender classes (i.e., the objects) from static code analysis. This paper describes how several dimensions of dynamic coupling can be calculated by tracing the flow of messages between objects at run-time. As a first evaluation of the proposed dynamic coupling measures, fairly accurate prediction models of the change proneness of classes have been developed using change data from nine maintenance releases of a large SmallTalk system. Preliminary results suggest that dynamic coupling may also be useful for developing prediction models and tools supporting change impact analysis. At present, work on developing a dynamic coupling tracer and ripple-effect prediction models for Java programs is underway.},
	author = {Arisholm, Erik}
}
@inbook {SE.5.Arisholm.2002.d,
	title = {Kapittel 10: Objektdesign},
	booktitle = {Systemutvikling - Fra kjernen og ut, fra skallet og inn},
	year = {2002},
	publisher = {H{\o}yskoleforlaget},
	organization = {H{\o}yskoleforlaget},
	chapter = {10},
	author = {Arisholm, Erik and Skagestein, G.},
	editor = {Skagestein, G.}
}
@inbook {SE.5.Arisholm.2002.b,
	title = {Kapittel 9: Fra Skallet Og Inn (kravspesifikasjon)},
	booktitle = {Systemutvikling - Fra kjernen og ut, fra skallet og inn},
	year = {2002},
	publisher = {H{\o}yskoleforlaget},
	organization = {H{\o}yskoleforlaget},
	chapter = {9},
	author = {Arisholm, Erik and Skagestein, G.},
	editor = {Skagestein, G.}
}
@inproceedings {SE.5.Arisholm.2002.c,
	title = {SESE - an Experiment Support Environment for Evaluating Software Engineering Technologies},
	journal = {NWPER{\textquoteright}2002 (Tenth Nordic Workshop on Programming and Software Development Tools and Techniques)},
	year = {2002},
	month = {18-20 August},
	pages = {81-98},
	type = {1},
	address = {Copenhagen, Denmark},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and G. J. {Carelius} and Lindsj{\o}rn, Y.}
}
@inproceedings {SE.5.Gallis.2002,
	title = {A Transition From Partner Programming to Pair Programming - an Industrial Case Study},
	journal = {Workshop "Pair Programming Installed" in 17th Annual ACM Conference on Object-Oriented Programming, Systems, Languages, and Applications (OOPSLA), Position paper},
	year = {2002},
	month = {November 4-8},
	publisher = {-},
	type = {1},
	address = {Seattle USA},
	abstract = {Pair programming is believed to provide many benefits compared with individual programming. However, implementing pair programming in a development project may not be trivial. At present, we are studying a small, industrial development project with two distributed pairs. Data has been collected through daily diaries, task questionnaires, feedback meetings and observation of the pairs. The preliminary results show that the two pairs are struggling to convert from their usual {\textquotedblleft}partner programming{\textquotedblright} habits to pair programming. We believe the main reason is that they mostly are used to working on their own individual tasks, but in close cooperation with another programmer in the same physical room (i.e., {\textquotedblleft}partner programming{\textquotedblright}). They are not convinced that a more structured collaboration (i.e., pair programming) will provide additional benefits. There is also a great difference between the pairs in the degree of resistance to pair programming. It seems that the pair with more experience and more knowledge about the system as a whole is more resistant to pair programming than the inexperienced pair. We believe that future empirical studies need to compare pair programming with other kinds of programmer collaboration, such as partner programming and team collocation. Research on pair programming should address when and how the various types of programmer collaboration are beneficial.},
	author = {Gallis, H. and Arisholm, Erik and Dyb{\r a}, Tore}
}
@article {SE.4.Arisholm.2002,
	title = {A Web-Based Support Environment for Software Engineering Experiments},
	journal = {Nordic Journal of Computing},
	volume = {9},
	number = {4},
	year = {2002},
	pages = {231-247},
	abstract = {The software engineering communities frequently propose new software engineering technologies, such as new development techniques, programming languages and tools, without rigorous scientific evaluation. One way to evaluate software engineering technologies is through controlled experiments where the effects of the technology can be isolated from confounding factors, i.e., establishing cause-effect relationships. For practical and financial reasons, however, such experiments are often quite unrealistic, typically involving students in a class-room environment solving small pen-and-paper tasks. A common criticism of the results of the experiments is their lack of external validity, i.e., that the results are not valid outside the experimental conditions. To increase the external validity of the experimental results, the experiments need to be more realistic. The realism can be increased using professional developers as subjects who conduct larger experimental tasks in their normal work environment. However, the logistics involved in running such experiments are tremendous. More specifically, the experimental materials (e.g., questionnaires, task descriptions, code and tools) must be distributed to each programmer, the progress of the experiment needs to be controlled and monitored, and the results of the experiment need to be collected and analyzed. To support this logistics for large-scale, controlled experiments, we have developed a web-based experiment support environment called SESE. This paper describes SESE, its development and the experiences from using it to conduct a large controlled experiment in industry.},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and G. J. {Carelius} and Lindsj{\o}rn, Y.}
}
@article {SE.4.Arisholm.2001,
	title = {Assessing the Changeability of Two Object-Oriented Design Alternatives - a Controlled Experiment},
	journal = {Empirical Software Engineering},
	volume = {6},
	number = {3},
	year = {2001},
	pages = {231-277},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Sjoeberg.2001,
	title = {Conducting Experiments on Software Evolution},
	journal = {4th International Workshop on Principles of Software Evolution (IWPSE 2001)},
	year = {2001},
	month = {September},
	pages = {142-145},
	publisher = {-},
	type = {Workshop},
	address = {Vienna, Austria},
	isbn = {1-58113-508-4},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Arisholm, Erik and J{\o}rgensen, Magne},
	editor = {Tamai, Tetsuo and Aoyama, Mikio and Bennet, Keith}
}
@phdthesis {SE.3.Arisholm.2001,
	title = {Empirical Assessment of Changeability in Object-Oriented Software},
	year = {2001},
	note = {ISSN 1501-7710, No. 143},
	school = {University of Oslo},
	type = {phd},
	author = {Arisholm, Erik}
}
@inproceedings {SE.5.Lien.2001.a,
	title = {Evolutionary Development of Web-Applications - Lessons Learned},
	journal = {European Software Process Improvement Conference (EuroSPI{\textquoteright}2001)},
	year = {2001},
	month = {November},
	publisher = {-},
	type = {1},
	address = {Limerick Institute of Technology, Ireland},
	abstract = {On evolutionary web development in three different companies.},
	author = {A. C. {Lien} and Arisholm, Erik}
}
@inproceedings {SE.5.Bratthall.2001.a,
	title = {Program Understanding Behaviour During Estimation of Enhancement on Small Java Programs},
	journal = {3rd International Conference on Product Focused Software Process Improvement (PROFES 2001)},
	year = {2001},
	month = {September},
	publisher = {-},
	type = {1},
	address = {Kaiserslautern, Germany},
	author = {Bratthall, Lars and Arisholm, Erik and J{\o}rgensen, Magne}
}
@article {SE.4.Arisholm.2000,
	title = {Towards a Framework for Empirical Assessment of Changeability Decay},
	journal = {Journal of Systems and Software},
	volume = {53},
	number = {1},
	year = {2000},
	pages = {3-14},
	abstract = {Evolutionary development allows early and frequent adaptations to new or changed requirements. However, such unanticipated changes may invalidate design documentation and cause structural degradations of the software, which in turn may accelerate changeability decay. Our definition of changeability decay focuses on the increased effort required to implement changes. We have identified three approaches to the assessment of changeability decay: (1) Structure measurement, (2) change complexity measurement and (3) benchmarking. Our research aims to evaluate and compare these approaches in order to develop an empirical assessment framework. In this paper we propose a set of change complexity measures (2) and compare them with structural attribute measures (1) using detailed process and product data collected from a commercial object-oriented development project. The preliminary results indicate that the change complexity measures capture some dimensions of changeability decay not accounted for with structural attribute measures. However, the current findings also suggest that many aspects of changeability decay cannot be accounted for by the indirect measures utilized in approach (1) and (2). As an alternative approach, we therefore propose using benchmarks (3) where change effort can be measured more directly. A research methodology for the development of benchmarks and benchmarking procedures are described.},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Arisholm.1999.b,
	title = {Empirical Assessment of Changeability Decay in Object-Oriented Software},
	journal = {ICSE{\textquoteright}99 Workshop on Empirical Studies of Software Development and Evolution},
	year = {1999},
	month = {May},
	pages = {62-69},
	publisher = {Springer Netherlands},
	type = {Workshop},
	address = {Los Angeles, CA},
	abstract = {Abstract Genera AS is a vendor of a CASE tool called Genova. The work described in this paper aims to define and evaluate an evolutionary development process (the Genova process) to complement and support the use of the Genova tool. As a starting point, we used a lightweight version of the Rational Unified Process in a development project for one of our customers. This process was instrumented to enable process improvement activities. An essential question is what to improve the defined or the actual process? Based on quantitative and qualitative data, we identified improvements related to the distribution of test effort throughout the life cycle. Furthermore, we gained useful experience on the management of evolutionary development projects. Our case study also provided insight for instrumentation of the process for collecting data related to the cost of changes.},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Simula.SE.42,
	title = {Empirical Studies of Object-Oriented Artifacts, Methods, and Processes: State of the Art and Future Directions},
	journal = {Empirical Software Engineering},
	volume = {4},
	number = {4},
	year = {1999},
	author = {Briand, Lionel and Arisholm, Erik and Counsell, Steve and Houdek, F. and Th{\'e}venod-Fosse, P.}
}
@article {SE.4.Briand.1999,
	title = {Empirical Studies of Object-Oriented Artifacts, Methods, and Processes: State of the Art and Future Directions},
	journal = {Empirical Software Engineering},
	volume = {4},
	number = {4},
	year = {1999},
	note = {(Also available as ISERN TR 99-12)},
	pages = {387-404},
	author = {Briand, Lionel and Arisholm, Erik and Counsell, Steve and Houdek, F.}
}
@inproceedings {SE.5.Arisholm.1999.c,
	title = {Guidelines on Conducting Software Process Improvement Studies in Industry},
	journal = {22nd IRIS Conference (Information Systems Research Seminar In Scandinavia)},
	year = {1999},
	month = {August 7-10,},
	pages = {87-102},
	type = {Conference},
	address = {Keuruu, Finland},
	author = {Arisholm, Erik and Anda, Bente Cecilie Dahlum and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Arisholm.1999.a,
	title = {Improving an Evolutionary Development Process - a Case Study},
	journal = {EuroSPI99},
	year = {1999},
	month = {25-27 October,},
	pages = {9.40-9.50},
	type = {1},
	address = {Pori, Finland},
	author = {Arisholm, Erik and Skandsen, J. and Sagli, K. and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Arisholm.1998,
	title = {Incorporating Rapid User Interface Prototyping in Object-Oriented Analysis and Design With Genova},
	journal = {Proceedings of NWPER{\textquoteright}98 Nordic Workshop on Programming Environment Research},
	year = {1998},
	month = {June 14-16,},
	pages = {155-161},
	publisher = {-},
	type = {1},
	address = {Bergen},
	author = {Arisholm, Erik and Hans Christian {Benestad} and Fredhall, H. and Skandsen, J.}
}
