@inproceedings {Simula.prepare.14,
	title = {Inferring Skill From Tests of Programming Performance: Combining Time and Quality},
	journal = {Proc. 5th Int{\textquoteright}l Symp.Empirical Software Engineering and Measurement (ESEM)},
	year = {2011},
	pages = {305-314},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {The skills of software developers are crucial to the success of software projects. Also, controlling for individual differences is important when studying the general effect of a method or a tool. However, the way skill is determined in industry and research settings is often ad hoc or based on unvalidated methods. According to established test theory, validated tests of skill should infer skill levels from welldefined performance measures on multiple small, representative tasks. We show how time and quality can be meaningfully combined as task performance, and subsequently aggregated into an approximate measure of programming skill. Our results show significant and positive correlations between our proposed measures of skill and variables such as seniority, lines of code written, and self-evaluated expertise. The method for combining time and quality are a promising first step to measuring programming skill in industry and research settings.},
	isbn = {978-1-4577-2203-5},
	author = {Gunnar Rye {Bergersen} and Jo Erskine {Hannay} and Sj{\o}berg, Dag Ingar Kondrup and Dyb{\r a}, Tore and Karahasanovic, Amela}
}
@article {Karahasanovic.2006.5,
	title = {Comparing of Feedback Collection and Think-Aloud Methods in Program Comprehension Studies},
	journal = {Behaviour \& Information Technology},
	volume = {28},
	number = {2},
	year = {2009},
	month = {March-April},
	pages = {139-164},
	publisher = {The University of Western Australia},
	abstract = {This paper reports an explorative experimental comparison of (i) an experience-sampling method called feedback collection and (ii) the think-aloud methods with respect to their usefulness in studies on program comprehension. Think-aloud methods are widely used in studies of cognitive processes, including program comprehension. Alternatively, as in the feedback-collection method (FCM), cognitive processes can be traced by collecting written feedback from the subjects at regular intervals. We compare FCM with concurrent think-aloud (CTA) and retrospective think-aloud (RTA) regarding type and usefulness of the collected information, costs related to analysis of the collected information, and effects of the data collection methods on the subjects\&$\#$39; performance. FCM allowed us to identify a greater number of comprehension problems that prevented progress or caused significant delay (FCM: 30 problems; CTA: 5; RTA: 15). It was less precise in identifying strategies for comprehension than CTA (92\% correctness for FCM; 100\% for CTA). FCM was less expensive in analysis (transcription and coding) than the other two methods (FCM: 0,7 hours of analysis per protocol; CTA: 31 hours; RTA: 7,9 hours). The results indicate that all three methods of data collection were intrusive and affected the performance of the subjects with respect to time and correctness (small to medium effect size). This research confirms that FCM can be used beneficially in studies that trace the cognitive processes involved in, and identify problems related to, the comprehension of software applications. Based on our experience, we recommend that FCM be used in studies that have a large number of subjects and as a complement to other methods for tracing cognitive processes, such as user log files. We recommend a design with two groups (verbalization and silent control) and a pretest task to be used in studies with FCM or CTA that focus on performances.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Thomas, Richard}
}
@article {Simula.SE.663,
	title = {Experience With an Extensible Workspace for Analysis of Low-Level Usage Data},
	journal = {Software: Practice and Experience},
	volume = {39},
	number = {14},
	year = {2009},
	pages = {1185-1213},
	publisher = {John Wiley \& Sons},
	abstract = {Analysis of low-level usage data collected in empirical studies of user interaction is well known as a demanding task. Existing techniques for data collection and analysis are either application specific or data-driven. This paper presents a workspace for data cleaning, transformation and analysis of low-level usage data that we have developed and reports our experience with it. By its five-level architecture, the workspace makes a distinction between more general data that typically can be used in initial data analysis and the data answering a specific research question. The workspace was used in four studies and in total 6.5M user actions were collected from 238 participants. The collected data have been proven to be useful for: (i) validating solution times, (ii) validating process conformances, (iii) exploratory studies on program comprehension for understanding use of classes and documents and (iv) testing hypotheses on keystroke latencies. We have found workspace creation to be demanding in time. Particularly demanding were determining the context of actions and dealing with deficiencies. However, once these processes were understood, it was easy to reuse the workspace for different experiments and to extend it to answer new research questions. Based on our experience, we give a set of guidelines that might help in setting up studies, collecting and preparing data. We recommend that designers of data collection instruments add context to each action. Furthermore, we recommend rapid iterations starting early in the process of data preparation and analysis, and covering both general and specific data. Copyright \&copy; 2009 John Wiley \&amp; Sons, Ltd.},
	doi = {10.1002/spe.v39:14},
	author = {Thomas, Richard and Karahasanovic, Amela}
}
@article {Karahasanovic.2006.4,
	title = {Comprehension Strategies and Difficulties in Maintaining Object-Oriented Systems: an Explorative Study},
	journal = {Journal of Systems and Software},
	volume = {80},
	number = {9},
	year = {2007},
	month = {September},
	pages = {1541-1559},
	abstract = {Program comprehension is a major time-consuming activity in software maintenance. Understanding the underlying mechanisms of program comprehension is therefore necessary for improving software maintenance. It has been argued that acquiring knowledge of how a program works before modifying it (the systematic strategy) is unrealistic in larger programs. The goal of the experiment presented in this paper is to explore this claim. The experiment examines strategies for program comprehension and cognitive difficulties of developers who maintain an unfamiliar object-oriented system. The subjects were 38 students in their third or fourth year of study in computer science. They used a professional Java tool to perform several maintenance tasks on a medium-size Java application system in a six-hour long experiment. The results showed that the subjects who applied the systematic strategy were more likely to produce correct solutions. Two major groups of difficulties were related to the comprehension of the application structure, namely to the understanding of GUI implementation and OO comprehension and programming. Acquisition of strategic knowledge might improve program comprehension in software maintenance.},
	author = {Karahasanovic, Amela and Annette Kristin {Levine} and Thomas, Richard}
}
@misc {Simula.SE.119,
	title = {Data Collection and Analysis for New Usability},
	howpublished = {Invited talk at the Research methods workshop: Capturing user experience, Telenor R\&I, Oslo},
	year = {2007},
	note = {May 10th 2007, Oslo, FUSE, Telenor R\&I},
	month = {May 2007},
	author = {Karahasanovic, Amela}
}
@inproceedings {Karahasanovic.2007.1,
	title = {Difficulties Experienced by Students in Maintaining Object-Oriented Systems: an Empirical Study},
	journal = {Australasian Computing Education Conference ACE 2007},
	year = {2007},
	month = {December 2006},
	pages = {81-87},
	publisher = {Australian Computer Society Inc., Vol 66},
	type = {Conference},
	abstract = {It is widely accepted that software maintenance absorbs a significant amount of the effort expended in software development. Proper training of both university students and professional developers is required in order to improve software maintenance. Understanding cognitive difficulties the students have while maintaining objectoriented systems is a prerequisite for improving their university education and preparing them for jobs in industry. The goal of the experiment reported in this paper is to explore the difficulties of students who maintain an unfamiliar object-oriented system. The subjects were 34 students in their third year of study in computer science. They used a professional Java tool to perform several maintenance tasks on a medium-size Java application system in a seven-hour long experiment. The major difficulties were related to understanding program logic, algorithms, finding change impacts, and inheritance of the functionality. Based on these results we suggest teaching the basics of impact analysis and introducing examples of modifying larger object-oriented programs in courses on object-oriented programming.},
	isbn = {1-920-68247-3},
	author = {Karahasanovic, Amela and Thomas, Richard},
	editor = {Mann, S.}
}
@inbook {Karahasanovic.2006.1,
	title = {Challenges of Learning Object-Oriented Analysis and Design Through a Modelling-First Approach},
	booktitle = {Comprehensive Object-Oriented Learning. The Learner{\textquoteright}s Perspective},
	year = {2006},
	pages = {49-66},
	publisher = {Informing Science Press},
	organization = {Informing Science Press},
	chapter = {4},
	address = {California, USA},
	isbn = {83-922337-4-3},
	author = {Karahasanovic, Amela and Holmboe, C.},
	editor = {Fjuk, A. and Kaasb{\o}ll, J.}
}
@book {Karahasanovic.2006.3,
	title = {Comprehensive Object-Oriented Learning: the Learners Perspective},
	year = {2006},
	publisher = {Informing Science Press},
	address = {California, USA},
	abstract = {The object-oriented paradigm is increasingly being integrated in computer science education as well as in industry. There is a high demand for understanding the learner{\textquoteright}s actions, strategies and thoughts while solving object-oriented problems. Little is known about these processes. Aimed at both teachers and researchers involved in teaching object-oriented technologies, this book reports findings from four case studies, two design experiments and three controlled experiments, with a total of 187 subjects. It provides new insight into knowledge-acquiring processes and shows how to successfully integrate the empirically based findings into pedagogical design. {\textquotedblleft}The area addressed in this book is interesting. I haven{\textquoteright}t found many texts on this issue and there is clearly need for more knowledge. It is very useful to get new perspectives on problems in learning object-oriented concepts. Advices given in the book are concise and concrete{\textquotedblright} Svein Erik Bratsberg, Norwegian University of Science and Technology. {\textquotedblleft}The book, interestingly recommends that the object-oriented education starts with using tools that hide the compilation and execution steps. Only later may the learners switch to more professional tools. This approach should be considered in any introductory course on object-oriented programming.{\textquotedblright} Gerhard Skagestein, University of Oslo.},
	isbn = {83-922337-4-3},
	author = {Fjuk, Annita and Karahasanovic, Amela and Kaasb{\o}ll, Jens}
}
@inbook {Kaasboll.2006.1,
	title = {Improvements of Teaching and Tools for Learning Object-Orientation},
	booktitle = {Comprehensive Object-Oriented Learning: The Learner{\textquoteright}s Perspective},
	year = {2006},
	pages = {205-220},
	publisher = {Informing Science Press},
	organization = {Informing Science Press},
	chapter = {11},
	address = {California, USA},
	isbn = {83-922337-4-3},
	author = {Kaasb{\o}ll, Jens and Fjuk, A. and Karahasanovic, Amela and Groven, A-K},
	editor = {Karahasanovic A. {Fjuk} and Kaasb{\o}ll, J.}
}
@inbook {Karahasanovic.2006.2,
	title = {Revealing Object-Oriented Comprehension by Feedback Collection},
	booktitle = {Comprehensive Object-Oriented Learning. The Learner{\textquoteright}s Perspective},
	year = {2006},
	pages = {111-130},
	publisher = {Informing Science Press},
	organization = {Informing Science Press},
	chapter = {7},
	address = {California, USA},
	isbn = {83-922337-4-3},
	author = {Karahasanovic, Amela and Fjuk, A. and Sj{\o}berg, Dag Ingar Kondrup and Thomas, R.},
	editor = {Fjuk, A. and Kaasb{\o}ll, J.}
}
@article {Karahasanovic.2005.2,
	title = {Collecting Feedback During Software Engineering Experiments},
	journal = {Empirical Software Engineering},
	volume = {10},
	number = {2},
	year = {2005},
	pages = {113-147},
	abstract = {Objective: To improve the qualitative data obtained from software engineering experiments by gathering feedback during experiments. Rationale: Existing techniques for collecting quantitative and qualitative data from software engineering experiments do not provide sufficient information to validate or explain all our results. Therefore, we would like a cost-effective and unobtrusive method of collecting feedback from subjects during an experiment to augment other sources of data. Design of study: We formulated a set of qualitative questions that might be answered by collecting feedback during software engineering experiments. We then developed a tool to collect such feedback from experimental subjects. This feedback-collection tool was used in four different experiments and we evaluated the usefulness of the feedback obtained in the context of each experiment. The feedback data was triangulated with other sources of quantitative and qualitative data collected for the experiments. Results: We have demonstrated that the collection of feedback during experiments provides useful additional data to: validate the data obtained from other sources about solution times and quality of solutions; check process conformance; understand problem solving processes; identify problems with experiments; and understand subjects{\textquoteright} perception of experiments. Conclusions: Feedback collection has proved useful in four experiments and we intend to use the feedback-collection tool in a range of other experiments to further explore the cost-effectiveness and limitations of this technique. It is also necessary to carry out a systematic study to more fully understand the impact of the feedback-collecting tool on subjects{\textquoteright} performance in experiments.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Welland, Ray}
}
@article {Karahasanovic.2005.3,
	title = {Comprehension Strategies and Difficulties in Maintaining Object-Oriented Systems: an Explorative Study},
	journal = {Journal of Systems and Software},
	volume = {80},
	number = {11},
	year = {2005},
	pages = {1541-1559},
	author = {Karahasanovic, Amela and Annette Kristin {Levine} and Thomas, R.}
}
@misc {Karahasanovic.2005.6,
	title = {Empirical Studies of Object-Oriented Comprehension},
	howpublished = {Invited talk at the Australian Defence Institute, Perth, Western Australia},
	year = {2005},
	month = {23.8.2005.},
	author = {Karahasanovic, Amela}
}
@inproceedings {Thomas.2005.1,
	title = {An Investigation Into Keystroke Metrics As an Indicator of Programming Performance},
	journal = {Proceedings of the Australian Computing Education Conference 2005, Conferences in Research and Practice in Information Technology, Vol. 42, Newcastle, Australia, January 31-February 3},
	year = {2005},
	pages = {127-134},
	publisher = {Australian Computer Society, Inc},
	type = {Conference},
	abstract = {Typing has long been studied in psychology and HCI, and strong cognitive models for transcription typing exist. However work on keystroke latencies has not yielded a stand alone technique for identity authentication. We present the results from two very different experiments with computer science students. Keystroke timings were recorded while they worked on Ada or Java source code. Programming quality of their finished work was measured mainly in terms of completeness. In the controlled experiment, 39 students undertook two tasks over more than 2 hours to modify an application consisting of 6000 lines of Java. In the field study experiment, data was collected over 6 weeks from 141 students while they worked unsupervised on Ada programming in first year laboratories. In both cases there were highly significant (P=0.001), but weak, negative correlations between speed and coding performance. With additional development, these techniques may have promise for user modelling and assessment as well as in educational diagnostics.},
	isbn = {1-920682-24-4},
	author = {Thomas, Richard and Karahasanovic, Amela and Kennedy, Gregor}
}
@misc {Karahasanovic.2005.5,
	title = {Object-Oriented Comprehension During Software Maintenance},
	howpublished = {Research Seminar at the Department of Computer Science and Software Engineering, University of Western Australia, Perth},
	year = {2005},
	note = {19.8.2005},
	author = {Karahasanovic, Amela}
}
@misc {Karahasanovic.2005.9,
	title = {Research, People and Economic Growth},
	howpublished = {Invited talk at the Conference Economic Growth and Diversity, Drammen},
	year = {2005},
	month = {17.2.2005.},
	author = {Karahasanovic, Amela}
}
@article {Sjoberg.2005.1,
	title = {A Survey of Controlled Experiments in Software Engineering},
	journal = {IEEE Transactions on Software Engineering},
	volume = {31},
	number = {9},
	year = {2005},
	month = {September},
	pages = {733-753},
	abstract = {The classical method for identifying cause-effect relationships is to conduct controlled experiments. This paper reports upon the present state of how controlled experiments in software engineering are conducted and the extent to which relevant information is reported. Among the 5,453 scientific articles published in 12 leading software engineering journals and conferences in the decade from 1993 to 2002, 103 articles (1.9 percent) reported controlled experiments in which individuals or teams performed one or more software engineering tasks. This survey quantitatively characterizes the topics of the experiments and their subjects (number of subjects, students versus professionals, recruitment, and rewards for participation), tasks (type of task, duration, and type and size of application) and environments (location, development tools). Furthermore, the survey reports on how internal and external validity is addressed and the extent to which experiments are replicated. The gathered data reflects the relevance of software engineering experiments to industrial practice and the scientific maturity of software engineering research.},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Jo Erskine {Hannay} and Hansen, Ove and Vigdis By {Kampenes} and Karahasanovic, Amela and Liborg, Nils-Kristian and Anette C. {Rekdal}}
}
@techreport {Karahasanovic.2004.1,
	title = {A Controlled Experiment to Evaluate Feedback-Collection Tool in Software Engineering Research, Technical Report 2004-8, Simula Research Laboratory},
	number = {2004-8},
	year = {2004},
	note = {Technical Report 2004-8, Simula Research Laboratory},
	publisher = {Simula Research Laboratory},
	abstract = {Objective: To compare the feedback-collection method with think-aloud methods regarding effects on subjects{\textquoteright} performance and type of collected information in software engineering experiments. Rationale: Think-aloud methods provide a rich set of data about experiments but are time-consuming in data collection. Alternatively, subjects can be interrupted during experiments and asked for written feedback. However, there has been no systematic comparison of these methods. Design of study: We conducted a controlled experiment to evaluate three methods for qualitative data collection: feedback-collection, concurrent think-aloud and retrospective think-aloud. Results: There is no significant difference in subjects{\textquoteright} performance for the three data collection methods. The types of information gathered by these three methods differed; in particular, the feedback collection method provided more explanations about the problem-solving process and more information about subjects{\textquoteright} perceptions of the experiment. Conclusions: This research confirms that the feedback-collection method is an appropriate data collection method in software engineering experiments.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Annette Kristin {Levine} and Thomas, Richard}
}
@inproceedings {SE.5.Karahasanovic.2004,
	title = {A Controlled Experiment to Evaluate the Reactivity and Usefulness of the Think-Aloud Tool},
	journal = {Information Resources Management Association International Conference (IRMA{\textquoteright}04), Human Computer Interaction Track},
	year = {2004},
	month = {May 23-26},
	pages = {1033-1034},
	publisher = {Idea Group Publishing},
	type = {1},
	address = {New Orleans, Louisiana, USA},
	abstract = {This paper presents ongoing work on the evaluation of a tool for collecting feedback in large-scale software engineering and program comprehension experiments. Two most important questions one can ask about a new verbal procedure are whether it is useful and whether it is reactive. We have developed a tool, called the think-aloud tool, to support the think-aloud method in a variety of experiments. This tool instructs the subjects at regular intervals to write down their thoughts on a web based screen, as opposed to the oral expression of thoughts in conventional think-aloud method. The tool collects the subjects\&$\#$39; feedback together with timestamps. This paper describes an experiment designed to investigate whether the think-aloud tool affects performance of the subjects and whether it collects information beyond that collected in traditional think-aloud and retrospective reports. Furthermore, we will conduct a qualitative comparison of mouse and keying rates with think-aloud and retrospective reports. In addition, the experiment aims to check for any effect from using the think-aloud tool on the subject\&$\#$39;s task-related understanding and problem solving.},
	author = {Karahasanovic, Amela and Fjuk, A. and Sj{\o}berg, Dag Ingar Kondrup and Thomas, R.}
}
@misc {SE.8.Joergensen.2004.p,
	title = {A Controlled Experiment to Evaluate the Reactivity and Usefulness of the Think-Aloud Tool},
	howpublished = {Presentation at the Oslo University College},
	year = {2004},
	note = {20 deltakere. Arrang{\o}r: HiO},
	month = {8. juni},
	author = {Karahasanovic, Amela and U. N. {Hinkel}}
}
@misc {SE.8.Karahasanovic.2004.b,
	title = {A Think-Aloud Tool As a Means for Understanding Processes of Programming and Process Evaluation},
	howpublished = {Presentation at the COOL seminar},
	year = {2004},
	note = {10-15 deltakere. Arrang{\o}r: Intermedia},
	month = {17. februar},
	author = {Karahasanovic, Amela}
}
@inbook {Sjoberg.2003.1,
	title = {Challenges and Recommendations When Increasing the Realism of Controlled Software Engineering Experiments},
	booktitle = {Empirical Methods and Studies in Software Engineering: Experiences from ESERNET},
	series = {Lecture Notes in Computer Science, Volume 2765},
	year = {2003},
	pages = {24-38},
	publisher = {Springer},
	organization = {Springer},
	chapter = {Part II: Method Chapters},
	address = {Berlin / Heidelberg},
	abstract = {An important goal of most empirical software engineering experiments is the transfer of the research results to industrial applications. To convince industry about the validity and applicability of the results of controlled software engineering experiments, the tasks, subjects and the environments should be as realistic as practically possible. Such experiments are, however, more demanding and expensive than experiments involving students, small tasks and pen-and-paper environments. This chapter describes challenges of increasing the realism of controlled experiments and lessons learned from the experiments that have been conducted at Simula Research Laboratory.},
	isbn = {978-3-540-40672-3},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Vok{\'a}c, Marek},
	editor = {Conradi, Reidar and Alf Inge {Wang}}
}
@inproceedings {SE.5.Karahasanovic.2003,
	title = {Is It Ethical to Log Users{\textquoteright} Actions in Software Engineering Experiments?},
	journal = {Informing Science and Information Technology Education Joint Conference},
	year = {2003},
	month = {June 24-27},
	pages = {1211-1214},
	publisher = {The Informing Science Institute, Santa Rosa, CA, USA},
	type = {1},
	address = {Pori, Finland},
	abstract = {In recent years, there has been a significant increase of empirical studies involving human participants in software engineering. Such studies heightened the need for considering ethical issues. Researchers have not only moral but also pragmatic reasons to treat participants of their studies ethically. They want to maintain their access to the data source and to get funding for their research. Thus the researchers have to be aware of general ethical issues and those that are specific for a particular research practice. This paper briefly describes our experiences with a tool that logs all actions of the users/participants (commands, keystrokes and mouse movements), and collects their comments in a web based screen. The tool was used in two usability studies and raised ethical issues concerning assuring confidentiality and minimization of inconvenience. The paper also discusses some conflicts between ethical standards and assuring validity of the studies that appeared during these studies.},
	author = {Karahasanovic, Amela}
}
@techreport {Karahasanovic.2003.3,
	title = {A Think-Aloud Support Tool for Collecting Feedback in Large-Scale Software Engineering Experiments},
	number = {2003-7},
	year = {2003},
	publisher = {Simula Research Laboratory},
	abstract = {The think-aloud method has proven useful for studying how a technology is used. However, it is labour-intensive and makes demands on resources for studies with many subjects. This paper describes a tool, called the think-aloud tool, that we have developed to support the collection of feedback from subjects in large-scale software engineering experiments. This tool instructs the subjects, at regular intervals, to write down their thoughts on a web-based screen and collects their feedback. An empirical evaluation of this tool is reported. The results from four controlled experiments suggest that the think-aloud tool is a valuable means for increasing the quality of large-scale software engineering experiments. The results also indicate that the participants were mostly positive to the tool and had no significant problems in verbalising their thoughts. The tool we proposed adds to an available set of tools for data collection in software engineering experiments. Whereas we have experience with the think-aloud tool in software engineering experiments, there is also potential for using such a tool in other areas in which the think-aloud method is used in studies involving a computer system.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@misc {SE.7.Karahasanovic.2003,
	title = {Think-Aloud Tool As a Means for Getting Insights Into Learning Object-Oriented Concepts},
	howpublished = {Extended abstract, Workshop on Learning and Teaching Object-Orientation -- Scandinavian Perspectives},
	year = {2003},
	month = {October},
	abstract = {The increasing need for getting insights into learning object-oriented concepts raises the challenge of adequate methods and tools for data collection. We have developed a tool, called the think-aloud tool (Karahasanovic et al., 2003), to support the think-aloud method in a variety of studies with large number of subjects. This tool instructs the subjects at regular intervals to write down their thoughts on a web based screen, as opposed to the oral expression of thoughts in conventional think-aloud method. The tool collects the subjects{\textquoteright} feedback together with timestamps. Whereas we have experience with the think-aloud tool in software engineering experiments, there is also potential for using such a tool in studies exploring how well the students understand the object-oriented concepts. We believe that the tool is particularly useful for collecting qualitative data about subjects in large-scale studies. Comments collected by the think-aloud tool may help us to identify problems that students have in understanding object-oriented concepts. They may also help us to get insight into learning processes. However, the think-aloud tool provides a less complete picture of learning activities than does the think-aloud method with a human observer and should therefore be used as complementary to other data collection methods or in explorative phases of a study. The think-aloud tool may also affect the student{\textquoteright}s task-related understanding and problem solving. As writing comments on the think-aloud screen demands explanations and reflections it may improve student{\textquoteright}s learning and understanding. We plan to conduct an experiment designed to investigate usefulness of the think-aloud tool as a mean for getting insight into learning activities. 56 third year students of Oslo College will be asked to conduct change task on a medium-size Java application (6000 LOC). The students will be divided into four groups. Assignment to the respective groups will be done at random. The first group will be asked to think aloud, the second to give immediate retrospective reports, the third to use the think-aloud tool and the fourth to work without giving feedback - the silent condition. For the third group, the think-aloud screen will appear every 15 minutes with the text {\textquotedblleft}What are you thinking now?{\textquotedblright} The subjects will be instructed to describe what they were thinking just before the screen appeared. The time available for writing comments will be limited to two minutes. The experiment will last six hours and data will be collected by the think-aloud tool and the classical think-aloud method. The students will be interviewed after the experiment.},
	author = {Karahasanovic, Amela}
}
@inproceedings {SE.5.Sjoeberg.2002,
	title = {Conducting Realistic Experiments in Software Engineering},
	journal = {ISESE{\textquoteright}2002 (First International Symposium on Empirical Software Engineering)},
	year = {2002},
	month = {October 3-4},
	pages = {17-26},
	publisher = {IEEE Computer Society},
	type = {1},
	abstract = {An important goal of most empirical software engineering research is the transfer of research results to industrial applications. Two important obstacles for this transfer are the lack of control of variables of case studies, i.e., the lack of explanatory power, and the lack of realism of controlled experiments. While it may be difficult to increase the explanatory power of case studies, there is a large potential for increasing the realism of controlled software engineering experiments. To convince industry about the validity and applicability of the experimental results, the tasks, subjects and the environments of the experiments should be as realistic as practically possible. Such experiments are, however, more expensive than experiments involving students, small tasks and pen-and-paper environments. Consequently, a change towards more realistic experiments requires a change in the amount of resources spent on software engineering experiments. This paper argues that software engineering researchers should apply for resources enabling expensive and realistic software engineering experiments similar to how other researchers apply for resources for expensive software and hardware that are necessary for their research. The paper describes some of the experiences from recent experiments that varied in size from involving one software professional for 5 days to 130 software professionals, from 9 consultancy companies, for one day each.},
	isbn = {0-7695-1796-X},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Koren, E. and Vok{\'a}c, Marek},
	editor = {found, not}
}
@phdthesis {SE.3.Karahasanovic.2002,
	title = {Supporting Application Consistency in Evolving Object-Oriented Systems by Impact Analysis and Visualisation},
	year = {2002},
	note = {ISSN 1501-7710, Nr. 234, Unipub AS},
	month = {May},
	school = {University of Oslo},
	type = {phd},
	abstract = {There is a growing number of object-oriented systems within areas such as CAD/CAM, software engineering, office automation and multimedia management where changes are rather a rule than an exception. Application systems in these areas are supposed to be easy to change due to encapsulation and inheritance. On the other hand, the transitivity of inheritance and aggregation structures makes it difficult to detect dependencies between classes, methods and fields of a system, which in turn makes it more difficult to change the system. The research presented in this thesis demonstrates that identifying and visualising impacts of changes in evolving object-oriented systems is a step towards improving the process of maintaining application consistency in such systems. A technology that identifies and visualises such impacts has been developed and evaluated. This technology helps discover dependencies and thus support maintaining application consistency. It is based on a component-based model of object-oriented systems and an improved version of a transitive closure algorithm. This technology identifies impacts of complex changes, i.e., changes involving several classes like merge and split. Furthermore, a visual language allows displaying relatively large amount of objects, both at a coarse level of granularity (packages, classes and interfaces) and at a fine level of granularity (fields and methods). Several empirical studies focusing on the usability of the proposed technology have been conducted. The results of these studies show that visualising impacts of changes improves the effectiveness of developers when conducting changes. Furthermore, the higher precision of impact analysis achieved by identifying impacts at the finer level of granularity (fields and methods) reduces the number of errors and the time needed to conduct changes. Identifying impacts of complex changes (merge, split) improves effectiveness. Empirical studies are a prerequisite for usability evaluation of any technology, including those supporting application consistency. Such studies, in turn, require efficient and reliable data collection. A tool for automatic data collection during software engineering experiments has been developed. This tool automatically collects data about the subjects and their interaction with the software technology under study. A {{\textquoteleft}}think-aloud{\textquoteright} screen was proposed as a means for collecting subjective information. High-level subjective data sources (the think-aloud screen and evaluation screen) are combined with a low-level objective data sources (user command logs). It has been demonstrated that this combination may increase the validity of the conducted studies.},
	author = {Karahasanovic, Amela}
}
@inproceedings {SE.5.Karahasanovic.2002,
	title = {Visualising Impacts of Change in Evolving Object-Oriented Systems: an Explorative Study},
	journal = {Proceedings of the International Workshop on Graph-Based Tools GraBaTs{\textquoteright}02},
	year = {2002},
	month = {October 7-8},
	pages = {22-31},
	publisher = {Technische Universitat Berlin},
	type = {1},
	address = {Barcelona, Spain},
	author = {Karahasanovic, Amela and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Karahasanovic.2001.c,
	title = {Data Collection in Software Engineering Experiments},
	journal = {Managing Information Technology in a Global Economy, Information Resources Management Association International Conference IRMA 2001, Software Engineering Track},
	year = {2001},
	month = {May 20 - 23},
	pages = {1027-1028},
	publisher = {Idea Group Publishing},
	type = {Conference},
	address = {Toronto, Ontario Canada},
	abstract = {This paper presents ongoing work on developing a mechanism for automatic data collection during software engineering experiments. The importance of experiments in software engineering can hardly be overemphasised. Conducting such experiments raises the challenge of efficient and reliable data collection and analysis. During the experiment subjects (students, experienced programmers, etc.) perform tasks on the software engineering artefacts (e.g., design documents and code) using the technology under study in the given context (e.g. operating system). We are typically interested in data concerning subjects, their interaction with the technology and context, and changes of the software engineering artefacts. To conduct a series of experiments on software engineering technology (methods and tools), we envisage an experimental environment for data collection and analysis. As a first step, we have developed a logging mechanism that collects data about subjects of the experiment and their usage of the technology under study. We believe that our logging mechanism will alleviate data collection and increase the validity of experiments.},
	author = {Karahasanovic, Amela and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Karahasanovic.2001.b,
	title = {Identifying Impacts of Database Schema Changes on Application},
	journal = {Proceedings of the 8th Doctoral Consortium at the CAiSE*01},
	year = {2001},
	note = {Technical Report No B 01-04, Institute of Computer Science},
	month = {June 4th-5th},
	pages = {93-104},
	publisher = {Freie Universit{\"a}t Berlin},
	type = {1},
	address = {Interlaken, Switzerland},
	author = {Karahasanovic, Amela},
	editor = {Hinze, A. and Herden, O.}
}
@inproceedings {SE.5.Karahasanovic.2001.a,
	title = {Visualizing Impacts of Database Schema Changes - a Controlled Experiment},
	journal = {2001 IEEE Symposium on Visual/Multimedia Approaches to Programming and Software Engineering},
	year = {2001},
	month = {September},
	pages = {358-365},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Stresa, Italy},
	abstract = {Research in schema evolution has been driven by the need for more effective software development and maintenance. Finding impacts of schema changes on the applications and presenting them in an appropriate way are particularly challenging. We have developed a tool that finds impacts of schema changes on applications in object-oriented systems. This tool displays components (packages, classes, interfaces, methods and fields) of a database application system as a graph. Components potentially affected by a change are indicated by changing the shape of the boxes representing those components. Two versions of the tool are available. One version identifies affected parts of applications at the granularity of packages, classes, and interfaces, whereas the other version identifies affected parts at the finer granularity of fields and methods. This paper presents the design and results of a controlled student experiment testing these two granularity levels with respect to productivity and user satisfaction. There are indications that identifying impacts at the finer granularity can reduce the time needed to conduct schema changes and reduce the number of errors. Our results also show that the subjects of the experiment appreciated the idea of visualizing the impacts of schema changes.},
	isbn = {0-7695-0474-4},
	author = {Karahasanovic, Amela and Sj{\o}berg, Dag Ingar Kondrup}
}
@misc {SE.7.Karahasanovic.2000.b,
	title = {Finding Impacts of Schema Changes},
	howpublished = {17th British National Conference on Databases, BNCOD 17},
	year = {2000},
	note = {Poster session},
	month = {July},
	author = {Karahasanovic, Amela}
}
@inproceedings {SE.5.Karahasanovic.2000.a,
	title = {SEMT - a Tool for Finding Impacts of Schema Changes},
	journal = {Proceedings of NWPER{\textquoteright}2000 Nordic Workshop on Programming Environment Research},
	year = {2000},
	note = {Research Report 283, Department of Informatics, University of Oslo, ISBN 82-7368-231-5, ISSN 0806-3036, Research Report, Department of Information Science, University of Bergen},
	month = {May 28-30},
	pages = {60-75},
	type = {1},
	address = {Lillehammer, Norway},
	author = {Karahasanovic, Amela}
}
@inproceedings {SE.5.Karahasanovic.1999,
	title = {Supporting Database Schema Evolution by Impact Analysis},
	journal = {Proceedings of Norwegian Conference in Informatics NIK{\textquoteright}99},
	year = {1999},
	month = {November 16-17,},
	pages = {303-314},
	publisher = {TAPIR},
	type = {Conference},
	address = {Trondheim, Norway},
	author = {Karahasanovic, Amela and Sj{\o}berg, Dag Ingar Kondrup}
}
