@article {Karahasanovic.2006.5,
	title = {Comparing of Feedback Collection and Think-Aloud Methods in Program Comprehension Studies},
	journal = {Behaviour \& Information Technology},
	volume = {28},
	number = {2},
	year = {2009},
	month = {March-April},
	pages = {139-164},
	publisher = {The University of Western Australia},
	abstract = {This paper reports an explorative experimental comparison of (i) an experience-sampling method called feedback collection and (ii) the think-aloud methods with respect to their usefulness in studies on program comprehension. Think-aloud methods are widely used in studies of cognitive processes, including program comprehension. Alternatively, as in the feedback-collection method (FCM), cognitive processes can be traced by collecting written feedback from the subjects at regular intervals. We compare FCM with concurrent think-aloud (CTA) and retrospective think-aloud (RTA) regarding type and usefulness of the collected information, costs related to analysis of the collected information, and effects of the data collection methods on the subjects\&$\#$39; performance. FCM allowed us to identify a greater number of comprehension problems that prevented progress or caused significant delay (FCM: 30 problems; CTA: 5; RTA: 15). It was less precise in identifying strategies for comprehension than CTA (92\% correctness for FCM; 100\% for CTA). FCM was less expensive in analysis (transcription and coding) than the other two methods (FCM: 0,7 hours of analysis per protocol; CTA: 31 hours; RTA: 7,9 hours). The results indicate that all three methods of data collection were intrusive and affected the performance of the subjects with respect to time and correctness (small to medium effect size). This research confirms that FCM can be used beneficially in studies that trace the cognitive processes involved in, and identify problems related to, the comprehension of software applications. Based on our experience, we recommend that FCM be used in studies that have a large number of subjects and as a complement to other methods for tracing cognitive processes, such as user log files. We recommend a design with two groups (verbalization and silent control) and a pretest task to be used in studies with FCM or CTA that focus on performances.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Thomas, Richard}
}
@article {Simula.SE.663,
	title = {Experience With an Extensible Workspace for Analysis of Low-Level Usage Data},
	journal = {Software: Practice and Experience},
	volume = {39},
	number = {14},
	year = {2009},
	pages = {1185-1213},
	publisher = {John Wiley \& Sons},
	abstract = {Analysis of low-level usage data collected in empirical studies of user interaction is well known as a demanding task. Existing techniques for data collection and analysis are either application specific or data-driven. This paper presents a workspace for data cleaning, transformation and analysis of low-level usage data that we have developed and reports our experience with it. By its five-level architecture, the workspace makes a distinction between more general data that typically can be used in initial data analysis and the data answering a specific research question. The workspace was used in four studies and in total 6.5M user actions were collected from 238 participants. The collected data have been proven to be useful for: (i) validating solution times, (ii) validating process conformances, (iii) exploratory studies on program comprehension for understanding use of classes and documents and (iv) testing hypotheses on keystroke latencies. We have found workspace creation to be demanding in time. Particularly demanding were determining the context of actions and dealing with deficiencies. However, once these processes were understood, it was easy to reuse the workspace for different experiments and to extend it to answer new research questions. Based on our experience, we give a set of guidelines that might help in setting up studies, collecting and preparing data. We recommend that designers of data collection instruments add context to each action. Furthermore, we recommend rapid iterations starting early in the process of data preparation and analysis, and covering both general and specific data. Copyright \&copy; 2009 John Wiley \&amp; Sons, Ltd.},
	doi = {10.1002/spe.v39:14},
	author = {Thomas, Richard and Karahasanovic, Amela}
}
@article {Karahasanovic.2006.4,
	title = {Comprehension Strategies and Difficulties in Maintaining Object-Oriented Systems: an Explorative Study},
	journal = {Journal of Systems and Software},
	volume = {80},
	number = {9},
	year = {2007},
	month = {September},
	pages = {1541-1559},
	abstract = {Program comprehension is a major time-consuming activity in software maintenance. Understanding the underlying mechanisms of program comprehension is therefore necessary for improving software maintenance. It has been argued that acquiring knowledge of how a program works before modifying it (the systematic strategy) is unrealistic in larger programs. The goal of the experiment presented in this paper is to explore this claim. The experiment examines strategies for program comprehension and cognitive difficulties of developers who maintain an unfamiliar object-oriented system. The subjects were 38 students in their third or fourth year of study in computer science. They used a professional Java tool to perform several maintenance tasks on a medium-size Java application system in a six-hour long experiment. The results showed that the subjects who applied the systematic strategy were more likely to produce correct solutions. Two major groups of difficulties were related to the comprehension of the application structure, namely to the understanding of GUI implementation and OO comprehension and programming. Acquisition of strategic knowledge might improve program comprehension in software maintenance.},
	author = {Karahasanovic, Amela and Annette Kristin {Levine} and Thomas, Richard}
}
@inproceedings {Karahasanovic.2007.1,
	title = {Difficulties Experienced by Students in Maintaining Object-Oriented Systems: an Empirical Study},
	journal = {Australasian Computing Education Conference ACE 2007},
	year = {2007},
	month = {December 2006},
	pages = {81-87},
	publisher = {Australian Computer Society Inc., Vol 66},
	type = {Conference},
	abstract = {It is widely accepted that software maintenance absorbs a significant amount of the effort expended in software development. Proper training of both university students and professional developers is required in order to improve software maintenance. Understanding cognitive difficulties the students have while maintaining objectoriented systems is a prerequisite for improving their university education and preparing them for jobs in industry. The goal of the experiment reported in this paper is to explore the difficulties of students who maintain an unfamiliar object-oriented system. The subjects were 34 students in their third year of study in computer science. They used a professional Java tool to perform several maintenance tasks on a medium-size Java application system in a seven-hour long experiment. The major difficulties were related to understanding program logic, algorithms, finding change impacts, and inheritance of the functionality. Based on these results we suggest teaching the basics of impact analysis and introducing examples of modifying larger object-oriented programs in courses on object-oriented programming.},
	isbn = {1-920-68247-3},
	author = {Karahasanovic, Amela and Thomas, Richard},
	editor = {Mann, S.}
}
@inproceedings {Thomas.2005.1,
	title = {An Investigation Into Keystroke Metrics As an Indicator of Programming Performance},
	journal = {Proceedings of the Australian Computing Education Conference 2005, Conferences in Research and Practice in Information Technology, Vol. 42, Newcastle, Australia, January 31-February 3},
	year = {2005},
	pages = {127-134},
	publisher = {Australian Computer Society, Inc},
	type = {Conference},
	abstract = {Typing has long been studied in psychology and HCI, and strong cognitive models for transcription typing exist. However work on keystroke latencies has not yielded a stand alone technique for identity authentication. We present the results from two very different experiments with computer science students. Keystroke timings were recorded while they worked on Ada or Java source code. Programming quality of their finished work was measured mainly in terms of completeness. In the controlled experiment, 39 students undertook two tasks over more than 2 hours to modify an application consisting of 6000 lines of Java. In the field study experiment, data was collected over 6 weeks from 141 students while they worked unsupervised on Ada programming in first year laboratories. In both cases there were highly significant (P=0.001), but weak, negative correlations between speed and coding performance. With additional development, these techniques may have promise for user modelling and assessment as well as in educational diagnostics.},
	isbn = {1-920682-24-4},
	author = {Thomas, Richard and Karahasanovic, Amela and Kennedy, Gregor}
}
@techreport {Karahasanovic.2004.1,
	title = {A Controlled Experiment to Evaluate Feedback-Collection Tool in Software Engineering Research, Technical Report 2004-8, Simula Research Laboratory},
	number = {2004-8},
	year = {2004},
	note = {Technical Report 2004-8, Simula Research Laboratory},
	publisher = {Simula Research Laboratory},
	abstract = {Objective: To compare the feedback-collection method with think-aloud methods regarding effects on subjects{\textquoteright} performance and type of collected information in software engineering experiments. Rationale: Think-aloud methods provide a rich set of data about experiments but are time-consuming in data collection. Alternatively, subjects can be interrupted during experiments and asked for written feedback. However, there has been no systematic comparison of these methods. Design of study: We conducted a controlled experiment to evaluate three methods for qualitative data collection: feedback-collection, concurrent think-aloud and retrospective think-aloud. Results: There is no significant difference in subjects{\textquoteright} performance for the three data collection methods. The types of information gathered by these three methods differed; in particular, the feedback collection method provided more explanations about the problem-solving process and more information about subjects{\textquoteright} perceptions of the experiment. Conclusions: This research confirms that the feedback-collection method is an appropriate data collection method in software engineering experiments.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Annette Kristin {Levine} and Thomas, Richard}
}
