@article {Karahasanovic.2006.5,
	title = {Comparing of Feedback Collection and Think-Aloud Methods in Program Comprehension Studies},
	journal = {Behaviour \& Information Technology},
	volume = {28},
	number = {2},
	year = {2009},
	month = {March-April},
	pages = {139-164},
	publisher = {The University of Western Australia},
	abstract = {This paper reports an explorative experimental comparison of (i) an experience-sampling method called feedback collection and (ii) the think-aloud methods with respect to their usefulness in studies on program comprehension. Think-aloud methods are widely used in studies of cognitive processes, including program comprehension. Alternatively, as in the feedback-collection method (FCM), cognitive processes can be traced by collecting written feedback from the subjects at regular intervals. We compare FCM with concurrent think-aloud (CTA) and retrospective think-aloud (RTA) regarding type and usefulness of the collected information, costs related to analysis of the collected information, and effects of the data collection methods on the subjects\&$\#$39; performance. FCM allowed us to identify a greater number of comprehension problems that prevented progress or caused significant delay (FCM: 30 problems; CTA: 5; RTA: 15). It was less precise in identifying strategies for comprehension than CTA (92\% correctness for FCM; 100\% for CTA). FCM was less expensive in analysis (transcription and coding) than the other two methods (FCM: 0,7 hours of analysis per protocol; CTA: 31 hours; RTA: 7,9 hours). The results indicate that all three methods of data collection were intrusive and affected the performance of the subjects with respect to time and correctness (small to medium effect size). This research confirms that FCM can be used beneficially in studies that trace the cognitive processes involved in, and identify problems related to, the comprehension of software applications. Based on our experience, we recommend that FCM be used in studies that have a large number of subjects and as a complement to other methods for tracing cognitive processes, such as user log files. We recommend a design with two groups (verbalization and silent control) and a pretest task to be used in studies with FCM or CTA that focus on performances.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Thomas, Richard}
}
@mastersthesis {Hinkel.2005.1,
	title = {Evaluating Methods for Data Collection During Software Engineering Experiments},
	year = {2005},
	note = {Supervisors Amela Karahasanovic and Dag Sj{\o}berg},
	month = {March},
	publisher = {Department of Informatics, University of Oslo},
	type = {masters},
	author = {Unni Nyhamar {Hinkel}}
}
@techreport {Karahasanovic.2004.1,
	title = {A Controlled Experiment to Evaluate Feedback-Collection Tool in Software Engineering Research, Technical Report 2004-8, Simula Research Laboratory},
	number = {2004-8},
	year = {2004},
	note = {Technical Report 2004-8, Simula Research Laboratory},
	publisher = {Simula Research Laboratory},
	abstract = {Objective: To compare the feedback-collection method with think-aloud methods regarding effects on subjects{\textquoteright} performance and type of collected information in software engineering experiments. Rationale: Think-aloud methods provide a rich set of data about experiments but are time-consuming in data collection. Alternatively, subjects can be interrupted during experiments and asked for written feedback. However, there has been no systematic comparison of these methods. Design of study: We conducted a controlled experiment to evaluate three methods for qualitative data collection: feedback-collection, concurrent think-aloud and retrospective think-aloud. Results: There is no significant difference in subjects{\textquoteright} performance for the three data collection methods. The types of information gathered by these three methods differed; in particular, the feedback collection method provided more explanations about the problem-solving process and more information about subjects{\textquoteright} perceptions of the experiment. Conclusions: This research confirms that the feedback-collection method is an appropriate data collection method in software engineering experiments.},
	author = {Karahasanovic, Amela and Unni Nyhamar {Hinkel} and Sj{\o}berg, Dag Ingar Kondrup and Annette Kristin {Levine} and Thomas, Richard}
}
