@article {28156,
	title = {When should we (not) use the mean magnitude of relative error (MMRE) as an error measure in software development effort estimation?},
	journal = {Information and Software Technology},
	volume = {143},
	year = {2022},
	month = {03/2022},
	pages = {106784},
	publisher = {Elsevier},
	abstract = {Context: The mean magnitude of relative error (MMRE) is an error measure frequently used to evaluate and compare the estimation performance of prediction models and software professionals.Objective: This paper examines conditions for proper use of MMRE in effort estimation contexts.Method: We apply research on scoring functions to identify the type of estimates that minimizes the expected value of the MMRE.Results: We show that the MMRE is a proper error measure for estimates of the most likely (mode) effort, but not for estimates of the median or mean effort, provided that the effort usage is approximately log-normally distributed, which we argue is a reasonable assumption in many software development contexts. The relevance of the findings is demonstrated on real-world software development data.Conclusion: MMRE is not a proper measure of the accuracy of estimates of the median or mean effort, but may be used for the accuracy evaluation of estimates of most likely effort.},
	author = {J{\o}rgensen, Magne and Halkjelsvik, Torleif and Liest{\o}l, Knut}
}
@inproceedings {42532,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill },
	journal = {ESEC/FSE},
	year = {2021},
	publisher = {Journal First publication (IEEE TSE)},
	type = {Journal first},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut}
}
@inproceedings {27934,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill (A Journal first conference publication)},
	journal = {CM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE)},
	year = {2021},
	publisher = {ACM},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut},
	editor = {Apel, Sven}
}
@article {26849,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill},
	journal = {Transactions on Software Engineering},
	year = {2020},
	month = {February 2020},
	publisher = {IEEE},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut}
}
@article {Simula.simula.2758,
	title = {Incorrect Results in Software Engineering Experiments: How to Improve Research Practices},
	journal = {Journal of Systems and Software},
	volume = {117},
	year = {2016},
	pages = {274-281},
	publisher = {Elsevier},
	abstract = {Context: The trustworthiness of research results is a growing concern in many empirical disciplines. Aim: The goals of this paper are to assess how much the trustworthiness of results reported in software engineering experiments is affected by researcher and publication bias and to suggest improved research practices. Method: First, we conducted a small-scale survey to document the presence of researcher and publication biases in software engineering experiments. Then, we built a model that estimates the proportion of correct results for different levels of researcher and publication bias. A review of 150 randomly selected software engineering experiments published in the period 2002-2012 was conducted to provide input to the model. Results: The survey indicates that researcher and publication bias is quite common. This finding is supported by the observation that the actual proportion of statistically significant results reported in the reviewed papers was about twice as high as the one expected assuming no researcher and publication bias. Our models suggest a high proportion of incorrect results even with quite conservative assumptions. Conclusion: Research practices must improve to increase the trustworthiness of software engineering experiments. A key to this improvement is to avoid conducting studies with unsatisfactory low statistical power.},
	author = {J{\o}rgensen, Magne and Dyb{\r a}, Tore and Liest{\o}l, Knut and Sj{\o}berg, Dag Ingar Kondrup}
}
