@article {28040,
	title = {Backsourcing of Information Technology - A Systematic Literature Review},
	journal = {Submitted to a journal},
	year = {2023},
	publisher = {x},
	abstract = {Context:\&nbsp;Backsourcing is the process of insourcing previously outsourced activities. When companies experience environmental or strategic changes, or challenges with outsourcing, backsourcing can be a viable alternative. While outsourcing and related processes have been extensively studied in information technology, few studies report experiences with backsourcing.Objectives:\&nbsp;We intend to summarize the results of the research literature on the backsourcing of IT, with a focus on software development. By identifying practical relevance experience, we aim to present findings that may help companies considering backsourcing. In addition, we aim to identify gaps in the current research literature and point out areas for future work.Method: Our systematic literature review (SLR) started with a search for empirical studies on the backsourcing of IT. From each study we identified the contexts in which backsourcing occurs, the factors leading to the decision to backsource, the backsourcing process itself, and the outcomes of backsourcing. We employed inductive coding to extract textual data from the papers identified and qualitative cross-case analysis to synthesize the evidence from backsourcing experiences.Results:\&nbsp;We identified 17 papers that reported 26 cases of backsourcing, six of which were related to software development. The cases came from a variety of contexts. The most common reasons for backsourcing were improving quality, reducing costs, and regaining control of outsourced activities. The backsourcing process can be described as containing five sub-processes: change management, vendor relationship management, competence building, organizational build-up, and transfer of ownership. Furthermore, we identified 14 positive outcomes and nine negative outcomes of backsourcing. Finally, we aggregated the evidence and detailed three relationships of potential use to companies considering backsourcing.Conclusion:\&nbsp;The backsourcing of IT is a complex process; its implementation depends on the prior outsourcing relationship and other contextual factors. Our systematic literature review may contribute to a better understanding of this process by identifying its components and their relationships based on the peer-reviewed literature. Our results may also serve as a motivation and baseline for further research on backsourcing and may provide guidelines and process fragments from which practitioners can benefit when they engage in backsourcing.},
	keywords = {backshoring, Backsourcing, information technology, software development, Software Engineering, systematic literature review},
	author = {Jefferson Seide {Moll{\'e}ri} and Lassenius, Casper and J{\o}rgensen, Magne}
}
@article {43186,
	title = {Characteristics and generative mechanisms of software development productivity distributions},
	journal = {Information and Software Technology},
	year = {2023},
	publisher = {Elsevier},
	abstract = {Context: There is considerable variation in the productivity of software developers. Better knowledge about this variation may provide valuable inputs for the design of skill tests and recruitment processes. Objective: This paper aims to identify properties of software development productivity distributions and gain insight into mechanisms that potentially explain these productivity differences. Method: Four data sets that contain the results of software developers solving the same programming tasks were collected. The properties of the productivity distributions were analyzed, the fits of different types of distributions to the productivity data were compared, and potential generative mechanisms that would lead to the types of distributions with the best fit to the productivity data were evaluated. Results: The coefficient of variance of the productivity of the software developers was, on average, 0.55, with the top 50\% of developers having average productivity that was 2.44 times higher than the bottom 50\% of developers. All productivity samples were right-skewed, with an average skew of 1.79. About 30\% of the observed productivity variance was explained by non-systematic, i.e., within-developer, variance. The distributions with the best fit to the empirical productivity data were the lognormal and power-law-with-an-exponential-cutoff distributions. The analysis of the mechanisms leading to productivity differences found no support for the "rich-getting-richer" explanation proposed for other disciplines. Instead, it suggests a constant productivity difference with increasing experience. Conclusion: The substantial difference in productivity among software developers solving programming tasks indicates that a thorough evaluation of skill in the recruitment process can be rewarding. In particular, the long tail towards higher productivity values demonstrates the large gains that can be achieved by detecting and recruiting developers with very high productivity. More research is needed to understand the mechanisms leading to the large productivity differences.},
	author = {J{\o}rgensen, Magne}
}
@article {43055,
	title = {Enabling Autonomous Teams and Continuous Deployment at Scale},
	journal = {IEEE IT Professional},
	year = {2023},
	publisher = {IEEE},
	abstract = {In this article, we give advice on transitioning to a more agile delivery model for large-scale agile development projects based on experience from the Parental Benefit Project of the Norwegian Labour and Welfare Administration. The project modernized a central part of the organization{\textquoteright}s IT portfolio and included up to ten development teams working in parallel. The project successfully changed from using a delivery model which combined traditional project management elements and agile methods to a more agile delivery model with autonomous teams and continuous deployment. This transition was completed in tandem with the project execution. We identify key lessons learned which will be useful for other organizations considering similar changes and report how the new delivery model reduced risk and opened up a range of new possibilities for delivering the benefits of digitalization.},
	author = {Dings{\o}yr, T. and J{\o}rgensen, Magne and Carlsen, F. and Carlstr{\"o}m, L. and Engelsrud, J. and Hansvold, K. and Heib{\o}, M. and K, R{\o}e. and K. O. {S{\o}rensen}}
}
@article {43106,
	title = {Improved Measurement of Software Development Effort Estimation Bias},
	journal = {Information and software technology},
	year = {2023},
	publisher = {Elsevier},
	abstract = {Context: While prior software development effort estimation research has examined the properties of estimation error measures, there has not been much research on the properties of measures of estimation bias. Objectives: Improved measurement of software development effort estimation bias. Methods: Analysis of the extent to which measures of estimation bias meet the criterion that perfect estimates should result in zero bias. Results: Recommendations for measurement of estimation bias for estimates of the mean, median, and mode software development effort. The results include the recommendation to avoid a commonly used measure of effort estimation bias. Conclusion: Proper evaluation of estimation bias requires knowledge about the type of estimates evaluated, together with the selection of a measure of estimation bias that gives zero bias for perfect estimates of that type.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {43187,
	title = {What can we learn from surveys on the importance of software development productivity factors?},
	journal = {CiBSE},
	year = {2023},
	publisher = {Brazilian Publishing Company},
	abstract = {A proper interpretation of survey results on the importance of software development productivity factors requires an understanding of what the responses reflect. To find out more about this, we conducted a survey with 79 experienced software professionals. The strongest connection found was between the high perceived importance of a productivity factor and how frequently the respondents had experienced that a low performance on that factor had caused productivity problems. We argue that it is challenging to interpret survey-based results on the importance of productivity factors. Instead of conducting more surveys on the importance of such factors, we recommend asking directly about previous experiences related to productivity factors and including contextual information that enables proper interpretation of the responses.},
	author = {J{\o}rgensen, Magne},
	editor = {, }
}
@misc {43061,
	title = {Digitalisering av offentlig sektor:  Utvalgte utviklingstrekk og trender},
	howpublished = {Brukerr{\r a}det, Oslo, Norway},
	year = {2022},
	author = {J{\o}rgensen, Magne}
}
@misc {43058,
	title = {Digitaliseringen av offentlig sektor i et fremtidsperspektiv},
	howpublished = {Statsministerens kontor (SMK), Oslo, Norway},
	year = {2022},
	type = {Presentation of research for the Norwegian prime minister and parts of the government},
	author = {J{\o}rgensen, Magne}
}
@article {43124,
	title = {Enabling Autonomous Teams and Continuous Deployment at Scale},
	journal = {IT Professional},
	volume = {24},
	year = {2022},
	month = {Jan-11-2022},
	pages = {47 - 53},
	publisher = {IEEE Computer Society},
	address = {New York City},
	issn = {1520-9202},
	doi = {10.1109/MITP.2022.3209871},
	url = {https://ieeexplore.ieee.org/document/10017407/http://xplorestaging.ieee.org/ielx7/6294/10017402/10017407.pdf?arnumber=10017407},
	author = {Dings{\o}yr, T. and J{\o}rgensen, Magne and Frode Odde {Carlsen} and Carlstr{\"o}m, Lena and Engelsrud, Jens and Hansvold, Kine and Heib{\o}-Bagheri, Mari and R{\o}e, Kjetil and Karl Ove Vika {S{\o}rensen}}
}
@misc {43060,
	title = {How Norwegian companies successfully combines benefits management and agile software development processes},
	howpublished = {SENOBR Industry day, Trondheim, Norway},
	year = {2022},
	author = {J{\o}rgensen, Magne}
}
@misc {43059,
	title = {How to Succeed with Digitalization? A Study of Benefit Management in Public IT Projects~},
	howpublished = {CONCEPT seminar, Oslo, Norway},
	year = {2022},
	author = {J{\o}rgensen, Magne and Berg, Helene}
}
@inproceedings {28160,
	title = {Measurement of software development effort estimation bias: Avoiding biased measures of estimation bias},
	journal = {11th International Conference on Software Engineering and Applications (SEA 2022)},
	year = {2022},
	publisher = {SEA},
	abstract = {In this paper, we propose improvements in how estimation bias, e.g., the tendency towards under-estimating the effort, is measured. The proposed approach emphasizes the need to know what the estimates are meant to represent, i.e., the type of estimate we evaluate and the need for a match between the type of estimate given and the bias measure used. We show that even perfect estimates of the mean effort will not lead to an expectation of zero estimation bias when applying the frequently used bias measure: (actual effort {\textendash} estimated effort)/actual effort. This measure will instead reward under-estimates of the mean effort. We also provide examples of bias measures that match estimates of the mean and the median effort, and argue that there are, in general, no practical bias measures for estimates of the most likely effort. The paper concludes with implications for the evaluation of bias of software development effort estimates.},
	author = {J{\o}rgensen, Magne},
	editor = {Evans, Harold}
}
@article {28411,
	title = {Realizing benefits in public IT projects: A multiple case study},
	journal = {IET Software},
	year = {2022},
	month = {12/2022},
	publisher = {IET},
	abstract = {IT investments in the public sector are large, and it is essential that they lead to benefits for the organizations themselves and for the wider society. While there is evidence suggesting a positive connection between the existence of benefits management practices and benefits realization, less is known about how to implement such practices effectively. The paper aims to provide insights into when benefits are most likely to be realized, and how benefits management practices and roles should be implemented in order to have a positive effect on the projects{\textquoteright} success in terms of realizing benefits. The authors collected data relating to ten Norwegian public IT projects. For each project, they collected data on benefits management from project documents, by interviewing the project owners and benefits owners, and follow-up surveys. The benefits internal to the organization were those with the highest degree of realization, while the societal benefits were those with the lowest degree. Projects assessed to have more specific, measurable, accountable, and realistically planned benefits were more successful in realizing benefits. Benefits owners were most effective when they were able to attract attention towards the benefits to be realized, had a strong mandate, and had domain expertise.},
	author = {Holgeid, Kjetil and J{\o}rgensen, Magne and Gro Holst {Volden} and Berg, Helene}
}
@article {28157,
	title = {Relative estimates of software development effort: Are they more accurate or less time-consuming to produce than absolute estimates, and to what extent are they person-independent?},
	journal = {Information and Software Technology},
	volume = {143},
	year = {2022},
	month = {03/2022},
	pages = {106782},
	publisher = {Wiley},
	abstract = {Context: Estimates of software development effort may be given as judgments of relationships between the use of efforts on different tasks {\textendash} that is, as relative estimates. The use of relative estimates has increased with the introduction of story points in agile software development contexts.Objective: This study examines to what extent relative estimates are likely to be more accurate or less time-consuming to produce than absolute software development effort estimates and to what extent relative estimates can be considered developer-independent.Method: We conducted two experiments. In the first experiment, we collected estimates from 102 professional software developers estimating the same tasks and randomly allocated to providing relative estimates in story points or absolute estimates in work-hours. In the second experiment, we collected the actual efforts of 20 professional software developers completing the same 5 programming tasks and used these to analyse the variance in relative efforts.Results: The results from the first experiment indicates that the relative estimates were less accurate than the absolute estimates, and that the time consumed completing the estimation work was higher for those using relative estimation, even when only considering developers with extensive prior experience in story point{\textendash}based estimation for both tasks. The second experiment revealed that the relative effort was far from developer-independent, especially for the least productive developers. This suggests that relative estimates to a large extent are developer-dependent.Conclusions: Although there may be good reasons for the continued use of relative estimates, we interpret our results as not supporting that the use of relative estimates is connected with higher estimation accuracy or less time consumed on producing the estimates. Neither do our results support a high degree of developer-independence in relative estimates.},
	author = {J{\o}rgensen, Magne and Escott, Eban}
}
@misc {43057,
	title = {Relative estimates of software development effort: Are they more accurate or less time-consuming to produce than absolute estimates, and to what extent are they person-independent?},
	howpublished = {ESEM 2022, Helsinki, Finland (Journal first publication)},
	year = {2022},
	type = {A Journal First presentation (refereed)},
	abstract = {Context:\&nbsp;Estimates of software development effort may be given as judgments of relationships between the use of efforts on different tasks-that is, as relative estimates. The use of relative estimates has increased with the introduction of story points in agile software development contexts.Objective:\&nbsp;This study examines to what extent relative estimates are likely to be more accurate or less time-consuming to produce than absolute software development effort estimates and to what extent relative estimates can be considered developer-independent.Method:\&nbsp;We conducted two experiments. In the first experiment, we collected estimates from 102 professional software developers estimating the same tasks and randomly allocated to providing relative estimates in story points or absolute estimates in work-hours. In the second experiment, we collected the actual efforts of 20 professional software developers completing the same 5 programming tasks and used these to analyze the variance in relative efforts.Results:\&nbsp;The results from the first experiment indicate that the relative estimates were less accurate than the absolute estimates, and that the time consumed completing the estimation work was higher for those using relative estimation, even when only considering developers with extensive prior experience in story point{\textendash}based estimation for both tasks. The second experiment revealed that the relative effort was far from developer-independent, especially for the least productive developers. This suggests that relative estimates to a large extent are developer-dependent.Conclusions:\&nbsp;Although there may be good reasons for the continued use of relative estimates, we interpret our results as not supporting that the use of relative estimates is connected with higher estimation accuracy or less time consumed on producing the estimates. Neither do our results support a high degree of developer-independence in relative estimates.},
	author = {J{\o}rgensen, Magne and Escott, Eban}
}
@misc {43062,
	title = {Smidig utvikling og nyttestyring {\textendash} som h{\r a}nd i hanske?},
	howpublished = {NAV-fagdag, Oslo, Norway},
	year = {2022},
	author = {J{\o}rgensen, Magne}
}
@misc {43063,
	title = {Understanding our own role in scientific predictions: (Over-)confidence},
	howpublished = {Seminar for FluidFlower (UiB), Bergen, Norway},
	year = {2022},
	author = {J{\o}rgensen, Magne}
}
@article {28155,
	title = {When 2 + 2 should be 5: The summation fallacy in time prediction},
	journal = {Journal of Behavioral Decision Making},
	volume = {35},
	year = {2022},
	pages = {e2265},
	publisher = {Wiley},
	abstract = {Predictions of time (e.g., work hours) are often based on the aggregation of estimates of elements (e.g., activities, subtasks). The only types of estimates that can be safely aggregated by summation are those reflecting predicted average outcomes (expected values). The sums of other types of estimates, such as bounds of confidence intervals or estimates of the mode, do not have the same interpretation as their components (e.g., the sum of the 90\% upper bounds is not the appropriate 90\% upper bound of the sum). This can be a potential source of bias in predictions of time, as shown in Studies 1 and 2, where professionals with experience in estimation provided total estimates of time that were inconsistent with their estimates of individual tasks. Study 3 shows that this inconsistency can be attributed to improper aggregation of time estimates and demonstrates how this can produce both over- and underestimation{\textemdash}and also time prediction intervals that are far too wide. Study 4 suggests that the results may reflect a more general fallacy in the aggregation of probabilistic quantities. Our observations are consistent with that inconsistencies and biases are driven by a tendency towards applying a na{\"\i}ve summation (2+2=4) of probabilistic (stochastic) values, in situations where this is not appropriate. This summation fallacy may be in particular consequential in a context where informal estimation methods (expert-judgment based estimation) are used.},
	author = {Halkjelsvik, Torleif and J{\o}rgensen, Magne}
}
@article {28156,
	title = {When should we (not) use the mean magnitude of relative error (MMRE) as an error measure in software development effort estimation?},
	journal = {Information and Software Technology},
	volume = {143},
	year = {2022},
	month = {03/2022},
	pages = {106784},
	publisher = {Elsevier},
	abstract = {Context: The mean magnitude of relative error (MMRE) is an error measure frequently used to evaluate and compare the estimation performance of prediction models and software professionals.Objective: This paper examines conditions for proper use of MMRE in effort estimation contexts.Method: We apply research on scoring functions to identify the type of estimates that minimizes the expected value of the MMRE.Results: We show that the MMRE is a proper error measure for estimates of the most likely (mode) effort, but not for estimates of the median or mean effort, provided that the effort usage is approximately log-normally distributed, which we argue is a reasonable assumption in many software development contexts. The relevance of the findings is demonstrated on real-world software development data.Conclusion: MMRE is not a proper measure of the accuracy of estimates of the median or mean effort, but may be used for the accuracy evaluation of estimates of most likely effort.},
	author = {J{\o}rgensen, Magne and Halkjelsvik, Torleif and Liest{\o}l, Knut}
}
@misc {43056,
	title = {XP 2022: It{\textquoteright}s all about the benefits: How to successfully implement benefits management in agile software development},
	howpublished = {XP 2022, Copenhagen, Denmark},
	year = {2022},
	publisher = {Agile Alliance},
	type = {Keynote XP 2022},
	edition = {https://www.agilealliance.org/xp2022/keynotes/},
	abstract = {The primary criterion for success in software development is that it leads to benefits (value, profit, gains, {\textellipsis}) to clients and/or other stakeholders, and that these benefits are sufficiently large to make the investment worthwhile. In spite of this pretty obvious criterion, most software development processes, including those belonging to the family of agile processes, often seems to have their main focus elsewhere.In this presentation, I will report on empirical studies conducted at SimulaMet/EDOS that aim at a better understanding of how to successfully integrate benefits management practices and roles in agile software development contexts. The studies give results on planning the realization of benefits, implementing continuous benefits management, designing the role of a {\textquotedblleft}benefits responsible,{\textquotedblright} and evaluating the realization of benefits.In general, the studies find a positive effect from benefits management practices on benefits realization. This was in particular the case in agile software development contexts. Not only did the benefits management practices give a better effect in agile contexts, they were also easier to implement and did not conflict with agile principles or values.},
	author = {J{\o}rgensen, Magne}
}
@misc {28161,
	title = {The 8th International Conference on Electrical Engineering and Informatics},
	howpublished = {ICEEI2021 - The 8th International Conference on Electrical Engineering and Informatics, Malaysia (virtual conference)},
	year = {2021},
	url = {www.ftsm.ukm.my/iceei2021},
	author = {J{\o}rgensen, Magne}
}
@article {28159,
	title = {Benefits management in software development: A systematic review of empirical studies},
	journal = {IET Software},
	volume = {15},
	year = {2021},
	pages = {1-24},
	publisher = {IET},
	abstract = {Considerable resources are wasted on software projects delivering less than the planned benefits. Herein, the objective is to synthesize empirical evidence of the adoption and impact of benefits management (BM) in software development, and to suggest directions for future research. A systematic review of the literature is performed and identified 4836 scientific papers of which the authors found 47 to include relevant research. While most organizations identify and structure benefits at the outset of a project, fewer organizations report implementing BM as a continuous process throughout the project lifecycle. Empirical evidence gives support for positive impact on project outcome from the following BM practices: identifying and structuring benefits, planning benefits realization, BM during project execution, benefits evaluation and the practice of having people responsible for benefits realization. The authors suggest four research directions to understand (1) why BM practices sometimes not are adopted, (2) BM in relation to other management practices, (3) BM in agile software development and (4) BM in the context of organizations{\textquoteright} value creation logics.},
	author = {Holgeid, Kjetil and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Krogstie, John}
}
@article {27620,
	title = {Evaluation of Probabilistic Project Cost Estimates},
	journal = {IEEE Transactions on Engineering Management},
	year = {2021},
	month = {08/2021},
	pages = {1-16},
	publisher = {IEEE},
	abstract = {Evaluation of cost estimates should be fair and give incentives for accuracy. These goals, we argue, are challenged by a lack of precision in what is meant by a cost estimate and the use of evaluation measures that do not reward the most accurate cost estimates. To improve the situation, we suggest the use of probabilistic cost estimates and propose guidelines on how to evaluate such estimates. The guidelines emphasize the importance of a match between the type of cost estimate provided by the estimators and the chosen cost evaluation measure, and the need for an evaluation of both the calibration and the informativeness of probabilistic cost estimates. The feasibility of the guidelines is exemplified in an analysis of a set of 69 large Norwegian governmental projects. The evaluation indicated that the projects had quite accurate and unbiased P50 estimates and that the prediction intervals were reasonably well calibrated. It also showed that the cost prediction intervals were non-informative with respect to differences in cost uncertainty and, consequently, not useful to identify projects with higher cost uncertainty. The results demonstrate the usefulness of applying the proposed cost estimation evaluation guidelines.},
	author = {J{\o}rgensen, Magne and Welde, Morten and Halkjelsvik, Torleif}
}
@article {27619,
	title = {A field experiment on trialsourcing and the effect of contract types on outsourced software development},
	journal = {Information and Software Technology},
	volume = {134},
	year = {2021},
	month = {June 2021},
	pages = {106559},
	publisher = {Elsevier},
	abstract = {Context: To ensure the success of software projects, it is essential to select skilled developers and to use suitable work contracts. Objective: This study tests two hypotheses: (i) the use of work-sample testing (trialsourcing) improves the selection of skilled software developers; and (ii) the use of contracts based on hourly payment leads to better software project outcomes than fixed-price contracts. Method: Fifty-seven software freelancers with relevant experience and good evaluation scores from previous clients were invited to complete a two-hour long trialsourcing task to qualify for a software development project. Thirty-six developers completed the trialsourcing task with acceptable performance, and, based on a stratified allocation process, were asked to give a proposal based on an hourly payment or a fixed-price contract. Eight hourly payment-based and eight fixed-priced proposals were accepted. The process and product characteristics of the completion of these 16 projects were collected and analysed. Results and Conclusion: Only partial support for our hypotheses was observed. While the use of trialsourcing may have prevented the selection of developers with insufficient skills, the performance on the trialsourcing task of the selected developers did not predict performance on the project. The use of hourly payments led to lower costs than fixed-price contracts, but not to improved processes or products. We plan to follow up these, to us unexpected, results with research on how to design more skill-predictive trialsourcing tasks, and when and why different project contexts give different contract consequences.},
	author = {J{\o}rgensen, Magne and Grov, Jon}
}
@misc {28158,
	title = {Hvordan lykkes med digitalisering? En unders{\o}kelse av nyttestyring i IT-prosjekter i offentlig sektor (Concept-rapport nr. 64)},
	year = {2021},
	publisher = {Ex Ante Akademisk forlag},
	address = {NTNU},
	isbn = {978-82-8433-004-4},
	author = {Berg, Helene and Holgeid, Kjetil and J{\o}rgensen, Magne and Gro Holst {Volden}}
}
@article {28410,
	title = {Jakten p{\r a} nytte i offentlige IT-prosjekter},
	journal = {Stat \& Styring},
	volume = {31},
	year = {2021},
	month = {10/2021},
	pages = {38-41},
	publisher = {Scandinavian University Press},
	abstract = {Det er {\o}kende oppmerksomhet p{\r a} at offentlige tiltak skal ha nytte for brukere og samfunn {\textendash} heldigvis. Det hjelper, for eksempel, lite at et nytt IT-system blir levert med spesifisert funksjonalitet, til avtalt tid og uten overskridelse av budsjettet dersom systemet ikke blir brukt og leverer nytte som forutsatt.},
	author = {Gro Holst {Volden} and J{\o}rgensen, Magne and Holgeid, Kjetil and Berg, Helene}
}
@inproceedings {27934,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill (A Journal first conference publication)},
	journal = {CM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE)},
	year = {2021},
	publisher = {ACM},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut},
	editor = {Apel, Sven}
}
@inproceedings {42532,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill },
	journal = {ESEC/FSE},
	year = {2021},
	publisher = {Journal First publication (IEEE TSE)},
	type = {Journal first},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut}
}
@inproceedings {27113,
	title = {Team resource management decisions in software development projects},
	journal = {IEEE 23rd Conference on Business Informatics (CBI) },
	year = {2021},
	publisher = {IEEE},
	abstract = {Purpose: This paper examines to what extent resource managers are likely to make normatively correct decisions in complex, but realistic, software development team resource management situations.Research design: Three scenarios were designed, all of them with a software development project consisting of a higher and a lower productivity team. The resource managers were asked to decide which team to increase (or decrease) the productivity by 10\% in order to minimize the total effort or the total duration of the software project. Ninety-nine software professionals, most of them with substantial resource management experience, were randomly allocated one of the scenarios.Findings: Most of the resource managers perceived their scenario as realistic and occurring in practice. The decisions tended, however, to be different from the normatively correct ones. In particular, when the scenario structure resembled that used to document a time-saving bias, the great majority of the managers made non-normative decisions. The findings suggest that the resource managers made decisions based on simple heuristics, often leading to non-normative decisions. When asked about the decisions they would make in practice, more of the resource managers gave normatively correct responses. Our findings suggest a cost-saving potential from more awareness of how to make team resource management decisions.Originality: The study may be the first to document non-normative team resource decisions, including those related to the time-saving bias, in the context of project management.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {27615,
	title = {Benefits management and agile practices in software projects: how perceived benefits are impacted},
	journal = {IEEE 22nd Conference on Business Informatics (CBI)},
	volume = {2},
	year = {2020},
	pages = {48-56},
	publisher = {IEEE},
	abstract = {Considerable resources are wasted on projects that deliver few or no benefits. The main objective is to better understand the characteristics of projects that are successful in delivering good client benefits. We asked 71 Norwegian software professionals to report information about projects completed between 2016 and 2018. We found that both benefits management and agile practices have a significant relationship with perceived realisation of client benefits. This includes the benefits management practices of having a plan for benefits realisation, individuals with assigned responsibility for benefits realisation, benefits management during project execution, quantification of realised benefits, evaluation of realised benefits, re-estimation of benefits during project execution, and the agile practices of a flexible scope and frequent deliveries to production. The software projects that were successful in delivering client benefits adopted benefits management and agile practices to a larger extent than the less successful ones. Future studies are required to establish more comprehensive understanding of what distinguishes projects that deliver good client benefits from the rest, including studies of the realisation of client benefits in agile software projects.},
	author = {Holgeid, Kjetil and J{\o}rgensen, Magne}
}
@article {23658,
	title = {How to pose for a professional photo: The effect of three facial expressions on perception of competence of a software developer},
	journal = {Australian Journal of Psychology},
	volume = {72},
	year = {2020},
	pages = {257-266},
	publisher = {Wiley},
	author = {Filkukova, Petra and J{\o}rgensen, Magne}
}
@article {27613,
	title = {Practices connected to perceived client benefits of software projects},
	journal = {IET Software},
	volume = {14},
	year = {2020},
	pages = {677-683},
	publisher = {The Institution of Engineering and Technology},
	abstract = {It is well-documented that many software projects deliver fewer benefits than planned. However prior research has had a stronger focus on the ability to deliver within budget, on time and with the specified functionality, than on what to do to successfully deliver client benefits. The authors have conducted a survey collecting information about benefits management practices, agile practices, use of contracts, and the perceived success in delivery of client benefits. The authors received responses from 83 software professionals with information about 73 recent and 74 older software projects. There was no statistically significant improvement of the delivered client benefits from the older to the recent projects. Statistically significant findings, applying a general linear model-based analysis, include that the degree of success in delivering client benefits is connected to a project having: (i) a plan for how to realise the benefits, (ii) implemented practices for benefits management during project execution, (iii) frequent deliveries to production during the project execution, and (iv) a process for the evaluation of realised benefits after project completion. The authors argue that greater use of these practices represents a potential for organisations to increase their success in delivering benefits from software projects.},
	author = {Holgeid, Kjetil and J{\o}rgensen, Magne}
}
@article {26849,
	title = {Relations Between Effort Estimates, Skill Indicators, and Measured Programming Skill},
	journal = {Transactions on Software Engineering},
	year = {2020},
	month = {February 2020},
	publisher = {IEEE},
	abstract = {There are large skill differences among software developers, and clients and managers will benefit from being able to identify those with better skill. This study examines the relations between low effort estimates, and other commonly used skill indicators, and measured programming skill. One hundred and four professional software developers were recruited. After skill-related information was collected, they were asked to estimate the effort for four larger and five smaller programming tasks. Finally, they completed a programming skill test. The lowest and most over-optimistic effort estimates for the larger tasks were given by those with the lowest programming skill, which is in accordance with the well-known Dunning-Kruger effect. For the smaller tasks, however, those with the lowest programming skill had the highest and most over-pessimistic estimates. The other programming skill indicators, such as length of experience, company assessed skill and self-assessed skill, were only moderately correlated with measured skill and not particularly useful in guiding developer skill identification. A practical implication is that for larger and more complex tasks, the use of low effort estimates and commonly used skill indicators as selection criteria leads to a substantial risk of selecting among the least skilled developers.},
	author = {J{\o}rgensen, Magne and Bergersen, Gunnar and Liest{\o}l, Knut}
}
@article {26848,
	title = {Sequence effects in the estimation of software development effort},
	journal = {Journal of Systems and Software},
	volume = {159},
	year = {2020},
	pages = {110448},
	publisher = {Elsevier},
	abstract = {Currently, little is known about how much the sequence in which software development tasks or projects are estimated affects judgment-based effort estimates. To gain more knowledge, we examined estimation sequence effects in two experiments. In the first experiment, 362 software professionals estimated the effort of three large tasks of similar sizes, whereas in the second experiment 104 software professionals estimated the effort of four large and five small tasks. The sequence of the tasks was randomised in both experiments. The first experiment, with tasks of similar size, showed a mean increase of 10\% from the first to the second and a 3\% increase from the second to the third estimate. The second experiment showed that estimating a larger task after a smaller one led to a mean decrease in the estimate of 24\%, and that estimating a smaller task after a larger one led to a mean increase of 25\%. There was no statistically significant reduction in the sequence effect with higher competence. We conclude that more awareness about how the estimation sequence affects the estimates may reduce potentially harmful estimation biases. In particular, it may reduce the likelihood of a bias towards too low effort estimates.},
	author = {J{\o}rgensen, Magne and Halkjelsvik, Torleif}
}
@misc {27123,
	title = {Agile software development and benefits management: A perfect match?},
	howpublished = {Seminar on effective software development, Kathmandu, Nepal},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27125,
	title = {Conducting realistic, controlled experiments in software engineering},
	howpublished = {Seminar for PhD-students at University of Manaus, Brazil},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27129,
	title = {Estimering av IT-utvikling},
	howpublished = {Seminar, Knowit, Norway},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@techreport {27114,
	title = {Estimering av kostnader i store statlige prosjekter: Hvor gode er estimatene og usikkerhetsanalysene i KS2-rapportene?},
	journal = {Concept-rapport nr. 59},
	year = {2019},
	publisher = {Ex ante akademisk forlag},
	address = {Trondheim},
	abstract = {The external quality assurance scheme for large government investment projects (the QA scheme / the state project model) aims, among other things, to ensure that budgets are realistic and that the risk analyses of the cost estimates reflect real cost uncertainty. The extent to which budgets, estimates and risk analyses are realistic, and where there may be potentials for improvements, are the main themes of this study.\&nbsp;Chapter 1 describes the background and motivation for the study. The starting point is that the Concept research programme collects final costs in projects that have been through QA2 (quality assurance of cost estimate before the parliament{\textquoteright}s investment decision).\&nbsp;That\&nbsp;provides\&nbsp;a\&nbsp;basis for studies of\&nbsp;cost performance. As the\&nbsp;sample of projects increase,\&nbsp;more detailed studies of the estimates\&nbsp;that formed the basis for the\&nbsp;parliament{\textquoteright}s investment decision becomes possible.\&nbsp;The study has three main topics. We look at:\&nbsp;The realism in the projects{\textquoteright} budgets\&nbsp;The realism in the point estimates in the QA2 reports, and\&nbsp;The realism and information value in the prediction intervals and estimation distributions.\&nbsp;Chapter 2 provides a review of previous studies of cost performance in projects that have been through QA2. They all show relatively good results both in terms of deviation from budgets and risk assessments. While average cost overruns reported in international studies typically have been around 30 per cent, Norwegian studies report average overruns of between two and six per\&nbsp;cent.\&nbsp;Other studies\&nbsp;also typically\&nbsp;report\&nbsp;a\&nbsp;strong\&nbsp;underestimation of uncertainty.\&nbsp;The P50 and P85 estimates from\&nbsp;the QA2 reports\&nbsp;on the other hand\&nbsp;(that is, estimates that are not expected to be exceeded in 50 and 85 per\&nbsp;cent of cases, respectively)\&nbsp;seem to\&nbsp;have\&nbsp;been reasonably well calibrated. However,\&nbsp;several authors have pointed out that the\&nbsp;distribution of final costs to the budgets\&nbsp;have been\&nbsp;somewhat higher than assumed at\&nbsp;the time of the investment decision.\&nbsp;The data used in the study, which is described in Chapter 3, is based on a larger sample of projects than previous studies.\&nbsp;The analyses focus more on the estimates than previous studies have\&nbsp;done. The analysis of the P50 and P85 estimates is based on\&nbsp;samples of\&nbsp;83 and 85 projects respectively.\&nbsp;Sufficient\&nbsp;data for our analysis of the cost estimates were found for 70 of these projects.\&nbsp;In Chapter 4, we\&nbsp;outline\&nbsp;detailed research questions and the methodology for the analyses. In this, we motivate and indicate, based on the latest research\&nbsp;on the area, how probability-based cost estimates should be evaluated.\&nbsp;We introduce an analysis of estimate deviations and estimation bias\&nbsp;based on what is a reasonable "loss function", where the loss function is what we attempt to minimise in the estimates. We evaluate\&nbsp;the extent to which we have been successful in estimating the real uncertainty of projects ex ante.\&nbsp;We also assess\&nbsp;how informative prediction intervals and estimate distributions have been. We argue that well-calibrated probability-based estimates (e.g.,\&nbsp;that\&nbsp;50 per\&nbsp;cent of P50 estimates\&nbsp;should not be\&nbsp;exceeded) are not a\&nbsp;sufficient\&nbsp;evaluation criterion. In addition, we need\&nbsp;indicators\&nbsp;for how informative the\&nbsp;probability-based estimates have been.\&nbsp;In Chapter 5,\&nbsp;we find that the median deviations between actual costs and the P50, measured as absolute percentage deviation, is 10 per cent (mean = 12.5), and that the median deviation from the P85 is 1.5 per cent (average = 3.4). In other words, for all the projects, there is only a slight tendency for overruns, and much lower than what has\&nbsp;been\&nbsp;reported in international studies. Over time, however, there has been a somewhat worrying development. While there was a tendency for cost\&nbsp;underruns in the past (an average of 6 per\&nbsp;cent\&nbsp;underruns of the P50\&nbsp;for projects with an investment decision between 2001 and\&nbsp;2003), there has been a tendency for cost overruns in the later years (an average of 12 per\&nbsp;cent overruns in the period 2010-2012).\&nbsp;Given well-calibrated estimates, the actual cost should be below the P50 in about 50 per cent of the cases and below the P85 in about 85 per cent of the cases. However, we find that this only applies in 40 per cent of the cases for the P50 and 73 per cent for the P85. The shares have been declining over time. While in 2001-2003, 62 and 100 per cent were within the P50 and P85 respectively, in 2010-2012 there were only 21 and 43 per cent within, albeit based on a\&nbsp;smaller sample\&nbsp;than in the time-periods before. The reason why hit rates for the P50 and the P85 for all projects together are not so far from the\&nbsp;intended\&nbsp;targets\&nbsp;is because we have gone from overestimation to underestimation. The tendency for underestimation should be reversed through better estimation and governance in future projects.\&nbsp;The analyses of the estimates in Chapter 6 find\&nbsp;about the same degree of overruns and estimate deviations for the P50 and P85\&nbsp;estimates as those reported in Chapter 5. The P50 estimates showed a median estimate bias of\&nbsp;-1 per cent (mean = 3 per cent). The median percentage deviation (regardless of sign) was 12 per cent (mean = 14 per cent). We calculated that the expected deviation\&nbsp;from the P50 budget\&nbsp;could not be less than 8-10 per cent, given some assumptions that the projects\&nbsp;do not\&nbsp;adapt deliveries to reduce deviations. Although the latter assumption hardly\&nbsp;is\&nbsp;met, this calculation suggests that the deviations are not particularly high.\&nbsp;We observe that there is typically a reduction from estimate to budget. The P50 budget was on average seven per cent lower than the P50 estimate and the P85 budget seven per cent lower than the P85 estimate. Although there were several projects that should have retained the original P50 and P85 estimates as P50 and P85 budget, respectively, we did not find that the adjustments overall reduced the realism. Many of the adjustments seem to be well justified.\&nbsp;The estimates in the QA2 reports include both point estimates, prediction intervals and estimate distributions (S-curves). Our analyses include all of these and have as their main findings are as follows:\&nbsp;\&nbsp;The estimation distributions and prediction intervals are typically too narrow to reflect actual uncertainty. For example, as many as 19 per cent of the projects have a lower cost than the P10 estimate and 20 per cent more than the P90 estimate. Future estimation should\&nbsp;take into account\&nbsp;that the scope for project costs is broader than previously typically assumed.\&nbsp;Estimated cost uncertainty, estimated through the width of the prediction interval and estimate distribution, does not correlate with actual cost uncertainty, measured by cost deviations and overruns. This indicates a low ability to distinguish between projects with high and\&nbsp;low cost\&nbsp;uncertainty.\&nbsp;If we become better at identifying the high-risk projects, we could potentially\&nbsp;reduce the need for risk contingency without compromising cost performance and project execution. We show, given some assumptions, that the P85 could be\&nbsp;17 per cent lower if the ability to distinguish between low and\&nbsp;high risk\&nbsp;projects had been better. Measures to improve this capability should be given priority in the estimation work.\&nbsp;There are differences in estimation performance between agencies and\&nbsp;between\&nbsp;consultancies carrying out the external QA.\&nbsp;Defence\&nbsp;projects stand out by having a strong tendency to overestimate costs (their average underrun of the P50 estimate is 19 per cent) and overly narrow prediction intervals (29 per\&nbsp;cent of projects within the 80 percent prediction range). The Norwegian Public Roads Administration also tends to\&nbsp;estimate too\&nbsp;narrow prediction intervals (57 per cent of projects within the 80 per cent prediction interval). Among the\&nbsp;QA consultancies, there are no major differences in estimate deviations, but larger differences in how realistic the uncertainty is estimated. There may be differences in project complexity or other\&nbsp;issues\&nbsp;that explain these differences.\&nbsp;Given the inability to distinguish between low- and high-risk projects in the estimation work, a simple mechanical mark-up model could in theory do just as well as the more demanding QA2 estimation work. We investigated this, where the uplifts were based on historical estimate deviations, but found that the QA2 estimates did better. This indicates that the work done in the QA2 estimation provides added value, measured against simple\&nbsp;mark-up\&nbsp;models.\&nbsp;In Chapter\&nbsp;7,\&nbsp;we summarize and discuss the findings. Overall, the main conclusions are that the QA2 framework is useful and that cost estimates appear to be realistic and reasonably well calibrated. However, developments over time are worrying and should lead to improvements in the estimation work. Two major areas of improvement are to specify broader estimate distributions, that is, to recognize that cost uncertainty is typically greater than that which has previously been\&nbsp;identified\&nbsp;in the estimation work, as well as to better distinguish between projects with low and high cost uncertainty.\&nbsp;},
	isbn = {78-82-93253-81-5},
	issn = {0803-9763},
	author = {Welde, Morten and J{\o}rgensen, Magne and Per Fridtjof {Larssen} and Halkjelsvik, Torleif}
}
@article {26645,
	title = {Evaluating Probabilistic Software Development Effort Estimates  Maximizing Informativeness Subject to Calibration},
	journal = {Information and Software Technology},
	volume = {115},
	year = {2019},
	pages = {93-96},
	publisher = {Elsevier},
	abstract = {Context: Probabilistic effort estimates inform about the uncertainty and may give useful input to plans, budgets and investment analyses. Objective \&amp; Method: This paper introduces, motivates and illustrates two principles on how to evaluate the accuracy and other performance criteria of probabilistic effort estimates in software development contexts. Results: The first principle emphasizes a consistency between the estimation error measure and the loss function of the chosen type of probabilistic single point effort estimates. The second principle points at the importance of not just measuring calibration, but also informativeness of estimated prediction intervals and distributions. The relevance of the evaluation principles is illustrated by a performance evaluation of estimates from twenty-eight software professionals using two different uncertainty assessment methods to estimate the effort of the same thirty software maintenance tasks.},
	author = {J{\o}rgensen, Magne}
}
@misc {27121,
	title = {Forstyrrelser, avbrudd og produktivitet},
	year = {2019},
	publisher = {Computerworld Norway},
	author = {J{\o}rgensen, Magne}
}
@misc {27122,
	title = {From myths and fashions to evidence-based software engineering},
	howpublished = {Seminar on effective software development, Kathmandu, Nepal},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27124,
	title = {Hva kjennetegner IT -utvikling som lykkes?},
	howpublished = {Seminar for Helsedirektoratet, H{\o}nefoss, Norway},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27127,
	title = {Hva kjennetegner IT-utvikling som lykkes?},
	howpublished = {Presentasjon for Skatteetaten (intern workshop), Norway},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27119,
	title = {I praksis riktig, men i teorien feil: Kostnadsbesparelser i team},
	year = {2019},
	publisher = {Computerworld Norway},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {27116,
	title = {Predicting software development skill from effort predictions (Un)skilled and unware of it?},
	journal = {International symposium on forecasting (ISF)},
	year = {2019},
	abstract = {If software developers{\textquoteright} estimates of how much effort they would need to complete tasks were strongly correlated with their actual use of efforts, we could safely select among the developers with the lowest effort estimates and predict that they will be among the most skilled ones. Unfortunately, this is not necessarily the case. As documented in studies in many domains, those with lower skill tend to know less about how little they know and, for this reason, give over-optimistic estimates of their own performance.This effect is termed the Dunning-Kruger effect. Does this mean that effort estimates are useless as predictors of software development skill?To find out more on this we requested 104 software developers to estimate four larger and five smaller software development tasks and measured their programming skills. We assessed the connection between lower estimates and higher skill through rank correlations and hit rates, where the hit rate measures the frequency of selecting the most skilled out of two developers when selecting the one with the lower effort estimate on the same task.The results were as predicted by the Dunning-Kruger effect for the larger tasks. The developers in the lowest skill quartile had on average lower estimates than those in the highest skill quartile. To predict relative programming skill based on these estimates would be very inaccurate. The correlations between task estimates and skill were between -0.15 and 0.01 and the hit rates between 44\% and 50\%.The developers{\textquoteright} effort estimates on the smaller tasks were, however, much better connected with their measured programming skills. The rank correlations were between 0.29 and 0.50 and the hit rates between 62\% and 68\%. Clustering the estimates of all the smaller tasks into the same factor gave a rank correlation of 0.45 between this factor and the measured skill and a hit rate of 70\%. While not very strong, we show that the estimates on these tasks is better connected with measured skill than those in use in software development contexts, i.e., company-assesses skill category, length of experience, self-assessed skill and confidence in knowing how to solve the tasks. These indicators had rank correlations between 0.14 and 0.29, and hit rates between 57\% and 65\%.A possible explanation of why lower skill was connected with higher effort estimates for the smaller, but not for the larger tasks, is that lower skill in solving a task can both lead to lower skill in identifying the complexity and the simplicity of a task. In the situation with the simpler tasks, we propose, those with better skill were also better in identifying how easily the tasks could be solved at the time of estimation. This explanation suggests that we may be able to design estimation tasks even better at separating those with high and low task completion skill than in our study, when focusing on hidden simplicity, i.e., simplicity that the more skilled are more likely to identify than the less skilled ones.},
	author = {J{\o}rgensen, Magne}
}
@article {26239,
	title = {Relations between Project Size, Agile Practices and Successful Software Development},
	journal = {IEEE Software},
	volume = {36},
	year = {2019},
	pages = {39-43},
	publisher = {IEEE},
	abstract = {The use of agile methods in the execution of large-scale software development is increasing. To find out more about the effect of this on project performance, information was collected about 196 Norwegian IT-projects. Increased project size was associated with decreased project performance for both agile and non-agile projects, but the projects using agile methods had better performance than the non-agile projects for all examined project size categories. Flexible scope, frequent deliveries to production, a high degree of requirement changes and more competent providers are candidates to explain the better performance of agile projects.},
	author = {J{\o}rgensen, Magne}
}
@misc {27118,
	title = {Sannsynlighetsvurderinger av kostnader},
	year = {2019},
	publisher = {Computerworld Norway},
	author = {J{\o}rgensen, Magne}
}
@misc {27120,
	title = {Smidig og stor - en selvmotsigelse innen IT-utvikling?},
	year = {2019},
	publisher = {Computerworld Norway},
	author = {J{\o}rgensen, Magne}
}
@misc {27117,
	title = {Usannsynlige sannsynligheter},
	year = {2019},
	publisher = {Computerworld Norway},
	author = {J{\o}rgensen, Magne}
}
@misc {27128,
	title = {What can - and should - software engineering learn from psychology?(with a focus on research method)},
	howpublished = {ICSIE, Cairo, Egypt},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {27126,
	title = {What makes software development projects successful, and what makes them fail? (and how to find out)},
	howpublished = {Presentation at Simula Summer School, Berlin, Germany},
	year = {2019},
	author = {J{\o}rgensen, Magne}
}
@misc {26278,
	title = {Agile software development and benefits management: A perfect match},
	howpublished = {PMI/Prosjekt Norge Workshop, Oslo, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26281,
	title = {Alltid smidig n{\r a}r du g{\r a}r?},
	howpublished = {Smidig-konferansen: Smidig i offentlig sektor, Oslo, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26287,
	title = {Alltid smidig n{\r a}r du g{\r a}r! Men hva med oppstartsfasen og forretningssiden?},
	howpublished = {Computerworld (Norge)},
	year = {2018},
	publisher = {IDG},
	author = {J{\o}rgensen, Magne}
}
@article {26241,
	title = {Combining Data Analytics with Team Feedback to Improve the Estimation Process in Agile Software Development},
	journal = {Foundations of Computing and Decision Sciences },
	volume = {43},
	number = {305-334},
	year = {2018},
	publisher = {De Gruyter},
	author = {Vetr{\'o}, Antonio and D{\"u}rre, Rupert and Conoscenti, Marco and Daniel Mendez {Fernandez} and J{\o}rgensen, Magne}
}
@misc {26276,
	title = {Cost and benefits of software development in a uncertain, skewed world},
	howpublished = {Software Analytics and its Impact on Industry Delft Data Science Seminar, Delft, The Netherlands},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26286,
	title = {De som vet lite vet ogs{\r a} lite om hvor lite de vet},
	howpublished = {Computerworld (Norge)},
	year = {2018},
	publisher = {IDG},
	author = {J{\o}rgensen, Magne}
}
@misc {26283,
	title = {Digitalisering = produktivitetsvekst?},
	howpublished = {Computerworld (Norge)},
	year = {2018},
	publisher = {IDG},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {26242,
	title = {Do Agile Methods Work for Large Software Projects?},
	journal = {19th International Conference on Agile Software Development (XP 2018)},
	year = {2018},
	pages = {179-190},
	publisher = {Springer},
	abstract = {Some people claim that agile methods do not scale well and are mainly useful for smaller software projects. Others claim that it is particularly in the context of large, complex software projects that the use of agile methods is likely to make the difference between success and failure. To find out more about whether or not agile methods work for larger projects, we conducted a questionnaire-based survey collecting information about 122 Norwegian software projects. Project success was measured as the combined performance of the project regarding delivered client benefits, cost control, and time control. We found that that projects using agile methods performed on average much better than those using non-agile methods for medium and large software projects, but not so much for smaller projects. This result gives support for the claim that agile methods are more rather than less successful compared to traditional methods when project size increases. There may consequently be more reasons to be concerned about how non-agile, rather than how agile methods, scale.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {26263,
	title = {An Experimental Evaluation of a De-biasing Intervention for Professional Software Developers},
	journal = {SAC 2018: Symposium on Applied Computing},
	year = {2018},
	publisher = {ACM},
	address = {New York},
	abstract = {CONTEXT:\&nbsp;The role of expert judgement is essential in our quest to improve software project planning and execution. However, its accuracy is dependent on many factors, not least the avoidance of judgement biases, such as the anchoring bias, arising from being influenced by initial information, even when it{\textquoteright}s misleading or irrelevant. This strong effect is widely documented. OBJECTIVE: We aimed to replicate this anchoring bias using professionals and, novel in a software engineering context, explore de-biasing interventions through increasing knowledge and awareness of judgement biases. METHOD: We ran two series of experiments in company settings with a total of 410 software developers. Some developers took part in a workshop to heighten their awareness of a range of cognitive biases, including anchoring. Later, the anchoring bias was induced by presenting low or high productivity values, followed by the participants{\textquoteright} estimates of their own project productivity. Our hypothesis was that the workshop would lead to reduced bias, i.e., work as a de-biasing intervention. RESULTS: The anchors had a large effect (robust Cohen{\textquoteright}s\&nbsp;d=1.19) in influencing estimates. This was substantially reduced in those participants who attended the workshop (robust Cohen{\textquoteright}s\&nbsp;d=0.72). The reduced bias related mainly to the high anchor. The de-biasing intervention also led to a threefold reduction in estimate variance. CONCLUSIONS: The impact of anchors upon judgement was substantial. Learning about judgement biases does appear capable of mitigating, although not removing, the anchoring bias. The positive effect of de-biasing through learning about biases suggests that it has value.},
	author = {Shepperd, Martin and Mair, Carolyn and J{\o}rgensen, Magne}
}
@misc {26268,
	title = {From myths and fashions to evidence-based software engineering},
	howpublished = {ICSIE, Cairo, Egypt},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26282,
	title = {Hva kjennetegner IT-prosjekter som lykkes?},
	howpublished = {Seminar Forsvaret, Kols{\r a}s, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26277,
	title = {Hvordan f{\r a} tak i reell usikkerhet av kost-nytte i en skjev verden?},
	howpublished = {Prosjekt Norges {\r a}rlige prosjektlederkonferanse, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26285,
	title = {Hvordan velge {\r a} velge - og litt om duer},
	howpublished = {Computerworld (Norge)},
	year = {2018},
	publisher = {IDG},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {26240,
	title = {Looking back on previous estimation error as a method to improve the uncertainty assessment of benefits and costs of software development projects},
	journal = {The 9th International Workshop on Empirical Software Engineering in Practice (IWESEP2018)},
	year = {2018},
	pages = {19-24},
	publisher = { IEEE},
	abstract = {Knowing the uncertainty of estimates of benefits and costs is useful when planning, budgeting and pricing projects. The traditional method for assessing such uncertainty is based on prediction intervals, e.g., asking for minimum and maximum values believed to be 90\% likely to include the actual outcome. Studies report that the traditional method typically results in too narrow intervals and intervals that are too symmetric around the estimated most likely outcome when compared with the actual uncertainty of outcomes. We examine whether an uncertainty assessment method based on looking back on the previous estimation error of similar projects leads to wider and less symmetric prediction intervals. Sixty software professionals, with experience from estimating software project costs and benefits, were randomly divided into a group with a traditional or a group with a looking back-based uncertainty assessment method. We found that those using the looking back-based method had much wider prediction intervals for both costs and benefits. The software professionals of both groups provided uncertainty assessment values suggesting a left-skewed distribution for benefits and a right-skewed distribution for cost, but with much more skew among those using the looking back-based method. We argue that a looking back-based method is promising for improved realism in uncertainty assessment of benefits and costs of software development projects.},
	author = {J{\o}rgensen, Magne}
}
@misc {26264,
	title = {Milliardinvesteringer i digitalisering. Hva gir det oss?},
	howpublished = {Software 2018, DnD, Norway},
	year = {2018},
	type = {Keynote},
	abstract = {Privat og offentlig sektor investerer milliarder av kroner p{\r a} {\r a} utvikle, videreutvikle og erstatte digitale l{\o}sninger med det form{\r a}l {\r a} forbedre og effektivisere.Hvor godt, og n{\r a}r, lykkes slike investeringene?B{\o}r vi roe ned eller heller {\o}ke tempoet i digitaliseringen?Har investeringene total sett gitt mer effektivitet og bedre liv?Det er overraskende vanskelig {\r a} kunne besvare disse sp{\o}rsm{\r a}lene godt. Noe vet vi imidlertid, og presentasjonen vil oppsummere kunnskap om dette basert p{\r a} internasjonale evalueringer, egen og andres forskning p{\r a} IT-utvikling, analyser av sammenheng mellom digitaliseringsinvesteringer og produktivitet, erfaringer fra gjennomganger i digitaliseringsr{\r a}det og andre kilder.},
	author = {J{\o}rgensen, Magne}
}
@misc {26284,
	title = {Sannheten om u{\ae}rlighet},
	howpublished = {Computerworld (Norge)},
	year = {2018},
	publisher = {IDG},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {26020,
	title = {Scope Creep or Embrace Change? A Survey of the Connections Between Requirement Changes, Use of Agile, and Software Project Success},
	journal = {12th International Conference on Project Management (ProMac)},
	year = {2018},
	pages = {673-689},
	publisher = {The Society of Project Management},
	address = {Tokyo, Japan},
	abstract = {Traditionally, a high degree of requirement change has been considered harmful for the success of software projects. Software professionals who use agile software development methods tend to view this topic differently. They tend to view requirement changes more as opportunities, which should be welcomed. Possibly, both views are correct but valid in different software development contexts. This paper aims at increasing the understanding of the connections between the degree of requirement change, choice of development method, and project success. Seventy software professionals were asked to provide information about their last software project. A higher degree of requirement changes, here defined as more than 30\% of the requirements added, deleted, or changed during the project{\textquoteright}s execution, was connected with a higher proportion of successful projects in an agile development context, but only when this included frequent deliveries to production. Our results consequently support that the agile claim of {\textquotedblleft}embrace change{\textquotedblright} has merit, but only in agile contexts.},
	author = {J{\o}rgensen, Magne}
}
@book {26129,
	title = {Time predictions: Understanding and avoiding unrealism in project planning and everyday life},
	series = {Simula SpringerBriefs on Computing},
	number = {5},
	year = {2018},
	pages = {110},
	publisher = {Springer},
	organization = {Springer},
	address = {Switzerland},
	author = {Halkjelsvik, Torleif and J{\o}rgensen, Magne}
}
@misc {26280,
	title = {Usikkerhetsanalyse },
	howpublished = {HIT-seminar, Oslo, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26266,
	title = {What can - and should - empirical software engineering learn from empirical studies in psychology?},
	howpublished = {12th International Symposium on    Empirical Software Engineering and Measurement, Oulu, Finland},
	year = {2018},
	abstract = {Software development is about people solving problems. They do this alone or in groups, for themselves or for others, with knowledge, experiences and biases. People solving problems have been empirically studied for more than two hundreds years in psychology, affecting not only treatment processes, but also economics, management, marketing, teaching and numerous other disciplines. What can we learn from how psychology researchers do their empirical studies and succeed in achieving useful results and affect practice? In this talk I will examine similarities and differences in use of empirical methods in psychology and software engineering and summarize this in what I argue has the potential of improving the quality and impact of our empirical software engineering studies.},
	author = {J{\o}rgensen, Magne}
}
@misc {26279,
	title = {What makes software projects successful?},
	howpublished = {Workshop with Government of Bangladesh Delegation (ECNEC), Oslo, Norway},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26267,
	title = {What makes software projects successful?},
	howpublished = {ICSSE, Prague, Czech Republic},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@misc {26265,
	title = {When is agile better? How the use of agile and autonomous teams affect success differently in different contexts (and other results)},
	howpublished = {A-teams (First International Workshop on Autonomous Agile Teams), XP-workshop, Porto, Portugal},
	year = {2018},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {25184,
	title = {Connections between Contract Type, Project Size, Benefits Management, Agile Practices and Success of Software Development Projects},
	journal = {FITAT 2017},
	year = {2017},
	publisher = {Mongolia University},
	address = {Ulan Bator, Mongolia},
	abstract = {The study examines connections between the outcome of software development projects and contract type, project size, benefits management practices, and agile practices. The collected information relates to 146 software projects{\textemdash}half from 2007-2012 and the other half from 2015-2017. Binary logistic models built to explain successful and problematic software development projects revealed that the use of time and materials contracts, as opposed to fixed price contracts, was associated with fourteen times higher likelihood of project success. Other practices with a strong connection to project outcome included frequent delivery to production and benefit management during project execution. We found that the explanatory strength of a practice sometimes differs substantially in models of successful and in models of problematic projects, as well as over time.},
	author = {J{\o}rgensen, Magne}
}
@article {25110,
	title = {Direct and indirect connections between type of contract and software project outcome},
	journal = {International Journal of Project Management},
	volume = {35},
	number = {1573-1586},
	year = {2017},
	publisher = {Elsevier},
	abstract = {This paper reports two empirical studies on how the use of different contract types affects the outcomes of software projects. The first study evaluates the effect of contract type on project failure using information from a large international dataset of small-scale, outsourced software projects and tasks. The second study proposes and tests how the use of contracts is connected with project outcome using information about Norwegian software projects with a public client. Both studies find that the use of fixed price contracts is connected with a higher risk of project failure compared to time and materials types of contracts. The results from the second study suggest that different project outcomes with different contract types is explained by differences in how the provider is selected, how the client is involved in the project, the use of agile practices and the use of benefit management during project execution.},
	doi = {10.1016/j.ijproman.2017.09.003},
	author = {J{\o}rgensen, Magne and Mohagheghi, Parastoo and Grimstad, Stein}
}
@misc {25746,
	title = {Empirical methods and evidence-based decisions in software engineering},
	howpublished = {Seminar at Innsbruck University, Austria},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@misc {25745,
	title = {Evaluering av digitalisering i offentlig sektor. Hvor gode er vi? Evaluerer vi det som er viktig?},
	howpublished = {Evalueringskonferansen, Oslo, Norway},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@misc {25748,
	title = {Evidensbasert praksis og misvisende nyheter},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {25752,
	title = {Femti, feit og ferdig, eller stadig mer produktiv?},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {25751,
	title = {Frederick Winslow Taylor: Prosessforbedring industrien},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@article {25111,
	title = {Guest editorial for special section on success and failure in software engineering},
	journal = {Empirical Software Engineering},
	volume = {22},
	number = {2281-2297},
	year = {2017},
	publisher = {Elsevier},
	author = {M{\"a}ntyl{\"a}, Mika and J{\o}rgensen, Magne and Ralph, Paul and Erdogmus, Hakan}
}
@misc {25741,
	title = {How to succeed and avoid failing with software projects},
	howpublished = {Seminar "International Trends in Software Development", Kathmandu, Nepal},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@misc {25749,
	title = {Husk {\r a} glemme},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {25744,
	title = {Hva kjennetegner IT-prosjekter som lykkes? (Resultater fra SMIOS-prosjektet)},
	howpublished = {Eleven presentations: 1) Oslo Kommune (Oslo), 2) Skatteetaten (T{\o}nsberg), 3) SINTEF (Trondheim), 4) SPK (Oslo), 5) Sparebank 1 (Oslo), 6) Miles (Oslo), 7) Posten (Oslo), 8) Vinmonopolet (Oslo), 9) FHI (Oslo), 10) Knowit (Oslo), 11) Eika IT (Oslo) , Norway},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@misc {25747,
	title = {Kravendringer i IT-prosjekter: trussel eller mulighet?},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {25750,
	title = {N{\r a}r regnereglene du l{\ae}rte p{\r a} skolen gir feil svar},
	year = {2017},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {25743,
	title = {Nyttestyring og viktigheten av den gode kunde},
	howpublished = {Prosjekt 2017 (Prosjekt Norges {\r a}rlige prosjektlederkonferanse), Oslo, Norway},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {25109,
	title = {Software development contracts: The impact of the provider{\textquoteright}s risk of financial loss on project success},
	journal = {ICSE-workshop: CHASE},
	year = {2017},
	pages = {30-35},
	publisher = {IEEE Press},
	author = {J{\o}rgensen, Magne}
}
@misc {25742,
	title = {Usikkerhetsanalyse med avhengigheter i en h{\o}yreskjev verden},
	howpublished = {HIT-seminar, Oslo, Norway},
	year = {2017},
	author = {J{\o}rgensen, Magne}
}
@article {25654,
	title = {What Contributes to the Success of IT Projects? An Empirical Study of IT Projects in the Norwegian Public Sector},
	journal = {Journal of Software},
	volume = {12},
	year = {2017},
	pages = {751-758},
	publisher = {IAP},
	author = {Mohagheghi, Parastoo and J{\o}rgensen, Magne}
}
@misc {25655,
	title = {What contributes to the success of IT projects?: success factors, challenges and lessons learned from an empirical study of software projects in the Norwegian public sector},
	howpublished = {ICSE},
	year = {2017},
	publisher = {IEEE Press},
	address = {Buenos Aires, Argentina},
	abstract = {Context.\&nbsp;Each year the public sector invests large amounts of money in the development and modifications of their software systems. These investments are not always successful and many public sector software projects fail to deliver the expected benefits.\&nbsp;Goal.\&nbsp;This study aims at reducing the waste of resources on failed software projects through better understanding of the success factors and challenges.\&nbsp;Method.Thirty-five completed software projects in 11 organizations in the public sector of Norway were analyzed. For each project, representatives from the project owners, project management and the user organization were interviewed.\&nbsp;Results.\&nbsp;Small and large software projects reported different challenges, especially related to project priority. Taking advantage of agile practices such as flexible scope and frequent delivery increased the success rate of the projects. Projects with time and material contracts and involved clients during execution were more successful than other projects. The respondents experienced that extensive involvement and good competence of the client, high priority of the project, good dialogue between client and provider and appliance of agile practices were main success factors. Main challenges were related to technical issues, project planning and management, transition of the product to the user organization, involvement and competence of the client, and benefit management.\&nbsp;Conclusions.\&nbsp;Success factors tend to focus on human factors, e.g., involvement, competence and collaboration. Challenges focus on human factors as well as issues of technical nature. Both aspects need to be addressed to enable successful and avoid failed software projects. Competence, client involvement and benefit management are among factors that the public sector should focus on for realizing client benefits.},
	author = {Mohagheghi, Parastoo and J{\o}rgensen, Magne}
}
@misc {25738,
	title = {What it actually is a working hour?},
	year = {2017},
	address = {15th European Congress of Psychology, Amsterdam, the Netherlands},
	author = {Filkukova, Petra and J{\o}rgensen, Magne}
}
@misc {25186,
	title = {What makes software projects successful?},
	howpublished = {ICSIE, Singapore},
	year = {2017},
	address = {Singapore},
	abstract = {Numerous research studies and consultancy reports make claims about how often, or rather how seldom, software projects are successful, why so many of them fail, and how to succeed more often. These studies and reports have reported very much the same success and failure factors and the same advices since the 1960s. If we already know how to make a successful software project, why is the proportion of failed software projects about the same as earlier? Are software professionals ignorant of the published knowledge or are there other reasons? One reason may be that previous studies have had very little focus on the most important success dimension, i.e., client benefits, contain very little practical advice on how to succeed, and have not managed to include the context-dependency and complexity of the connections between process choices and outcome. In this keynote I present recent attempts to better define and operationalize project success and better analyse and describe the context-dependent and probabilistic network of connections between essential choices made early and the outcome of software projects. The hope is that this may inspire other researchers to emphasise research on software project management that results in more context-aware results of practical use for software professionals.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {25185,
	title = {Working with Industry},
	journal = {5th IEEE/ACM International Workshop on Conducting Empirical Studies in Industry, CESI@ICSE 2017},
	year = {2017},
	pages = {46-52},
	publisher = {IEEE Press},
	abstract = {The software engineering industry should be the laboratory of much, perhaps most, of the empirical software engineering research. Not only would this create a more realistic context and higher external validity of the empirical research, it would also ease the result transfer and make the results more convincing for the industry. Unfortunately, this is currently not the case. About 90\% of software engineering experiments are, for example, conducted with students instead of software professionals as subjects. One reason for the lack of industry studies may be that an efficient and sustainable give-and-take-based collaboration between research and industry can be difficult to establish. The collaborations are frequently fragile, end before the research is completed, and lead to a waste of resources for both the researchers and the industrial partners. This paper presents stories and lessons learned from failed and successful research-industry collaborations. It has a focus on experience with the use of non-traditional collaboration types, such as payment to get industry participation in experiments, trade-based collaboration, lightweight collaborations at industry venues, and network-based collaborations. It is argued that empirical software engineering research should more often consider the use of alternative types of research{\textendash}industry collaborations than those traditionally chosen.},
	doi = {10.1109/CESI.2017.3},
	author = {J{\o}rgensen, Magne}
}
@misc {25188,
	title = {Working with industry to conduct empirical software engineering research: Patterns of successful and failed collaborations},
	howpublished = {ICSE-workshop: CESI, Buenos Aires, Argentina},
	year = {2017},
	publisher = {IEEE Press},
	abstract = {The industry should be the laboratory of a large part of empirical SE research. Not only does this create a more realistic context for the empirical research, it also eases the result transfer and makes them more convincing for the industry. Unfortunately, this is currently not the case. About 90\% of SE experiments are, for example, conducted with students instead of software professionals as subjects. One reason for the lack of industry studies is that a good and sustainable {\textquotedblleft}give-and-take{\textquotedblright}-based collaboration with industry can be difficult to establish. The collaborations are frequently fragile, end before the research is completed, and lead to a waste of resources for both the researchers and the industrial partners. This keynote presents experience from different, failed and successful, academia-industry collaborations with the goal of identifying approaches that are more likely to give the researchers insight and opportunities to evaluate methods and tools, and the industry benefits that make the collaborations worthwhile for them. A special focus will be on experiences with the use of {\textquotedblleft}lightweight{\textquotedblright} collaborations, collaboration where the researchers use part of their budget to create monetary incentives for the industry to collaborate, and the use of evidence-based SE as a supporting tool for industry collaborations.},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.2116,
	title = {Better Selection of Software Providers Through Trialsourcing},
	journal = {IEEE Software},
	volume = {33},
	year = {2016},
	pages = {48-53},
	publisher = {Elsevier},
	abstract = {In this article we show that differences between software providers in terms of productivity and quality can be very large and that traditional means of evaluating software providers, such as CVs and evaluations by reference clients, fail to separate the competent from the incompetent ones. We argue and motivate through analyses of empirical data that software clients would benefit from using work sample-based tests of software development providers\&$\#$39; performance (trialsourcing) to guide their selection of providers. We discuss potential challenges of trialsourcing and provide recommendations on what to consider when designing proper trialsourcing processes.},
	author = {J{\o}rgensen, Magne}
}
@article {24496,
	title = {Cultural characteristics and their connection to increased risk of software project failure},
	journal = {Journal of Software},
	volume = {11},
	number = {6},
	year = {2016},
	pages = {606-614},
	publisher = {International Academy Publisher},
	abstract = {Abstract\&mdash;Offshoring software projects to low cost countries has the potential of reducing the cost of software development and increasing the availability of competent development resources. It has, however, also been documented that it can increase the risk of project failure. In particular, the cultural differences between software client and provider countries are believed to increase the risk of project failure. The main goal of our study is to empirically examine how cultural characteristics of and cultural differences between software client and provider countries are connected with an increased risk of project failure. We analyze a large data set of small software projects with providers and clients from various countries. For each provider and client country, and each combination of provider and client country, we calculate the project failure rate. We use Hofstede\&rsquo;s culture dimensions power distance, individualism, masculinity and uncertainty avoidance together with Hall\&rsquo;s concept of high and low-context cultures (communication style) to measure cultural characteristics and differences. We found a statistically significant increase in failure rate with increased difference in the provider and client country\&rsquo;s communication style, but no connection between cultural differences and project failure rate using Hofstede\&rsquo;s cultural characteristics. Provider countries, and in particular, client countries with a low-context culture had lower proportions of project failures than countries with high-context cultures. A decrease in power distance and increase in individualism were both connected with a decrease in project failure rate. In total, the cultural differences between countries seem to matter less than cultural characteristics of the countries. In particular, some of the cultural characteristics of the client country were strongly connected with project failure rates. Software offshoring clients may benefit from an awareness of the importance of their role in avoiding project failures and consider adopting the cultural characteristics of the most successful client offshoring countries. They may also benefit from selecting software providers from low-context cultures.},
	author = {J{\o}rgensen, Magne and Yamashita, Aiko}
}
@misc {24465,
	title = {Enkelt er bra, uforst{\r a}elig er bedre?},
	howpublished = {Article in Computerworld Norway},
	year = {2016},
	month = {12/2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24464,
	title = {Estimering av IT-utvikling},
	howpublished = {KnowIT, Estimation seminar, Oslo},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24921,
	title = {Evidence-based software engineering: A framework for collaboration between researchers and software professionals},
	howpublished = {Amsterdam},
	year = {2016},
	publisher = {Seminar at Centrum voor Wiskunde en Informatica (CWI)},
	author = {J{\o}rgensen, Magne}
}
@misc {23971,
	title = {Fixed Price without Fixed Specifications},
	howpublished = {G{\o}teborg},
	volume = {Keynote at REFSQ (Requirement Engineering Foundation for Software Engieering)},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24463,
	title = {How to identify risky IT projects and avoid them turning into black swans},
	howpublished = {Ernst \& Young: Nordic Advisory Learning Weekend, Riga},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24919,
	title = {Hva kjennetegner IT-prosjekter som lykkes},
	howpublished = {Oslo},
	year = {2016},
	publisher = {Seminars/conferences organised by LO-IT, Econa and Tekna (Det digitale skiftet)},
	author = {J{\o}rgensen, Magne}
}
@misc {24917,
	title = {Hva kjennetegner IT-prosjekter som lykkes (Resultater fra SMIOS-prosjektet)},
	howpublished = {Oslo},
	year = {2016},
	publisher = {Presentations at: HIT-seminar, NSB-seminar, nav-seminar, spk-seminar, Lyse-seminar, Thales-days, Digitaliseringsr{\r a}det, Finansdepartementet, KMD, transportkomiteen},
	author = {J{\o}rgensen, Magne and Mohagheghi, Parastoo and Grimstad, Stein}
}
@misc {24461,
	title = {Hva skiller IT-prosjekter som lykkes fra de som mislykkes? Hva vil det si {\r a} lykkes?},
	howpublished = {Computas brukerforum},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24914,
	title = {Hva skiller IT-prosjekter som lykkes fra de som mislykkes? Hva vil det si {\r a} lykkes?},
	howpublished = {Seminar at Fornebu Consulting},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.2758,
	title = {Incorrect Results in Software Engineering Experiments: How to Improve Research Practices},
	journal = {Journal of Systems and Software},
	volume = {117},
	year = {2016},
	pages = {274-281},
	publisher = {Elsevier},
	abstract = {Context: The trustworthiness of research results is a growing concern in many empirical disciplines. Aim: The goals of this paper are to assess how much the trustworthiness of results reported in software engineering experiments is affected by researcher and publication bias and to suggest improved research practices. Method: First, we conducted a small-scale survey to document the presence of researcher and publication biases in software engineering experiments. Then, we built a model that estimates the proportion of correct results for different levels of researcher and publication bias. A review of 150 randomly selected software engineering experiments published in the period 2002-2012 was conducted to provide input to the model. Results: The survey indicates that researcher and publication bias is quite common. This finding is supported by the observation that the actual proportion of statistically significant results reported in the reviewed papers was about twice as high as the one expected assuming no researcher and publication bias. Our models suggest a high proportion of incorrect results even with quite conservative assumptions. Conclusion: Research practices must improve to increase the trustworthiness of software engineering experiments. A key to this improvement is to avoid conducting studies with unsatisfactory low statistical power.},
	author = {J{\o}rgensen, Magne and Dyb{\r a}, Tore and Liest{\o}l, Knut and Sj{\o}berg, Dag Ingar Kondrup}
}
@misc {24915,
	title = {Judgment and decision-making in software engineering: How rational are we?},
	howpublished = {Tokyo, Japan},
	year = {2016},
	publisher = {ICSIE},
	author = {J{\o}rgensen, Magne}
}
@misc {24538,
	title = {Klarer vi {\r a} kontrollere kunstig superintelligens?},
	howpublished = {Computerworld},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {24911,
	title = {Kulturelle egenskaper og offshoring av IT-utvikling},
	year = {2016},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@misc {24912,
	title = {Kunde eller leverand{\o}rstyrt bemanning av IT-prosjekter},
	year = {2016},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@article {19127,
	title = {Numerical anchors and their strong effects on software development effort estimates},
	journal = {Journal of Systems and Software},
	volume = {116},
	year = {2016},
	pages = {49-56},
	publisher = {Elsevier},
	author = {L{\o}hre, Erik and J{\o}rgensen, Magne}
}
@misc {24918,
	title = {Successful IT projects: The role of the contract},
	howpublished = {Z{\"u}rich},
	year = {2016},
	publisher = {Seminar organized by the Swiss Association for Quality},
	author = {J{\o}rgensen, Magne}
}
@misc {24916,
	title = {Suksess med offentlige IT-prosjekter. Hva er det og hvordan f{\r a}r vi det til?},
	howpublished = {Trondheim},
	year = {2016},
	publisher = {Hemit-konferansen},
	author = {J{\o}rgensen, Magne}
}
@misc {24909,
	title = {Suksess som indikator for fremtidige skuffelser},
	year = {2016},
	publisher = {Article in Computerworld},
	author = {J{\o}rgensen, Magne}
}
@article {24528,
	title = {A Survey on the Characteristics of Projects with Success in Delivering Client Benefits},
	journal = {Information and Software Technology},
	volume = {78},
	year = {2016},
	pages = {83-94},
	publisher = {Elsevier},
	abstract = {Context: A large waste of resources in software development projects currently results from being unable to produce client benefits.Objective: The main objective is to better understand the characteristics of successful software projects and contribute to software projects that are more likely to produce the planned client benefits.Method: We asked 63 Norwegian software professionals, representing both the client and the provider role, to report information about their last completed project. In a follow-up survey with 64 Norwegian software professionals, we addressed selected findings from the first survey.Results: The analysis of the project information showed the following: i) The project management triangle criteria of being on time, on budget, and having the specified functionality are poor correlates of the essential success dimension client benefits. ii) Benefit management planning before the project started and benefit management activities during project execution were connected with success in delivering client benefits. iii) Fixed-price projects and projects in which the selection of providers had a strong focus on low price were less successful in delivering project benefits than other projects. iv) Agile projects were in general more successful than other projects, but agile projects without flexible scope to reflect changed user needs and learning, or without frequent delivery to the client, had less than average success in delivering client benefits.Conclusions: The software projects that were successful in delivering client benefits differed from the less successful ones in several ways. In particular, they applied benefit management practices during project execution, they avoided fixed-price contracts, they had less focus on low price in the selection of providers, and they applied the core agile practices - frequent delivery to the client and scope flexibility.},
	author = {J{\o}rgensen, Magne}
}
@article {24527,
	title = {Unit effects in software project effort estimation: Work-hours gives lower effort estimates than workdays},
	journal = {Journal of Systems and Software},
	volume = {117},
	year = {2016},
	pages = {274-281},
	publisher = {Elsevier},
	abstract = {Software development effort estimates are typically expert judgment-based and too low to reflect the actual use of effort. Our goal is to understand how the choice of effort unit affects expert judgement-based effort estimates, and to use this knowledge to increase the realism of effort estimates. We conducted two experiments where the software professionals were randomly instructed to estimate the effort of the same projects in work-hours or in workdays. In both experiment, the software professionals estimating in work-hours had much lower estimates (on average 33\%\&ndash;59\% lower) than those estimating in workdays. We argue that the\&nbsp;unitosity\&nbsp;effect\&mdash;i.e., that we tend to infer information about the quantity from the choice of unit\&mdash;is the main explanation for the large difference in effort estimates. A practical implication of the unit effect is that, in contexts where there is a tendency toward effort under estimation, the instruction to estimate in higher granularity effort units, such as workdays instead of work-hours, is likely to lead to more realistic effort estimates.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {23969,
	title = {The Use of Precision of Software Development Effort Estimates to Communicate Uncertainty},
	journal = {Software Quality Days. The Future of Systems-and Software Development},
	year = {2016},
	pages = {156-168},
	publisher = {Springer International Publishing},
	abstract = {The precision of estimates may be applied to communicate theuncertainty of required software development effort. The effort estimates 1000and 975 work-hours, for example, communicate different levels of expectedestimation accuracy. Through observational and experimental studies we foundthat software professionals (i) sometimes, but not in the majority of theexamined projects, used estimate precision to convey effort uncertainty, (ii)tended to interpret overly precise, inaccurate effort estimates as indicating lowdeveloper competence and low trustworthiness of the estimates, while toonarrow effort prediction intervals had the opposite effect. This differenceremained even when the actual effort was known to be outside the narrow effortprediction interval. We identified several challenges related to the use of theprecision of single value estimates to communicate effort uncertainty andrecommend that software professionals use effort prediction intervals, and notthe preciseness of single value estimates, to communicate effort uncertainty.},
	author = {J{\o}rgensen, Magne}
}
@misc {24462,
	title = {Vurderinger og beslutninger. Hvor rasjonelle er vi?},
	howpublished = {DND{\textquoteright}s Software Conference},
	year = {2016},
	author = {J{\o}rgensen, Magne}
}
@misc {23646,
	title = {{\r A} (mis)lykkes med IT-prosjekter},
	howpublished = {Presentation at an IKT Norge-seminar},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23919,
	title = {Are most published research findings in empirical software engineering wrong or with exaggerated effect sizes? How to improve?},
	howpublished = {ISERN-meeting, Beijing, China},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {23599,
	title = {Believing is Seeing: Confirmation Bias Studies in Software Engineering},
	journal = {41st Euromicro Conference on Software Engineering and Advanced Applications (SEAA), Madeira},
	volume = { },
	year = {2015},
	pages = {92-95},
	publisher = {IEEE},
	abstract = {Abstract{\textemdash}Confirmation bias is the human tendency to search for, collect, interpret, analyse, or recall information in a way that confirms one{\textquoteright}s prior beliefs or preferences. In this paper, we review previous research and demonstrate confirmation bias and its effect in two software engineering contexts. The first study documents that managers bias their interpretation of randomly generated project data towards confirmation of their preferred contract type. The second study reports from an analysis of the results of 35 published comparisons of regression and analogy-based cost estimation models. Twenty of these comparisons evaluate the performance of a self-developed analogy-based estimation model relative to a regression-based model and may be biased towards finding evidence confirming a better accuracy of their own model. A statistical meta-analysis of all 35 comparisons showed that the analogy-based models were more accurate than the regression-based models, whereas removing the 20 comparisons where the researchers evaluated their own models gave the opposite result. Our results support the idea that it is important to consider the presence and degree of confirmation bias when collecting information about topics where there are prior preferences, beliefs, or vested interests to avoid misleading conclusions.},
	author = {J{\o}rgensen, Magne and Papatheocharous, Efi}
}
@misc {23917,
	title = {Den gode kunde: Kompetanse, involvering og kultur},
	howpublished = {HIT-seminar, Oslo},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {23897,
	title = {The Effect of the Time Unit on Software Development Effort Estimates},
	journal = {9th International Conference on Software, Knowledge, Information Management \& Applications (SKIMA), Kathmandu, Nepal},
	year = {2015},
	pages = {178-182},
	publisher = {IEEE},
	abstract = {Estimates of software development effort are frequently inaccurate and over-optimistic. In this paper we describe how changes in the granularity of the unit of estimation, e.g., work-days instead of work-hours, affects the effort estimates. We describe four psychological mechanisms, how they interact and discuss the expected total effect of higher granularity units on effort estimates. We argue that the mechanisms in general imply that higher granularity effort units will result in higher effort estimates, e.g., that estimating software development work in work-days or weeks will lead to higher estimates than when estimating in work-hours. A possible implication of this predicted effect is that, in contexts where there is a tendency towards under-estimation, estimation in work-days or weeks instead of work-hours leads to more realistic estimates.},
	author = {J{\o}rgensen, Magne}
}
@misc {23650,
	title = {Empirical Methods and Evidence-Based Decisions in Software Engineering},
	howpublished = {Oulu, Finland},
	year = {2015},
	type = {Seminar at the Software Engineering Department},
	author = {J{\o}rgensen, Magne}
}
@misc {23657,
	title = {Er tiden for de store IT-anskaffelsene n{\r a} over?},
	howpublished = {Article in Computerworld Norge},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23921,
	title = {Estimering av IT-prosjekter: Hvordan bli bedre til {\r a} treffe med estimater, planer og budsjetter},
	howpublished = {Politiet, Oslo},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23922,
	title = {From fashion-based to evidence-based software engineering},
	howpublished = {Teleplan, Oslo \& Wroclaw, Poland},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23651,
	title = {From myths and fashions to evidence-based software engineering},
	howpublished = {Stockholm, SICS seminar},
	year = {2015},
	type = {Seminar talk at software engineering department},
	author = {J{\o}rgensen, Magne}
}
@misc {23920,
	title = {Hva skal til for {\r a} lykes i IT-prosjekter? Hvor mye og hvordan kan man l{\ae}re av andres suksesser og fiaskoer?},
	howpublished = {H{\o}gskolen i Hamar},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23652,
	title = {Hva skal til for {\r a} lykkes i IT-prosjekter? Hvor mye og hvordan kan man l{\ae}re av andres suksesser og fiaskoer?},
	howpublished = {DnD{\textquoteright}s Software Conference, Oslo},
	year = {2015},
	type = {DnD Software Conference},
	author = {J{\o}rgensen, Magne}
}
@misc {23647,
	title = {Hva vet vi om IT-bransjens evne til {\r a} levere nyttige l{\o}sninger med god kvalitet?},
	howpublished = {Keynote at Computerworld Norges Round Table seminar},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23918,
	title = {Hvilke IT-prosjekter lykkes best},
	howpublished = {Trondheim, NOKIOS},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23645,
	title = {Hvilken betydning har kontrakten for suksess i IT-prosjekter?},
	howpublished = {Presentation at a DnD seminar},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23643,
	title = {Ingen flere store offentlige IT-prosjekter?},
	howpublished = {Presentation at Difi seminar},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23653,
	title = {Med h{\o}y presisjon},
	howpublished = {Article in Computerworld Norge},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23923,
	title = {Offshoring av IT-utvikling: Hva er viktig for {\r a} lykkes},
	howpublished = {Digin-seminar, Kristiansand},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23656,
	title = {Sitte i celle eller i {\r a}pent kontorlandskap?},
	howpublished = {Article in Computerworld Norge},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23915,
	title = {Software development effort estimation: How to improve it},
	howpublished = {Deerwalk seminar, Kathmandu, Nepal},
	year = {2015},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {23644,
	title = {Suksess og fiasko i offentlige IKT-prosjekter},
	howpublished = {Presentation at a seminar organized by the Ministry of Local Government and Modernisation},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23654,
	title = {Tallverdieffekten: Vi rekker mer p{\r a} tolv m{\r a}neder enn ett {\r a}r},
	howpublished = {Article in Computerworld Norge},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23924,
	title = {Trenger vi mer lokal eksperimentering?},
	howpublished = {Article in Computerworld Norway},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23649,
	title = {The use and misuse of statistics in studies on software development. Things you should know about statistics that you didn{\textquoteright}t learn in school},
	howpublished = {Keynote at ICSIE, Dubai, UAE},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23925,
	title = {Vellykkede IT-prosjekter krever gode kunder},
	howpublished = {Article in Computerworld Norway},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23655,
	title = {Verden er stort sett ikke normal, men h{\o}yreskjev},
	howpublished = {Article in Computerworld Norge},
	year = {2015},
	author = {J{\o}rgensen, Magne}
}
@misc {23916,
	title = {The world is skewed},
	howpublished = {CREST-seminar, London},
	year = {2015},
	publisher = {CREST},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.2498,
	title = {Communication of Software Cost Estimates},
	journal = {Proceedings of the 18th International Conference on Evaluation and Assessment in Software Engineering},
	year = {2014},
	pages = {241-245},
	publisher = {ACM},
	type = {Conference},
	address = { },
	abstract = {The meaning of an effort or cost estimate should be understood and communicated consistently and clearly to avoid planning and budgeting mistakes. Results from two studies, one of 42 software companies and one of 423 individual software developers, suggest that this is far from being the case. In both studies we found a large variety in what was meant by an effort estimate and that the meaning was frequently not communicated. To improve the planning and budgeting of software projects we recommend that the meaning of effort estimates is understood and communicated using a probability-based terminology.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2882,
	title = {Den Gode Kunden - Viktigere Enn Du Kanskje Tror},
	howpublished = {Difi-seminar + NSB seminar},
	year = {2014},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.simula.2738,
	title = {The Effect of Facial Expression on Evaluation of Competence: Keep Smiling!},
	howpublished = {Poster at: 17th General Meeting of the European Association of Social Psychology, The Netherlands, Amsterdam},
	year = {2014},
	month = {July},
	author = {Filkukova, Petra and J{\o}rgensen, Magne}
}
@misc {19111,
	title = {Erfaringer fra it-prosjekter. Fokus p{\r a} kompetanse og kvalitet, som basis for {\r a} levere et forventet resultat},
	howpublished = {Invited talk: Computerworld Round Table},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {19117,
	title = {Estimering av IT-prosjekter: Hva vet vi virker?},
	howpublished = {Presentation at Thales seminar},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {19115,
	title = {Estimering av IT-prosjekter: Hvordan bli bedre til {\r a} treffe med estimater, planer, budsjetter og pris?},
	howpublished = {Presentation at Teleplan-seminar},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.1986,
	title = {Failure Factors of Small Software Projects at a Global Outsourcing Marketplace},
	journal = {Journal of Systems and Software},
	volume = {92},
	number = {x},
	year = {2014},
	pages = {157-169},
	abstract = {The presented study aims at a better understanding of when and why software projects fail. The analysis is based on a data set of 785,325 small-scale outsourced software projects. A binary logistic regression model relying solely on information known at the time of a project{\textquoteright}s start-up correctly predicted 74\% of the project failures and 67\% of the non-failures. The model-predicted failure probability corresponded well with the actual frequencies of failures for most levels of failure risk. The model suggests that the factors connected to the strongest reduction in the risk of failure are related to previous collaboration between the client and the provider and a low failure rate of previous projects completed by the provider. We found the characteristics of the client to be almost as important as those of the provider in explaining project failures and that the risk of project failure increased with an increased client emphasis on low price and with an increased project size. The identified relationships seem to be stable across the studied project size categories, which indicate that the results may potentially be applicable to larger projects than the small-scale projects dominating this data set.},
	keywords = {Conference, Internal Seminar, University of Oslo, Workshop},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.2762,
	title = {Fallacies and Biases When Adding Effort Estimates},
	journal = {Proceedings at Euromicro/SEEA},
	year = {2014},
	pages = {277-284},
	publisher = {IEEE},
	type = {Conference},
	address = { },
	abstract = {Software professionals do not always clarify what they mean by their effort estimates. Knowing what is meant by an estimate is, however, essential when adding individual effort estimates from a work breakdown structure to find the estimated total effort. Adding the most likely instead of the mean effort of a set of cost elements may result in substantial underestimation of the total effort. In a survey of forty-four software companies we found only two companies that clarified the meaning of their estimates and had a proper method for adding these estimates. The other companies typically added single point estimates without clarifying what they added or with types of estimates likely to give too low estimates of the total effort. We examine the effect of improper addition of estimates and find, for the studied contexts, that summing the most likely effort estimates would lead to a substantial under-estimation of the most likely total effort. We also find that the use of the PERT-method, which provides a proper statistical basis for adding effort estimates and is used by many software companies, is likely to underestimate the effort in software development contexts. This is caused by, we argue and illustrate with empirical data, people{\textquoteright}s tendency towards providing too narrow minimum and maximum effort intervals. We outline a method that, we believe, better ensures that proper estimates of the total effort are produced.},
	author = {J{\o}rgensen, Magne}
}
@misc {19114,
	title = {Fra myter og moter til evidensbasert IT-utvikling},
	howpublished = {Presentation at TEKNA-seminar},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.3006,
	title = {From Myths and Fashions to Evidence-Based Software Engineering},
	howpublished = {Keynote at: Lindholmen Software Day, Gothenburg, Sweeden},
	year = {2014},
	abstract = {The software engineering discipline contains numerous myths and over-simplifications. Some of them may be harmless, but others may lead to inefficient practices and contribute to a fashion- and myth-based software engineering discipline. In this presentation I give examples of software engineering myths and over-simplifications, discuss how they are created and spread and illustrate how it is possible to base important software engineering decision and practice on available evidence from research, practice and own empirical studies. A move towards evidence-based software engineering requires that software professionals become more critical towards claims, know how to formulate answerable questions, collect and evaluate evidence and use evidence to guide important decisions. This requires training and, not least, a change in mindset. Results on how to do this are presented.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2760,
	title = {From Myths, Fashion, Over-Simplifications and Non-Validated Claims to Evidence-Based Software Engineering},
	howpublished = {Distinguished Lecturers Series Leon the Mathematician, University of Thessaloniki},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {19125,
	title = {Hva er et kostnadsestimat?},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {19120,
	title = {Hvor ofte og n{\r a}r lykkes IT-prosjekter?},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {19112,
	title = {Hvordan ansl{\r a} usikkerhet til estimater},
	howpublished = {Presentation at JavaZone (repeated for NTNU-students as JavaZone Academy presentation)},
	year = {2014},
	publisher = { },
	address = { },
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {19124,
	title = {Hvordan gj{\o}re statistikk mer overbevisende?},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2880,
	title = {Hvordan Unng{\r a} Skuffelser i IT-Prosjekter og Ellers i Livet},
	howpublished = {Miles seminar},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.2500,
	title = {The Ignorance of Confidence Levels in Minimum-Maximum Software Development Effort Intervals},
	journal = {ICSIE, Dubai},
	volume = {Lecture Notes on Software Engineering, 2(4)},
	year = {2014},
	pages = {327-340},
	publisher = { },
	type = {Workshop},
	address = { },
	abstract = {Software professionals are frequently asked to provide minimum-maximum effort intervals for a given confidence level. They may for example be asked to provide minimum-maximum intervals where it is 90\% likely to include the actual use of effort. If their response is the interval from 800 to 1200 work-hours this should correspond to that it is 90\% likely that the actual effort will be more than 800 and less than 1200 work-hours. This effort interval information is, amongst others, used in the planning and budgeting of software projects. In this paper we show that software professionals tend to ignore the confidence levels connected with the minimum-maximum effort intervals. As a consequence, the meaning of minimum-maximum effort interval is unclear and the use of such intervals questionable. The experiment used to document the ignorance of the confidence level is based on requesting one group of software developers to be 98\% confident, and another group to be 80\% confident when providing their effort intervals. In spite of a difference in confidence levels that should generate quite difference effort intervals, the actual intervals were almost the same. This finding challenge commonly recommended effort uncertainty assessment practices, e.g., those implemented in the PERT method, which are based on the assumption that software professionals are able to provide minimum-maximum effort that reflect the stated confidence levels. The finding does also challenge the explanation typically given for too narrow confidence intervals, i.e., that people are over-confident. Instead, we propose that a more likely explanation is that people ignore the confidence level when setting the minimum and maximum values.},
	author = {J{\o}rgensen, Magne}
}
@misc {19126,
	title = {Kommunikasjon av kostnadsestimater},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {19123,
	title = {L{\o}nner det seg med mer utdannelse?},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2690,
	title = {Myths, Over-Simplifications and Unconfirmed Claims in Software Engineering},
	howpublished = {Keynote at: EUROMICRO DSD/SEAA, Verona, Italy},
	year = {2014},
	abstract = {The software engineering discipline contains numerous myths, over-simplifications and unconfirmed claims. Some of them may be harmless, but others may hamper the use of good practices. Examples of myths, over-simplifications and unconfirmed claims are presented to demonstrate the mechanisms contributing to the creation, spread and survival of them. Ten years ago we (Barbara Kitchenham, Tore Dyb{\r a} and myself) introduced evidence-based software engineering. The goal was to support a move towards a discipline with practices based on valid scientific and experience-based evidence, i.e., to reduce the influence from myths, over-simplifications and unconfirmed claims. I report from my experience in providing training in the use of evidence-based practices to software professionals and computer science students. The experience includes the need to emphasize and train software professionals in the collection and evaluation of practice-based experience and in the design and completion of studies within their own context, not so much in collecting and evaluating research-based evidence.},
	author = {J{\o}rgensen, Magne}
}
@misc {19122,
	title = {N{\r a}r vi har bestemt oss, ser mye annerledes ut},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {19113,
	title = {Offshoring av IT-utvikling: Hva er viktig for {\r a} lykkes?},
	year = {2014},
	publisher = { },
	type = {Presentation at DIGIN-seminar},
	author = {J{\o}rgensen, Magne}
}
@misc {19119,
	title = {Praedico ergo sum},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {19118,
	title = {Prosjektstyring: Ikke mye nytt siden pyramidene},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2876,
	title = {Skumle IT-Prosjekter Og Sorte Svaner},
	howpublished = {Keynote at CIO Forum on Project Management},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {24102,
	title = {Ten years with Evidence-Based Software Engineering. What is it? Has it had any impact? What{\textquoteright}s next?},
	howpublished = {Hong Kong},
	year = {2014},
	publisher = {Impact paper award talk at: FSE},
	address = {Hong Kong},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2878,
	title = {Ten Years With Evidence-Based Software Engineering. What Is It? Has It Had Any Impact? What{\textquoteright}s Next?},
	howpublished = {Keynote on 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering},
	year = {2014},
	publisher = {ACM},
	abstract = {An evidence-based software engineer is one who is able to: 1) Formulate a question, related to a decision or judgment, so that it can be answered by the use of evidence, 2) Collect, critically evaluate and summarise relevant evidence from research, practise and local studies, 3) Apply the evidence, integrated with knowledge about the local context, to guide decisions and judgments. The keynote reflects on what it in practise means to be evidence-based in software engineering contexts, where the number of different contexts is high and the research-based evidence sparse, and why there is a need for more evidence-based practises. We summarise our experience from ten years of Evidence-Based Software Engineering in the context of university courses, training of software engineers and systematic literature reviews of software engineering research. While there are challenges in training people in evidence-based practise, our experience suggest that it is feasible and that the training can make an important difference in terms of quality of software engineering judgment and decisions. Based on our experience we suggest changes in how evidence-based software engineering should be presented and taught, and how we should ease the transfer of research results into evidence-based practises.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2688,
	title = {Things You Were Never Told, Did Not Understand, Forgot, Or Chose to Ignore in Statistics},
	howpublished = {Keynote at 1st Conference on Advanced Probability and Statistics in Information Systems (APSIS)},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {19116,
	title = {Usikkerhetsestimering av IT-prosjekter},
	howpublished = {Presentation at seminar for Kongsberg FMC Subsea},
	year = {2014},
	author = {J{\o}rgensen, Magne}
}
@misc {19121,
	title = {Visdommen til sm{\r a} grupper},
	year = {2014},
	publisher = { },
	type = {Article in Computerworld Norge},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.2440,
	title = {What We Do and Don{\textquoteright}t Know About Software Development Effort Estimation},
	journal = {IEEE Software},
	volume = {31},
	number = {March/April},
	year = {2014},
	pages = {37-40},
	abstract = {OVERWHELMING EVIDENCE DOCUMENTS a tendency toward cost and effort overruns in software projects. On average, this overrun seems to be around 30 percent.1 Furthermore, comparing the estimation accuracy of the 1980s with that reported in more recent surveys suggests that the estimation accuracy hasn{\textquoteright}t changed much since then. Estimation methods haven{\textquoteright}t changed much either. In spite of an extensive research on formal estimation models, the dominating estimation method is still expert estimation. An apparent lack of improvement in estimation accuracy doesn{\textquoteright}t mean that we don{\textquoteright}t know more about effort estimation than before. In this article, I try to summarize some of the knowledge I believe we{\textquoteright}ve gained. Some of this knowledge has the potential of improving the estimation accuracy, some is about what most likely will not lead to improvements, and some is about what we know we don{\textquoteright}t know about effort estimation.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2160,
	title = {Cognitive Bias Remains After De-Biasing Information},
	year = {2013},
	month = {July},
	abstract = {Context: The role of humans is becoming increasingly appreciated in our quest to improve software project prediction. The accuracy of the predictions is dependent on many factors, not least cognitive biases, which influence thinking and can often lead to poor decisions. Objective: The strong impact of the anchoring bias is well known. This study investigated the impact of anchoring on professional software engineers{\textquoteright} estimation accuracy after they had attended a workshop specifically aimed at addressing this bias. We hypothesised the bias would remain despite the workshop and that location could be a confound. Method: We ran four studies with a total of 118 software engineers to explore the impact of the anchoring bias on the accuracy of productivity estimations made by professional software engineers. The participants attended a workshop introducing the concept of cognitive bias and the influence on decision making and were introduced to strategies for de-biasing. Immediately after the workshop, participants were randomly allocated to a high or a low anchor group and asked to estimate their productivity as lines of code per hour (LOC{-}hr) on a recently completed project. Results: The anchor had a large effect (partial η2 \sim 0.22; p < 0.0001) in influencing estimates. The median estimate of hourly productivity for the high anchor was 30 LOC{-}hr and 7 LOC{-}hr for the low anchor. Lower values might reasonably be interpreted as more realistic estimates of hourly productivity. Conclusion: The impact of bias upon expert judgement is substantial, seemingly ubiquitous and difficult to eradicate. This study demonstrates the pervasive nature of the anchoring bias despite de-biasing information.},
	keywords = {Conference},
	author = {Mair, Carolyn and Shepperd, Martin and J{\o}rgensen, Magne}
}
@misc {Simula.simula.2348,
	title = {Det {\o}konomisk Mest Fordelaktige Tilbudet},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2332,
	title = {D{\o}dssynder Innen Kostnadsestimering},
	howpublished = {Presentation at SMIDIG 2013},
	year = {2013},
	keywords = {Conference},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.simula.2358,
	title = {Er {\textquotedblright}kopier Og Lim{\textquotedblright}-Programmering Skadelig?},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2346,
	title = {Estimater, Usikkerhet, Kommunikasjon. Hva Er Et Estimat? Hvordan Be Om Og Kommunisere Usikkerhet? Effekt Av Ulike Avtaletyper.},
	howpublished = {Presentation at internal seminar at Creuna},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2338,
	title = {Estimering Av IT-Prosjekter: Hvorfor Bommer Vi Og Hvordan Kan Vi Bli Bedre?},
	howpublished = {Presentation at five company internal seminars (KnowIT, Bouvet, Mesan, Politiet, Storebrand)},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2122,
	title = {Finding Results Where There Are None},
	howpublished = {Keynote at: IWSM-Mensura, Ankara, Turkey},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2342,
	title = {From Myths and Fashions to Evidence-Based Software Engineering},
	howpublished = {Presented at internal seminars at OneCall, Deerwalk (Nepal), Thales and at Certus workshop (Simula)},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2334,
	title = {How Much Is a Great Developer Worth?},
	howpublished = {Presentation at Norwegian Developer Conference (NDC) 2013},
	year = {2013},
	keywords = {Conference},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.simula.2336,
	title = {How to Avoid Disappointments in Software Projects (and Life)},
	howpublished = {Presentation at Gilb Week (London)},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2340,
	title = {Hvorfor (ikke) Fastpris? Vinnerens Forbannelse, Informasjonsasymmetri, Utvalgsrisiko, Opportunistisk Adferd Og IT-Kontrakter},
	howpublished = {Presented at internal seminar at Norsvin},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.1478,
	title = {The Influence of Selection Bias on Effort Overruns in Software Development Projects},
	journal = {Information and Software Technology},
	volume = {55},
	number = {9},
	year = {2013},
	pages = {1640-1650},
	abstract = {Context: A potentially important, but neglected, reason for effort overruns in software projects is related to selection bias. Selection bias-induced effort overruns occur when proposals are more likely to be accepted and lead to actual projects when based on effort estimates that are too low rather than on realistic estimates or estimates that are too high. The effect of this bias may be particularly important in bidding rounds, but is potentially relevant in all situations where there is effort or cost-based selection between alternatives. Objective: To better understand the relevance and management of selection bias effects in software development contexts. Method: First, we present a statistical model illustrating the relation between selection bias in bidding and other contexts and effort overruns. Then, we examine this relation in an experiment with software professionals who estimated and completed a set of development tasks and examine relevant field study evidence. Finally, we use a selection bias scenario to assess awareness of the effect of selection bias among software providers. Results: The results from the statistical model and the experiment demonstrated that selection bias is capable of explaining much of the effort overruns. The field evidence was also consistent with a substantial effect of selection bias on effort overruns, although there are alternative explanations for the findings. We found a low awareness of selection bias among the software providers. Conclusion: Selection bias is likely to be an important source of effort overruns and should be addressed to reduce problems related to over-optimistic effort estimates.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2118,
	title = {Judgement and Decision-Making in Software Engineering},
	howpublished = {Keynote at: QSIC, Nanjing, china},
	year = {2013},
	month = {July},
	keywords = {Conference},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2350,
	title = {Lederkompetanse Evaluert P{\r a} 100 Millisekunder},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2344,
	title = {Offshoring Av IT-Utvikling: Hva Er Viktig for {\r a} Lykkes?},
	howpublished = {Presented at internal seminar at Telenor},
	year = {2013},
	keywords = {Workshop},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2360,
	title = {Oppsiktsvekkende Resultater},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2356,
	title = {Presten Bayes, Kravspesifikasjoner Og Skandalejournalisme},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2352,
	title = {Problemer Med Offshoring Av IT. Kultur- Eller Kvalitetsproblemer?},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.814,
	title = {Relative Estimation of Software Development Effort: It Matters With What and How You Compare},
	journal = {IEEE Software},
	number = {March},
	year = {2013},
	pages = {74-79},
	abstract = {Software development effort estimation is frequently based on comparing the effort of one task relative to that of another. We present empirical results that show how relative estimation may result in biased assessments of similarity and over-optimistic effort estimates. We observe that tasks tend to be assessed as more similar than they in reality are when compared with each other, that the similarity of two tasks depends on the direction of the comparison and that it matters whether the comparison is based on difference in work-hours or as the ratio. We use the observations and other evidence to suggest ways of improving the accuracy of relative estimation.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.1924,
	title = {A Strong Focus on Low Price When Selecting Software Providers Increases the Likelihood of Failure in Software Outsourcing Projects},
	journal = {7th International Conference on Evaluation and Assessment in Software Engineering},
	year = {2013},
	pages = {220-227},
	type = {Conference},
	abstract = {Context: Bidding rounds are frequently used to select competent and cost-efficient providers for software projects. Objective: We hypothesize that emphasizing low price when selecting software providers in such bidding rounds substantially increases the likelihood the project will fail. Method: The hypothesis is tested by analyzing a dataset of 4,791,067 bids for 785,326 small-scale projects registered at a web-based marketplace connecting software clients and providers. Results: We find evidence supporting our hypothesis. For example, selecting providers with bids 25\% lower than the average bid is connected to a 9\% increase in the frequency of project failures for the same level of provider skill. In addition, we found that clients emphasizing a low price, on average, selected providers with lower skill levels. This decrease in provider skill level further strengthened the negative effect of a strong focus on low price on project failures. For example, selecting a provider with a 15\% failure rate for previous projects instead of 5\% increased the failure rate by 33\%. Conclusion: We interpret the findings to suggest that a client may substantially reduce the likelihood of project failure by reducing the emphasis on low price when selecting a provider.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.2354,
	title = {Ustrukturerte Ansettelsesintervjuer},
	year = {2013},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.1282,
	title = {Does the Use of Fibonacci Numbers in Planning Poker Affect Effort Estimates?},
	journal = {16th Int. Conference on Evaluation \& Assessment in Software Engineering},
	year = {2012},
	pages = {228-232},
	publisher = {IET},
	type = {Conference},
	abstract = {Background: The estimation technique Planning Poker is common in agile software development. The cards used to propose an estimate in Planning Poker do not include all numbers, but for example only the numbers 0, {\textonehalf}, 1, 2, 3, 5, 8, 13, 20, 40 and 100. We denote this, somewhat inaccurately, a Fibonacci scale in this paper. In spite of the widespread use of the Fibonacci scale in agile estimation, we do not know much about how this scale influences the estimation process. Aim: Better understanding of the effect of going from a linear scale to a Fibonacci scale in effort estimation. Method: We conducted two empirical studies. In the first study, we gave computer science students the same estimation task. Half of the students estimated the task using the Fibonacci scale and the other half a linear scale. The second study included four estimation teams, each composed of four software professionals, estimating the effort to complete the same ten tasks. Two of the teams estimated the first five tasks using the Fibonacci scale and the last five using the linear scale. The two other teams used the scales in the opposite sequence. Results: We found a median decrease in the effort estimates of 60\% (first study) and 26\% (second study) when using a Fibonacci scale instead of the traditional linear scale. The scale difference in the effort estimates decreased as the developers{\textquoteright} skill increased. Conclusion: The use of a Fibonacci scale, and possibly other non-linear scales, is likely to affect the effort estimates towards lower values compared to linear scales. A possible explanation for this scale-induced effect is that people tend to be biased towards toward the middle of the provided scale, especially when the uncertainty is substantial. The middle value is likely to be perceived as lower for the Fibonacci than for the linear scale.},
	author = {Tamrakar, Ritesh and J{\o}rgensen, Magne},
	editor = {M. T. {Baldassarre} and Genero, M. and Mendes, M. and Piattini, M.}
}
@inproceedings {Simula.simula.740,
	title = {First Impressions in Software Development Effort Estimation: Easy to Create and Difficult to Neutralize},
	journal = {16th Int. Conference on Evaluation \& Assessment in Software Engineering},
	year = {2012},
	pages = {216-222},
	publisher = {IET},
	type = {Conference},
	abstract = {The four studies included in this paper examine the strength of the first impression in the estimation of work effort. The studied context is mainly one where the first impressions of software development effort is manipulated (biased) through comparisons with either much too low or too high reference effort values, e.g., by responding to a question from the client whether one believes that the effort will be less than 10 work-hours when the likely usage of effort typically would be in the range of 100-300 work-hours. Then, the software developers are exposed to a subsequent comparison with reference effort values in the opposite direction, e.g., by responding to a question from the client whether one believes that the effort would be less than 800 work-hours. The results from the four studies suggest a strong first impression effect, but also a noticeable effect from the subsequent comparisons. We also observe that the instruction to {\textquotedblleft}forget{\textquotedblright} the first impression seems to have the opposite effect, i.e., it seems to increase the strength of the first impression. A practical implication of the results is that it is essential that software professionals ensure that their first impression of a project{\textquoteright}s development effort is based on comparisons with representative reference values and objects. First impressions in software development seem to be easy to manipulate with misleading reference values and difficult to replace.},
	author = {J{\o}rgensen, Magne and L{\o}hre, Erik},
	editor = {Maria Therese {Baldassarre} and Genero, Marcela and Mendes, Emilia and Piattini, Mario}
}
@misc {Simula.simula.1686,
	title = {Fra Myter Til Evidens},
	howpublished = {Invited talk (TAD-seminar)},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.30,
	title = {From Origami to Software Development: a Review of Studies on Judgment-Based Predictions of Performance Time},
	journal = {Psychological Bulletin},
	volume = {138},
	number = {2},
	year = {2012},
	pages = {238-271},
	author = {Halkjelsvik, Torleif and J{\o}rgensen, Magne}
}
@article {Simula.simula.742,
	title = {How Does Project Size Affect Cost Estimation Error? Statistical Artifacts and Methodological Challenges},
	journal = {International Journal of Project Management},
	volume = {30},
	number = {7},
	year = {2012},
	pages = {751-862},
	abstract = {Empirical studies differ in what they report as the underlying relation between project size and percent cost overrun. As a consequence, the studies also differ in their project management recommendations. We show that studies with a project size measure based on the actual cost systematically report an increase in percent cost overrun with increased project size, whereas studies with a project size measure based on the estimated cost report a decrease or no change in percent cost overrun with increased project size. The observed pattern is, we argue, to some extent a statistical artifact caused by imperfect correlation between the estimated and the actual cost. We conclude that the previous observational studies cannot be considered as providing reliable evidence in favor of an underlying project size related cost estimation bias. The more robust evidence from controlled experiments, limited to small tasks, suggests an increase in underestimation with increased project size.},
	author = {J{\o}rgensen, Magne and Halkjelsvik, Torleif and Kitchenham, Barbara}
}
@misc {Simula.simula.1670,
	title = {Hvor Gode Er Vi Til {\r a} Forutse Kriser?},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1682,
	title = {Hvor Mye Er En Sv{\ae}rt Dyktig IT-Utvikler Verdt?},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.simula.126,
	title = {Interpretation Problems Related to the Use of Regression Models to Decide on Economy of Scale in Software Development},
	journal = {Journal of Systems and Software},
	volume = {85},
	number = {11},
	year = {2012},
	pages = {2494-2503},
	abstract = {Many research studies report an economy of scale in software development, i.e., an increase in productivity with increasing project size. Several software practitioners seem, on the other hand, to believe in a diseconomy of scale, i.e., a decrease in productivity with increasing project size. In this paper we argue that violations of essential regression model assumptions in the research studies to a large extent may explain this disagreement. Particularly illustrating is the finding that the use of the production function (Size = a\bulletEffortb), instead of the factor input model (Effort=a\bulletSizeb), would most likely have led to the opposite result, i.e., a tendency towards reporting diseconomy of scale in the research studies. We conclude that there are good reasons to warn against the use of regression analysis parameters to investigate economies of scale and to look for other analysis methods when studying economy of scale in software development contexts.},
	author = {J{\o}rgensen, Magne and Kitchenham, Barbara}
}
@misc {Simula.simula.1676,
	title = {Jo Mer Usikker, Jo Mer Middels},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1680,
	title = {L{\o}nnsdiskriminering I IT-Bransjen},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.simula.1496,
	title = {Myths and Over-Simplifications in Software Engineering},
	journal = {Proceedings of ICSCT},
	year = {2012},
	publisher = {Lecture Notes on Software Engineering},
	type = {Conference},
	abstract = {The software engineering discipline contains numerous myths and over-simplifications. Some of them may be harmless, but others may hamper evidence-based practices and contribute to a fashion- and myth-based software engineering discipline. In this article we give examples of software engineering myths and over-simplifications and discuss how they are created and spread. One essential mechanism of the creation and spread of myths and over-simplifications are, we argue, people{\textquoteright}s tendency towards searching for confirming and neglecting disconfirming evidence. We report from a study examining this tendency. The study demonstrated that the developers who believed in a positive effect of agile methods tended to interpret randomly generated (neutral) project data as evidence confirming the benefit of agile methods. For the purpose of supporting evidence-based practice and avoiding unwanted influence from myths and over-simplifications, we provide a checklist to be used to evaluate the validity of software engineering claims.},
	author = {J{\o}rgensen, Magne},
	editor = {Yulin, Wang and Othman, Mohamed}
}
@misc {Simula.simula.1684,
	title = {Relative Estimation of Software Development Effort: It Matter With What and How You Compare},
	howpublished = {Presentation at JavaZone},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1678,
	title = {Sm{\r a}tt Er Godt, Eller Jo St{\o}rre Jo Bedre},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1688,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Steria-seminar},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.667,
	title = {Software Development Estimation Biases: the Role of Interdependence},
	journal = {IEEE Transactions on Software Engineering},
	volume = {38},
	number = {3},
	year = {2012},
	pages = {677-693},
	abstract = {Software development effort estimates are frequently too low, which may lead to poor project plans and project failures. One reason for this bias seems to be that the effort estimates produced by software developers are affected by information that has no relevance for the actual use of effort. We attempted to acquire a better understanding of the underlying mechanisms and the robustness of this type of estimation bias. For this purpose, we hired 374 software developers working in outsourcing companies to participate in a set of three experiments. The experiments examined the connection between estimation bias and developer dimensions: Self-construal (how one sees oneself), thinking style, nationality, experience, skill, education, sex, and organizational role. We found that estimation bias was present along most of the studied dimensions. The most interesting finding may be that the estimation bias increased significantly with higher levels of interdependence, i.e., with stronger emphasis connectedness, social context and relationships. We propose that this connection may be enabled by an activation of one{\textquoteright}s self-construal when engaging in effort estimation, and, a connection between a more interdependent self-construal and increased search for indirect messages, lower ability to ignore irrelevant context, and a stronger emphasis on socially desirable responses.},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.simula.1668,
	title = {Things aren{\textquoteright}t always what they seem: Misinterpretation of observational data in software engineering},
	howpublished = {Keynote at Euromicro, CESME},
	year = {2012},
	month = {August},
	keywords = {Conference},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1672,
	title = {Vakker Programvarekode},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.1674,
	title = {Vil Vi Noen Gang F{\r a} Feilfri Programvare?},
	year = {2012},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.908,
	title = {{\r A} Svare P{\r a} Hvor Forn{\o}yd Du Er Gj{\o}r Deg Mindre Forn{\o}yd},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.prepare.4,
	title = {Contrasting Ideal and Realistic Conditions As a Means to Improve Judgment-Based Software Development Effort Estimation},
	journal = {Information and Software Technology},
	volume = {53},
	number = {12},
	year = {2011},
	pages = {1382-1390},
	abstract = {Context: The effort estimates of software development work are on average too low. A possible reason for this tendency is that software developers, perhaps unconsciously, assume ideal conditions when they estimate the most likely use of effort. In this article, we propose and evaluate a two-step estimation process that may induce more awareness of the difference between idealistic and realistic conditions and as a consequence more realistic effort estimates. The proposed process differs from traditional judgment-based estimation processes in that it starts with an effort estimation that assumes ideal conditions before the most likely use of effort is estimated. Objective: The objective of the paper is to examine the potential of the proposed method to induce more realism in the judgment-based estimates of work effort. Method: Three experiments with software professionals as participants were completed. In all three experiments there was one group of participants which followed the proposed and another group which followed the traditional estimation process. In one of the experiments there was an additional group which started with a probabilistically defined estimate of minimum effort before estimating the most likely effort. Results: We found, in all three experiments, that estimation of most likely effort seems to assume rather idealistic assumptions and that the use of the proposed process seems to yield more realistic effort estimates. In contrast, starting with an estimate of the minimum effort, rather than an estimate based on ideal conditions, did not have the same positive effect on the subsequent estimate of the most likely effort. Conclusion: The empirical results from our studies together with similar results from other domains suggest that the proposed estimation process is promising for the improvement of the realism of software development effort estimates.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.914,
	title = {Du Er Mer Lik Meg Enn Jeg Er Lik Deg},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.900,
	title = {Du Er Mer Lik Meg Enn Jeg Er Lik Deg: Asymmetri I Relativ Estimering},
	howpublished = {XP-meeting},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.896,
	title = {Estimering Av IT-Prosjekter: Hva Vet Vi? Hvordan Bli Bedre?},
	howpublished = {Sparebank 1-seminar},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.910,
	title = {FLOSS, Flott Eller Flopp},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.888,
	title = {From Fashion and Opinion-Based to Evidence-Based Software and Knowledge Engineering},
	howpublished = {Keynote at: ICKE (India)},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.902,
	title = {How We Know What Isn{\textquoteright}t So: Common Myths in Daily Life and Software Development. Why Do We Believe in Myths?},
	howpublished = {Chalmers-seminar},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.912,
	title = {Huleboere Bodde Ikke I Huler Men I Telt},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.886,
	title = {Human Judgment in Planning and Estimation of Software Projects},
	howpublished = {ICKE 2011 (India)},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.918,
	title = {Identification and Management of IT-Projects With High Risk of Cost Overrun},
	howpublished = {Bearing Point-seminar},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.892,
	title = {Identification and Management of IT-Projects With High Risk of Cost Overrun},
	howpublished = {ABB seminar},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.299,
	title = {The Impact of Irrelevant and Misleading Information on Software Development Effort Estimates: a Randomized Controlled Field Experiment},
	journal = {IEEE Transactions on Software Engineering},
	volume = {37},
	number = {5},
	year = {2011},
	pages = {695-707},
	abstract = {Several studies have reported that software development effort estimates can be strongly affected by effort-irrelevant and misleading information without the estimators being aware of this effect. These studies were conducted in laboratory (artificial) estimation contexts. To increase our knowledge about the importance of these effects in field settings, we paid 46 outsourcing companies from Eastern European and East Asian countries to estimate the required effort of the same five software development projects. The companies were allocated randomly to either the original requirement specification or a manipulated version of the original requirement specification. The manipulations were as follows: i) reduced length of requirement specification with no change of content, ii) information about the low effort spent on the development of the old system to be replaced, iii) information about the client{\textquoteright}s unrealistic expectations about low cost, and iv) a restriction of a short development period with start up a few months ahead (which should, rationally speaking, lead to an increase in effort). All manipulations led to decreased median effort estimates, but only manipulation iv) led to a large, statistically significant decrease. A comparison of the effects of similar types of irrelevant and misleading information in laboratory and field settings suggests that the effect of manipulations i), ii) and iii) where much lower in field settings than in laboratory settings, while the effect of manipulation iv) was almost at the same level. We conclude that the tendency towards a smaller effect in field settings means that laboratory studies are frequently only useful for demonstrating the existence of a software engineering phenomenon, or for understanding it better, and that we need field studies to analyze its importance.},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.simula.904,
	title = {IT-Fiaskoer: Djevelen Er I Detaljene},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.906,
	title = {L{\o}nner Det Seg {\r a} F{\o}lge IT-Moten?},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.890,
	title = {L{\o}nner Det Seg {\r a} Investere I IT-Prosjekter? Hva Vet Vi Om ROI-Prediksjoner?},
	howpublished = {Keynote at: IDG, Oslo},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@inbook {Simula.simula.848,
	title = {Overconfidence in the Accuracy of Own Work Effort Predictions: the Role of Interval Width},
	booktitle = {Perspectives on thinking, judging, and decision making},
	year = {2011},
	pages = {47-56},
	publisher = {Universitetsforlaget},
	organization = {Universitetsforlaget},
	chapter = {3},
	isbn = {978-82-15-01878-2},
	author = {J{\o}rgensen, Magne},
	editor = {Brun, Wibecke and Keren, Gideon and Kirkeb{\o}en, Geir and Montogomery, Henry}
}
@misc {Simula.simula.898,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Seminar held for various companies in Ukraine, Mexico and Nepal},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.916,
	title = {Testere B{\o}r V{\ae}re Destruktive},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.673,
	title = {To Read Two Pages, I Need 5 Minutes, But Give Me 5 Minutes and I Will Read Four: How to Change Productivity Estimates by Inverting the Question},
	journal = {Applied Cognitive Psychology},
	volume = {25},
	number = {2},
	year = {2011},
	pages = {314-323},
	abstract = {Past research has shown that people underestimate the time they need to complete large tasks, whereas completion times for smaller tasks are often overestimated, suggesting higher productivity estimates for larger than for smaller tasks. By replacing the traditional question about how much time a given work will take with a question about how much work can be completed within a given amount of time, we found the opposite pattern. This could reflect a general tendency to underestimate large amounts relatively to small ones both for durations and for amounts of work. We explored this idea in two studies where students estimated reading tasks, a third where IT-professionals estimated software projects, and a fourth where participants imagined a familiar walk, divided into time segments or part distances of varying lengths.},
	author = {Halkjelsvik, Torleif and J{\o}rgensen, Magne and Teigen, Karl-Halvor}
}
@misc {Simula.simula.894,
	title = {Vinnerens Forbannelse, Informasjonsasymmetri, Utvalgsrisiko, Moralsk Risiko Og IT-Kontrakter},
	howpublished = {Confex Seminar},
	year = {2011},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.140,
	title = {Blir Vi Mer Og Mer Middelm{\r a}dige?},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.136,
	title = {{\textquotedblright}Det Meste Av Kommunikasjon Er Ikke-Verbal" Og Annen Misbruk Av Forskning},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.150,
	title = {The Economic Benefits of Publicly Funded IT-Research. How Research and Education on Software Engineering Can Contribute to Costa Rica{\textquoteright}s ICT Capability},
	howpublished = {Invited presentation at the Omar Dengo fundacion seminar: Software Engineering and Emerging Markets, Costa Rica},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.311,
	title = {The Effects of Request Formats on Judgment-Based Effort Estimation},
	journal = {Journal of Systems and Software},
	volume = {83},
	number = {1},
	year = {2010},
	pages = {29-36},
	abstract = {In this paper we study the effects of a change from the traditional request {\textquotedblleft}How much effort is required to complete X?{\textquotedblright} to the alternative {\textquotedblleft}How much can be completed in Y work-hours?{\textquotedblright}. Studies 1 and 2 report that software professionals receiving the alternative format provided much lower, and presumably more optimistic, effort estimates of the same software development work than those receiving the traditional format. Studies 3 and 4 suggest that the effect belongs to the family of anchoring effects. An implication of our results is that project managers and clients should avoid the alternative estimation request format.},
	author = {J{\o}rgensen, Magne and Halkjelsvik, Torleif}
}
@misc {Simula.simula.172,
	title = {Estimering Av IT-Prosjekter: Hva Vet Vi? Hvordan Bli Bedre?},
	howpublished = {Presentation at Computas seminar},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.156,
	title = {How to Become an Excellent IT-Research Institute},
	howpublished = {Seminar at University of Auckland},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.152,
	title = {How to Become an Excellent IT-Research Institute (with Substantial Impact on the IT-Intensive Industry)},
	howpublished = {Invited presentation at the Omar Dengo fundacion seminar: Research, Software Development, and Strategic Investments, Costa Rica},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.164,
	title = {How We Know What Isn{\textquoteright}t So. Common Myths in Business, Health and Daily Life. Why Do We Believe in Myths?},
	howpublished = {Presentation at IT-Fornebu seminar},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.130,
	title = {Hva Skjer N{\r a}r Budsjettet Eller Leveransedato Presses?},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.168,
	title = {Hvem Passer Offshoring For? Hva Er Viktig for {\r a} Lykkes?},
	howpublished = {Presentation at CIO Forum (IDG)},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.166,
	title = {Hvordan Estimering Av Ideell Tid Gj{\o}r Deg Mer Realistisk (med Innlagt NM I Estimering)},
	howpublished = {Presentation at JavaZone},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.144,
	title = {Hvordan Forbedre Estimering Av Tid Og Kostnader I IT-Prosjekter},
	howpublished = {Presentation at TrygVesta seminar},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.170,
	title = {Identification and Management of IT-Projects With High Risk of Cost Overrun},
	howpublished = {Presentation at Prepare{\textquoteright}s seminar for the IT-industry (and at an internal Bearing Point seminar)},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {publicoutreachreference.2010-12-29.6234757596,
	title = {Identification and Management of IT-Projects With High Risk of Cost Overrun},
	howpublished = {Presentation at Prepare{\textquoteright}s seminar for the IT-industry (and at an internal Bearing Point seminar)},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.621,
	title = {Identification of More Risks Can Lead to Increased Over-Optimism of and Over-Confidence in Software Development Effort Estimates},
	journal = {Information and Software Technology},
	volume = {52},
	number = {5},
	year = {2010},
	pages = {506-516},
	abstract = {Context: Software professionals are, on average, over-optimistic about the required effort usage and over-confident about the accuracy of their effort estimates. Objective: A better understanding of the mechanisms leading to the over-optimism and over-confidence may enable better estimation processes and, as a consequence, better managed software development projects. Method: We hypothesize that there are situations where more work on risk identification leads to increased over-optimism and over-confidence in software development effort estimates, instead of the intended improvement of realism. Four experiments with software professionals are conducted to test the hypothesis. Results: All four experiments provide results in support of the hypothesis. Possible explanations of the counter-intuitive finding relate to results from cognitive science on {\textquotedblleft}illusion-of-control{\textquotedblright}, {\textquotedblleft}cognitive accessibility{\textquotedblright}, {\textquotedblleft}the peak-end rule{\textquotedblright} and {\textquotedblleft}risk as feeling.{\textquotedblright} Conclusions: Thorough work on risk identification is essential for many purposes and our results should not lead to less emphasis on this activity. Our results do, however, suggest that it matters how risk identification and judgment-based effort estimation processes are combined. A simple approach for better combination of risk identification work and effort estimation is suggested.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.162,
	title = {Jeg Ser Det N{\r a}r Jeg Tror Det! Moter, Retorikk Og Systemutviklingsmetoder},
	howpublished = {Seminar for Telenor},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.146,
	title = {Jeg Vet Ikke Hva Et {\textquotedblright}story Point{\textquotedblright} Er, Men Det Virker Bra ...},
	howpublished = {Presentation at DnD Software conference},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.174,
	title = {Moter, Myter Og Overforenklinger I IT-Bransjen},
	howpublished = {Keynote at: Norsk informatikk-konferanse (NIK)},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.142,
	title = {Offshoring: V{\ae}r En God Kunde - Og Du Vil Lykkes},
	howpublished = {Dagens N{\ae}ringsliv},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.148,
	title = {Research Quality. What Is It? How to Achieve It?},
	howpublished = {Presentation at Telenor seminar},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.235,
	title = {Selection of Effort Estimation Strategies},
	journal = {Journal of Systems and Software},
	volume = {83},
	number = {6},
	year = {2010},
	pages = {1039-1050},
	abstract = {We currently know little about the factors that motivate the selection and change of estimation strategy in judgment-based effort estimation context. A better understanding of these issues may lead to more accurate judgment-based effort estimates and motivates the four experiments reported in this paper. The experiments{\textquoteright} two main results are the identification of the importance of {\textquotedblleft}estimation surprises{\textquotedblright} (large estimation errors) to motivate estimation strategy change and the large individual variation in the initial choice of estimation strategy. The individual variation seems not only to be a result of differences in previous experiences, but also a result of differences in the mental {\textquotedblleft}accessibility{\textquotedblright} of the strategies. We found, for example, that the use of a strategy was increased when we instructed a developer to use the same type of strategy on unrelated tasks immediately before. The laboratory contexts of the studies means that the results should be interpreted as a first step towards more knowledge about expert estimation strategies and that there is a strong need for more studies, preferably in field situations, before recommending actions on the basis of the findings.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.138,
	title = {Sm{\r a} gaver er ogs{\r a} egnet til {\r a} p{\r a}virke},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.158,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Seminar for the IT-industry in Auckland, organized by University of Auckland},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.176,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Presentations at seminars at Orient, VietSoft and IMT, Vietnam},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.160,
	title = {The Value of Empirical Software Engineering Research},
	howpublished = {Presentation at Chalmers University},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.132,
	title = {V{\r a}r Medf{\o}dte Tallforst{\r a}else},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.simula.128,
	title = {{\O}nsketenkning},
	howpublished = {Computerworld},
	year = {2010},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.719,
	title = {{\r A}tti Prosent Av Systemutviklere Er Bedre Enn Gjennomsnittet},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.725,
	title = {Bindersmyten Og {\textquotedblright}45\% Av All Funksjonalitet Brukes Aldri{\textquotedblright}},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.717,
	title = {Er IT-Utviklere Historiel{\o}se? Gj{\o}r Det Noe?},
	howpublished = {Article in Computerworld Norway},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.703,
	title = {Feill{\ae}ring Skaper Overtro P{\r a} Smidige Metoder},
	howpublished = {Presentation at Smidig},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.159,
	title = {How to Avoid Selecting Providers With Bids Based on Over-Optimistic Cost Estimates},
	journal = {IEEE Software Magazine},
	volume = {26},
	number = {3},
	year = {2009},
	pages = {79-84},
	abstract = {It is well known that software development companies tend to produce over-optimistic cost estimates and that this over-optimism may lead to delivery problems for the clients as well as the providers. In this paper we summarize evidence suggesting that the clients can reduce the likelihood of selecting providers with bids based on over-optimistic cost estimates through their control of the bidding processes. Important means for this purpose include: avoid inviting many bidders when price is an important criterion for selection; avoid using price as an important criterion for selection when the ability to assess provider competence is low; apply bidding processes that ensure that the provider understands the complexity of the project; avoid budget or price expectation related information in the bidding material and avoid a negotiation process in which you ask for bid updates on reduced functionality. The evidence presented in this paper can also be used by software providers to identify bidding rounds where it is likely that they win only when strongly over-optimistic about the cost.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.623,
	title = {How to Impact Software Engineering Practice Through Empirical Research},
	howpublished = {Keynote at: EASE},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.693,
	title = {Hvordan Tilrettelegge for Realisme I Estimater Av Arbeidsmengde I IT-Prosjekter},
	howpublished = {Presentation at LinPro seminar},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.223,
	title = {The Impact of Lessons-Learned Sessions on Effort Estimation and Uncertainty Assessments},
	journal = {IEEE Transactions on Software Engineering},
	volume = {35},
	number = {3},
	year = {2009},
	pages = {368-383},
	abstract = {Inaccurate estimates of software development effort is a frequently reported cause of IT-project failures. We report results from a study that investigated the effect of introducing lessons learned sessions on estimation accuracy and the assessment of uncertainty. Twenty software professionals were randomly allocated to a Learning group or a Control group and instructed to estimate and complete the same five development tasks. Those in the Learning group, but not those in the Control group, were instructed to spend at least 30 minutes on identifying, analyzing, and summarizing their effort estimation and uncertainty assessment experience after completing each task. We found that the estimation accuracy and the realism of the uncertainty assessment were not better in the Learning group than in the Control group. A follow-up study with 83 software professionals was completed to better understand this lack of improvement from lessons-learned sessions. The follow-up study found that receiving feedback about other software professionals{\textquoteright} estimation performance led to more realistic uncertainty assessments than receiving the same feedback of one{\textquoteright}s own estimates. Lessons-learned sessions, we argue, have to be carefully designed to avoid wasting resources on learning processes that stimulate rather than reduce learning biases related to assessment of own estimation performance.},
	author = {J{\o}rgensen, Magne and Tanja Milijana {Gruschke}}
}
@misc {Simula.se.723,
	title = {Jeg Ser Det N{\r a}r Jeg Tror Det},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.695,
	title = {Jeg Ser Det N{\r a}r Jeg Tror Det! Moter, Retorikk Og Systemutviklingsmetoder},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.721,
	title = {Jo Mer Jeg Tenker P{\r a} Det, Desto Sikrere Blir Jeg},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.689,
	title = {Myter Og Empiri Innen Systemutvikling},
	howpublished = {Presentation at DnD{\textquoteright}s Software Conference},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.691,
	title = {Myter Og Empiri (og Hvorfor Vi Aldri Vil F{\r a} Svaret P{\r a} Om Smidige Metoder Virker)},
	howpublished = {Presentation at a Bouvet seminar},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.383,
	title = {A Preliminary Study of Sequence Effects in Judgment-Based Software Development Work- Effort Estimation},
	journal = {IET Software},
	volume = {3},
	number = {5},
	year = {2009},
	pages = {435-441},
	abstract = {Context: Software development effort estimates are often inaccurate, and this inaccuracy cause problems for the clients as well as the providers. Consequently, we need more knowledge about the estimation processes, so that we can improve them. Objective: This study investigates how initial judgment-based estimation of work effort in software development affects subsequent, unrelated estimation work. Method: Fifty-six software professionals from the same company were allocated randomly to two groups. One group estimated the most likely effort required to complete a small software development task, while the other group estimated the effort required to complete a large task. After that, all the subjects estimated the effort required to complete the same medium-sized task. We replicated the experiment in another company (with 17 software professionals). Results: We found that sequence effects may have a strong impact on judgment-based effort estimates. Both in the first experiment and in the replication, the subsequent estimates were assimilated towards the subjects{\textquoteright} initial estimate, i.e., the group that began with a small task supplied, on average, lower estimates of the medium-sized task than the group that began with the large task. Conclusion: Our findings suggest that knowledge about sequence effects may be important in order to improve estimation processes. However, currently we have a quite incomplete understanding of how, when and how much sequence effects affect effort estimation. Consequently, further research is needed.},
	doi = {10.1049/iet-sen.2008.0110},
	author = {Grimstad, Stein and J{\o}rgensen, Magne}
}
@misc {Simula.se.701,
	title = {Realistisk Estimering Av Prosjekt, Release Og Iterasjon},
	howpublished = {Presentation at an Ergo seminar},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {23914,
	title = {The role of judgment in software estimation},
	journal = {International Conference on Software Engineering},
	volume = {Companion Volume},
	year = {2009},
	pages = {13-17},
	publisher = {IEEE},
	abstract = {The consequences of poor (or good) judgment in estimation are reflected in\&nbsp;software quality, cost, time-tomarket, and operational reliability. While judgment plays a\&nbsp;prominent role in aviation and medicine{\textendash}the same care arguably needs to be taken in\&nbsp;making tradeoffs impacting software development. This panel will bring togetherpractitioners to discuss aspects of judgment in software estimation including a discussion of\&nbsp;current practices and influences from other disciplines.},
	author = {Fraser, Steven and Boehm, Barry and Erdogmus, Hakan and J{\o}rgensen, Magne and Rifkin, Stan and Ross, Mike}
}
@inproceedings {Simula.SE.421,
	title = {The Role of Judgment in Software Estimation},
	journal = {ICSE 2009 Panel},
	year = {2009},
	type = {Conference},
	abstract = {The consequences of poor (or good) judgment in es-timation are reflected in software quality, cost, time-to-market, and operational reliability. While judgment plays a prominent role in aviation and medicine - the same care arguably needs to be taken in making trade-offs impacting software development. This panel will bring together practitioners to discuss aspects of judgment in software estimation including a discussion of current practices and influences from other disci-plines.},
	author = {Fraser, Steven and Boehm, Barry and Erdogmus, Hakan and J{\o}rgensen, Magne and Rifkin, Stan and Ross, Mike}
}
@misc {Simula.se.727,
	title = {S{\r a} Du Gorillaen?},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@inbook {Simula.SE.427,
	title = {Software Development Effort Estimation: Demystifying and Improving Expert Estimation},
	booktitle = {Simula Research Laboratory - by thinking constantly about it},
	year = {2009},
	pages = {381-404},
	publisher = {Springer},
	organization = {Springer},
	chapter = {26},
	address = {Heidelberg},
	abstract = {Research background: Inaccurate estimation of software development effort is one of the most important reasons of IT-project failures. While too low effort estimates may lead to project management problems, delayed deliveries, budget overruns and low software quality, too high effort estimates may lead to lost business opportunities and inefficient use of resources. These IT-project problems motivated the BEST (Better Estimation of Software Tasks) project at Simula Research Laboratory to conduct research with the goal of improving effort estimation methods. Scientific challenges: The project{\textquoteright}s main focus is to improve judgment-based effort estimation (expert estimation), which is the estimation approach most frequently used by the software industry. We argue that a better understanding of the mental steps involved in expert estimation is necessary to achieve robust improvement of software development estimation processes. A great challenge when studying expert judgment is to understand the unconscious steps involved. To study these steps, we make use of multidisciplinary competencies, especially psychology and software engineering, and financial resources enabling studies in realistic software development effort estimation contexts. Obtained and expected results: In this section, we present the main results of the project from its beginnings in 2001 until today and describe how the project has benefited from being an integrated part of Simula Research Laboratory, especially through the opportunity to conduct large-scale, controlled experiments in field settings. The section includes practical results on how to improve estimation methods, scientific results leading to a better understanding of the mental processes involved in judgment-based effort estimation, and our innovative research methods in the field of empirical software engineering. Results of the BEST project are currently in use by several software companies, as well as by researchers in forecasting and psychology, and they are also included in project management and software engineering textbooks. The BEST project plans to continue its multidisciplinary effort with the goal of constructing and evaluating better models of the mental steps involved in judgment-based effort estimation. An improved model of these steps will enhance our ability to accurately estimate software development effort and to predict when different types of estimation methods can be expected to deliver accurate effort estimates. This will in turn lead to a reduction of the number of IT-project failures and make better use of scarce financial and human resources.},
	isbn = {978-3642011559},
	author = {J{\o}rgensen, Magne and Grimstad, Stein},
	editor = {Tveito, Aslak and Are Magnus {Bruaset} and Lysne, Olav}
}
@article {Simula.SE.321,
	title = {Software Development Effort Estimation: Formal Models Or Expert Judgment? (Viewpoints Article)},
	journal = {IEEE Software},
	number = {March/April},
	year = {2009},
	pages = {14-19},
	author = {J{\o}rgensen, Magne and Boehm, Barry}
}
@misc {Simula.se.697,
	title = {Software Development Effort Estimation: What Do We Know About It?},
	howpublished = {Presentation at PGS (Wroclaw) seminar},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.715,
	title = {Spillet Som Kun Har Tapere},
	howpublished = {Computerworld},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.se.699,
	title = {Utvikling Av Programvare I Lavkostland},
	howpublished = {Presentation at DnD seminar on outsourcing},
	year = {2009},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.369,
	title = {Avoiding Impact From Irrelevant and Misleading Information},
	howpublished = {Presentation at Chalmers University, G{\o}teborg},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.112,
	title = {Avoiding Irrelevant and Misleading Information When Estimating Development Effort},
	journal = {IEEE Software},
	number = {May/June},
	year = {2008},
	pages = {78-83},
	abstract = {Software development effort estimates are reported to be highly inaccurate and systematically over-optimistic. We provide empirical evidence that suggests that this problem is caused, to some extent, by the influence of irrelevant and misleading information, e.g., information about the client{\textquoteright}s budget, present in the estimation material. The only really effective means of eliminating this influence is to avoid exposure to this type of information. Other means, such as more use of formal effort estimation models, improved analysis of requirement specifications, and better selection of estimators, had a positive effect but did not eliminate the influence. We propose process elements that are designed to avoid irrelevant and misleading information and illustrate how this process may lead to more realism in effort estimation.},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.SE.341,
	title = {Best for Hvem},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.345,
	title = {Hjelper Smidige Metoder?},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.359,
	title = {How to Avoid Impact From Irrelevant and Misleading Information on Your Cost Estimates},
	howpublished = {Presentation at Itestra Seminar (M{\"u}nchen)},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.353,
	title = {Human Biases in Forecasting of Work-Effort: Differences in Effect Sizes in Laboratory and Field Settings},
	howpublished = {Presentation at International Symposium on Forecasting},
	year = {2008},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.SE.365,
	title = {Hvorfor L{\ae}ring Av Erfaring Er Vanskelig Og Hvordan Bli Bedre},
	howpublished = {Presentation at JavaZone},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.361,
	title = {The Impact of Irrelevant and Misleading Information on Software Development Effort Estimation: a Randomized, Controlled Field Experiment of Forty-Six Outsourcing Companies},
	howpublished = {Presentation at IT University Copenhagen},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.160,
	title = {The Impact of Irrelevant Information on Estimates of Software Development Effort},
	journal = {Journal of Systems and Software},
	year = {2008},
	abstract = {Software professionals typically estimate software development effort on the basis of a requirement specification. Parts of this specification frequently contain information that is irrelevant to the estimation of the actual effort involved in the development of software. We hypothesize that effort-irrelevant information sometimes has a strong impact on effort estimates. To test this hypothesis, we conducted three controlled experiments with software professionals. In each of the experiments, the software professionals received specifications describing the same requirements. However, we gave one group of the software professionals a version of the requirement specification where we had included additional, effort-irrelevant, information. In all three experiments, we observed that the estimates of most likely effort increased when the estimates were based on requirement specifications that contained the information irrelevant to development effort. The results suggest that when estimation-irrelevant information is included as input to expert judgment-based estimation processes, the estimators find it difficult to ignore it. The results also show that it is difficult to predict the impact of estimation-irrelevant information and that software professionals seem to be unaware of the impact. One possible (and advisable) course of action, given our findings, would be to remove estimation-irrelevant information from the requirement specification prior to the use of it as input to estimation work.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne}
}
@inproceedings {Simula.SE.215,
	title = {Judgment-Updating Among Software Professionals},
	journal = {The 2nd international conference on software knowledge information management and applications (SKIMA)},
	year = {2008},
	pages = {62-67},
	publisher = {SKIMA conference organising committee},
	type = {Conference},
	abstract = {Initial judgments related to key decisions in software projects are often based on one-sided or misleading information. The initial assessment of the benefits of introducing a new development tool may for example be based a vendor{\textquoteright}s sales demonstration or a reference client{\textquoteright}s favorable description. In this paper we study software professionals{\textquoteright} abilities to adjust their early, biased judgments when receiving contradicting or less biased information. The first study, involving 160 software professionals, found a strong under-adjustment for the impact of misleading information and one-sided argument. A follow-up two weeks later found that this under-adjustment was not removed over time. The second study, involving 65 software professionals, found that the ability to update biased judgments may sometimes be quite good, but that it is hard to predict when. A practical consequence of our results is that software professionals should strongly emphasize the avoidance of biased and potentially misleading information and not trust that they are able to adjust their judgments and beliefs when more reliable and unbiased information are available.},
	isbn = {9781851432516},
	author = {J{\o}rgensen, Magne and Grimstad, Stein},
	editor = {Hossain, Ziaul and Ouzrout, }
}
@misc {Simula.SE.339,
	title = {Kritisk Tenkning},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.347,
	title = {Kulturforskjeller Og Offshoring-Prosjekter},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.371,
	title = {Kulturforskjeller Og Outsourcing Av IT-Prosjekter Inni Er Vi Like - Eller Er Vi Egentlig Det?},
	howpublished = {Presentation at Simula{\textquoteright}s Industry Seminar},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.367,
	title = {L{\ae}ring Av Erfaring},
	howpublished = {Presentation at Nordnet (project management conference)},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.351,
	title = {Peer, Du Lyver! Nei, Jeg Gj{\o}r Ei!},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.SE.385,
	title = {A Preliminary Study of Sequence Effects in Judgment-Based Software Development Work- Effort Estimation},
	journal = {Proceedings, 12th International Conference on Evaluation and Assessment in Software Engineering},
	year = {2008},
	publisher = {University of Bari, Italy},
	type = {Conference},
	abstract = {Context: Software development effort estimates are often inaccurate, and this inaccuracy cause problems for the clients as well as the providers. Consequently, we need more knowledge about the estimation processes, so that we can improve them. Objective: This study investigates how initial judgment-based estimation of work effort in software development affects subsequent, unrelated estimation work. Method: Fifty-six software professionals from the same company were allocated randomly to two groups. One group estimated the most likely effort required to complete a small software development task, while the other group estimated the effort required to complete a large task. After that, all the subjects estimated the effort required to complete the same medium-sized task. We replicated the experiment in another company (with 17 software professionals). Results: We found that sequence effects may have a strong impact on judgment-based effort estimates. Both in the first experiment and in the replication, the subsequent estimates were assimilated towards the subjects{\textquoteright} initial estimate, i.e., the group that began with a small task supplied, on average, lower estimates of the medium-sized task than the group that began with the large task. Conclusion: The results of this study suggest that knowledge about sequence effects may be important in order to improve estimation processes. However, currently we have a quite incomplete understanding of how, when and how much sequence effects affect effort estimation. Consequently, further research is needed.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne},
	editor = {Visaggio, Guiseppe and Maria Teresa {Baldassarre} and Linkman, Stephen and Turner, Mark}
}
@misc {Simula.SE.343,
	title = {Priming},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.363,
	title = {Prinsipper for Estimering Av Utviklingskostnader I IT-Prosjekter},
	howpublished = {Keynote at Geiloseminaret 2008},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {23926,
	title = {Prinsipper for estimering av utviklingskostnader i IT-prosjekter},
	howpublished = {Geiloseminaret, Geilo},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.355,
	title = {Rasjonalitetens Makt Og Maktens Rasjonalitet},
	howpublished = {Presentation at Objectnet seminar (Dublin)},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@article {Simula.SE.9,
	title = {The Role of Artificial Design Elements in Software Engineering Experiments},
	journal = {IEEE Transactions on Software Engineering},
	volume = {34},
	number = {2},
	year = {2008},
	month = {March/April},
	pages = {242-259},
	abstract = {Increased realism in software engineering experiments is often promoted as an important means to increase generalizability and industrial relevance. In this context, artificiality, e.g., the use of constructed tasks in place of realistic tasks, is seen as a threat. In this article, we examine the opposite view, that deliberately introduced artificial design elements may increase knowledge gain and enhance both generalizability and relevance. In the first part of the article, we identify and evaluate arguments and examples in favor of, and against, deliberately introducing artificiality into software engineering experiments. In the second part of the article, we summarize a content analysis of articles reporting software engineering experiments published over the ten-year period 1993-2002. The analysis reveals a striving for realism and external validity, but little awareness of for what and when, various degrees of artificiality and realism are appropriate. We conclude that an increased awareness and deliberation in these respects is essential. However, arguments in favor of artificial design elements should not be used to justify studies that are badly designed or that have research questions of low relevance.},
	author = {Jo Erskine {Hannay} and J{\o}rgensen, Magne}
}
@article {Gruschke.2007.1,
	title = {The Role of Outcome Feedback in Improving the Uncertainty Assessment of Software Development Effort Estimates},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {17},
	number = {4},
	year = {2008},
	pages = {20-35},
	abstract = {To enable properly-sized software project budgets and plans it is important to be able to assess the uncertainty of the estimates of the most likely effort required to complete the projects. Previous studies show that software professionals tend to be too optimistic about the uncertainty of their effort estimates, i.e., they tend to be overconfident. This paper reports the results from a study on the role of outcome feedback in the uncertainty assessment learning process. Software developers were given repeated and immediate outcome feedback about the discrepancy between the estimated most likely effort and the actual effort. We found that one condition for the improvement of uncertainty assessments of effort estimates may be the use of explicitly-formulated strategies for uncertainty assessment. By contrast, intuition-based uncertainty assessment strategies may lead to no or little improvement. This has the implication that we cannot expect much improvement in uncertainty assessment unless we train and instruct software developers to use the previous estimation accuracy experience as input to an explicitly defined uncertainty assessment process.},
	author = {Tanja Milijana {Gruschke} and J{\o}rgensen, Magne}
}
@misc {Simula.SE.387,
	title = {Sequence Effects in Judgment-Based Software Development Work-Effort Forecasting},
	howpublished = {Proceedings of the 28th International Symposium on Forecasting},
	year = {2008},
	note = {Talk at ISF 2008, Nice, France (June 2008)},
	abstract = {Accurate forecasts of work-effort are essential to enable successful software development projects. Such forecasts are typically judgment-based and inaccurate. One important reason for the inaccuracy may be due to the {\textquotedblleft}sequence effect{\textquotedblright}, i.e., that initial software development work-effort forecasts may affect subsequent, even unrelated, judgment-based forecasting work. If, for example, the previously forecasted development task was small, there seems to be a tendency to under-estimate the work-effort of the following task. We report sequence effects found in the laboratory and in the field. In a laboratory-based experiment, we randomly divided 56 software professionals into two groups. One group started with work-effort forecasts of a small and the other of a larger software development task. Subsequently, all the software professionals were asked to forecast the work-effort of the same medium-sized task. We found that the forecasts of the medium-sized tasks were assimilated towards the initial forecasts, i.e., the group that initially forecasted a small task submitted, on average, lower forecasts of the medium-sized task than the group that initially forecasted a larger task. We found similar results in a field experiment. Forty software companies from Asia and Eastern Europe were hired to forecast the work-effort of the same five software development projects. The companies that started with forecasts of the largest projects submitted, on average, the highest forecasts of the other projects. The typical approach for software development work-effort forecasting is based on a decomposition of projects into sub-tasks. A possible consequence of our results is that software professionals should start with forecasts of medium complex, medium sized sub-tasks of the project, or, with large and complex task if there is a strong tendency towards over-optimistic forecasts.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne}
}
@misc {Simula.SE.379,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Presentation at D2Hawkeye (Nepal) seminar},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.377,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Presenation at TMA Solution (Vietnam) seminar},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.375,
	title = {Software Development Effort Estimation: Why It Fails and How to Improve It},
	howpublished = {Presentation at Silverstone (India) Seminar},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.373,
	title = {Software Project Planning and Management: Why It Fails and How to Improve It},
	howpublished = {Presentation at Kathmandu University},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.357,
	title = {When Should We Trust Expert Judgment in Software Development?},
	howpublished = {Presentation at Technische Universit{\"a}t M{\"u}nchen},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.349,
	title = {{\O}kt Psykologisk Distanse F{\o}rer Til {\o}kt Optimisme},
	howpublished = {Article in Computerworld Norge},
	year = {2008},
	author = {J{\o}rgensen, Magne}
}
@article {Jorgensen.2007.3,
	title = {Characteristics of Software Engineers With Optimistic Predictions},
	journal = {Journal of Systems and Software},
	volume = {80},
	number = {9},
	year = {2007},
	pages = {1472-1482},
	abstract = {This paper examines to which degree level of optimism in software engineers{\textquoteright} predictions is related to optimism on previous predictions, scores on tests of general level of optimism (explanatory style, life orientation and self-assessed optimism), development skill, confidence in accuracy of own predictions, and, ability to recall effort used on previous tasks. Results from four experiments suggest that more optimistic software engineers are characterized by more optimistic previous predictions, higher confidence in accuracy of own predictions, lower development skills, poorer ability to recall effort on previous tasks, and, higher optimism scores. A substantial part of the variation of level of optimism seems, however, to be random.},
	author = {J{\o}rgensen, Magne and Faugli, Bj{\o}rn and Tanja Milijana {Gruschke}}
}
@inproceedings {Simula.SE.164,
	title = {A Critique of How We Measure and Interpret the Accuracy of Software Development Effort Estimation},
	journal = {1st International Workshop on Software Productivity Analysis and Cost Estimation},
	year = {2007},
	pages = {15-22},
	publisher = {Information Processing Society of Japan},
	type = {Workshop},
	abstract = {This paper criticizes current practice regarding the measurement and interpretation of the accuracy of software development effort estimation. The shortcomings we discuss are related to: 1) the meaning of {{\textquoteleft}}effort estimate{\textquoteright}, 2) the meaning of {{\textquoteleft}}estimation accuracy{\textquoteright}, 3) estimation of moving targets, and 4) assessment of the estimation process, and not only the discrepancy between the estimated and the actual effort, to evaluate estimation skill. It is possible to correct several of the discussed shortcomings by better practice. However, there are also inherent problems related to both laboratory and field analyses of the accuracy of software development effort estimation. It is essential that both software researchers and professionals are aware of these problems and their implications for the analysis of the measurement of effort estimation accuracy.},
	isbn = {ISBN 978-4-915256-72-1 C3040},
	author = {J{\o}rgensen, Magne},
	editor = {Keung, Jacky}
}
@misc {Simula.SE.187,
	title = {Det Kommer an P{\r a} Hvordan Man Sp{\o}r},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.190,
	title = {Er Selvbeherskelsen Din Sliten Og Utrent?},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@article {Jorgensen.2007.2,
	title = {Estimation of Software Development Work Effort:Evidence on Expert Judgment and Formal Models},
	journal = {International Journal of Forecasting},
	volume = {23},
	number = {3},
	year = {2007},
	pages = {449-462},
	abstract = {The main goal of the review presented in this paper is to examine when to use expert judgment, when to use formal models, and, when to combine these two approaches when estimating software development work effort. Sixteen relevant studies were identified and reviewed. The review found that the average accuracy of expert judgment-based effort estimates was better than the average accuracy of the models in ten of the sixteen studies. Two indicators of higher accuracy of judgment-based effort estimates seem to be that: i) the estimation models are not calibrated to the organization using the model, and, ii) the experts possess important context information not included in the formal estimation models. The use of models, on the other hand, may be particularly useful in estimation situations believed to lead to stronger than usual degree of over-optimism, particularly very large projects. Five of the reviewed studies evaluated estimates based on a combination of expert judgment and model. In all five studies the average accuracy of combination-based effort estimates had similar or more accurate estimates than the average accuracy of the expert estimates and of the best model.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.184,
	title = {Estimering Av IT-Prosjekter},
	howpublished = {Presentation at Lindorff-seminar},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.183,
	title = {Estimering Av IT-Prosjekter: Ekspertestimat Eller Modell?},
	howpublished = {Presentation at Simula{\textquoteright}s annual estimation seminar},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Simula.SE.13,
	title = {The Future of Empirical Methods in Software Engineering Research},
	journal = {Future of Software Engineering (FOSE {\textquoteright}07)},
	year = {2007},
	month = {May},
	pages = {358-378},
	publisher = {IEEE-CS Press},
	type = {Conference},
	abstract = {We present the vision that for all fields of software engineering (SE), empirical research methods should enable the development of scientific knowledge about how useful different SE technologies are for different kinds of actors, performing different kinds of activities, on different kinds of systems. It is part of the vision that such scientific knowledge will guide the development of new SE technology and is a major input to important SE decisions in industry. Major challenges to the pursuit of this vision are: more SE research should be based on the use of empirical methods; the quality, including relevance, of the studies using such methods should be increased; there should be more and better synthesis of empirical evidence; and more theories should be built and tested. Means to meet these challenges include (1) increased competence regarding how to apply and combine alternative empirical methods, (2) tighter links between academia and industry, (3) the development of common research agendas with a focus on empirical methods, and (4) more resources for empirical research.},
	isbn = {432663526},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Dyb{\r a}, Tore and J{\o}rgensen, Magne},
	editor = {Briand, Leon and Wolf, A.}
}
@misc {Simula.SE.180,
	title = {Help! My Brain Is Out of Control! Impact From Irrelevant and Misleading Information in Software Effort Estimation},
	howpublished = {Presentation at BEKK-seminar},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.194,
	title = {How Should We (not) Design Empirical Studies of Software Development?},
	howpublished = {Invited presentation at the University of New South Wales, Sydney},
	year = {2007},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Simula.SE.152,
	title = {How to Get a Low Price on Your Next Software Development Project and Why You Should Avoid It},
	howpublished = {International Symposium on Forecasting},
	year = {2007},
	abstract = {Software providers are well known for over-optimistic forecasts of how much the development of software systems will cost. We claim that the software clients, to a far greater degree than is commonly believed, are responsible for the providers{\textquoteright} over-optimism. Our claim is based on empirical studies of how client controlled factors impact the level of optimism in providers{\textquoteright} software development cost forecasts. The studies suggest that forecasting over-optimism is related to clients who communicate unrealistic price expectations or present attractive future opportunities related to winning the bidding round systematically. The studies also suggest that there are commonly used formats of software project bidding processes that strongly increase the risk of selecting providers with prices based on over-optimistic cost forecasts. We find, for example, that bidding processes with negotiation tend to increase the level of over-optimism when the negotiation involves asking for price updates on a reduced version of the initial software requirements. Evidence suggests that the clients may not benefit from a low price based on an over-optimistic cost forecast. One reason for this, applying concepts from the principal-agency theory, is the high level of information asymmetry in most software development project. As an illustration, the clients{\textquoteright} poor ability to specify and monitor software quality attributes, such as the maintainability of the software, makes it possible for the providers to deliver software with poorer than expected quality. We observe that social factors that otherwise may be sufficient to avoid opportunistic behaviour by the clients, e.g., the software developers{\textquoteright} self-imposed code of ethics, are less important in situations with over-optimistic effort estimates, e.g., in situations where the provider tries to avoid financial losses. Based on the above findings we outline elements of how the bidding processes should be designed to reduce the probability of receiving bids based on over-optimistic forecasts.},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.189,
	title = {Hva Slags Informasjon Stoler Vi Mest P{\r a}?},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.182,
	title = {Hvordan Kundens Anbudsprosess F{\r a}r Deg Til {\r a} Estimere Overoptimistisk Og Hva Du Kan Gj{\o}re Med Det},
	howpublished = {Presentation at JavaZone 2007},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.191,
	title = {Hypnose, Rasjonalisering Og Litt M{\r a}leteori},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Grimstad.2007.2,
	title = {The Impact of Irrelevant Information on Estimates of Software Development Effort},
	journal = {The Australian Software Engineering Conference (Paper received "Best Paper Award")},
	year = {2007},
	note = {Awarded a prize for the conference{\textquoteright}s best paper.},
	pages = {359-368},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Software professionals typically estimate software development effort based on a requirement specification. Parts of this specification frequently contain information that is irrelevant to the estimation of the actual effort involved in the development of software. We hypothesize that effort-irrelevant information sometimes has a strong impact on effort estimates. To test this hypothesis, we conducted two controlled experiments with software professionals. In each of the experiments, the software professionals received specifications describing the same requirements. However, we gave one group of the software professionals a version of the requirement specification where we had included additional, effort-irrelevant, information. In both experiments we observed that the estimates of most likely effort increased when the estimates were based on requirement specifications that contained the information irrelevant to development effort. The results suggest that when estimation-irrelevant information is included as input to expert judgment-based estimation processes, the estimators find it difficult to distinguish between the estimation-relevant and the estimation-irrelevant information. A possible consequence of our findings is that estimation-irrelevant information should be removed from the requirement specification prior to the use of it as input to estimation work.},
	isbn = {999},
	author = {Grimstad, Stein and J{\o}rgensen, Magne},
	editor = {Grant, Doug}
}
@misc {Simula.SE.122,
	title = {The Impact of Irrelevant Information on Inconsistency in Expert Judgment-Based Estimates of Software Development Work},
	year = {2007},
	abstract = {Effort estimation is an important activity in software development and provides essential input to pricing, planning and budgeting processes. Unfortunately, many software effort estimates are inaccurate. The consequences of inaccurate estimates can be severe, e.g., budget-overruns, delayed time-to-market, and poor quality software. Most software effort estimation work is at least party based on expert judgment, i.e. non-mechanical and unconscious processes. For this reason, a certain degree of intra-person inconsistency is expected, i.e., the same information presented to the same individual at different occasions sometimes lead to different effort estimates. We have conducted an experiment where seven experienced software professionals estimated the same sixty software development tasks over a period of three months. Six of the sixty tasks were estimated twice. We found a surprisingly high degree of inconsistency in the software professionals{\textquoteright} effort. The mean difference of the effort estimates of the same task by the same estimator was as much 71\% (median 50\%) The correlation between the corresponding estimates was 0,7. Highly inconsistent effort estimates will, on average, be inaccurate and difficult to learn from. It is consequently important to focus estimation process improvement on consistency issues. Evidence from other forecasting fields recommends that to reduce inconsistency, only the most important estimation information is used as input to the estimation work. We have empirically examined this advice{\textquoteright}s applicability to software effort estimation by analysing inter-estimator agreement in six software effort estimation experiments that report on the impact of including information of low or no relevance in the input to estimation work. The main findings are that inconsistency can increase when information of low or no relevance is present, and that this happens in spite of the software professionals knowing and accepting the lack of relevance. At present, the only safe way to remove the impact seems to be to remove the irrelevant information from the estimation material before it is given to the estimators.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne}
}
@article {Grimstad.2007.1,
	title = {Inconsistency in Expert Judgment-Based Estimates of Software Development Effort},
	journal = {Journal of Systems and Software},
	volume = {80},
	number = {11},
	year = {2007},
	pages = {1770-1777},
	abstract = {Effort estimation of software development work is partly based on non-mechanical and unconscious processes, i.e., human judgment. For this reason, a certain degree of intra-person inconsistency is expected, i.e., the same information presented to the same individual at different occasions may lead to different effort estimates. In this paper, we report from an experiment where seven experienced software professionals estimated the same sixty software development tasks over a period of three months. Six of the tasks were estimated twice. We found a high degree of inconsistency in a subject{\textquoteright}s effort estimates of the same task. The mean difference of the effort estimates of the same task by the same estimator was as much as 71\%! Surprisingly, the software professionals{\textquoteright} estimation accuracy on previously completed development tasks did not correlate with their degree of estimation inconsistency in the experiment. We discuss reasons for and consequences of our findings.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne}
}
@inproceedings {Simula.SE.113,
	title = {Individual Differences in How Much People Are Affected by Irrelevant and Misleading Information},
	journal = {Second European Conference on Cognitive Science},
	year = {2007},
	pages = {347-352},
	publisher = {Hellenic Cognitive Science Society},
	type = {Conference},
	abstract = {People differ in how much they update their beliefs based on new information. According to a recently proposed theory, individual differences in belief updating are, to some extent, determined by neurological differences, i.e., differences in the organization of the brain. The same neurological differences that affect belief updating may also affect handedness. In particular, more mixed-handed people may have a lower threshold for updating beliefs than strongly right-handed people. On the basis of the proposed theory, we hypothesize that mixed-handed software engineers will be more affected by irrelevant and misleading information when providing expert judgments. This hypothesis is tested in five experiments conducted in software engineering contexts. All five experiments supported the hypothesis and suggest that a low threshold for updating beliefs, as measured by degree of mixed-handedness correlates with inaccurate judgment in situations that contain irrelevant or misleading information. On the basis of the results, we argue that software engineering decisions, problem solving and estimation processes should take into account differences in individuals{\textquoteright} threshold for updating belief and not be based on the assumption that one {\textquotedblleft}process fits all{\textquotedblright}.},
	isbn = {978-1-84169-696-6},
	author = {J{\o}rgensen, Magne},
	editor = {Vosniadou, Stella and Kayser, Daniel and Protopapaps, Athanassios}
}
@misc {Simula.SE.192,
	title = {Kunder, Leverand{\o}rer, Yrkesetikk Og Informasjonsasymmetri},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.188,
	title = {Overtro P{\r a} F{\o}rsteinntrykket},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.181,
	title = {Rasjonelle Beslutninger Ved Valg Av Metoder Og Verkt{\o}y},
	howpublished = {Presentation at JavaZone 2007},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.193,
	title = {Sannsynligvis Vil Noe Usannsynlig Skje},
	howpublished = {Article in Computerworld Norge},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.185,
	title = {Software Cost Estimation: When to Use Estimation Models and When to Use Expert Judgment},
	year = {2007},
	author = {J{\o}rgensen, Magne}
}
@misc {Simula.SE.186,
	title = {Software Development Effort Estimation},
	howpublished = {Presentation at Making Waves-seminar},
	year = {2007},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@article {Jorgensen.2007.1,
	title = {A Systematic Review of Software Development Cost Estimation Studies},
	journal = {IEEE Transactions on Software Engineering},
	volume = {33},
	number = {1},
	year = {2007},
	month = {January},
	pages = {33-53},
	abstract = {This paper aims to provide a basis for the improvement of software estimation research through a systematic review of previous work. The review identifies 304 software cost estimation papers in 76 journals and classifies the papers according to research topic, estimation approach, research approach, study context and data set. Based on the review, we provide recommendations for future software cost estimation research: 1) Increase the breadth of the search for relevant studies, 2) Search manually for relevant papers within a carefully selected set of journals when completeness is essential, 3) Conduct more research on basic software cost estimation topics, 4) Conduct more studies of software cost estimation in real-life settings, 5) Conduct more studies on estimation methods commonly used by the software industry, and, 6) Conduct fewer studies that evaluate methods based on arbitrarily chosen data sets.},
	author = {J{\o}rgensen, Magne and Shepperd, Martin}
}
@article {Jorgensen.2006.3,
	title = {The Effects of the Format of Software Project Bidding Processes},
	journal = {International Journal of Project Management},
	volume = {24},
	number = {6},
	year = {2006},
	pages = {522-528},
	abstract = {This study investigates how differences in format of the bidding process affect companies{\textquoteright} bids for software projects. Thirty outsourcing companies from different Asian and European countries participated in the bidding for a software project. A participating company either started with the provision of a bid based on a reduced version of the specification and then continued with a bid based on the full specification (the Increase situation), or started with the full specification and then continued with the reduced one (the Decrease situation). We observed important differences in bids for the same project as a result of different bidding sequences. Our results constitute evidence that in situations similar to the one we studied, the client will typically select a provider with about a 40\% lower price in the Decrease situation than in the Increase situation. The difference in bids seems to be explained by whether the first bid was provided on the full or the reduced specification, not by the process of updating, i.e., increasing or decreasing, the bids. We warn against manipulation of the bidding processes to receive lower bids. This increases the risk of over-optimistic bids. Over-optimistic bids frequently have as a consequence "the winner{\textquoteright}s curse", which leads easily to "the client{\textquoteright}s curse".},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.10,
	title = {En Jungel Av Forskningsresultater},
	howpublished = {Article in Computerworld Norge, No 36, p 40},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@inbook {Jorgensen.2006.9,
	title = {Expert Estimation of Software Development Work},
	booktitle = {Software Evolution and Feedback: Theory and Practice},
	year = {2006},
	pages = {489-503},
	publisher = {Wiley},
	organization = {Wiley},
	chapter = {25},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup},
	editor = {Nazim H. {Madhavji} and Fernandez-Ramli, Juan and Perry, Dewayne}
}
@inproceedings {Grimstad.2006.2,
	title = {A Framework for the Analysis of Software Cost Estimation Accuracy},
	journal = {ISESE 2006},
	year = {2006},
	pages = {58-65},
	publisher = {ACM Press},
	type = {Conference},
	isbn = {1-59593-218-6},
	author = {Grimstad, Stein and J{\o}rgensen, Magne},
	editor = {Jos{\'e} Carlos {Maldonado} and Wohlin, Claes and Mendes, Emilia}
}
@misc {Jorgensen.2006.14,
	title = {Herre I Eget Hus?},
	howpublished = {Article in Computerworld Norge, nr 24, s. 40.},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.12,
	title = {How Large Are Software Cost Overruns? a Review of the 1994 CHAOS Report},
	howpublished = {Article in Software Practitioner (Vol 16, no 4\&5, p 13-14)},
	year = {2006},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Jorgensen.2006.4,
	title = {How Large Are Software Cost Overruns? Critical Comments on the Standish Group{\textquoteright}s CHAOS Reports},
	journal = {Information and Software Technology},
	volume = {48},
	number = {4},
	year = {2006},
	pages = {297-301},
	abstract = {The Standish Group reported in 1994 that the average cost overrun of software projects was as high as 189\%. This cost overrun number is used as input in recent governmental reports on software development and as benchmark for several recent projects{\textquoteright} estimation performances. It is therefore important that we can trust this number. More recent cost overrun results reported by the Standish Group and others, however, show much lower cost overrun. Does this mean that software companies have improved their estimation ability strongly the last 10 years? In this paper we argue that there are reasons to doubt the validity of the Standish Group{\textquoteright}s 1994 cost overrun results and that a continued use of these results may hinder progress.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Simula.SE.151,
	title = {How Much Does Feedback and Performance Review Improve Software Development Effort Estimation? an Empirical Study (extended Abstract)},
	howpublished = {In International Symposium on Forecasting, edited by Antonio Garc{\`\i}a-Ferrer, Santander, 12-14 June 2006. International Institute of Forecasters, pages 103},
	year = {2006},
	abstract = {Over-optimistic and over-confident software development effort estimates are more the rule than the exception. Several software improvement process frameworks, e.g., the Capability Maturity Model, are based on the assumption that improved feedback and use of performance reviews leads to higher degree of realism and better project performance. This paper investigates that assumptions, i.e., whether effort estimation and uncertainty assessment skills improve with better feedback processes and mandatory estimation performance reviews. We recruited 20 professional software developers with the necessary technological experience and skill to complete the same 5 software development tasks on an in-use web-system. They were paid standard industry wage corresponding to their level of experience and expertise. Work conditions were like those in industry. Ten of the participants received outcome feedback on their estimation performance and followed our estimation performance review instructions, the other ten acted as a control group with no performance review instructions and only on-the-job feedback. We found no or only minor differences in performance between the treatment groups and conclude that the feedback and performance reviews did not lead to improvement of estimation or uncertainty assessment skill. Possible reasons for this surprising finding include: a) Low motivation for learning (A high performance on effort estimation and uncertainty assessment may have been perceived as a much less important goal than being perceived as a skilled programmer through low effort estimates and high confidence.), and, b) The software developers who were instructed to learn from previous performance may have believed that they have learned more than they actually did. In a follow-up experiment 81 software professionals assessed the uncertainty of the effort estimates provided by the developers in the original experiment. We found that the software professionals assessing the uncertainty of other developers{\textquoteright} effort estimates achieved a much higher realism and better learning from history.},
	author = {Tanja Milijana {Gruschke} and J{\o}rgensen, Magne}
}
@misc {Gruschke.2006.4,
	title = {How Much Does Feedback and Performance Review Improve Software Development Effort Estimation? an Empirical Study},
	howpublished = {International Symposium of Forecasting (ISF) 2006},
	year = {2006},
	month = {June},
	abstract = {Over-optimistic and over-confident software development effort estimates are more the rule than the exception. Several software improvement process frameworks, e.g., the Capability Maturity Model, are based on the assumption that improved feedback and use of performance reviews leads to higher degree of realism and better project performance. This paper investigates that assumption, i.e., whether effort estimation and uncertainty assessment skills improve with better feedback processes and mandatory estimation performance reviews. We recruited 20 professional software developers with the necessary technological experience and skill to complete the same 5 software development tasks on an in-use web-system. They were paid standard industry wage corresponding to their level of experience and expertise. Work conditions were like those in industry. Ten of the participants received outcome feedback on their estimation performance and followed our estimation performance review instructions, the other ten acted as a control group with no performance review instructions and only on-the-job feedback. We found no or only minor differences in performance between the treatment groups and conclude that the feedback and performance reviews did not lead to improvement of estimation or uncertainty assessment skill. Possible reasons for this surprising finding include: a) Low motivation for learning (A good performance on effort estimation and uncertainty assessment may have been perceived as a much less important goal than being perceived as a skilled programmer, communicated by providing low effort estimates and high confidences.),and, b) The software developers who were instructed to learn from previous performance may have believed that they had learned more than they actually did. In a follow-up experiment 81 software professionals assessed the uncertainty of the effort estimates provided by the developers in the original experiment. We found that the software professionals assessing the uncertainty of other developers{\textquoteright} effort estimates achieved a much higher realism and better learning from history.},
	author = {Tanja Milijana {Gruschke} and J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.13,
	title = {How to Achieve Lower Software Bids and Why Not to Do This},
	howpublished = {Article in Improve (Newsletter 2-2006, p 1-2)},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.22,
	title = {How to Avoid Impact From Irrelevant and Misleading Information on Your Cost Estimates},
	howpublished = {Presentation at Simula{\textquoteright}s Annual Estimation Seminar, November 21},
	year = {2006},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Jorgensen.2006.16,
	title = {Hvor Rasjonelle Er V{\r a}re Beslutninger Ved Valg Av Metoder Og Verkt{\o}y?},
	howpublished = {Keynote at SPIKE conference, Oslo, April 4},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.21,
	title = {Hvordan Bli Bedre Til {\r a} Estimere? (Inkludert NM I Estimering)},
	howpublished = {Presentation at JavaZone, September 13},
	year = {2006},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Jorgensen.2006.23,
	title = {Hvorfor Kostnadsestimater Er Overoptimistiske. Kundens Ansvar Er Sterkt Undervurdert!},
	howpublished = {Presenation at IKT-Norge seminar, Desember 5},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.6,
	title = {L{\ae}ring Av Erfaring},
	howpublished = {Article in Computerworld Norge, No 30, p 36},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.5,
	title = {Min Venstre Hjernehalvdel H{\o}rer Ikke P{\r a} Meg},
	howpublished = {Article in Computerworld Norge, No 42, p 40},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.20,
	title = {N{\r a}r Skal Vi Bruke Hodet Og N{\r a}r Skal Vi Bruke Estimeringsmodeller?},
	howpublished = {Presentation at JavaZone, September 13},
	year = {2006},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Jorgensen.2006.19,
	title = {Overoptimisme I IT-Prosjekter?},
	howpublished = {Presentation at Veivesenets 11th summer meeting, Svolv{\ae}r, Juni 1},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2006.17,
	title = {Overoptimisme I IT-Prosjekter: Hvorfor L{\ae}rer Vi Aldri?},
	howpublished = {Presentation at OMG-seminar, Oslo, April 19},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Jorgensen.2006.1,
	title = {Prediction of Optimistic Expert Judgment-Based Predictions},
	journal = {EASE 2006},
	year = {2006},
	pages = {40-49},
	publisher = {BCS},
	type = {1},
	address = {UK},
	abstract = {(Paper selected for publication in JSS)},
	isbn = {ISSN 1477-9358},
	author = {J{\o}rgensen, Magne and Faugli, Bj{\o}rn},
	editor = {Kitchenham, Barbara}
}
@inproceedings {Jorgensen.2006.8,
	title = {A Preliminary Model of Judgment-Based Project Software Effort Predictions},
	journal = {IRNOP VIII, Project Research Conference},
	year = {2006},
	pages = {661-668},
	publisher = {Publishing House of Electronci Industry},
	type = {1},
	address = {Beijing},
	abstract = {An improvement of the software industry{\textquoteright}s software development effort estimation processes may benefit from a better understanding of the mental, partly unconscious, processes involved in estimating effort. This paper proposes and tests a theory potentially explaining essential parts of typical judgment-based effort estimation processes ({\textquotedblleft}expert estimation{\textquotedblright}). The theory is based on findings from the human judgment research literature and proposes that judgment-based effort estimation is based on: i) an early categorization of the project to be estimated, ii) a resistance towards a change of the chosen category, and, iii) a {\textquotedblleft}regression{\textquotedblright} of the effort estimate towards a reference value of the chosen category, where the amount of regression depends on the level of uncertainty of the project work. Implications of the theory are tested with results from three software effort estimation experiments. All examined studies confirmed the theory. There is, however, a strong need for more work, independent evidence and clearer description of scope and concepts part of the theory. Finally, we outline a study planned for further testing of essential parts of the theory.},
	isbn = {7-121-03252-X},
	author = {J{\o}rgensen, Magne},
	editor = {Rodney Turner Lix {Ou}}
}
@misc {Jorgensen.2006.15,
	title = {Software Cost Estimation Research at Simula Research Laboratory},
	howpublished = {Seminar presentation at New Bulgarian University, Sofia},
	year = {2006},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Jorgensen.2006.7,
	title = {Software Cost Estimation: When to Use Expert Judgment and When to Use Models},
	journal = {International Symposium on Forecasting},
	year = {2006},
	pages = {53},
	publisher = {International Institute of Forecasters},
	type = {Conference},
	isbn = {999-999-999},
	author = {J{\o}rgensen, Magne},
	editor = {Garc{\'\i}a-Ferrer, Antonio}
}
@misc {Jorgensen.2006.2,
	title = {Software Cost Overruns - How Large Are They and How Should They Be Measured? a Critique of the Standish Group Chaos Report.},
	howpublished = {Invited talk at the conference: OOP (Object oriented programming), M{\"u}nchen, January,},
	year = {2006},
	month = {January},
	author = {J{\o}rgensen, Magne}
}
@article {Grimstad.2006.1,
	title = {Software Effort Estimation Terminology: the Tower of Babel},
	journal = {Information and Software Technology},
	volume = {48},
	number = {4},
	year = {2006},
	pages = {302-310},
	abstract = {This paper provides a review of how software development effort estimation terms are used in software engineering textbooks and research papers. We found that the term {{\textquoteleft}}effort estimate{\textquoteright} frequently is applied without any clarification of its meaning. It is therefore difficult to determine whether the authors{\textquoteright} intended interpretation is an estimate of {{\textquoteleft}}most likely effort{\textquoteright}, {{\textquoteleft}}planned effort{\textquoteright}, {{\textquoteleft}}budgeted effort{\textquoteright}, or something else. This is problematic as these terms are not equivalent and are used for different purposes. The lack of clarity of {{\textquoteleft}}effort estimate{\textquoteright} lowers the quality and interpretability of surveys on software effort estimation accuracy, i.e., it is not clear what the estimation accuracy results really mean. This reduces the estimation evaluation and learning possibilities. We suggest guidelines on how to reduce this terminology ambiguity. To the authors{\textquoteright} knowledge, this is the first published review of software cost estimation terminology.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Gruschke.2006.5,
	title = {To Know Or Not to Know: When Does Feedback Lead to Better Assessment of Uncertainty of Own Beliefs?},
	journal = {IAREP/SABE congress (Behavioural Economics and Economic Psychology)},
	year = {2006},
	pages = {O-226},
	publisher = {Elsevier},
	type = {1},
	abstract = {People are frequently overconfident about the accuracy of their own beliefs. The goal of this experiment is to examine whether, and if so under what conditions, very large amounts of feedback lead to better assessments of the uncertainty of one{\textquoteright}s own beliefs. Fifteen participants answered the same 960 general knowledge questions over two days. Questions were selected from the board game {\textquotedblleft}Who wants to be a millionaire?{\texttrademark}{\textquotedblright}. Each question had four answer alternatives. When an answer alternative had been chosen, the participants were asked to assess the probability that it was the correct answer. They did this by choosing from a list of predefined confidence intervals. The questions belonged to one out of six difficulty categories, as decided by the board game developers. Participants answered piles of 80 questions of the same difficulty at a time. Feedback about the correct answer was received immediately, and feedback about correspondence between {\textquotedblleft}hit rate{\textquotedblright} and confidence level was received immediately after a pile of questions had been answered. In addition, a summary of the first day{\textquoteright}s performance was provided at the start of Day 2. We found that thirteen out of the fifteen participants improved in the correspondence between hit rate and confidence level on the second day, which suggests that the feedback had an effect. The strongest improvement was achieved on questions with high {\textquotedblleft}global{\textquotedblright} difficulty, i.e., high difficulty as assessed by the board game developers, and low {\textquotedblleft}internal{\textquotedblright} difficulty, i.e., low difficulty as perceived by the participants.},
	isbn = {999-999-999},
	author = {Tanja Milijana {Gruschke} and J{\o}rgensen, Magne},
	editor = {Roland-L{\'e}vy, Christine}
}
@misc {Jorgensen.2006.11,
	title = {Wie Gross Sind Die Kosten{\"u}berschreitungen Tats{\"a}chlich},
	howpublished = {Article in OBJEKTspektrum (in German)},
	year = {2006},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Mair.2005.2,
	title = {An Analysis of Data Sets Used to Train and Validate Cost Prediction Systems},
	journal = {International Conference on Software Engineering: Proceedings of the 2005 workshop on Predictor models in software engineering},
	year = {2005},
	note = {ISBN:-159593-125-2},
	pages = {1-6},
	publisher = {ACM Press},
	type = {1},
	address = {New York, NY, USA},
	author = {Mair, Carolyn and Shepperd, Martin and J{\o}rgensen, Magne}
}
@inproceedings {Gruschke.2005.1,
	title = {Assessing Uncertainty of Software Development Effort Estimates:The Learning From Outcome Feedback},
	journal = {11th International Software Metrics Symposium (METRICS 2005), Como, Italy, September 19-22},
	year = {2005},
	note = {Proceedings not published on paper, only on CD-ROM. Therefore, the page numbers are displayed differently than usual.},
	month = {Sept},
	pages = {p.4 (1-10)},
	publisher = {IEEE},
	type = {1},
	abstract = {To enable properly sized software projects budgets and plans it is important to be able to assess the uncertainty of the estimates of most likely effort required to complete the projects. Previous studies show that software professionals tend to be too optimistic about the uncertainty of their effort estimates. This paper reports the results from a preliminary study on the role of outcome feedback in the learning process on effort estimation uncertainty assessment. Software developers were given repeated and immediate outcome feedback, i.e. feedback about the discrepancy between the estimated most likely effort and the actual effort, for the purpose of investigating how much, and how, they improve (learn). We found that a necessary condition for improvement of uncertainty assessments of effort estimates may be the use of explicitly formulated uncertainty assessment strategies. By contrast, intuitionbased uncertainty assessment strategies may lead to no or little learning.},
	author = {Tanja Milijana {Gruschke} and J{\o}rgensen, Magne}
}
@inproceedings {Grimstad.2005.2,
	title = {The Clients{\textquoteright} Impact on Effort Estimation Accuracy in Software Development Projects},
	journal = {11th IEEE International Software Metrics Symposium (METRICS 2005), Como, Italy, September 19-22},
	year = {2005},
	pages = {3},
	publisher = {IEEE},
	type = {1},
	abstract = {It seems clear that there is no simple solution to improved predictability of software development projects. Over several decades various aspects of software development and their relationship to estimation accuracy have been investigated. This paper focus on one such relationship; the clients{\textquoteright} impact on estimation accuracy in software development projects. Factors contributing to overruns as well as factors preventing overruns are investigated. Based on a literature review and a survey of 300 software professionals we find that: 1) It is software professionals{\textquoteright} perception that clients impact estimation accuracy. Changed and new requirements are perceived as the clients{\textquoteright} most frequent contribution to overruns, while overruns are prevented by the availability of competent clients and capable decision makers. 2) Survey results should not be used in estimation accuracy improvement initiatives without further analysis. Surveys typically identify direct and project specific causes for overruns, while substantial improvement is only possible when the underlying causes are understood. Keywords: software estimation, survey, client-vendor relationship, overrun causes},
	keywords = {client-vendor relationship, overrun causes, software estimation, survey},
	author = {Grimstad, Stein and J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Karahasanovic.2005.2,
	title = {Collecting Feedback During Software Engineering Experiments},
	journal = {Empirical Software Engineering},
	volume = {10},
	number = {2},
	year = {2005},
	pages = {113-147},
	abstract = {Objective: To improve the qualitative data obtained from software engineering experiments by gathering feedback during experiments. Rationale: Existing techniques for collecting quantitative and qualitative data from software engineering experiments do not provide sufficient information to validate or explain all our results. Therefore, we would like a cost-effective and unobtrusive method of collecting feedback from subjects during an experiment to augment other sources of data. Design of study: We formulated a set of qualitative questions that might be answered by collecting feedback during software engineering experiments. We then developed a tool to collect such feedback from experimental subjects. This feedback-collection tool was used in four different experiments and we evaluated the usefulness of the feedback obtained in the context of each experiment. The feedback data was triangulated with other sources of quantitative and qualitative data collected for the experiments. Results: We have demonstrated that the collection of feedback during experiments provides useful additional data to: validate the data obtained from other sources about solution times and quality of solutions; check process conformance; understand problem solving processes; identify problems with experiments; and understand subjects{\textquoteright} perception of experiments. Conclusions: Feedback collection has proved useful in four experiments and we intend to use the feedback-collection tool in a range of other experiments to further explore the cost-effectiveness and limitations of this technique. It is also necessary to carry out a systematic study to more fully understand the impact of the feedback-collecting tool on subjects{\textquoteright} performance in experiments.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Welland, Ray}
}
@article {Molokken-ostvold.2005.5,
	title = {A Comparison of Software Project Overruns - Flexible Vs. Sequential Development Models},
	journal = {IEEE Transactions on Software Engineering},
	volume = {31},
	number = {9},
	year = {2005},
	note = {Project questions are dowloadable},
	pages = {754-766},
	abstract = {Flexible software development models, e.g., evolutionary and incremental models, have become increasingly popular. Advocates of these models claim that among the benefits of these models are reduced software project overruns, which is one of the main challenges of software project management. This paper describes an in-depth survey of software development projects. The results support the claim that projects which employ a flexible development model experience less effort overruns than do those who employ a sequential model. The reason for the difference is not obvious. We found, for example, no variation in project size, estimation process, or delivered proportion of planned functionality between projects applying different types of development model. When the managers were asked to provide reasons for software overruns and/or estimation accuracy, the largest difference were that more of flexible projects cited good requirement specifications and good collaboration/communication with clients as contributing to accurate estimates.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@article {Jorgensen.2005.5,
	title = {Evidence-Based Guidelines for Assessment of Software Development Cost Uncertainty},
	journal = {IEEE Transactions on Software Engineering},
	volume = {31},
	number = {11},
	year = {2005},
	pages = {942-954},
	author = {J{\o}rgensen, Magne}
}
@article {Dyba.2005.1,
	title = {Evidence-Based Software Engineering for Practitioners},
	journal = {IEEE Software},
	volume = {22},
	number = {1},
	year = {2005},
	pages = {58-65},
	publisher = {-},
	abstract = {Objective: Our objective is to explain how practitioners in industry can use evidencebased software engineering (EBSE) to assist decisions concerning the adoption of new techniques. Rationale: Practitioners may make incorrect technology adoption decisions if they do not consider scientific evidence about the efficacy of techniques. Method: We adapt procedures used for evidence-based medicine to software engineering and discuss how these procedures map to software process improvement. Results: We found EBSE fitted well with current ideas concerning process improvement and that it could be an important means for closing the gap between research and practice. However EBSE presents difficulties for practitioners because current software engineering research is limited and not reported in a manner that assists accumulation and evaluation of evidence. Conclusions: EBSE has potential benefits for software engineering practitioners, but will be hindered without changes to software engineering research. Keywords: Evidence, empirical software engineering, evaluation.}, how these procedures map to software process improvement. Results = {We found EBSE},
	keywords = {empirical software engineering, evaluation., Evidence},
	author = {Dyb{\r a}, Tore and Kitchenham, Barbara and J{\o}rgensen, Magne}
}
@article {Molokken-ostvold.2005.4,
	title = {Expert Estimation of the Effort of Web-Development Projects: Are Software Professionals in Technical Roles More Optimistic Than Those in Non-Technical Roles?},
	journal = {Empirical Software Engineering},
	volume = {10},
	number = {1},
	year = {2005},
	pages = {7-30},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@misc {Jorgensen.2005.17,
	title = {Hvordan Unng{\r a} VINNERENS FORBANNELSE I Anbudsrunder Med Mange Tilbydere Og Sterk Fokusering P{\r a} Pris?},
	howpublished = {Presenation at SPIKE seminar, June},
	year = {2005},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2005.20,
	title = {Hvorfor Er Vi Overoptimistiske Igjen Og Igjen Og Igjen ...},
	howpublished = {Presentation IT-Fornebu halvtimen},
	year = {2005},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Jorgensen.2005.1,
	title = {Industrial Use of Formal Software Cost Estimation Models: Expert Estimation in Disguise?},
	journal = {Proceedings of EASE, Keele, UK, April 11-13},
	year = {2005},
	month = {April 2005},
	pages = {1-7},
	publisher = {Keele University},
	type = {1},
	abstract = {The goal of this paper is to propose and evaluate the hypothesis that software cost estimates based on formal estimation models are frequently expert estimation in disguise, i.e., that the cost estimates are not as mechanically derived as prescribed and assumed. We test implications of the hypothesis through discussion of related work and an empirical study of function point-based effort estimation of software projects. The actual effort estimates of the projects were compared with the effort estimates one would expect if the formal function point model was applied as prescribed. We observed several large deviations between the actual and the mechanically derived effort estimates, which we interpret as indications of a strong impact from expert judgment. Important limitations of our study are that the hypothesis is formulated vaguely, that there is not much evidence available, and, that we may have had a tendency to bias our search towards supporting evidence. More studies are therefore needed, preferably from independent researchers. If our hypothesis is correct, implementation of formal software cost estimation models should include means to avoid unwanted effects of initial beliefs and irrelevant information.},
	author = {J{\o}rgensen, Magne and Tanja Milijana {Gruschke}}
}
@misc {Jorgensen.2005.10,
	title = {Kan Vi Unng{\r a} at "s{\r a} {\r a} Si Helt Sikkert" Bare Betyr "60\% Sikkert"?},
	howpublished = {Prosjektledelse (norsk fagtidsskrift for prosjektledere), Nr 2, s. 29-31},
	year = {2005},
	month = {September},
	author = {J{\o}rgensen, Magne and Teigen, Karl-Halvor}
}
@misc {Jorgensen.2005.18,
	title = {"The Magic Step" I Ekspertestimering Av IT-Utviklingskostnader. Hvorfor F{\o}les 1000 Timeverk Mer Riktig Enn 1500?},
	howpublished = {Invited talk at JavaZone 2005},
	year = {2005},
	author = {J{\o}rgensen, Magne and Grimstad, Stein and Tanja Milijana {Gruschke}}
}
@misc {Jorgensen.2005.11,
	title = {The Magic Step in Expert Estimation of Software Cost. Why Does 1000 Feel More Right Than 1500?},
	howpublished = {Invited talk at: JavaZone, 2005},
	year = {2005},
	month = {September},
	author = {J{\o}rgensen, Magne and Grimstad, Stein and Tanja Milijana {Gruschke}}
}
@inproceedings {Jorgensen.2005.7,
	title = {The "Magic Step" of Judgment-Based Software Effort Estimation},
	journal = {International Conference on Cognitive Economics},
	year = {2005},
	pages = {105-114},
	publisher = {New Bulgarian University, Sofia, August 5-8},
	type = {1},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Molokken-ostvold.2005.1,
	title = {Management of Public Software Projects: Avoiding Overruns},
	journal = {Hawaiian International Conference on Business, May 25-28, Hawaii, USA},
	year = {2005},
	month = {May},
	publisher = {Hawaiian International Conference on Business},
	type = {1},
	abstract = {Avoiding overruns is one of the major challenges in public software projects. In order to reduce overruns and, in turn, increase the chances of project success, it is essential to have an unbiased overview of the frequency and magnitude of overruns, as well as knowledge about factors which may prevent overruns. In a recent survey of the software industry it was found that the average magnitude of overruns in work-hours was 67\% for projects with a public client, compared to 21\% for projects with a private client. This paper presents results from two surveys of software companies and professionals that investigate the possible differences between public and private clients, and a review of research on public IT-projects. The results indicate that there are several properties of public projects on the political, organizational and individual level which may contribute to the large overruns. These include artificial deadlines based on political terms, regulations on procurement and development processes, and lack of technology and project management skills. The challenges appear to be common for several countries in the OECD region which have investigated this topic. In order to reduce overruns of public projects, both contractors and government officials have to address these issues.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and S{\o}rgaard, P{\r a}l and Grimstad, Stein}
}
@inproceedings {Jorgensen.2005.2,
	title = {Over-Optimism in Software Development Projects: {\textquotedblleft}The Winner{\textquoteright}s Curse{\textquotedblright}},
	journal = {Proceedings of IEEE CONIELECOMP, Puebla, Mexico, February 28-March 2},
	year = {2005},
	month = {March},
	pages = {280-285},
	publisher = {IEEE Computer Society},
	type = {1},
	author = {J{\o}rgensen, Magne and Grimstad, Stein}
}
@misc {Jorgensen.2005.19,
	title = {Overoptimisme I IT-Prosjekter: Hvorfor L{\ae}rer Vi Aldri?},
	howpublished = {Simula seminar with approx. 100 software professionals},
	year = {2005},
	author = {J{\o}rgensen, Magne}
}
@article {Jorgensen.2005.3,
	title = {Practical Guidelines for Better Support of Expert Judgement-Based Software Effort Estimation},
	journal = {IEEE Software},
	volume = {22},
	number = {3},
	year = {2005},
	pages = {57-63},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2005.21,
	title = {Practical Guidelines for Expert-Judgment-Based Software Effort Estimation},
	howpublished = {Invited article to SPIKE newsletter IMPROVE},
	year = {2005},
	author = {J{\o}rgensen, Magne}
}
@misc {Teigen.2005.2,
	title = {On the Psychology of Uncertainty Intervals},
	howpublished = {Presentation at the seminar: Confidence in Interactive Decisions. University of Leicester},
	year = {2005},
	author = {Teigen, Karl-Halvor and J{\o}rgensen, Magne and Halberg, Anne-M}
}
@misc {Jorgensen.2005.16,
	title = {Teaching Evidence-Based Software Engineering},
	howpublished = {EASE, Keele, England},
	year = {2005},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {Jorgensen.2005.4,
	title = {Teaching Evidence-Based Software Engineering to University Students},
	journal = {11th IEEE International Software Metrics Symposium, Como, Italy, September 19-22},
	year = {2005},
	month = {September},
	type = {1},
	abstract = {Evidence-based software engineering (EBSE) describes a process of identifying, understanding and evaluating findings from research and practice-based experience. This process aims at improving software engineering decisions. For the last three years, EBSE has been taught to university students at Hedmark University College, Rena, Norway. The motivation for the EBSE-course is that it is essential for the students, as future practitioners, to learn how to base important software engineering decisions on the systematic and critical evaluation of the best available evidence. The main purpose of this paper is to inspire and support other universities in their work on developing their own EBSE-courses. For this purpose we report on how our course has been organized and what lessons have been learned. There are currently no studies available on the effects of teaching EBSE and, as far as we know, only we have gained practice-based experience. To acquire more knowledge about the costs and benefits of teaching EBSE we hope that other universities will develop their own EBSE-courses and report their experience.},
	author = {J{\o}rgensen, Magne and Kitchenham, Barbara and Dyb{\r a}, Tore}
}
@misc {Jorgensen.2005.15,
	title = {Valg Av IT-Leverand{\o}r Basert P{\r a} Laveste Pris:VINNERENS FORBANNELSE},
	howpublished = {Presentation at the DnD conference Software 2005},
	year = {2005},
	author = {J{\o}rgensen, Magne}
}
@article {Teigen.2005.1,
	title = {When 90\% Confidence Intervals Are Only 50\% Certain: on the Credibility of Credible Intervals},
	journal = {Applied Cognitive Psychology},
	volume = {19},
	number = {4},
	year = {2005},
	pages = {455-475},
	abstract = {Estimates of confidence intervals for general knowledge items are usually too narrow. We report five experiments showing that people have much less confidence in these intervals than dictated by the assigned level of confidence. For instance, 90\% intervals can be associated with an estimated confidence of 50\% or less (and still lower hit rates). Moreover, interval width appears to remain stable over a wide range of instructions (high and low numeric and verbal confidence levels). This leads to a high degree of overconfidence for 90\% intervals, but less for 50\% intervals or for free choice intervals (without an assigned degree of confidence). To increase interval width one may have to ask exclusion rather than inclusion questions, for instance by soliciting {\textquotedblleft}improbable{\textquotedblright} upper and lower values (Experiment 4), or by asking separate {\textquotedblleft}more than{\textquotedblright} and {\textquotedblleft}less than{\textquotedblright} questions (Experiment 5). We conclude that interval width and degree of confidence have different determinants, and cannot be regarded as equivalent ways of expressing uncertainty.},
	author = {Teigen, Karl-Halvor and J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.2004.a,
	title = {Better Sure Than Safe? Overconfidence in Judgment Based Software Development Effort Prediction Intervals},
	journal = {Journal of Systems and Software},
	volume = {70},
	number = {1-2},
	year = {2004},
	pages = {79-93},
	abstract = {The uncertainty of a software development effort estimate can be indicated through a prediction interval, i.e., the estimated minimum and maximum effort corresponding to a specific confidence level. For example, a project manager may be {\textquotedblleft}90\% confident{\textquotedblright} or believe that is it {\textquotedblleft}very likely{\textquotedblright} that the effort required to complete a project will be between 8,000 and 12,000 work-hours. This paper describes results from four studies (Studies A-D) on human judgement (expert) based prediction intervals of software development effort. Study A examines the accuracy of the prediction intervals in real software projects. The results suggest that the prediction intervals were generally much too narrow to reflect the chosen level of confidence, i.e., that there was a strong over-confidence. Studies B, C and D try to understand the reasons for the observed over-confidence. Study B examines the possibility that the over-confidence is related to type of experience or estimation process. Study C examines the possibility that the concept of confidence level is difficult to interpret for software estimators. Finally, Study D examines the possibility that there are unfortunate feedback mechanisms that reward over-confidence.},
	author = {J{\o}rgensen, Magne and Teigen, Karl-Halvor and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2004.b,
	title = {Does Use of Development Model Affect Estimation Accuracy and Bias?},
	journal = {Product Focused Software Process Improvement: 5th International Conference (PROFES 2004)},
	year = {2004},
	note = {ISBN: 3-540-21421-6},
	month = {April 5-8},
	pages = {17-29},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Kansai Science City, Japan},
	abstract = {Objective. To investigate how the use of incremental and evolutionary development models affects the accuracy and bias of effort and schedule estimates of software projects. Rationale. Advocates of incremental and evolutionary development models often claim that use of these models results in improved estimation accuracy. Design of study. We conducted an in-depth survey, where information was collected through structured interviews with 22 software project managers in 10 different companies. We collected and analyzed information about estimation approach, effort estimation accuracy and bias, schedule estimation accuracy and bias, completeness of delivered functionality and other estimation related information. Results. We found no impact from the development model on the estimation approach. However, we found that incremental and evolutionary projects were less prone to effort overruns. The degree of delivered functionality and schedule estimation accuracy, on the other hand, were seemingly independent of development model. Conclusion. The use of incremental and evolutionary development models may reduce the chance of effort overruns.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and A. C. {Lien} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and S. E. {Hove}}
}
@inproceedings {SE.5.Joergensen.2004.b,
	title = {Eliminating Over-Confidence in Software Development Effort Estimates},
	journal = {Conference on Product Focused Software Process Improvement},
	year = {2004},
	month = {April},
	pages = {174-184},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Japan},
	abstract = {Minimum-maximum effort intervals are applied in the planning of software development projects in order to, among other things, determine the contingency buffer. Several studies suggest that judgment-based minimummaximum intervals are based on a systematic over-confidence in the accuracy of the effort estimates. In this paper, we investigate whether the possession by estimators of information about previous estimation error for similar projects reduces this over-confidence. Nineteen realistically composed estimation teams provided minimum-maximum effort intervals for the same software project. Ten of the teams (Group A) received no instructions about the uncertainty assessment process. The remaining nine teams (Group B) were instructed to begin the minimum-maximum effort interval assessment by recalling the distribution of estimation error for similar projects. We found that the recall of the error distribution of the Group B teams did have an impact, but mainly on the assessment of the estimated minimum effort, not on the maximum effort. We discuss reasons for this finding and provide recommendations.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {SE.4.Joergensen.2004.g,
	title = {An Empirical Study of Software Project Bidding},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {12},
	year = {2004},
	pages = {953-969},
	abstract = {The study described in this paper reports from a real-life bidding process in which 35 companies were bidding for the same contract. The bidding process consisted of two separate phases: A pre-study phase and a bidding phase. In the pre-study phase 17 of the 35 bidding companies provided rough non-binding price indications based on a brief, incomplete description of user requirements. In the bidding phase, all 35 companies provided bids based on a more complete requirement specification that described a software system with substantially more functionality than the system indicated in the pre-study phase. The main result of the study is that the 17 companies involved in the pre-study phase presented bids that were on average about 70\% higher than the bids of the other companies. We propose a preliminary theory that has the potential to explain this difference. This preliminary theory is based, amongst other things, on the {\textquotedblleft}precautionary bidding effect{\textquotedblright} found in auctioning studies. Two important implications of our preliminary theory are that: 1) Software clients tend to achieve better price/uncertainty relationships, i.e., better prices, when the requirement uncertainty perceived by the bidders is low. 2) Software clients should not request early price indications based on limited and uncertain information when the final bids can be based on more complete and reliable information.},
	author = {J{\o}rgensen, Magne and G. J. {Carelius}}
}
@misc {SE.8.Joergensen.2004.d,
	title = {Erfaring Er Ikke Ekspertise},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {4. juni},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.n,
	title = {Estimering Av IT-Prosjekter},
	howpublished = {Simula-seminar: Presentasjon},
	year = {2004},
	month = {mars},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Kitchenham.2004,
	title = {Evidence-Based Software Engineering},
	journal = {International Conference on Software Engineering (ICSE{\textquoteright}04)},
	year = {2004},
	month = {23-28 May},
	pages = {273-281},
	publisher = {IEEE Computer Society, Washington DC, USA},
	type = {1},
	address = {Edinburgh},
	abstract = {Objective: Our objective is to describe how software engineering might benefit from an evidence-based approach and to identify the potential difficulties associated with the approach. Method: We compared the organisation and technical infrastructure supporting evidence-based medicine (EBM) with the situation in software engineering. We considered the impact that factors peculiar to software engineering (i.e. the skill factor and the lifecycle factor) would have on our ability to practice evidence-based software engineering (EBSE). Results: EBSE promises a number of benefits by encouraging integration of research results with a view to supporting the needs of many different stakeholder groups. However, we do not currently have the infrastructure needed for widespread adoption of EBSE. The skill factor means software engineering experiments are vulnerable to subject and experimenter bias. The lifecycle factor means it is difficult to determine how technologies will behave once deployed. Conclusions: Software engineering would benefit from adopting what it can of the evidence approach provided that it deals with the specific problems that arise from the nature of software engineering.},
	author = {Kitchenham, Barbara and Dyb{\r a}, Tore and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2004.c,
	title = {Generalization and Theory-Building in Software Engineering Research},
	journal = {Empirical Assessment in Software Engineering (EASE2004)},
	year = {2004},
	pages = {29-36},
	publisher = {IEE Proceedings},
	type = {1},
	abstract = {The main purpose of this paper is to generate discussions which may improve how we conduct empirical software engineering studies. Our position is that statistical hypothesis testing plays a too large role in empirical software engineering studies. The problems of applying statistical hypothesis testing in empirical software engineering studies is illustrated by the finding: Only 3 out of the 47 studies in Journal of Empirical Software Engineering which applied statistical hypothesis testing, were able to base their statistical testing on well-defined populations and random samples from those populations. The frequent use of statistical hypothesis testing may also have had unwanted consequences on the study designs, e.g., it may have contributed to a too low focus on theory building. We outline several steps we believe are useful for a change in focus from \&ldquo;generalizing from a random sample to a larger population\&rdquo; to \&ldquo;generalizing across populations through theory-building\&rdquo;},
	isbn = {0 86341 435 4},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Moloekken-Oestvold.2004.b,
	title = {Group Processes in Software Effort Estimation},
	journal = {Empirical Software Engineering},
	volume = {9},
	number = {4},
	year = {2004},
	pages = {315-334},
	abstract = {The effort required to complete software projects is often estimated, completely or partially, using the judgement of experts, whose assessment may be biased. In general, such bias as there is seems to be towards estimates that are overly optimistic. The degree of bias varies from expert to expert, and seems to depend on both conscious and unconscious processes. One possible approach to reduce this bias towards over-optimism is to combine the judgments of several experts. This paper describes an experiment in which experts with different backgrounds combined their estimates in group discussion. First, twenty software professionals were asked to provide individual estimates of the effort required for a software development project. Subsequently, they formed five estimation groups, each consisting of four experts. Each of these groups agreed on a project effort estimate via the pooling of knowledge in discussion. We found that the groups submitted less optimistic estimates than the individuals. Interestingly, the group discussion-based estimates were closer to the effort expended on the actual project than the average of the individual expert estimates were, i.e., the group discussions led to better estimates than a mechanical averaging of the individual estimates. The groups{\textquoteright} ability to identify a greater number of the activities required by the project is among the possible explanations for this reduction of bias.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.e,
	title = {Hvor Gode Er V{\r a}re It-Akt{\o}rer},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {18. juni},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.l,
	title = {Hvordan F{\r a} Tak I Den Reelle Usikkerheten I IT-Prosjekter},
	howpublished = {Den Norske Dataforening: Presentasjon p{\r a} seminar},
	year = {2004},
	month = {8. juni},
	author = {J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.2004.b,
	title = {The Impact of Customer Expectation on Software Development Effort Estimates},
	journal = {International Journal of Project Management},
	volume = {22},
	number = {4},
	year = {2004},
	month = {May},
	pages = {317-325},
	abstract = {The results from the study described in this paper suggest that customer expectations of a project{\textquoteright}s total cost can have a very large impact on human judgment-based estimates (expert estimates) of the most likely use of software development effort. The information that the customer expectations are not valid estimation information did not remove the impact. Surprisingly, the estimators did not notice this impact or assessed it to be low. An implication of the results is that the provision of realistic project estimate of most likely use of effort may require that the estimators do not know the customer{\textquoteright}s expectations of the total cost of the project.},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Joergensen.2004.e,
	title = {Increasing Realism in Effort Estimation Uncertainty Assessments: It Matters How You Ask},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {4},
	year = {2004},
	pages = {209-217},
	abstract = {Traditionally, software professionals are requested to provide minimum-maximum intervals to indicate the uncertainty of their effort estimates. In this paper we claim that the traditional request is not optimal and leads to over-optimistic views about the level of estimation uncertainty. Instead, we propose, a person different from the estimator should identify minimum and maximum values relevant for planning or bidding purposes and request that the software professionals assess how likely it is that these values are exceeded. Not only does this seem to increase realism, but it also leads to more useful uncertainty assessments. Our claims are based on the results of a previously reported experiment and field studies in two companies. The two software companies were instructed to apply the traditional, and our alternative, framing on random samples of their projects. In total, we collected information about 47 projects applying the traditional framing and 23 projects applying the alternative framing.},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.m,
	title = {IT-Sprekk Skyldes Urealistiske Kunder},
	howpublished = {digi.no},
	year = {2004},
	month = {30. mars},
	publisher = {digi.no},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.f,
	title = {Kunsten {\r a} M{\r a}le},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {13. august},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.c,
	title = {Lav Pris Gir Lav Kvalitet},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {21. mai},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Moloekken-Oestvold.2004.j,
	title = {Offentlig Fare},
	howpublished = {Newspaper article in Computerworld},
	year = {2004},
	month = {24. september},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and S{\o}rgaard, P{\r a}l}
}
@misc {SE.8.Joergensen.2004.h,
	title = {Optimisme: Styrke Og Svakhet},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {22. oktober},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.k,
	title = {Overskridelser I Offentlige IT-Prosjekter},
	howpublished = {Computerworld: Kronikk},
	year = {2004},
	month = {april},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.i,
	title = {Produktet Som Ikke Kan M{\r a}lbindes},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {17. desember},
	author = {J{\o}rgensen, Magne}
}
@techreport {SE.7.Moloekken-Oestvold.2004,
	title = {Project Estimation in the Norwegian Software Industry - a Summary},
	number = {2004-03},
	year = {2004},
	note = {Results from the BEST-Pro (Better Estimation of Software Tasks and Process Improvement) survey},
	publisher = {Simula Research Laboratory},
	abstract = {This report provides an overview of the results obtained from a survey on project estimation in Norwegian software companies. The survey was conducted between February and November 2003. The main results are: {\textbullet} We observed that 76\% of the projects used more effort than estimated, while 19\% used less. The average effort overrun was 41\%. {\textbullet} Average effort overrun was 67\% in projects with a public client, compared to an average effort overrun of 21\% for projects with a private client. {\textbullet} Projects that used an incremental or evolutionary development approach had an average effort overrun of 24\%, as opposed to the average 55\% overrun for projects that used a waterfall-based development approach. The frequency and magnitude of effort overruns found in this survey seems to be similar to results reported from surveys conducted in other countries in the past 20 years. The observed differences in effort overruns between private and public projects may be caused by differences in bidding procedures, level of client involvement or acceptance procedures. In order to reduce the risk of effort overruns, software companies should: {\textbullet} Focus on analyzing their own estimation performance, and invest in estimation improvement (for instance through experience databases or work breakdown structures). {\textbullet} Differentiate risk buffers based on the type of the customer, development approach and the size of the project. {\textbullet} Try to establish an {\textquotedblleft}as close as possible{\textquotedblright} dialogue with the customers (e.g. through an incremental development approach). All companies should analyse completed projects, in order to benchmark their performance. This allows for improvement efforts to be identified.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and A. C. {Lien} and S. E. {Hove}}
}
@article {SE.4.Joergensen.2004.h,
	title = {Reasons for Software Effort Estimation Error: Impact of Respondents Role, Information Collection Approach, and Data Analysis Method},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {12},
	year = {2004},
	pages = {993-1007},
	abstract = {This study is a first step towards better processes of understanding why errors occur in software effort estimation. Within one software development company, we collected information about estimation errors through: (1) Interviews with estimation responsible employees in different roles, (2) Estimation experience reports from 68 completed projects, and, (3) Statistical analysis of relations between characteristics of the 68 completed projects and estimation error. We found that the role of the respondents, the data collection approach, and the type of analysis had an important impact on the reasons for estimation error that were given. We found, for example, a strong tendency to perceive factors outside the respondents{\textquoteright} own control as important reasons for inaccurate estimates. Reasons given for accurate estimates, on the other hand, typically cited factors that were within the respondents{\textquoteright} own control, and were determined by the estimators{\textquoteright} skill or experience. This bias in types of reason means that the collection only of project managers{\textquoteright} viewpoints will not yield balanced models of reasons for estimation error. Unfortunately, previous studies on reasons for estimation error have tended to collect information from project managers only. We recommend that software companies combine estimation error information from in-depth interviews with stakeholders in all relevant roles, estimation experience reports, and results from statistical analyses of project characteristics.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {SE.4.Joergensen.2004.f,
	title = {Regression Models of Software Development Effort Estimation Accuracy and Bias},
	journal = {Empirical Software Engineering},
	volume = {9},
	number = {4},
	year = {2004},
	pages = {297-314},
	abstract = {Traditionally, software professionals are requested to provide minimum-maximum intervals to indicate the uncertainty of their effort estimates. In this paper we claim that the traditional request is not optimal and leads to over-optimistic views about the level of estimation uncertainty. Instead, we propose, a person different from the estimator should identify minimum and maximum values relevant for planning or bidding purposes and request that the software professionals assess how likely it is that these values are exceeded. Not only does this seem to increase realism, but it also leads to more useful uncertainty assessments. Our claims are based on the results of a previously reported experiment and field studies in two companies. The two software companies were instructed to apply the traditional, and our alternative, framing on random samples of their projects. In total, we collected information about 47 projects applying the traditional framing and 23 projects applying the alternative framing.},
	author = {J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.2004.c,
	title = {A Review of Studies on Expert Estimation of Software Development Effort},
	journal = {Journal of Systems and Software},
	volume = {70},
	number = {1-2},
	year = {2004},
	pages = {37-60},
	abstract = {This paper provides an extensive review of studies related to expert estimation of software development effort. The main goal and contribution of the review is to support the research on expert estimation, e.g., to ease other researcher{\textquoteright}s search for relevant expert estimation studies. In addition, we provide software practitioners with useful estimation guidelines, based on the research-based knowledge of expert estimation processes. The review results suggest that expert estimation is the most frequently applied estimation strategy for software projects, that there is no substantial evidence in favour of use of estimation models, and that there are situations where we can expect expert estimates to be more accurate than formal estimation models. The following twelve expert estimation {\textquotedblleft}best practice{\textquotedblright} guidelines are evaluated through the review: 1) Evaluate estimation accuracy, but avoid high evaluation pressure, 2) Avoid conflicting estimation goals, 3) Ask the estimators to justify and criticize their estimates, 4) Avoid irrelevant and unreliable estimation information, 5) Use documented data from previous development tasks, 6) Find estimation experts with relevant domain background and good estimation records, 7) Estimate top-down and bottom-up, independently of each other, 8) Use estimation checklists, 9) Combine estimates from different experts and estimation strategies, 10) Assess the uncertainty of the estimate, 11) Provide feedback on estimation accuracy and development task relations, and, 12) Provide estimation training opportunities. We found supporting evidence for all twelve estimation principles, and provide suggestions on how to implement them in software organizations.},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.g,
	title = {Simpsons Paradoks},
	howpublished = {Computerworld, Under lupen},
	year = {2004},
	month = {27. august},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.b,
	title = {Software Cost Overruns - How Large Are They and How Should They Be Measured?},
	howpublished = {Invited talk at: 60th seminar of the Software Reliability and Metrics Club, London},
	year = {2004},
	month = {November},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.o,
	title = {Software Cost Overruns - How Large Are They? Critical Comments on the CHAOS Report},
	howpublished = {SPIKE-seminar: Presentasjon},
	year = {2004},
	month = {September},
	abstract = {The Standish Group (www.standishgroup.com) claims that the results of their CHAOS research, i.e., their large-scaled surveys conducted in 1994, 1996, 1998, 2000 and 2002, are the most widely quoted statistics in the IT industry. This may very well be true. Quoted with particular frequency are the results described in the 1994 CHAOS report [1], probably because the 1994 CHAOS report is free and easily can be downloaded from the web. The results of that report have been used in several recent governmental reports, project reviews, and research studies. Examples are the PITAC 1999 report [2] and the cost estimation study described in [3]. An important result from the 1994 CHAOS research is the reported 189\% average cost overrun of so-called challenged projects, i.e., projects not on time, on cost, and with all specified functionality. In this paper we argue that the 189\% average cost overrun number, as it is commonly interpreted, is not consistent with results of other cost accuracy surveys and probably far too high to reflect the average cost overrun in that period. The measures and the research method of the CHAOS survey are insufficiently described to evaluate the quality of the results, e.g., there are many possible interpretations of what is meant by {{\textquoteleft}}cost overrun{\textquoteright} in the CHAOS reports. We should therefore cease to trust the 189\% average cost overrun as a reference point for performance of software projects until such time as the Standish Group disclose how they measure cost overrun and how they conduct their research.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Moloekken-Oestvold.2004.a,
	title = {A Survey on Software Estimation in the Norwegian Industry},
	journal = {10th International Software Metrics Symposium (METRICS 2004)},
	year = {2004},
	month = {September 16-18},
	pages = {208-219},
	publisher = {IEEE Computer Society Press},
	type = {1},
	address = {Chicago, USA},
	abstract = {This paper seeks to provide an overview of the estimation methods that software companies apply to estimate their projects, why those methods are chosen, and how accurate they are. In order to improve software estimation accuracy, such knowledge is essential. We conducted an in-depth survey, where information was collected through structured interviews with senior managers from 18 different companies and project managers of 52 different projects. We collected and analyzed information about estimation approach, effort estimation accuracy and bias, schedule estimation accuracy and bias, completeness of delivered functionality and other estimation related information. Our results suggest, for example, that average effort overruns are 41\%, that the software estimation performance has not changed much the last 10-20 years, that expert estimation is the dominating estimation method, that estimation accuracy is not much impacted by use of formal estimation models, and that software managers tend to believe that the estimation accuracy of their company is better than it actually is.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and A. C. {Lien} and S. E. {Hove}}
}
@article {SE.4.Joergensen.2004.d,
	title = {Top-Down and Bottom-Up Expert Estimation of Software Development Effort},
	journal = {Information and Software Technology},
	volume = {46},
	number = {1},
	year = {2004},
	pages = {3-16},
	abstract = {Expert estimation of software development effort may follow top-down or bottom-up strategies, i.e., the total effort estimate may be based on properties of the project as a whole and distributed over project activities (top-down) or calculated as the sum of the project activity estimates (bottom-up). The study reported in this paper examines differences between these two strategies based on measurement and video recording of the discussions of seven estimation teams. Each estimation team applied a top-down estimation strategy on one project and a bottom-up estimation strategy on another. The results from the study contribute, we believe, to an improved understanding of when to apply top-down and when to apply bottom-up estimation strategies.},
	author = {J{\o}rgensen, Magne}
}
@misc {Jorgensen.2004.1,
	title = {Uncertainty Assessments in Situations With Unknown Probabilities, Unknown Events and Partial Impact of Outcome},
	howpublished = {Misc},
	year = {2004},
	note = {Presentation at conference on "Causality, Uncertainty and Ignorance", Koblenz, August, 2004},
	month = {8},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.8.Joergensen.2004.a,
	title = {Uncertainty Assessments in Software Development Projects},
	howpublished = {Presentation at: Causality, Uncertainty \& Ignorance, 3rd International Summer School 2004, University of Konstanz, Germany},
	year = {2004},
	month = {16-21 August},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2003.b,
	title = {An Attempt to Model the Software Development Effort Estimation Accuracy and Bias},
	journal = {Proceedings of Conference on Empirical Assessment in Software Engineering},
	year = {2003},
	month = {April},
	pages = {117-128},
	publisher = {-},
	type = {1},
	address = {Keele, England},
	abstract = {This paper describes regression models of estimation accuracy and bias, i.e., models aiming at explaining the accuracy and bias variation of an organization{\textquoteright}s software development effort estimates. We collected information about variables we believed would impact the estimation accuracy or bias of tasks completed by the organization. In total, information about 49 software development tasks was collected. We found that the following conditions led to decrease in estimation accuracy: 1) Estimates were provided by a person in the role {{\textquoteleft}}software developer{\textquoteright} instead of {{\textquoteleft}}project leader{\textquoteright}, 2) The project had its highest priority on time-to-delivery instead of quality or cost, and, 3) The estimator did not participate in the completion of the task. The following conditions led to increased bias towards under-estimation: 1) Estimates were provided by a person with the role {{\textquoteleft}}software developer{\textquoteright} instead of {{\textquoteleft}}project leader{\textquoteright}, and, 2) The estimator assessed the accuracy of own estimates of similar, previously completed tasks to be low (more than 20\% deviation). Although all variables included in the regression models were significant (p@inbook {Sjoberg.2003.1,
	title = {Challenges and Recommendations When Increasing the Realism of Controlled Software Engineering Experiments},
	booktitle = {Empirical Methods and Studies in Software Engineering: Experiences from ESERNET},
	series = {Lecture Notes in Computer Science, Volume 2765},
	year = {2003},
	pages = {24-38},
	publisher = {Springer},
	organization = {Springer},
	chapter = {Part II: Method Chapters},
	address = {Berlin / Heidelberg},
	abstract = {An important goal of most empirical software engineering experiments is the transfer of the research results to industrial applications. To convince industry about the validity and applicability of the results of controlled software engineering experiments, the tasks, subjects and the environments should be as realistic as practically possible. Such experiments are, however, more demanding and expensive than experiments involving students, small tasks and pen-and-paper environments. This chapter describes challenges of increasing the realism of controlled experiments and lessons learned from the experiments that have been conducted at Simula Research Laboratory.},
	isbn = {978-3-540-40672-3},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Vok{\'a}c, Marek},
	editor = {Conradi, Reidar and Alf Inge {Wang}}
}
@article {SE.4.Joergensen.2003.b,
	title = {An Effort Prediction Interval Approach Based on the Empirical Distribution of Previous Estimation Accuracy},
	journal = {Information and Software Technology},
	volume = {45},
	number = {3},
	year = {2003},
	month = {March},
	pages = {123-136},
	abstract = {When estimating software development effort, it may be useful to describe the uncertainty of the estimate through an effort prediction interval (PI). An effort PI consists of a minimum and a maximum effort value and a confidence level. We introduce and evaluate a software development effort PI approach that is based on the assumption that the estimation accuracy of earlier software projects predicts the effort PIs of new projects. First, we demonstrate the applicability and different variants of the approach on a data set of 145 software development tasks. Then, we experimentally compare the performance of one variant of the approach with human (software professionals{\textquoteright}) judgment and regression analysis-based effort PIs on a data set of 15 development tasks. Finally, based on the experiment and analytical considerations, we discuss when to base effort PIs on human judgment, regression analysis, or our approach.},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Joergensen.2003.c,
	title = {How Much Does a Vacation Cost? Or What Is a Software Cost Estimate?},
	journal = {ACM Software Engineering Notes},
	volume = {28},
	number = {6},
	year = {2003},
	pages = {30},
	abstract = {What is a software cost estimate? Is it the most likely cost, the planned cost, the budget, the price, or, something else? Through comparison with vacation cost estimation and a real-life case we illustrate that it is not meaningful to compare and analyze cost estimates unless it is clear which interpretation is applied. Unfortunately, the software industry, software engineering textbooks and scientific estimation studies do frequently not clarify how they apply the term {{\textquoteleft}}cost estimate{\textquoteright}. We argue that this lack of clarity may lead to conflicting estimation goals, communication problems, and, learning problems, and provide recommendations on how to deal with these problems.},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2003,
	title = {A Preliminary Checklist for Software Cost Management},
	journal = {IEEE International Conference on Quality Software},
	year = {2003},
	month = {November},
	pages = {134-140},
	publisher = {-},
	type = {1},
	address = {Dallas, USA},
	abstract = {This paper presents a process framework and a preliminary checklist for software cost management. While most textbooks and research papers on cost estimation look mainly at the {\textquotedblleft}Estimation{\textquotedblright} phase, our framework and checklist includes the phases relevant to estimation: {\textquotedblleft}Preparation{\textquotedblright}, {\textquotedblleft}Estimation{\textquotedblright}, {\textquotedblleft}Application{\textquotedblright}, and {\textquotedblleft}Learning{\textquotedblright}. We believe that cost estimation processes and checklists should support these phases to enable high estimation accuracy. The checklist we suggest is based on checklists from a number of sources, e.g., a handbook in forecasting and checklists present in several Norwegian software companies. It needs, however, to be extended through feedback from other researchers and software practitioners. There is also a need for a provision of conditions for meaningful use of the checklist issues and descriptions of the strength and sources of evidence in favor of the checklist issues. The present version of the checklist should therefore be seen as preliminary and we want to get feedback from the conference participants and other readers of this paper for further improvements.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2003.a,
	title = {A Review of Surveys on Software Effort Estimation},
	journal = {IEEE International Symposium on Empirical Software Engineering (ISESE 2003)},
	year = {2003},
	note = {ISBN 0-7695-2002-2},
	month = {September 30 - October 1},
	pages = {223-230},
	publisher = {IEEE Computer Society},
	type = {1},
	address = {Rome, Italy},
	abstract = {This paper tries to summarize estimation knowledge through a review of surveys on software effort estimation. Main findings were that: (1) Most projects (60-80\%) encounter effort and/or schedule overruns. The overruns, however, seem to be lower than the overruns reported by some consultancy companies. For example, Standish Group{\textquoteright}s {{\textquoteleft}}Chaos Report{\textquoteright} describes an average cost overrun of 89\%, which is much higher than the average overruns found in other surveys, i.e., 30-40\%. (2) The estimation methods in most frequent use are expert judgment-based. A possible reason for the frequent use of expert judgment is that there is no evidence that formal estimation models lead to more accurate estimates. (3) There is a lack of surveys including extensive analyses of the reasons for effort and schedule overruns.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2003.a,
	title = {Situational and Task Characteristics Systematically Associated With Accuracy of Software Development Effort Estimates},
	journal = {Information Resources Management Association Conference},
	year = {2003},
	month = {May},
	pages = {824-826},
	publisher = {-},
	type = {1},
	address = {Philadelphia, USA},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {SE.4.Joergensen.2003.a,
	title = {Software Effort Estimation by Analogy and Regression Toward the Mean},
	journal = {Journal of Systems and Software},
	volume = {68},
	number = {3},
	year = {2003},
	pages = {253-262},
	abstract = {Estimation by analogy is, simplified, the process of finding one or more projects that are similar to the one to be estimated and then derive the estimate from the values of these projects. If the selected projects have an unusual high or low productivity, then we should adjust the estimates toward productivity values of more average projects. The size of the adjustments depends on the expected accuracy of the estimation model. This paper evaluates one adjustment approach, based on the findings made by Sir Francis Galton in the late 1800s regarding the statistical phenomenon {\textquotedblleft}regression toward the mean{\textquotedblright} (RTM). We evaluate this approach on several data sets and find indications that it improves the estimation accuracy. Surprisingly, current analogy based effort estimation models do not, as far as we know, include adjustments related to extreme analogues and inaccurate estimation models. An analysis of several industrial software development and maintenance projects indicates that the effort estimates provided by software professionals, i.e., expert estimates, to some extent are RTM-adjusted. A student experiment confirms this finding, but also indicates a rather large variance in how well the need for RTM-adjustments is understood among software developers.},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Ulf G. {Indahl}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2003,
	title = {Software Effort Estimation: Unstructured Group Discussion As a Method to Reduce Individual Biases},
	journal = {The 15th Annual Workshop of the Psychology of Programming Interest Group (PPIG 2003)},
	year = {2003},
	month = {8-10 April},
	pages = {285-296},
	publisher = {-},
	type = {1},
	address = {Keele University, UK},
	abstract = {The effort of software projects is often estimated, completely or partially, using expert judgement. This estimation process is subject to biases of the expert responsible. Generally, this bias seems to be towards too optimistic estimates regarding the effort needed to complete the project. The degree of bias varies depending on the expert involved, and seems to be connected to both conscious and unconscious decisions. One possible way to reduce this bias towards over-optimism is to combine the judgments of several experts. This paper describes an experiment where experts with different backgrounds combined their estimates through group discussion. Twenty software professionals were asked to provide individual effort estimates of a software development project. Subsequently, they formed five estimation groups, each consisting of four experts. Each of these groups agreed on a project effort estimate through discussion and combination of knowledge. We found that the groups were less optimistic in their estimates than the individual experts. Interestingly, the group discussion-based estimates were closer to the effort used by the actual project than the average individual expert, i.e., the group discussions led to better estimates than a mechanical combination of the individual estimates. The groups{\textquoteright} ability to identify more project activities is among the possible explanations for this reduction of bias.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@techreport {Karahasanovic.2003.3,
	title = {A Think-Aloud Support Tool for Collecting Feedback in Large-Scale Software Engineering Experiments},
	number = {2003-7},
	year = {2003},
	publisher = {Simula Research Laboratory},
	abstract = {The think-aloud method has proven useful for studying how a technology is used. However, it is labour-intensive and makes demands on resources for studies with many subjects. This paper describes a tool, called the think-aloud tool, that we have developed to support the collection of feedback from subjects in large-scale software engineering experiments. This tool instructs the subjects, at regular intervals, to write down their thoughts on a web-based screen and collects their feedback. An empirical evaluation of this tool is reported. The results from four controlled experiments suggest that the think-aloud tool is a valuable means for increasing the quality of large-scale software engineering experiments. The results also indicate that the participants were mostly positive to the tool and had no significant problems in verbalising their thoughts. The tool we proposed adds to an available set of tools for data collection in software engineering experiments. Whereas we have experience with the think-aloud tool in software engineering experiments, there is also potential for using such a tool in other areas in which the think-aloud method is used in studies involving a computer system.},
	author = {Karahasanovic, Amela and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Siw Elisabeth {Hove} and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Bratthall.2002,
	title = {Can You Trust a Single Data-Source Exploratory Software Engineering Case-Study},
	journal = {Empirical Software Engineering},
	volume = {7},
	year = {2002},
	pages = {9-26},
	author = {Bratthall, Lars and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2002.a,
	title = {Combination of Software Development Effort Prediction Intervals: Why, When and How?},
	journal = {Fourteenth IEEE Conference on Software Engineering and Knowledge Engineering (SEKE{\textquoteright}02)},
	year = {2002},
	month = {July 15-19},
	pages = {425-428},
	publisher = {-},
	type = {1},
	address = {Ischia, Italy},
	abstract = {The uncertainty of a software development effort estimate may be described through a prediction interval, e.g., that the most likely use of effort is 1.500 work-hours and that it is 90 \% probable (90\% confidence level) that the actual use of effort will be between 1.000 (minimum) and 2.000 (maximum) work-hours. Previous studies suggest that software development effort prediction intervals are, on average, much too narrow to reflect high confidence levels, i.e., the uncertainty is under-estimated. This paper analyses when and how a combination of several individual prediction intervals of the same task improves the correspondence between hit rate and confidence level of effort prediction intervals. We analyse three combination strategies: (1) Average of the individual minimum and maximum values, (2) Maximum and minimum of the individual maximum and minimum values, and (3) Group process (discussion) based prediction intervals. Based on an empirical study with software professionals we found that strategy (1) did not lead to much correspondence improvement compared with the individual prediction intervals, mainly because of a, as expected, strong individual bias towards too narrow prediction intervals. Strategy (2) and (3) both improved the correspondence. However, Strategy (3) used the uncertainty information more efficiently, i.e., had narrower prediction intervals for the same hit rate. Our empirical results suggest that group discussion based combination of prediction intervals should be used instead of {\`{\"\i}}mechanical{\^{\"\i}} combinations of individual prediction intervals. Clearly, there is no best combination strategy for all prediction interval situations, and the choice of strategy should be based on an investigation of factors that impact the usefulness of a strategy.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Joergensen.2002.c,
	title = {Combining Quantitative Software Development Cost Estimation Precision Data With Qualitative Data From Project Experience Reports at Ericsson Design Center in Norway},
	journal = {Proceedings of the Conference on Empirical Assesssments of Software Engineering},
	year = {2002},
	publisher = {-},
	type = {1},
	address = {Keele, UK},
	author = {J{\o}rgensen, Magne and L{\o}vstad, N. and Moen, L.}
}
@article {SE.4.Joergensen.2002.b,
	title = {Comments On: a Simulation Tool for Efficient Analogy Based Cost Estimation},
	journal = {Empirical Software Engineering},
	volume = {7},
	year = {2002},
	pages = {375-376},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Sjoeberg.2002,
	title = {Conducting Realistic Experiments in Software Engineering},
	journal = {ISESE{\textquoteright}2002 (First International Symposium on Empirical Software Engineering)},
	year = {2002},
	month = {October 3-4},
	pages = {17-26},
	publisher = {IEEE Computer Society},
	type = {1},
	abstract = {An important goal of most empirical software engineering research is the transfer of research results to industrial applications. Two important obstacles for this transfer are the lack of control of variables of case studies, i.e., the lack of explanatory power, and the lack of realism of controlled experiments. While it may be difficult to increase the explanatory power of case studies, there is a large potential for increasing the realism of controlled software engineering experiments. To convince industry about the validity and applicability of the experimental results, the tasks, subjects and the environments of the experiments should be as realistic as practically possible. Such experiments are, however, more expensive than experiments involving students, small tasks and pen-and-paper environments. Consequently, a change towards more realistic experiments requires a change in the amount of resources spent on software engineering experiments. This paper argues that software engineering researchers should apply for resources enabling expensive and realistic software engineering experiments similar to how other researchers apply for resources for expensive software and hardware that are necessary for their research. The paper describes some of the experiences from recent experiments that varied in size from involving one software professional for 5 days to 130 software professionals, from 9 consultancy companies, for one day each.},
	isbn = {0-7695-1796-X},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Arisholm, Erik and Dyb{\r a}, Tore and J{\o}rgensen, Magne and Karahasanovic, Amela and Koren, E. and Vok{\'a}c, Marek},
	editor = {found, not}
}
@article {SE.4.Joergensen.2002.a,
	title = {Impact of Experience on Maintenance Skills},
	journal = {Journal of Software Maintenance and Evolution: Research and Practice},
	volume = {14},
	number = {2},
	year = {2002},
	month = {March},
	pages = {123-146},
	abstract = {This study reports results from an empirical study of 54 software maintainers in the software maintenance department of a Norwegian company. The study addresses the relationship between amount of experience and maintenance skills. The findings were, amongst others: (1) While there may have been a reduction in the frequency of major unexpected problems from tasks solved by very inexperienced to medium experienced maintainers, additional years of general software maintenance experience did not lead to further reduction. More application specific experience, however, further reduced the frequency of major unexpected problems. (2) The most experienced maintainers did not predict maintenance problems better than maintainers with little or medium experience. (3) A simple one-variable model outperformed the maintainers{\textquoteright} predictions of maintenance problems, i.e., the average prediction performance of the maintainers seems poor. An important reason for the weak correlation between length of experience and ability to predict maintenance problems may be the lack of meaningful feedback on the predictions.},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Teigen.2002,
	title = {Probability Intervals and Interval Probabilities Are Not the Same},
	journal = {Annual Meeting Society for Judgment and Decision Making (Poster)},
	year = {2002},
	month = {22-25 November},
	publisher = {-},
	type = {1},
	address = {Kansas, USA},
	author = {Teigen, Karl-Halvor and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2002.d,
	title = {A Simple Effort Prediction Interval Approach},
	journal = {Proceedings of the Conference on Achieving Quality in Information Systems},
	year = {2002},
	month = {13.-15. March},
	pages = {19-30},
	type = {1},
	address = {Venezia},
	isbn = {0000-0000},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Joergensen.2002.b,
	title = {Uncertainty Intervals Versus Interval Uncertainty: an Alternative Method for Eliciting Effort Prediction Intervals in Software Development Projects},
	journal = {Proceedings of International Conference on Project Management (ProMAC)},
	year = {2002},
	pages = {343-352},
	publisher = {-},
	type = {1},
	address = {Singapore},
	abstract = {Frequently, there is a poor correspondence between the judged and the actual uncertainty of effort usage in software development projects. This may to some extent be a consequence of the uncertainty elicitation process. Traditionally, software developers are asked to provide the minimum and maximum effort of development work for a given confidence level, e.g., minimum and maximum effort that includes the actual effort usage with a 90\% probability. An alternative uncertainty elicitation process is to instruct the software developers to provide the uncertainty of a given effort interval, e.g., the probability that the actual effort is between 50\% and 200\% of the estimated most likely effort. In an empirical investigation, this alternative process led to significant improvement of prediction interval accuracy. The observed improvement using this alternative elicitation process can, we believe, be explained through a simplified interpretation of historical prediction accuracy data, less \&ldquo;conflicting estimation goal\&rdquo;, and less influence from the \&ldquo;anchoring effect\&rdquo;.},
	author = {J{\o}rgensen, Magne and Teigen, Karl-Halvor}
}
@inproceedings {SE.5.Joergensen.2001.a,
	title = {Anchoring Effects: an Important Cause of Too Optimistic Effort Estimates in Software Development Projects},
	journal = {24nd IRIS Conference (Information Systems Research Seminar In Scandinavia)},
	year = {2001},
	month = {August},
	type = {Conference},
	address = {Ulvik, Norway},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Arisholm.2001,
	title = {Assessing the Changeability of Two Object-Oriented Design Alternatives - a Controlled Experiment},
	journal = {Empirical Software Engineering},
	volume = {6},
	number = {3},
	year = {2001},
	pages = {231-277},
	author = {Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Bratthall.2001.b,
	title = {Can You Trust a Single Data-Source Exploratory Software Engineering Case Study},
	journal = {EASE 2001},
	year = {2001},
	month = {April},
	publisher = {-},
	type = {1},
	address = {Keele, England},
	author = {Bratthall, Lars and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Sjoeberg.2001,
	title = {Conducting Experiments on Software Evolution},
	journal = {4th International Workshop on Principles of Software Evolution (IWPSE 2001)},
	year = {2001},
	month = {September},
	pages = {142-145},
	publisher = {-},
	type = {Workshop},
	address = {Vienna, Austria},
	isbn = {1-58113-508-4},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Arisholm, Erik and J{\o}rgensen, Magne},
	editor = {Tamai, Tetsuo and Aoyama, Mikio and Bennet, Keith}
}
@inproceedings {SE.5.Karahasanovic.2001.c,
	title = {Data Collection in Software Engineering Experiments},
	journal = {Managing Information Technology in a Global Economy, Information Resources Management Association International Conference IRMA 2001, Software Engineering Track},
	year = {2001},
	month = {May 20 - 23},
	pages = {1027-1028},
	publisher = {Idea Group Publishing},
	type = {Conference},
	address = {Toronto, Ontario Canada},
	abstract = {This paper presents ongoing work on developing a mechanism for automatic data collection during software engineering experiments. The importance of experiments in software engineering can hardly be overemphasised. Conducting such experiments raises the challenge of efficient and reliable data collection and analysis. During the experiment subjects (students, experienced programmers, etc.) perform tasks on the software engineering artefacts (e.g., design documents and code) using the technology under study in the given context (e.g. operating system). We are typically interested in data concerning subjects, their interaction with the technology and context, and changes of the software engineering artefacts. To conduct a series of experiments on software engineering technology (methods and tools), we envisage an experimental environment for data collection and analysis. As a first step, we have developed a logging mechanism that collects data about subjects of the experiment and their usage of the technology under study. We believe that our logging mechanism will alleviate data collection and increase the validity of experiments.},
	author = {Karahasanovic, Amela and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Anda.2001.a,
	title = {Estimating Software Development Effort Based on Use Cases - Experiences From Industry},
	journal = {4th International Conference on the Unified Modeling Language (UML2001)},
	year = {2001},
	month = {October},
	pages = {487-502},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Toronto, Canada},
	abstract = {Use case models are used in object-oriented analysis for capturing and describing the functional requirements of a system. Several methods for estimating software development effort are based on attributes of a use case model. This paper reports the results of three industrial case studies on the application of a method for effort estimation based on use case points. The aim of this paper is to provide guidance for other organizations that want to improve their estimation process applying use cases. Our results support existing claims that use cases can be used successfully in estimating software development effort. The results indicate that the guidance provided by the use case points method can support expert knowledge in the estimation process. Our experience is also that the design of the use case models has a strong impact on the estimates.},
	isbn = {3-540-42667-1},
	author = {Anda, Bente Cecilie Dahlum and Dreiem, Hege and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne},
	editor = {Gogolla, M. and Kobryn, C.}
}
@article {SE.4.Joergensen.2001.a,
	title = {Impact of Effort Estimates on Software Project Work},
	journal = {Information and Software Technology},
	volume = {43},
	number = {15},
	year = {2001},
	pages = {939-948},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Bratthall.2001.a,
	title = {Program Understanding Behaviour During Estimation of Enhancement on Small Java Programs},
	journal = {3rd International Conference on Product Focused Software Process Improvement (PROFES 2001)},
	year = {2001},
	month = {September},
	publisher = {-},
	type = {1},
	address = {Kaiserslautern, Germany},
	author = {Bratthall, Lars and Arisholm, Erik and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Anda.2001.b,
	title = {Quality and Understandability of Use Case Models},
	journal = {ECOOP 2001 - Object-Oriented Programming, 15th European Conference},
	year = {2001},
	month = {June 18-22},
	pages = {402-428},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Budapest, Hungary},
	abstract = {Use case models are used in object-oriented analysis for capturing and describing the functional requirements of a system. Use case models are also used in communication between stakeholders in development projects. It is therefore important that the use case models are constructed in such a way that they support the development process and promote a good understanding of the requirements among the stakeholders. Despite this, there are few guidelines on how to construct use case models. This paper describes an explorative study where three different sets of guidelines were used for constructing and documenting use case models. An experiment with 139 undergraduate students divided into 31 groups was conducted. Each group used one out of the three sets of guidelines when constructing a use case model from an informal requirements specification. After completing the use case model, each student answered a questionnaire. The results of the experiment indicate that guidelines based on templates support the construction of use case models that are easier to understand for the readers, than guidelines without specific details on how to document each use case. The guidelines based on templates were also considered as the most useful when constructing use cases. In addition to better understandability, our experiment indicates that the guidelines based on templates result in better use case models regarding also other quality attributes. Our results further indicate that it may be beneficial to combine the template guidelines with another set of guidelines that focus on the documentation of the flow of events of each use case. Keywords. Object-oriented analysis, Requirements specification, Use Cases, UML, Understandability, Experiment},
	isbn = {3-540-42206-4},
	author = {Anda, Bente Cecilie Dahlum and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne},
	editor = {Knudsen, Lindskov}
}
@inproceedings {SE.5.Joergensen.2001.b,
	title = {Software Effort Estimation by Analogy and Regression Toward the Mean},
	journal = {The Thirteenth International Conference on Software Engineering \& Knowledge Engineering (SEKE01)},
	year = {2001},
	month = {June 13-15},
	pages = {253 - 262},
	publisher = {Elsevier Science Inc},
	type = {Conference},
	address = {Buenos Aires, Argentina},
	abstract = {Estimation by analogy is, simplified, the process of finding one or more projects that are similar to the one to be estimated and then derive the estimate from the values of these projects. If the selected projects have an unusual high or low productivity, then we should adjust the estimates toward productivity values of more average projects. The size of the adjustments depends on the expected accuracy of the estimation model. This paper evaluates one adjustment approach, based on the findings made by Sir Francis Galton in the late 1800s regarding the statistical phenomenon \&ldquo;regression toward the mean\&rdquo; (RTM). We evaluate this approach on several data sets and find indications that it improves the estimation accuracy. Surprisingly, current analogy based effort estimation models do not, as far as we know, include adjustments related to extreme analogues and inaccurate estimation models. An analysis of several industrial software development and maintenance projects indicates that the effort estimates provided by software professionals, i.e., expert estimates, to some extent are RTM-adjusted. A student experiment confirms this finding, but also indicates a rather large variance in how well the need for RTM-adjustments is understood among software developers.},
	author = {J{\o}rgensen, Magne and Ulf G. {Indahl} and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Joergensen.2001.b,
	title = {Software Process Improvement and Human Judgement Heuristics},
	journal = {Scandinavian Journal of Information Systems},
	volume = {13},
	number = {1},
	year = {2001},
	pages = {99-122},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Joergensen.2000.c,
	title = {Empirical Studies on Effort Estimation in Software Development Projects},
	journal = {IRMA 2000},
	year = {2000},
	month = {May},
	pages = {778-779},
	publisher = {IGI Publishing},
	type = {Conference},
	address = {Alaska,},
	isbn = {1-878-28984-5},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Joergensen.2000.b,
	title = {Human Judgement in Effort Estimation of Software Projects},
	journal = {Beg, Borrow, or Steal Workshop, International Conference on Software Engineering},
	year = {2000},
	month = {June},
	pages = {45-51},
	type = {Workshop},
	address = {Limerick, Ireland},
	author = {J{\o}rgensen, Magne and Kirkeb{\o}en, Geir and Sj{\o}berg, Dag Ingar Kondrup and Anda, Bente Cecilie Dahlum and Bratthall, Lars}
}
@inproceedings {SE.5.Joergensen.2000.a,
	title = {The Importance of NOT Learning From Experience},
	journal = {EuroSPI{\textquoteright}2000 -European Software Process Improvement},
	year = {2000},
	month = {November 7-9,},
	pages = {2.2-2.8},
	type = {Workshop},
	address = {Copenhagen, Denmark},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {SE.5.Joergensen.2000.d,
	title = {The Prediction Ability of Experienced Software Maintainers},
	journal = {4th Europan Conference on Software Maintenance and Reengineering},
	year = {2000},
	pages = {93-100},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Zurich, Switzerland},
	abstract = {Reports an empirical study of 109 randomly selected maintenance tasks in a large Norwegian software organization. When the maintainers had understood the maintenance task specifications, we asked whether they knew how to solve the task. A high confidence in knowing how to solve the task meant that the maintainers did not expect any major difficulties. Then, immediately after the task was completed, we asked whether there had been any major unexpected difficulties. A comparison of the answers gave the seemingly surprising result that one could not, except for corrective, small and simple maintenance tasks, have more confidence in the predictions of an experienced maintainer than the predictions of an inexperienced maintainer. We believe that better quality of the feedback on previous predictions and more training in probabilistic thinking are important means to improve the prediction abilities of maintainers. Decision aids, such as maintenance effort estimation models, should enable the analysis of previous predictions and stimulate probabilistic thinking},
	isbn = {0-7695-0546-5},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Kirkeb{\o}en, Geir},
	editor = {Ebert, Jurgen and Verhoef, Chris}
}
@techreport {SE.7.Dybaa.2000,
	title = {SPIQ Metodeh{\r a}ndbok},
	number = {282},
	year = {2000},
	note = {ISBN 82-7368-230-7, ISSN 0806-3036},
	publisher = {Department of Informatics, University of Oslo},
	author = {Dyb{\r a}, Tore and K. J. {Wedde} and St{\r a}lhane, T. and N. B. {Moe} and Conradi, R. and Dings{\o}yr, T. and Sj{\o}berg, Dag Ingar Kondrup and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Anda.2000,
	title = {Understanding Use Case Models},
	journal = {Beg, Borrow, or Steal Workshop, International Conference on Software Engineering},
	year = {2000},
	month = {June 5,},
	pages = {94-102},
	publisher = {ICSE Workshop},
	type = {Workshop},
	address = {Limerick, Ireland},
	abstract = {Use Case Modeling is a technique for handling the functional requirements in a software development project. The Use Case Model can serve as a means of communication between the different stakeholders in a project. It is used in planning the project and is updated and used during the project. In order to reduce the possibilities for misunderstandings and differences in understanding, it would be useful to be able to evaluate to what extent the different stakeholders have understood the model and also to detect differences in interpretation. Low comprehension or differences in interpretation may indicate a need for more effort on specifying the requirements. If this is not feasible, it may be necessary to assume a higher risk when planning and estimating the project. We propose using knowledge on how humans understand text from cognitive psychology in the design of an experiment with a twofold goal: Investigate methods for measuring comprehension of Use Case Models and analyze the differences in understanding.},
	author = {Anda, Bente Cecilie Dahlum and J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.1999.c,
	title = {An Empirical Study of the Correlation Between Development Efficiency and Software Development Tools},
	journal = {Telektronikk},
	volume = {95},
	number = {1},
	year = {1999},
	pages = {54-62,},
	author = {J{\o}rgensen, Magne and Bygd{\r a}s, S.}
}
@inproceedings {SE.5.Arisholm.1999.c,
	title = {Guidelines on Conducting Software Process Improvement Studies in Industry},
	journal = {22nd IRIS Conference (Information Systems Research Seminar In Scandinavia)},
	year = {1999},
	month = {August 7-10,},
	pages = {87-102},
	type = {Conference},
	address = {Keuruu, Finland},
	author = {Arisholm, Erik and Anda, Bente Cecilie Dahlum and J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {SE.4.Joergensen.1999.a,
	title = {Reuse of Software Development Experience - a Case Study},
	journal = {Telektronikk},
	volume = {95},
	number = {1},
	year = {1999},
	pages = {48-54,},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Conradi, R.}
}
@article {SE.4.Joergensen.1999.b,
	title = {Software Quality According to Measurement Theory},
	journal = {Telektronikk},
	volume = {95},
	number = {1},
	year = {1999},
	pages = {12-17,},
	author = {J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.1999.d,
	title = {Software Quality Measurement},
	journal = {Advances in Engineering Software},
	volume = {30},
	number = {12},
	year = {1999},
	pages = {907-912,},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.1998,
	title = {Reuse of Software Development Experience at Telenor Telecom Software},
	journal = {European Software Process Improvement Conference (EuroSPI{\textquoteright}98)},
	year = {1998},
	month = {November 16-18,},
	pages = {10.19-10.31},
	type = {Conference},
	address = {Gothenburg, Sweden},
	author = {J{\o}rgensen, Magne and Sj{\o}berg, Dag Ingar Kondrup and Conradi, R.}
}
@inproceedings {SE.5.Joergensen.1997,
	title = {Measurement of Software Quality},
	journal = {First International Conference Software Quality Engineering},
	year = {1997},
	pages = {257-266},
	type = {1},
	address = {Udine, Italy,},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Sjoeberg.1996.a,
	title = {Evaluating Software Maintenance Technology},
	journal = {Norwegian Conference in Informatics},
	year = {1996},
	month = {November 18-20,},
	pages = {49-61},
	publisher = {TAPIR},
	type = {Conference},
	address = {Alta, Norway},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Welland, Ray and M. P. {Atkinson} and J{\o}rgensen, Magne and John Petter {Martinussen} and Maus, Arne}
}
@techreport {SE.7.Joergensen.1995,
	title = {Efficiency Evaluation of CASE Tools - Method and Results},
	year = {1995},
	note = {Research Report 38/95},
	publisher = {Norwegian Telecom},
	author = {J{\o}rgensen, Magne and Bygd{\r a}s, S. S. and Lunde, T.}
}
@inproceedings {SE.5.Joergensen.1995.a,
	title = {Empirical Evaluation of CASE Tool Efficiency},
	journal = {Sixth International Conference On applications of Software Measurement},
	year = {1995},
	pages = {207-230},
	type = {1},
	address = {Orlando,},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.1995.b,
	title = {Empirical Evaluation of CASE Tools},
	journal = {Norwegian Informatics Conference},
	year = {1995},
	pages = {71-82,},
	type = {1},
	author = {J{\o}rgensen, Magne and Bygd{\r a}s, S. S.}
}
@article {SE.4.Joergensen.1995.a,
	title = {An Empirical Study of Software Maintenance Tasks},
	journal = {Journal of Software Maintenance and Evolution: Research and Practice},
	volume = {7},
	year = {1995},
	pages = {27-48,},
	author = {J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.1995.c,
	title = {Experience With the Accuracy of Software Maintenance Task Effort Prediction Models},
	journal = {IEEE Transactions on Software Engineering},
	volume = {21},
	number = {8},
	year = {1995},
	pages = {674-681,},
	author = {J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.1995.b,
	title = {The Quality of Questionnaire Based Software Maintenance Studies},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {20},
	number = {1},
	year = {1995},
	pages = {71-73,},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.1994.a,
	title = {A Comparative Study of Software Maintenance Effort Prediction Models},
	journal = {5th Australasian Conference on Information Systems},
	year = {1994},
	type = {1},
	address = {Melbourne, Australia,},
	author = {J{\o}rgensen, Magne}
}
@phdthesis {SE.3.Joergensen.1994,
	title = {Empirical Studies of Software Maintenance},
	year = {1994},
	note = {Research Report 1880},
	school = {University of Oslo},
	type = {phd},
	author = {J{\o}rgensen, Magne}
}
@article {SE.5.Joergensen.1994.b,
	title = {Prediction of Software Maintenance Effort},
	journal = {Norwegian Informatics Conference},
	year = {1994},
	pages = {81-92,},
	author = {J{\o}rgensen, Magne}
}
@misc {SE.7.Joergensen.1994,
	title = {Systemutvikling Og Vitenskapsteori},
	howpublished = {Norwegian Telecom Research Note, 18/94, (in Norwegian)},
	year = {1994},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.1993,
	title = {A Case Study of Software Maintenance Tasks},
	journal = {Norwegian Informatics Conference},
	year = {1993},
	pages = {101-112,},
	type = {1},
	author = {J{\o}rgensen, Magne and Maus, Arne}
}
@article {SE.4.Bygdaas.1993,
	title = {Software Development Methods and Life Cycle Models},
	journal = {Technical Journal of Norwegian Telecom},
	volume = {2/3},
	year = {1993},
	pages = {44-51,},
	author = {Bygd{\r a}s, S. S. and J{\o}rgensen, Magne}
}
@article {SE.4.Joergensen.1993,
	title = {Software Maintainability - on Purpose Or Accidentally},
	journal = {Technical Journal of Norwegian Telecom},
	volume = {1},
	year = {1993},
	pages = {150-156,},
	author = {J{\o}rgensen, Magne}
}
@inproceedings {23445,
	title = {The use of precision of software development effort estimates to communicate uncertainty},
	journal = { },
	abstract = {The precision of estimates may be applied to communicate the uncertainty of required software development effort. The effort estimates 1000 and 975 work-hours, for example, communicate different levels of expected estimation accuracy. Through observational and experimental studies we found that software professionals (i) sometimes, but not in the majority of the examined projects, used estimate precision to convey effort uncertainty, (ii) tended to interpret overly precise, inaccurate effort estimates as indicating low developer competence and low trustworthiness of the estimates, while too narrow effort prediction intervals had the opposite effect. This difference remained even when the actual effort was known to be outside the effort prediction interval. We identified several challenges related to the use of the precision of single value estimates to communicate effort uncertainty and recommend that software professionals use effort prediction intervals, and not the preciseness of single value estimates, to communicate effort uncertainty. They should also avoid presenting overly precise single value effort estimates.},
	author = {J{\o}rgensen, Magne}
}
