@article {Simula.SE.150,
	title = {Using Planning Poker for Combining Expert Estimates in Software Projects},
	journal = {Journal of Systems and Software},
	volume = {81},
	number = {12},
	year = {2008},
	pages = {2106-2117},
	abstract = {Combination of expert opinion is frequently used to produce estimates in software projects. However, whether, when, and how to combine expert estimates is poorly understood. In order to study the effects of a combination technique called planning poker, the technique was introduced in a software project for half of the tasks. The tasks estimated with planning poker provided: 1) group consensus estimates that were less optimistic than the statistical combination (mean) of individual estimates for the same tasks, and 2) group consensus estimates that were more accurate than the statistical combination of individual estimates for the same tasks. The set of control tasks in the same project, estimated by individual experts, achieved similar estimation accuracy as the planning poker tasks. Moreover, for both planning poker and the control group, measures of the median estimation bias indicated that both groups had unbiased estimates, because the typical estimated task was perfectly on target. A code analysis revealed that for tasks estimated with planning poker, more effort was expended due to the complexity of the changes to be made, possibly triggered by the group discussions.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and Nils Christian {Haugen} and Hans Christian {Benestad}}
}
@inproceedings {Molokken-ostvold.2007.1,
	title = {Combining Estimates With Planning Poker - an Empirical Study},
	journal = {18th Australian Conference on Software Engineering},
	year = {2007},
	month = {April},
	pages = {349-358},
	publisher = {IEEE},
	type = {Conference},
	abstract = {Combination of expert opinion is frequently used to produce estimates in software projects. However, if, when and how to combine expert estimates, is poorly understood. In order to study the effects of a combination technique called planning poker, the technique was introduced in a software project for half of the tasks. The tasks estimated with planning poker provided: 1) group consensus estimates that were less optimistic than the mechanical combination of individual estimates for the same tasks, and 2) group consensus estimates that were more accurate than the mechanical combination of individual estimates for the same tasks. The set of control tasks in the same project, estimated by individual experts, achieved similar estimation accuracy as the planning poker tasks. However, for both planning poker and the control group, the median estimation bias indicated that both groups had unbiased estimates, as the typical estimated task was perfectly on target.},
	isbn = {0-7695-2778-7},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and Nils Christian {Haugen}},
	editor = {Grant, Doug}
}
@misc {Simula.SE.177,
	title = {Hvordan Hyppige Leveranser Kan Hjelpe Deg I Ukjente Prosjektomgivelser - Erfaringer Fra En Doktorgrad Levert Under Budsjett Og F{\o}r Deadline},
	howpublished = {Smidig 2007},
	year = {2007},
	month = {November},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Simula.SE.176,
	title = {Lightning-Talk: Estimation},
	howpublished = {XP-meetup},
	year = {2007},
	month = {March},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Simula.SE.14,
	title = {The Relationship Between Customer Collaboration and Software Project Overruns},
	journal = {Agile 2007},
	year = {2007},
	month = {August},
	pages = {72-83},
	publisher = {IEEE},
	type = {Conference},
	abstract = {Most agile projects rely heavily on good collaboration with the customer in order to achieve project goals and avoid overruns. However, the role of the customer in software projects is not fully understood. Often, successful projects are attributed to developer competence, while unsuccessful projects are attributed to customer incompetence. A study was conducted on eighteen of the latest projects of a software contractor. Quantitative project data was collected, and project managers interviewed, on several issues related to estimates, key project properties, and project outcome. It was found that in projects where collaboration was facilitated by daily communication between the contractor and the customer, they experienced a lesser magnitude of effort overruns. In addition, employing a contract that facilitates risk-sharing may also have a positive impact.},
	isbn = {0-7695-2872-4},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and Kristian Marius {Furulund}}
}
@inproceedings {Simula.SE.117,
	title = {Using Project Experience to Increase Software Estimation Accuracy - an Empirical Study},
	journal = {The Seventh International Conference on Quality Software - QSIC 2007},
	year = {2007},
	publisher = {IEEE Computer Society Press},
	type = {Conference},
	abstract = {The use of experience data is frequently suggested as a mean to increase software effort estimation accuracy. However, there has been done limited empirical research on the subject. To investigate the effect of using experience data, we conducted a study on eighteen of the latest projects of a software contractor. Quantitative project data was collected, and project managers interviewed, on several issues related to estimates, key project properties, and project outcome. It was found that in projects where experience data was utilized in the estimation process, they experienced a lesser magnitude of effort overruns. The use of a checklist also appeared to increase estimation accuracy. However, the utilization of an estimation model in the estimation process appeared not to have any impact on the estimation accuracy.},
	isbn = {ISBN},
	author = {Kristian Marius {Furulund} and Kjetil Johan {Mol{\o}kken-{\O}stvold}},
	editor = {Ramamoorthy, C.V.}
}
@misc {Molokken-ostvold.2006.3,
	title = {Combining Expert Estimates},
	howpublished = {Presentation during MeLLow Workshop, Brunel University, London.},
	year = {2006},
	month = {October},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Jorgensen.2006.12,
	title = {How Large Are Software Cost Overruns? a Review of the 1994 CHAOS Report},
	howpublished = {Article in Software Practitioner (Vol 16, no 4\&5, p 13-14)},
	year = {2006},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Jorgensen.2006.4,
	title = {How Large Are Software Cost Overruns? Critical Comments on the Standish Group{\textquoteright}s CHAOS Reports},
	journal = {Information and Software Technology},
	volume = {48},
	number = {4},
	year = {2006},
	pages = {297-301},
	abstract = {The Standish Group reported in 1994 that the average cost overrun of software projects was as high as 189\%. This cost overrun number is used as input in recent governmental reports on software development and as benchmark for several recent projects{\textquoteright} estimation performances. It is therefore important that we can trust this number. More recent cost overrun results reported by the Standish Group and others, however, show much lower cost overrun. Does this mean that software companies have improved their estimation ability strongly the last 10 years? In this paper we argue that there are reasons to doubt the validity of the Standish Group{\textquoteright}s 1994 cost overrun results and that a continued use of these results may hinder progress.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2006.2,
	title = {Know When to Hold {\textquoteright}em, Know When to Fold {\textquoteright}em - Combining Estimates With Planning Poker},
	howpublished = {Presentation at Simula{\textquoteright}s annual estimation seminar},
	year = {2006},
	month = {November},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and Nils Christian {Haugen}}
}
@misc {Molokken-ostvold.2006.5,
	title = {Kombinasjon Av Estimater - En Enkel Og Praktisk Metode for {\r a} {\o}ke Presisjonen},
	howpublished = {Presentation for IKT-Norge/REFLEKS-project},
	year = {2006},
	month = {December},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2006.1,
	title = {Overskridelser Og Skandaler I Offentlige IT-Prosjekter - Hvorfor Vi Heller Ikke Kommer Til {\r a} L{\ae}re Av Problemene Til Golf, MinSide Og Flexus.},
	howpublished = {Presentation for Abelia{\textquoteright}s 5th anniversary seminar},
	year = {2006},
	note = {Se also published newsarticle in dagensit.no (http://www.dagensit.no/bedrifts-it/article803978.ece)},
	month = {June},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Haugen.2006.1,
	title = {Planning Poker - Playing for Better Estimates},
	howpublished = {Presentation at JavaZone},
	year = {2006},
	month = {September},
	author = {Nils Christian {Haugen} and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Grimstad.2006.1,
	title = {Software Effort Estimation Terminology: the Tower of Babel},
	journal = {Information and Software Technology},
	volume = {48},
	number = {4},
	year = {2006},
	pages = {302-310},
	abstract = {This paper provides a review of how software development effort estimation terms are used in software engineering textbooks and research papers. We found that the term {{\textquoteleft}}effort estimate{\textquoteright} frequently is applied without any clarification of its meaning. It is therefore difficult to determine whether the authors{\textquoteright} intended interpretation is an estimate of {{\textquoteleft}}most likely effort{\textquoteright}, {{\textquoteleft}}planned effort{\textquoteright}, {{\textquoteleft}}budgeted effort{\textquoteright}, or something else. This is problematic as these terms are not equivalent and are used for different purposes. The lack of clarity of {{\textquoteleft}}effort estimate{\textquoteright} lowers the quality and interpretability of surveys on software effort estimation accuracy, i.e., it is not clear what the estimation accuracy results really mean. This reduces the estimation evaluation and learning possibilities. We suggest guidelines on how to reduce this terminology ambiguity. To the authors{\textquoteright} knowledge, this is the first published review of software cost estimation terminology.},
	author = {Grimstad, Stein and J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Jorgensen.2006.11,
	title = {Wie Gross Sind Die Kosten{\"u}berschreitungen Tats{\"a}chlich},
	howpublished = {Article in OBJEKTspektrum (in German)},
	year = {2006},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.10,
	title = {Bedre Estimater I IT-Prosjekter. N{\o}kkelen Ligger I Grenselandet Mellom Kunde Og Leverand{\o}r},
	howpublished = {Hovedinnledning p{\r a} Abelias seminar om endringsvennlig prosjektstyring},
	year = {2005},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Grimstad.2005.2,
	title = {The Clients{\textquoteright} Impact on Effort Estimation Accuracy in Software Development Projects},
	journal = {11th IEEE International Software Metrics Symposium (METRICS 2005), Como, Italy, September 19-22},
	year = {2005},
	pages = {3},
	publisher = {IEEE},
	type = {1},
	abstract = {It seems clear that there is no simple solution to improved predictability of software development projects. Over several decades various aspects of software development and their relationship to estimation accuracy have been investigated. This paper focus on one such relationship; the clients{\textquoteright} impact on estimation accuracy in software development projects. Factors contributing to overruns as well as factors preventing overruns are investigated. Based on a literature review and a survey of 300 software professionals we find that: 1) It is software professionals{\textquoteright} perception that clients impact estimation accuracy. Changed and new requirements are perceived as the clients{\textquoteright} most frequent contribution to overruns, while overruns are prevented by the availability of competent clients and capable decision makers. 2) Survey results should not be used in estimation accuracy improvement initiatives without further analysis. Surveys typically identify direct and project specific causes for overruns, while substantial improvement is only possible when the underlying causes are understood. Keywords: software estimation, survey, client-vendor relationship, overrun causes},
	keywords = {client-vendor relationship, overrun causes, software estimation, survey},
	author = {Grimstad, Stein and J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Molokken-ostvold.2005.5,
	title = {A Comparison of Software Project Overruns - Flexible Vs. Sequential Development Models},
	journal = {IEEE Transactions on Software Engineering},
	volume = {31},
	number = {9},
	year = {2005},
	note = {Project questions are dowloadable},
	pages = {754-766},
	abstract = {Flexible software development models, e.g., evolutionary and incremental models, have become increasingly popular. Advocates of these models claim that among the benefits of these models are reduced software project overruns, which is one of the main challenges of software project management. This paper describes an in-depth survey of software development projects. The results support the claim that projects which employ a flexible development model experience less effort overruns than do those who employ a sequential model. The reason for the difference is not obvious. We found, for example, no variation in project size, estimation process, or delivered proportion of planned functionality between projects applying different types of development model. When the managers were asked to provide reasons for software overruns and/or estimation accuracy, the largest difference were that more of flexible projects cited good requirement specifications and good collaboration/communication with clients as contributing to accurate estimates.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@inproceedings {Molokken-ostvold.2005.2,
	title = {Ethical Concerns When Increasing Realism in Controlled Experiments With Industrial Participants},
	journal = {38th Annual Hawaii International Conference on System Sciences (HICSS{\textquoteright}05), January 3-6, Hawaii, USA},
	year = {2005},
	month = {January},
	publisher = {Computer Society Press},
	type = {1},
	abstract = {The emerging interest for realistic controlled experiments in computer science has created a need for focus on related research ethics. Increased realism and scale in experimental studies pose new challenges which have not been debated to a sufficient extent. Specifically, there can be conflicts between the ethical principles of scientific value and informed consent. This paper provides an account of related previous work in computer science research ethics. To illustrate, two large scale software engineering experiments with industrial participants are described. Challenges and solutions in these experiments are discussed in light of current ethical guidelines. Interviews and debriefing sessions with industrial participants from these, and other, experiments are also provided. These reveal that there not necessarily will be ethical problems with increased realism, given that the researchers respect the principles of informed consent, benefice and confidentiality.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {Molokken-ostvold.2005.4,
	title = {Expert Estimation of the Effort of Web-Development Projects: Are Software Professionals in Technical Roles More Optimistic Than Those in Non-Technical Roles?},
	journal = {Empirical Software Engineering},
	volume = {10},
	number = {1},
	year = {2005},
	pages = {7-30},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@misc {Jorgensen.2005.20,
	title = {Hvorfor Er Vi Overoptimistiske Igjen Og Igjen Og Igjen ...},
	howpublished = {Presentation IT-Fornebu halvtimen},
	year = {2005},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.3,
	title = {Kan {\textquotedblright}moderne{\textquotedblright} Utviklingsmetodikker Redusere Overskridelser I IT-Prosjekter?},
	howpublished = {Presentation for Abelia seminar on agile development},
	year = {2005},
	month = {April},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.7,
	title = {Lynkurs},
	howpublished = {Lynkurs i estimering for Sterias kunder},
	year = {2005},
	note = {Lynkurs for Sterias kunder},
	month = {January},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.11,
	title = {Lynkurs II},
	howpublished = {Lynkurs i estimering for Sterias kunder},
	year = {2005},
	note = {Lynkurs for Steria-kunder},
	month = {February},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {Molokken-ostvold.2005.1,
	title = {Management of Public Software Projects: Avoiding Overruns},
	journal = {Hawaiian International Conference on Business, May 25-28, Hawaii, USA},
	year = {2005},
	month = {May},
	publisher = {Hawaiian International Conference on Business},
	type = {1},
	abstract = {Avoiding overruns is one of the major challenges in public software projects. In order to reduce overruns and, in turn, increase the chances of project success, it is essential to have an unbiased overview of the frequency and magnitude of overruns, as well as knowledge about factors which may prevent overruns. In a recent survey of the software industry it was found that the average magnitude of overruns in work-hours was 67\% for projects with a public client, compared to 21\% for projects with a private client. This paper presents results from two surveys of software companies and professionals that investigate the possible differences between public and private clients, and a review of research on public IT-projects. The results indicate that there are several properties of public projects on the political, organizational and individual level which may contribute to the large overruns. These include artificial deadlines based on political terms, regulations on procurement and development processes, and lack of technology and project management skills. The challenges appear to be common for several countries in the OECD region which have investigated this topic. In order to reduce overruns of public projects, both contractors and government officials have to address these issues.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and S{\o}rgaard, P{\r a}l and Grimstad, Stein}
}
@misc {Molokken-ostvold.2005.12,
	title = {Nyskapning Basert P{\r a} Forskning Innenfor Kostnadsberegning Og Gjennomf{\o}ring Av IT-Prosjekter},
	howpublished = {Presentation for Stortingets N{\ae}ringskomit{\'e}},
	year = {2005},
	month = {February},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.9,
	title = {Research on Software Estimation - Have We Made Progress the Last 30 Years?},
	howpublished = {Invited talk at Brunel University},
	year = {2005},
	month = {October},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {Molokken-ostvold.2005.8,
	title = {Software Estimation},
	howpublished = {Presentation for Norgesgruppen},
	year = {2005},
	month = {February},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and Hans Enger {Gallis} and Grimstad, Stein}
}
@misc {SE.8.Moloekken-Oestvold.2004.a,
	title = {Best-Pro Estimeringsseminar},
	howpublished = {Presentasjon for ledere og prosjektledere i norske IT-bedrifter},
	year = {2004},
	note = {60-70 deltakere. Arrang{\o}r: Simula},
	month = {3. mars},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {SE.4.Joergensen.2004.a,
	title = {Better Sure Than Safe? Overconfidence in Judgment Based Software Development Effort Prediction Intervals},
	journal = {Journal of Systems and Software},
	volume = {70},
	number = {1-2},
	year = {2004},
	pages = {79-93},
	abstract = {The uncertainty of a software development effort estimate can be indicated through a prediction interval, i.e., the estimated minimum and maximum effort corresponding to a specific confidence level. For example, a project manager may be {\textquotedblleft}90\% confident{\textquotedblright} or believe that is it {\textquotedblleft}very likely{\textquotedblright} that the effort required to complete a project will be between 8,000 and 12,000 work-hours. This paper describes results from four studies (Studies A-D) on human judgement (expert) based prediction intervals of software development effort. Study A examines the accuracy of the prediction intervals in real software projects. The results suggest that the prediction intervals were generally much too narrow to reflect the chosen level of confidence, i.e., that there was a strong over-confidence. Studies B, C and D try to understand the reasons for the observed over-confidence. Study B examines the possibility that the over-confidence is related to type of experience or estimation process. Study C examines the possibility that the concept of confidence level is difficult to interpret for software estimators. Finally, Study D examines the possibility that there are unfortunate feedback mechanisms that reward over-confidence.},
	author = {J{\o}rgensen, Magne and Teigen, Karl-Halvor and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2004.b,
	title = {Does Use of Development Model Affect Estimation Accuracy and Bias?},
	journal = {Product Focused Software Process Improvement: 5th International Conference (PROFES 2004)},
	year = {2004},
	note = {ISBN: 3-540-21421-6},
	month = {April 5-8},
	pages = {17-29},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Kansai Science City, Japan},
	abstract = {Objective. To investigate how the use of incremental and evolutionary development models affects the accuracy and bias of effort and schedule estimates of software projects. Rationale. Advocates of incremental and evolutionary development models often claim that use of these models results in improved estimation accuracy. Design of study. We conducted an in-depth survey, where information was collected through structured interviews with 22 software project managers in 10 different companies. We collected and analyzed information about estimation approach, effort estimation accuracy and bias, schedule estimation accuracy and bias, completeness of delivered functionality and other estimation related information. Results. We found no impact from the development model on the estimation approach. However, we found that incremental and evolutionary projects were less prone to effort overruns. The degree of delivered functionality and schedule estimation accuracy, on the other hand, were seemingly independent of development model. Conclusion. The use of incremental and evolutionary development models may reduce the chance of effort overruns.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and A. C. {Lien} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and S. E. {Hove}}
}
@misc {SE.8.Moloekken-Oestvold.2004.f,
	title = {Effin-Seminar},
	howpublished = {Prosjektledere og softwareutviklere for offentlige IT-prosjekter, spesialister innen Human Computer Interaction, brukerrepresentanter og forskere},
	year = {2004},
	note = {25-30 deltakere. Arrang{\o}r: SINTEF},
	month = {12.-13. mai},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@phdthesis {SE.3.Moloekken-Oestvold.2004,
	title = {Effort and Schedule Estimation of Software Development Projects},
	year = {2004},
	school = {University of Oslo},
	type = {phd},
	abstract = {This thesis explores different topics related to software project effort and schedule estimation. It consists of three sections which address 1) state of practice in the software industry, 2) different methods for improving estimation accuracy, and 3) general methodological aspects in research on software estimation. Related to international estimation practices and performance, a review of previous research performed as part of this thesis revealed that a majority of software projects (60-80\%) had encountered effort overruns. The average magnitude of these overruns was 30-40\%. The frequency (65-80\%) and magnitude (20-25\%) of schedule overruns was similar. The dominating estimation approach was expert judgement. Similar, a survey of software projects in Norway undertaken as part of this thesis found a frequency of 76\% effort overruns, with an average magnitude of 41\%. The frequency and magnitude of schedule overruns was 62\% and 25\% respectively. Expert estimation was by far the preferred estimation method. It was also observed that the type of customer had an impact on the magnitude of effort overruns. Public projects had an average effort overrun of 67\%, as opposed to the 21\% average in private projects. This observed difference appears to be caused by systematic differences between private and public organizations found at 1) the political level, 2) the organizational level, and 3) and the individual level. In an experiment on expert judgment, the results indicated that professionals in technical roles (project managers and developers) were significantly more (over)optimistic than professionals in non-technical roles (sales managers and user analysts) when estimating project effort. Regarding improvement of estimation practices, two different approaches were investigated. A controlled experiment showed that combination of expert estimates through unstructured group discussion can reduce the existing bias towards too optimistic estimates. In addition, it appears as if use of flexible development models (e.g. incremental, iterative or agile) leads to a lesser magnitude of effort overruns when compared to the traditional sequential waterfall model. It was also found that results in some previous studies on estimation accuracy may have been impacted by methodological shortcomings, and that there is little attention directed at ethical aspects in software engineering research in general. The main research contributions presented in this thesis indicate that most software projects face schedule and effort overruns. In addition, the frequency and magnitude of software effort and schedule overruns are significant, and appear to be independent of location and stable over time. Most practitioners favour expert judgement over formal estimation models. In order to improve estimation accuracy in ways that are easy and cost-efficient to implement, professional software organizations may rely more on flexible development methods, and combine estimates from different expert through group interaction.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.b,
	title = {E-Kontaktgruppen},
	howpublished = {Foredrag for e-kontaktgruppen, med representanter for 6 forskjellige departementer},
	year = {2004},
	note = {12-15 deltakere. Arrang{\o}r: e-kontaktgruppen},
	month = {10. mars},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Joergensen.2004.b,
	title = {Eliminating Over-Confidence in Software Development Effort Estimates},
	journal = {Conference on Product Focused Software Process Improvement},
	year = {2004},
	month = {April},
	pages = {174-184},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Japan},
	abstract = {Minimum-maximum effort intervals are applied in the planning of software development projects in order to, among other things, determine the contingency buffer. Several studies suggest that judgment-based minimummaximum intervals are based on a systematic over-confidence in the accuracy of the effort estimates. In this paper, we investigate whether the possession by estimators of information about previous estimation error for similar projects reduces this over-confidence. Nineteen realistically composed estimation teams provided minimum-maximum effort intervals for the same software project. Ten of the teams (Group A) received no instructions about the uncertainty assessment process. The remaining nine teams (Group B) were instructed to begin the minimum-maximum effort interval assessment by recalling the distribution of estimation error for similar projects. We found that the recall of the error distribution of the Group B teams did have an impact, but mainly on the assessment of the estimated minimum effort, not on the maximum effort. We discuss reasons for this finding and provide recommendations.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.d,
	title = {Genovus},
	howpublished = {Presentasjon for Generas kunder},
	year = {2004},
	note = {50-60 deltakere. Arrang{\o}r: Genera},
	month = {26.-29. mars},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@article {SE.4.Moloekken-Oestvold.2004.b,
	title = {Group Processes in Software Effort Estimation},
	journal = {Empirical Software Engineering},
	volume = {9},
	number = {4},
	year = {2004},
	pages = {315-334},
	abstract = {The effort required to complete software projects is often estimated, completely or partially, using the judgement of experts, whose assessment may be biased. In general, such bias as there is seems to be towards estimates that are overly optimistic. The degree of bias varies from expert to expert, and seems to depend on both conscious and unconscious processes. One possible approach to reduce this bias towards over-optimism is to combine the judgments of several experts. This paper describes an experiment in which experts with different backgrounds combined their estimates in group discussion. First, twenty software professionals were asked to provide individual estimates of the effort required for a software development project. Subsequently, they formed five estimation groups, each consisting of four experts. Each of these groups agreed on a project effort estimate via the pooling of knowledge in discussion. We found that the groups submitted less optimistic estimates than the individuals. Interestingly, the group discussion-based estimates were closer to the effort expended on the actual project than the average of the individual expert estimates were, i.e., the group discussions led to better estimates than a mechanical averaging of the individual estimates. The groups{\textquoteright} ability to identify a greater number of the activities required by the project is among the possible explanations for this reduction of bias.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@misc {SE.8.Moloekken-Oestvold.2004.i,
	title = {IKT-2010},
	howpublished = {Presentasjon for deltakere p{\r a} IKT-2010},
	year = {2004},
	note = {70-80 deltakere. Arrang{\o}r: NFR},
	month = {20.-21. september},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.k,
	title = {LCSC Kick-Off},
	howpublished = {Presentasjon p{\r a} LCSC kick-off},
	year = {2004},
	note = {15 deltakere. Arrang{\o}r: DNV},
	month = {28. oktober},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.j,
	title = {Offentlig Fare},
	howpublished = {Newspaper article in Computerworld},
	year = {2004},
	month = {24. september},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and S{\o}rgaard, P{\r a}l}
}
@techreport {SE.7.Moloekken-Oestvold.2004,
	title = {Project Estimation in the Norwegian Software Industry - a Summary},
	number = {2004-03},
	year = {2004},
	note = {Results from the BEST-Pro (Better Estimation of Software Tasks and Process Improvement) survey},
	publisher = {Simula Research Laboratory},
	abstract = {This report provides an overview of the results obtained from a survey on project estimation in Norwegian software companies. The survey was conducted between February and November 2003. The main results are: {\textbullet} We observed that 76\% of the projects used more effort than estimated, while 19\% used less. The average effort overrun was 41\%. {\textbullet} Average effort overrun was 67\% in projects with a public client, compared to an average effort overrun of 21\% for projects with a private client. {\textbullet} Projects that used an incremental or evolutionary development approach had an average effort overrun of 24\%, as opposed to the average 55\% overrun for projects that used a waterfall-based development approach. The frequency and magnitude of effort overruns found in this survey seems to be similar to results reported from surveys conducted in other countries in the past 20 years. The observed differences in effort overruns between private and public projects may be caused by differences in bidding procedures, level of client involvement or acceptance procedures. In order to reduce the risk of effort overruns, software companies should: {\textbullet} Focus on analyzing their own estimation performance, and invest in estimation improvement (for instance through experience databases or work breakdown structures). {\textbullet} Differentiate risk buffers based on the type of the customer, development approach and the size of the project. {\textbullet} Try to establish an {\textquotedblleft}as close as possible{\textquotedblright} dialogue with the customers (e.g. through an incremental development approach). All companies should analyse completed projects, in order to benchmark their performance. This allows for improvement efforts to be identified.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and A. C. {Lien} and S. E. {Hove}}
}
@article {SE.4.Joergensen.2004.h,
	title = {Reasons for Software Effort Estimation Error: Impact of Respondents Role, Information Collection Approach, and Data Analysis Method},
	journal = {IEEE Transactions on Software Engineering},
	volume = {30},
	number = {12},
	year = {2004},
	pages = {993-1007},
	abstract = {This study is a first step towards better processes of understanding why errors occur in software effort estimation. Within one software development company, we collected information about estimation errors through: (1) Interviews with estimation responsible employees in different roles, (2) Estimation experience reports from 68 completed projects, and, (3) Statistical analysis of relations between characteristics of the 68 completed projects and estimation error. We found that the role of the respondents, the data collection approach, and the type of analysis had an important impact on the reasons for estimation error that were given. We found, for example, a strong tendency to perceive factors outside the respondents{\textquoteright} own control as important reasons for inaccurate estimates. Reasons given for accurate estimates, on the other hand, typically cited factors that were within the respondents{\textquoteright} own control, and were determined by the estimators{\textquoteright} skill or experience. This bias in types of reason means that the collection only of project managers{\textquoteright} viewpoints will not yield balanced models of reasons for estimation error. Unfortunately, previous studies on reasons for estimation error have tended to collect information from project managers only. We recommend that software companies combine estimation error information from in-depth interviews with stakeholders in all relevant roles, estimation experience reports, and results from statistical analyses of project characteristics.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.e,
	title = {Riksrevisjonen},
	howpublished = {Presentasjon av v{\r a}r forskning for Riksrevisjonen},
	year = {2004},
	note = {12-15 deltakere. Arrang{\o}r: Riksrevisjonen},
	month = {28. april},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.h,
	title = {SPIKE-Seminar},
	howpublished = {Presentasjon for forskere og bedrifter tilknyttet SPIKE},
	year = {2004},
	note = {30-40 deltakere. Arrang{\o}r: Simula},
	month = {7.-8. september},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.c,
	title = {Steria Lederseminar},
	howpublished = {Seminar for Sterias internasjonale ledergruppe},
	year = {2004},
	note = {12-15 deltakere. Arrang{\o}r: Steria},
	month = {24. mars},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@misc {SE.8.Moloekken-Oestvold.2004.g,
	title = {Steria Lynkurs},
	howpublished = {Seminar for Sterias kunder},
	year = {2004},
	note = {80-90 deltakere. Arrang{\o}r: Steria},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2004.a,
	title = {A Survey on Software Estimation in the Norwegian Industry},
	journal = {10th International Software Metrics Symposium (METRICS 2004)},
	year = {2004},
	month = {September 16-18},
	pages = {208-219},
	publisher = {IEEE Computer Society Press},
	type = {1},
	address = {Chicago, USA},
	abstract = {This paper seeks to provide an overview of the estimation methods that software companies apply to estimate their projects, why those methods are chosen, and how accurate they are. In order to improve software estimation accuracy, such knowledge is essential. We conducted an in-depth survey, where information was collected through structured interviews with senior managers from 18 different companies and project managers of 52 different projects. We collected and analyzed information about estimation approach, effort estimation accuracy and bias, schedule estimation accuracy and bias, completeness of delivered functionality and other estimation related information. Our results suggest, for example, that average effort overruns are 41\%, that the software estimation performance has not changed much the last 10-20 years, that expert estimation is the dominating estimation method, that estimation accuracy is not much impacted by use of formal estimation models, and that software managers tend to believe that the estimation accuracy of their company is better than it actually is.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne and Sinan Sigurd {Tanilkan} and Gallis, H. and A. C. {Lien} and S. E. {Hove}}
}
@inproceedings {SE.5.Joergensen.2003,
	title = {A Preliminary Checklist for Software Cost Management},
	journal = {IEEE International Conference on Quality Software},
	year = {2003},
	month = {November},
	pages = {134-140},
	publisher = {-},
	type = {1},
	address = {Dallas, USA},
	abstract = {This paper presents a process framework and a preliminary checklist for software cost management. While most textbooks and research papers on cost estimation look mainly at the {\textquotedblleft}Estimation{\textquotedblright} phase, our framework and checklist includes the phases relevant to estimation: {\textquotedblleft}Preparation{\textquotedblright}, {\textquotedblleft}Estimation{\textquotedblright}, {\textquotedblleft}Application{\textquotedblright}, and {\textquotedblleft}Learning{\textquotedblright}. We believe that cost estimation processes and checklists should support these phases to enable high estimation accuracy. The checklist we suggest is based on checklists from a number of sources, e.g., a handbook in forecasting and checklists present in several Norwegian software companies. It needs, however, to be extended through feedback from other researchers and software practitioners. There is also a need for a provision of conditions for meaningful use of the checklist issues and descriptions of the strength and sources of evidence in favor of the checklist issues. The present version of the checklist should therefore be seen as preliminary and we want to get feedback from the conference participants and other readers of this paper for further improvements.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2003.a,
	title = {A Review of Surveys on Software Effort Estimation},
	journal = {IEEE International Symposium on Empirical Software Engineering (ISESE 2003)},
	year = {2003},
	note = {ISBN 0-7695-2002-2},
	month = {September 30 - October 1},
	pages = {223-230},
	publisher = {IEEE Computer Society},
	type = {1},
	address = {Rome, Italy},
	abstract = {This paper tries to summarize estimation knowledge through a review of surveys on software effort estimation. Main findings were that: (1) Most projects (60-80\%) encounter effort and/or schedule overruns. The overruns, however, seem to be lower than the overruns reported by some consultancy companies. For example, Standish Group{\textquoteright}s {{\textquoteleft}}Chaos Report{\textquoteright} describes an average cost overrun of 89\%, which is much higher than the average overruns found in other surveys, i.e., 30-40\%. (2) The estimation methods in most frequent use are expert judgment-based. A possible reason for the frequent use of expert judgment is that there is no evidence that formal estimation models lead to more accurate estimates. (3) There is a lack of surveys including extensive analyses of the reasons for effort and schedule overruns.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2003.a,
	title = {Situational and Task Characteristics Systematically Associated With Accuracy of Software Development Effort Estimates},
	journal = {Information Resources Management Association Conference},
	year = {2003},
	month = {May},
	pages = {824-826},
	publisher = {-},
	type = {1},
	address = {Philadelphia, USA},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2003.b,
	title = {Software Effort Estimation: Planning XP Guidelines Compared to Research on Traditional Software Development},
	journal = {Presentation at the Ph.D. Symposium in Fourth International Conference on eXtreme Programming and Agile Processes in Software Engineering (XP 2003)},
	year = {2003},
	month = {May 25-29},
	pages = {441-442},
	publisher = {Springer-Verlag},
	type = {1},
	address = {Genova, Italy},
	abstract = {In the {\textquotedblleft}Planning Extreme Programming{\textquotedblright} framework by Beck and Fowler, they propose several guidelines for project estimation. This paper focuses on the guideline to use estimation teams. There exist several methods from software engineering research on how to do this, with varying degree of formalism involved. We present an account of earlier research on how to combine expert estimates, and compare it with the XP guidelines. It seems that the proposed framework, with some modifications, is a reasonable way to improve accuracy.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
@inproceedings {SE.5.Moloekken-Oestvold.2003,
	title = {Software Effort Estimation: Unstructured Group Discussion As a Method to Reduce Individual Biases},
	journal = {The 15th Annual Workshop of the Psychology of Programming Interest Group (PPIG 2003)},
	year = {2003},
	month = {8-10 April},
	pages = {285-296},
	publisher = {-},
	type = {1},
	address = {Keele University, UK},
	abstract = {The effort of software projects is often estimated, completely or partially, using expert judgement. This estimation process is subject to biases of the expert responsible. Generally, this bias seems to be towards too optimistic estimates regarding the effort needed to complete the project. The degree of bias varies depending on the expert involved, and seems to be connected to both conscious and unconscious decisions. One possible way to reduce this bias towards over-optimism is to combine the judgments of several experts. This paper describes an experiment where experts with different backgrounds combined their estimates through group discussion. Twenty software professionals were asked to provide individual effort estimates of a software development project. Subsequently, they formed five estimation groups, each consisting of four experts. Each of these groups agreed on a project effort estimate through discussion and combination of knowledge. We found that the groups were less optimistic in their estimates than the individual experts. Interestingly, the group discussion-based estimates were closer to the effort used by the actual project than the average individual expert, i.e., the group discussions led to better estimates than a mechanical combination of the individual estimates. The groups{\textquoteright} ability to identify more project activities is among the possible explanations for this reduction of bias.},
	author = {Kjetil Johan {Mol{\o}kken-{\O}stvold} and J{\o}rgensen, Magne}
}
@inproceedings {SE.5.Joergensen.2002.a,
	title = {Combination of Software Development Effort Prediction Intervals: Why, When and How?},
	journal = {Fourteenth IEEE Conference on Software Engineering and Knowledge Engineering (SEKE{\textquoteright}02)},
	year = {2002},
	month = {July 15-19},
	pages = {425-428},
	publisher = {-},
	type = {1},
	address = {Ischia, Italy},
	abstract = {The uncertainty of a software development effort estimate may be described through a prediction interval, e.g., that the most likely use of effort is 1.500 work-hours and that it is 90 \% probable (90\% confidence level) that the actual use of effort will be between 1.000 (minimum) and 2.000 (maximum) work-hours. Previous studies suggest that software development effort prediction intervals are, on average, much too narrow to reflect high confidence levels, i.e., the uncertainty is under-estimated. This paper analyses when and how a combination of several individual prediction intervals of the same task improves the correspondence between hit rate and confidence level of effort prediction intervals. We analyse three combination strategies: (1) Average of the individual minimum and maximum values, (2) Maximum and minimum of the individual maximum and minimum values, and (3) Group process (discussion) based prediction intervals. Based on an empirical study with software professionals we found that strategy (1) did not lead to much correspondence improvement compared with the individual prediction intervals, mainly because of a, as expected, strong individual bias towards too narrow prediction intervals. Strategy (2) and (3) both improved the correspondence. However, Strategy (3) used the uncertainty information more efficiently, i.e., had narrower prediction intervals for the same hit rate. Our empirical results suggest that group discussion based combination of prediction intervals should be used instead of {\`{\"\i}}mechanical{\^{\"\i}} combinations of individual prediction intervals. Clearly, there is no best combination strategy for all prediction interval situations, and the choice of strategy should be based on an investigation of factors that impact the usefulness of a strategy.},
	author = {J{\o}rgensen, Magne and Kjetil Johan {Mol{\o}kken-{\O}stvold}}
}
