@misc {43251,
	title = {The Center for Effective Digitalization of the Public Sector (EDOS)},
	howpublished = {SimulaMet},
	year = {2023},
	type = {Presentation for Board},
	author = {Jo Erskine {Hannay}}
}
@article {43184,
	title = {A logic-based event controller for means-end reasoning in simulation environments},
	journal = {SIMULATION},
	volume = {61},
	year = {2023},
	month = {03/2023},
	publisher = {SAGE journals},
	abstract = {Simulation games are designed to cultivate expertise and rehearse particular skill sets. To yield longitudinal effects, sequences of events must be crafted to yield intended learning outcomes, sometimes by focusing on particularly difficult situations and replaying variants. The present paper develops a logic-based approach for encoding the interrelation between action, events, and objects in a manner that allows the resulting scenario description to immediately be executed in a game development environment. This has the dual effect of decoupling the description of a scenario from the simulation platform itself, as well as supporting iterative and flexible development of learning content. To this end, we provide three interrelated components: First, we develop a scenario description language based on Answer Set Programming. The language is designed to allow an automated reasoner to deduce a schedule of the future events that are caused by an action taken in a given simulation environment. Second, we define a protocol for exchanging actions and computed futures between, respectively, the simulation environment and the external automated reasoner. Finally, as a proof of concept, we develop an Application Programming Interface (API) for the Unity Real-Time Development Platform that implements the protocol and offers a software framework for connecting the computed future events to concrete game objects. This allows the game to evolve coherently from the specification. We argue that the resulting system inherits capabilities for artificial commonsense reasoning from its declarative basis which are useful for reasoning about an evolving emergency incident or training scenario.},
	issn = {0037-5497},
	doi = {10.1177/00375497231157384},
	url = {http://journals.sagepub.com/doi/10.1177/00375497231157384http://journals.sagepub.com/doi/pdf/10.1177/00375497231157384http://journals.sagepub.com/doi/pdf/10.1177/00375497231157384http://journals.sagepub.com/doi/full-xml/10.1177/00375497231157384},
	author = {Stolpe, Audun and Rummelhoff, Ivar and Jo Erskine {Hannay}}
}
@misc {43253,
	title = {Nyttestyring og kontinuerlig produktutvikling},
	howpublished = {Innovasjon Norge},
	year = {2023},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {43238,
	title = {An Overview of the Understanding and Use of Assessment Methods in Benefits Management},
	journal = {25th IEEE International Conference on Business Informatics (CBI 2023)},
	year = {2023},
	month = {06/2023},
	publisher = { IEEE},
	abstract = {Evidence suggests that benefits management is critical for enabling greater value and for enhancing the positive impact of programs, projects, and portfolios. However, many organizations only focus their efforts on identifying the intended benefits, without conducting a proper follow-up through ongoing assessment or evaluation to ensure benefits realization. For this reason, this study aims to obtain an overview of the actual use of the assessment methods in benefits management, through a compilation and analysis of the literature following the Systematic Mapping Study methodology. The results and findings obtained show a significant gap in the interplay between the fields of assessment methods and benefits management. Based on this, we suggest opportunities for improvement, such as developing audit guides and frameworks for enhancing the monitoring of the benefits, strengthening the focus on external benefits of organizations, and for adapting benefits assessment to the changed perceptions of intended benefits, in line with continuous improvement.},
	keywords = {Assessment, benefits management, Literature Analysis, Literature Review, Project Management},
	isbn = {979-8-3503-1515-8},
	doi = {https://dx.doi.org/10.1109/CBI58679.2023.10187441},
	author = {Jose David {Pat{\'o}n-Romero} and Jo Erskine {Hannay}}
}
@inproceedings {43165,
	title = {Projects VS Continuous Product Development - Does it Affect Benefits Realization?},
	journal = {The Ninth International Conference on Advances and Trends in Software Engineering (SOFTENG)},
	year = {2023},
	month = {04/2023},
	pages = {20-25},
	publisher = {Xpert Publishing Services (XPS)},
	abstract = {Software investments are traditionally implemented using project organization, which often leads project participants to focus on time, cost and scope, rather than the intended benefits of the investment. We conducted a survey to compare work organized as projects against work organized as Continuous Product Development (CPD). Our results indicate that: 1. Both project organization and CPD are commonly used in practice. 2. Agile is very popular, but DevOps and the use of linear models for organizing work are also frequent. 3. CPD is perceived to outperform projects in realization of benefits. 4. We found no difference in perceived realization of benefits between those using or not using a set of ways of organizing work (including linear models, agile, DevOps, BizDev or program organization). We conclude that organizing work using CPD is a viable alternative to project organization, especially in situations where failure must be avoided. Also, we suggest that more research should be conducted to better understand what factors of the different ways of organizing work affects the realization of benefits.},
	author = {Sinan Sigurd {Tanilkan} and Jo Erskine {Hannay}},
	editor = {Lavazza, Luigi}
}
@misc {43252,
	title = {Senter for Effektiv Digitalisering av Offentlig Sektor (EDOS)},
	howpublished = {Helse S{\o}r{\o}st},
	year = {2023},
	type = {Presentation for portfolio managers},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {43344,
	title = {A Survey on the Perception of Opportunities and Limitations of Generative AI in the Public Sector},
	journal = {2nd World Conference on Information Systems for Business Management (ISBM 2023)},
	year = {2023},
	month = {09/2023},
	publisher = { Springer},
	abstract = {Recent breakthroughs in Artificial Intelligence (AI) have increased the interest in applications of AI in general and Generative AI in particular. This interest is manifest in public debate on the promise and limitations of AI in various sectors, but also on the risks of relying upon such technology. However, little is known about the interest in and perceptions of Generative AI among Information Technology (IT) professionals likely to take responsibility for developing and implementing AI-based solutions in the public sector. We present findings from a survey we developed (with help from ChatGPT) and conducted among IT practitioners in the Norwegian public sector. Through this survey, we investigated how opportunities and limitations of the use Generative AI were perceived and understood. We found ambivalent attitudes: while the interest in Generative AI is high, the level of perceived knowledge is low; there is considerable optimism about the value of AI, but an equivalent level of concern about threats this technology represents; and although respondents suspected AI had helped generate the survey, they found the survey questions relevant and fully context-oriented. These findings provide background and context for further research into the adoption of AI and help practitioners establish a baseline for efforts to build and manage AI-based solutions in public institutions.},
	keywords = {artificial intelligence, ChatGPT, Digitalization, Generative AI, Public sector, survey},
	author = {Knutsen, Leif and Jose David {Pat{\'o}n-Romero} and Jo Erskine {Hannay} and Sinan Sigurd {Tanilkan}}
}
@inproceedings {43091,
	title = {A Survey on the Use and Effects of Goal Hierarchies in Digitalization Efforts},
	journal = {Portland International Conference on Management of Engineering and Technology: Managing Technology, Engineering and Manufacturing for a Sustainable World (PICMET 2023)},
	year = {2023},
	month = {07/2023},
	publisher = { IEEE},
	abstract = {Digitalization has become a primary goal for organizations. Successfully adopting the digital context both in daily operations and in business management and strategy entails great benefits at different levels (organizational,\&nbsp;economic, social, environmental...).\&nbsp;Thus, it is very important that practitioners have clear conceptions of the goals in this regard and that those goals are {\textquotedblleft}alive{\textquotedblright} in organizations.For this reason, in this study we present a survey that we performed among practitioners related to the management of Information Technology (IT) from both the private and public sectors in Norway. Through this survey we have tried to find out how organizations understand and translate the current context of digitalization from different goal levels. For that, we asked respondents to relate to one of three goal hierarchies: A) a classical governance approach; B) an organizational tier approach; and C) an effects-based approach.Among the results obtained we found that the first two are the most used and the goal achievement is slightly higher for the classical governance approach than for the organizational tier approach. Likewise, we identified that while top level management has a good understanding of the goals, this understanding deteriorates as one moves down the organizational hierarchy.},
	keywords = {Digitalization, information technology, Management, strategy, sustainability},
	author = {Sinan Sigurd {Tanilkan} and Knutsen, Leif and Jose David {Pat{\'o}n-Romero} and Jo Erskine {Hannay}}
}
@inproceedings {42770,
	title = {Benefit Considerations in Project Decisions},
	journal = {International Conference on Product-Focused Software Process Improvement. PROFES 2022},
	year = {2022},
	month = {11/2022},
	pages = {217-234},
	publisher = {Springer, Cham},
	edition = {23},
	abstract = {Software project success is often characterized in terms of time, cost and scope {\textendash} despite that delivering benefit is the main purpose of a project. In this paper, we explore 1) to what degree benefit considerations influence major project decisions, 2) to what degree a specific set of benefits management challenge are handled and influence major project decisions and 3) if there is any realization (over time) that benefit considerations should receive greater attention. We investigate influence in projects with four types of problem severity: completed projects with only minor problems, completed projects with major problems, projects that were disrupted but completed, and projects that were terminated before completion. We asked 45 software professionals to what degree time, cost, scope, benefit and benefit/cost, as well as benefits management challenges, influence major  roject decisions. Our findings indicate that time, cost and scope have a significantly higher degree of influence on project decisions than benefit and benefit/cost. However, practitioners think that benefit and benefit/cost should have significantly more influence on decisions than cost. The benefits management challenges are found to have less influence in the more severe projects. We argue that giving benefits considerations a stronger voice in project decisions would be in line with the desire of practitioners and the prime objective of delivering benefit to stakeholders. We conclude that it is important to understand how to handle benefits management challenges at different stages of project life and that handling such challenges should be integrated with other prime drivers of project success.},
	keywords = {Benefit, Benefits management challenges, Cost, Scope, Software project severity, Time},
	isbn = {978-3-031-21388-5},
	doi = {https://doi.org/10.1007/978-3-031-21388-5_15},
	author = {Sinan Sigurd {Tanilkan} and Jo Erskine {Hannay}},
	editor = {Taibi, Davide and Kuhrmann, Marco and Mikkonen, Tommi and Kl{\"u}nder, Jil and Abrahamsson, Pekka}
}
@inbook {42503,
	title = {Eliciting and Prioritizing Services for Accessible Information - for  Residential Real Estate Transactions},
	booktitle = {HCI for Health, Well-being, Universal Access and Healthy Aging},
	number = {LNCS 13521},
	year = {2022},
	pages = {378-395},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {A number of initiatives are underway for digitalizing real estate transaction processes. Public and private sector bodies are working to automate information retrieval and processing of the financial, ordinance\&nbsp; and\&nbsp; fiscal aspects of such transactions. Other initiatives, such as ours,\&nbsp; are targeted toward helping\&nbsp; stakeholders directly involved in selling and buying\&nbsp; real estate. We present the results from a set of\&nbsp; group sessions, where the focus was on improving the presentation of salient information to sellers and buyers of property. Based on an earlier conceptualization of perceived\&nbsp; information difficulties, we\&nbsp; \&nbsp;elicited user stories\&nbsp; for facilitating a better generation, provision and consumption of relevant information for the\&nbsp; residential real estate transaction process. A total of ten services were aggregated from the user stories. We then asked a set of stakeholders to rate the effect of the services on functional objectives; i.e., on how they will affect the transaction process.\&nbsp; We asked stakeholders at the managerial level to rate the functional objectives on strategic objectives. Combining the two sets of ratings, one obtains a rating of perceived benefit for the services, which can help in prioritzing which services to start developing first. In the outset, real estate transactions involve stakeholders with opposing interests. We conclude that multi-stakeholder group sessions can help generate services that serve these conflicting interests on a common ground.\&nbsp;},
	doi = {10.1007/978-3-031-17902-0_27},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-17902-0_27},
	author = {Jo Erskine {Hannay} and Bjarte M. {{\O}stvold} and Kristin Skeide {Fuglerud}},
	editor = {Vincent G. {Duffy} and Gao, Qin and Zhou, Jia and Antona, Margherita and Stephanidis, Constantine}
}
@inproceedings {42484,
	title = {Perceived Challenges in Benefits Management - A Study of Public Sector Information Systems Engineering Projects},
	journal = {Conference on Business Informatics (CBI)},
	year = {2022},
	month = {06/2022},
	pages = {156-165},
	publisher = {IEEE Computer Society Digital Library},
	edition = {24},
	abstract = {The field of benefits management gives guidelines on how to plan and realize benefits\&nbsp; throughout the life-cycle of a system. However, realizing benefits from information systems projects has proven to be challenging in practice. In this paper, we investigate specific benefits management challenges as perceived by practitioners involved in information systems engineering projects. We conducted 22 interviews with respondents representing nine public sector projects, where challenges in managing benefits were elicited and identified. We elicited six specific benefits management challenges: A - Identifying and describing benefits, B - Alignment of work with planned benefits, C - Reception and acceptance of the planned benefits, D - Organizational issues, E - Alternative or competing solutions, F - Measuring and evaluating benefits. Overlaying these challenges with current normative models on benefits management, we find\&nbsp; that: 1. Normative models on benefits management lack sufficient guidance on operative work on how to create information systems fit for realizing benefits and how to introduce these solutions to ensure benefits realization, and 2. Normative models on benefits management do not explicitly leverage the\&nbsp; rapid project learning promoted by modern engineering methods. We conclude that more specific benefits management models should be elaborated, which are integrated into modern information systems engineering practices. This will enable best practices on the continuous adjustments of cost and scope according to evolving knowledge in projects to also be adapted to the management of benefits.},
	keywords = {Benefits management challenges, Benefits management models, Information systems engineering, Public sector},
	doi = {10.1109/CBI54897.2022.00024},
	author = {Sinan Sigurd {Tanilkan} and Jo Erskine {Hannay}}
}
@article {doi:10.1177/00375497221095070,
	title = {Quantifying means-end reasoning skills in simulation-based training: a logic-based approach},
	journal = {SIMULATION},
	volume = {98},
	year = {2022},
	month = {05/2022},
	pages = {933-957},
	publisher = {SAGE journals},
	abstract = {We develop a logic-based approach for designing simulation-based training scenarios. Our methodology embodies a concise definition of the scenario concept and integrates the notions of training goals, acceptable versus unacceptable actions and performance scoring. The approach applies classical artificial intelligence (AI) planning to extract coherent plays from a causal description of the training domain. The domain- and task-specific parts are defined in a high-level action description language AL. Generic causal and temporal logic is added when the causal theory is compiled into the underlying Answer Set Programming (ASP) language. The ASP representation is used to derive a scoring function that reflects the quality of a play or training session, based on a distinction of states and actions into green (acceptable) and red (unacceptable) ones. To that end, we add to the casual theory a set of norms that specify an initial assignment of colors. The ASP engine uses these norms as axioms and propagates colors by consulting the causal theory. We prove that any set of such norms constitutes a conservative extension of the underlying causal theory. With this work, we hope to lay the foundation for the development of design and analysis tools for exercise managers. We envision a software system that lets an exercise manager view all plays of a tentative scenario design, with expediency information and scores for each possible play. Our approach is applicable to any domain in which means-ends reasoning is pertinent. We illustrate the approach in the domain of crisis response and management.},
	keywords = {Answer Set Programming, automated scoring, deontic logic, means-end reasoning, Simulation-based training},
	doi = {10.1177/00375497221095070},
	url = {https://doi.org/10.1177/00375497221095070},
	author = {Stolpe, Audun and Jo Erskine {Hannay}}
}
@inbook {42472,
	title = {Scenario Design for Healthcare Collaboration Training under Suboptimal Conditions},
	booktitle = {Digital Human Modeling and Applications in Health, Safety, Ergonomics and Risk Management. Health, Operations Management, and Design},
	volume = {19},
	number = {LNCS 13320},
	year = {2022},
	pages = {197-214},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {Health care today\&nbsp;usually consists of various services covering various parts of the\&nbsp;total health care of a region or country. These services are\&nbsp;required to coordinate and collaborate, often using procedures and IT collaboration\&nbsp;tools that may not be designed for interoperating across the\&nbsp;evolving wider landscape of health care services. We posit that it\&nbsp;is necessary to train personnel in collaboration skills using\&nbsp;whatever infrastructure is in place. To this end, we present design\&nbsp;principles for simulation-based collaboration training scenarios that emphasizes the inclusion of suboptimal infrastructure elements. We\&nbsp;applied the principles in a co-creational workshop with healthcare stakeholders\&nbsp;from a hospital and surrounding municipalities in Norway where we\&nbsp;discussed cases where collaboration training is perceived as\&nbsp;critical. We elicited five training vignettes concerning the general case of\&nbsp;\&nbsp; detecting, and following up on, clinical deterioration in a patient at home or in a nursing home. We found that the design principles spurred highly relevant discussions among participants and that novel ideas for collaboration training were brought forth on the basis of these principles. We conclude\&nbsp;that there is a potential in using these principles for eliciting training vignettes that address the actual situation more accurately.\&nbsp;},
	keywords = {Healthcare Collaboration, IT Services, Procedures, Scenario Design, Simulation-based training, Stakeholder Journey Analysis},
	author = {Jo Erskine {Hannay} and Kristin Skeide {Fuglerud} and Leister, Wolfgang and Schulz, Trenton},
	editor = {Duffy, Vincent}
}
@inbook {42470,
	title = {Stakeholder Perceptions on Requirements for Accessible Technical Condition Information in Residential Real Estate Transactions},
	booktitle = {Universal Access in Human-Computer Interaction. Novel Design Approaches and Technologies  },
	volume = {7},
	number = {LNCS 13308},
	year = {2022},
	pages = {242-259},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	chapter = {16},
	address = {Cham},
	abstract = {Buyers of residential real estate frequently experience dissatisfaction with the property they have purchased. Recent findings suggest that insufficient knowledge about the property is a key trigger to ensuing disappointment and claims for compensation. Further, a good technical condition report reduces the probability of dissatisfaction and insurance claims. For the purpose of designing services for improving technical condition information and its flow, we elicited stakeholder perceptions on the suitability of residential real estate technical condition reports.\&nbsp; Specifically, we conducted multiple surveys which we content analyzed and used as the basis for a conceptual model of information products and dependencies needed to deliver better information to stakeholders in a real estate transaction process. The conceptual model, in turn, forms the basis for specific service design in future work.},
	keywords = {Conflict Reduction, Information Services, Residential Real Estate Transactions, Technical Condition Information},
	isbn = {978-3-031-05027-5},
	author = {Jo Erskine {Hannay} and Kristin Skeide {Fuglerud} and Bjarte M. {{\O}stvold}},
	editor = {Antona, Margherita and Stephanidis, Constantine}
}
@inproceedings {Stolpe-Hannay-2021,
	title = {On the Adaptive Delegation and Sequencing of Actions},
	journal = {Proceedings of the 18th Int{\textquoteright}l Conf. Information on Information Systems for Crisis Response and Management (ISCRAM{\textendash}-AI and Intelligent Systems for Crises and Risks},
	year = {2021},
	month = {2021},
	publisher = {International Association for Information Systems for Crisis Response and Management},
	abstract = {Information systems support to crisis response and management relies on presenting actionable information in a manner that supports cognitive processes. We outline how AI Planning can be used viably to support the delegation and sequencing of tasks, using standard operating procedures as initial specifications of plans. When expressed in the AI planning language Answer set Programming (ASP), machine reasoning can be used in a pre-incident review to display the cooperative structure inherent in a plan. The purpose of this is to uncover weaknesses and to optimize the plan. Further, adaptive planning can be supported in during-incident reviews by updating the current status and recomputing the consequences. At this point, initial goals may no longer be viable and the explicit suggestion of prior sub-optimal goals now worth pursuing can be a game-changer under stress.},
	author = {Stolpe, Audun and Jo Erskine {Hannay}},
	editor = {Amanda Lee {Hughes} and McNeill, Fiona and Zobel, Christopher}
}
@book {27964,
	title = {Benefit/Cost-Driven Software Development - with Benefit Points and Size Points},
	year = {2021},
	publisher = {Springer},
	organization = {Springer},
	address = {Cambridge, UK},
	abstract = {This open access book presents a set of basic techniques for estimating the benefit of IT development projects and portfolios. It also offers methods for monitoring how much of that estimated benefit is being achieved during projects.Readers can then use these benefit estimates together with cost estimates to create a benefit/cost index to help them decide which functionalities to send into construction and in what order. This allows them to focus on constructing the functionality that offers the best value for money at an early stage.Although benefits management involves a wide range of activities in addition to estimation and monitoring, the techniques in this book provides a clear guide to achieving what has always been the goal of project and portfolio stakeholders: developing systems that produce as much usefulness and value as possible for the money invested. The techniques can also help deal with vicarious motives and obstacles that prevent this happening. The book equips readers to recognize when a project budget should not be spent in full and resources be allocated elsewhere in a portfolio instead. It also provides development managers and upper management with common ground as a basis for making informed decisions.},
	keywords = {benefit points, benefit/costs index, benefits management, earned business value management, periodization, uncertainty assessment},
	isbn = {978-3-030-74217-1},
	doi = {10.1007/978-3-030-74218-8},
	url = {https://link.springer.com/book/10.1007/978-3-030-74218-8$\#$toc},
	author = {Jo Erskine {Hannay}}
}
@article {Hannay-Gjorven-2021,
	title = {Leveraging Network-Centric Strategic Goals in Capabilities},
	journal = {Journal of Military Studies},
	volume = {10},
	number = {1},
	year = {2021},
	pages = {90-104},
	publisher = {De Gruyter},
	abstract = {The vision of network-centric operations is to increase operational capabilities through networked collaboration. NATO and its member nations state this vision in strategic documents at a very high level of abstraction. While suitable for giving an overall feel, current documentation renders the steps toward implementing those visions largely unsupported. We outline a method, based on agile requirements engineering, for converting high-level strategic visions into capabilities, whose form lend themselves to incremental implementation. We illustrate the use of this method in two cases that concern both operational capabilities and technical capabilities. We also\&nbsp; show how\&nbsp; the method enables one to prioritize\&nbsp; which capabilities to develop first. We conclude that some form of explicit methodology to span the gap between strategic visions and implementing those visions is necessary.},
	keywords = {benefits management, capabilities, requirements engineering, strategic goals},
	doi = {https://doi.org/10.2478/jms-2021-0001},
	url = {https://www.sciendo.com/article/10.2478/jms-2021-0001},
	author = {Jo Erskine {Hannay} and Gj{\o}rven, Eli}
}
@inbook {10.1007/978-3-030-90963-5_32,
	title = {Toward AI-Based Scenario Management for Cyber Range Training},
	booktitle = {HCI International 2021 - Late Breaking Papers: Multimodality, eXtended Reality, and Artificial Intelligence},
	number = {LNCS 13095},
	year = {2021},
	month = {11/2021},
	pages = {423{\textendash}436},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {There is an immediate need for a greater number of highly skilled cybersecurity personnel to meet intensified cyber attacks. We propose a cyber range exercise management architecture that employs machine reasoning to structure the design, execution and analysis of cyber range training scenarios. The scenarios are then used in simulation-based training in an emulated IT infrastructure environment. The machine reasoning is obtained by combining four AI methods: attack-defence trees, formal argumentation theory, answer set programming and multiagent systems. We argue that this type of advanced functionality that supports exercise managers in their design and analysis of scenarios is strictly necessary to improve current exercise management systems and build the required cybersecurity expertise.},
	isbn = {978-3-030-90963-5},
	author = {Jo Erskine {Hannay} and Stolpe, Audun and Muhammad Mudassar {Yamin}},
	editor = {Stephanidis, Constantine and Kurosu, Masaaki and Jessie Y. C. {Chen} and Fragomeni, Gino and Streitz, Norbert and Konomi, Shin{\textquoteright}ichi and Degen, Helmut and Ntoa, Stavroula}
}
@inproceedings {RouwendalvanSchijndel-Stolpe-Hannay-2021,
	title = {Toward an AI-based external scenario event controller for crisis response simulations},
	journal = {Proceedings of the 18th Int{\textquoteright}l Conf. Information on Information Systems for Crisis Response and Management (ISCRAM{\textendash}-AI and Intelligent Systems for Crises and Risks},
	year = {2021},
	publisher = {International Association for Information Systems for Crisis Response and Management},
	abstract = {There is a need for tool support for structured planning, execution and analysis of simulation-based training for crisis response and management. As a central component of an architecture for such tool support, we outline the design of an AI-based scenario event controller. The event controller is a component that uses machine reasoning to compute the next state in a scenario, given the actions performed in the corresponding simulation (execution of the scenario). Scenarios are specified in Answer Set Programming (ASP), which is a logic programming language we use for automated planning of training scenarios. A plan encoding in ASP adds expressivity in scenario specification and enables machine reasoning. For exercise managers this gives AI-based tool support for before-action and during-action reviews to optimize learning. In line with Modelling and Simulation as as Service, our approach externalizes event control from any particular simulation platform. The scenario, and its unfolding in terms of events, is externalized as a service. This increases interoperability and enables scenarios to be designed and modified readily and rapidly to adapt to new training requirements.},
	author = {Dashley Kevin {Rouwendal} and Stolpe, Audun and Jo Erskine {Hannay}},
	editor = {Adrot, Anouck and Grace, Rob and Moore, Kathleen and Zobel, Christopher}
}
@article {Hannay2020ModelingAS,
	title = {Modeling and Simulation as a Service infrastructure capabilities for discovery, composition and execution of simulation services},
	journal = {The Journal of Defense Modeling and Simulation: Applications, Methodology, Technology},
	volume = {18},
	year = {2020},
	pages = {5 - 28},
	publisher = {SAGE journals},
	abstract = {Modeling and Simulation as a Service (MSaaS) embodies the idea that simulations should be composed quickly for the task at hand from loosely coupled shared components, simulation services, in a cloud-based environment. These simulations are then offered, as composed simulation services, to human and technical consumers. Instrumental to this, is functionality that lets a simulation operator discover and compose simulation services and execute the composition. We describe this functionality in terms of what we call MSaaS infrastructure capabilities. Following the idea of stepwise refinement, the discovery and composition of simulation services can be done at design time using implementation-independent information about simulation services and at implementation time using implementation-specific information about simulation services. The execution environment can also be set up at design time and at implementation time. We therefore describe the MSaaS infrastructure capabilities in terms of how they are used on both implementation-independent and implementation-specific service information. By doing these elaborations, we intend to gain greater insight into how to perform simulation service discovery, composition, and execution. We conclude that although much of the required functionality for a MSaaS infrastructure is available through existing platforms and frameworks, it is necessary to offer this functionality as services, alongside (composed) simulation services, to fulfill the MSaaS vision.},
	doi = {10.1177/1548512919896855},
	author = {Jo Erskine {Hannay} and van den Berg, Tom and Gallant, Scott and Gupton, Kevin}
}
@inproceedings {RouwendalvanSchijndel-Hannay-Stolpe-2020,
	title = {Simulation Vignette Generation from Answer Set Specifications},
	journal = {Proc. 17th Int{\textquoteright}l Conf. Information Systems for Crisis Response and Management (ISCRAM){\textendash}-AI Systems for Crisis and Risks},
	year = {2020},
	month = {2020},
	publisher = {International Association for Information Systems for Crisis Response and Management},
	abstract = {We investigate an approach that allows exercise managers to design simulations with an explicit focus on building skills, rather than having to focus on all the objects and interactions that a simulation must have. Exercise managers may design exercises at various levels of abstraction and always independently of how those sessions are implemented in simulations, while simulation components that implement the design are assembled and to some extent, automatically, behind the scenes. We outline (1) how Answer Set Programming can assist exercise managers in exercise planning and (2) how automated stage and content generation may be used to invoke appropriate simulation components to realize the design. For deliberate and recurrent training of decision-making skills, stages and content must vary to avoid familiarity (testing effects). We conclude by distilling a main research hypothesis that stipulates how (1) and (2) represent two modes of automated reasoning (so-called deductive versus abductive) and how that distinction clarifies the planning task.},
	author = {D. K. Rouwend {van Schijndel} and Jo Erskine {Hannay} and Stolpe, A.},
	editor = {Amanda Lee {Hughes} and McNeill, Fiona and Zobel, Christopher}
}
@inbook {10.1007/978-3-030-49108-6_27,
	title = {Stakeholder Journey Analysis for Innovation},
	booktitle = {Universal Access in Human-Computer Interaction. Applications and Practice},
	number = {LNCS 12189},
	year = {2020},
	pages = {370{\textendash}389},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {When analysing how the information-technological innovation of a startup company is perceived to affect the market, we encountered challenges when using existing customer journey analysis frameworks. In particular, we identified a need to express critical events for technology adoption, a need to express universal access principles and a need to express journeys for multiple stakeholders in the same diagram; all in an easily readable manner. To do this, we extended an existing stripped-down customer journey framework with elements for the above aspects. We present this extended framework as a stakeholder journey framework. Based on the initial application of this framework on the startup company{\textquoteright}s innovation product, we conclude that the stakeholder journey framework aided in uncovering issues within technology adoption and universal access that would otherwise not have been addressed.},
	isbn = {978-3-030-49108-6},
	author = {Jo Erskine {Hannay} and Kristin Skeide {Fuglerud} and Bjarte M. {{\O}stvold}},
	editor = {Antona, Margherita and Stephanidis, Constantine}
}
@inbook {10.1007/978-3-030-50732-9_60,
	title = {Using Block-Based Programming and Sunburst Branching to Plan and Generate Crisis Training Simulations},
	booktitle = { Communications in Computer and Information Science },
	volume = {1226},
	year = {2020},
	pages = {463{\textendash}471},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	edition = {HCI International 2020 - Posters},
	address = {Cham},
	abstract = {Simulation-based exercises for crisis response are difficult to plan. We suggest an intuitive planning interface where exercise managers can use block-based programming to create machine-readable training vignettes. An answer set programming module interprets these vignettes and generates all possible actions and causal relations, which are then visualised for the exercise manager in a sunburst diagram. This allows exercise managers to create training vignettes using visual techniques and to see the possible results and causal relations of the trainees{\textquoteright} actions. This facilitates using exercise results to adjust further vignette design.},
	isbn = {978-3-030-50732-9},
	author = {Dashley K. Rouwend {van Schijndel} and Stolpe, Audun and Jo Erskine {Hannay}},
	editor = {Stephanidis, Constantine and Antona, Margherita}
}
@article {25405,
	title = {Agile Uncertainty Assessment for Benefit Points and Story Points},
	journal = {IEEE Software},
	volume = {36},
	year = {2019},
	month = {10/2018 },
	pages = {50--62},
	publisher = {IEEE Computer Society},
	abstract = {Estimates are inherently uncertain, and it{\textquoteright}s fair to expect that business value estimates are even more so. By making this uncertainty explicit, you can manage and monitor your agile project according to upper and lower margins of uncertainty. The trick is to instantiate benefit points and story points with values reflecting various scenarios.},
	author = {Jo Erskine {Hannay} and Hans Christian {Benestad} and Strand, Kjetil}
}
@inproceedings {Hannay-Kikke-2019,
	title = {Structured crisis training with mixed reality simulations},
	journal = {Proc. 16th Int{\textquoteright}l Conf. Information Systems for Crisis Response and Management (ISCRAM)},
	year = {2019},
	month = {2019},
	pages = {1310{\textendash}1319},
	publisher = {International Association for Information Systems for Crisis Response and Management},
	abstract = {We argue that current technology for crisis training does not explicitly cater well enough for managing training objectives and skill building metrics throughout the lifespan of training. We suggest how successful crisis training may be enabled by interoperating next-generation exercise management tools with mixed-reality simulations. We propose an architecture consisting of (1) a front-end in which training objectives, essential skills, corresponding events and metrics can be declared, (2) a back-end consisting of simulations that implement the events and metrics and (3) a middleware which transfers information between the front-end and back-end to enable semi-automatic composition of the simulations and performance analysis. The purpose of this architecture is to facilitate learning through the principles of deliberate practice. We indicate where emerging technologies are necessary to achieve this.},
	author = {Jo Erskine {Hannay} and Kikke, Y.},
	editor = {Franco, Zeno and Jos{\'e} J. {Gonz{\'a}lez} and Jos{\'e} H. {Can{\'o}s}}
}
@inproceedings {26412,
	title = {Simulation-Supported Wargaming using M\&S as a Service (MSaaS)},
	journal = {MSG-159 Symposium on Multinational Interoperability},
	year = {2018},
	publisher = {STO},
	abstract = {The Allied Framework for Modelling and Simulation as a Service (MSaaS) proposes that simulations should be composed of services and applications, which may be stored in repositories, found by registry services, and composed by modelling services. Docker containerization may implement several of these aspects of the Allied Framework, and has emerged as a promising technology for MSaaS. We provide a case study of using Docker containerization to close in on the vision of the Allied Framework. Containerizing a simulation system is not a trivial task, and we describe a set of issues and challenges that must be considered. Docker has no mechanisms for describing which services are offered by an image, so this must be handled by an external registry service. As future work, we outline how service descriptions might be written using semantic\&nbsp; technology, so that containerized applications implementing these descriptions might be composed on the fly.},
	author = {Asprusten, Martin and Jo Erskine {Hannay}}
}
@misc {25863,
	title = {Usikkerhetsvurdering for nyttepoeng og kostpoeng},
	howpublished = {Hovedstadsomr{\r a}dets nettverk for IT-ledelse og styring (HIT), Norway},
	year = {2018},
	author = {Jo Erskine {Hannay}}
}
@article {25406,
	title = {Benefit and Cost Periodized {\textendash} Stretching Your Points},
	journal = {IEEE Software},
	year = {2017},
	publisher = {IEEE Computer Society},
	abstract = {When you estimate lifecycle cost and benefit of your software product, your stakeholders should be assured not only that you{\textquoteright}lldeliver value but should also be informed when that value is expected to manifest itself. To do this you{\textquoteright}ll need tools to layout your estimates in time, so that you can plan and monitor your project{\textquoteright}s investments and returns. After reading this, you{\textquoteright}ll beable to take your points-based estimates for both cost and benefit (story points and benefit points) and distribute them out in timeaccording to the best of your project{\textquoteright}s knowledge. You{\textquoteright}ll also be able to project cost and return according to most likely, bad caseand good case uncertainty assessments.},
	author = {Jo Erskine {Hannay} and Hans Christian {Benestad} and Strand, Kjetil}
}
@article {24652,
	title = {Benefit Points {\textendash} The Best Part of the Story},
	journal = {IEEE Software},
	volume = {34},
	year = {2017},
	pages = { 73 - 85},
	publisher = {IEEE Computer Society},
	abstract = {\&quot;Value for customer\&quot; is the mantra in agile management and development. The product owner is involved along the way, and backlogs are prioritized, with the best of intents to maximize business value early and to eliminate waste. Yet, in many IT development projects bewilderment remains as to how exactly value for customer should be expressed in process decisions and in the delivered system. And projects continue to implement functionality which turns out to be off the mark or is never used. The reason, we say, is that there isn\&rsquo;t sufficient methodological support to determine and to monitor value for customer. Nor is there sufficient methodological support to link business value decisions to the mechanics of the development project.},
	author = {Jo Erskine {Hannay} and Hans Christian {Benestad} and Strand, Kjetil}
}
@article {24653,
	title = {Earned Business Value Management {\textendash} See that You Deliver Value to Your Customer {\textendash}},
	journal = {IEEE Software},
	volume = {34},
	year = {2017},
	pages = {58 - 70},
	publisher = {IEEE Computer Society},
	abstract = {The order in which you send your backlog items into construction determines when stakeholders will be able to reap benefit from\&nbsp;what functionality. This can have substantial impact on market timing, enterprise earnings and project manager survivor rate.\&nbsp;There are several ways to order a backlog, and sophisticated methods and tools exist to do so\&mdash;for example, in release planning.\&nbsp;But the important point we\&rsquo;ll make here, is that no matter what scheme for backlog ordering you choose to use, you ought to be\&nbsp;explicit on the order in which you realize potential business value. To this end, we\&rsquo;ll present methods to express business value\&nbsp;relative to cost in your backlog and methods to monitor how much potential business value you\&rsquo;re realizing along the way\&mdash;in\&nbsp;addition to cost expended.},
	author = {Jo Erskine {Hannay} and Hans Christian {Benestad} and Strand, Kjetil}
}
@article {25455,
	title = {A Hybrid Architecture Framework for Simulations in a Service-Oriented Environment},
	journal = {Systems Engineering},
	volume = {20},
	year = {2017},
	month = {07/2017},
	pages = {235-256},
	publisher = {Wiley},
	abstract = {Service orientation, and more recently, the notions of cloud technology that service orientation enable, are designed to accommodate the need for flexible enterprise business processes. Through standardized interfaces, a service-oriented architecture (SOA) should enable one to build and rebuild software systems readily and rapidly in a methodological manner. However, certain domains have specialized architectural standards; an example in point is modeling and simulation (M\&amp;S), for which there exist mature architectural standards, that may even have many of the characteristics strived for in SOA. An important issue is, therefore, how to integrate specialized architectures into a wider SOA. Using defense information systems and M\&amp;S as a case, we outline a hybrid architecture framework for specialized architectures in an encompassing SOA. Although it may be possible to dissolve a specialized architecture into the encompassing SOA at implementation time, we argue that it is important to be able to model the specialized architecture as an integral intact part. We further advocate a pragmatic notion of reference architecture in terms of appropriate level of abstraction and domain specificity to avoid pitfalls that may render architecture work unusable.},
	issn = {1520-6858},
	doi = {10.1002/sys.21390 },
	url = {http://onlinelibrary.wiley.com/doi/10.1002/sys.21390/abstract;jsessionid=01E67303DB2210DA93510424ED6D0887.f01t04},
	author = {Jo Erskine {Hannay} and Brathen, Karsten and Ole Martin {Mevassvik}}
}
@misc {25675,
	title = {Inntjent forretningsverdi},
	howpublished = {Hovedstadsomr{\r a}dets nettverk for IT-ledelse og styring (HIT), Oslo, Norway},
	year = {2017},
	type = {For industry},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {Hannay-vandenBerg-2017,
	title = {The NATO MSG-136 Reference Architecture for M\&S as a Service},
	journal = {Proc. NATO Modelling and Simulation Group Symp. on M\&S Technologies and Standards for Enabling Alliance Interoperability and Pervasive M\&S Applications (STO-MP-MSG-149)},
	year = {2017},
	publisher = {NATO Science and Technology Organization},
	abstract = {The Allied Framework for Modeling and Simulation (M\&amp;S) as a Service (MSaaS) is proposed by NATO MSG-136 as a permanent service- and cloud-based M\&amp;S ecosystem for use by NATO and partner nations. The framework is designed to aid stakeholders to utilize state-of-the-art service-oriented and cloud-based methodology and technology to achieve interoperability between participating simulation systems and ensuring credibility of results. This paper presents the reference architecture that is currently developed as part of the technical concept for the Allied Framework for MSaaS. The reference architecture is structured on The Open Group SOA Reference Architecture, while its contents is supplied via the NATO C3 Taxonomy in the form of architecture building blocks and architecture patterns. The MSaaS Reference Architecture is not a final product. It will change over time as new architecture building blocks and patterns are identified and added, and existing ones modified and improved.},
	isbn = {978-92-837-2137-6},
	author = {Jo Erskine {Hannay} and van den Berg, Tom}
}
@misc {25676,
	title = {Nyttepoengbasert usikkerhetsvurdering},
	howpublished = {Hovedstadsomr{\r a}dets nettverk for IT-ledelse og styring (HIT), Oslo, Norway},
	year = {2017},
	type = {For industry},
	author = {Jo Erskine {Hannay}}
}
@misc {25769,
	title = {Nyttestyring - noen metodiske prinsipper. Miniseminar Forsvarsmateriell},
	howpublished = {Forsvarsmateriell, Norway},
	year = {2017},
	month = {01/2018},
	author = {Jo Erskine {Hannay}}
}
@article {24654,
	title = {Agile requirements handling in a service-oriented taxonomy of capabilities},
	journal = {Requirements Engineering},
	year = {2016},
	pages = {1-26},
	publisher = {Springer},
	abstract = {To get to grips with information systemsportfolio development, strategic decisions tend towardservice orientation and cloud deployment. Functionalityshould be presented as services that can be consumedfrom secure clouds in a range of contexts, and serviceorientedarchitectures should enable one to build andrebuild systems portfolios readily and rapidly. However,there is little practical guidance on how to organize andcoordinate the multiple lines of work that developing, ormodernizing to, a service-oriented portfolio entails. Weoutline a method framework that uses the structure ofa service-oriented taxonomy of capabilities to organizerequirements and development in terms of elaborationand refinement of requirements. The method compilesseveral best practices, and supports independent, butintegral, lines of work that can be organized in smallscaleprojects. We illustrate the framework on threecases that involve computer- and simulation-assistedbusiness processes. We conclude that service-orientedcapability taxonomies can be used to structure and disciplinerequirements handling at all levels; from enterprisestrategy to technical systems.We suggest that ourframework supports the development of capabilities andservices that are persistent in the service-oriented senserelative to each other and to implementation. We suggestfurther that the framework supports collaborativework by facilitating shared conceptions across lines ofwork. We emphasize that empirical studies should beconducted to evaluate and refine the framework.},
	doi = {10.1007/s00766-016-0244-8},
	url = {http://rdcu.be/nB7c},
	author = {Jo Erskine {Hannay} and Brathen, Karsten and Ole Martin {Mevassvik}}
}
@article {24657,
	title = {Architectural work for Modelling and Simulation combining the NATO Architecture Framework and C3 Taxonomy},
	journal = {Journal of Defense Modeling and Simulation},
	year = {2016},
	publisher = {Sage Publications},
	abstract = {To provide modeling and simulation functionality as services is strategically leveraged in the defense domain\&nbsp;and elsewhere. To describe and understand the context\&mdash;the ecosystem\&mdash;wherein such services are used and\&nbsp;interoperate with other services and capabilities, one needs tools that capture the simulation services themselves\&nbsp;as well as the capability landscape they operate in. By using the NATO Consultation, Command and Control (C3)\&nbsp;Taxonomy to structure architecture design in the NATO Architecture Framework (NAF), cohesive descriptions of\&nbsp;modeling and simulation capabilities within larger contexts can be given. We show how a basic seven-step approach\&nbsp;may benefit architecture work for modeling and simulation at the overarching, reference and target architectural\&nbsp;levels; in particular for (1) hybrid architectures that embed simulation architectures within a larger service-oriented\&nbsp;architecture, and (2) for architectural design of simulation scenarios. Central to the approach is the use of the C3\&nbsp;Taxonomy as a repository for overarching architecture building blocks and patterns. We conclude that the promotion\&nbsp;of technical functionality as capabilities in their own right helps delineate simulation environment boundaries, helps\&nbsp;delineate services within and outside the boundary and is an enabler for defining the service concepts in cloud-based\&nbsp;approaches to Modeling and Simulation as a Service (MSaaS).},
	author = {Jo Erskine {Hannay}}
}
@article {24656,
	title = {Are team personality and climate related to satisfaction and software quality? Aggregating results from a twice replicated experiment},
	journal = {Information and Software Technology},
	volume = {57},
	year = {2015},
	pages = {141{\textendash}156},
	publisher = {Elsevier},
	abstract = {ContextResearch into software engineering teams focuses on human and social team factors. Social psychology deals with the study of team formation and has found that personality factors and group processes such as team climate are related to team effectiveness. However, there are only a handful of empirical studies dealing with personality and team climate and their relationship to software development team effectiveness.ObjectiveWe present aggregate results of a twice replicated quasi-experiment that evaluates the relationships between personality, team climate, product quality and satisfaction in software development teams.MethodOur experimental study measures the personalities of team members based on the Big Five personality traits (openness, conscientiousness, extraversion, agreeableness, neuroticism) and team climate factors (participative safety, support for innovation, team vision and task orientation) preferences and perceptions. We aggregate the results of the three studies through a meta-analysis of correlations. The study was conducted with students.ResultsThe aggregation of results from the baseline experiment and two replications corroborates the following findings. There is a positive relationship between all four climate factors and satisfaction in software development teams. Teams whose members score highest for the agreeableness personality factor have the highest satisfaction levels. The results unveil a significant positive correlation between the extraversion personality factor and software product quality. High participative safety and task orientation climate perceptions are significantly related to quality.ConclusionsFirst, more efficient software development teams can be formed heeding personality factors like agreeableness and extraversion. Second, the team climate generated in software development teams should be monitored for team member satisfaction. Finally, aspects like people feeling safe giving their opinions or encouraging team members to work hard at their job can have an impact on software quality. Software project managers can take advantage of these factors to promote developer satisfaction and improve the resulting product.},
	author = {Silvia Teresita {Acu{\~n}a} and Marta N. {Gomez} and Jo Erskine {Hannay} and Pfahl, Dietmar}
}
@inproceedings {24655,
	title = {On How Simulations Can Support Adaptive Thinking in Operations Planning},
	journal = {NATO Modelling and Simulation Group Symp. M\&S Support to Operational Tasks Including War Gaming, Logistics, Cyber Defence (MSG-133)},
	year = {2015},
	publisher = {NATO Science and Technology Organization},
	abstract = {Modern operations planning directives have well-defined stages for developing, testing and rehearsing plans. At\&nbsp;the same time, it is acknowledged that warfare is highly unpredictable and hard to plan. However, it is also\&nbsp;acknowledged that the planning process in itself gives insights that may prepare for unpredictable situations. We\&nbsp;outline how simulations might support and augment operations planning processes by supporting and developing\&nbsp;so-called adaptive thinking. In particular, simulation-supported course of action analysis enables multiple trials and\&nbsp;analyses, which, (1) enables planners to investigate a larger number of COAs with consequences, thereby broadening\&nbsp;the perspective of planners to the possibility of diverse outcomes, (2) may include critical cases generalizable to\&nbsp;ranges of COAs, and (3) may include elements targeted to induce adaptive thinking directly. The latter may employ\&nbsp;artificial elements to target task processes central to adaptive performance. We conclude that so-called passive\&nbsp;decision support systems in a case-driven simulation mode is appropriate for enhancing adaptive thinking.},
	author = {Jo Erskine {Hannay} and Brathen, Karsten and Jens Inge {Hynd{\o}y}}
}
@inproceedings {24659,
	title = {Simulation-Supported Wargaming for Analysis of Plans},
	journal = {NATO Modelling and Simulation Group Symp. M\&S Support to Operational Tasks Including War Gaming, Logistics, Cyber Defence (MSG-133)},
	year = {2015},
	publisher = {NATO Science and Technology Organization},
	abstract = {Wargaming is used in the military decision making process to visualize the execution of a preliminary plan or course of action in order to analyze and discover weaknesses and possibilities. The wargaming is traditionally done manually on a paper map, and the course of events is determined based on the experience and assumptions of the officers conducting the wargame. This paper describes ongoing research in Norway on the development of a demonstrator for Simulation-supported Wargaming for Analysis of Plans \&ndash; SWAP. The focus is particularly on the synchronization of cooperating and supporting units, aiming to enable the planning group to more easily distribute supporting units to its subordinates when the support is most needed. This tool is intended to integrate simulated wargaming in the planning process and thereby increase the quality of plans and decrease the planning time. SWAP uses a COTS computer generated forces federated with an agent-based simulation of C2 and combat management for simulation of the plan. It takes as input elements of a brigade plan from the Norwegian Command and Control Information System (C2IS) (NORCCIS). A web-based tool has been developed to support the officers in creating a synchronization matrix and to review the results of the simulation. The user can follow the simulated execution of the plan in the C2IS and receive information, such as fuel and ammo consumption and casualties on both sides. C2 to Simulation (C2SIM) standards and a service-based approach is used to promote interoperability, while the simulation comprises a time-managed High Level Architecture (HLA) federation.},
	author = {Bruvoll, Solveig and Jo Erskine {Hannay} and Guro Kristin {Svendsen} and Martin Leira {Asprusten} and Kjell Magne {Fauske} and Vegard Berg {Kvernelv} and Rikke Amilde {L{\o}vlid} and Jens Inge {Hynd{\o}y}}
}
@inproceedings {24658,
	title = {Live, Virtual, Constructive (LVC) simulation for land operations training: Concept Development \& Experimentation (CD\&E)},
	journal = {NATO Modelling and Simulation Group Symp. Integrating Modelling \& Simulation in the Defence Acquisition Lifecycle and Military Training Curriculum (MSG-126)},
	year = {2014},
	publisher = {NATO Science and Technology Organization},
	abstract = {Augmenting and extending military training by combining Live, Virtual and Constructive (LVC) simulation is thought to yield a range of benefits. A Concept Development and Experimentation (CD\&amp;E) activity was initiated to investigate the feasibility of complementing the Live training range at The Norwegian Army Combat \&amp; Manoeuvre Training Centre (NACMTC) with Virtual and Constructive simulations of BLUEFOR and OPFOR vehicles, UAV and artillery. The focus was on leadership training, and the military situation was augmented by activating contextual forces from the scenario and introducing more resources to administrate. The LVC simulation system prototype was run as a trial during an actual military exercise over four days. We found that the prototype did enable enhanced training, and that it is feasible and desirable to establish the LVC capability in full. Further, it is desirable to extend the LVC capability with augmented reality for Live forces and Joint Terminal Attack Controller (JTAC)/Close Air Support (CAS) simulation. We found it premature to evaluate the effect of LVC simulation training; because this is a longer-term activity, and because of a lack of validated instruments for measuring skill acquisition; in particular for decision making and judgement tasks. We conclude that there is a clear desire among operational personnel to acquire the LVC capabilities, and that several aspects of the capabilities can be established quickly. However, the use of LVC simulation for training and education must be mandated at all levels and incorporated explicitly in training plans and curricula, with a sufficient business case, so that political decisions for acquiring LVC capabilities can be made.},
	author = {Jo Erskine {Hannay} and Ole Martin {Mevassvik} and Skjeltorp, Arild and Brathen, Karsten}
}
@inproceedings {Simula.simula.1224,
	title = {Does the Prioritization Technique Affect Stakeholders{\textquoteright} Selection of Essential Software Product Features?},
	journal = {Proceedings of IEEE International Conference on Empirical Software Engineering and Measurement (ESEM 2012)},
	year = {2012},
	pages = {261-270},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Context: To select the essential, non-negotiable product features is a key skill for stakeholders in software projects. Such selection relies on human judgment, possibly supported by structured prioritization techniques and tools. Goal: Our goal was to investigate whether certain attributes of prioritization techniques affect stakeholders\&$\#$39; threshold for judging product features as essential. The four investigated techniques represent four combinations of granularity (low, high) and cognitive support (low, high). Method: To control for robustness and masking effects when investigating in the field, we conducted both an artificial experiment and a field experiment using the same prioritization techniques. In the artificial experiment, 94 subjects in four treatment groups indicated the features (from a list of 16) essential when buying a new cell phone. In the field experiment, 44 domain experts indicated the software product features that were essential for the fulfillment of the project\&$\#$39;s vision. The effects of granularity and cognitive support on the number of essential ratings were analyzed and compared between the experiments. Result: With lower granularity, significantly more features were rated as essential. The effect was large in the general experiment and extreme in the field experiment. Added cognitive support had medium effect, but worked in opposite directions in the two experiments, and was not statistically significant in the field experiment. Implications: Software projects should avoid taking stakeholders\&$\#$39; judgments of essentiality at face value. Practices and tools should be designed to counteract biases and to support the conscious knowledge-based elements of prioritizing.},
	doi = {10.1145/2372251.2372300},
	author = {Hans Christian {Benestad} and Jo Erskine {Hannay}}
}
@article {Simula.simula.1198,
	title = {Software Effort Estimation-A Matter of Skill Or Environment?},
	journal = {IEEE Transactions on Software Engineering},
	year = {2012},
	abstract = {Estimating the effort of software development is fraught with difficulties, and it is clear that effort should be invested in improving the accuracy and the reliability (consistency) of effort estimates, as well as the assessment of estimate uncertainty. However, it is less clear where to target such improvement efforts. We discuss the degree to which it is feasible to improve the expertise of the person(s) who estimate(s), and the environment in which the estimation is performed. The former hinges on what there is to say about the development of estimation expertise and the task characteristics of effort estimation. The latter hinges on what contextual support may be developed in terms of environment control and tools and methodology. We integrate several theories to make a framework for discussing software effort estimation and planning. On the basis of that discussion, we conclude that present guidelines almost exclusively concern the environment and its influence on broad psychological factors, that task-specific estimation expertise is too weak a signal in the noise of biases, and that strengthening this expertise requires new efforts in understanding the task-specific elements in software effort estimation and planning, as well as environmental measures (tools and methodologies) that support expert behavior and expert learning.},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {Simula.simula.28,
	title = {A Comparison of Model-Based and Judgment-Based Release Planning in Incremental Software Projects},
	journal = {ACM/IEEE International Conference on Software Engineering},
	year = {2011},
	pages = {766-775},
	publisher = {ACM New York, NY, USA},
	type = {Conference},
	abstract = {Numerous factors are involved when deciding when to implement which features in incremental software development. To facilitate a rational and efficient planning process, release planning models make such factors explicit and compute release plan alternatives according to optimization principles. However, experience indicates that industrial use of such models is limited. To investigate the feasibility of model and tool support, we compared input factors assumed by release planning models with factors considered by expert planners. The former factors were cataloged by systematically surveying release planning models, while the latter were elicited through repertory grid interviews in three software organizations. The findings indicate a substantial overlap between the two approaches. However, a detailed analysis reveals that models focus on only select parts of a possibly larger space of relevant planning factors. Three concrete areas of mismatch were identified: (1) continuously evolving requirements and specifications, (2) continuously changing prioritization criteria, and (3) authority-based decision processes. With these results in mind, models, tools and guidelines can be adjusted to better address real-life development processes.},
	isbn = {978-1-4503-0445-0},
	author = {Hans Christian {Benestad} and Jo Erskine {Hannay}}
}
@inproceedings {Simula.simula.520,
	title = {Does the Prioritization Technique Affect Stakeholders{\textquoteright} Selection of Essential Software Product Features?},
	journal = {5th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement},
	year = {2011},
	pages = {261{\textendash}270},
	publisher = {ACM},
	type = {Conference},
	abstract = {Context: Being able to select the essential, nonnegotiable product features is a key skill for stakeholders of software projects. Such selection relies on human judgment, sometimes supported by structured prioritization techniques and associated tools. Goal: Our goal was to investigate whether certain attributes of prioritization techniques affect stakeholders\&$\#$39; threshold for judging product features as essential. The four investigated techniques reflect four combinations of granularity (low, high) and cognitive support (low, high). Method: In one experiment, 94 subjects in four treatment groups indicated the features (from a list of 16) that would be essential in their decision to buy a new cell phone. With a similar setup in a controlled field experiment, 44 domain experts indicated the software product features that were essential for the fulfillment of the project\&$\#$39;s vision. The effects of granularity and cognitive support on the number of essential ratings were analyzed and compared between the experiments. Result: With lower granularity, significantly more features were rated as essential. The effect was large in the first experiment and extreme (Cohen\&$\#$39;s d=2.40) in the second. Added cognitive support had medium effect (Cohen\&$\#$39;s d=0.43 and 0.50), but worked in opposite directions in the two experiments, and was not statistically significant in the second. Implications: The results of the study imply that software projects should avoid taking stakeholders\&$\#$39; judgments of essentiality at face value. Practices and tools should be designed to counteract potentially harmful biases; however, more empirical work is needed to obtain more insight into the causes of these biases.},
	doi = {10.1145/2372251.2372300},
	author = {Hans Christian {Benestad} and Jo Erskine {Hannay}}
}
@inproceedings {Simula.prepare.14,
	title = {Inferring Skill From Tests of Programming Performance: Combining Time and Quality},
	journal = {Proc. 5th Int{\textquoteright}l Symp.Empirical Software Engineering and Measurement (ESEM)},
	year = {2011},
	pages = {305-314},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {The skills of software developers are crucial to the success of software projects. Also, controlling for individual differences is important when studying the general effect of a method or a tool. However, the way skill is determined in industry and research settings is often ad hoc or based on unvalidated methods. According to established test theory, validated tests of skill should infer skill levels from welldefined performance measures on multiple small, representative tasks. We show how time and quality can be meaningfully combined as task performance, and subsequently aggregated into an approximate measure of programming skill. Our results show significant and positive correlations between our proposed measures of skill and variables such as seniority, lines of code written, and self-evaluated expertise. The method for combining time and quality are a promising first step to measuring programming skill in industry and research settings.},
	isbn = {978-1-4577-2203-5},
	author = {Gunnar Rye {Bergersen} and Jo Erskine {Hannay} and Sj{\o}berg, Dag Ingar Kondrup and Dyb{\r a}, Tore and Karahasanovic, Amela}
}
@inproceedings {Simula.simula.386,
	title = {A Literature Review of Agile Practices and Their Effects in Scientific Software Development},
	journal = {Proceedings of SE-CSE11},
	year = {2011},
	pages = {1-9},
	type = {Workshop},
	abstract = {The nature of scientific research and the development of scientific software have similarities with processes that follow the agile manifesto: responsiveness to change and collaboration are of the utmost importance. But how well do current scientific software development processes match the practices found in agile development methods, and what are the effects of using agile practices in such processes? In order to investigate this, we conduct a literature review, focusing on evaluating the agility present in a selection of scientific software projects. Both projects with intentionally agile practices and projects with a certain degree of agile elements are taken into consideration. In the agility assessment, we define and utilize an agile mapping chart. The elements of the mapping chart are based on Scrum and XP, thus covering two of the most prominent agile reference models. We compared the findings of the literature review to results of a previously conducted survey. The comparison indicates that scientific software development projects adopting agile practices perceive their testing to be better than average. No difference to average projects was perceived regarding requirements-related activities. Future work includes an in-depth case study to further investigate the existence and impact of agility in three large scientific software projects, ultimately aiming at a better understanding of the particularities involved in developing scientific software.},
	author = {Magnus Thorstein {Sletholt} and Jo Erskine {Hannay} and Pfahl, Dietmar and Hans Christian {Benestad} and Hans Petter {Langtangen}},
	editor = {Carver, J.}
}
@misc {Simula.simula.370,
	title = {Stort \& Smidig Eller Stort \& Tungt},
	howpublished = {Talk given at Software 2011, Oslo.},
	year = {2011},
	author = {Jo Erskine {Hannay}}
}
@article {Simula.simula.722,
	title = {What Do We Know About Agile Practices in Scientific Software Development?},
	journal = {Computing in Science \& Engineering},
	volume = {14},
	number = {2},
	year = {2011},
	month = {June},
	pages = {24-37},
	abstract = {The development of scientific software has similarities with processes that follow the software engineering Agile Manifesto: responsiveness to change and collaboration are of utmost importance. But how well do current scientific software development processes match the practices found in agile development methods, and what are the effects of using agile practices in such processes? To find out, the authors conducted a literature review of past projects and a multiple case study of three ongoing scientific software development projects.},
	author = {Magnus Thorstein {Sletholt} and Jo Erskine {Hannay} and Pfahl, Dietmar and Hans Petter {Langtangen}}
}
@inproceedings {Simula.simula.275,
	title = {Detecting Learning and Fatigue Effects by Inspection of Person-Item Residuals},
	journal = {Probabilistic models for measurement in education, psychology, social science and health},
	year = {2010},
	publisher = {Copenhagen Business School and the University of Copenhagen},
	type = {1},
	isbn = {000-0-000-00000-0},
	author = {Gunnar Rye {Bergersen} and Jo Erskine {Hannay}},
	editor = {Nielsen, Tine and Kreiner, Svend and Brodersen, John}
}
@misc {Simula.simula.46,
	title = {Expertise in Planning \& Estimation: What Is It and Can One Improve It?},
	howpublished = {Talk given at the 2010 PREPARE Industry Seminar, October 15, 2010},
	year = {2010},
	author = {Jo Erskine {Hannay}}
}
@misc {Simula.prepare.24,
	title = {Forskning P{\r a} PERFORM Hva Har Vi Funnet Ut?},
	howpublished = {Talk given at Agile fagdag Statens Pensjonskasse August 27},
	year = {2010},
	author = {Petersen, J{\o}rgen and Jo Erskine {Hannay} and Hans Christian {Benestad}}
}
@misc {Simula.prepare.8,
	title = {Kontrakter Og Prosjektstyring I Store, Smidige IT-Prosjekter (Contracts and Project Management in Large, Agile Development Projects)},
	howpublished = {Talk given at Agile Project Management seminar at The Norwegian Center of Project Management, Trondheim, May 31.},
	year = {2010},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {Simula.simula.50,
	title = {Making Modern Scientific Software Development Explicitly Agile},
	journal = {The Changing Dynamics of Scientific Collaborations},
	year = {2010},
	type = {Workshop},
	abstract = {The nature of scientific research and the development of scientific software have similarities with processes that follow the agile manifesto: responsiveness to change and collaboration are of the utmost importance. But how well do current scientific software development processes match the practices found in agile development methods? As a representative for a modern scientific software project, we briefly outline the FEniCS project whose objective is to automate solution of differential equations. Based on initial investigations, we propose a casestudy where we investigate how the development process in FEniCS can be mapped to agile development methods, and whether SCRUM can be suitable to administer such a project. The case study will use another project, the more traditional Dalton project, as a contrast.},
	author = {Magnus Thorstein {Sletholt} and Jo Erskine {Hannay} and Hans Petter {Langtangen} and Pfahl, Dietmar}
}
@inproceedings {Simula.se.759,
	title = {Perceived Productivity Threats in Large Agile Development Projects},
	journal = {International Symposium on Empirical Software Engineering and Measurement (ESEM 2010)},
	year = {2010},
	publisher = {ACM},
	type = {Conference},
	abstract = {Applying agile methodology in large software development projects introduces many challenges. For example, one may expect that the combination of autonomous teams and the necessity for an overall organizational control structure may lead to conflicts, and one may expect that Agile{\textquoteright}s informal means of knowledge sharing breaks down as the number of project participants increases. Issues such as these may in turn compromise the project{\textquoteright}s productivity. In order to better understand potential threats to productivity in large agile development projects, we conducted repertory grid interviews with 13 project members on their perceptions of threats to productivity. The project was a large software development project consisting of 11 Scrum teams from three different subcontractors. The repertory grid sessions produced 100 issues, which were content analyzed into 10 main problem areas: (1) Restraints on collaboration due to contracts, ownership, and culture, (2) Architectural and technical qualities are given low priority, (3) Conflicts between organizational control and flexibility, (4) Volatile and late requirements from external parties, (5) Lack of a shared vision for the end product, (6) Limited dissemination of functional knowledge, (7) Excessive dependencies within the system, (8) Overloading of key personnel, (9) Difficulties in maintaining well-functioning technical environments, (10) Difficulties in coordinating test and deployment with external parties. Using critical-case reasoning, we claim that projects deploying agile practices in projects with less favorable conditions than those enjoyed in the current project, and that are larger and more complex, are likely to face similar challenges.},
	isbn = {978-1-4503-0039-01},
	author = {Jo Erskine {Hannay} and Hans Christian {Benestad}},
	editor = {Morisio G. {Succi} and Nagappan, N.}
}
@inbook {Simula.se.755,
	title = {Personality, Intelligence, and Expertise: Impacts on Software Development},
	booktitle = {Making Software. What Really Works, and Why We Believe It},
	year = {2010},
	pages = {79-110},
	publisher = {O{\textquoteright}Reilly},
	organization = {O{\textquoteright}Reilly},
	chapter = {6},
	isbn = {978-0-596-80832-7},
	author = {Jo Erskine {Hannay}},
	editor = {Wilson, Greg and Oram, Andy}
}
@inproceedings {Simula.se.761,
	title = {Software Engineering Group Work - Personality, Patterns and Performance},
	journal = {Special Interest Group of the ACM on Management Information Systems and Special Interest Group of the ACM on Computer Personnel Research},
	year = {2010},
	pages = {43-47},
	publisher = {ACM},
	type = {Conference},
	abstract = {Software Engineering has been a fundamental part of many computing undergraduate courses for a number of years. Although many of the tools and techniques used to undertake software engineering have changed, the assessment has typically stayed the same. Students are commonly tasked with producing a number of software artefacts, for example designs using the Unified Modelling Language (UML). We recently attempted to extend the software engineering experience for a group of second year students with them participating in groups that attempt to replicate industrial practice. This paper reports our investigation into the correlation between the personality of group members, their approach with respect to using design patterns and their learning achievements.},
	isbn = {978-1-4503-0004-9},
	author = {Bell, David and Hall, Tracy and Jo Erskine {Hannay} and Pfahl, Dietmar and Silvia Teresita {Acu{\~n}a}},
	editor = {Downey, J. and Joseph, D.}
}
@inproceedings {Simula.SE.583,
	title = {Challenges in Enterprise Software Integration: an Industrial Study Using Repertory Grids},
	journal = {Empirical Software Engineering and Measurement (ESEM)},
	year = {2009},
	month = {March},
	publisher = {IEEE},
	type = {Conference},
	abstract = {To identify and systematize software practitioners{\textquoteright} perceptions of a problem is an important first step toward analyzing and searching for a solution to the problem. This paper reports on an industrial study, in which the repertory grid technique was used to elicit practitioners{\textquoteright} perceptions of key challenges in the company{\textquoteright}s software integration practices. The perceptions of a total of nine practitioners from three organizational tiers (Developer, QA Manager, Project Manager) were elicited and analyzed. We found that perceptions differ markedly between tiers, but that on some issues, there is a consensus across all tiers. The elicited information may be used to plan process improvement for integration projects in the company, and may also be used in building a general ontology for integration challenges and their solutions.},
	isbn = {978-1-4244-4842-5},
	author = {Heidi Jacobsen {Rognerud} and Jo Erskine {Hannay}},
	editor = {Carver, J.}
}
@article {Simula.SE.309,
	title = {The Effectiveness of Pair-Programming: a Meta-Analysis},
	journal = {Information and Software Technology},
	volume = {55},
	number = {7},
	year = {2009},
	month = {June},
	pages = {1110-1122},
	abstract = {Several experiments on the effects of pair versus solo programming have been reported in the literature. We present a metaanalysis of these studies. The analysis shows a small significant positive overall effect of pair programming on quality, a medium significant positive overall effect on duration, and a medium significant negative overall effect on effort. However, between-study variance is significant, and there are signs of publication bias among published studies on pair programming. A more detailed examination of the evidence suggests that pair programming is faster than solo programming when programming task complexity is low and yields code solutions of higher quality when task complexity is high. The higher quality for complex tasks comes at a price of considerably greater effort, while the reduced completion time for the simpler tasks comes at a price of noticeably lower quality. We conclude that greater attention should be given to moderating factors on the effects of pair programming.},
	author = {Jo Erskine {Hannay} and Dyb{\r a}, Tore and Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Simula.SE.305,
	title = {Effects of Personality on Pair Programming},
	journal = {IEEE Transactions on Software Engineering},
	volume = {36},
	number = {1},
	year = {2009},
	pages = {61-80},
	abstract = {Personality tests in various guises are commonly used in recruitment and career counseling industries. Such tests have also been considered as instruments for predicting programming and team performance in software engineering. However, research suggests that other human-related factors, such as motivation, general mental ability, expertise and task complexity also affect performance in general. This paper reports on a study of the impact of Big-Five personality traits on the performance of pair programmers together with the impact of expertise and task complexity. The study involved 196 software professionals from three countries forming 98 pairs. The analysis consisted of a confirmatory part and an exploratory part, and the results show that (1) our data does not confirm a meta-analysis-based model of the impact of certain personality traits on performance, and (2) personality traits in general have modest predictive value on pair programming performance compared with expertise, task complexity, and even country. Rather than focusing on direct effects of personality on pair programming performance, we conclude that (a) effort should be spent on elaborating on personality{\textquoteright}s (and other factors{\textquoteright}) indirect effects on performancemediated by formalized scales of collaboration, and (b) more effort should be spent on investigating other performance-related predictors such as programming skill, learning, motivation, expertise, and task complexity.},
	author = {Jo Erskine {Hannay} and Arisholm, Erik and Engvik, Harald and Sj{\o}berg, Dag Ingar Kondrup}
}
@inproceedings {Simula.SE.425,
	title = {How Do Scientists Develop and Use Scientific Software?},
	journal = {Software Engineering for Computational Science and Engineering},
	year = {2009},
	pages = {1-8},
	publisher = {IEEE Computer Society},
	type = {Workshop},
	abstract = {New knowledge in science and engineering relies increasingly on results produced by scientific software. Therefore, knowing how scientists develop and use software in their research is critical to assessing the necessity for improving current development practices and to making decisions about the future allocation of resources. To that end, this paper presents the results of a survey conducted online in October-December 2008 which received almost 2000 responses. Our main conclusions are that (1) the knowledge required to develop and use scientific software is primarily acquired from peers and through self-study, rather than from formal education and training; (2) the number of scientists using supercomputers is small compared to the number using desktop or intermediate computers; (3) most scientists rely primarily on software with a large user base; (4) while many scientists believe that software testing is important, a smaller number believe they have sufficient understanding about testing concepts; and (5) that there is a tendency for scientists to rank standard software engineering concepts higher if they work in large software development projects and teams, but that there is no uniform trend of association between rank of importance of software engineering concepts and project/team size.},
	isbn = {978-1-4244-3737-5},
	author = {Jo Erskine {Hannay} and Hans Petter {Langtangen} and MacLeod, Carolyn and Pfahl, Dietmar and Singer, Janice and Wilson, Greg},
	editor = {Carver, Jeff}
}
@inproceedings {Simula.SE.585,
	title = {Personality and the Nature of Collaboration in Pair Programming},
	journal = {Empirical Software Engineering and Measurement (ESEM)},
	year = {2009},
	month = {March},
	publisher = {IEEE},
	type = {Conference},
	abstract = {The benefits of synergistic collaboration are at the heart of arguments in favor of pair programming. However, empirical studies usually investigate direct effects of various factors on pair programming performance without looking into the details of collaboration. This paper reports from an empirical study that (1) investigated the nature of pair programming collaboration, and (2) subsequently investigated postulated effects of personality on pair programming collaboration. Audio recordings of 44 professional programmer pairs were categorized according to a taxonomy of collaboration. We then measured postulated relationships between the collab- oration categories and the personality of the individuals in the pairs. We found evidence that personality generally af- fects the type of collaboration that occurs in pairs, and that different levels of a given personality trait between two pair members increases the amount of communication-intensive collaboration exhibited by a pair.},
	isbn = {978-1-4244-4842-5},
	author = {Walle, Thorbj{\o}rn and Jo Erskine {Hannay}},
	editor = {Carver, J.}
}
@article {Simula.SE.124,
	title = {A Systematic Review of Quasi-Experiments in Software Engineering},
	journal = {Information and Software Technology},
	volume = {51},
	number = {1},
	year = {2009},
	month = {January},
	pages = {71-82},
	author = {Vigdis By {Kampenes} and Dyb{\r a}, Tore and Jo Erskine {Hannay} and Sj{\o}berg, Dag Ingar Kondrup}
}
@inbook {Sjoberg.2007.1,
	title = {Building Theories in Software Engineering},
	booktitle = {Advanced Topics in Empirical Software Engineering},
	year = {2008},
	pages = {312-336},
	publisher = {Springer},
	organization = {Springer},
	chapter = {12},
	abstract = {Empirical studies have become an integral element of software engineering research and practice. This unique text/reference includes chapters from some of the top international empirical software engineering researchers and focuses on the practical knowledge necessary for conducting, reporting and using empirical methods in software engineering.},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Dyb{\r a}, Tore and Anda, Bente Cecilie Dahlum and Jo Erskine {Hannay}},
	editor = {Shull, Forrest and Singer, Janice and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Simula.SE.9,
	title = {The Role of Artificial Design Elements in Software Engineering Experiments},
	journal = {IEEE Transactions on Software Engineering},
	volume = {34},
	number = {2},
	year = {2008},
	month = {March/April},
	pages = {242-259},
	abstract = {Increased realism in software engineering experiments is often promoted as an important means to increase generalizability and industrial relevance. In this context, artificiality, e.g., the use of constructed tasks in place of realistic tasks, is seen as a threat. In this article, we examine the opposite view, that deliberately introduced artificial design elements may increase knowledge gain and enhance both generalizability and relevance. In the first part of the article, we identify and evaluate arguments and examples in favor of, and against, deliberately introducing artificiality into software engineering experiments. In the second part of the article, we summarize a content analysis of articles reporting software engineering experiments published over the ten-year period 1993-2002. The analysis reveals a striving for realism and external validity, but little awareness of for what and when, various degrees of artificiality and realism are appropriate. We conclude that an increased awareness and deliberation in these respects is essential. However, arguments in favor of artificial design elements should not be used to justify studies that are badly designed or that have research questions of low relevance.},
	author = {Jo Erskine {Hannay} and J{\o}rgensen, Magne}
}
@article {Simula.SE.162,
	title = {Are Two Heads Better Than One? on the Effectiveness of Pair Programming},
	journal = {IEEE Software},
	volume = {24},
	number = {6},
	year = {2007},
	month = {November/December},
	pages = {12-15},
	author = {Dyb{\r a}, Tore and Arisholm, Erik and Sj{\o}berg, Dag Ingar Kondrup and Jo Erskine {Hannay} and Shull, Forrest}
}
@article {Kampenes.2006.1,
	title = {A Systematic Review of Effect Size in Software Engineering Experiments},
	journal = {Information and Software Technology},
	volume = {49},
	number = {11-12},
	year = {2007},
	pages = {1073-1086},
	author = {Vigdis By {Kampenes} and Dyb{\r a}, Tore and Jo Erskine {Hannay} and Sj{\o}berg, Dag Ingar Kondrup}
}
@article {Hannay.2006.1,
	title = {A Systematic Review of Theory Use in Software Engineering Experiments},
	journal = {IEEE Transactions on Software Engineering},
	volume = {33},
	number = {2},
	year = {2007},
	month = {February},
	pages = {87-107},
	abstract = {Empirically-based theories are generally perceived as foundational to science. However, in many disciplines, the nature, role and even the necessity of theories remain matters for debate, particularly in young or practical disciplines such as software engineering. This article reports a systematic review of the explicit use of theory in a comprehensive set of 103 articles reporting controlled experiments, from of a total of 5,453 articles published in major software engineering journals and conferences in the decade 1993-2002. Of the 103 articles, 24 use a total of 39 theories in various ways to explain the cause-effect relationship(s) under investigation. The majority of these use theory in the experimental design to justify research questions and hypotheses; some use theory to provide \emph{post-hoc} explanations of their results; while a few test or modify theory. A third of the theories are proposed by authors of the reviewed articles. The interdisciplinary nature of the theories used is greater than that of research in software engineering in general. We found that theory use and awareness of theoretical issues are present, but that theory-driven research is, as yet, not a major issue in empirical software engineering. Several articles comment explicitly on the lack of relevant theory. We call for an increased awareness of the potential benefits of involving theory, when feasible. To support software engineering researchers who wish to use theory, we give an overview that shows which of the reviewed articles on which topics use which theories for what purposes, as well as details of the theories{\textquoteright} characteristics.},
	author = {Jo Erskine {Hannay} and Sj{\o}berg, Dag Ingar Kondrup and Dyb{\r a}, Tore}
}
@article {Sjoberg.2005.1,
	title = {A Survey of Controlled Experiments in Software Engineering},
	journal = {IEEE Transactions on Software Engineering},
	volume = {31},
	number = {9},
	year = {2005},
	month = {September},
	pages = {733-753},
	abstract = {The classical method for identifying cause-effect relationships is to conduct controlled experiments. This paper reports upon the present state of how controlled experiments in software engineering are conducted and the extent to which relevant information is reported. Among the 5,453 scientific articles published in 12 leading software engineering journals and conferences in the decade from 1993 to 2002, 103 articles (1.9 percent) reported controlled experiments in which individuals or teams performed one or more software engineering tasks. This survey quantitatively characterizes the topics of the experiments and their subjects (number of subjects, students versus professionals, recruitment, and rewards for participation), tasks (type of task, duration, and type and size of application) and environments (location, development tools). Furthermore, the survey reports on how internal and external validity is addressed and the extent to which experiments are replicated. The gathered data reflects the relevance of software engineering experiments to industrial practice and the scientific maturity of software engineering research.},
	author = {Sj{\o}berg, Dag Ingar Kondrup and Jo Erskine {Hannay} and Hansen, Ove and Vigdis By {Kampenes} and Karahasanovic, Amela and Liborg, Nils-Kristian and Anette C. {Rekdal}}
}
@misc {Simula.SE.611,
	title = {Controlled Experiments in Software Engineering},
	howpublished = {Talk given at SINTEF ICT Oslo},
	year = {2004},
	author = {Jo Erskine {Hannay}}
}
@misc {Simula.SE.609,
	title = {Empirically-Based Theories for Software Engineering},
	howpublished = {Talk given at SINTEF ICT Oslo},
	year = {2004},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.2003.b,
	title = {Abstraction Barrier-Observing Relational Parametricity},
	journal = {Typed Lambda Calculi and Applications. Proceedings of TLCA, 6th International Conference, Valencia, Spain, LNCS volume 2701},
	year = {2003},
	month = {June 10 - 12},
	pages = {135-152},
	publisher = {Springer-Verlag},
	type = {1},
	abstract = {A concept of relational parametricity is developed where the encapsulation mechanism inherent in universal types is taken into account. This is then applied in the context of data types and refinement, naturally giving rise to a notion of simulation relations that compose for data types with higher-order operations, and whose existence coincides with observational equivalence. The ideas are developed syntactically in lambda calculus with a relational logic. The new notion of relational parametricity is asserted axiomatically, and a corresponding parametric PER-semantics is devised.},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.2003.a,
	title = {Axiomatic Criteria for Quotients and Subobjects at Higher Order},
	journal = {Automata, Languages and Programming. Proceedings of ICALP 2003, 30th International Colloquium, Eindhoven, the Netherlands, LNCS volume 2719},
	year = {2003},
	month = {June 30 - July 4},
	pages = {903-917},
	publisher = {Springer-Verlag},
	type = {1},
	abstract = {Axiomatic criteria are given for the existence of higher-order maps over subobjects and quotients. These criteria are applied in showing the soundness of a method for proving specification refinement up to observational equivalence. This generalises the method to handle data types with higher-order operations. We also give a direct setoid-based model satisfying the criteria. The setting is the second-order polymorphic lambda calculus and the assumption of relational parametricity.},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.2003.c,
	title = {Semantic and Syntactic Approaches to Simulation Relations},
	journal = {Mathematical Foundations of Computer Science. Proceedings of MFCS, 28th International Symposium on Mathematical Foundations of Computer Science, Bratislava, Slovak Republic, LNCS volume 2747},
	year = {2003},
	note = {Invited paper.},
	month = {August 25 - 29},
	pages = {68-91},
	publisher = {Springer-Verlag},
	type = {1},
	abstract = {Simulation relations are tools for establishing the correctness of data refinement steps. In the simply-typed lambda calculus, logical relations are the standard choice for simulation relations, but they suffer from certain shortcomings; these are resolved by use of the weaker notion of pre-logical relations instead. Developed from a syntactic setting, abstraction barrier-observing simulation relations serve the same purpose, and also handle polymorphic operations. Meanwhile, second-order pre-logical relations directly generalise pre-logical relations to polymorphic lambda calculus (System F). We compile the main refinement-pertinent results of these various notions of simulation relation, and try to raise some issues for aiding their comparison and reconciliation.},
	author = {Jo Erskine {Hannay} and S. -Y {Katsumata} and D. T. {Sannella}}
}
@misc {Simula.SE.613,
	title = {System F - Relational Parametricity and Refinement},
	howpublished = {Three lectures given at the University of Oslo},
	year = {2003},
	author = {Jo Erskine {Hannay}}
}
@phdthesis {SE.3.Hannay.2001,
	title = {Abstraction Barriers and Refinement in the Polymorphic Lambda Calculus},
	year = {2001},
	note = {Doctor of Philosophy thesis},
	school = {University of Edinburgh, Scotland},
	type = {phd},
	abstract = {The use of safety-critical software relies crucially on the certified verification relative to an unambiguous specification that the software will perform correctly, even on the first occasion it is put to use. One way of achieving unambiguity and assurance is to use mathematical or formal methods in the development process of such software, so that the result is software that is mathematically proven correct relative to a formal specification. The concept of algebraic specification refinement embodies a framework in which to develop software formally. In this thesis, this concept is expressed in a different, and in many respects, richer mathematical framework, namely that of type theory, polymorphic lambda-calculus, and an intuitionistic logic extended to assert relational parametricity, as well as key features of specification refinement. This translation into a different framework enables important extensions to the concept of algebraic specification refinement that enhance the proving power, the applicability, and hence the usefulness of the concept.},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.2000,
	title = {A Higher-Order Simulation Relation for System F},
	journal = {Foundations of Software Science and Computation Structures, Proceedings of FOSSACS 2000, a part of ETAPS 2000 (European Joint Conferences on Theory and Practice of Software), Berlin, Germany, LNCS volume 1784},
	year = {2000},
	pages = {130-145},
	publisher = {Springer-Verlag},
	type = {1},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.1999.a,
	title = {Specification Refinement in System F, the Higher-Order Case},
	journal = {Recent Trends in Data Type Specification, Selected papers of the 14th International Workshop on Algebraic Development Techniques (WADT{\textquoteright}99), Chateau de Bonas, France, LNCS volume 1827},
	year = {1999},
	pages = {162-181},
	publisher = {Springer-Verlag},
	type = {1},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.1999.b,
	title = {Specification Refinement With System F},
	journal = {Computer Science Logic, Proceedings of CSL{\textquoteright}99, Madrid, Spain, LNCS volume 1683},
	year = {1999},
	pages = {530-545},
	publisher = {Springer-Verlag},
	type = {1},
	author = {Jo Erskine {Hannay}}
}
@inproceedings {SE.5.Hannay.1998,
	title = {Abstraction Barriers in Equational Proof},
	journal = {Algebraic Methodology and Software Technology, Proceedings of AMAST{\textquoteright}98, Aria{\'u}, Amazonas, Brasil, LNCS volume 1548},
	year = {1998},
	pages = {196-213},
	publisher = {Springer-Verlag},
	type = {1},
	author = {Jo Erskine {Hannay}}
}
@techreport {SE.7.Hannay.1998,
	title = {Referential Opacity in Equational Reasoning},
	number = {ECS-LFCS-98-398},
	year = {1998},
	publisher = {University of Edinburgh},
	author = {Jo Erskine {Hannay}}
}
@techreport {SE.7.Hannay.1997,
	title = {Relative Equational Specification and Semantics},
	number = {ECS-LFCS-97-366},
	year = {1997},
	note = {Technical report},
	publisher = {University of Edinburgh},
	author = {Jo Erskine {Hannay}}
}
@techreport {Hannay.1995.2,
	title = {Generalisert Algebraisk Spesifikasjon Med En Anvendelse P{\r a} Indirekte Semantikk},
	year = {1995},
	publisher = {Department of Informatics, University of Oslo},
	abstract = {The thesis deals with topics in the field of formal abstract specification and verification of programs, particularly within the framework of algebraic methods restricted to equational logic. The discussion views equational specification in conjunctiom with the closely related topic of simple term-rewriting as more concrete and nearer to implementation than specification in general. Therefore initial and final models are considered, rather than complete model-classes. Also mostly constructive, {\textquoteleft}{\textquoteleft}executable{\textquoteright}{\textquoteright} equational specifications are considered giving an inclination towards viewing specifications as abstract programs. The thesis is structured around the following two topics:  {\textendash}{\textendash}  1. The method of equational algebraic specification is generalized in a manner allowing a certain mode of modular specification. The generalization has two forms accommodating initial and final algebra semantics respectively. The generalization is modular in the sense that complex specifications may be constructed stepwise from simpler kernel specifications. Each construction step provides the choice of initial or final generalized form. The resulting complex equational specifications differ from general case predicate logic hierarchical specifications, in the sense that the polarization into initial and final semantics gives inhomogeneous specifications; a complex specification cannot in principle be viewed as a non-hierarchical simple specification.  A concept of consistency relative to kernel specifications is developed and it is possible to reason about such relative consistency without specific knowledge about the kernel specification.  The possibility of analogously constructing complex formal-mechanical proof methods from simpler methods is briefly discussed.  2. A variant of (initial) algebraic specification called indirect specification is developed. Motivation and a foundation for indirect specification is given by syntactical specifier functions; i.e. terms are to be understood identical iff the their values under some given function (with syntactical codomain) are identical. A special and interesting case of syntactical specifier functions are functions giving canonical representatives in some well-defined sense.  Besides providing the specification language with additional means of specifying equality, indirect specification expands the class of congruences (over ground terms) decidable by simple term-rewriting. In addition, although the class of initial congruences equationally specifiable is identical to the class specifiable by syntactical specifier functions with canonical representatives as codomain, there exist syntactical specifier functions which do not give canonical representatives. (Such a function may for instance give a representative in some other class than in which the argument belongs.) It may therefore be the fact that the class of congruences indirectly specifiable is greater than the class equationally specifiable. Characteristically, indirect specification represents a more operational mode of specification than does usual algebraic specification.  Indirect specification introduces further inhomogeneousity to equational algebraic specification. The modularity of the generalized specification strategy discussed in part 1 is used to encapsulate indirect specification in the hierarchical framework.  It is then shown that a trivial augmentation of indirect specification can be reduced to standard non-generalized initial or final semantics specification; under certain interesting circumstances. It is also shown that Knuth-Bendix completion of an augmentation of indirect specification, if successful and under a certain congruency condition, will give an equivalent standard algebraic specification. Viewing equational specifications as abstract programs, this might be seen as a program transformation. (A more applicable strategy for automatic transformation in this sense is sketched just as an idea without any sort of further proof.)  {\textendash}{\textendash}  The overall structure sees part 1 primarily as giving a framework for the discussion in part 2.  The subject of consistency permeates the entire discussion. Consistency is viewed as related to basic ideas of some semantic domain which are expressed as untamperable presuppositions in the act of specifying. For example, the basic idea that a mathematical proposition cannot simultaneously be true and false is presupposed in predicate calculus by predefined interpretations of the symbols true and false, and by referring to a set of axioms as inconsistent iff the predicate true = false is deducible from the axioms. The concept of (in)consistency seen always relatively to such presuppositions allows a generalized notion of consistency in hierarchical specification relative to kernel specifications as discussed in part 1. (This also generalizes Guttag{\textquoteright}s notion of consistency.) Some simple methods for detecting (generalized) inconsistency and for establishing consistency are briefly presented.  The notion of artificial inconsistency is introduced, as inconsistency due to auxiliary functions; i.e. functions helpful or necessary during constructive definition or implementation, but otherwise really not belonging to the semantical objects under implementation. Formalisms and results are developed showing that auxiliary functions and hence artificial inconsistency can be hidden from the formal reasoning, still allowing for the full implementatory benefits. This goes beyond the model-theoretical notions of hidden sorts and symbols. The theory developed can probably be used and implemented by modifying existing proof methods (based on the concepts of proof by consistency and inductive completion).  Finally an extension of Knuth-Bendix completion is presented as a step towards mechanical generation of constructive proofs in the input theory given to the process. Constructive proofs may give deeper insight into a theory. It also turns out that the extension of Knuth-Bendix completion under certain circumstances can be used in establishing consistency in conjunction with indirect specification (part 2). The extension also turns out to be instrumental in proving that hiding of auxiliary functions and artificial inconsistency may be implemented in proof methods based on inductive completion.},
	author = {Jo Erskine {Hannay}}
}
@techreport {Hannay.1995.3,
	title = {Generalisert Algebraisk Spesifikasjon Med En Anvendelse P{\r a} Indirekte Semantikk},
	year = {1995},
	publisher = {Department of Informatics, University of Oslo},
	abstract = {The thesis deals with topics in the field of formal abstract specification and verification of programs, particularly within the framework of algebraic methods restricted to equational logic. The discussion views equational specification in conjunctiom with the closely related topic of simple term-rewriting as more concrete and nearer to implementation than specification in general. Therefore initial and final models are considered, rather than complete model-classes. Also mostly constructive, {\textquoteleft}{\textquoteleft}executable{\textquoteright}{\textquoteright} equational specifications are considered giving an inclination towards viewing specifications as abstract programs. The thesis is structured around the following two topics:  {\textendash}{\textendash}  1. The method of equational algebraic specification is generalized in a manner allowing a certain mode of modular specification. The generalization has two forms accommodating initial and final algebra semantics respectively. The generalization is modular in the sense that complex specifications may be constructed stepwise from simpler kernel specifications. Each construction step provides the choice of initial or final generalized form. The resulting complex equational specifications differ from general case predicate logic hierarchical specifications, in the sense that the polarization into initial and final semantics gives inhomogeneous specifications; a complex specification cannot in principle be viewed as a non-hierarchical simple specification.  A concept of consistency relative to kernel specifications is developed and it is possible to reason about such relative consistency without specific knowledge about the kernel specification.  The possibility of analogously constructing complex formal-mechanical proof methods from simpler methods is briefly discussed.  2. A variant of (initial) algebraic specification called indirect specification is developed. Motivation and a foundation for indirect specification is given by syntactical specifier functions; i.e. terms are to be understood identical iff the their values under some given function (with syntactical codomain) are identical. A special and interesting case of syntactical specifier functions are functions giving canonical representatives in some well-defined sense.  Besides providing the specification language with additional means of specifying equality, indirect specification expands the class of congruences (over ground terms) decidable by simple term-rewriting. In addition, although the class of initial congruences equationally specifiable is identical to the class specifiable by syntactical specifier functions with canonical representatives as codomain, there exist syntactical specifier functions which do not give canonical representatives. (Such a function may for instance give a representative in some other class than in which the argument belongs.) It may therefore be the fact that the class of congruences indirectly specifiable is greater than the class equationally specifiable. Characteristically, indirect specification represents a more operational mode of specification than does usual algebraic specification.  Indirect specification introduces further inhomogeneousity to equational algebraic specification. The modularity of the generalized specification strategy discussed in part 1 is used to encapsulate indirect specification in the hierarchical framework.  It is then shown that a trivial augmentation of indirect specification can be reduced to standard non-generalized initial or final semantics specification; under certain interesting circumstances. It is also shown that Knuth-Bendix completion of an augmentation of indirect specification, if successful and under a certain congruency condition, will give an equivalent standard algebraic specification. Viewing equational specifications as abstract programs, this might be seen as a program transformation. (A more applicable strategy for automatic transformation in this sense is sketched just as an idea without any sort of further proof.)  {\textendash}{\textendash}  The overall structure sees part 1 primarily as giving a framework for the discussion in part 2.  The subject of consistency permeates the entire discussion. Consistency is viewed as related to basic ideas of some semantic domain which are expressed as untamperable presuppositions in the act of specifying. For example, the basic idea that a mathematical proposition cannot simultaneously be true and false is presupposed in predicate calculus by predefined interpretations of the symbols true and false, and by referring to a set of axioms as inconsistent iff the predicate true = false is deducible from the axioms. The concept of (in)consistency seen always relatively to such presuppositions allows a generalized notion of consistency in hierarchical specification relative to kernel specifications as discussed in part 1. (This also generalizes Guttag{\textquoteright}s notion of consistency.) Some simple methods for detecting (generalized) inconsistency and for establishing consistency are briefly presented.  The notion of artificial inconsistency is introduced, as inconsistency due to auxiliary functions; i.e. functions helpful or necessary during constructive definition or implementation, but otherwise really not belonging to the semantical objects under implementation. Formalisms and results are developed showing that auxiliary functions and hence artificial inconsistency can be hidden from the formal reasoning, still allowing for the full implementatory benefits. This goes beyond the model-theoretical notions of hidden sorts and symbols. The theory developed can probably be used and implemented by modifying existing proof methods (based on the concepts of proof by consistency and inductive completion).  Finally an extension of Knuth-Bendix completion is presented as a step towards mechanical generation of constructive proofs in the input theory given to the process. Constructive proofs may give deeper insight into a theory. It also turns out that the extension of Knuth-Bendix completion under certain circumstances can be used in establishing consistency in conjunction with indirect specification (part 2). The extension also turns out to be instrumental in proving that hiding of auxiliary functions and artificial inconsistency may be implemented in proof methods based on inductive completion.},
	author = {Jo Erskine {Hannay}}
}
@mastersthesis {Hannay.1995.1,
	title = {Generalisert Algebraisk Spesifikasjon Med En Anvendelse P{\r a} Indirekte Semantikk},
	year = {1995},
	publisher = {Department of Informatics, University of Oslo},
	type = {masters},
	abstract = {The thesis deals with topics in the field of formal abstract specification and verification of programs, particularly within the framework of algebraic methods restricted to equational logic. The discussion views equational specification in conjunctiom with the closely related topic of simple term-rewriting as more concrete and nearer to implementation than specification in general. Therefore initial and final models are considered, rather than complete model-classes. Also mostly constructive, {\textquoteleft}{\textquoteleft}executable{\textquoteright}{\textquoteright} equational specifications are considered giving an inclination towards viewing specifications as abstract programs. The thesis is structured around the following two topics:  {\textendash}{\textendash}  1. The method of equational algebraic specification is generalized in a manner allowing a certain mode of modular specification. The generalization has two forms accommodating initial and final algebra semantics respectively. The generalization is modular in the sense that complex specifications may be constructed stepwise from simpler kernel specifications. Each construction step provides the choice of initial or final generalized form. The resulting complex equational specifications differ from general case predicate logic hierarchical specifications, in the sense that the polarization into initial and final semantics gives inhomogeneous specifications; a complex specification cannot in principle be viewed as a non-hierarchical simple specification.  A concept of consistency relative to kernel specifications is developed and it is possible to reason about such relative consistency without specific knowledge about the kernel specification.  The possibility of analogously constructing complex formal-mechanical proof methods from simpler methods is briefly discussed.  2. A variant of (initial) algebraic specification called indirect specification is developed. Motivation and a foundation for indirect specification is given by syntactical specifier functions; i.e. terms are to be understood identical iff the their values under some given function (with syntactical codomain) are identical. A special and interesting case of syntactical specifier functions are functions giving canonical representatives in some well-defined sense.  Besides providing the specification language with additional means of specifying equality, indirect specification expands the class of congruences (over ground terms) decidable by simple term-rewriting. In addition, although the class of initial congruences equationally specifiable is identical to the class specifiable by syntactical specifier functions with canonical representatives as codomain, there exist syntactical specifier functions which do not give canonical representatives. (Such a function may for instance give a representative in some other class than in which the argument belongs.) It may therefore be the fact that the class of congruences indirectly specifiable is greater than the class equationally specifiable. Characteristically, indirect specification represents a more operational mode of specification than does usual algebraic specification.  Indirect specification introduces further inhomogeneousity to equational algebraic specification. The modularity of the generalized specification strategy discussed in part 1 is used to encapsulate indirect specification in the hierarchical framework.  It is then shown that a trivial augmentation of indirect specification can be reduced to standard non-generalized initial or final semantics specification; under certain interesting circumstances. It is also shown that Knuth-Bendix completion of an augmentation of indirect specification, if successful and under a certain congruency condition, will give an equivalent standard algebraic specification. Viewing equational specifications as abstract programs, this might be seen as a program transformation. (A more applicable strategy for automatic transformation in this sense is sketched just as an idea without any sort of further proof.)  {\textendash}{\textendash}  The overall structure sees part 1 primarily as giving a framework for the discussion in part 2.  The subject of consistency permeates the entire discussion. Consistency is viewed as related to basic ideas of some semantic domain which are expressed as untamperable presuppositions in the act of specifying. For example, the basic idea that a mathematical proposition cannot simultaneously be true and false is presupposed in predicate calculus by predefined interpretations of the symbols true and false, and by referring to a set of axioms as inconsistent iff the predicate true = false is deducible from the axioms. The concept of (in)consistency seen always relatively to such presuppositions allows a generalized notion of consistency in hierarchical specification relative to kernel specifications as discussed in part 1. (This also generalizes Guttag{\textquoteright}s notion of consistency.) Some simple methods for detecting (generalized) inconsistency and for establishing consistency are briefly presented.  The notion of artificial inconsistency is introduced, as inconsistency due to auxiliary functions; i.e. functions helpful or necessary during constructive definition or implementation, but otherwise really not belonging to the semantical objects under implementation. Formalisms and results are developed showing that auxiliary functions and hence artificial inconsistency can be hidden from the formal reasoning, still allowing for the full implementatory benefits. This goes beyond the model-theoretical notions of hidden sorts and symbols. The theory developed can probably be used and implemented by modifying existing proof methods (based on the concepts of proof by consistency and inductive completion).  Finally an extension of Knuth-Bendix completion is presented as a step towards mechanical generation of constructive proofs in the input theory given to the process. Constructive proofs may give deeper insight into a theory. It also turns out that the extension of Knuth-Bendix completion under certain circumstances can be used in establishing consistency in conjunction with indirect specification (part 2). The extension also turns out to be instrumental in proving that hiding of auxiliary functions and artificial inconsistency may be implemented in proof methods based on inductive completion.},
	author = {Jo Erskine {Hannay}}
}
