@article {Simula.simula.2514,
	title = {Applying UML/MARTE on Industrial Projects: Challenges, Experiences, and Guidelines},
	journal = {Software and Systems Modeling},
	number = {1619-1366},
	year = {2014},
	month = {March 2014},
	pages = {1-19},
	publisher = {Springer Publishing Company},
	abstract = {MARTE (Modeling and Analysis of Real-Time and Embedded Systems) is a UML profile, which has been developed to model concepts specific to Real-Time and Embedded Systems (RTES). In the last five years, we have applied UML/MARTE to three distinct industrial problems in three industry sectors: architecture modeling and configuration of large-scale and highly configurable integrated control systems, model-based robustness testing of communication-intensive systems, and model-based environment simulator generation of large-scale RTES for testing. In this paper, we report on our experience of solving these problems by applying UML/MARTE on four industrial case studies. We highlight the challenges we faced with respect to the industrial adoption of MARTE. Based on our combined experience, we derive a framework to guide practitioners for future applications of UML/MARTE in an industrial context. The framework provides a set of detailed guidelines that help reduce the gap between the modeling notations and real-world industrial application needs.},
	author = {Muhammad Zohaib {Iqbal} and Ali, Shaukat and Yue, Tao and Briand, Lionel Claude}
}
@article {Simula.simula.2598,
	title = {Architecture-Level Configuration of Large-Scale Embedded Software Systems},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {23},
	number = {3},
	year = {2014},
	publisher = {ACM},
	abstract = {Configuration in the domain of integrated control systems (ICS) is largely manual, laborious, and error-prone. In this paper, we propose a model-based configuration approach that provides automation support for reducing configuration effort and the likelihood of configuration errors in the ICS domain. We ground our approach on component-based specifications of ICS families. We then develop a configuration algorithm using constraint satisfaction techniques over finite domains to generate products that are consistent with respect to their ICS family specifications. We reason about the termination and consistency of our configuration algorithm analytically. We evaluate the effectiveness of our configuration approach by applying it to a real subsea oil production system. Specifically, we have rebuilt a number of existing verified product configurations of our industry partner. Our experience shows that our approach can automatically infer up to 50\% of the configuration decisions, and reduces the complexity of making configuration decisions.},
	author = {Behjati, Razieh and Nejati, Shiva and Briand, Lionel Claude}
}
@article {IST2014,
	title = {An Extended Systematic Literature Review on Provision of Evidence for Safety Certification},
	journal = {Information and Software Technology},
	volume = {56},
	number = {7},
	year = {2014},
	pages = {689-717},
	publisher = {Elsevier},
	abstract = {Context: Critical systems in domains such as aviation, railway, and automotive are often subject to a formal process of safety certification. The goal of this process is to ensure that these systems will operate safely without posing undue risks to the user, the public, or the environment. Safety is typically ensured via complying with safety standards. Demonstrating compliance to these standards involves providing evidence to show that the safety criteria of the standards are met. Objective: In order to cope with the complexity of large critical systems and subsequently the plethora of evidence information required for achieving compliance, safety professionals need in-depth knowledge to assist them in classifying different types of evidence, and in structuring and assessing the evidence. This paper is a step towards developing such a body of knowledge that is derived from a large-scale empirically rigorous literature review. Method: We use a Systematic Literature Review (SLR) as the basis for our work. The SLR builds on 218 peer-reviewed studies, selected through a multi-stage process, from 4,963 studies published between 1990 and 2012. Results: We develop a taxonomy that classifies the information and artefacts considered as evidence for safety. We review the existing techniques for safety evidence structuring and assessment, and further study the relevant challenges that have been the target of investigation in the academic literature. We analyse commonalities in the results among different application domains and discuss implications of the results for both research and practice. Conclusion: The paper is, to our knowledge, the largest existing study on the topic of safety evidence. The results are particularly relevant to practitioners seeking a better grasp on evidence requirements as well as to researchers in the area of system safety. As a major finding of the review, the results strongly suggest the need for more practitioner-oriented and industry-driven empirical studies in the area of safety certification.},
	author = {Nair, Sunil and Jose Luis {de la Vara} and Sabetzadeh, Mehrdad and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.2848,
	title = {Model-Based Testing of Obligations},
	journal = {The 14th International Conference on Quality Software (QSIC)},
	year = {2014},
	publisher = {IEEE},
	type = {Conference},
	address = {Allen, TX, USA},
	abstract = {Obligations are mandatory actions that users must perform, addressing access control requirements. To ensure that such obligations are implemented correctly, an automated and systematic testing approach is often recommended. One such approach is Model-Based Testing (MBT) that allows defining cost-effective testing strategies to support rigorous testing via automation. In this paper, we present MBT for obligations by extending the Unified Modeling Language (UML) via a profile called the Obligations Profile. Based on the profile, we define a modeling methodology based on the concepts of Obligations Class Diagrams (OCDs) and Obligations State Machines (OSMs), which are standard UML Class Diagrams and UML State Machines with stereotypes from the Obligations Profile. Our methodology, using OCDs and OSMs, is automatically enforced by the validation of constraints defined in the profile. To assess the completeness and applicability of the profile and methodology, we modeled 47 obligations from four different systems. The results of our experiment show that we successfully modeled all the obligations and used 75\% of the stereotypes that we defined in the profile. In addition, using OCDs and OSMs, we automatically generate executable test cases using a standard state machine structural coverage criterion and common test data generation strategies. The effectiveness of generated test cases is assessed using mutation analysis on two systems, using mutation operators specifically designed for obligation faults. Test case executions killed 75\% of the mutants and a careful analysis further suggests that more sophisticated testing strategies must be defined to further improve testing effectiveness.},
	keywords = {Conference},
	author = {Rubab, Iram and Ali, Shaukat and Briand, Lionel Claude and Le Traon, Yves},
	editor = {T. H. {Tse} and Wong, Eric and McMillin, Bruce}
}
@inproceedings {dialesio2014worstcase,
	title = {Worst-case Scheduling of Software Tasks {\textendash} A Constraint Optimization Model to Support Performance Testing},
	journal = {The 20th International Conference on Principles and Practice of Constraint Programming (CP 2014)},
	year = {2014},
	type = {Conference},
	abstract = {Real-Time Embedded Systems (RTES) in safety-critical domains, such as maritime and energy, must satisfy strict performance requirements to be deemed safe. Therefore, such systems have to be thoroughly tested to ensure their correct behavior even under the worst operating conditions. In this paper, we address the need of deriving worst case scenarios with respect to three common performance requirements, namely task deadlines, response time, and CPU usage. Specifically, we investigate whether this worst-case analysis can be effectively re-expressed as a Constrained Optimization Problem (COP) over the space of possible inputs to the system. Solving this problem means finding the sets of inputs that maximize the chance to violate performance requirements at runtime. Such inputs can in turn be used to test if the target RTES meets the expected performance even in the worst case. We develop an OPL model for IBM ILOG CP Optimizer that implements a task priority-based preemptive scheduling, and apply it to a case study from the maritime and energy domain. Our validation shows that (1) the input to our model can be provided with reasonable effort in an industrial setting, and (2) the COP effectively identifies test cases that maximize deadline misses, response time, and CPU usage.},
	doi = {10.1007/978-3-319-10428-7_58},
	author = {Di Alesio, Stefano and Nejati, Shiva and Briand, Lionel Claude and Gotlieb, Arnaud}
}
@article {Simula.simula.116,
	title = {Achieving Scalable Model-Based Testing Through Test Case Diversity},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {22},
	number = {1},
	year = {2013},
	publisher = {ACM},
	abstract = {The increase in size and complexity of modern software systems entails scalable, systematic, and automated testing approaches. Model-based testing (MBT), as a systematic and automated test case generation technique, is being successfully applied to verify industrial-scale systems and is supported by commercial tools. However, scalability is still an open issue for large systems as in practice there are limits to the amount of testing that can be performed in industrial contexts. Even with standard coverage criteria, the resulting test suites generated by MBT techniques can be very large and expensive to execute, especially for system level testing on real deployment platforms and network facilities. Therefore, a scalable MBT technique should be flexible regarding the size of the generated test suites and should be easily accommodated to fit resource and time constraints. Our approach is to select a subset of the generated test suite in such a way that it can be realistically executed and analyzed within the time and resource constraints, while preserving the fault revealing power of the original test suite to a maximum extent. In this paper, to address this problem, we introduce a family of similarity-based test case selection techniques. We evaluate 320 different similarity-based selection techniques and then compare the effectiveness of the best similarity-based selection technique with other common selection techniques in the literature. The results based on two industrial case studies show significant benefits and a large improvement in performance when using a similarity-based approach. We complement these analyses with further studies on the scalability of the technique and the effects of failure rate on its effectiveness. We also propose a method to identify optimal tradeoffs between the number of test cases to run and fault detection.},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1654,
	title = {Assessing Quality and Effort of Applying Aspect State Machines for Robustness Testing: a Controlled Experiment},
	journal = {International Conference on Software Testing, Verification and Validation (ICST)},
	year = {2013},
	month = {March},
	publisher = {IEEE},
	type = {Conference},
	address = {Luembourg},
	abstract = {Aspect-Oriented Modeling (AOM) has been the subject of intense research over the last decade and aims to provide numerous benefits to modeling, such as enhanced modularization, easier evolution, higher quality as well as reduced modeling effort. However, these benefits can only be obtained at the cost of learning and applying new modeling approaches. Studying their applicability is therefore important to assess whether they are worth using in practice. In this paper, we report a controlled experiment to assess the applicability of AOM, focusing on a recently published UML profile (AspectSM). This profile was originally designed to support model-based robustness testing in an industrial context but is applicable to the behavioral modeling of other crosscutting concerns. This experiment assesses the applicability of AspectSM from two aspects: the quality of derived state machines and the effort required to build them. With AspectSM, a crosscutting behavior is modeled using an {\textquotedblleft}aspect state machine{\textquotedblright}. The applicability of aspect state machines is evaluated by comparing them with standard UML state machines that directly model the entire system behavior, including crosscutting concerns. The quality of both aspect and standard UML state machines derived by subjects is measured by comparing them against predefined reference state machines. Results show that aspect state machines derived with AspectSM are significantly more complete and correct though AspectSM took significantly more time than the standard approach.},
	keywords = {Conference},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude},
	editor = {Le Traon, Yves}
}
@article {Simula.simula.1922,
	title = {Environment Modeling and Simulation for Automated Testing of Soft Real-Time Embedded Software},
	journal = {Software and Systems Modeling},
	number = {online},
	year = {2013},
	publisher = {Springer-Verlag},
	abstract = {Given the challenges of testing at the system level, only a fully automated approach can really scale up to industrial real-time embedded systems (RTES). Our goal is to provide a practical approach to the model-based testing of RTES by allowing system testers, who are often not familiar with the system{\textquoteright}s design but are application domain experts, to model the system environment in such a way as to enable its black-box test automation. Environment models can support the automation of three tasks: the code generation of an environment simulator to enable testing on the development platform or without involving actual hardware, the selection of test cases, and the evaluation of their expected results (oracles). From a practical standpoint{\r u}le{1em}{1pt}}and such considerations are crucial for industrial adoption{\r u}le{1em}{1pt}}environment modeling should be based on modeling standards (1) that are at an adequate level of abstraction, (2) that software engineers are familiar with, and (3) that are well supported by commercial or open source tools. In this paper, we propose a precise environment modeling methodology fitting these requirements and discuss how these models can be used to generate environment simulators. The environment models are expressed using UML/MARTE and OCL, which are international standards for real-time systems and constraint modeling. The presented techniques are evaluated on a set of three artificial problems and on two industrial RTES.},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@article {Simula.simula.812,
	title = {Facilitating the Transition From Use Case Models to Analysis Models: Approach and Experiments},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {22},
	number = {1},
	year = {2013},
	publisher = {ACM},
	abstract = {Use case modeling, including use case diagrams and use case specifications (UCSs), is commonly applied to structure and document requirements. UCSs are usually structured but unrestricted textual documents complying with a certain use case template. However, because Use Case Models (UCMods) remain essentially textual, ambiguity is inevitably introduced. In this paper, we propose a use case modeling approach, called Restricted Use Case Modeling (RUCM), which is composed of a set of well-defined restriction rules and a modified use case template. The goal is two-fold: (1) restrict the way users can document UCSs in order to reduce ambiguity and (2) facilitate the manual derivation of initial analysis models which, when using the Unified Modeling Language (UML), are typically composed of class diagrams, sequence diagrams, and possibly other types of diagrams. Though the proposed restriction rules and template are based on a clear rationale, two main questions need to be investigated. First, do users find them too restrictive or impractical in certain situations? In other words, can users express the same requirements with RUCM as with unrestricted use cases? Second, do the rules and template have a positive, significant impact on the quality of the constructed analysis models? To investigate these questions, we performed and report on two controlled experiments, which evaluate the restriction rules and use case template in terms of (1) whether they are easy to apply while developing UCMods and facilitate the understanding of UCSs, and (2) whether they help users manually derive higher quality analysis models than what can be generated when they are not used, in terms of correctness, completeness, and redundancy. This paper reports on the first controlled experiments that evaluate the applicability of restriction rules on use case modeling and their impact on the quality of analysis models. The measures we have defined to characterize restriction rules and the quality of analysis class and sequence diagrams can be reused to perform similar experiments in the future, either with RUCM or other approaches. Results show that, the restriction rules are overall easy to apply and that RUCM results into significant improvements over traditional approaches (i.e., with standard templates, without restrictions) in terms of class correctness and class diagram completeness, message correctness and sequence diagram completeness, and understandability of UCSs.},
	author = {Yue, Tao and Briand, Lionel Claude and Labiche, Yvan}
}
@article {TestData,
	title = {Generating Test Data From OCL Constraints With Search Techniques},
	journal = {IEEE Transactions on Software Engineering},
	volume = {39},
	number = {10},
	year = {2013},
	publisher = {IEEE},
	abstract = {Model-based testing (MBT) aims at automated, scalable, and systematic testing solutions for complex industrial software systems. To increase chances of adoption in industrial contexts, software systems can be modeled using well-established standards such as the Unified Modeling Language (UML) and the Object Constraint Language (OCL). Given that test data generation is one of the major challenges to automate MBT, we focus on test data generation from OCL constraints in this paper. This endeavor is all the more challenging given the numerous OCL constructs and operations that are designed to facilite the definition of constraints. Though search-based software testing has been applied to test data generation for white-box testing (e.g., branch coverage), its application to the MBT of industrial software systems has been limited. In this paper, we propose a set of search heuristics targeted to OCL constraints to guide test data generation and automate MBT in industrial applications. We evaluate these heuristics for three search algorithms: Genetic Algorithm, (1+1) Evolutionary Algorithm, and Alternating Variable Method. We empirically evaluate our heuristics using complex artificial problems, followed by empirical analyses of the feasibility of our approach on one industrial system in the context of robustness testing. Our approach is also compared with the most widely referenced OCL solver (UMLtoCSP) in the literature and shows to be significantly more efficient.},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@article {Simula.simula.1556,
	title = {SimPL: a Product-Line Modeling Methodology for Families of Integrated Control Systems},
	journal = {Information and Software Technology},
	volume = {55},
	number = {3},
	year = {2013},
	pages = {607-629},
	publisher = {.},
	abstract = {Context: Integrated control systems (ICSs) are heterogeneous systems where software and hardware components are integrated to control and monitor physical devices and processes. A family of ICSs share the same software code base, which is configured differently for each product to form a unique installation. Due to the complexity of ICSs and inadequate automation support, product configuration in this context is typically error-prone and costly. Objective: As a first step to overcome these challenges, we propose a UML-based product-line modeling methodology that provides a foundation for semi-automated product configuration in the specific context of ICSs. Method: We performed a comprehensive domain analysis to identify characteristics of ICS families, and their configuration challenges. Based on this, we formulated the characteristics of an adequate configuration solution, and derived from them a set of modeling requirements for a model-based solution to configuration. The SimPL methodology is proposed to fulfill these requirements. Results: To evaluate the ability of SimPL to fulfill the modeling requirements, we applied it to a large-scale industrial case study. Our experience with the case study shows that SimPL is adequate to provide a model of the product family that meets the modeling requirements. Further evaluation is still required to assess the applicability and scalability of SimPL in practice. Doing this requires conducting field studies with human subjects and is left for future work. Conclusion: We conclude that configuration in ICSs requires better automation support, and UML-based approaches to product family modeling can be tailored to provide the required foundation.},
	author = {Behjati, Razieh and Yue, Tao and Briand, Lionel Claude and Selic, Bran}
}
@article {Simula.simula.1992,
	title = {Test Case Selection for Black-Box Regression Testing of Database Applications},
	journal = {Information and Software Technology},
	year = {2013},
	month = {August},
	publisher = {Elsevier},
	abstract = {Context: This paper presents an approach for selecting regression test cases in the context of large-scale, database applications. We focus on a black-box (specification-based) approach, relying on classification tree models to model the input domain of the system under test (SUT), in order to obtain a more practical and scalable solution. We perform an industrial case study where the SUT is a large database application in Norway{\textquoteright}s tax department. Objective: We investigate the use of similarity-based test case selection for supporting black box regression testing of database applications. We have developed a practical approach and tool (DART) for functional black-box regression testing of database applications. In order to make the regression test approach scalable for large database applications, we needed a test case selection strategy that reduces the test execution costs and analysis effort. We used classification tree models to partition the input domain of the SUT in order to then select test cases. Rather than selecting test cases at random from each partition, we incorporated a similarity-based test case selection, hypothesizing that it would yield a higher fault detection rate. Method: An experiment was conducted to determine which similarity-based selection algorithm was the most suitable in selecting test cases in large regression test suites, and whether similarity-based selection was a worthwhile and practical alternative to simpler solutions. Results: The results show that combining similarity measurement with partition-based test case selection, by using similarity-based test case selection within each partition, can provide improved fault detection rates over simpler solutions when specific conditions are met regarding the partitions. Conclusions: Under the conditions present in the experiment the improvements were marginal. However, a detailed analysis concludes that the similarity-based selection strategy should be applied when a large number of test cases are contained in each partition and there is significant variability within partitions. If these conditions are not present, incorporating similarity measures is not worthwhile, since the gain is negligible over a random selection within each partition.},
	author = {Rogstad, Erik and Briand, Lionel Claude and Torkar, Richard}
}
@article {Simula.approve.88,
	title = {The Value of Design Rationale Information},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {22},
	year = {2013},
	publisher = {ACM},
	abstract = {A complete and detailed Design Rationale Documentation (DRD) could support many software development activities, such as an impact analysis or a major redesign. However, a full DRD is too onerous for systematic industrial use as it is not cost-effective to write, maintain, or read. The key idea investigated in this paper is that DRD should be adopted only to the extent required to support activities particularly difficult to enact or in need of significant improvement in a particular context. The aim of this paper is to empirically investigate the possibility to customize the DRD by documenting only the information that will probably be required for executing an activity. This customization strategy relies on the hypothesis that the value of a DRD information item depends on its category (e.g. Assumptions, Related requirements, etc.) and on the activity it is meant to support. This hypothesis is investigated through two controlled experiments involving a total of seventy-five post-graduate master students as experimental subjects. The value of DRD information was shown to significantly depend on its category and, within the same category, on the activity it supports. Furthermore, our results show that, on average among activities, documenting only the information that has been required at least half of the time (i.e., the information that will probably be required in the future) generates a customized DRD containing about half the information of a full documentation. Such a significant reduction of information to document is expected to mitigate the effects of inhibitors that are currently preventing practitioners from documenting the rationale of their design decisions.},
	author = {Falessi, Davide and Briand, Lionel Claude and Cantone, Giovanni and Capilla, Rafael and Kruchten, Philippe}
}
@misc {Simula.simula.2038,
	title = {Automatic Derivation of Test Execution Plans From a Video Conferencing System Product Line Modelling},
	year = {2012},
	address = {.},
	author = {Wang, Shuai and Gotlieb, Arnaud and Liaaen, Marius and Briand, Lionel Claude}
}
@techreport {Simula.simula.1298,
	title = {Combining Search-Based and Adaptive Random Testing for Black-Box System Testing of Real-Time Embedded Systems},
	number = {2012-12},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Effective system testing of real-time embedded systems (RTES) requires a fully automated approach. One such black-box system testing approach is to use environment models to automatically generate test cases and test oracles along with an environment simulator to enable early testing of RTES. In this paper, we propose a hybrid strategy, which combines (1+1) Evolutionary Algorithm (EA) and Adaptive Random Testing (ART), to improve the overall performance of system testing that is obtained when using each single strategy in isolation. An empirical study is carried out on a number of artificial problems and one industrial case study. The novel strategy shows significant overall improvement in terms of fault detection compared to individual performances of both (1+1) EA and ART.},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1474,
	title = {Combining Search-Based and Adaptive Random Testing Strategies for Environment Model-Based Testing of Real-Time Embedded Systems},
	journal = {Symposium on Search Based Software Engineering (SSBSE)},
	year = {2012},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Effective system testing of real-time embedded systems (RTES) requires a fully automated approach. One such black-box system testing approach is to use environment models to automatically generate test cases and test oracles along with an environment simulator to enable early testing of RTES. In this paper, we propose a hybrid strategy, which combines (1+1) Evolutionary Algorithm (EA) and Adaptive Random Testing (ART), to improve the overall performance of system testing that is obtained when using each single strategy in isolation. An empirical study is carried out on a number of artificial problems and one industrial case study. The novel strategy shows significant overall improvement in terms of fault detection compared to individual performances of both (1+1) EA and ART.},
	keywords = {Conference},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude},
	editor = {Fraser, Gordon and Susi, Angelo and Jerffeson Teixeira {de Souza}}
}
@article {Simula.simula.760,
	title = {Does Aspect-Oriented Modeling Help Improve the Readability of UML State Machines?},
	journal = {Software and Systems Modeling},
	number = {DOI 10.1007/s10270-012-0293-5},
	year = {2012},
	publisher = {Springer Berlin Heidelberg},
	abstract = {Aspect-oriented Modeling (AOM) is a relatively recent and very active field of research, which application has however been limited in practice. AOM is assumed to yield several potential benefits such as enhanced modularization, easier evolution, increased reusability, and improved readability of models, as well as reduced modeling effort. However, credible, solid empirical evidence of such benefits is lacking. We evaluate the {\textquotedblleft}readability{\textquotedblright} of state machines when modeling crosscutting behavior using AOM and more specifically AspectSM, a UML profile extending the UML state machine notation to provide mechanisms to define aspects using state machines. With AspectSM, crosscutting behavior is modeled using so-called {\textquotedblleft}aspect state machines{\textquotedblright}. Their readability is compared with that of system state machines directly modeling crosscutting and standard behavior together. A controlled experiment was conducted with 25 trained, graduate students to achieve the above objective. Readability is defined based on the identification of defects seeded in state machines and the score obtained when answering a comprehension questionnaire about the system behavior. We use two baselines of comparisons-standard UML state machines without hierarchical features (flat state machines) and standard state machines with hierarchical features (hierarchical state machines)-in order to assess the role of hierarchies in the benefits brought by AspectSM. The results showed that defect identification with AspectSM is significantly better than that with both flat and hierarchical state machines. In terms of the comprehension questionnaire, the AspectSM scores are better than flat state machines, but worse than hierarchical state machines. Regarding the effort for inspecting the state machines, no significant difference could be observed. Conclusion: Results of the experiment suggest using hierarchical features of UML state machines for crosscutting concerns where this is possible and resort otherwise to aspect state machines or use them together for better readability.},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1274,
	title = {Empirical Investigation of Search Algorithms for Environment Model-Based Testing of Real-Time Embedded Software},
	journal = {International Symposium on Software Testing and Analysis (ISSTA), ACM},
	year = {2012},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {System testing of real-time embedded systems (RTES) is a challenging task and only a fully automated testing approach can scale up to the testing requirements of industrial RTES. One such approach, which offers the advantage for testing teams to be black-box, is to use environment models to automatically generate test cases and oracles and an environment simulator to enable earlier and more practical testing. In this paper, we propose novel heuristics for search-based, RTES system testing which are based on these environment models. We evaluate the fault detection effectiveness of two search-based algorithms, i.e., Genetic Algorithms and (1+1) Evolutionary Algorithm, when using these novel heuristics and their combinations. Preliminary experiments on 13 carefully selected, non-trivial artificial problems, show that, under certain conditions, these novel heuristics are effective at bringing the environment into a state exhibiting a system fault. The heuristic combination that showed the best overall performance on the artificial problems was applied on an industrial case study where it showed consistent results.},
	keywords = {Conference},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1472,
	title = {Experiences of Applying UML/MARTE on Three Industrial Projects},
	journal = {ACM/IEEE International Conference on Model Driven Engineering Languages and Systems (MODELS)},
	volume = {7590},
	year = {2012},
	pages = {642-658},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {MARTE (Modeling and Analysis of Real-Time and Embedded Systems) is a UML profile, which has been developed to model concepts specific to Real-Time and Embedded Systems (RTES). In previous years, we have applied UML/MARTE to three distinct industrial problems in various industry sectors: architecture modeling and configuration of large-scale and highly configurable integrated control systems, model-based robustness testing of communication-intensive systems, and model-based environment simulator generation of large-scale RTES for testing. In this paper, we report on our experiences of solving these problems by applying UML/MARTE on four industrial case studies. Based on our common experiences, we derive a framework to help practitioners for future applications of UML/MARTE. The framework provides a set of detailed guidelines on how to apply MARTE in industrial contexts and will help reduce the gap between the modeling standards and industrial needs.},
	doi = {10.1007/978-3-642-33666-9\_41},
	author = {Muhammad Zohaib {Iqbal} and Ali, Shaukat and Yue, Tao and Briand, Lionel Claude},
	editor = {Robert B. {France} and Kazmeier, J{\"u}rgen and Breu, Ruth and Atkinson, Colin}
}
@techreport {Simula.simula.1242,
	title = {Experiences With Model-Based Product Line Engineering for Developing a Family of Integrated Control Systems: an Industrial Case Study},
	number = {2012-06},
	year = {2012},
	month = {March},
	publisher = {Simula Research Laboratory},
	abstract = {Integrated Control Systems (ICSs) are often large-scale and highly configurable software-intensive systems-of-systems, with software and hardware components integrated to control and monitor physical devices and processes (e.g., oil and gas production platforms). Employing Product Line Engineering (PLE) is believed to bring potential benefits with respect to reduced cost, higher productivity, higher quality, and faster time-to-market. However, relatively few industrial field studies are reported regarding the application of PLE to develop large-scale systems, and more specifically ICSs. In this paper, we report about our experiences and insights gained from investigating the application of model-based PLE at FMC Technologies, a company developing subsea production systems (typical ICSs) to manage the exploitation of oil and gas production fields. We first discuss the benefits and challenges of applying systematic domain analysis-the first major step of PLE, report on the key domain analysis results and then assess the benefits and challenges of employing model-based PLE at FMC, as a means to improve the quality of their products and the productivity of their product development process.},
	author = {Yue, Tao and Briand, Lionel Claude and Selic, Bran and Gan, Qitao}
}
@article {Simula.simula.186,
	title = {Formal Analysis of the Probability of Interaction Fault Detection Using Random Testing},
	journal = {IEEE Transactions on Software Engineering},
	volume = {38},
	number = {5},
	year = {2012},
	pages = {1088-1099},
	publisher = {IEEE},
	abstract = {Modern systems are becoming highly configurable to satisfy the varying needs of customers and users. Software product lines are hence becoming a common trend in software development to reduce cost by enabling systematic, large scale reuse. However, high levels of configurability entail new challenges. Some faults might be revealed only if a particular combination of features is selected in the delivered products. But testing all combinations is usually not feasible in practice, due to their extremely large numbers. Combinatorial testing is a technique to generate smaller test suites for which all combinations of t features are guaranteed to be tested. In this paper, we formally prove that, when there are no constraints among the features that can be part of a product, then random testing is much more effective than expected at detecting feature interaction faults when compared with combinatorial testing. More importantly, this trend gets even stronger as the number of features increases and converges towards equal effectiveness. Given that combinatorial testing entails significant computational overhead in the presence of many features, the results suggest that random testing may outperform combinatorial testing in large systems. However, when constraints are present among features, then random testing can fare much worse than combinatorial testing. As a result, we claim that research effort should focus on improving combinatorial testing in the presence of feature constraints in order to have a practical impact.},
	author = {Arcuri, Andrea and Briand, Lionel Claude}
}
@techreport {Simula.simula.14,
	title = {Generating Test Data From OCL Constraints With Search Techniques},
	number = {2010-16},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Model-based testing (MBT) aims at automated, scalable, and systematic testing solutions for complex industrial software systems. To increase chances of adoption in industrial contexts, software systems should be modeled using well-established standards such as the Unified Modeling Language (UML) and Object Constraint Language (OCL). Given that test data generation is one of the major challenges to automate MBT, we focus on test data generation from OCL constraints in this paper. Though search-based software testing has been applied to test data generation for white-box testing (e.g., branch coverage), its application to the MBT of industrial software systems has been limited. In this paper, we propose a set of search heuristics targeted to OCL constraints to guide test data generation and automate MBT in industrial applications. These heuristics are used to develop an efficient OCL solver exclusively based on search. We evaluate these heuristics for search algorithms such as Genetic Algorithms, (1+1) Evolutionary Algorithm and Alternating Variable Method. We empirically evaluate our heuristics using complex artificial problems followed by empirical analyses to evaluate the feasibility of our approach on one industrial system. Our approach is compared with a downloadable, standard OCL solver and shows to be significantly more efficient and more widely applicable.},
	author = {Ali, Shaukat and Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@article {Simula.simula.1564,
	title = {A Hitchhiker{\textquoteright}s Guide to Statistical Tests for Assessing Randomized Algorithms in Software Engineering},
	journal = {Software Testing, Verification and Reliability},
	volume = {24},
	year = {2012},
	month = {05/2014},
	pages = {219-250},
	publisher = {John Wiley \& Sons},
	abstract = {Randomized algorithms are widely used to address many types of software engineering problems, especially in the area of software verification and validation with a strong emphasis on test automation. However, randomized algorithms are affected by chance, and so require the use of appropriate statistical tests to be properly analyzed in a sound manner. This paper features a systematic review regarding recent publications in 2009 and 2010 showing that, overall, empirical analyses involving randomized algorithms in software engineering tend to not properly account for the random nature of these algorithms. Many of the novel techniques presented clearly appear promising, but the lack of soundness in their empirical evaluations casts unfortunate doubts on their actual usefulness. In software engineering, though there are guidelines on how to carry out empirical analyses involving human subjects, those guidelines are not directly and fully applicable to randomized algorithms. Furthermore, many of the text books on statistical analysis are written from the viewpoints of social and natural sciences, which present different challenges from randomized algorithms. To address the questionable overall quality of the empirical analyses reported in the systematic review, this paper provides guidelines on how to carry out and properly analyze randomized algorithms applied to solve software engineering tasks, with a particular focus on software testing which is by far the most frequent application area of randomized algorithms within software engineering.},
	doi = {10.1002/stvr.1486},
	author = {Arcuri, Andrea and Briand, Lionel Claude}
}
@techreport {Simula.simula.1260,
	title = {A Model-Based Approach to the Automated Reuse of Configuration Data Based on Internal Similarities},
	number = {2012-09},
	year = {2012},
	publisher = {Simula Research Laboratory},
	abstract = {Product configuration in families of embedded software systems, such as Integrated Control Systems (ICSs), involves resolving thousands of configurable parameters and is, therefore, time-consuming and error-prone. Typically, these systems consist of highly similar components that need to be configured similarly. For large-scale systems, a considerable portion of the configuration data can be reused, based on such similarities, during the configuration of each individual product. In this paper, we propose a model-based approach to automate the reuse of configuration data based on the similarities within an ICS product. Our approach provides configuration engineers with appropriate means for manipulating the reuse of configuration data, and provides the required formalism for ensuring the consistency of the reused data. Our investigation of a number of product configurations with an industry partner shows that more than 60\% of configuration data can be automatically reused using our similarity-based approach, thereby reducing configuration effort.},
	author = {Behjati, Razieh and Yue, Tao and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1280,
	title = {Model-Based Automated and Guided Configuration of Embedded Software Systems},
	journal = {ECMFA 2012},
	volume = {7349},
	year = {2012},
	pages = {226-243},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Configuring Integrated Control Systems (ICSs) is largely manual, time-consuming and error-prone. In this paper, we propose a model-based configuration approach that interactively guides engineers to configure software embedded in ICSs. Our approach verifies engineers{\textquoteright} decisions at each configuration iteration, and further, automates some of the decisions. We use a constraint solver, SICStus Prolog, to automatically infer configuration decisions and to ensure the consistency of configuration data. We evaluated our approach by applying it to a real subsea oil production system. Specifically, we rebuilt a number of existing verified product configurations of our industry partner. Our experience shows that our approach successfully enforces consistency of configurations, can automatically infer up to 50\% of the configuration decisions, and reduces the complexity of making configuration decisions.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-31491-9\_18},
	author = {Behjati, Razieh and Nejati, Shiva and Yue, Tao and Gotlieb, Arnaud and Briand, Lionel Claude},
	editor = {Vallecillo, Antonio and Tolvanen, Juha-Pekka and Kindler, Ekkart and St{\"o}rrle, Harald and Kolovos, Dimitris}
}
@inproceedings {16146,
	title = {Modeling and Analysis of CPU Usage in Safety-Critical Embedded Systems to Support Stress Testing},
	journal = {The ACM/IEEE 15th International Conference on Model Driven Engineering Languages \& Systems (MODELS 2012)},
	year = {2012},
	type = {Conference},
	abstract = {Software safety certification needs to address non-functional constraints with safety implications, e.g., deadlines, throughput, and CPU and memory usage. In this paper, we focus on CPU usage constraints and provide a framework to support the derivation of test cases that maximize the chances of violating CPU usage requirements. We develop a conceptual model specifying the generic abstractions required for analyzing CPU usage and provide a mapping between these abstractions and UML/MARTE. Using this model, we formulate CPU usage analysis as a constraint optimization problem and provide an implementation of our approach in a state-of-the-art optimization tool. We report an application of our approach to a case study from the maritime and energy domain. Through this case study, we argue that our approach (1) can be applied with a practically reasonable overhead in an industrial setting, and (2) is effective for identifying test cases that maximize CPU usage.},
	doi = {10.1007/978-3-642-33666-9_48},
	author = {Nejati, Shiva and Di Alesio, Stefano and Sabetzadeh, Mehrdad and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1468,
	title = {A Modeling Approach to Support the Similarity-Based Reuse of Configuration Data},
	journal = {Model Driven Engineering Languages and Systems, 15th International Conference, MODELS 2012},
	volume = {7590},
	year = {2012},
	pages = {497-513},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Product configuration in families of embedded software systems, such as Integrated Control Systems (ICSs), involves resolving thousands of configurable parameters and is, therefore, time-consuming and error-prone. Typically, these systems consist of highly similar components that need to be configured similarly. For large-scale systems, a considerable portion of the configuration data can be reused, based on such similarities, during the configuration of each individual product. In this paper, we propose a model-based approach to automate the reuse of configuration data based on the similarities within an ICS product. Our approach provides configuration engineers with appropriate means for manipulating the reuse of configuration data, and provides the required formalism for ensuring the consistency of the reused data. Our investigation of a number of product configurations with an industry partner shows that more than 60\% of configuration data can be automatically reused using our similarity-based approach, thereby reducing configuration effort.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-33666-9\_32},
	author = {Behjati, Razieh and Yue, Tao and Briand, Lionel Claude},
	editor = {France, Robert and Kazmeier, Juergen and Atkinson, Colin and Breu, Ruth}
}
@article {Simula.simula.118,
	title = {Modeling Robustness Behavior Using Aspect-Oriented Modeling to Support Robustness Testing of Industrial Systems},
	journal = {Software and Systems Modeling},
	volume = {11},
	number = {4},
	year = {2012},
	pages = {633-670},
	publisher = {Springer-Verlag},
	abstract = {Model-based robustness testing requires, precise and complete behavioral, robustness modeling, for example using state machines, to support automated test case generation. Robustness behavior is a crosscutting behavior and, if modeled directly, often results in large, complex state machines. These in practice tend to be error-prone and difficult to read and understand. As a result, modeling robustness behavior in this way is not scalable for complex industrial systems. To overcome these problems, Aspect Oriented Modeling (AOM) can be employed to model robustness behavior as aspects in the form of state machines specifically designed to model robustness behavior. In this paper, we present a RobUstness Modeling Methodology (RUMM) that allows modeling robustness behavior as aspects. At the core of RUMM is a UML profile (AspectSM) that allows modeling UML state machine aspects as UML state machines (aspect state machines). Such an approach, relying on a standard and using the target notation as the basis to model the aspects themselves, is expected to make the practical adoption of aspect modeling easier, and thus in our context support robustness modeling for the purpose of automated model-based testing in industry. We discuss the benefits, such as reduced modeling effort and improved readability, achieved when applying RUMM to an industrial case study where we have used the profile to model the crosscutting robustness behavior of a videoconferencing system.},
	author = {Ali, Shaukat and Briand, Lionel Claude and Hemmati, Hadi}
}
@inproceedings {Simula.simula.1228,
	title = {MuTIL: Mutation-Based Statistical Test Inputs Generation for Automatic Fault Localization},
	journal = {Proceedings of the Sixth International Conference on Software Security and Reliability (SERE-12), Washington, USA},
	year = {2012},
	month = {06/2012},
	publisher = {.},
	type = {Conference},
	address = {.},
	abstract = {Automatic Fault Localization (AFL) is a process to locate faults automatically in software programs. Essentially, an AFL method takes as input a set of test cases including failed test cases, and ranks the statements of a program from the most likely to the least likely to contain a fault. As a result, the efficiency of an AFL method depends on the "quality" of the test cases used to rank statements. More specifically, in order to improve the accuracy of their ranking within test budget constraints, we have to ensure that program statements are executed by a reasonably large number of test cases which provide a coverage as uniform as possible of the input domain. This paper proposes μTIL, a new statistical test inputs generation method dedicated to AFL, based on constraint solving and mutation testing. Using mutants where the locations of injected faults are known, μTIL is able to significantly reduce the length of an AFL test suite while retaining its accuracy (i.e., the code size to examine before spotting the fault). In order to address the motivations stated above, the statistical generator objectives are two-fold: 1) each feasible path of the program is activated with the same probability; 2) the subdomain associated to each feasible path is uniformly covered. Using several widely used ranking techniques (i.e., Tarantula, Jaccard, Ochiai), we show on a small but realistic program that a proof-of-concept implementation of μTIL can generate test sets with significantly better fault localization accuracy than both random testing and adaptive random testing. We also show on the same program that using mutation testing enables a 75\% length reduction of the AFL test suite without decrease in accuracy.},
	keywords = {Conference},
	author = {Delahaye, Mickael and Briand, Lionel Claude and Gotlieb, Arnaud and Petit, Matthieu}
}
@article {Simula.approve.50,
	title = {A Precise Method-Method Interaction-Based Cohesion Metric for Object-Oriented Classes},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {21},
	number = {2},
	year = {2012},
	publisher = {ACM},
	abstract = {The building of highly cohesive classes is an important objective in object-oriented design. Class cohesion refers to the relatedness of the class members, and it indicates one important aspect of the class design quality. A meaningful class cohesion metric helps object-oriented software developers detect class design weaknesses and refactor classes accordingly. Several class cohesion metrics have been proposed in the literature. Most of these metrics are applicable based on low-level design information such as attribute references in methods. Some of these metrics capture class cohesion by counting the number of method pairs sharing common attributes. A few metrics measure cohesion more precisely by considering the degree of interaction, through attribute references, between each pair of methods. However, the formulas applied by these metrics to measure the degree of interaction cause the metrics to violate important mathematical properties, thus undermining their construct validity and leading to misleading cohesion measurement. In this paper, we propose a formula that precisely measures the degree of interaction between each pair of methods, and we use it as a basis to introduce a low-level design class cohesion metric (LSCC). We verify that the proposed formula does not cause the metric to violate important mathematical properties. In addition, we provide a mechanism to use this metric as a useful indicator for refactoring weakly cohesive classes, thus showing its usefulness in improving class cohesion. Finally, we empirically validate LSCC. Using four open source software systems and eleven cohesion metrics, we investigate the relationship between LSCC, other cohesion metrics, and fault occurrences in classes. Our results show that LSCC is one of three metrics that explains more accurately the presence of faults in classes. LSCC is the only one among the three metrics to comply with important mathematical properties, and statistical analysis shows it captures a measurement dimension of its own. This suggests that LSCC is a better alternative, when taking into account both theoretical and empirical results, as a measure to guide the refactoring of classes. From a more general standpoint, the results suggest that class quality, as measured in terms of fault occurrences, can be more accurately explained by cohesion metrics that account for the degree of interaction between each pair of methods.},
	author = {Jehad Al {Dallal} and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1476,
	title = {A Product Line Modeling and Configuration Methodology to Support Model-Based Testing: an Industrial Case Study},
	journal = {ACM/IEEE 15th International Conference on Model Driven Engineering Languages \& Systems (MODELS)},
	year = {2012},
	pages = {726-742},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Product Line Engineering (PLE) is expected to enhance quality and productivity, speed up time-to-market and decrease development effort, through reuse-the key mechanism of PLE. In addition, one can also apply PLE to support systematic testing and more specifically model-based testing (MBT) of product lines-the original motivation behind this work. MBT has shown to be cost-effective in many industry sectors but at the expense of building models of the system under test (SUT). However, the modeling effort to support MBT can significantly be reduced if an adequate product line modeling and configuration methodology is followed, which is the main motivation of this paper. The initial motivation for this work emerged while working with MBT for a Video Con-ferencing product line at Cisco Systems, Norway. In this paper, we report on our experience in modeling product family models and various types of behavioral variability in the Saturn product line. We focus on behavioral variability in UML state machines since the Video Conferencing Systems (VCSs) exhibit strong state-based behavior and these models are the main drivers for MBT; however, the approach can be also tailored to other UML diagrams. We also provide a mechanism to specify and configure various types of variability using stereotypes and Aspect-Oriented Modeling (AOM). Results of applying our methodology to the Saturn product line modeling and configuration process show that the effort required for modeling and configuring products of the product line family can be significantly reduced.},
	keywords = {Conference},
	doi = {10.1007/978-3-642-33666-9\_46},
	author = {Ali, Shaukat and Yue, Tao and Briand, Lionel Claude and Walawege, Suneth}
}
@article {Simula.simula.1122,
	title = {A SysML-Based Approach to Traceability Management and Design Slicing in Support of Safety Certification: Framework, Tool Support, and Case Studies},
	journal = {Information and Software Technology},
	volume = {54},
	number = {6, June 2012},
	year = {2012},
	pages = {569-590},
	publisher = {Butterworth-Heinemann/ Elsevier },
	abstract = {Traceability is one of the basic tenets of all safety standards and a key prerequisite for software safety certification. In the current state of practice, there is often a significant traceability gap between safety requirements and software design. Poor traceability, in addition to being a non-compliance issue on its own, makes it difficult to determine whether the design fulfills the safety requirements, mainly because the design aspects related to safety cannot be clearly identified. The goal of this article is to develop a framework for specifying and automatically extracting design aspects relevant to safety requirements. This goal is realized through the combination of two components: (1) A methodology for establishing traceability between safety requirements and design, and (2) an algorithm that can extract for any given safety requirement a minimized fragment (slice) of the design that is sound, and yet easy to understand and inspect. We ground our framework on System Modeling Language (SysML). The framework includes a traceability information model, a methodology to establish traceability, and mechanisms for model slicing based on the recorded traceability information. The framework is implemented in a tool, named SafeSlice. We prove that our slicing algorithm is sound for temporal safety properties, and argue about the completeness of slices based on our practical experience. We report on the lessons learned from applying our approach to two case studies, one benchmark and one industrial case. Both studies indicate that our approach substantially reduces the amount of information that needs to be inspected for ensuring that a given (behavioral) safety requirement is met by the design.},
	author = {Nejati, Shiva and Sabetzadeh, Mehrdad and Falessi, Davide and Briand, Lionel Claude and Thierry, Coq}
}
@inproceedings {14931,
	title = {Testing Deadline Misses for Real-Time Systems Using Constraint Optimization Techniques},
	journal = {The 4th Workshop on Constraints in Software Testing, Verification, and Analysis (CSTVA 2012)},
	year = {2012},
	type = {Workshop},
	abstract = {Safety-critical real-time applications are typically subject to stringent timing constraints which are dictated by the surrounding physical environments. Specifically, tasks in these applications need to finish their execution before given deadlines, otherwise the system is deemed unsafe. It is therefore important to test real-time systems for deadline misses. In this paper, we present a strategy for testing real-time applications that aims at finding test scenarios in which deadline misses become more likely. We identify such test scenarios by searching the possible ways that a set of real-time tasks can be executed according to the scheduling policy of the operating system on which they are running. We formulate this search problem using a constraint optimization model that includes (1) a set of constraints capturing how a given set of tasks with real-time constraints are executed according to a particular scheduling policy, and (2) a cost function that estimates how likely the given tasks are to miss their deadlines. We implement our constraint optimization model in ILOG, apply our model to several examples, and report on the performance results.},
	doi = {10.1109/ICST.2012.171},
	author = {Di Alesio, Stefano and Gotlieb, Arnaud and Nejati, Shiva and Briand, Lionel Claude}
}
@article {Simula.approve.46,
	title = {UML/MARTE Model Analysis Method for Uncovering Scenarios Leading to Starvation and Deadlocks in Concurrent Systems},
	journal = {IEEE Transactions on Software Engineering},
	volume = {38},
	number = {2},
	year = {2012},
	publisher = {IEEE},
	abstract = {Concurrency problems, such as starvation and deadlocks, should be identified early in the design process. As larger, more complex concurrent systems are being developed, this is made increasingly difficult. We propose here a general approach, based on the analysis of specialized design models expressed in the Unified Modeling Language (UML) that uses a specifically designed genetic algorithm to detect concurrency problems. Though the current paper addresses deadlocks and starvation, we will show how the approach can be easily tailored to other concurrency issues. Our main motivations are (1) to devise solutions that are applicable in the context of the UML design of concurrent systems without requiring additional modeling and (2) to use a search technique to achieve scalable automation in terms of concurrency problem detection. To achieve the first objective, we show how all relevant concurrency information is extracted from systems{\textquoteright} UML models that comply with the UML Modeling and Analysis of Real-Time and Embedded Systems (MARTE) profile. For the second objective, a tailored genetic algorithm is used to search for execution sequences exhibiting deadlock or starvation problems. Scalability in terms of problem detection is achieved by showing that the detection rates of our approach are in general high and are not strongly affected by large increases in the size of complex search spaces.},
	author = {Shousha, Marwa and Briand, Lionel Claude and Labiche, Yvan}
}
@techreport {Simula.simula.380,
	title = {An AADL-Based SysML Profile for Architecture Level Systems Engineering: Approach, Metamodels, and Experiments},
	number = {2011-03},
	year = {2011},
	month = {February},
	publisher = {Simula Research Laboratory},
	abstract = {Recent years have seen a proliferation of languages for describing embedded control systems. Some of these languages have emerged from domain-specific frameworks, and some are adaptions or extensions of more general-purpose languages. In this paper, we focus on two widely-used standard languages: the Architecture Analysis and Design Language (AADL) and the Systems Modeling Language (SysML). AADL was born as an avionics-focused domain-specific language and later on has been revised to represent and support a more general category of embedded real-time systems. SysML is an extension of the Unified Modeling Language (UML) intended to support modeling system engineering applications. We propose the ExSAM profile that extends SysML by adding AADL concepts to it with the goal of exploiting the key advantages of both languages in a seamless way. We describe this profile through several examples and compare it with existing alternatives. We have implemented ExSAM using IBM Rational Rhapsody and evaluated its completeness and usefulness through two case studies.},
	author = {Behjati, Razieh and Yue, Tao and Nejati, Shiva and Briand, Lionel Claude and Selic, Bran}
}
@inproceedings {Simula.simula.614,
	title = {Adaptive Random Testing: an Illusion of Effectiveness?},
	journal = {ACM International Conference on Software Testing and Analysis (ISSTA)},
	year = {2011},
	pages = {265-275},
	publisher = {ACM},
	type = {Conference},
	address = {New York, NY, USA},
	abstract = {Adaptive Random Testing (ART) has been proposed as an enhancement to random testing, based on assumptions on how failing test cases are distributed in the input domain. The main assumption is that failing test cases are usually grouped into contiguous regions. Many papers have been published in which ART has been described as an effective alternative to random testing when using the average number of test case executions needed to find a failure (F-measure). But all the work in the literature is based either on simulations or case studies with unreasonably high failure rates. In this paper, we report on the largest empirical analysis of ART in the literature, in which 3727 mutated programs and nearly ten trillion test cases were used. Results show that ART is highly inefficient even on trivial problems when accounting for distance calculations among test cases, to an extent that probably prevents its practical use in most situations. For example, on the infamous Triangle Classification program, random testing finds failures in few milliseconds whereas ART execution time is prohibitive. Even when assuming a small, fixed size test set and looking at the probability of failure (P-measure), ART only fares slightly better than random testing, which is not sufficient to make it applicable in realistic conditions. We provide precise explanations of this phenomenon based on rigorous empirical analyses. For the simpler case of single-dimension input domains, we also perform formal analyses to support our claim that ART is of little use in most situations, unless drastic enhancements are developed. Such analyses help us explain some of the empirical results and identify the components of ART that need to be improved to make it a viable option in practice.},
	isbn = {978-1-4503-0562-4},
	author = {Arcuri, Andrea and Briand, Lionel Claude},
	editor = {Dwyer, Matthew and Tip, Frank}
}
@article {Simula.SE.403,
	title = {Assessing, Comparing, and Combining State Machine-Based Testing and Structural Testing: a Series of Experiments},
	journal = {IEEE Transactions on Software Engineering},
	volume = {37},
	number = {2},
	year = {2011},
	publisher = {IEEE},
	author = {Mouchawrab, Samar and Briand, Lionel Claude and Labiche, Yvan and Di Penta, Massimiliano}
}
@techreport {Simula.simula.368,
	title = {Code Generation From UML/MARTE/OCL Environment Models to Support Automated System Testing of Real-Time Embedded Software},
	number = {2011-04, Version 2},
	year = {2011},
	publisher = {Simula Research Laboratory},
	abstract = {Given the challenges of testing at the system level, only a fully automated approach can really scale up to industrial real-time embedded systems (RTES). Our goal is to provide a practical approach to the model-based testing of RTES by allowing system testers, who are often not familiar with the system\&$\#$39;s design but are application domain experts, to model the system environment in such a way as to enable its black-box test automation. Environment models can support the automation of three tasks: the code generation of an environment simulator to enable testing on the development platform or without involving actual hardware, the selection of test cases, and the evaluation of their expected results (oracles). From a practical standpoint, such environment modeling should be based on modeling standards (1) that are at the right level of abstraction, (2) that software engineers are familiar with, and (3) that are well supported by commercial or open source tools. In this paper, we augment our previous works on environment modeling and test case selection by describing in detail our solution for the code generation of environment simulators from environment models expressed with UML/MARTE and OCL, which are international standards for real-time systems and constraint modeling. The presented techniques are evaluated on a set of three artificial problems and on two industrial RTES.},
	author = {Muhammad Zohaib {Iqbal} and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.1120,
	title = {CRESCO: Construction of Evidence Repositories for Managing Standards Compliance},
	journal = {30th ACM International Conference on Conceptual Modeling (ER2011)},
	year = {2011},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {We describe CRESCO, a tool for Construction of Evidence REpositories for Managing Standards COmpliance. CRESCO draws on Model Driven Engineering (MDE) technologies to generate a database repository schema from the evidence requirements of a given standard, expressed as a UML class diagram. CRESCO in addition generates a web-based user interface for building and manipulating evidence repositories based on the schema. CRESCO is targeted primarily at addressing the tool infrastructure needs for supporting the collection and management of safety evidence data. A systematic treatment of evidence information is a key prerequisite for demonstration of compliance to safety standards, such as IEC 61508, during the safety certification process.},
	isbn = {978-3-642-24574-9},
	author = {Rajwinder Kaur {Panesar-Walawege} and Torbj{\o}rn Skyberg {Knutsen} and Sabetzadeh, Mehrdad and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.672,
	title = {On the Effectiveness of Contracts As Test Oracles in the Detection and Diagnosis of Race Conditions and Deadlocks in Concurrent Object-Oriented Software},
	journal = {ACM/IEEE Int. Symposium on Empirical Software Engineering and Measurement (ESEM)},
	year = {2011},
	pages = {10-19},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Banff, AB},
	abstract = {The idea behind Design by Contract (DbC) is that a method defines a contract stating the requirements a client needs to fulfill to use it, the precondition, and the properties it ensures after its execution, the postcondition. Though there exists ample support for DbC for sequential programs, applying DbC to concurrent programs presents several challenges. We have proposed a solution to these challenges in the context of Java as programming language and the Java Modeling language as specification language. This paper presents our findings when applying our DbC technique on a case study to evaluate the ability of contract-based, runtime assertion checking code at detecting and diagnosing race conditions and deadlocks during system testing. The case study is a highly concurrent industrial system from the telecommunications domain, with actual faults. It is the first work to systematically investigate the impact of contract assertions for the detection of race conditions and deadlocks, along with functional properties, in an industrial system.},
	isbn = {978-1-4577-2203-5},
	author = {Araujo, Wladimir and Briand, Lionel Claude and Labiche, Yvan}
}
@inproceedings {Simula.simula.8,
	title = {Empirical Investigation of the Effects of Test Suite Properties on Similarity-Based Test Case Selection},
	journal = {4th International Conference on Software Testing, Verification and Validation (ICST2011)},
	year = {2011},
	pages = {327-336},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Berlin},
	abstract = {Our experience with applying model-based testing on industrial systems showed that the generated test suites are often too large and costly to execute given project deadlines and the limited resources for system testing on real platforms. In such industrial contexts, it is often the case that only a small subset of test cases can be run. In previous work, we proposed novel test case selection techniques that minimize the similarities among selected test cases and outperforms other selection alternatives. In this paper, our goal is to gain insights into why and under which conditions similarity-based selection techniques, and in particular our approach, can be expected to work. We investigate the properties of test suites with respect to similarities among fault revealing test cases. We thus identify the ideal situation in which a similarity-based selection works best, which is useful for devising more effective similarity functions. We also address the specific situation in which a test suite contains outliers, that is a small group of very different test cases, and show that it decreases the effectiveness of similarity-based selection. We then propose, and successfully evaluate based on two industrial systems, a solution based on rank scaling to alleviate this problem.},
	isbn = {978-1-61284-174-8},
	author = {Hemmati, Hadi and Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.434,
	title = {Extending SysML With AADL Concepts for Comprehensive System Architecture Modeling},
	journal = {7th European Conference on Modelling Foundations and Applications},
	number = {LNCS 6698},
	year = {2011},
	month = {June},
	pages = {236-252},
	publisher = {Springer},
	type = {Conference},
	address = {Berlin Heidelberg},
	abstract = {Recent years have seen a proliferation of languages for describing embedded systems. Some of these languages have emerged from domain-specific frameworks, and some are adaptions or extensions of more general-purpose languages. In this paper, we focus on two widely-used standard languages: the Architecture Analysis and Design Language (AADL) and the Systems Modeling Language (SysML). AADL was born as an avionics-focused domain-specific language and later on has been revised to represent and support a more general category of embedded real-time systems. SysML is an extension of the Unified Modeling Language (UML) intended to support modeling system engineering applications. We propose the ExSAM profile that extends SysML by adding AADL concepts to it with the goal of exploiting the key advantages of both languages in a seamless way. More precisely, by using one language (ExSAM) and any SysML modeling environment, we should be able to both model system engineering concepts and use AADL analysis tools where needed. We describe this profile through several examples and compare it with existing alternatives. We have implemented ExSAM using IBM Rational Rhapsody and evaluated its completeness and usefulness through two case studies.},
	isbn = {978-3-642-21469-1},
	author = {Behjati, Razieh and Yue, Tao and Nejati, Shiva and Briand, Lionel Claude and Selic, Bran}
}
@techreport {Simula.simula.670,
	title = {A Hitchhiker{\textquoteright}s Guide to Statistical Tests for Assessing Randomized Algorithms in Software Engineering},
	number = {2011-13},
	year = {2011},
	publisher = {Simula Research Laboratory},
	abstract = {Randomized algorithms have been used to successfully address many different types of software engineering problems. This type of algorithms entail a significant degree of randomness as part of their logic. Randomized algorithms are useful to address difficult problems where a precise solution cannot be derived in a deterministic way within reasonable time. However, randomized algorithms can produce different results on every run when applied to the same problem instance. It is hence important to assess the effectiveness of randomized algorithms by collecting data from a large enough number of runs. The rigorous use of statistical tests is then essential to provide support to the conclusions derived by analyzing such data. In this paper, we provide a systematic review of the use of randomized algorithms in selected software engineering venues in 2009/2010. Its goal is not to perform a complete survey but to get a representative and up-to-date snapshot of current practice in software engineering research. We show that randomized algorithms are used in a significant percentage of papers but that, in most cases, randomness is not properly accounted for. This casts doubts on the validity of most empirical results assessing randomized algorithms for various applications. There are numerous statistical tests, based on different assumptions, and it is not always clear when and how to use these tests. We hence provide practical guidelines to support empirical research on randomized algorithms in software engineering.},
	author = {Arcuri, Andrea and Briand, Lionel Claude}
}
@inproceedings {Simula.simula.694,
	title = {An Industrial Application of Robustness Testing Using Aspect-Oriented Modeling, UML/MARTE, and Search Algorithms},
	journal = {ACM/IEEE 14th International Conference on Model Driven Engineering Languages and Systems (Models 2011)},
	year = {2011},
	pages = {108-122},
	publisher = {ACM/IEEE},
	type = {Conference},
	address = {.},
	abstract = {Systematic and rigorous robustness testing is very critical for embedded systems, as for example communication and control systems. Robustness testing aims at testing the behavior of a system in the presence of faulty situations in its operating environment (e.g., sensors and actuators). In such situations, the system should gracefully degrade its performance instead of abruptly stopping execution. To systematically perform robustness testing, one option is to resort to model-based robustness testing (MBRT), based for example on UML/MARTE models. However, to successfully apply MBRT in industrial contexts, new technology needs to be developed to scale to the complexity of real industrial systems. In this paper, we report on our experience of performing MBRT on video conferencing systems developed by Cisco Systems, Norway. We discuss how we developed and integrated various techniques and tools to achieve a fully automated MBRT that is able to detect previously uncaught software faults in those systems. We provide an overview of how we achieved scalable modeling of robustness behavior using aspect-oriented modeling, test case generation using search algorithms, and environment emulation for test case execution. Our experience and lessons learned identify challenges and open research questions for the industrial application of MBRT.},
	author = {Ali, Shaukat and Briand, Lionel Claude and Arcuri, Andrea and Walawege, Suneth},
	editor = {Whittle, Jon and Clark, Tony and K{\"u}hne, Thomas}
}
@article {Simula.simula.182,
	title = {Planning for Safety Evidence Collection: a Tool-Supported Approach Based on Modeling of Standards Compliance Information},
	journal = {IEEE Software},
	volume = {pp},
	number = {99},
	year = {2011},
	publisher = {IEEE},
	abstract = {Safety-critical software-dependent systems such as those found in the avionics, automotive, maritime, and energy domains often need to be certified by a licensing or regulatory body based on one or more safety standards. Safety standards often do not specify the details of the evidence that needs to be collected for the certification of a particular system because these standards need to remain generic and applicable to a wide variety of systems and domains. Without an upfront agreement between the system supplier and the certifier about the details of the evidence that needs to be collected, there will invariably be important omissions in the evidence information provided by the supplier, which need to be remedied after the fact and at significant costs. In this paper, we present a flexible and tool-supported approach for assisting suppliers and certifiers in developing an agreement about the evidence necessary to demonstrate compliance to a safety standard. Our approach is model-based; specifically, the safety standard of interest is expressed via an information model. This helps manage the complexity of safety standards, provides precise interpretation for the standards, and ensures the completeness and consistency of the generated agreement.},
	author = {Falessi, Davide and Sabetzadeh, Mehrdad and Briand, Lionel Claude and Turella, Emanuele and Coq, Thierry and Rajwinder Kaur {Panesar-Walawege}}
}
@techreport {Simula.simula.193,
	title = {A SysML-Based Approach to Traceability Management and Design Slicing in Support of Safety Certification: Framework, Tool Support, and Case Studies},
	number = {2011-01},
	year = {2011},
	month = {January},
	publisher = {Simula Research Laboratory},
	abstract = {Context: Traceability is one of the basic tenets of all software safety standards and a key prerequisite for certification of software. Despite this, the safety-critical software industry is still suffering from a chronic lack of guidelines on traceability. An acute traceability problem that we have identified through observing software safety certification processes has to do with the link between safety requirements and software design. In the current state of practice, this link often lacks sufficient detail to support the systematic inspections conducted by the certifiers of the software safety documentation. As a result, the suppliers often have to remedy the traceability gaps after the fact which can be very expensive and the outcome often is far from satisfactory. Objective: The objective of this article is developing a framework to enable systematic and efficient software design inspections during safety certification. In particular, the framework enables safety engineers and certifiers to extract design slices (model fragments) that filter out irrelevant details but keep enough context information for the slices to be easy to inspect and understand. This helps reduce cognitive load and thus makes it less likely that serious safety issues would be overlooked. Method: Our framework is grounded on SysML which is rapidly becoming the notation of choice for developing safety-critical systems. The framework includes a traceability information model, a methodology to establish traceability, and mechanisms to use traceability for extracting slices of models relevant to a particular safety requirement. The framework is implemented in a tool, named SafeSlice, that supports establishing the traceability links envisaged by the methodology, automated consistency checking of these links, and automated generation of SysML design slices. Results: We provide a formal proof that our slicing algorithm is sound for temporal safety properties, and argue about the completeness of the slices based on our practical experience. We report on the lessons learned from applying our approach to two case studies, one benchmark case and one industrial case. Both case studies indicate that our approach offers benefits by substantially reducing the amount of information that needs to be inspected in order to ensure that a given safety requirement is met by the design.},
	author = {Nejati, Shiva and Sabetzadeh, Mehrdad and Falessi, Davide and Briand, Lionel Claude and Coq, Thierry}
}
@inproceedings {Simula.simula.804,
	title = {Using SysML for Modeling of Safety-Critical Software-Hardware Interfaces: Guidelines and Industry Experience},
	journal = {13th IEEE International High Assurance Systems Engineering Symposium (HASE{\textquoteright}11)},
	year = {2011},
	publisher = {IEEE},
	type = {Conference},
	address = {Boca Raton, FL},
	abstract = {Safety-critical embedded systems often need to undergo a rigorous certification process to ensure that the safety risks associated with the use of the systems are adequately mitigated. Interfaces between software and hardware components (SW/HW interfaces) play a fundamental role in these systems by linking the systems{\textquoteright} control software to either the physical hardware components or to a hardware abstraction layer. Subsequently, safety certification of embedded systems necessarily has to cover the SW/HW interfaces used in these systems. In this paper, we describe a Model Driven Engineering (MDE) approach based on the SysML language, targeted at facilitating the certification of SW/HW interfaces in embedded systems. Our work draws on our experience with maritime and energy systems, but the work should also apply to a broader set of domains, e.g., the automotive sector, where similar design principles are used for (SW/HW) interface design. Our approach leverages our previous work on the development of SysML-based modeling and analysis techniques for safety-critical systems. Specifically, we tailor the methodology developed in our previous work to the development of safety-critical interfaces, and provide step-by-step and practical guidelines aimed at providing the evidence necessary for arguing that the safety-related requirements of an interface are properly addressed by its design. We describe an application of our proposed guidelines to a representative safety-critical interface in the maritime and energy domain.},
	author = {Sabetzadeh, Mehrdad and Nejati, Shiva and Briand, Lionel Claude and Anne-Heidi Evensen {Mills}}
}
@inproceedings {Vokac.2005.3,
	title = {Using a Domain-Specific Language and Custom Tools to Model a Multi-Tier Service-Oriented Application-Experiences and Challenges},
	journal = {Models 2005, Montego Bay, Jamaica October 2-7},
	year = {2005},
	note = {ISSN: 0302-9743 ISBN: 3-540-29010-9 DOI: 10.1007/11557432\_37},
	month = {October 2005},
	pages = {492-506},
	publisher = {Springer-Verlag GmbH},
	type = {1},
	address = {Heidelberg},
	abstract = {A commercial Customer Relationship Management application of approx. 1.5 MLOC of C++ code is being reimplemented, in stages, as a service-oriented, multi-tier application in C$\#$ on Microsoft .NET. We have chosen to use a domain-specific language both to model the external service-oriented interfaces, and to manage the transition to the internal, object-oriented implementation. Generic UML constructs such as class diagrams do not capture enough semantics to model these concepts. By defining a UML Profile that incorporates the concepts we wish to model, we have in effect created a Domain-Specific Language for our application. The models are edited using Rational XDE, but we have substituted our own code generator. This generator is a relatively generic text-substitution engine, which takes a template text and performs substitutions based on the model. The generator uses reflection to convert the UML and Profile concepts into substitution tags, which are in turn used in the template text. In this way, we can translate the semantics of the model into executable code, WSDL or other formats in a flexible way. We have successfully used this approach on a prototype scale, and are now transitioning to full-scale development.},
	isbn = {3-540-29010-9},
	doi = {10.1007/11557432_37},
	author = {Vok{\'a}c, Marek and Jens M. {Glattetre} and Williams, Clay and Briand, Lionel Claude}
}
