@phdthesis {26043,
	title = {Uncertainty-wise Cyber-Physical Systems Testing},
	volume = {PhD},
	year = {2018},
	month = {26/06/2018},
	pages = {292},
	school = {The University of Oslo},
	address = {Norway},
	abstract = {A Cyber-Physical Systems (CPS), as an integration of computing, communication, and control for making intelligent and autonomous systems, has been widely applied in various safety-critical domains, e.g., avionics and automotive. However, uncertainty is inherent in CPSs due to various reasons such as unpredictable environment under which the CPSs are operated. And, uncertainties may cause irreparable accidents once they cannot be handled properly by CPSs. Therefore, it is crucial to identify uncertainties in CPSs and test CPSs under the uncertainties, to ensure that CPSs are capable of handling the uncertainties during their actual operations, i.e., making CPSs less uncertain.Towards this direction, five contributions were made in the thesis corresponding to five papers respectively: (C1) a conceptual model, named as U-Model, for helping develop a systematic and comprehensive understanding of uncertainty in CPSs; (C2) an use case modeling methodology, named as U-RUCM, for identifying, qualifying, and, where possible, quantifying uncertainty in requirements engineering; (C3) a test modeling methodology, named as UncerTum, for supporting the construction of test ready models with the explicit representation of uncertainties in CPSs; (C4) an evolution framework, named as UncerTolve, for interactively evolving test ready models specified with UncerTum based on real operational data; and (C5) a testing framework, named as UncerTest, for testing CPSs in the presence of uncertainties in their operating environments in a cost-effective manner using model-based and search-based testing techniques.Based on our evaluations of the five contributions with the industrial CPS case studies, we observed that U-Model, as the foundation for this research, is sufficiently complete for characterizing and classifying uncertainties in CPSs. Then, the U-Model based modeling methodologies U-RUCM and UncerTum offer solutions to enable the identification and specification of uncertainties at two critical phases of a system development lifecycle: requirements engineering and testing. Furthermore, UncerTolve can successfully evolve model elements of the test ready models specified with UncerTum and calculate objective uncertainty measurements based on real operational data. Last, UncerTest managed to cost-effectively test CPSs in the presence of uncertainties and proactively identify unknown uncertainties by introducing the sources of the uncertainties into the test environments during test case execution.},
	keywords = {Cyber-Physical System, Model-based Testing, Uncertainty},
	author = {Zhang, Man}
}
@article {25456,
	title = {Uncertainty-Wise Cyber-Physical System Test Modeling},
	journal = {Software \& Systems Modeling},
	year = {2017},
	publisher = { Springer},
	abstract = {It is important that a Cyber-Physical System (CPS) deals with uncertainty in its behavior caused by its unpredictable operating environment, to ensure its reliable operation. One method to ensure that the CPS will handle such uncertainty during its operation is by testing the CPS with Model-based Testing (MBT) techniques. However, existing MBT techniques do not explicitly capture uncertainty in test ready models i.e., capturing the uncertain expected behavior of a CPS in the presence of environment uncertainty. To fill this gap, we present an Uncertainty-Wise test-modeling framework, named as Uncertum, to create test ready models to support MBT of CPSs facing uncertainty. Uncertum relies on the definition of a UML profile (the UML Uncertainty Profile (UUP)) and a set of UML model libraries extending the UML profile for Modeling and Analysis of Real-Time and Embedded Systems (MARTE). Uncertum also benefits from the UML Testing Profile (UTP) V.2 to support standard-based MBT. Uncertum was evaluated with two industrial CPS case studies, one real-world case study, and one open source CPS case study from the following four perspectives: 1) Completeness and Coverage of the profiles and model libraries in terms of concepts defined in their underlying uncertainty conceptual model for CPSs (i.e., U-Model and MARTE, 2) Effort required to model uncertainty with Uncertum, and 3) Correctness of the developed test ready models, which was assessed via model execution. Based on the evaluation, we can conclude that we were successful in modeling all the uncertainties identified in the four case studies, which gives us an indication that Uncertum is sufficiently complete. In terms of modeling effort, we concluded that on average Uncertum requires18.5\% more time to apply stereotypes from UUP on test ready models.},
	keywords = {Cyber-Physical System; UML, Model-based Testing, Uncertainty},
	issn = {1619-1374},
	doi = {10.1007/s10270-017-0609-6},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland and Okariz, Oscar}
}
@article {25187,
	title = {Uncertainty-Wise Evolution of Test Ready Models},
	journal = {Information and Software Technology (IST)},
	volume = {87},
	year = {2017},
	pages = {140-159},
	publisher = {Elsevier},
	abstract = {Context: Cyber-Physical Systems (CPSs), when deployed for operation, are inherently prone to uncertainty. Considering their applications in critical domains (e.g., healthcare), it is important that such CPSs are tested sufficiently, with the explicit consideration of uncertainty. Model-based testing (MBT) involves creating test ready models capturing the expected behavior of a CPS and its operating environment. These test ready models are then used for generating executable test cases. It is, therefore, necessary to develop methods that can continuously evolve, based on real operational data collected during the operation of CPSs, test ready models and uncertainty captured in them, all together termed as Belief Test Ready Models (BMs)Objective: Our objective is to propose a model evolution framework that can interactively improve the quality of BMs, based on operational data. Such BMs are developed by one or more test modelers (belief agents) with their assumptions about the expected behavior of a CPS, its expected physical environment, and potential future deployments. Thus, these models explicitly contain subjective uncertainty of the test modelers.Method: We propose a framework (named as UncerTolve) for interactively evolving BMs (specified with extended UML notations) of CPSs with subjective uncertainty developed by test modelers. The key inputs of UncerTolve include initial BMs of CPSs with known subjective uncertainty and real data collected from the operation of CPSs. UncerTolve has three key features: 1) Validating the syntactic correctness and conformance of BMs against real operational data via model execution, 2) Evolving objective uncertainty measurements of BMs via model execution, and 3) Evolving state invariants (modeling test oracles) and guards of transitions (modeling constraints for test data generation) of BMs with a machine learning technique.Results: As a proof-of-concept, we evaluated UncerTolve with one industrial CPS case study, i.e., GeoSports from the healthcare domain. Using UncerTolve, we managed to evolve 51\% of belief elements, 18\% of states, and 21\% of transitions as compared to the initial BM developed in an industrial setting.Conclusion: UncerTolve can successfully evolve model elements of the initial BM, in addition to objective uncertainty measurements using real operational data. The evolved model can be used to generate additional test cases covering evolved model elements and objective uncertainty. These additional test cases can be used to test the current and future deployments of a CPS to ensure that it will handle uncertainty gracefully during its operations.\&nbsp;\&nbsp;\&nbsp;\&nbsp;\&nbsp;},
	keywords = {Belief Model, Belief Test Ready Model, Model Evolution, Model-based Testing, Uncertainty},
	issn = {0950-5849},
	doi = {10.1016/j.infsof.2017.03.003},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland}
}
@techreport {24660,
	title = {Interactively Evolving Test Ready Models with Uncertainty Developed for Testing Cyber-Physical Systems},
	year = {2016},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Context: Cyber-Physical Systems (CPSs), when deployed for operation, are inherently prone to uncertainty. Considering their applications in critical domains (e.g., healthcare), it is important that such CPSs are tested sufficiently, with the explicit consideration of uncertainty. Model-based testing (MBT) involves creating test ready models capturing the expected behavior of a CPS and its operating environment. These test ready models are then used for generating executable test cases. It is, therefore, necessary to develop methods that can continuously evolve, based on real operational data collected during the operation of CPSs, test ready models and uncertainty captured in them, all together termed as Belief Test Ready Models (BMs)Objective: Our objective is to propose a model evolution framework that can interactively improve the quality of BMs, based on operational data. Such BMs are developed by one or more test modelers (belief agents) with their assumptions about the expected behavior of a CPS, its expected physical environment, and potential future deployments. Thus, these models explicitly contain subjective uncertainty of the test modelers.Method: We propose a framework (named as UncerTolve) for interactively evolving BMs (specified with extended UML notations) of CPSs with subjective uncertainty developed by test modelers. The key inputs of UncerTolve include initial BMs of CPSs with known subjective uncertainty and real data collected from the operation of CPSs. UncerTolve has three key features: 1) Validating the syntactic correctness and conformance of BMs against real operational data via model execution, 2) Evolving objective uncertainty measurements of BMs via model execution, and 3) Evolving state invariants (modeling test oracles) and guards of transitions (modeling constraints for test data generation) of BMs with a machine learning technique.Results: As a proof-of-concept, we evaluated UncerTolve with one industrial CPS case study, i.e., GeoSports from the healthcare domain. Using UncerTolve, we managed to evolve 51\% of belief elements, 18\% of states, and 21\% of transitions as compared to the initial BM developed in an industrial setting.Conclusion: UncerTolve can successfully evolve model elements of the initial BM, in addition to objective uncertainty measurements using real operational data. The evolved model can be used to generate additional test cases covering evolved model elements and objective uncertainty. These additional test cases can be used to test the current and future deployments of a CPS to ensure that it will handle uncertainty gracefully during its operations.\&nbsp;\&nbsp;\&nbsp;\&nbsp;\&nbsp;},
	keywords = {Belief State Machine, Cyber-Physical System, Model Evolution, Model-based Testing, Uncertainty},
	author = {Zhang, Man and Ali, Shaukat and Yue, Tao and Norgren, Roland}
}
@misc {24941,
	title = {Model-Driven Testing of Cyber-Physical Systems with the Explicit Consideration of Uncertainty},
	howpublished = {MPM4CPS WG meetings in Malaga, Spain, 24-25 November 2016},
	year = {2016},
	abstract = {It is a well-recognized fact that Cyber-Physical Systems (CPSs) face both known and unknown uncertainty during their operation. This demands the development of testing techniques that must take into account known uncertainty both in the CPS and its environment with the final goal of discovering unknown uncertainty against which the CPS can be tested. Eventually, the implementation of the CPS can be improved to shield it against the newly discovered uncertainty. In this presentation, we will present some of the results that we have achieved in an EU Horizon2020 project (U-Test) in this regard. We, first, demonstrate the modeling of test ready models of CPSs together with subjective uncertainty using the Uncertainty Modeling Framework. Second, we present two uncertainty-based test case generation and four test case minimization techniques relying on the test ready models founded on Uncertainty theory and multi-objective search. Third, we present the evaluation of the test case generation and minimization techniques that was conducted to select the best strategy to be used in the practice to test a real CPS. Fourth, we present the results of testing a real CPS with the selected best strategy.},
	keywords = {Cyber-Physical System, Model-based Testing, Search-Based Testing, Uncertainty},
	author = {Ali, Shaukat}
}
@misc {24661,
	title = {Tackling Uncertainty in Cyber-Physical Systems with Automated Testing},
	howpublished = {DE-CPS Workshop, Pisa, Italy},
	year = {2016},
	abstract = {The U-Test project (www.u-test.eu) aims at developing new methods and techniques for testing Cyber-Physical Systems (CPSs) under uncertainty. This position paper aims to provide the current status of the results achieved in the project during the first year. Our ultimate aim is to facilitate collaboration among various Horizon2020 projects focusing on CPSs. This position paper focuses on the research results from the following four perspectives: 1) Understanding Uncertainty in CPSs, 2) Modeling Uncertainty in CPSs, 3) Discovering Uncertainties, and 4) Testing Uncertainty. In addition to the research results, we also present a set of standardization activities that are planned in the project.},
	keywords = {Cyber-Physical Systems, Model-based Testing, Testing, Uncertainty, Uncertainty Modeling},
	author = {Ali, Shaukat and Yue, Tao and Zhang, Man}
}
@misc {24942,
	title = {U-TCsGM: Generating and Minimizing Uncertainty-Based Test Cases for Cyber-Physical Systems (Tool Demo)},
	howpublished = {MPM4CPS WG meetings in Malaga, Spain, 24-25 November 2016},
	year = {2016},
	abstract = {In this tool demo, we will present the implementation of our recent research work on generating and minimizing executable test cases from the test models of a Cyber-Physical System tagged with subjective uncertainty. The algorithms are founded on uncertainty theory and NSGA-II\&mdash;the most commonly used multi-objective search algorithm. We will demonstrate the complete process starting from creating a test model with uncertainty, generating test cases, minimizing test cases, and finally executing the minimized test cases using a real CPS case study.},
	keywords = {Model-based Testing, Search-Based Testing, Uncertainty},
	author = {Zhang, Man}
}
@techreport {23476,
	title = {Understanding Uncertainty in Cyber-Physical Systems: A Conceptual Model},
	year = {2015},
	note = {This work is supported by U-Test project (Testing Cyber-Physical Systems under Uncertainty).},
	publisher = {Simula Research Laboratory},
	abstract = {Uncertainty is intrinsic in most technical systems, including Cyber-Physical Systems (CPS). Therefore, handling uncertainty in a graceful manner during the real operation of CPS is critical. Since designing, developing, and testing modern and highly sophisticated CPS is an expanding field, a step to-wards dealing with uncertainty is to identify, define, and classify uncertainties at various levels of CPS. This will help develop a systematic and comprehen-sive understanding of uncertainty. To that end, we propose a conceptual model for uncertainty specifically designed for CPS. Since the study of uncertainty in CPS development and testing is still irrelatively unexplored, this conceptual model was derived in a large part by reviewing existing work on uncertainty in other fields, including philosophy, physics, statistics, and healthcare. The con-ceptual model is mapped to the three logical levels of CPS: Application, Infra-structure, and Integration. It is captured using UML class diagrams, including relevant OCL constraints. To validate the conceptual model, we identified, clas-sified, and specified uncertainties in two distinct industrial case studies.},
	keywords = {Cyber-Physical Systems, Model-based Testing, Search-Based Testing, UML, Uncertainty},
	author = {Zhang, Man and Selic, Bran and Ali, Shaukat and Yue, Tao and Okariz, Oscar and Norgren, Roland}
}
