@article {43318,
	title = {Comparative study of machine learning test case prioritization for continuous integration testing},
	journal = {Software Quality Journal},
	year = {2023},
	month = {07/2023},
	publisher = { Springer},
	doi = {doi.org/10.1007/s11219-023-09646-0},
	author = {Marijan, Dusica}
}
@article {43173,
	title = {Interpolation-Based Inference of Vessel Trajectory Waypoints from Sparse AIS Data in Maritime},
	journal = {Journal of Marine Science and Engineering},
	volume = {11},
	number = {615},
	year = {2023},
	publisher = {Ocean Engineering},
	author = {Zaman, Bakht and Marijan, Dusica and Kholodna, Tetyana}
}
@inproceedings {43347,
	title = {Measuring the Effect of Causal Disentanglement on the Adversarial Robustness of Neural Network Models},
	journal = { ACM International Conference on Information and Knowledge Management},
	year = {2023},
	abstract = {Causal Neural Network models have shown high levels of robustness to adversarial attacks as well as an increased capacity for generalisation tasks such as few-shot learning and rare-context classification compared to traditional Neural Networks. This robustness is argued to stem from the disentanglement of causal and confounder input signals. However, no quantitative study has yet measured the level of disentanglement achieved by these types of causal models or assessed how this relates to their adversarial robustness.\&nbsp;\&nbsp; Existing causal disentanglement metrics are not applicable to deterministic models trained on real-world datasets. We, therefore, utilise metrics of content/style disentanglement from the field of Computer Vision to measure different aspects of the causal disentanglement for four state-of-the-art causal Neural Network models. By re-implementing these models with a common ResNet18 architecture we are able to fairly measure their adversarial robustness on three standard image classification benchmarking datasets under seven common white-box attacks. We find a strong association (r=0.820, p=0.001) between the degree to which models decorrelate causal and confounder signals and their adversarial robustness. Additionally, we find a moderate negative association between the pixel-level information content of the confounder signal and adversarial robustness (r=-0.597, p=0.040).},
	author = {Preben Monteiro {Ness} and Marijan, Dusica and Bose, Sunanda}
}
@article {43331,
	title = {Reactive buffering window trajectory segmentation: RBW-TS},
	journal = {Journal of Big Data},
	volume = {10},
	number = {123},
	year = {2023},
	month = {07/2023},
	publisher = { Springer},
	doi = {https://doi.org/10.1186/s40537-023-00799-0},
	author = {Zaman, Bakht and Altan, Dogan and Marijan, Dusica and Kholodna, Tetyana}
}
@article {43392,
	title = {ReMAV: Reward Modeling of Autonomous Vehicles for Finding Likely Failure Events},
	journal = {IEEE Transactions on Software Engineering (TSE)},
	year = {2023},
	publisher = { IEEE},
	abstract = {Autonomous vehicles are advanced driving systems that are well known for being vulnerable to various adversarial attacks, compromising the vehicle{\textquoteright}s safety, and posing danger to other road users. Rather than actively training complex adversaries by interacting with the environment, there is a need to first intelligently find and reduce the search space to only those states where autonomous vehicles are found less confident. In this paper, we propose a blackbox testing framework\&nbsp;ReMAV\&nbsp;using offline trajectories first to analyze the existing behavior of autonomous vehicles and determine appropriate thresholds for finding the probability of failure events. To this end, we introduce a three-step methodology which i) uses offline state action pairs of any autonomous vehicle under test, ii) builds an abstract behavior representation using our designed reward modeling technique in order to analyze states with uncertain driving decisions, and iii) uses a disturbance model for minimal perturbation attacks where the driving decisions are less confident. Our reward modeling technique helps in creating a behavior representation that allows us to highlight regions of likely uncertain behavior even when the baseline autonomous vehicle is performing well. This approach allows for more efficient testing without the need for computational and inefficient active adversarial learning techniques. We perform our experiments in a high-fidelity urban driving environment using three different driving scenarios containing single and multi-agent interactions. Our experiment shows a 35\%, 23\%, 48\%, and 50\% increase in occurrences of vehicle collision, road objects collision, pedestrian collision, and offroad steering events respectively by the autonomous vehicle under test, demonstrating a significant increase in failure events. We also perform a comparative analysis with prior testing frameworks and show that they underperform in terms of training-testing efficiency, finding total infractions, and simulation steps to identify the first failure compared to our approach. The results show that the proposed framework can be used to understand existing weaknesses of the autonomous vehicles under test in order to only attack those regions, starting with the simplistic perturbation models. We demonstrate that our approach is capable of finding failure events in terms of increased collision and offroad steering errors compared to the baseline driving behavior.},
	keywords = {Autonomous Vehicle Testing, Behavior Modeling, deep reinforcement learning, Inverse Reinforcement Learning},
	url = {https://arxiv.org/abs/2308.14550},
	author = {Sharif, Aizaz and Marijan, Dusica}
}
@article {43087,
	title = {SafeWay: Improving the Safety of Autonomous Waypoint Detection in Maritime using Transformer and Interpolation},
	journal = {Maritime Transport Research},
	volume = {4},
	number = {100086},
	year = {2023},
	note = {},
	month = {03/2023},
	publisher = { Elsevier},
	abstract = {},
	issn = {2666-822X},
	doi = {https://doi.org/10.1016/j.martra.2023.100086},
	author = {Altan, Dogan and Marijan, Dusica and Kholodna, Tetyana}
}
@article {42905,
	title = {A Survey on Blockchain for Healthcare: Challenges, Benefits, and Future Directions},
	journal = {IEEE Communications Surveys \& Tutorials},
	volume = {25},
	year = {2023},
	pages = {386 - 424},
	publisher = { IEEE},
	author = {Mohammad Salar {Arbabi} and Lal, Chhagan and Narasimha Raghavan {Veeraragavan} and Marijan, Dusica and Jan F. {Nyg{\r a}rd} and Vitenberg, Roman}
}
@inproceedings {42658,
	title = {Adversarial Deep Reinforcement Learning for Improving the Robustness of Multi-agent Autonomous Driving Policies},
	journal = {29th Asia-Pacific Software Engineering Conference (APSEC)},
	year = {2022},
	publisher = { IEEE},
	address = {29th Asia-Pacific Software Engineering Conference (APSEC)},
	abstract = {\&nbsp;Autonomous cars are well known for being vulnerable to adversarial attacks that can compromise the safety of the car and pose danger to other road users. To effectively defend against adversaries, it is required to not only test autonomous cars for finding driving errors but to improve the robustness of the cars to these errors. To this end, in this paper, we propose a two-step methodology for autonomous cars that consists of (i) finding failure states in autonomous cars by training the adversarial driving agent, and (ii) improving the robustness of autonomous cars by retraining them with effective adversarial inputs. Our methodology supports testing autonomous cars in a multi-agent environment, where we train and compare adversarial car policy on two custom reward functions to test the driving control decision of autonomous cars. We run experiments in a vision-based high-fidelity urban driving simulated environment. Our results show that adversarial testing can be used for finding erroneous autonomous driving behavior, followed by adversarial training for improving the robustness of deep reinforcement learning-based autonomous driving policies. We demonstrate that the autonomous cars retrained using the effective adversarial inputs noticeably increase the performance of their driving policies in terms of reduced collision and offroad steering errors.\&nbsp;},
	keywords = {adversarial testing, AI testing, autonomous car, autonomous driving, deep reinforcement learning, multi-agent, Robustness, self-driving car, simulation testing},
	doi = {10.1109/APSEC57359.2022.00018},
	url = {https://ieeexplore.ieee.org/abstract/document/10043282},
	author = {Sharif, Aizaz and Marijan, Dusica}
}
@article {42577,
	title = {Blockchain verification and validation: Techniques, challenges, and research directions},
	journal = {Computer Science Review},
	volume = {45},
	year = {2022},
	pages = {100492},
	publisher = { Elsevier},
	author = {Marijan, Dusica and Lal, Chhagan}
}
@inproceedings {42454,
	title = {Discovering Gateway Ports in Maritime using Temporal Graph Neural Network Port Classification},
	journal = {35th Canadian Conference on Artificial Intelligence},
	year = {2022},
	note = {},
	publisher = {Canadian Artificial Intelligence Association},
	abstract = {Vessel navigation is influenced by various factors, such as dynamic environmental factors that change over time or static features such as vessel type or depth of the ocean. These dynamic and static navigational factors impose limitations on vessels, such as long waiting times in regions outside the actual ports, and we call these waiting regions gateway ports. Identifying gateway ports and their associated features such as congestion and available utilities can enhance vessel navigation by planning on fuel optimization or saving time in cargo operation. In this paper, we propose a novel temporal graph neural network (TGNN) based port classification method to enable vessels to discover gateway ports efficiently, thus optimizing their operations. The proposed method processes vessel trajectory data to build dynamic graphs capturing spatio-temporal dependencies between a set of static and dynamic navigational features in the data, and it is evaluated in terms of port classification accuracy on a real-world data set collected from ten vessels operating in Halifax, NS, Canada. The experimental results indicate that our TGNN-based port classification method provides an f-score of 95\% in classifying ports.},
	keywords = {actual ports, AIS data, gateway ports, maritime situational awareness, maritime traffic, port area, port classification, port congestion, spatio-temporal, temporal graph neural networks, vessel trajectories},
	doi = {https://doi.org/10.21428/594757db.a76bcb9d},
	author = {Altan, Dogan and Etemad, Mohammad and Marijan, Dusica and Kholodna, Tetyana}
}
@inproceedings {42848,
	title = {Evaluating the Robustness of Deep Reinforcement Learning for Autonomous Policies in a Multi-agent Urban Driving Environment},
	journal = {22nd IEEE International Conference on Software Quality, Reliability, and Security (QRS)},
	year = {2022},
	publisher = { IEEE},
	address = {22nd International Conference on Software Quality, Reliability and Security (QRS)},
	abstract = {Background: Deep reinforcement learning is actively used for training autonomous car policies in a simulated driving environment. Due to the large availability of various reinforcement learning algorithms and the lack of their systematic comparison across different driving scenarios, we are unsure of which ones are more effective for training autonomous car software in single-agent as well as multi-agent driving environments. Aims: A benchmarking framework for the comparison of deep reinforcement learning in a vision-based autonomous driving will open up the possibilities for training better autonomous car driving policies. Method: To address these challenges, we provide an open and reusable benchmarking framework for systematic evaluation and comparative analysis of deep reinforcement learning algorithms for autonomous driving in a single- and multi-agent environment. Using the framework, we perform a comparative study of four discrete and two continuous action space deep reinforcement learning algorithms. We also propose a comprehensive multi-objective reward function designed for the evaluation of deep reinforcement learning-based autonomous driving agents. We run the experiments in a vision-only high-fidelity urban driving simulated environments. Results: The results indicate that only some of the deep reinforcement learning algorithms perform consistently better across single and multi-agent scenarios when trained in various multi-agent-only environment settings. For example, A3C- and TD3-based autonomous cars perform comparatively better in terms of more robust actions and minimal driving errors in both single and multi-agent scenarios. Conclusions: We conclude that different deep reinforcement learning algorithms exhibit different driving and testing performance in different scenarios, which underlines the need for their systematic comparative analysis. The benchmarking framework proposed in this paper facilitates such a comparison.},
	keywords = {autonomous cars, autonomous driving, deep reinforcement learning, multi-agent systems, testing autonomous driving},
	doi = {10.1109/QRS57517.2022.00084},
	url = {https://ieeexplore.ieee.org/abstract/document/10062456},
	author = {Sharif, Aizaz and Marijan, Dusica}
}
@article {42455,
	title = {Industry-Academia Research Collaboration and Knowledge Co-creation: Patterns and Anti-patterns},
	journal = {ACM Transactions on Software Engineering and Methodology},
	volume = {31},
	year = {2022},
	pages = {1-52},
	publisher = {ACM},
	issn = {1049-331X},
	doi = {10.1145/3494519},
	url = {https://dl.acm.org/doi/10.1145/3494519https://dl.acm.org/doi/pdf/10.1145/3494519},
	author = {Marijan, Dusica and Sen, Sagar}
}
@article {42506,
	title = {Bridging Software Engineering Research and Industrial Practice},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {46},
	year = {2021},
	publisher = {ACM},
	author = {Marijan, Dusica and Zimmermann, Thomas and Ham, MyungJoo and Selic, Bran}
}
@inproceedings {28131,
	title = {DeepOrder: Deep Learning for Test Case Prioritization in Continuous Integration Testing},
	journal = {2021 IEEE International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2021},
	pages = {525-534},
	publisher = { IEEE},
	abstract = {Continuous integration testing is an important step in the modern software engineering life cycle. Test prioritization is a method that can improve the efficiency of continuous integration testing by selecting test cases that can detect faults in the early stage of each cycle. As continuous integration testing produces voluminous test execution data, test history is a commonly used artifact in test prioritization. However, existing test prioritization techniques for continuous integration either cannot handle large test history or are optimized for using a limited number of historical test cycles. We show that such a limitation can decrease fault detection effectiveness of prioritized test suites.This work introduces DeepOrder, a deep learning-based model that works on the basis of regression machine learning. DeepOrder ranks test cases based on the historical record of test executions from any number of previous test cycles. DeepOrder learns failed test cases based on multiple factors including the duration and execution status of test cases. We experimentally show that deep neural networks, as a simple regression model, can be efficiently used for test case prioritization in continuous integration testing. DeepOrder is evaluated with respect to time-effectiveness and fault detection effectiveness in comparison with an industry practice and the state of the art approaches. The results show that DeepOrder outperforms the industry practice and state-of-the-art test prioritization approaches in terms of these two metrics.},
	doi = {10.1109/ICSME52107.2021.00053},
	author = {Sharif, Aizaz and Marijan, Dusica and Liaaen, Marius},
	editor = {, }
}
@inproceedings {27711,
	title = {Improving the Reliability of Autonomous Software Systems through  Metamorphic Testing},
	journal = {Proceedings of the 31st European Safety and Reliability Conference (ESREL)},
	year = {2021},
	pages = {1-page abstract},
	publisher = {ESREL},
	isbn = {ISBN: 978-981-18-2016-8},
	author = {Mohit Kumar {Ahuja} and Mohamed Bachir {Belaid} and Bernab{\'e}, Pierre and Gotlieb, Arnaud and Marijan, Dusica and Sharif, Aizaz and Spieker, Helge},
	editor = {Castanier, Bruno and Cepin, Marko and Bigaud, David and Berenguer, Christophe}
}
@article {27493,
	title = {Industry-Academia research collaboration in software engineering: The Certus model},
	journal = {Information and Software Technology},
	volume = {132},
	number = {106473},
	year = {2021},
	publisher = {Elsevier},
	author = {Marijan, Dusica and Gotlieb, Arnaud}
}
@article {42505,
	title = {Industry-Academia Research Collaborations During and After COVID-19},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {46},
	year = {2021},
	publisher = {ACM},
	author = {Marijan, Dusica and Bansal, Chetan and Lopez, Tamara}
}
@inbook {27608,
	title = {Testing Industrial Robotic Systems: A New Battlefield!},
	booktitle = {Software Engineering for Robotics},
	year = {2021},
	month = {06/2021},
	pages = {109-137},
	publisher = {Springer Nature},
	organization = {Springer Nature},
	chapter = {5},
	address = {Cham},
	isbn = {ISBN 978-3-030-66493-0},
	url = {https://www.springer.com/gp/book/9783030664930$\#$aboutBook},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge},
	editor = {Cavalcanti, Ana and Dongol, Brijesh and Hierons, Robert and Timmis, Jon and Woodcock, Jim}
}
@techreport {27459,
	title = {Blockchain for Healthcare: Opportunities, Challenges, and Future Directions},
	year = {2020},
	publisher = {Simula Research Laboratory},
	abstract = {The continuous generation of large volume of medical data from different sources, such as patient monitoring, clinical trials management, and processing payments and reimbursement claims, makes healthcare a data-intensive domain. This data needs to be shared among different medical facilities for various purposes, such as in-depth data analysis and collaborative research, to achieve innovative advances in the medical treatment procedures and drug developments, and providing personalised healthcare services to the patients. However, the existing data exchange solutions in healthcare domain exhibits several challenges, like data security, patient privacy, data owner consent management, interoperability, and chain-of-custody. Recently, the industry and research community turned its focus on the possible use of blockchain technology to solve one or more of the above challenges in healthcare domain. The blockchain technology along with the support from smart contracts is considered as an adequate solution for secure and efficient medical data sharing. It is due to its unique features, such as decentralization, trustlessness, immutability, traceability, and security. In this paper, we provide a comprehensive survey on the state-of-the-art efforts that envision the usage of blockchain-based solutions to improve one or more aspects in healthcare domain. 
To this end, first we categorize the existing works based on a set of key challenge(s) that they address in healthcare by using blockchain technology. The particular challenges that we consider for the categorization of the surveyed articles are as follows: (i) security and privacy, (ii) interoperability, (iii) compliance management, (iv) medical records management, and (v) chain-of-custody. Second, we discuss the benefits and limitations that the surveyed solutions exhibit when incorporating blockchain in healthcare. We also investigate the causal relationships among these challenges to demonstrate how various factors causing these challenges are connected with each other. Third, we identify the practical issues that blockchain-driven healthcare must overcome to become a success, and we suggest the additional future research required to address the identified issues. Our survey shows that there is an exponential increase in the number of efforts towards the use blockchain and smart contracts to improve various functionalities in several healthcare services such as clinical trials, medical data management, and drug supply chain management. However, most of these efforts are still in their initial phases, and a lot of work still remains before their integration in practical healthcare scenarios. This work is currently in-progress, and it is planned to be performed in an accepted research-based innovation project by Research Council of Norway called {\textquotedblleft}SMARTMED {\textendash} Secure and accountable sharing of medical records using smart contracts and blockchain{\textquotedblright}.},
	keywords = {blockchain, Cloud computing, Electronic Health Records, Internet of things, Medical data, security and privacy},
	author = {Lal, Chhagan and Marijan, Dusica}
}
@article {27357,
	title = {ITE: A Lightweight Implementation of Stratified Reasoning for Constructive Logical Operators},
	journal = {International Journal on Artificial Intelligence Tools},
	volume = {29},
	year = {2020},
	month = {06/2020},
	pages = {23},
	publisher = {World Scientific Publishing},
	doi = {10.1142/S0218213020600064.},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge}
}
@inproceedings {27373,
	title = {Lessons Learned on Research Co-Creation: Making Industry-Academia Collaboration Work},
	journal = {The 46th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)},
	year = {2020},
	publisher = {Euromicro},
	author = {Marijan, Dusica and Gotlieb, Arnaud},
	editor = {IEEE, }
}
@inproceedings {27372,
	title = {Neural Network Classification for Improving Continuous Regression Testing},
	journal = {The IEEE Second International Conference On Artificial Intelligence Testing (AITest 2020)},
	year = {2020},
	publisher = { IEEE},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Sapkota, Abhijeet},
	editor = {IEEE, }
}
@inproceedings {27341,
	title = {Opening the Software Engineering Toolbox for the Assessment of Trustworthy AI},
	journal = {1st International Workshop on New Foundations for Human-Centered AI @ ECAI 2020},
	year = {2020},
	month = {08/2020},
	pages = {67-70},
	publisher = {CEUR Workshop Proceedings},
	abstract = {Trustworthiness is a central requirement for the acceptance and success of human-centered artificial intelligence (AI). To deem an AI system as trustworthy, it is crucial to assess its behaviour and characteristics against a gold standard of Trustworthy AI, consisting of guidelines, requirements, or only expectations. While AI systems are highly complex, their implementations are still based on software. The software engineering community has a long-established toolbox for the assessment of software systems, especially in the context of software testing. In this paper, we argue for the application of software engineering and testing practices for the assessment of trustworthy AI. We make the connection between the seven key requirements as defined by the European Commission{\textquoteright}s AI high-level expert group and established procedures from software engineering and raise questions for future work.},
	url = {http://ceur-ws.org/Vol-2659/ahuja.pdf},
	author = {Mohit Kumar {Ahuja} and Mohamed Bachir {Belaid} and Bernab{\'e}, Pierre and Collet, Mathieu and Gotlieb, Arnaud and Lal, Chhagan and Marijan, Dusica and Sen, Sagar and Sharif, Aizaz and Spieker, Helge},
	editor = {Saffiotti, Alessandro and Serafini, Luciano and Lukowicz, Paul}
}
@misc {27578,
	title = {Requirements collection for the AI4EU platform development},
	howpublished = {Big Data Value Forum, 3-5 November, Berlin, Germany},
	year = {2020},
	author = {Marijan, Dusica}
}
@inproceedings {26799,
	title = {RobTest: A CP Approach to Generate Maximal Test Trajectories for Industrial Robots},
	journal = {International Conference on Principles and Practice of Constraint Programming},
	volume = {12333},
	year = {2020},
	month = {08/2020},
	pages = {1-17},
	publisher = {Springer},
	edition = {LNCS},
	author = {Collet, Mathieu and Gotlieb, Arnaud and Lazaar, Nadjib and Carlsson, Mats and Marijan, Dusica and Mossige, Morten},
	editor = {Simonis, Helmut}
}
@book {27434,
	title = {Seventh International Workshop on Software Engineering Research and Industrial Practice},
	journal = {International Conference on Software Engineering (ICSE)},
	year = {2020},
	publisher = {ACM},
	address = {South Korea},
	author = {Marijan, Dusica and Zimmermann, Thomas and Ham, MyungJoo and Selic, Bran}
}
@inproceedings {27371,
	title = {Software Testing for Machine Learning},
	journal = {The Thirty-Fourth AAAI Conference on Artificial Intelligence},
	year = {2020},
	month = {2020},
	publisher = {AAAI},
	address = {AAAI{\textquoteright}20},
	author = {Marijan, Dusica and Gotlieb, Arnaud},
	editor = {, }
}
@inproceedings {26562,
	title = {Challenges of Testing Machine Learning Based Systems},
	journal = {Proceedings of the 1st IEEE Artificial Intelligence Testing Conference (AI Test 2019)},
	year = {2019},
	publisher = { IEEE},
	address = {San Francisco, CA, USA},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Mohit Kumar {Ahuja}},
	editor = {IEEE, }
}
@article {26956,
	title = {Good Practices in Aligning Software Engineering Research and Industry Practice},
	journal = { ACM SIGSOFT Software Engineering Notes},
	volume = {44},
	year = {2019},
	publisher = {ACM},
	author = {Marijan, Dusica and Sen, Sagar}
}
@article {26957,
	title = {Implications of Resurgence in Artificial Intelligence for Research Collaborations in Software Engineering},
	journal = {ACM SIGSOFT Software Engineering Notes},
	volume = {44},
	year = {2019},
	publisher = {ACM},
	author = {Marijan, Dusica and Shang, Weiyi and Shukla, Rakesh}
}
@article {26228,
	title = {A learning algorithm for optimizing continuous integration development and testing practice},
	journal = {Software: Practice and Experience},
	volume = {49},
	year = {2019},
	pages = {192-213},
	publisher = {Wiley Online Library},
	keywords = {Continuous Integration, Decision Trees, Machine Leanring, Software Testing},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius}
}
@book {26968,
	title = {Proceedings of the Joint 7th International Workshop on Conducting Empirical Studies in Industry and 6th International Workshop on Software Engineering Research and Industrial Practice},
	journal = {International Conference on Software Engineering (ICSE)},
	year = {2019},
	publisher = {ACM/IEEE},
	author = {Marcus Ciolkowski {Jedlitschka} and Marijan, Dusica and Galster, Matthias and Shang, Weiyi and Jedlitschka, Andreas and Shukla, Rakesh}
}
@techreport {26810,
	title = {SARCoS: Machine Learning Models for the Selection of Security Test Scenarios},
	year = {2019},
	publisher = {Simula Research Laboratory},
	author = {Gotlieb, Arnaud and Bernab{\'e}, Pierre and Marijan, Dusica and Spieker, Helge}
}
@misc {27720,
	title = {Statistics AIS Dataset from Statsat},
	howpublished = {Simula Research Laboratory},
	year = {2019},
	author = {Bernab{\'e}, Pierre and Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Legeard, Bruno}
}
@techreport {26779,
	title = {Survey on Testing of Deep Learning Systems},
	year = {2019},
	publisher = {Simula Research Laboratory},
	abstract = {Recent studies have shown that deep learning algorithms used for image classification or object recognition are not sufficiently reliable. These algorithms can be easily fooled by applying perturbations to images or generating artificial images that result in misclassification. In this paper, we provide an overview of software testing methods present in literature to test deep learning systems. We have explored different methods of testing deep neural networks, namely metamorphic testing, mutation testing, differential testing, and adversarial perturbation testing. We present the main findings available from the literature and compare these methods systematically and comprehensively. The results show that systematic testing of deep learning systems can further help to increase the performance of state-of-the-art systems.},
	author = {Mohit Kumar {Ahuja} and Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@misc {26809,
	title = {The TSAR Project},
	howpublished = {14th Certus User Partner Workshop, Sep. 2019, Larvik, Norway},
	year = {2019},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@article {25983,
	title = {Certus: an organizational effort towards research-based innovation in software verification and validation},
	journal = {International Journal of Systems Assurance Engineering and Management },
	volume = {9},
	year = {2018},
	pages = {313-322},
	publisher = { Springer},
	author = {Sen, Sagar and Marijan, Dusica and Gotlieb, Arnaud}
}
@inproceedings {25977,
	title = {DevOps Enhancement with Continuous Test Optimization},
	journal = {The 30th International Conference on Software Engineering and Knowledge Engineering (SEKE)},
	year = {2018},
	pages = {535-536},
	publisher = {KSI Research Inc. and Knowledge Systems Institute Graduate School},
	doi = {10.18293/SEKE2018-168},
	author = {Marijan, Dusica and Sen, Sagar}
}
@inproceedings {25976,
	title = {DevOps Improvements for Reduced Cycle Times with Integrated Test Optimizations for Continuous Integration},
	journal = {2018 IEEE 42nd Annual Computer Software and Applications Conference (COMPSAC)},
	year = {2018},
	pages = {22-27},
	publisher = {IEEE},
	doi = {10.1109/COMPSAC.2018.00012},
	author = {Marijan, Dusica and Sen, Sagar and Liaaen, Marius}
}
@book {26025,
	title = {Fifth International Workshop on Software Engineering Research and Industrial Practice},
	journal = {International Conference on Software Engineering (ICSE)},
	year = {2018},
	month = {May},
	publisher = {ACM},
	address = {Gothenburg, Sweden},
	author = {Shukla, Rakesh and Marijan, Dusica and Borg, Markus and Yang, Ye}
}
@misc {26026,
	title = {Practical selective regression testing with effective redundancy in interleaved tests},
	howpublished = {International Conference on Software Engineering (ICSE), Gothenburg, Sweden},
	year = {2018},
	address = {Software Engineering in Practice},
	author = {Marijan, Dusica}
}
@inproceedings {25975,
	title = {Practical Selective Regression Testing with Effective Redundancy in Interleaved Tests},
	journal = {Proceedings of the 40th International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP{\textquoteright}18)},
	year = {2018},
	pages = {153-162},
	publisher = {ACM},
	isbn = {978-1-4503-5659-6},
	doi = {10.1145/3183519.3183532},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@misc {25657,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	howpublished = {Gesellschaft f{\"u}r Informatik Software Engineering Conference 2018 (SE18), Ulm, Germany},
	year = {2018},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@inproceedings {26109,
	title = {Stratified Constructive Disjunction and Negation in Constraint Programming},
	journal = {Proc. of IEEE Int. Conf. on Tools with Artificial Intelligence (ICTAI-18). Volos, Greece. Nov. 2018},
	year = {2018},
	pages = {106-113},
	publisher = {IEEE},
	abstract = {Constraint Programming (CP) is a powerful declarative programming paradigm combining inference and search in order to find solutions to various type of constraint systems. Dealing with highly disjunctive constraint systems is notoriously difficult in CP. Apart from trying to solve each disjunct independently from each other, there is little hope and effort to succeed in constructing intermediate results combining the knowledge originating from several disjuncts. In this paper, we propose If Then Else (ITE), a lightweight approach for implementing stratified constructive disjunction and negation on top of an existing CP solver, namely SICStus Prolog clp(FD). Although constructive disjunction is known for more than three decades, it does not have straightforward implementations in most CP solvers. ITE is a freely available library proposing stratified and constructive reasoning for various operators, including disjunction and negation, implication and conditional. Our preliminary experimental results show that ITE is competitive with existing approaches that handle disjunctive constraint systems.},
	doi = {10.1109/ICTAI.2018.00026},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Spieker, Helge}
}
@inproceedings {25065,
	title = {Detecting and Reducing Redundancy in Software Testing for Highly Configurable Systems},
	journal = {IEEE International Symposium on High Assurance Systems Engineering},
	year = {2017},
	publisher = {ACM/IEEE},
	isbn = {978-1-5090-4636-2},
	doi = {10.1109/HASE.2017.31},
	author = {Marijan, Dusica and Sen, Sagar}
}
@article {25664,
	title = {Modeling and Verifying Combinatorial Interactions to Test Data Intensive Systems: Experience at the Norwegian Customs Directorate},
	journal = {IEEE Transactions on Reliability},
	volume = {66},
	number = {1},
	year = {2017},
	pages = {3{\textendash}16},
	publisher = {IEEE Transactions on Reliability},
	doi = {10.1109/TR.2016.2618121},
	url = {https://doi.org/10.1109/TR.2016.2618121},
	author = {Sen, Sagar and Marijan, Dusica and Ieva, Carlo and Grime, Astrid and Sander, Atle}
}
@inproceedings {25382,
	title = {Reinforcement Learning for Automatic Test Case Prioritization and Selection in Continuous Integration},
	journal = {Proceedings of the 26th ACM SIGSOFT International Symposium on Software Testing and Analysis},
	year = {2017},
	pages = {12-22},
	publisher = {ACM},
	address = {New York, NY, USA},
	isbn = {978-1-4503-5076-1},
	doi = {10.1145/3092703.3092709},
	author = {Spieker, Helge and Gotlieb, Arnaud and Marijan, Dusica and Mossige, Morten}
}
@inbook {25729,
	title = {Software Product Line Test Suite Reduction with Constraint Optimization},
	booktitle = {Software Technologies},
	volume = {743},
	year = {2017},
	pages = {68-87},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	author = {Carlsson, Mats and Gotlieb, Arnaud and Marijan, Dusica},
	editor = {Cabello, Enrique and Cardoso, Jorge and Ludwig, Andr{\'e} and Leszek A. {Maciaszek} and van Sinderen, Marten}
}
@inproceedings {25064,
	title = {Test Prioritization with Optimally Balanced Configuration Coverage},
	journal = {IEEE International Symposium on High Assurance Systems Engineering},
	year = {2017},
	publisher = {ACM/IEEE},
	isbn = {978-1-5090-4636-2},
	doi = {10.1109/HASE.2017.26},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {25227,
	title = {TITAN: Test Suite Optimization for Highly Configurable Software},
	journal = {International Conference on Software Testing, Verification and Validation (ICST 2017) },
	year = {2017},
	publisher = {IEEE},
	isbn = {978-1-5090-6031-3},
	doi = {10.1109/ICST.2017.60},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Liaaen, Marius and Sen, Sagar and Ieva, Carlo}
}
@article {25231,
	title = {Using Global Constraints to Automate Regression Testing},
	journal = {AI Magazine},
	volume = {38},
	number = {1},
	year = {2017},
	publisher = {AAAI},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@inproceedings {23736,
	title = {Automated Regression Testing Using Constraint Programming},
	journal = {Twenty-Eighth Conference on Innovative Applications of Artificial Intelligence (IAAI-16), Phoenix, AZ, USA, Feb. 2016},
	year = {2016},
	month = {02/2016},
	abstract = {\&nbsp; In software validation, regression testing aims to check the absence of regression faults in new releases of a software system. Typically, test cases used in regression testing are executed during a limited amount of time and are selected to check a given set of user requirements. When testing large systems the number of regression tests grows quickly over the years, and yet the available time slot stays limited. In order to overcome this problem, an approach known as test suite reduction (TSR), has been developed in software engineering to select a smallest subset of test cases, so that each requirement remains covered at least once. However solving the TSR problem is difficult as the underlying optimization problem is NP-hard, but it is also crucial for vendors interested in reducing the time to market of new software releases.\&nbsp; In this paper, we address regression testing and TSR with Constraint Programming (CP).\&nbsp; More specifically, we propose new CP models to solve TSR that exploit global constraints, namely NValue and GCC. We reuse a set of preprocessing rules to reduce a priori each instance, and we introduce a structure-aware search heuristic. We evaluated our CP models and proposed improvements\&nbsp; against existing approaches, including a simple greedy approach and MINTS, the state-of-the-art tool of the software engineering community. Our experiments show that CP outperforms both the greedy approach and MINTS when it is interfaced with MiniSAT, in terms of percentage of reduction and execution time. When MINTS is interfaced with CPLEX, we show that our CP model performs better only on percentage of reduction. Finally, by working closely with validation engineers from Cisco Systems, Norway, we integrated our CP model into an industrial regression testing process.},
	author = {Gotlieb, Arnaud and Carlsson, Mats and Liaaen, Marius and Marijan, Dusica and Petillon, Alexandre}
}
@inproceedings {24939,
	title = {Coverage-based Test Prioritization for Regression Testing of Configurable Software},
	journal = {IEEE 27th International Symposium on Software Reliability Engineering (ISSRE)},
	year = {2016},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {24868,
	title = {Effect of Time Window on the Performance of Continuous Regression Testing},
	journal = {32nd IEEE International Conference on Software Maintenance and Evolution (ICSME)},
	year = {2016},
	author = {Marijan, Dusica and Liaaen, Marius}
}
@inproceedings {24869,
	title = {Improving Configurable Software Testing with Statistical Test Selection},
	journal = {31st IEEE/ACM International Conference on Automated Software Engineering (ASE) Workshops},
	year = {2016},
	author = {Marijan, Dusica}
}
@article {23816,
	title = {Modelling and Verifying Combinatorial Interactions to Test Data Intensive Systems: Experience with Optimal Archiving at the Norwegian Customs and Excise Directorate},
	journal = {IEEE Transaction on Reliability},
	year = {2016},
	pages = {1-14},
	publisher = {IEEE},
	abstract = {Testing data-intensive systems is paramount to increaseour reliance on information processed in e-governance,scientific/medical research, and social networks. Data accrued inthese systems often go through several manual and computationalsteps involving human inputs in interactive media and complexbatch appications that aim to ensure high quality of data interms of validity, correctness, and adherence to business rules. Acommon industrial practice in testing data-intensive systems isto extract test databases from live production streams and verifythe data in them through a checklist of requirements eitherby tedious manual observation or by executing complex SQLqueries composed and understood by very few domain experts.We elevate the specification of such requirements on data bymodelling data interactions between fields cross-cutting the testdatabase\&rsquo;s schema. These interactions are modelled as test casesin a classification tree model. The model documents intuitiveexpert knowledge about what to expect in the test databaseand is given executable semantics using our human-in-the-looptool DEPICT. DEPICT verifies if interactions occurred or notin systematically extracted test databases. Non-occurrence ofexpected interactions or occurrence of unexpected interactionsindicate faults in the data. We present experiences on how ourmodel-driven approach has been successfully applied to verifytest databases in the Norwegian Public Sector. In particular, wepresent case studies at (1) the Norwegian Customs and ExciseDirectorate for verifying the adherence to customs regulationsand (2) the Cancer Registry of Norway to verify its data qualitymanagement process involving both human coders and complexlegacy batches.},
	author = {Sen, Sagar and Marijan, Dusica and Ieva, Carlo and Grime, Astrid and Sander, Atle}
}
@inproceedings {24525,
	title = {A New Approach to Feature-based Test Suite Reduction in Software Product Line Testing},
	journal = {ICSOFT-EA 2016, 11th Int. Conf. on Software Engineering and Applications, Lisbon, July 2016, Awarded Best Paper},
	year = {2016},
	month = {07/2016},
	publisher = {INSTICC Press},
	author = {Gotlieb, Arnaud and Carlsson, Mats and Marijan, Dusica and Petillon, Alexandre}
}
@article {Simula.simula.2182,
	title = {Practical Minimization of Pairwise-Covering Test Configurations Using Constraint Programming},
	journal = {Information and Software Technology},
	volume = {71},
	year = {2016},
	month = {03/2016},
	pages = {129-146},
	publisher = {Elsevier},
	abstract = {Context: Testing highly-configurable software systems is challenging due to a large number of test configurations that have to be carefully selected in order to reduce the testing effort as much as possible, while maintaining high software quality. Finding the smallest set of valid test configurations that ensure sufficient coverage of the system\&$\#$39;s feature interactions is thus the objective of validation engineers, especially when the execution of test configurations is costly or time-consuming. However, this problem is NP-hard in general and approximation algorithms have often been used to address it in practice.Objective: In this paper, we explore an alternative exact approach based on constraint programming that will allow engineers to increase the effectiveness of configuration testing while keeping the number of configurations as low as possible.Method: Our approach consists in using a (time-aware) minimization algorithm based on constraint programming. Given the amount of time, our solution generates a minimized set of valid test configurations that ensure coverage of all pairs of feature values (a.k.a. pairwise coverage). The approach has been implemented in a tool called PACOGEN.Results: PACOGEN was evaluated on 224 feature models from the standard benchmark repository SPLOT, and comparedin comparison with the two existing tools that are based on a greedy algorithm. For 79\% of 224 feature models, PACOGEN generated up to 60\% fewer test configurations than the competitor tools.\&nbsp; We further evaluated PACOGEN in the case study of an industrial video conferencing product line with a feature model of 169 features, and found 60\% fewer configurations compared with the manual approach followed by test engineers. The set of test configurations generated by PACOGEN decreased the time required by test engineers in manual test configuration by 85\%, increasing the feature-pairs coverage at the same timeConclusion: Our experimental evaluation concluded that optimal time-aware minimization of pairwise-covering test configurations is efficiently addressed using constraint programming techniques.},
	author = {Hervieu, Aymeric and Marijan, Dusica and Gotlieb, Arnaud and Baudry, Benoit}
}
@article {23696,
	title = {Certus: Glimpses of a Centre for Research-Based Innovation in Software Verification and Validation},
	journal = {International Journal of System Assurance Engineering and Management},
	year = {2015},
	month = {03/2015},
	pages = {1-25},
	publisher = {Springer},
	abstract = {What is gratifying to a software engineering researcher? Three of many possible answers to this question are (a) the intellectual exercise in developing/disseminating approaches that address emerging and existing challenges, (b) recognition from impact in a community of researchers and (c) widespread use of novel ideas, including software, in the society at large leading to enhancement of human ability and job creation. A culmination of these sources requires an organizational effort. This article presents a detailed account of a research-based innovation centre, Certus, to facilitate such a culmination for software engineering researchers. Certus has established a body of knowledge, methods and tools for the validation and verification of software systems in the Norwegian private and public sector. Certus works in close cooperation with five founding user partners and is hosted by the Simula Research Laboratory. We present the organizational structure of Certus and describe how Certus{\textquoteright}s life and health is planned and evaluated on a regular basis as a research-based innovation centre. We expound two successful collaborations, with (a) the private sector entity Cisco systems, Norway and (b) the public sector entity the Norwegian Customs and Excise. We hope that this document will serve as a basis to encourage national/international funding schemes to create call for proposals for long-term research-based innovation centres. This, we believe, is one way to justify use of tax payers resources in creating a win{\textendash}win situation for the triple helix: government, researchers and industry.},
	author = {Sen, Sagar and Marijan, Dusica and Gotlieb, Arnaud}
}
@inproceedings {sen2015evaluating,
	title = {Evaluating Reconfiguration Impact in Self-Adaptive Systems {\textendash} An Approach Based on Combinatorial Interaction Testing},
	journal = {The 41st Euromicro Conference on Software Engineering and Advanced Applications (SEAA). Madeira, Portugal, August 26-28},
	year = {2015},
	month = {08/2015},
	pages = {250 - 254},
	publisher = {IEEE},
	address = {Funchal},
	abstract = {Self-adaptive software adapts its behavior to the operational context via automatic run-time reconfiguration of software components. Particular reconfigurations may negatively affect the system Quality of Service (QoS), and therefore their impact over the system performance needs to be thoroughly evaluated. In this paper, we present an approach, based on Combinatorial Interaction Testing (CIT), that generates a sequence of configurations aimed at evaluating the extent to which reconfigurations affect the system QoS. Specifically, we transform a Classification Tree Models (CTM) of the configurations domain to a Constraint Satisfaction Problem (CSP) in Alloy, whose solution is a sequence of reconfigurations achieving T-wise coverage between system features, and R-wise coverage between configurations in the sequence. The resolution of the CSP is performed by an incremental growth algorithm that divides the generation of the sequence into sub-problems, and merges the results into a final set of test configurations. Preliminary validation in a self-adaptive vision system shows that our methodology effectively identifies critical reconfigurations exhibiting a high variation in QoS. This result encourages the use of CIT as a strategy to evaluate the performance of self-adaptive systems.},
	doi = {10.1109/SEAA.2015.72},
	author = {Sen, Sagar and Di Alesio, Stefano and Marijan, Dusica and Sarkar, Arnab}
}
@inproceedings {23807,
	title = {Multi-perspective Regression Test Prioritization for Time-Constrained Environments},
	journal = {IEEE International Conference on Software Quality, Reliability and Security},
	year = {2015},
	address = {Vancouver, Canada, August 3-5},
	author = {Marijan, Dusica}
}
@inproceedings {23638,
	title = {Towards More Relational Feature Models},
	journal = {ICSOFT-EA 2015 - Proceedings of the 10th International Conference on Software Engineering and Applications, Colmar, Alsace, France, 20-22 July.},
	year = {2015},
	month = {07/2015},
	pages = {381-386},
	publisher = {SciTePress},
	abstract = {Feature modeling is of paramount importance to capture variabilities and commonalities within a software product line. Nevertheless, current feature modeling notations are limited, representing only propositional formulae over attributed variables. This position paper advocates the extension of feature modeling formalisms with richer computational domains and relational operations. In particular, it proposes to extend feature modeling with finite and continuous domain variables, with first-order logic quantifiers, and with N-ary relations between features attributes, and with so-called global constraints. In order to extend the expressiveness while preserving automated analysis facilities, feature models could be semantically interpreted as first-order logic formulae (instead of propositional logic formulae), including global and continuous dependency between features. In simpler words, this paper emphasizes the importance of having more relational feature models and presents next generation applications.},
	isbn = {978-989-758-114-4},
	author = {Gotlieb, Arnaud and Marijan, Dusica and Sen, Sagar}
}
@inproceedings {Simula.simula.2710,
	title = {FLOWER: Optimal Test Suite Reduction As a Network Maximum Flow},
	journal = {Proceedings of Int. Symp. on Soft. Testing and Analysis (ISSTA{\textquoteright}14), San Jos{\'e}, CA, USA, Jul. 2014},
	year = {2014},
	month = {07/2014},
	pages = {171-180},
	publisher = {ACM},
	type = {Conference},
	address = {New York, USA},
	keywords = {Conference},
	isbn = {978-1-4503-2645-2},
	doi = {10.1145/2610384.2610416},
	author = {Gotlieb, Arnaud and Marijan, Dusica}
}
@misc {Simula.simula.3054,
	title = {Variability Testing of Highly-Configurable Software},
	howpublished = {The Norwegian Computer Society},
	year = {2014},
	publisher = {.},
	address = {.},
	author = {Marijan, Dusica}
}
@misc {Simula.simula.2176,
	title = {Managing Test Configurations in High-Variability Testing Environments With TITAN and Pure::variants},
	howpublished = {pure::variants Solutions Forum, SPLC},
	year = {2013},
	publisher = {.},
	address = {.},
	author = {Marijan, Dusica}
}
@inproceedings {Simula.simula.2172,
	title = {Practical Pairwise Testing for Software Product Lines},
	journal = {Proceedings of the International Software Product Line Conference (SPLC)},
	year = {2013},
	pages = {227-235},
	publisher = {ACM},
	type = {1},
	address = {New York, NY, USA},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Hervieu, Aymeric and Sen, Sagar}
}
@inproceedings {Simula.simula.2178,
	title = {Test Case Prioritization for Continuous Regression Testing: an Industrial Case Study},
	journal = {Proceedings of the International Conference on Software Maintenance (ICSM)},
	year = {2013},
	publisher = {IEEE},
	type = {1},
	address = {Eindhoven},
	author = {Marijan, Dusica and Gotlieb, Arnaud and Sen, Sagar},
	editor = {IEEE, }
}
@techreport {Simula.simula.2280,
	title = {Test Selection Based on Data Interactions in Data-Intensive Systems},
	number = {2013-03},
	year = {2013},
	month = {10/2013},
	publisher = {Simula Research Laboratory},
	abstract = {Testing data-intensive systems is paramount to increase our reliance on information in e-governance, scientific/ medical research, and social networks. Common practice to test these systems is by using a live production database. This testing approach is space and time inefficient and lacks clarity about what test cases or scenarios are covered. In this paper, we leverage classification tree modelling to specify desired test cases as data interactions between a set of fields across multiple tables of an existing database. Our methodology and tool, DEPICT, uses test case specifications in classification tree models to (a) automatically derive a spanning tree representing a relationship between any set of fields for any given database schema (b) generates queries to create an efficient inner join between related tables in the spanning tree (c) extract records from various tables that satisfy data interactions in the classification tree model (d) discovers holes or unsatisfied test cases in the test databases. We perform experiments to show that our approach is fast and scalable to extract test databases. Our experiments are based on selecting test databases from 8000 declarations for 60,000 items from the Norwegian Customs and Excise information system TVINN.},
	keywords = {Conference},
	author = {Sen, Sagar and Ieva, Carlo and Marijan, Dusica and Sarkar, Arnab and Gotlieb, Arnaud}
}
@techreport {27056,
	title = {Research report on test configuration generation},
	year = {2012},
	publisher = {Simula Research Laboratory},
	author = {Marijan, Dusica and Gotlieb, Arnaud}
}
@inproceedings {Simula.simula.2168,
	title = {A Review of Two Experiences From Applying Model Based Testing in Practice},
	journal = {International Symposium on Software Reliability Engineering Workshops (ISSREW)},
	year = {2012},
	publisher = {IEEE},
	type = {1},
	address = {Dallas, TX},
	author = {Marijan, Dusica},
	editor = {IEEE, }
}
