@article {25664,
	title = {Modeling and Verifying Combinatorial Interactions to Test Data Intensive Systems: Experience at the Norwegian Customs Directorate},
	journal = {IEEE Transactions on Reliability},
	volume = {66},
	number = {1},
	year = {2017},
	pages = {3{\textendash}16},
	publisher = {IEEE Transactions on Reliability},
	doi = {10.1109/TR.2016.2618121},
	url = {https://doi.org/10.1109/TR.2016.2618121},
	author = {Sen, Sagar and Marijan, Dusica and Ieva, Carlo and Grime, Astrid and Sander, Atle}
}
@article {23816,
	title = {Modelling and Verifying Combinatorial Interactions to Test Data Intensive Systems: Experience with Optimal Archiving at the Norwegian Customs and Excise Directorate},
	journal = {IEEE Transaction on Reliability},
	year = {2016},
	pages = {1-14},
	publisher = {IEEE},
	abstract = {Testing data-intensive systems is paramount to increaseour reliance on information processed in e-governance,scientific/medical research, and social networks. Data accrued inthese systems often go through several manual and computationalsteps involving human inputs in interactive media and complexbatch appications that aim to ensure high quality of data interms of validity, correctness, and adherence to business rules. Acommon industrial practice in testing data-intensive systems isto extract test databases from live production streams and verifythe data in them through a checklist of requirements eitherby tedious manual observation or by executing complex SQLqueries composed and understood by very few domain experts.We elevate the specification of such requirements on data bymodelling data interactions between fields cross-cutting the testdatabase\&rsquo;s schema. These interactions are modelled as test casesin a classification tree model. The model documents intuitiveexpert knowledge about what to expect in the test databaseand is given executable semantics using our human-in-the-looptool DEPICT. DEPICT verifies if interactions occurred or notin systematically extracted test databases. Non-occurrence ofexpected interactions or occurrence of unexpected interactionsindicate faults in the data. We present experiences on how ourmodel-driven approach has been successfully applied to verifytest databases in the Norwegian Public Sector. In particular, wepresent case studies at (1) the Norwegian Customs and ExciseDirectorate for verifying the adherence to customs regulationsand (2) the Cancer Registry of Norway to verify its data qualitymanagement process involving both human coders and complexlegacy batches.},
	author = {Sen, Sagar and Marijan, Dusica and Ieva, Carlo and Grime, Astrid and Sander, Atle}
}
@inproceedings {Simula.simula.2988,
	title = {Experience Report: Verifying Data Interaction Coverage to Improve Testing of Data-Intensive Systems},
	journal = {International Symposium on Software Reliability Engineering},
	year = {2014},
	month = {November},
	pages = {223-234},
	publisher = {IEEE},
	type = {Conference},
	abstract = {Testing data-intensive systems is paramount to increase our reliance on information processed in e-governance, scientific/ medical research, and social networks. A common practice in the industrial testing process is to use test databases copied from live production streams to test functionality of complex database applications that manage well-formedness of data and its adherence to business rules in these systems. This practice is often based on the assumption that the test database adequately covers realistic scenarios to test, hopefully, all functionality in these applications. There is a need to systematically evaluate this assumption. We present a tool-supported method to model realistic scenarios and verify whether copied test databases actually cover them and consequently facilitate adequate testing. We conceptualize realistic scenarios as data interactions between fields cross-cutting a complex database schema and model them as test cases in a classification tree model. We present a human-inthe- loop tool, DEPICT, that uses the classification tree model as input to (a) facilitate interactive selection of a connected subgraph from often many possible paths of interactions between tables specified in the model (b) automatically generate SQL queries to create an inner join between tables in the connected subgraph (c) extract records from the join and generate a visual report of satisfied and unsatisfied interactions hence quantifying test adequacy of the test database. We report our experience as a qualitative evaluation of approach and with a large industrial database from the Norwegian Customs and Excise information system TVINN featuring large and complex databases with millions of records.},
	keywords = {Conference},
	isbn = {978-1-4799-6032-3},
	doi = {10.1109/ISSRE.2014.22 },
	author = {Sen, Sagar and Ieva, Carlo and Sarkar, Arnab and Grime, Astrid and Sander, Atle}
}
