@article {43210,
	title = {Detailed Modeling of Heterogeneous and Contention-Constrained Point-to-Point MPI Communication},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {34},
	year = {2023},
	month = {03/2023},
	pages = {1580 - 1593},
	publisher = {IEEE},
	abstract = {The network topology of modern parallel computing systems is inherently heterogeneous, with a variety of latency and bandwidth values. Moreover, contention for the bandwidth can exist on different levels when many processes communicate with each other. Many-pair, point-to-point MPI communication is thus characterized by heterogeneity and contention, even on a cluster of homogeneous multicore CPU nodes. To get a detailed understanding of the individual communication cost per MPI process, we propose a new modeling methodology that incorporates both heterogeneity and contention. First, we improve the standard max-rate model to better quantify the actually achievable bandwidth depending on the number of MPI processes in competition. Then, we make a further extension that more detailedly models the bandwidth contention when the competing MPI processes have different numbers of neighbors, with also non-uniform message sizes. Thereafter, we include more flexibility by considering interactions between intra-socket and inter-socket messaging. Through a series of experiments done on different processor architectures, we show that the new heterogeneous and contention-constrained performance models can adequately explain the individual communication cost associated\&nbsp;with each MPI process. The largest test of realistic point-to-point MPI communication involves 8,192 processes and in total 2,744,632 simultaneous messages over 64 dual-socket AMD Epyc Rome compute nodes connected by InfiniBand, for which the overall prediction accuracy achieved is 84\%.},
	issn = {1045-9219},
	doi = {10.1109/TPDS.2023.3253881},
	url = {https://ieeexplore.ieee.org/document/10064025},
	author = {Thune, Andreas and Reinemo, Sven-Arne and Skeie, Tor and Cai, Xing}
}
@article {27917,
	title = {On the impact of heterogeneity-aware mesh partitioning and non-contributing computation removal on parallel reservoir simulations},
	journal = {Journal of Mathematics in Industry},
	volume = {11},
	year = {2021},
	month = {06/2021},
	publisher = { Springer},
	abstract = {Parallel computations have become standard practice for simulating the complicated multi-phase flow in a petroleum reservoir. Increasingly sophisticated numerical techniques have been developed in this context. During the chase of algorithmic superiority, however, there is a risk of forgetting the ultimate goal, namely, to efficiently simulate real-world reservoirs on realistic parallel hardware platforms. In this paper, we quantitatively analyse the negative performance impact caused by non-contributing computations that are associated with the {\textquotedblleft}ghost computational cells{\textquotedblright} per subdomain, which is an insufficiently studied subject in parallel reservoir simulation. We also show how these non-contributing computations can be avoided by reordering the computational cells of each subdomain, such that the ghost cells are grouped together. Moreover, we propose a new graph-edge weighting scheme that can improve the mesh partitioning quality, aiming at a balance between handling the heterogeneity of geological properties and restricting the communication overhead. To put the study in a realistic setting, we enhance the open-source Flow simulator from the OPM framework, and provide comparisons with industrial-standard simulators for real-world reservoir models.},
	doi = {10.1186/s13362-021-00108-5},
	url = {https://mathematicsinindustry.springeropen.com/articles/10.1186/s13362-021-00108-5},
	author = {Thune, Andreas and Cai, Xing and Alf Birger {Rustad}}
}
@article {27942,
	title = {The Open Porous Media Flow reservoir simulator},
	journal = {Computers \& Mathematics with Applications},
	volume = {81},
	year = {2021},
	month = {06/2020},
	pages = {159 - 185},
	publisher = {Elsevier},
	abstract = {The Open Porous Media (OPM) initiative is a community effort that encourages open\&nbsp;innovation and reproducible research for simulation of porous media processes. OPM\&nbsp;coordinates collaborative software development, maintains and distributes open-source\&nbsp;software and open data sets, and seeks to ensure that these are available under a free\&nbsp;license in a long-term perspective.In this paper, we present OPM Flow, which is a reservoir simulator developed for\&nbsp;industrial use, as well as some of the individual components used to make OPM Flow.\&nbsp;The descriptions apply to the 2019.10 release of OPM.},
	issn = {08981221},
	doi = {10.1016/j.camwa.2020.05.014},
	author = {Atgeirr Fl{\o} {Rasmussen} and Tor Harald {Sandve} and Bao, Kai and Lauser, Andreas and Hove, Joakim and Skaflestad, B{\r a}rd and Kl{\"o}fkorn, Robert and Blatt, Markus and Alf Birger {Rustad} and S{\ae}vareid, Ove and Lie, Knut-Andreas and Thune, Andreas}
}
@misc {26986,
	title = {Balancing the numerical and parallel performance for reservoir simulations},
	howpublished = {SIAM Conference on Computational Science and Engineering (CSE19), Spokane, Washington, USA},
	year = {2019},
	abstract = {The overall performance of a PDE-based simulator depends on two factors: the algorithmic efficiency of the numerical scheme chosen and the parallel efficiency of the software implementation. Since aspects from the two factors may influence each other{\textquoteright}s performance, a suitable balance between the two is important. The focus of this talk is on the OPM framework of oil reservoir simulation, for which the computational core is to solve the black-oil model: a coupled system of nonlinear PDEs. Due to large variations in the geological properties of a reservoir, the sparse matrix that arises from discretizing the coupled PDEs exhibits a strong heterogeneity in its nonzero values. These reflect the strength of coupling between the degrees of freedom. It is thus necessary to consider this heterogeneity in the unstructured mesh partitioning process, typically translated to partitioning a graph with weighted edges. Particularly, we study the impact of different strategies of edge weighting on both the numerical and parallel performance. The ordering of the degrees of freedom, which also affects both sides, is studied in addition. Our purpose is to shed some light on a suitable mesh partitioning and ordering methodolgy, which is also relevant beyond the context of reservoir simulation. The issue of how to allow users of OPM to inject such flexibility into the existing software framework is also discussed.},
	keywords = {HPC},
	author = {Thune, Andreas and Cai, Xing}
}
@misc {26359,
	title = {Unstructured mesh partitioning in the presence of strong coefficient heterogeneity},
	howpublished = {PDESoft 2018 Conference, Bergen, Norway},
	year = {2018},
	abstract = {Mesh partitioning is the first step in enabling parallel computation\&nbsp;for solving PDEs. For an unstructured computational mesh, the\&nbsp;task of partitioning is nontrivial, which can be\&nbsp;formulated as an optimization problem with two goals. The first\&nbsp;goal is load balancing, i.e.,\&nbsp;dividing\&nbsp;the computational work evenly among the subdomains.\&nbsp;The second goal is communication overhead minimization,\&nbsp;i.e.,\&nbsp;limiting the subsequent inter-subdomain communication.\&nbsp;Traditionally, an unstructured mesh is translated to a graph before partitioning,\&nbsp;where the graph{\textquoteright}s nodes correspond to the mesh entities and the\&nbsp;edges reflect the direct couplings between\&nbsp;mesh entities.In the presence of strong coefficient heterogeneity in a PDE, the\&nbsp;coupling between the degrees of freedom (e.g.,\&nbsp;the nonzero values in\&nbsp;a linear system arisen from the discretization) will also exhibit strong heterogeneity. It is numerically beneficial to group the\&nbsp;degrees of freedom that have strong in-between couplings in the same\&nbsp;subdomain, whereas the weaker couplings are prioritized as\&nbsp;places for the separation cuts between subdomains. This is achieved by\&nbsp;heterogeneously weighting the edges of the graph, which is then\&nbsp;partitioned by a graph partitioning algorithm that almost exclusively\&nbsp;aims to minimize the so-called edge cut\&nbsp;(sum of weights of all\&nbsp;the cut-through edges).Such an approach requires care, because an over-weighting of edges that\&nbsp;represent strong couplings may result in too few {\textquoteleft}{\textquoteleft}light-weight{\textquoteright}{\textquoteright} edges that can be\&nbsp;candidates for cut between subdomains. This may again lead to\&nbsp;bad partitioning results where e.g.,\&nbsp;a subdomain consists of several\&nbsp;disjoint patches. An accompanying weakness is that the associated edge-cut value\&nbsp;no longer\&nbsp;bears resemblance to the true communication overhead that will arise in\&nbsp;the parallel solution process of the PDE.This talk will report the results of ongoing work on investigating the\&nbsp;weaknesses of the edge-cut-oriented graph partitoning strategy in the\&nbsp;presence of strong\&nbsp;coefficient heterogeneity. We will use concrete examples from reservoir\&nbsp;simulations that involve strong disparity in key geological\&nbsp;coefficients.\&nbsp;Moreover, we aim to experiment with possible improvements to the\&nbsp;current graph partitioning paradigm by giving less emphasis to\&nbsp;edge cut while incorporating a new metric that better resembles the\&nbsp;overhead of inter-subdomain communication.},
	author = {Thune, Andreas and Cai, Xing}
}
