@article {43210,
	title = {Detailed Modeling of Heterogeneous and Contention-Constrained Point-to-Point MPI Communication},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {34},
	year = {2023},
	month = {03/2023},
	pages = {1580 - 1593},
	publisher = {IEEE},
	abstract = {The network topology of modern parallel computing systems is inherently heterogeneous, with a variety of latency and bandwidth values. Moreover, contention for the bandwidth can exist on different levels when many processes communicate with each other. Many-pair, point-to-point MPI communication is thus characterized by heterogeneity and contention, even on a cluster of homogeneous multicore CPU nodes. To get a detailed understanding of the individual communication cost per MPI process, we propose a new modeling methodology that incorporates both heterogeneity and contention. First, we improve the standard max-rate model to better quantify the actually achievable bandwidth depending on the number of MPI processes in competition. Then, we make a further extension that more detailedly models the bandwidth contention when the competing MPI processes have different numbers of neighbors, with also non-uniform message sizes. Thereafter, we include more flexibility by considering interactions between intra-socket and inter-socket messaging. Through a series of experiments done on different processor architectures, we show that the new heterogeneous and contention-constrained performance models can adequately explain the individual communication cost associated\&nbsp;with each MPI process. The largest test of realistic point-to-point MPI communication involves 8,192 processes and in total 2,744,632 simultaneous messages over 64 dual-socket AMD Epyc Rome compute nodes connected by InfiniBand, for which the overall prediction accuracy achieved is 84\%.},
	issn = {1045-9219},
	doi = {10.1109/TPDS.2023.3253881},
	url = {https://ieeexplore.ieee.org/document/10064025},
	author = {Thune, Andreas and Reinemo, Sven-Arne and Skeie, Tor and Cai, Xing}
}
