@article {23383,
	title = {Early experiences with live migration of SR-IOV enabled InfiniBand},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {78},
	year = {2015},
	month = {04/2015},
	pages = {39-52},
	publisher = {Elsevier},
	abstract = {Virtualization is the key to efficient resource utilization and elastic resource allocation in cloud computing. It enables consolidation, the on-demand provisioning of resources, and elasticity through live migration. Live migration makes it possible to optimize resource usage by moving virtual machines (VMs) between physical servers in an application transparent manner. It does, however, require a flexible, high-performance, scalable virtualized I/O architecture to reach its full potential. This is challenging to achieve with high-speed networks such as InfiniBand and remote direct memory access enhanced Ethernet, because these devices usually maintain their connection state in the network device hardware. Fortunately, the single root IO virtualization (SR-IOV) specification addresses the performance and scalability issues. With SR-IOV, each VM has direct access to a hardware assisted virtual device without the overhead introduced by emulation or para-virtualization. However, SR-IOV does not address the migration of the network device state. In this paper we present and evaluate the first available prototype implementation of live migration over SR-IOV enabled InfiniBand devices.},
	keywords = {Architecture, IO virtualization, SR-IOV, VM migration},
	doi = {http://dx.doi.org/10.1016/j.jpdc.2015.01.004},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Yen, Chien-Hua and Skeie, Tor and Lysne, Olav and T{\o}rudbakken, Ola}
}
@phdthesis {Simula.simula.2802,
	title = {Dynamic Reconfiguration in Interconnections Network},
	year = {2014},
	school = {Faculty of Mathematics and Natural Sciences, University of Oslo},
	type = {phd},
	author = {Wei Lin {Guay}}
}
@misc {27080,
	title = {System and method for signaling dynamic reconfiguration events in a middleware machine environment},
	number = {13/649,689},
	year = {2014},
	month = {11/2014},
	edition = {G06F11/1423},
	chapter = {US20130124910A1},
	abstract = {Rerouting around faulty components and migration of jobs both require reconfiguration of data structures in the Queue Pairs residing in the hosts on an InfiniBand cluster. In this patent we describe\&nbsp;an implementation of dynamic reconfiguration of such host side data-structures. Our implementation preserves the Queue Pairs, and lets the application run without being interrupted. With this implementation, we demonstrate a complete solution to fault tolerance in an InfiniBand network, where dynamic network reconfiguration to a topology-agnostic routing function is used to avoid malfunctioning components. This solution is in principle able to let applications run uninterruptedly on the cluster, as long as the topology is physically connected. Through measurements on our test-cluster we show that the increased cost of our method in setup latency is negligible, and that there is only a minor reduction in throughput during reconfiguration.},
	url = {https://patents.google.com/patent/US8880932B2},
	author = {Wei Lin {Guay} and Bj{\o}rn Dag {Johnsen} and Brean, David and Reinemo, Sven-Arne}
}
@misc {Simula.simula.2496,
	title = {Prototyping Live Migration With SR-IOV Supported InfiniBand HCAs},
	howpublished = {HPC Advisory Council Spain Conference},
	year = {2013},
	month = {September},
	abstract = {Live migration is challenging to achieve with high-speed networks because these devices usually maintain their connection state in the network device hardware. In this work we i) describe the challenges with live migration over SR-IOV enabled InfiniBand devices, ii) present and evaluate the first available prototype implementation of live migration over SR-IOV enabled InfiniBand devices.},
	keywords = {Conference},
	author = {Tasoulas, Evangelos and Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Yen, Chien-Hua and Skeie, Tor and Lysne, Olav and Torudbakken, Ola}
}
@misc {Simula.simula.1254,
	title = {Prototyping Live Migration With SR-IOV Supported InfiniBand},
	howpublished = {Contributed talk at the 2012 OpenFabrics International Workshop, Monterey, USA},
	year = {2012},
	month = {March},
	keywords = {Workshop},
	author = {Wei Lin {Guay} and Bj{\o}rn Dag {Johnsen} and Torudbakken, Ola and Yen, Chien-Hua and Reinemo, Sven-Arne and Frank, Richard}
}
@inproceedings {Simula.simula.1528,
	title = {A Scalable Signalling Mechanism for VM Migration With SR-IOV Over InfiniBand},
	journal = {18th IEEE International Conference on Parallel and Distributed Systems (ICPADS)},
	year = {2012},
	pages = {384-391},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Single Root I/O Virtualization (SR-IOV) is a promising I/O virtualization approach for achieving high performance in the virtualization over InfiniBand (IB) network. One challenge is related to the hardware address assignment for each virtual IB device. There are two schemes for the hardware address assignment; static assignment and dynamic assignment. Static assignment always preserves the hardware address of a virtual IB device that is attached to a VM, but the dynamic assignment does not. A drawback, however, using static assignment is that its communication will be disconnected after VM migration. In this paper, we point out the problem related to SRIOV over IB that breaks the network connections after VM migration when the static assignment is deployed. Then, we propose a signalling mechanism that can maintain the network connectivity after VM migration. The performance evaluation using an experimental test bed shows that the proposed signalling mechanism does not increase the service downtime during hot migration. We also optimize the signalling method, where the same event can only be forwarded to a physical server once regardless of the hosted VMs, to reduce the management message overhead from O(n {\_\ast} m) to O(n).},
	keywords = {Conference},
	isbn = {978-0-7695-4903-3},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Skeie, Tor and Torudbakken, Ola},
	editor = {Tang, Xueyan and Xu, Cheng-Zhong and He, Bingsheng}
}
@inproceedings {Simula.simula.818,
	title = {DFtree - a Fat-Tree Routing Algorithm Using Dynamic Allocation of Virtual Lanes to Alleviate Congestion in InfiniBand Networks},
	journal = {The Network-Aware Data Management Workshop to be held in conjunction with the IEEE/ACM International Conference for High Performance Computing, Networking, Storage and Analysis (SC{\textquoteright}11)},
	year = {2011},
	month = {November},
	pages = {1-10},
	publisher = {ACM},
	type = {Workshop},
	abstract = {End-point hotspots can cause major slowdowns in interconnection networks due to head-of-line blocking and congestion. Therefore, avoiding congestion is important to ensure high performance for the network traffic. It is especially important in situations where permanent congestion, which results in permanent slowdown, can occur. Permanent congestion occurs when traffic has been moved away from a failed link, when multiple jobs run on the same system, and compete for network resources, or when a system is not balanced for the application that runs on it. In this paper we suggest a mechanism for dynamic allocation of virtual lanes and live optimisation of the distribution of flows between the allocated virtual lanes. The purpose is to alleviate the negative effect of permanent congestion by separating network flows into slow lane and fast lane traffic. Flows destined for an end-point hot-spot is placed in the slow lane and all other flows are placed in the fast lane. Consequently, the flows in the fast lane are unaffected by the head-of-line blocking created by the hot-spot traffic. We demonstrate the feasbility of this approach using a modified version of OFED and OpenSM with fat-tree routing on a small InfiniBand cluster. Our experiments show an increase in throughput ranging from 150\% to 468\% compared to the conventional fat-tree algorithm in OFED.},
	isbn = {978-1-4503-1132-8},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Lysne, Olav and Skeie, Tor},
	editor = {Balman, Mehmet and Byna, Surendra}
}
@inproceedings {Simula.simula.358,
	title = {A Scalable Method for Signalling Dynamic Reconfiguration Events With OpenSM},
	journal = {11th IEEE/ACM International Symposium on Cluster, Cloud, and Grid Computing (CCGrid 2011)},
	year = {2011},
	pages = {332-341},
	publisher = {IEEE Computer Society Press},
	type = {Conference},
	abstract = {Rerouting around faulty components, on-the-fly policy changes, and migration of jobs all require reconfiguration of data structures in the Queue Pairs residing in the hosts on an InfiniBand cluster. In addition to a proper implementation at the host, the subnet manager needs to implement a scalable method for signaling reconfiguration events to the hosts. In this paper we propose and evaluate three different implementations for signalling dynamic reconfiguration events with OpenSM. Through our evaluation we demonstrate a scalable solution for signalling host-side reconfiguration events in an InfiniBand network based on an example where dynamic network reconfiguration combined with a topology-agnostic routing function is used to avoid malfunctioning components. Through measurements on our test-cluster and an analytical study we show that our best proposal reduces reconfiguration latency by more than 90\% and in certain situations eliminates it completely. Furthermore, the processing overhead in the subnet manager is shown to be minimal.},
	isbn = {978-1-4577-0129-0},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne},
	editor = {Buyya, Rajkumar}
}
@misc {Simula.simula.824,
	title = {VFtree - a Fat-Tree Routing Algorithm Using Virtual Lanes to Alleviate Congestion},
	howpublished = {Invited talk at the HPC Advisory Council Switzerland Workshop 2011, Lugano, Switzerland.},
	year = {2011},
	month = {March},
	abstract = {A large number of clusters using InfiniBand is based on fat-tree topologies that can be routed deadlock-free using only one virtual lane. Consequently, all the remaining virtual lanes are left unused. In this talk we present an enhancement to the fat-tree algorithm that utilizes virtual lanes to improve performance when hot-spots are present. In a congested environment, results show that we are able to achieve throughput increases up to 38\% on a small cluster and from 221\% to 757\% depending on the hot-spot scenario for a 648-port simulated cluster.},
	author = {Reinemo, Sven-Arne and Wei Lin {Guay} and Bogdanski, Bartosz and Lysne, Olav and Skeie, Tor}
}
@inproceedings {Simula.simula.184,
	title = {VFtree - a Fat-Tree Routing Algorithm Using Virtual Lanes to Alleviate Congestion},
	journal = {Proceedings of the 25th IEEE International Parallel \& Distributed Processing Symposium},
	year = {2011},
	pages = {197-208},
	publisher = {IEEE Computer Society Press},
	type = {Conference},
	abstract = {It is a well known fact that multiple virtual lanes can improve performance in interconnection networks, but this knowledge has had little impact on real clusters. Currently, a large number of clusters using InfiniBand is based on fat-tree topologies that can be routed deadlock-free using only one virtual lane. Consequently, all the remaining virtual lanes are left unused. In this paper we suggest an enhancement to the fat-tree algorithm that utilizes virtual lanes to improve performance when hot-spots are present. Even though the bisection bandwidth in a fat-tree is constant, hot-spots are still possible and they will degrade performance for flows not contributing to them due to head-of-line blocking. Such a situation may be alleviated through adaptive routing or congestion control, however, these methods are not yet readily available in InfiniBand technology. To remedy this problem, we have implemented an enhanced fat-tree algorithm in OpenSM that distributes traffic across all available virtual lanes without any configuration needed. We evaluated the performance of the algorithm on a small cluster and done a large-scale evaluation through simulations. In a congested environment, results show that we are able to achieve throughput increases up to 38\% on a small cluster and from 221\% to 757\% depending on the hot-spot scenario for a 648-port simulated cluster.},
	isbn = {978-1-61284-372-8},
	author = {Wei Lin {Guay} and Bogdanski, Bartosz and Reinemo, Sven-Arne and Lysne, Olav and Skeie, Tor},
	editor = {Xin, Yuan}
}
@inproceedings {Simula.netsys.12,
	title = {Host Side Dynamic Reconfiguration With InfiniBand},
	journal = {2010 IEEE International Conference on Cluster Computing},
	year = {2010},
	pages = {126-135},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Rerouting around faulty components and migration of jobs both require reconfiguration of data structures in the Queue Pairs residing in the hosts on an InfiniBand cluster. In this paper we report an implementation of dynamic reconfiguration of such host side data-structures. Our implementation preserves the Queue Pairs, and lets the application run without being interrupted. With this implementation, we demonstrate a complete solution to fault tolerance in an InfiniBand network, where dynamic network reconfiguration to a topology-agnostic routing function is used to avoid malfunctioning components. This solution is in principle able to let applications run uninterruptedly on the cluster, as long as the topology is physically connected. Through measurements on our test-cluster we show that the increased cost of our method in setup latency is negligible, and that there is only a minor reduction in throughput during reconfiguration.},
	isbn = {978-0-7695-4220-1},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Lysne, Olav and Skeie, Tor and Bj{\o}rn Dag {Johnsen} and Holen, Line},
	editor = {Gu, Xiaohui and Ma, Xiaosong}
}
@misc {Simula.netsys.14,
	title = {VFtree- Fat Tree Routing With Virtual Lanes in InfiniBand},
	howpublished = {Poster, HiPEAC ACACES, Barcelona, Spain},
	year = {2010},
	abstract = {Even though the bisectional bandwidth in a fat-tree is constant, hot spots are still possible. Such a situation may be alleviated through adaptive routing or congestion control, however, these methods are not fully supported in InfiniBand technology. We propose an inexpensive approach for reducing the hot-spot problem in fat-trees, which is based on the application of virtual lanes.},
	author = {Wei Lin {Guay} and Bogdanski, Bartosz and Reinemo, Sven-Arne and Lysne, Olav}
}
