@article {23942,
	title = {Efficient Routing and Reconfiguration in Virtualized HPC Environments with vSwitch-enabled Lossless Networks},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {31},
	year = {2018},
	month = {02/2018},
	publisher = {John Wiley \& Sons},
	abstract = {To meet the demands of communication-intensive workloads in the cloud, virtual machines (VMs) should utilize low overhead network communication paradigms. In general, such paradigms enable VMs to directly communicate with the hardware by means of a passthrough technology like Single-Root I/O Virtualization (SR-IOV). However, when passthrough-based virtualization is coupled with lossless interconnection networks, live-migrations introduce scalability challenges due to the substantial network reconfiguration overhead. With these challenges in mind we proposed a virtual switch (vSwitch) SR-IOV architecture for InfiniBand in (33). In this paper, we first suggest solutions to rectify the space-domain scalability issues that are present in vSwitch-enabled subnets as a result of the VMs using dedicated layer-two addresses. Then we discuss routing strategies for virtualized environments using vSwitches, and present a routing algorithm for Fat-Trees. We also present a reconfiguration method that minimizes imposed reconfiguration overhead on Fat-Trees. We perform an extensive evaluation of our prototype algorithms, and as vSwitch-enabled hardware does not yet exist, we deduce from empirical observations by emulating vSwitches with existing hardware, as well as large-scale simulations. Our results show significant reduction in the reconfiguration times as route recalculations can be eliminated, and for certain scenarios, the number of reconfiguration subnet management packets sent to switches is reduced from several hundred thousand down to a single one without degrading the routing quality.},
	keywords = {InfiniBand, Lossless Interconnection Networks, Network Reconfiguration, Network Routing, SR-IOV, Virtualization},
	author = {Tasoulas, Evangelos and Zahid, Feroz and Ernst Gunnar {Gran} and Begnum, Kyrre and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@article {25984,
	title = {A Self-Adaptive Network for HPC Clouds: Architecture, Framework, and Implementation},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	pages = {2658-2671},
	publisher = {IEEE},
	abstract = {Clouds offer flexible and economically attractive compute and storage solutions for enterprises. However, the effectiveness of cloud computing for high-performance computing (HPC) systems still remains questionable. When clouds are deployed on lossless interconnection networks, like InfiniBand (IB), challenges related to load-balancing, low-overhead virtualization, and performance isolation hinder full potential utilization of the underlying interconnect. Moreover, cloud data centers incorporate a highly dynamic environment rendering static network reconfigurations, typically used in IB systems, infeasible. In this paper, we present a framework for a self-adaptive network architecture for HPC clouds based on lossless interconnection networks, demonstrated by means of our implemented IB prototype. Our solution, based on a feedback control and optimization loop, enables the lossless HPC network to dynamically adapt to the varying traffic patterns, current resource availability, workload distributions, and also in accordance with the service provider-defined policies. Furthermore, we present IBAdapt, a simplified ruled-based language for the service providers to specify adaptation strategies used by the framework. Our developed self-adaptive IB network prototype is demonstrated using state-of-the-art industry software. The results obtained on a test cluster demonstrate the feasibility and effectiveness of the framework when it comes to improving Quality-of-Service compliance in HPC clouds.},
	doi = {10.1109/TPDS.2018.2842224},
	author = {Zahid, Feroz and Taherkordi, Amir and Ernst Gunnar {Gran} and Skeie, Tor and Bj{\o}rn Dag {Johnsen}}
}
@article {23944,
	title = {Compact Network Reconfiguration in Fat-Trees},
	journal = {The Journal of Supercomputing},
	volume = {72},
	year = {2016},
	pages = {4438{\textendash}4467},
	publisher = {Springer},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Skeie, Tor and Tasoulas, Evangelos}
}
@article {23945,
	title = {Efficient Network Isolation and Load Balancing in Multi-Tenant HPC Clusters},
	journal = {Journal of Future Generation Computer Systems},
	year = {2016},
	month = {04/2016},
	publisher = {Elsevier},
	doi = {http://dx.doi.org/10.1016/j.future.2016.04.003},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@inproceedings {24836,
	title = {Fast Hybrid Network Reconfiguration for Large-Scale Lossless Interconnection Networks},
	journal = {15th IEEE International Symposium on Network Computing and Applications (NCA 2016)},
	year = {2016},
	pages = {101-108},
	publisher = {IEEE},
	abstract = {Reconfiguration of high performance lossless interconnection networks is a cumbersome and time-consuming task. For that reason reconfiguration in large networks are typically limited to situations where it is absolutely necessary, for instance when severe faults occur. On the contrary, due to the shared and dynamic nature of modern cloud infrastructures, performance-driven reconfigurations are necessary to ensure efficient utilization of resources. In this work we present a scheme that allows for fast reconfigurations by limiting the task to sub-parts of the network that can benefit from a local reconfiguration. Moreover, our method is able to use different routing algorithms for different sub-parts within the same subnet. We also present a Fat-Tree routing algorithm that reconfigures a network given a user-provided node ordering. Hardware experiments and large scale simulation results show that we are able to significantly reduce reconfiguration times from 50\% to as much as 98.7\% for very large topologies, while improving performance.},
	keywords = {Cloud computing, Fat-Tree, HPC, InfiniBand, Network Reconfiguration, scalability},
	author = {Tasoulas, Evangelos and Ernst Gunnar {Gran} and Skeie, Tor and Bj{\o}rn Dag {Johnsen}},
	editor = {Pellegrini, Alessandro and Gkoulalas-Divanis, Aris and Di Sanzo, Pierangelo and Dimiter R. {Avresky}}
}
@misc {24958,
	title = {System and method for efficient network reconfiguration in fat-trees},
	number = {US15/073,022},
	year = {2016},
	month = {03/2016},
	type = {Pending},
	abstract = {Systems and methods are provided for supporting efficient reconfiguration of an interconnection network having a pre-existing routing comprising. An exemplary method can provide, a plurality of switches, the plurality switches comprising at least one leaf switch, wherein each of the one or more switches comprise a plurality of ports, and a plurality of end nodes, wherein the plurality of end nodes are interconnected via the one or more switches. The method can detect, by a subnet manager, a reconfiguration triggering event. The method can compute, by the subnet manager, a new routing for the interconnection network, wherein the computing by the subnet manager of the new routing for the interconnection network takes into consideration the pre-existing routing and selects the new routing for the interconnection network that is closest to the pre-existing routing. The method can reconfigure the interconnection network according to the new routing.},
	author = {Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Zahid, Feroz and Ernst Gunnar {Gran}}
}
@misc {24899,
	title = {System and method for efficient network reconfiguration in fat-trees},
	number = {US14927085},
	year = {2016},
	month = {05/2016},
	type = {Pending},
	abstract = {Systems and methods are provided for supporting efficient reconfiguration of an interconnection network having a pre-existing routing. An exemplary method can provide a plurality of switches, a plurality of end nodes, and one or more subnet managers, including a master subnet manager. The method can calculate, via the master subnet manager, a first set of one or more leaf-switch to leaf-switch multipaths. The method can store this first set of one or more leaf-switch to leaf-switch multipaths at a metabase. The method can detect a reconfiguration triggering event, and call a new routing for the interconnection network. Finally, the method can reconfigure the network according to the new routing for the interconnection network.},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen}}
}
@misc {24957,
	title = {System and Method for Providing an InfiniBand SR-IOV vSwitch Architecture for a High Performance Cloud Computing Environment},
	number = {US15/050,901},
	year = {2016},
	month = {02/2016},
	type = {Pending},
	abstract = {Systems and methods are provided for implementing a Virtual Switch (vSwitch) architecture that supports transparent virtualization and live migration. In an embodiment, a vSwitch with prepopulated Local Identifiers (LIDs). Another embodiment provides for vSwitch with dynamic LID assignment. Another embodiment provides for vSwitch with prepopulated LIDS and dynamic LID assignment Moreover, embodiments of the present invention provide scalable dynamic network reconfiguration methods which enable live migrations of VMs in network environments.},
	author = {Tasoulas, Evangelos and Bj{\o}rn Dag {Johnsen} and Ernst Gunnar {Gran}}
}
@misc {24634,
	title = {System and method for supporting efficient load-balancing in a high performance computing (HPC) environment},
	number = {US14792070},
	year = {2016},
	note = {Date Filed: October 19, 2015,\&nbsp;Published Online: Jan\&nbsp;14, 2016.},
	month = {01/16},
	type = {Pending},
	abstract = {Methods and systems for supporting efficient load balancing among a plurality of switches and a plurality of end nodes arranged in a tree topology in a network environment. The methods and systems can sort the plurality of end nodes, wherein the plurality of end nodes are sorted in a decreasing order of a receive weight. The method and system may further route, in the decreasing order of receive weights, the plurality of end nodes, wherein the routing comprises selecting at least one down-going port and at least one up-going port. Further, the method and system can increase an accumulated downward weight on each selected down-going port by the receive weight of the routed end node, and increase an accumulated upward weight on each selected up-going port by the receive weight of the routed end node},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen}}
}
@misc {24635,
	title = {System and method for supporting partition-aware routing in a multi-tenant cluster environment},
	number = {US14927085},
	year = {2016},
	note = {Date Filed: July 6, 2015,\&nbsp;Published Online: May 5, 2016.},
	month = {05/2016},
	type = {Pending},
	abstract = {A system and method can support partition-aware routing in a multi-tenant cluster environment. An exemplary method can support one or more tenants within the multi-tenant cluster environment. The method can associate each of the one or more tenants with a partition of a plurality of partitions. The method can then associate each of the plurality of partitions with one or more nodes of a plurality of nodes, each of the plurality of nodes being associated with a leaf switch of a plurality of switches, the plurality of switches comprising a plurality of leaf switches and a plurality of root switches. Finally, the method can generate one or more linear forwarding tables, the one or more linear forwarding tables providing isolation between the plurality of partitions, wherein each of the plurality of nodes is associated with a partitioning order.},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen}}
}
@misc {24956,
	title = {Systems and Method for Providing a Dynamic Cloud with Subnet Administration (SA) Query Caching},
	number = {US14924281},
	year = {2016},
	month = {05/2016},
	type = {Pending},
	abstract = {A system and method support can subnet management in a cloud environment. During a virtual machine migration in a cloud environment, a subnet manager can become a bottleneck point that delays efficient service. A system and method can alleviate this bottleneck point by ensuring a virtual machine retains a plurality of addresses after migration. The system and method can further allow for each host node within the cloud environment to be associated with a local cache that virtual machines can utilize when re-establishing communication with a migrated virtual machine.},
	issn = {20160127495},
	author = {Tasoulas, Evangelos and Bj{\o}rn Dag {Johnsen} and Ernst Gunnar {Gran}}
}
@article {23383,
	title = {Early experiences with live migration of SR-IOV enabled InfiniBand},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {78},
	year = {2015},
	month = {04/2015},
	pages = {39-52},
	publisher = {Elsevier},
	abstract = {Virtualization is the key to efficient resource utilization and elastic resource allocation in cloud computing. It enables consolidation, the on-demand provisioning of resources, and elasticity through live migration. Live migration makes it possible to optimize resource usage by moving virtual machines (VMs) between physical servers in an application transparent manner. It does, however, require a flexible, high-performance, scalable virtualized I/O architecture to reach its full potential. This is challenging to achieve with high-speed networks such as InfiniBand and remote direct memory access enhanced Ethernet, because these devices usually maintain their connection state in the network device hardware. Fortunately, the single root IO virtualization (SR-IOV) specification addresses the performance and scalability issues. With SR-IOV, each VM has direct access to a hardware assisted virtual device without the overhead introduced by emulation or para-virtualization. However, SR-IOV does not address the migration of the network device state. In this paper we present and evaluate the first available prototype implementation of live migration over SR-IOV enabled InfiniBand devices.},
	keywords = {Architecture, IO virtualization, SR-IOV, VM migration},
	doi = {http://dx.doi.org/10.1016/j.jpdc.2015.01.004},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Yen, Chien-Hua and Skeie, Tor and Lysne, Olav and T{\o}rudbakken, Ola}
}
@inproceedings {23376,
	title = {A Novel Query Caching Scheme for Dynamic InfiniBand Subnets},
	journal = {15th IEEE/ACM International Symposium on Cluster, Cloud and Grid Computing (CCGrid)},
	year = {2015},
	publisher = {ACM/IEEE},
	abstract = {In large InfiniBand subnets the Subnet Manager (SM) is a potential bottleneck. When an InfiniBand subnet grows in size, the number of paths between hosts increases polynomially and the SM may not be able to serve the network in a timely manner when many concurrent path resolution requests are received. This scalability challenge is further amplified in a dynamic virtualized cloud environment. When a Virtual Machine (VM) with InfiniBand interconnect live migrates, the VM addresses change. These address changes result in additional load to the SM as communicating peers send Subnet Administration (SA) path record queries to the SM to resolve new path characteristics.In this paper we benchmark OpenSM to empirically demonstrate the SM scalability problems. Then we show that our novel SA Path Record Query caching scheme significantly reduces the load towards the SM. In particular, we show by using the Reliable Datagram Socket protocol that only a single initial SA path query is needed per communicating peer, independent of any subsequent (re)connection attempts.},
	author = {Tasoulas, Evangelos and Ernst Gunnar {Gran} and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@inproceedings {19134,
	title = {Partition-aware routing to improve network isolation in InfiniBand based multi-tenant clusters},
	journal = {15th IEEE/ACM International Symposium on Cluster, Cloud and Grid Computing (CCGrid)},
	year = {2015},
	month = {07/2015},
	pages = {189-198},
	publisher = {ACM/IEEE},
	address = {Shenzhen, China},
	abstract = {InfiniBand (IB) is a widely used network interconnect for modern high-performance computing systems.\&nbsp;In large IB fabrics, network isolation is provided through partitioning.\&nbsp;However, routing is oblivious to the partitions in the network. Hence, physical links\&nbsp;share flows from different partitions. This sharing of the intermediate links creates\&nbsp;interference, which is particularly critical to avoid in multi-tenant environments, like cloud\&nbsp;computing. In such systems, each tenant needs predictable network performance, unaffected by the workload of\&nbsp;the other tenants. In addition, using the current routing schemes, despite that the links connecting nodes outside\&nbsp;partitions are never used, they are routed the same way as the other functional links. This may result in\&nbsp;degraded load-balancing.In this paper, we present an implementation of a partition-aware fat-tree routing algorithm, pFTree. The pFTree\&nbsp;utilizes a multifold mechanism to provide performance isolation among partitions belonging to the different tenant groups.\&nbsp;Given the available network resources, pFTree starts isolating partitions\&nbsp;at the physical link level, and then it moves on to utilize virtual lanes when needed. Our\&nbsp;experiments and simulations show that pFTree is able to significantly reduce the affect of\&nbsp;inter-partition interference effectively without any additional functional overhead. Furthermore, pFTree also provides\&nbsp;improved load-balancing over the state-of-the-art fat-tree routing algorithm.},
	isbn = {978-1-4799-8006-2},
	doi = {10.1109/CCGrid.2015.96},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@inproceedings {23558,
	title = {SlimUpdate: Minimal Routing Update for Performance-based Reconfigurations in Fat-Trees},
	journal = {1st IEEE International Workshop on High-Performance Interconnection Networks Towards the Exascale and Big-Data Era (HiPINEB 2015)},
	year = {2015},
	month = {10/2015},
	pages = {849-856},
	publisher = {IEEE Computer Society},
	abstract = {As the size of high-performance computing systems grows, the number of events requiring a network\&nbsp;reconfiguration, as well as the complexity of each reconfiguration, is likely to increase.\&nbsp;In large systems, the probability of component failure is high. At the same time, with more\&nbsp;network components, ensuring high utilization of network resources becomes challenging.\&nbsp;Reconfiguration in interconnection networks,\&nbsp;like InfiniBand (IB), typically involves computation and distribution of a new set of routes in order\&nbsp;to maintain connectivity and performance. In general, current routing algorithms do not consider the\&nbsp;existing routes in a network when calculating new ones. Such configuration-oblivious routing\&nbsp;might result in substantial modifications to the existing paths, and the reconfiguration\&nbsp;becomes costly as it potentially involves a large number of source-destination pairs.\&nbsp;In this paper, we propose a novel routing algorithm for IB based fat-tree topologies, SlimUpdate.\&nbsp;SlimUpdate employs techniques to preserve existing forwarding entries in switches to ensure a minimal\&nbsp;routing update, without any performance penalty, and with minimal computational overhead.\&nbsp;We present an implementation of SlimUpdate in OpenSM,\&nbsp;and compare it with the current de facto fat-tree routing algorithm. Our experiments and simulations show\&nbsp;a decrease of up to 80\% in the number of total path modifications when using SlimUpdate routing, while achieving\&nbsp;similar or even better performance than the fat-tree routing in most reconfiguration scenarios.},
	isbn = {978-1-4673-6598-7},
	doi = {10.1109/CLUSTER.2015.142},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@inproceedings {23478,
	title = {Towards the InfiniBand SR-IOV vSwitch Architecture},
	journal = {IEEE Cluster 2015},
	year = {2015},
	month = {09/2015},
	publisher = {IEEE},
	address = {IEEE Cluster 2015},
	abstract = {To meet the demands of the Exascale era and facilitate Big Data analytics in the cloud while maintaining flexibility, cloud providers will have to offer efficient virtualized High Performance Computing clusters in a pay-as-you-go model. As a consequence, high performance network interconnect solutions, like InfiniBand (IB), will be beneficial. Currently, the only way to provide IB connectivity on Virtual Machines (VMs) is by utilizing direct device assignment. At the same time to be scalable, Single-Root I/O Virtualization (SR-IOV) is used. However, the current SR-IOV model employed by IB adapters is a Shared Port implementation with limited flexibility, as it does not allow transparent virtualization and live-migration of VMs.In this paper, we explore an alternative SR-IOV model for IB, the virtual switch (vSwitch), and propose and analyze two vSwitch implementations with different scalability characteristics. Furthermore, as network reconfiguration time is critical to make live-migration a practical option, we accompany our proposed architecture with a scalable and topology agnostic dynamic reconfiguration method, implemented and tested using OpenSM. Our results show that we are able to significantly reduce the reconfiguration time as route recalculations are no longer needed, and in large IB subnets, for certain scenarios, the number of reconfiguration subnet management packets (SMPs) sent is reduced from several hundred thousand down to a single one.},
	keywords = {Dynamic Network Reconfiguration, InfiniBand, Live Migration, SR-IOV, Virtualization},
	author = {Tasoulas, Evangelos and Ernst Gunnar {Gran} and Bj{\o}rn Dag {Johnsen} and Begnum, Kyrre and Skeie, Tor}
}
@inproceedings {19102,
	title = {A weighted fat-tree routing algorithm for efficient load-balancing in InfiniBand enterprise clusters},
	journal = {Proceedings of the 23rd Euromicro International Conference on Parallel, Distributed Network-based Processing (PDP 2015)},
	year = {2015},
	month = {03/2015},
	pages = {35-42},
	publisher = {IEEE},
	address = {Turku, Finland},
	abstract = {InfiniBand (IB) has become a popular network interconnect for\&nbsp;high-performance computing (HPC) systems. Many of the large\&nbsp;IB-based HPC systems use some variant of the fat-tree topology to take advantage\&nbsp;of the useful properties fat-trees offer. The fat-tree routing algorithm\&nbsp;is one of the most efficient deterministic routing\&nbsp;algorithms for fat-tree topologies.\&nbsp;The algorithm ensures that the number of routes assigned to each link are balanced across the fabric. However,\&nbsp;one problem with its load-balancing technique is that it assumes\&nbsp;uniform traffic distribution in the network. When routes towards nodes that mainly consume\&nbsp;large amount of data are assigned to share links in the fabric while alternative links are underutilized,\&nbsp;sub-optimal network throughput is obtained. Also, as the fat-tree algorithm routes\&nbsp;nodes according to the indexing order, the performance may differ\&nbsp;for two systems cabled in the exact same way.In this paper, we propose wFatTree, a novel fat-tree routing algorithm, which considers node traffic characteristics to balance load\&nbsp;across the network links more evenly, and with predictable network performance. Our experiments\&nbsp;and simulations show an improvement of up to 60\% in total network throughput on large\&nbsp;fat-tree installations when using wFatTree routing. Furthermore, wFatTree can also be used to prioritize traffic flowing towards\&nbsp;the critical nodes in the network.},
	keywords = {fat-tree networks, InfiniBand, Load-balancing, Routing algorithms},
	issn = {1066-6192},
	doi = {10.1109/PDP.2015.111},
	author = {Zahid, Feroz and Ernst Gunnar {Gran} and Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Skeie, Tor}
}
@inproceedings {Simula.simula.2226,
	title = {Multi-Homed Fat-Tree Routing With InfiniBand},
	journal = {22nd Euromicro International Conference on Parallel, Distributed and Network-based Processing},
	year = {2014},
	month = {02/2014},
	pages = {122-129},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {For clusters where the topology consists of a fat-tree or more than one fat-tree combined into one subnet, there are several properties that the routing algorithms should support, beyond what exists today. One of the missing properties is that current fat-tree routing algorithm does not guarantee that each port on a multi-homed node is routed through redundant spines, even if these ports are connected to redundant leaves. As a consequence, in case of a spine failure, there is a small window where the node is unreachable until the subnet manager has rerouted to another spine. In this paper, we discuss the need for independent routes for multi-homed nodes in fat-trees by providing real-life examples when a single point of failure leads to complete outage of a multi-port node. We present and implement the methods that may be used to alleviate this problem and perform simulations that demonstrate improvements in performance, scalability, availability and predictability of InfiniBand fat-tree topologies. We show that our methods not only increase the performance by up to 52.6\%, but also, and more importantly, that there is no downtime associated with spine switch failure.},
	issn = {1066-6192},
	doi = {http://dx.doi.org/10.1109/PDP.2014.22},
	author = {Bogdanski, Bartosz and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen}}
}
@misc {27080,
	title = {System and method for signaling dynamic reconfiguration events in a middleware machine environment},
	number = {13/649,689},
	year = {2014},
	month = {11/2014},
	edition = {G06F11/1423},
	chapter = {US20130124910A1},
	abstract = {Rerouting around faulty components and migration of jobs both require reconfiguration of data structures in the Queue Pairs residing in the hosts on an InfiniBand cluster. In this patent we describe\&nbsp;an implementation of dynamic reconfiguration of such host side data-structures. Our implementation preserves the Queue Pairs, and lets the application run without being interrupted. With this implementation, we demonstrate a complete solution to fault tolerance in an InfiniBand network, where dynamic network reconfiguration to a topology-agnostic routing function is used to avoid malfunctioning components. This solution is in principle able to let applications run uninterruptedly on the cluster, as long as the topology is physically connected. Through measurements on our test-cluster we show that the increased cost of our method in setup latency is negligible, and that there is only a minor reduction in throughput during reconfiguration.},
	url = {https://patents.google.com/patent/US8880932B2},
	author = {Wei Lin {Guay} and Bj{\o}rn Dag {Johnsen} and Brean, David and Reinemo, Sven-Arne}
}
@inproceedings {Simula.simula.1996,
	title = {Making the Network Scalable: Inter-Subnet Routing in InfiniBand},
	journal = {Proceedings from the 19th International Euro-Par Conference on Parallel Processing},
	volume = {8097},
	year = {2013},
	month = {August},
	pages = {685-698},
	publisher = {Springer Berlin Heidelberg},
	type = {Conference},
	abstract = {As InfiniBand clusters grow in size and complexity, the need arises to segment the network into manageable sections. Up until now, InfiniBand routers have not been used extensively and little research has been done to accommodate them. However, the limits imposed on local addressing space, inability to logically segment fabrics, long reconfiguration times for large fabrics in case of faults, and, finally, performance issues when interconnecting large clusters, have rekindled the industry\&$\#$39;s interest into IB-IB routers. In this paper, we examine the routing problems that exist in the current implementation of OpenSM and we introduce two new routing algorithms for inter-subnet IB routing. We evaluate the performance of our routing algorithms against the current solution and we show an improvement of up to 100 times that of OpenSM.},
	keywords = {Conference},
	isbn = {978-3-642-40046-9},
	doi = {10.1007/978-3-642-40047-6\_69},
	author = {Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Reinemo, Sven-Arne and Flich, Jos{\'e}},
	editor = {Bernd Mohr Felix {Wolf} and Dieter an {Mey}}
}
@misc {Simula.simula.2496,
	title = {Prototyping Live Migration With SR-IOV Supported InfiniBand HCAs},
	howpublished = {HPC Advisory Council Spain Conference},
	year = {2013},
	month = {September},
	abstract = {Live migration is challenging to achieve with high-speed networks because these devices usually maintain their connection state in the network device hardware. In this work we i) describe the challenges with live migration over SR-IOV enabled InfiniBand devices, ii) present and evaluate the first available prototype implementation of live migration over SR-IOV enabled InfiniBand devices.},
	keywords = {Conference},
	author = {Tasoulas, Evangelos and Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Yen, Chien-Hua and Skeie, Tor and Lysne, Olav and Torudbakken, Ola}
}
@inproceedings {Simula.simula.1554,
	title = {Discovery and Routing of Degraded Fat-Trees},
	journal = {2012 13th International Conference on Parallel and Distributed Computing, Applications and Technologies},
	year = {2012},
	month = {December},
	pages = {689-694},
	publisher = {IEEE Computer Society},
	type = {Conference},
	address = {Los Alamitos},
	abstract = {The fat-tree topology has become a popular choice for InfiniBand enterprise systems due to its deadlock freedom, fault-tolerance and full bisection bandwidth. In the HPC domain, InfiniBand fabric is used in almost 42\% of the systems on the latest Top 500 list, and many of those systems are based on the fat-tree topology. Despite the popularity of the fat-tree topology, little research has been done to compare the behavior of InfiniBand routing algorithms on degraded fat-tree topologies. In this paper, we identify the weaknesses of the current fat-tree routing and propose enhancements that liberalize the restrictions imposed on the routed fabric. Furthermore, we present a thorough analysis of non-proprietary routing algorithms that are implemented in the InfiniBand Open Subnet Manager. Our results show that even though the performance of a fat-tree routed network deteriorates predictably with the number of failed links, fat-tree routing algorithm is still the best choice for severely degraded fat-tree fabrics.},
	keywords = {Conference},
	author = {Bogdanski, Bartosz and Bj{\o}rn Dag {Johnsen} and Reinemo, Sven-Arne and Frank Olaf {Sem-Jacobsen}},
	editor = {Shen, Hong and Sang, Yingpeng and Li, Yidong and Qian, Depai and Albert Y. {Zomaya}}
}
@misc {Simula.simula.1254,
	title = {Prototyping Live Migration With SR-IOV Supported InfiniBand},
	howpublished = {Contributed talk at the 2012 OpenFabrics International Workshop, Monterey, USA},
	year = {2012},
	month = {March},
	keywords = {Workshop},
	author = {Wei Lin {Guay} and Bj{\o}rn Dag {Johnsen} and Torudbakken, Ola and Yen, Chien-Hua and Reinemo, Sven-Arne and Frank, Richard}
}
@inproceedings {Simula.simula.1528,
	title = {A Scalable Signalling Mechanism for VM Migration With SR-IOV Over InfiniBand},
	journal = {18th IEEE International Conference on Parallel and Distributed Systems (ICPADS)},
	year = {2012},
	pages = {384-391},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Single Root I/O Virtualization (SR-IOV) is a promising I/O virtualization approach for achieving high performance in the virtualization over InfiniBand (IB) network. One challenge is related to the hardware address assignment for each virtual IB device. There are two schemes for the hardware address assignment; static assignment and dynamic assignment. Static assignment always preserves the hardware address of a virtual IB device that is attached to a VM, but the dynamic assignment does not. A drawback, however, using static assignment is that its communication will be disconnected after VM migration. In this paper, we point out the problem related to SRIOV over IB that breaks the network connections after VM migration when the static assignment is deployed. Then, we propose a signalling mechanism that can maintain the network connectivity after VM migration. The performance evaluation using an experimental test bed shows that the proposed signalling mechanism does not increase the service downtime during hot migration. We also optimize the signalling method, where the same event can only be forwarded to a physical server once regardless of the hosted VMs, to reduce the management message overhead from O(n {\_\ast} m) to O(n).},
	keywords = {Conference},
	isbn = {978-0-7695-4903-3},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Bj{\o}rn Dag {Johnsen} and Skeie, Tor and Torudbakken, Ola},
	editor = {Tang, Xueyan and Xu, Cheng-Zhong and He, Bingsheng}
}
@inproceedings {Simula.netsys.12,
	title = {Host Side Dynamic Reconfiguration With InfiniBand},
	journal = {2010 IEEE International Conference on Cluster Computing},
	year = {2010},
	pages = {126-135},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Rerouting around faulty components and migration of jobs both require reconfiguration of data structures in the Queue Pairs residing in the hosts on an InfiniBand cluster. In this paper we report an implementation of dynamic reconfiguration of such host side data-structures. Our implementation preserves the Queue Pairs, and lets the application run without being interrupted. With this implementation, we demonstrate a complete solution to fault tolerance in an InfiniBand network, where dynamic network reconfiguration to a topology-agnostic routing function is used to avoid malfunctioning components. This solution is in principle able to let applications run uninterruptedly on the cluster, as long as the topology is physically connected. Through measurements on our test-cluster we show that the increased cost of our method in setup latency is negligible, and that there is only a minor reduction in throughput during reconfiguration.},
	isbn = {978-0-7695-4220-1},
	author = {Wei Lin {Guay} and Reinemo, Sven-Arne and Lysne, Olav and Skeie, Tor and Bj{\o}rn Dag {Johnsen} and Holen, Line},
	editor = {Gu, Xiaohui and Ma, Xiaosong}
}
@article {Simula.ND.18,
	title = {The Interconnection Network - Architectural Challenges for Utility Computing Data Centres},
	journal = {IEEE Computer Magazine},
	volume = {41},
	number = {9},
	year = {2008},
	month = {September},
	pages = {62-69},
	publisher = {IEEE},
	abstract = {The mode of operation employed by Computational Data Centres that offer Utility Computing differs significantly from that of traditional supercomputers and server clusters and as such present new architectural problems that should be studied and solved. In this paper we concentrate on issues facing the interconnection network. We argue that this is a part of the overall architecture where shortcomings in present day solutions are most severe and present a model for the mode of operation of a Utility Computing Data Centre where virtualisation is a main ingredient. Based on this model we identify several areas where the interconnection network faces new challenges and needs new solutions. In each of these areas we give a brief introduction to previous results before we identify the new challenges.},
	author = {Lysne, Olav and Reinemo, Sven-Arne and Skeie, Tor and {\r A}shild Gr{\o}nstad {Solheim} and S{\o}dring, Thomas and Lars Paul {Huse} and Bj{\o}rn Dag {Johnsen}}
}
@techreport {25342,
	title = {The Interconnection Network - Architectural Challenges for Utility Computing Data Centres},
	year = {2007},
	note = {This technical report is an earlier version of a published journal article. The published version can be found here:https://www.simula.no/publications/interconnection-network-architectural-challenges-utility-computing-data-centres},
	publisher = {Simula Research Laboratory},
	author = {Lysne, Olav and Reinemo, Sven-Arne and Skeie, Tor and {\r A}shild Gr{\o}nstad {Solheim} and S{\o}dring, Thomas and Lars Paul {Huse} and Bj{\o}rn Dag {Johnsen}}
}
@inproceedings {Sem-jacobsen.2005.1,
	title = {Siamese-Twin: a Dynamically Fault-Tolerant Fat Tree},
	journal = {International Parallel and Distributed Processing Symposium (IPDPS), Denver, Colorado, USA, April 4-8},
	year = {2005},
	publisher = {IEEE Computer Society},
	type = {Conference},
	abstract = {Fat-trees are a special case of multistage interconnection networks with quite good static fault tolerance capabilities. They are however straightforwardly unable to provide local dynamic fault tolerance. In this paper we propose a network topology based on the fat-tree using two parallel networks with crossover links between them in an effort to enable dynamic fault tolerance. We evaluate and compare this topology with two other similar fat-tree topologies and show through simulations that the new topology is able to improve slightly upon the ability to tolerate faults statically. More importantly, we show that the new network topology is the only one of the evaluated topologies able to tolerate one fault dynamically, with a superior network performance in the face of dynamically handled faults.},
	isbn = {0-7695-2312-9},
	author = {Frank Olaf {Sem-Jacobsen} and Skeie, Tor and Lysne, Olav and T{\o}rudbakken, Ola and Rongved, Eivind and Bj{\o}rn Dag {Johnsen}}
}
@misc {ND.8.Musunoori.2004.d,
	title = {The Interconnection Network: Architectural Challenges for Datacenters in the Computational GRID},
	howpublished = {Presented at the eVITA Workshop},
	year = {2004},
	note = {Presented at the eVITA Workshop on eScience and Applications},
	month = {October 12-13},
	author = {Lysne, Olav and Skeie, Tor and Theiss, I. and Bj{\o}rn Dag {Johnsen} and Rognved, E. and T{\o}rudbakken, O.}
}
