@article {,
	title = {Revisiting I/O bandwidth-sharing strategies for HPC applications},
	number = {RR-9502},
	year = {2023},
	month = {2023-03},
	institution = {INRIA},
	abstract = {This work revisits I/O bandwidth-sharing strategies for HPC applications. When several applications post concurrent I/O operations, well-known approaches include serializing these operations (First-Come First-Served) or fair-sharing the bandwidth across them (FairShare). Another recent approach, I/O-Sets, assigns priorities to the applications, which are classified into different sets based upon the average length of their iterations. We introduce several new bandwidth-sharing strategies, some of them simple greedy algorithms, and some of them more complicated to implement, and we compare them with existing ones. Our new strategies do not rely on any a-priori knowledge of the behavior of the applications, such as the length of work phases, the volume of I/O operations, or some expected periodicity. We introduce a rigorous framework, namely steady-state windows, which enables to derive bounds on the competitive ratio of all bandwidth-sharing strategies for three different objectives: minimum yield, platform utilization, and global efficiency. To the best of our knowledge, this work is the first to provide a quantitative assessment of the online competitiveness of any bandwidth-sharing strategy. This theory-oriented assessment is complemented by a comprehensive set of simulations, based upon both synthetic and realistic traces. The main conclusion is that our simple and low-complexity greedy strategies significantly outperform First-Come First-Served, FairShare and I/O-Sets, and we recommend that the I/O community implements them for further assessment.},
	keywords = {bandwidth sharing, HPC applications, I/O, scheduling strategy},
	url = {https://hal.inria.fr/hal-04038011},
	author = {Anne Benoit and Thomas Herault and Lucas Perotin and Yves Robert and Frederic Vivien}
}
@conference {,
	title = {When to checkpoint at the end of a fixed-length reservation?},
	booktitle = {Fault Tolerance for HPC at eXtreme Scales (FTXS) Workshop},
	year = {2023},
	month = {2023-08},
	address = {Denver, United States},
	abstract = {This work considers an application executing for a fixed duration, namely the length of the reservation that it has been granted. The checkpoint duration is a stochastic random variable that obeys some well-known probability distribution law. The question is when to take a checkpoint towards the end of the execution, so that the expectation of the work done is maximized. We address two scenarios. In the first scenario, a checkpoint can be taken at any time; despite its simplicity, this natural problem has not been considered yet (to the best of our knowledge). We provide the optimal solution for a variety of probability distribution laws modeling checkpoint duration. The second scenario is more involved: the application is a linear workflow consisting of a chain of tasks with IID stochastic execution times, and a checkpoint can be taken only at the end of a task. First, we introduce a static strategy where we compute the optimal number of tasks before the application checkpoints at the beginning of the execution. Then, we design a dynamic strategy that decides whether to checkpoint or to continue executing at the end of each task. We instantiate this second scenario with several examples of probability distribution laws for task durations.},
	url = {https://inria.hal.science/hal-04215554},
	author = {Quentin Barbut and Anne Benoit and Thomas Herault and Yves Robert and Frederic Vivien}
}
@inproceedings {,
	title = {Checkpointing {\`a} la Young/Daly: An Overview},
	journal = {IC3-2022: Proceedings of the 2022 Fourteenth International Conference on Contemporary Computing},
	year = {2022},
	month = {2022-08},
	pages = {701-710},
	publisher = {ACM Press},
	address = {Noida, India},
	abstract = {The Young/Daly formula provides an approximation of the optimal checkpoint period for a parallel application executing on a supercomputing platform. The Young/Daly formula was originally designed for preemptible tightly-coupled applications. We provide some background and survey various application scenarios to assess the usefulness and limitations of the formula.},
	isbn = {9781450396752},
	doi = {10.1145/3549206},
	url = {https://dl.acm.org/doi/fullHtml/10.1145/3549206.3549328},
	author = {Anne Benoit and Yishu Du and Thomas Herault and Loris Marchal and Guillaume Pallez and Lucas Perotin and Yves Robert and Hongyang Sun and Frederic Vivien}
}
@conference {,
	title = {Distributed-Memory Multi-GPU Block-Sparse Tensor Contraction for Electronic Structure},
	booktitle = {35th IEEE International Parallel \&  Distributed Processing Symposium (IPDPS 2021)},
	year = {2021},
	month = {2021-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Portland, OR},
	abstract = {Many domains of scientific simulation (chemistry, condensed matter physics, data science) increasingly eschew dense tensors for block-sparse tensors, sometimes with additional structure (recursive hierarchy, rank sparsity, etc.). Distributed-memory parallel computation with block-sparse tensorial data is paramount to minimize the time-tosolution (e.g., to study dynamical problems or for real-time analysis) and to accommodate problems of realistic size that are too large to fit into the host/device memory of a single node equipped with accelerators. Unfortunately, computation with such irregular data structures is a poor match to the dominant imperative, bulk-synchronous parallel programming model. In this paper, we focus on the critical element of block-sparse tensor algebra, namely binary tensor contraction, and report on an efficient and scalable implementation using the task-focused PaRSEC runtime. High performance of the block-sparse tensor contraction on the Summit supercomputer is demonstrated for synthetic data as well as for real data involved in electronic structure simulations of unprecedented size.},
	keywords = {block-sparse matrix multiplication, distributed-memory, Electronic structure, multi-GPU node, parsec, tensor contraction},
	url = {https://hal.inria.fr/hal-02970659/document},
	author = {Thomas Herault and Yves Robert and George Bosilca and Robert Harrison and Cannada Lewis and Edward Valeev and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications},
	year = {2021},
	month = {2021-04},
	publisher = {2021 Exascale Computing Project Annual Meeting},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {,
	title = {Revisiting Credit Distribution Algorithms for Distributed Termination Detection},
	journal = {2021 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2021},
	pages = {611{\textendash}620},
	publisher = {IEEE},
	abstract = {This paper revisits distributed termination detection
algorithms in the context of High-Performance Computing (HPC)
applications. We introduce an efficient variant of the Credit
Distribution Algorithm (CDA) and compare it to the original
algorithm (HCDA) as well as to its two primary competitors: the
Four Counters algorithm (4C) and the Efficient Delay-Optimal
Distributed algorithm (EDOD). We analyze the behavior of each
algorithm for some simplified task-based kernels and show the
superiority of CDA in terms of the number of control messages.},
	keywords = {control messages, credit distribution algorithms, task-based HPC application, Termination detection},
	doi = {10.1109/IPDPSW52791.2021.00095},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Le F{\`e}vre, Valentin and Robert, Yves and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Systems and Interfaces (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {1460,
	title = {Overhead of Using Spare Nodes},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2020},
	month = {2020-02},
	abstract = {With the increasing fault rate on high-end supercomputers, the topic of fault tolerance has been gathering attention. To cope with this situation, various fault-tolerance techniques are under investigation; these include user-level, algorithm-based fault-tolerance techniques and parallel execution environments that enable jobs to continue following node failure. Even with these techniques, some programs with static load balancing, such as stencil computation, may underperform after a failure recovery. Even when spare nodes are present, they are not always substituted for failed nodes in an effective way. This article considers the questions of how spare nodes should be allocated, how to substitute them for faulty nodes, and how much the communication performance is affected by such a substitution. The third question stems from the modification of the rank mapping by node substitutions, which can incur additional message collisions. In a stencil computation, rank mapping is done in a straightforward way on a Cartesian network without incurring any message collisions. However, once a substitution has occurred, the optimal node-rank mapping may be destroyed. Therefore, these questions must be answered in a way that minimizes the degradation of communication performance. In this article, several spare node allocation and failed node substitution methods will be proposed, analyzed, and compared in terms of communication performance following the substitution. The proposed substitution methods are named sliding methods. The sliding methods are analyzed by using our developed simulation program and evaluated by using the K computer, Blue Gene/Q (BG/Q), and TSUBAME 2.5. It will be shown that when failures occur, the stencil communication performance on the K and BG/Q can be slowed around 10 times depending on the number of node failures. The barrier performance on the K can be cut in half. On BG/Q, barrier performance can be slowed by a factor of 10. Further, it will also be shown that almost no such communication performance degradation can be seen on TSUBAME 2.5. This is because TSUBAME 2.5 has an Infiniband network connected with a FatTree topology, while the K computer and BG/Q have dedicated Cartesian networks. Thus, the communication performance degradation depends on network characteristics.},
	keywords = {communication performance, fault mitigation, Fault tolerance, sliding method, spare node},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177\%2F1094342020901885},
	url = {https://journals.sagepub.com/doi/10.1177/1094342020901885},
	author = {Atsushi Hori and Kazumi Yoshinaga and Thomas Herault and Aurelien Bouteiller and George Bosilca and Yutaka Ishikawa}
}
@conference {,
	title = {The Template Task Graph (TTG) - An Emerging Practical Dataflow Programming Paradigm for Scientific Simulation at Extreme Scale},
	booktitle = { 2020 IEEE/ACM 5th International Workshop on Extreme Scale Programming Models and Middleware (ESPM2)},
	year = {2020},
	month = {2020-11},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We describe TESSE, an emerging general-purpose, open-source software ecosystem that attacks the twin challenges of programmer productivity and portable performance for advanced scientific applications on modern high-performance computers. TESSE builds upon and extends the ParsecDAG/-dataflow runtime with a new Domain Specific Languages (DSL) and new integration capabilities. Motivating this work is our belief that such a dataflow model, perhaps with applications composed in domain specific languages, can overcome many of the challenges faced by a wide variety of irregular applications that are poorly served by current programming and execution models. Two such applications from many-body physics and applied mathematics are briefly explored. This paper focuses upon the Template Task Graph (TTG), which is TESSE{\textquoteright}s main C++ Api that provides a powerful work/data-flow programming model. Algorithms on spatial trees, block-sparse tensors, and wave fronts are used to illustrate the API and associated concepts, as well as to compare with related approaches.},
	keywords = {dag, dataflow, exascale, graph, High-performance computing, workflow},
	doi = {https://doi.org/10.1109/ESPM251964.2020.00011},
	author = {George Bosilca and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Poornima Nookala and Edward Valeev}
}
@article {1305,
	title = {Checkpointing Strategies for Shared High-Performance Computing Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {9},
	number = {1},
	year = {2019},
	pages = {28{\textendash}52},
	abstract = {Input/output (I/O) from various sources often contend for scarcely available bandwidth. For example, checkpoint/restart (CR) protocols can help to ensure application progress in failure-prone environments. However, CR I/O alongside an application{\textquoteright}s normal, requisite I/O can increase I/O contention and might negatively impact performance. In this work, we consider different aspects (system-level scheduling policies and hardware) that optimize the overall performance of concurrently executing CR-based applications that share I/O resources. We provide a theoretical model and derive a set of necessary constraints to minimize the global waste on a given platform. Our results demonstrate that Young/Daly{\textquoteright}s optimal checkpoint interval, despite providing a sensible metric for a single, undisturbed application, is not sufficient to optimally address resource contention at scale. We show that by combining optimal checkpointing periods with contention-aware system-level I/O scheduling strategies, we can significantly improve overall application performance and maximize the platform throughput. Finally, we evaluate how specialized hardware, namely burst buffers, may help to mitigate the I/O contention problem. Overall, these results provide critical analysis and direct guidance on how to design efficient, CR ready, large -scale platforms without a large investment in the I/O subsystem.},
	issn = {2185-2847},
	url = {http://www.ijnc.org/index.php/ijnc/article/view/195},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@article {1301,
	title = {Comparing the Performance of Rigid, Moldable, and Grid-Shaped Applications on Failure-Prone HPC Platforms},
	journal = {Parallel Computing},
	volume = {85},
	year = {2019},
	month = {2019-07},
	pages = {1{\textendash}12},
	doi = {https://doi.org/10.1016/j.parco.2019.02.002},
	author = {Valentin Le F{\`e}vre and Thomas Herault and Yves Robert and Aurelien Bouteiller and Atsushi Hori and George Bosilca and Jack Dongarra}
}
@conference {1410,
	title = {Generic Matrix Multiplication for Multi-GPU Accelerated Distributed-Memory Platforms over PaRSEC},
	booktitle = {ScalA{\textquoteright}19: 10th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	author = {Thomas Herault and Yves Robert and George Bosilca and Jack Dongarra}
}
@conference {1452,
	title = {Performance Analysis of Tile Low-Rank Cholesky Factorization Using PaRSEC Instrumentation Tools},
	booktitle = {Workshop on Programming and Performance Visualization Tools (ProTools 19) at SC19},
	year = {2019},
	month = {2019-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	author = {Qinglei Cao and Yu Pei and Thomas Herault and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@conference {1371,
	title = {Replication is More Efficient Than You Think},
	booktitle = {The IEEE/ACM Conference on High Performance Computing Networking, Storage and Analysis (SC19)},
	year = {2019},
	month = {2019-11},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Denver, CO},
	author = {Anne Benoit and Thomas Herault and Valentin Le F{\`e}vre and Yves Robert}
}
@conference {1378,
	title = {Software-Defined Events through PAPI},
	booktitle = {2019 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {PAPI has been used for almost two decades as an abstraction and standardization layer for profiling hardware-specific performance metrics. However, application developers-and profiling software packages-are quite often interested in information beyond hardware counters, such as the behavior of libraries used by the software that is being profiled. So far, accessing this information has required interfacing directly with the libraries on a case-by-case basis, or low-level binary instrumentation. In this paper, we introduce the new Software-Defined Event (SDE) component of PAPI which aims to enable PAPI to serve as an abstraction and standardization layer for events that originate in software layers as well. Extending PAPI to include SDEs enables monitoring of both types of performance events-hardware-and software-related events-in a uniform way, through the same consistent PAPI interface. Furthermore, implementing SDE as a PAPI component means that the new API is aimed only at the library developers who wish to export events from within their libraries. The API for reading PAPI events-both hardware and software-remains the same, so all legacy codes and tools that use PAPI will not only continue to work, but they will automatically be able to read SDEs wherever those are available. The goal of this paper is threefold. First, we outline our design decisions regarding the functionality we offer through the new SDE interface, and offer simple examples of usage. Second, we illustrate how those events can be utilized by different software packages, specifically, by showcasing their use in the task-based runtime PaRSEC, and the HPCG supercomputing benchmark. Third, we provide a thorough performance analysis of the overhead that results from monitoring different types of SDEs, and showcase the negligible overhead of using PAPI SDE even in cases of extremely heavy use.},
	doi = {https://doi.org/10.1109/IPDPSW.2019.00069},
	author = {Anthony Danalis and Heike Jagode and Thomas Herault and Piotr Luszczek and Jack Dongarra}
}
@inbook {1302,
	title = {System Software for Many-Core and Multi-Core Architectures},
	booktitle = {Advanced Software Technologies for Post-Peta Scale Computing: The Japanese Post-Peta CREST Research Project},
	year = {2019},
	pages = {59{\textendash}75},
	publisher = {Springer Singapore},
	organization = {Springer Singapore},
	address = {Singapore},
	abstract = {In this project, the software technologies for the post-peta scale computing were explored. More specifically, OS technologies for heterogeneous architectures, lightweight thread, scalable I/O, and fault mitigation were investigated. As for the OS technologies, a new parallel execution model, Partitioned Virtual Address Space (PVAS), for the many-core CPU was proposed. For the heterogeneous architectures, where multi-core CPU and many-core CPU are connected with an I/O bus, an extension of PVAS, Multiple-PVAS, to have a unified virtual address space of multi-core and many-core CPUs was proposed. The proposed PVAS was also enhanced to have multiple processes where process context switch can take place at the user level (named User-Level Process: ULP). As for the scalable I/O, EARTH, optimization techniques for MPI collective I/O, was proposed. Lastly, for the fault mitigation, User Level Fault Mitigation, ULFM was improved to have faster agreement process, and sliding methods to substitute failed nodes with spare nodes was proposed. The funding of this project was ended in 2016; however, many proposed technologies are still being propelled.},
	isbn = {978-981-13-1924-2},
	doi = {10.1007/978-981-13-1924-2_4},
	url = {https://doi.org/10.1007/978-981-13-1924-2_4},
	author = {Atsushi Hori and Tsujita, Yuichi and Shimada, Akio and Yoshinaga, Kazumi and Mitaro, Namiki and Fukazawa, Go and Sato, Mikiko and George Bosilca and Aurelien Bouteiller and Thomas Herault},
	editor = {Sato, Mitsuhisa}
}
@techreport {1200,
	title = {Data Movement Interfaces to Support Dataflow Runtimes},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-03},
	year = {2018},
	month = {2018-05},
	publisher = {University of Tennessee},
	abstract = {This document presents the design study and reports on the implementation of a portable hosted accelerator device support in the PaRSEC Dataflow Tasking at Exascale runtime, undertaken as part of the ECP contract 17-SC-20-SC. The document discusses different technological approaches to transfer data to/from hosted accelerators, issues recommendations for technology providers, and presents the design of an OpenMP-based accelerator support in PaRSEC.},
	author = {Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@techreport {1320,
	title = {Distributed Termination Detection for HPC Task-Based Environments},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-14},
	year = {2018},
	month = {2018-06},
	publisher = {University of Tennessee},
	abstract = {This paper revisits distributed termination detection algorithms in the context of high-performance computing applications in task systems. We first outline the need to efficiently detect termination in workflows for which the total number of tasks is data dependent and therefore not known statically but only revealed dynamically during execution. We introduce an efficient variant of the Credit Distribution Algorithm (CDA) and compare it to the original algorithm (HCDA) as well as to its two primary competitors: the Four Counters algorithm (4C) and the Efficient Delay-Optimal Distributed algorithm (EDOD). On the theoretical side, we analyze the behavior of each algorithm for some simplified task-based kernels and show the superiority of CDA in terms of the number of control messages. On the practical side, we provide a highly tuned implementation of each termination detection algorithm within PaRSEC and compare their performance for a variety of benchmarks, extracted from scientific applications that exhibit dynamic behaviors.},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Valentin Le F{\`e}vre and Yves Robert and Jack Dongarra}
}
@conference {1214,
	title = {Do moldable applications perform better on failure-prone HPC platforms?},
	booktitle = {11th Workshop on Resiliency in High Performance Computing in Clusters, Clouds, and Grids},
	series = {LNCS},
	year = {2018},
	month = {2018-08},
	publisher = {Springer Verlag},
	organization = {Springer Verlag},
	address = {Turin, Italy},
	abstract = {This paper compares the performance of different approaches to tolerate failures using checkpoint/restart when executed on large-scale failure-prone platforms. We study (i) Rigid applications, which use a constant number of processors throughout execution; (ii) Moldable applications, which can use a different number of processors after each restart following a fail-stop error; and (iii) GridShaped applications, which are moldable applications restricted to use rectangular processor grids (such as many dense linear algebra kernels). For each application type, we compute the optimal number of failures to tolerate before relinquishing the current allocation and waiting until a new resource can be allocated, and we determine the optimal yield that can be achieved. We instantiate our performance model with a realistic applicative scenario and make it publicly available for further usage.},
	author = {Valentin Le F{\`e}vre and George Bosilca and Aurelien Bouteiller and Thomas Herault and Atsushi Hori and Yves Robert and Jack Dongarra}
}
@article {1089,
	title = {A Failure Detector for HPC Platforms},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-01},
	pages = {139{\textendash}158},
	abstract = {Building an infrastructure for exascale applications requires, in addition to many other key components, a stable and efficient failure detector. This article describes the design and evaluation of a robust failure detector that can maintain and distribute the correct list of alive resources within proven and scalable bounds. The detection and distribution of the fault information follow different overlay topologies that together guarantee minimal disturbance to the applications. A virtual observation ring minimizes the overhead by allowing each node to be observed by another single node, providing an unobtrusive behavior. The propagation stage uses a nonuniform variant of a reliable broadcast over a circulant graph overlay network and guarantees a logarithmic fault propagation. Extensive simulations, together with experiments on the Titan Oak Ridge National Laboratory supercomputer, show that the algorithm performs extremely well and exhibits all the desired properties of an exascale-ready algorithm.},
	keywords = {failure detection, Fault tolerance, MPI},
	doi = {https://doi.org/10.1177/1094342017711505},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@conference {1196,
	title = {Optimal Cooperative Checkpointing for Shared High-Performance Computing Platforms},
	booktitle = {2018 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW), Best Paper Award},
	year = {2018},
	month = {2018-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Vancouver, BC, Canada},
	abstract = {In high-performance computing environments, input/output (I/O) from various sources often contend for scarce available bandwidth. Adding to the I/O operations inherent to the failure-free execution of an application, I/O from checkpoint/restart (CR) operations (used to ensure progress in the presence of failures) place an additional burden as it increase I/O contention, leading to degraded performance. In this work, we consider a cooperative scheduling policy that optimizes the overall performance of concurrently executing CR-based applications which share valuable I/O resources. First, we provide a theoretical model and then derive a set of necessary constraints needed to minimize the global waste on the platform. Our results demonstrate that the optimal checkpoint interval, as defined by Young/Daly, despite providing a sensible metric for a single application, is not sufficient to optimally address resource contention at the platform scale. We therefore show that combining optimal checkpointing periods with I/O scheduling strategies can provide a significant improvement on the overall application performance, thereby maximizing platform throughput. Overall, these results provide critical analysis and direct guidance on checkpointing large-scale workloads in the presence of competing I/O while minimizing the impact on application performance.},
	doi = {10.1109/IPDPSW.2018.00127},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@techreport {1280,
	title = {Tensor Contraction on Distributed Hybrid Architectures using a Task-Based Runtime System},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-13},
	year = {2018},
	month = {2018-12},
	publisher = {University of Tennessee},
	abstract = {The needs for predictive simulation of electronic structure in chemistry and materials science calls for fast/reduced-scaling formulations of quantum n-body methods that replace the traditional dense tensors with element-, block-, rank-, and block-rank-sparse (data-sparse) tensors. The resulting, highly irregular data structures are a poor match to imperative, bulk-synchronous parallel programming style due to the dynamic nature of the problem and to the lack of clear domain decomposition to guarantee a fair load-balance. TESSE runtime and the associated programming model aim to support performance-portable composition of applications involving irregular and dynamically changing data. In this paper we report an implementation of irregular dense tensor contraction in a paradigmatic electronic structure application based on the TESSE extension of PaRSEC, a distributed hybrid task runtime system, and analyze the resulting performance on a distributed memory cluster of multi-GPU nodes. Unprecedented strong scaling and promising efficiency indicate a viable future for task-based programming of complete production-quality reduced scaling models of electronic structure.},
	author = {George Bosilca and Damien Genet and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Chong Peng and Edward Valeev}
}
@article {1176,
	title = {Argobots: A Lightweight Low-Level Threading and Tasking Framework},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	year = {2017},
	month = {2017-10},
	abstract = {In the past few decades, a number of user-level threading and tasking models have been proposed in the literature to address the shortcomings of OS-level threads, primarily with respect to cost and flexibility. Current state-of-the-art user-level threading and tasking models, however, are either too specific to applications or architectures or are not as powerful or flexible. In this paper, we present Argobots, a lightweight, low-level threading and tasking framework that is designed as a portable and performant substrate for high-level programming models or runtime systems. Argobots offers a carefully designed execution model that balances generality of functionality with providing a rich set of controls to allow specialization by the user or high-level programming model. We describe the design, implementation, and optimization of Argobots and present integrations with three example high-level models: OpenMP, MPI, and co-located I/O service. Evaluations show that (1) Argobots outperforms existing generic threading runtimes; (2) our OpenMP runtime offers more efficient interoperability capabilities than production OpenMP runtimes do; (3) when MPI interoperates with Argobots instead of Pthreads, it enjoys reduced synchronization costs and better latency hiding capabilities; and (4) I/O service with Argobots reduces interference with co-located applications, achieving performance competitive with that of the Pthreads version.},
	keywords = {Argobots, context switch, I/O, interoperability, lightweight, MPI, OpenMP, stackable scheduler, tasklet, user-level thread},
	doi = {10.1109/TPDS.2017.2766062},
	url = {http://ieeexplore.ieee.org/document/8082139/},
	author = {Sangmin Seo and Abdelhalim Amer and Pavan Balaji and Cyril Bordage and George Bosilca and Alex Brooks and Philip Carns and Adrian Castello and Damien Genet and Thomas Herault and Shintaro Iwasaki and Prateek Jindal and Sanjay Kale and Sriram Krishnamoorthy and Jonathan Lifflander and Huiwei Lu and Esteban Meneses and Mar Snir and Yanhua Sun and Kenjiro Taura and Pete Beckman}
}
@inproceedings {1183,
	title = {Dynamic Task Discovery in PaRSEC- A data-flow task-based Runtime},
	journal = {ScalA17},
	year = {2017},
	month = {2017-09},
	publisher = {ACM},
	address = {Denver},
	abstract = {Successfully exploiting distributed collections of heterogeneous many-cores architectures with complex memory hierarchy through a portable programming model is a challenge for application developers. The literature is not short of proposals addressing this problem, including many evolutionary solutions that seek to extend the capabilities of current message passing paradigms with intranode features (MPI+X). A different, more revolutionary, solution explores data-flow task-based runtime systems as a substitute to both local and distributed data dependencies management. The solution explored in this paper, PaRSEC, is based on such a programming paradigm, supported by a highly efficient task-based runtime. This paper compares two programming paradigms present in PaRSEC, Parameterized Task Graph (PTG) and Dynamic Task Discovery (DTD) in terms of capabilities, overhead and potential benefits.},
	keywords = {data-flow, dynamic task-graph, parsec, task-based runtime},
	isbn = {978-1-4503-5125-6},
	doi = {10.1145/3148226.3148233},
	url = {https://dl.acm.org/citation.cfm?doid=3148226.3148233},
	author = {Reazul Hoque and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {924,
	title = {Assessing the Cost of Redistribution followed by a Computational Kernel: Complexity and Performance Results},
	journal = {Parallel Computing},
	volume = {52},
	year = {2016},
	month = {2016-02},
	pages = {22-41},
	abstract = {The classical redistribution problem aims at optimally scheduling communications when reshuffling from an initial data distribution to a target data distribution. This target data distribution is usually chosen to optimize some objective for the algorithmic kernel under study (good computational balance or low communication volume or cost), and therefore to provide high efficiency for that kernel. However, the choice of a distribution minimizing the target objective is not unique. This leads to generalizing the redistribution problem as follows: find a re-mapping of data items onto processors such that the data redistribution cost is minimal, and the operation remains as efficient. This paper studies the complexity of this generalized problem. We compute optimal solutions and evaluate, through simulations, their gain over classical redistribution. We also show the NP-hardness of the problem to find the optimal data partition and processor permutation (defined by new subsets) that minimize the cost of redistribution followed by a simple computational kernel. Finally, experimental validation of the new redistribution algorithms are conducted on a multicore cluster, for both a 1D-stencil kernel and a more compute-intensive dense linear algebra routine.},
	keywords = {Data partition, linear algebra, parsec, QR factorization, Redistribution, Stencil},
	doi = {doi:10.1016/j.parco.2015.09.005},
	author = {Julien Herrmann and George Bosilca and Thomas Herault and Loris Marchal and Yves Robert and Jack Dongarra}
}
@techreport {978,
	title = {Context Identifier Allocation in Open MPI},
	journal = {University of Tennessee Computer Science Technical Report},
	number = { ICL-UT-16-01},
	year = {2016},
	month = {2016-01},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	abstract = {The concept of communicators is a central notion in Message Passing Interface, allowing on one side the MPI implemen- tation to specialize it{\textquoteright}s matching and deliver messages in the right context, and on the other side the library developers to contextualize their message exchanges, and scope different algorithms to well-defined groups of processes. More pre- cisely, all communication objects in MPI are derived from a communicator at some point. All MPI functions allowing the creation of new communicators have a collective mean- ing, either over the group of processes from the parent com- municator or those from the target communicator. Thus, the perfromance of the communicator creation is tied to the col- lective communication performance, as well as the amount of data needed to be exchanged in order to consistently create this new communicator. We introduce several communica- tor creation algorithms, and present their implementation in the context of Open MPI. We explore the performance of these new algorithms and compare them with state-of-the- art algorithms available in other MPI implementations.},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {979,
	title = {Failure Detection and Propagation in HPC Systems},
	journal = { Proceedings of the The International Conference for High Performance Computing, Networking, Storage and Analysis (SC{\textquoteright}16)},
	year = {2016},
	month = {2016-11},
	pages = {27:1-27:11},
	publisher = {IEEE Press},
	address = {Salt Lake City, Utah},
	keywords = {failure detection, fault-tolerance, MPI},
	isbn = {978-1-4673-8815-3},
	url = {http://dl.acm.org/citation.cfm?id=3014904.3014941},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@article {820,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations, Multiple Failures, and Accuracy},
	journal = {ACM Transactions on Parallel Computing},
	volume = {1},
	number = {10},
	year = {2015},
	month = {2015-01},
	pages = {10:1-10:28},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that
require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations
are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the
Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault
Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme
conditions, such as the absence of any reliable node and the possibility of losing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead decreases inversely to the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors. Applicability to tolerate multiple failures and accuracy after multiple recovery is also considered.},
	keywords = {ABFT, algorithms, fault-tolerance, High Performance Computing, linear algebra},
	doi = {10.1145/2686892},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Peng Du and Jack Dongarra},
	editor = {Phillip B. Gibbons}
}
@article {852,
	title = {Composing Resilience Techniques: ABFT, Periodic, and Incremental Checkpointing},
	journal = {International Journal of Networking and Computing},
	volume = {5},
	number = {1},
	year = {2015},
	month = {2015-01},
	pages = {2-15},
	abstract = {Algorithm Based Fault Tolerant (ABFT) approaches promise unparalleled scalability and performance in failure-prone environments. Thanks to recent advances in the understanding of the involved mechanisms, a growing number of important algorithms (including all widely used factorizations) have been proven ABFT-capable. In the context of larger applications, these algorithms provide a temporal section of the execution, where the data is protected by its own intrinsic properties, and can therefore be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only practical fault-tolerance approach for these applications is checkpoint/restart. In this paper we propose a model to investigate the efficiency of a composite protocol, that alternates between ABFT and checkpoint/restart for the effective protection of an iterative application composed of ABFT- aware and ABFT-unaware sections. We also consider an incremental checkpointing composite approach in which the algorithmic knowledge is leveraged by a novel optimal dynamic program- ming to compute checkpoint dates. We validate these models using a simulator. The model and simulator show that the composite approach drastically increases the performance delivered by an execution platform, especially at scale, by providing the means to increase the interval between checkpoints while simultaneously decreasing the volume of each checkpoint. },
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, model, performance evaluation, resilience},
	issn = {ISSN 2185-2839},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@conference {840,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and George Bosilca and Thomas Herault and Jack Dongarra}
}
@techreport {919,
	title = {Fault Tolerance Techniques for High-performance Computing},
	journal = {University of Tennessee Computer Science Technical Report (also LAWN 289)},
	number = {UT-EECS-15-734},
	year = {2015},
	month = {2015-05},
	publisher = {University of Tennessee},
	abstract = {This report provides an introduction to resilience methods. The emphasis is on checkpointing, the de-facto standard technique for resilience in High Performance Computing. We present the main two protocols, namely coordinated checkpointing and hierarchical checkpointing. Then we introduce performance models and use them to assess the performance of theses protocols. We cover the Young/Daly formula for the optimal period and much more! Next we explain how the efficiency of checkpointing can be improved via fault prediction or replication. Then we move to application-specific methods, such as ABFT. We conclude the report by discussing techniques to cope with silent errors (or silent data corruption).},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn289.pdf},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@inproceedings {1310,
	title = {From MPI to OpenSHMEM: Porting LAMMPS},
	journal = {OpenSHMEM and Related Technologies. Experiences, Implementations, and Technologies},
	year = {2015},
	pages = {121{\textendash}137},
	publisher = {Springer International Publishing},
	address = {Annapolis, MD, USA},
	abstract = {This work details the opportunities and challenges of porting a Petascale, MPI-based application {\textendash}-LAMMPS{\textendash}- to OpenSHMEM. We investigate the major programming challenges stemming from the differences in communication semantics, address space organization, and synchronization operations between the two programming models. This work provides several approaches to solve those challenges for representative communication patterns in LAMMPS, e.g., by considering group synchronization, peer{\textquoteright}s buffer status tracking, and unpacked direct transfer of scattered data. The performance of LAMMPS is evaluated on the Titan HPC system at ORNL. The OpenSHMEM implementations are compared with MPI versions in terms of both strong and weak scaling. The results outline that OpenSHMEM provides a rich semantic to implement scalable scientific applications. In addition, the experiments demonstrate that OpenSHMEM can compete with, and often improve on, the optimized MPI implementation.},
	isbn = {978-3-319-26428-8},
	doi = {10.1007/978-3-319-26428-8_8},
	author = {Tang, Chunyan and Aurelien Bouteiller and Thomas Herault and Manjunath Gorentla Venkata and George Bosilca},
	editor = {Manjunath Gorentla Venkata and Shamis, Pavel and Imam, Neena and M. Graham Lopez}
}
@techreport {872,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems: Formal Proof},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-15-01},
	year = {2015},
	month = {2015-04},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@conference {889,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {The ability to consistently handle faults in a distributed environment requires, among a small set of basic routines, an agreement algorithm allowing surviving entities to reach a consensual decision between a bounded set of volatile resources. This paper presents an algorithm that implements an Early Returning Agreement (ERA) in pseudo-synchronous systems, which optimistically allows a process to resume its activity while guaranteeing strong progress. We prove the correctness of our ERA algorithm, and expose its logarithmic behavior, which is an extremely desirable property for any algorithm which targets future exascale platforms. We detail a practical implementation of this consensus algorithm in the context of an MPI library, and evaluate both its efficiency and scalability through a set of benchmarks and two fault tolerant scientific applications.},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@conference {708,
	title = {Assessing the Impact of ABFT and Checkpoint Composite Strategies},
	booktitle = {16th Workshop on Advances in Parallel and Distributed Computational Models, IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFTunaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@techreport {822,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	journal = {ICL Technical Report},
	number = {ICL-UT-14-04},
	year = {2014},
	month = {2014-11},
	publisher = {University of Tennessee},
	abstract = {Abstract{\textemdash}As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {867,
	title = {A Multithreaded Communication Substrate for OpenSHMEM},
	booktitle = {8th International Conference on Partitioned Global Address Space Programming Models (PGAS)},
	year = {2014},
	month = {2014-10},
	address = {Eugene, OR},
	abstract = {OpenSHMEM scalability is strongly dependent on the capa- bility of its communication layer to efficiently handle multi- ple threads. In this paper, we present an early evaluation of the thread safety specification in the Unified Common Com- munication Substrate (UCCS) employed in OpenSHMEM. Results demonstrate that thread safety can be provided at an acceptable cost and can improve efficiency for some op- erations, compared to serializing communication.},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca}
}
@article {814,
	title = {Performance and Reliability Trade-offs for the Double Checkpointing Algorithm},
	journal = {International Journal of Networking and Computing},
	volume = {4},
	number = {1},
	year = {2014},
	month = {2014},
	pages = {32-41},
	chapter = {32},
	abstract = {Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach based upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kal{\'e} [23], with the non-blocking algorithm of Ni, Meneses and Kal{\'e} [15] in terms of both performance and risk. We also extend the model proposedcan provide a better efficiency in [23, 15] to assess the impact of the overhead associated to non-blocking communications. In addition, we deal with arbitrary failure distributions (as opposed to uniform distributions in [23]). We then provide a new peer-to-peer checkpointing algorithm, called the triple checkpointing algorithm, that can work without additional memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	keywords = {communication contention, in-memory checkpoint, performance, resilience, risk},
	issn = {2185-2847},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@conference {871,
	title = {PTG: An Abstraction for Unhindered Parallelism},
	booktitle = {International Workshop on Domain-Specific Languages and High-Level Frameworks for High Performance Computing (WOLFHPC)},
	year = {2014},
	month = {2014-11},
	publisher = {IEEE Press},
	organization = {IEEE Press},
	address = {New Orleans, LA},
	abstract = {<p>
Increased parallelism and use of heterogeneous computing resources is now an established trend in High Performance Computing (HPC), a trend that, looking forward to Exascale, seems bound to intensify. Despite the evolution of hardware over the past decade, the programming paradigm of choice was invariably derived from Coarse Grain Parallelism with explicit data movements. We argue that message passing has remained the de facto standard in HPC because, until now, the ever increasing challenges that application developers had to address to create efficient portable applications remained manageable for expert programmers.
</p>
<p>
Data-flow based programming is an alternative approach with significant potential. In this paper, we discuss the Parameterized Task Graph (PTG) abstraction and present the specialized input language that we use to specify PTGs in our data-flow task-based runtime system, PaRSEC. This language and the corresponding execution model are in contrast with the execution model of explicit message passing as well as the model of alternative task based runtime systems. The Parameterized Task Graph language decouples the expression of the parallelism in the algorithm from the control-flow ordering, load balance, and data distribution. Thus, programs are more adaptable and map more efficiently on challenging hardware, as well as maintain portability across diverse architectures. To support these claims, we discuss the different challenges of HPC programming and how PaRSEC can address them, and we demonstrate that in today{\textquoteright}s large scale supercomputers, PaRSEC can significantly outperform state-of-the-art MPI applications and libraries, a trend that will increase with future architectural evolution.
</p>},
	keywords = {dte, parsec, plasma},
	author = {Anthony Danalis and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@techreport {694,
	title = {Assessing the impact of ABFT and Checkpoint composite strategies},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ICL-UT-13-03},
	year = {2013},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFT-unaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@techreport {684,
	title = {On the Combination of Silent Error Detection and Checkpointing},
	journal = {UT-CS-13-710},
	year = {2013},
	month = {2013-06},
	publisher = {University of Tennessee Computer Science Technical Report},
	abstract = {In this paper, we revisit traditional checkpointing and rollback recovery strategies, with a focus on silent data corruption errors. Contrarily to fail-stop failures, such latent errors cannot be detected immediately, and a mechanism to detect them must be provided. We consider two models: (i) errors are detected after some delays following a probability distribution (typically, an Exponential distribution); (ii) errors are detected through some verification mechanism. In both cases, we compute the optimal period in order to minimize the waste, i.e., the fraction of time where nodes do not perform useful computations. In practice, only a fixed number of checkpoints can be kept in memory, and the first model may lead to an irrecoverable failure. In this case, we compute the minimum period required for an acceptable risk. For the second model, there is no risk of irrecoverable failure, owing to the verification mechanism, but the corresponding overhead is included in the waste. Finally, both models are instantiated using realistic scenarios and application/architecture parameters.},
	keywords = {checkpointing, error recovery, High-performance computing, silent data corruption, verification},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn278.pdf},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@article {750,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {25},
	year = {2013},
	month = {2013-03},
	pages = {572-585},
	abstract = {With our current expectation for the exascale systems, composed of hundred of thousands of many-core nodes, the mean time between failures will become small, even under the most optimistic assumptions. One of the most scalable checkpoint restart techniques, the message logging approach, is the most challenged when the number of cores per node increases because of the high overhead of saving the message payload. Fortunately, for two processes on the same node, the failure probability is correlated, meaning that coordinated recovery is free. In this paper, we propose an intermediate approach that uses coordination between correlated processes but retains the scalability advantage of message logging between independent ones. The algorithm still belongs to the family of event logging protocols but eliminates the need for costly payload logging between coordinated processes.},
	doi = {10.1002/cpe.2859},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {icl:698,
	title = {Dense Linear Algebra on Distributed Heterogeneous Hardware with a Symbolic DAG Approach},
	journal = {Scalable Computing and Communications: Theory and Practice},
	year = {2013},
	month = {2013-03},
	pages = {699-735},
	publisher = {John Wiley \& Sons},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Piotr Luszczek and Jack Dongarra},
	editor = {Samee Khan and Lin-Wang Wang and Albert Zomaya}
}
@article {869,
	title = {An evaluation of User-Level Failure Mitigation support in MPI},
	journal = {Computing},
	volume = {95},
	year = {2013},
	month = {2013-12},
	pages = {1171-1184},
	abstract = {As the scale of computing platforms becomes increasingly extreme, the requirements for application fault tolerance are increasing as well. Techniques to address this problem by improving the resilience of algorithms have been developed, but they currently receive no support from the programming model, and without such support, they are bound to fail. This paper discusses the failure-free overhead and recovery impact of the user-level failure mitigation proposal presented in the MPI Forum. Experiments demonstrate that fault-aware MPI has little or no impact on performance for a range of applications, and produces satisfactory recovery times when there are failures.},
	keywords = {Fault tolerance, MPI, User-level fault mitigation},
	doi = {10.1007/s00607-013-0331-3},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@article {691,
	title = {Extending the scope of the Checkpoint-on-Failure protocol for forward recovery in standard MPI},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-07},
	abstract = {Most predictions of exascale machines picture billion ways parallelism, encompassing not only millions of cores but also tens of thousands of nodes. Even considering extremely optimistic advances in hardware reliability, probabilistic amplification entails that failures will be unavoidable. Consequently, software fault tolerance is paramount to maintain future scientific productivity. Two major problems hinder ubiquitous adoption of fault tolerance techniques: (i) traditional checkpoint-based approaches incur a steep overhead on failure free operations and (ii) the dominant programming paradigm for parallel applications (the message passing interface (MPI) Standard) offers extremely limited support of software-level fault tolerance approaches. In this paper, we present an approach that relies exclusively on the features of a high quality implementation, as defined by the current MPI Standard, to enable advanced forward recovery techniques, without incurring the overhead of customary periodic checkpointing. With our approach, when failure strikes, applications regain control to make a checkpoint before quitting execution. This checkpoint is in reaction to the failure occurrence rather than periodic. This checkpoint is reloaded in a new MPI application, which restores a sane environment for the forward, application-based recovery technique to repair the failure-damaged dataset. The validity and performance of this approach are evaluated on large-scale systems, using the QR factorization as an example. Published 2013. This article is a US Government work and is in the public domain in the USA.},
	doi = {10.1002/cpe.3100},
	url = {http://doi.wiley.com/10.1002/cpe.3100},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {752,
	title = {Hierarchical QR Factorization Algorithms for Multi-core Cluster Systems},
	journal = {Parallel Computing},
	volume = {39},
	year = {2013},
	month = {2013-05},
	pages = {212-232},
	abstract = {This paper describes a new QR factorization algorithm which is especially designed for massively parallel platforms combining parallel distributed nodes, where a node is a multi-core processor. These platforms represent the present and the foreseeable future of high-performance computing. Our new QR factorization algorithm falls in the category of the tile algorithms which naturally enables good data locality for the sequential kernels executed by the cores (high sequential performance), low number of messages in a parallel distributed setting (small latency term), and fine granularity (high parallelism). Each tile algorithm is uniquely characterized by its sequence of reduction trees. In the context of a cluster of nodes, in order to minimize the number of inter-processor communications (aka, {\textquoteleft}{\textquoteleft}communication-avoiding{\textquoteright}{\textquoteright}), it is natural to consider hierarchical trees composed of an {\textquoteleft}{\textquoteleft}inter-node{\textquoteright}{\textquoteright} tree which acts on top of {\textquoteleft}{\textquoteleft}intra-node{\textquoteright}{\textquoteright} trees. At the intra-node level, we propose a hierarchical tree made of three levels: (0) {\textquoteleft}{\textquoteleft}TS level{\textquoteright}{\textquoteright} for cache-friendliness, (1) {\textquoteleft}{\textquoteleft}low-level{\textquoteright}{\textquoteright} for decoupled highly parallel inter-node reductions, (2) {\textquoteleft}{\textquoteleft}domino level{\textquoteright}{\textquoteright} to efficiently resolve interactions between local reductions and global reductions. Our hierarchical algorithm and its implementation are flexible and modular, and can accommodate several kernel types, different distribution layouts, and a variety of reduction trees at all levels, both inter-node and intra-node. Numerical experiments on a cluster of multi-core nodes (i) confirm that each of the four levels of our hierarchical tree contributes to build up performance and (ii) build insights on how these levels influence performance and interact within each other. Our implementation of the new algorithm with the DAGUE scheduling tool significantly outperforms currently available QR factorization software for all matrix shapes, thereby bringing a new advance in numerical linear algebra for petascale and exascale platforms.},
	keywords = {Cluster, Distributed memory, Hierarchical architecture, multi-core, numerical linear algebra, QR factorization},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Mathias Jacquelin and Julien Langou and Yves Robert}
}
@techreport {icl:733,
	title = {Multi-criteria checkpointing strategies: optimizing response-time versus resource utilization},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ICL-UT-13-01},
	year = {2013},
	month = {2013-02},
	abstract = {Failures are increasingly threatening the eciency of HPC systems, and current projections of Exascale platforms indicate that rollback recovery, the most convenient method for providing fault tolerance to generalpurpose applications, reaches its own limits at such scales. One of the reasons explaining this unnerving situation comes from the focus that has been given to per-application completion time, rather than to platform efficiency. In this paper, we discuss the case of uncoordinated rollback recovery where the idle time spent waiting recovering processors is used to progress a different, independent application from the system batch queue. We then propose an extended model of uncoordinated checkpointing that can discriminate between idle time and wasted computation. We instantiate this model in a simulator to demonstrate that, with this strategy, uncoordinated checkpointing per application completion time is unchanged, while it delivers near-perfect platform efficiency.},
	author = {Aurelien Bouteiller and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert}
}
@conference {868,
	title = {Multi-criteria Checkpointing Strategies: Response-Time versus Resource Utilization},
	booktitle = {Euro-Par 2013},
	year = {2013},
	month = {2013-08},
	publisher = {Springer},
	organization = {Springer},
	address = {Aachen, Germany},
	abstract = {Failures are increasingly threatening the efficiency of HPC systems, and current projections of Exascale platforms indicate that roll- back recovery, the most convenient method for providing fault tolerance to general-purpose applications, reaches its own limits at such scales. One of the reasons explaining this unnerving situation comes from the focus that has been given to per-application completion time, rather than to platform efficiency. In this paper, we discuss the case of uncoordinated rollback recovery where the idle time spent waiting recovering processors is used to progress a different, independent application from the sys- tem batch queue. We then propose an extended model of uncoordinated checkpointing that can discriminate between idle time and wasted com- putation. We instantiate this model in a simulator to demonstrate that, with this strategy, uncoordinated checkpointing per application comple- tion time is unchanged, while it delivers near-perfect platform efficiency.},
	author = {Aurelien Bouteiller and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert}
}
@techreport {702,
	title = {Optimal Checkpointing Period: Time vs. Energy},
	journal = {University of Tennessee Computer Science Technical Report (also LAWN 281)},
	number = {ut-eecs-13-718},
	year = {2013},
	month = {2013-10},
	publisher = {University of Tennessee},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Jack Dongarra}
}
@article {749,
	title = {PaRSEC: Exploiting Heterogeneity to Enhance Scalability},
	journal = {IEEE Computing in Science and Engineering},
	volume = {15},
	year = {2013},
	month = {2013-11},
	pages = {36-45},
	abstract = {New high-performance computing system designs with steeply escalating processor and core counts, burgeoning heterogeneity and accelerators, and increasingly unpredictable memory access times call for dramatically new programming paradigms. These new approaches must react and adapt quickly to unexpected contentions and delays, and they must provide the execution environment with sufficient intelligence and flexibility to rearrange the execution to improve resource utilization.},
	doi = {10.1109/MCSE.2013.98},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Thomas Herault and Jack Dongarra}
}
@article {693,
	title = {Post-failure recovery of MPI communication capability: Design and rationale},
	journal = {International Journal of High Performance Computing Applications},
	volume = {27},
	year = {2013},
	month = {2013-01},
	pages = {244 - 254},
	abstract = {As supercomputers are entering an era of massive parallelism where the frequency of faults is increasing, the MPI Standard remains distressingly vague on the consequence of failures on MPI communications. Advanced fault-tolerance techniques have the potential to prevent full-scale application restart and therefore lower the cost incurred for each failure, but they demand from MPI the capability to detect failures and resume communications afterward. In this paper, we present a set of extensions to MPI that allow communication capabilities to be restored, while maintaining the extreme level of performance to which MPI users have become accustomed. The motivation behind the design choices are weighted against alternatives, a task that requires simultaneously considering MPI from the viewpoint of both the user and the implementor. The usability of the interfaces for expressing advanced recovery techniques is then discussed, including the difficult issue of enabling separate software layers to coordinate their recovery. },
	issn = {1094-3420},
	doi = {10.1177/1094342013488238},
	url = {http://hpc.sagepub.com/cgi/doi/10.1177/1094342013488238},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@techreport {icl:735,
	title = {Revisiting the Double Checkpointing Algorithm},
	journal = {University of Tennessee Computer Science Technical Report (LAWN 274)},
	number = {ut-cs-13-705},
	year = {2013},
	month = {2013-01},
	abstract = {Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach base upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kal{\'e}, with the non-blocking algorithm of Ni, Meneses and Kal{\'e} in terms of both performance and risk. We also extend the model that they have proposed to assess the impact of the overhead associated to non-blocking communications. We then provide a new peer-to-peer checkpointing algorithm, called the triple checkpointing algorithm, that can work at constant memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	keywords = {checkpoint algorithm, communication overlap, fault-tolerance, performance model, resilience},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@conference {717,
	title = {Revisiting the Double Checkpointing Algorithm},
	booktitle = {15th Workshop on Advances in Parallel and Distributed Computational Models, at the IEEE International Parallel \& Distributed Processing Symposium},
	year = {2013},
	month = {2013-05},
	address = {Boston, MA},
	abstract = {Abstract{\textemdash}Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach base upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kale [1], with the non-blocking algorithm of Ni, Meneses and Kale [2] in terms of both performance and risk. We also extend the model proposed in [1], [2] to assess the impact of the overhead associated to non-blocking communications. We then provide a new peer-topeer checkpointing algorithm, called the triple checkpointing algorithm, that can work at constant memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@inbook {695,
	title = {Scalable Dense Linear Algebra on Heterogeneous Hardware},
	booktitle = {HPC: Transition Towards Exascale Processing, in the series Advances in Parallel Computing},
	year = {2013},
	abstract = {Abstract. Design of systems exceeding 1 Pflop/s and the push toward 1 Eflop/s, forced a dramatic shift in hardware design. Various physical and engineering constraints resulted in introduction of massive parallelism and functional hybridization with the use of accelerator units. This paradigm change brings about a serious challenge for application developers, as the management of multicore proliferation and heterogeneity rests on software. And it is reasonable to expect, that this situation will not change in the foreseeable future. This chapter presents a methodology of dealing with this issue in three common scenarios. In the context of shared-memory multicore installations, we show how high performance and scalability go hand in hand, when the well-known linear algebra algorithms are recast in terms of Direct Acyclic Graphs (DAGs), which are then transparently scheduled at runtime inside the Parallel Linear Algebra Software for Multicore Architectures (PLASMA) project. Similarly, Matrix Algebra on GPU and Multicore Architectures (MAGMA) schedules DAG-driven computations on multicore processors and accelerators. Finally, Distributed PLASMA (DPLASMA), takes the approach to distributed-memory machines with the use of automatic dependence analysis and the Direct Acyclic Graph Engine (DAGuE) to deliver high performance at the scale of many thousands of cores.},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@article {748,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-11},
	abstract = {In this paper, we present a unified model for several well-known checkpoint/restart protocols. The proposed model is generic enough to encompass both extremes of the checkpoint/restart space, from coordinated approaches to a variety of uncoordinated checkpoint strategies (with message logging). We identify a set of crucial parameters, instantiate them, and compare the expected efficiency of the fault tolerant protocols, for a given application/platform pair. We then propose a detailed analysis of several scenarios, including some of the most powerful currently available high performance computing platforms, as well as anticipated Exascale designs. The results of this analytical comparison are corroborated by a comprehensive set of simulations. Altogether, they outline comparative behaviors of checkpoint strategies at very large scale, thereby providing insight that is hardly accessible to direct experimentation.},
	doi = {10.1002/cpe.3173},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@inproceedings {icl:672,
	title = {Algorithm-Based Fault Tolerance for Dense Matrix Factorization},
	journal = {Proceedings of the 17th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP 2012},
	year = {2012},
	month = {2012-02},
	pages = {225-234},
	publisher = {ACM},
	address = {New Orleans, LA, USA},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme conditions, such as the absence of any reliable component and the possibility of loosing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead sharply decreases with the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors.},
	keywords = {ft-la, ftmpi},
	doi = {10.1145/2145816.2145845},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra},
	editor = {J. Ramanujam and P. Sadayappan}
}
@inproceedings {icl:679,
	title = {A Checkpoint-on-Failure Protocol for Algorithm-Based Recovery in Standard MPI},
	journal = {18th International European Conference on Parallel and Distributed Computing (Euro-Par 2012) (Best Paper Award)},
	year = {2012},
	month = {2012-08},
	publisher = {Springer-Verlag},
	address = {Rhodes, Greece},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Christos Kaklamanis and Theodore Papatheodorou and Paul Spirakis}
}
@article {icl:670,
	title = {DAGuE: A generic distributed DAG Engine for High Performance Computing.},
	journal = {Parallel Computing},
	volume = {38},
	number = {1-2},
	year = {2012},
	month = {2012-00},
	pages = {27-51},
	publisher = {Elsevier},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@inproceedings {icl:680,
	title = {An Evaluation of User-Level Failure Mitigation Support in MPI},
	journal = {Proceedings of Recent Advances in Message Passing Interface - 19th European MPI Users{\textquoteright} Group Meeting, EuroMPI 2012},
	year = {2012},
	month = {2012-09},
	publisher = {Springer},
	address = {Vienna, Austria},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@techreport {icl:724,
	title = {Extending the Scope of the Checkpoint-on-Failure Protocol for Forward Recovery in Standard MPI},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ut-cs-12-702},
	year = {2012},
	month = {2012-00},
	keywords = {ftmpi},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {icl:699,
	title = {From Serial Loops to Parallel Execution on Distributed Systems},
	booktitle = {International European Conference on Parallel and Distributed Computing (Euro-Par {\textquoteright}12)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes, Greece},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jack Dongarra}
}
@inproceedings {icl:687,
	title = {Hierarchical QR Factorization Algorithms for Multi-Core Cluster Systems},
	journal = {IPDPS 2012, the 26th IEEE International Parallel and Distributed Processing Symposium},
	year = {2012},
	month = {2012-05},
	publisher = {IEEE Computer Society Press},
	address = {Shanghai, China},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Julien Langou and Yves Robert}
}
@techreport {icl:667,
	title = {A Proposal for User-Level Failure Mitigation in the MPI-3 Standard},
	journal = {University of Tennessee Electrical Engineering and Computer Science Technical Report},
	number = {ut-cs-12-693},
	year = {2012},
	month = {2012-02},
	publisher = {University of Tennessee},
	keywords = {ftmpi},
	author = {Wesley Bland and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@techreport {icl:716,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	journal = {University of Tennessee Computer Science Technical Report (also LAWN 269)},
	number = {UT-CS-12-697},
	year = {2012},
	month = {2012-06},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@techreport {icl:626,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-11-676},
	year = {2011},
	month = {2011-08},
	address = {Knoxville, TN},
	keywords = {ft-la},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {icl:673,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	journal = {Proceedings of 17th International Conference, Euro-Par 2011, Part II},
	volume = {6853},
	year = {2011},
	month = {2011-08},
	pages = {51-64},
	publisher = {Springer},
	address = {Bordeaux, France},
	keywords = {ftmpi},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Emmanuel Jeannot and Raymond Namyst and Jean Roman}
}
@inproceedings {icl:675,
	title = {DAGuE: A Generic Distributed DAG Engine for High Performance Computing},
	journal = {Proceedings of the Workshops of the 25th IEEE International Symposium on Parallel and Distributed Processing (IPDPS 2011 Workshops)},
	year = {2011},
	month = {2011-00},
	pages = {1151-1158},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@inproceedings {icl:676,
	title = {Flexible Development of Dense Linear Algebra Algorithms on Massively Parallel Architectures with DPLASMA},
	journal = {Proceedings of the Workshops of the 25th IEEE International Symposium on Parallel and Distributed Processing (IPDPS 2011 Workshops)},
	year = {2011},
	month = {2011-05},
	pages = {1432-1441},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, dplasma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@techreport {icl:645,
	title = {Hierarchical QR Factorization Algorithms for Multi-Core Cluster Systems},
	journal = {University of Tennessee Computer Science Technical Report (also Lawn 257)},
	number = {UT-CS-11-684},
	year = {2011},
	month = {2011-10},
	keywords = {magma, plasma},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Julien Langou and Yves Robert}
}
@article {icl:636,
	title = {Performance Portability of a GPU Enabled Factorization with the DAGuE Framework},
	journal = {IEEE Cluster: workshop on Parallel Programming on Accelerator Clusters (PPAC)},
	year = {2011},
	month = {2011-06},
	keywords = {dague, magma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {icl:648,
	title = {Process Distance-aware Adaptive MPI Collective Communications},
	journal = {IEEE Int{\textquoteright}l Conference on Cluster Computing (Cluster 2011)},
	year = {2011},
	month = {2011-00},
	address = {Austin, Texas},
	author = {Teng Ma and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {icl:677,
	title = {QCG-OMPI: MPI Applications on Grids.},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2011},
	month = {2011-01},
	pages = {435-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@techreport {icl:612,
	title = {On Scalability for MPI Runtime Systems},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ICL-UT-11-05},
	year = {2011},
	month = {2011-05},
	address = {Knoxville, TN},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:671,
	title = {On Scalability for MPI Runtime Systems},
	journal = {International Conference on Cluster Computing (CLUSTER)},
	year = {2011},
	month = {2011-09},
	pages = {187-195},
	publisher = {IEEEE},
	address = {Austin, TX, USA},
	keywords = {harness},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:674,
	title = {Scalable Runtime for MPI: Efficiently Building the Communication Infrastructure},
	journal = {Proceedings of Recent Advances in the Message Passing Interface - 18th European MPI Users{\textquoteright} Group Meeting, EuroMPI 2011},
	volume = {6960},
	year = {2011},
	month = {2011-09},
	pages = {342-344},
	publisher = {Springer},
	address = {Santorini, Greece},
	keywords = {ftmpi},
	author = {George Bosilca and Thomas Herault and Pierre Lemariner and Jack Dongarra and A. Rezmerita},
	editor = {Yiannis Cotronis and Anthony Danalis and Dimitrios S. Nikolopoulos and Jack Dongarra}
}
@inproceedings {icl:593,
	title = {A Unified HPC Environment for Hybrid Manycore/GPU Distributed Systems},
	journal = {IEEE International Parallel and Distributed Processing Symposium (submitted)},
	year = {2011},
	month = {2011-05},
	address = {Anchorage, AK},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@article {icl:555,
	title = {Constructing Resiliant Communication Infrastructure for Runtime Environments in Advances in Parallel Computing},
	journal = {Advances in Parallel Computing - Parallel Computing: From Multicores and GPU{\textquoteright}s to Petascale},
	volume = {19},
	year = {2010},
	pages = {441-451},
	doi = {10.3233/978-1-60750-530-3-441},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Barbara Chapman and Frederic Desprez and Gerhard R. Joubert and Alain Lichnewsky and Frans Peters and T. Priol}
}
@techreport {icl:528,
	title = {DAGuE: A generic distributed DAG engine for high performance computing},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-10-01},
	year = {2010},
	month = {2010-04},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@techreport {icl:563,
	title = {Distributed Dense Numerical Linear Algebra Algorithms on Massively Parallel Architectures: DPLASMA},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-10-660},
	year = {2010},
	month = {2010-09},
	keywords = {dague, dplasma, parsec, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@techreport {icl:529,
	title = {Distributed-Memory Task Execution and Dependence Tracking within DAGuE and the DPLASMA Project},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-10-02},
	year = {2010},
	month = {2010-00},
	keywords = {dague, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@inproceedings {icl:534,
	title = {Dodging the Cost of Unavoidable Memory Copies in Message Logging Protocols},
	journal = {Proceedings of EuroMPI 2010},
	year = {2010},
	month = {2010-09},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {ftmpi},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Jack Dongarra and Michael Resch and Rainer Keller and Edgar Gabriel}
}
@article {icl:574,
	title = {QCG-OMPI: MPI Applications on Grids},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2010},
	month = {2010-03},
	pages = {357-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@inproceedings {icl:532,
	title = {QR Factorization of Tall and Skinny Matrices in a Grid Computing Environment},
	journal = {24th IEEE International Parallel and Distributed Processing Symposium (also LAWN 224)},
	year = {2010},
	month = {2010-04},
	address = {Atlanta, GA},
	author = {Emmanuel Agullo and Camille Coti and Jack Dongarra and Thomas Herault and Julien Langou}
}
@techreport {icl:484,
	title = {Constructing resiliant communication infrastructure for runtime environments},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-09-02},
	year = {2009},
	month = {2009-07},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@article {icl:517,
	title = {Constructing Resilient Communication Infrastructure for Runtime Environments},
	journal = {ParCo 2009},
	year = {2009},
	month = {2009-09},
	address = {Lyon France},
	author = {Pierre Lemariner and George Bosilca and Camille Coti and Thomas Herault and Jack Dongarra}
}
