\author{Raphaël Couturier}
\maketitle
-
+\cleardoublepage
\frontmatter
%\include{frontmatter/Foreword}
-\include{frontmatter/preface}
\tableofcontents
\listoffigures
\listoftables
+\include{frontmatter/preface}
\mainmatter
\label{chapter1}
\section{Introduction}\label{ch1:intro}
-
+``test" "test" ``test''
This chapter introduces the Graphics Processing Unit (GPU) architecture and all
the concepts needed to understand how GPUs work and can be used to speed up the
execution of some algorithms. First of all this chapter gives a brief history of
threads are different from traditional threads for a CPU. In
Chapter~\ref{chapter2}, some examples of GPU programming will explain the
details of the GPU threads. Threads are gathered into blocks of 32
-threads, called warps. These warps are important when designing an algorithm
+threads, called ``warps''. These warps are important when designing an algorithm
for GPU.
@Book{succi-book,
author = {Succi, S.},
- title = {The lattice Boltzmann equation and beyond},
+ title = {The lattice {B}oltzmann {E}quation and {B}eyond},
publisher = {Oxford University Press, Oxford},
year = {2001},
}
@Article{desplat,
author = {Desplat, J.-C. and Pagonabarraga, I. and Bladon P.},
- title = {LUDWIG: A parallel lattice-Boltzmann code for complex fluids},
+ title = {LUDWIG: A parallel lattice-{B}oltzmann code for complex fluids},
journal = {Comput. Phys. Comms.},
year = {2001},
volume = {134},
@Article{aidun2010,
author = { Aidun, C. K. and Clausen, J. R.},
- title = {Lattice Boltzmann method for complex flows},
+ title = {Lattice {B}oltzmann method for complex flows},
journal = {Ann. Rev. Fluid Mech.},
year = {2010},
volume = {42},
@Article{stratford2008,
author = {Stratford, K. and Pagonabarraga, I.},
- title = {Parallel domain decomposition for lattice Boltzmann with moving particles},
+ title = {Parallel domain decomposition for lattice {B}oltzmann with moving particles},
journal = {Comput. Math. with Applications},
year = {2008},
volume = {55},
@Article{wei2004,
author = {Wei, X. and Li, W. and M\"uller, K. and Kaufman, A. E.},
- title = {The lattice Boltzmann method for simulating gaseous phenomena},
+ title = {The lattice {B}oltzmann method for simulating gaseous phenomena},
journal = {IEEE Transactions on Visualization and Computer Graphics},
year = {2004},
volume = {10},
@Article{zhu2006,
author = {Zhu, H. and Liu, X. and Liu, Y. and Wu, E.},
- title = {Simulation of miscible binary mixtures based on lattice Boltzmann method},
+ title = {Simulation of miscible binary mixtures based on lattice {B}oltzmann method},
journal = {Comp. Anim. Virtual Worlds},
year = {2006},
volume = {17},
@Article{zhao2007,
author = {Zhao, Y.},
- title = {Lattice Boltzmann based PDE solver on the GPU},
+ title = {Lattice {B}oltzmann based {PDE} solver on the {GPU}},
journal = {Visual Comput.},
year = {2008},
volume = {24},
@Article{toelke2010,
author = {T\"olke, J.},
- title = {Implementation of a lattice Boltzmann kernel using the compute unified device architecture developed by nVIDIA},
+ title = {Implementation of a lattice {B}oltzmann kernel using the compute unified device architecture developed by {NVIDIA}},
journal = {Comput. Visual Sci.},
year = {2010},
volume = {13},
@InProceedings{fan2004,
author = { Fan, Z. and Qiu, F. and Kaufman, A. and Yoakum-Stover, S.},
- title = {GPU cluster for high performance computing},
+ title = {{GPU} cluster for high performance computing},
booktitle = {Proceedings of ACM/IEEE Supercomputing Conference},
pages = {47--59},
year = {2004},
@Article{myre2011,
author = { Myre, J. and Walsh, S. D. C. and Lilja, D. and Saar, M. O.},
- title = {Performance analysis of single-phase, multiphase, and multicomponent lattice Boltzmann fluid flow simulations on GPU clusters},
+ title = {Performance analysis of single-phase, multiphase, and multicomponent lattice {B}oltzmann fluid flow simulations on {GPU} clusters},
journal = {Concurrency Computat.: Pract. Exper.},
year = {2011},
volume = {23},
@Article{obrecht2011,
author = { Obrecht, C. and Kuznik, F. and Tourancheau, B. and Roux, J.-J.},
- title = {Multi-GPU implementation of the lattice Boltzmann method},
+ title = {Multi-{GPU} implementation of the lattice {B}oltzmann method},
journal = {Comput. Math. with Applications},
year = {2013},
volume = {65},
@Article{bernaschi2010,
author = {Bernaschi, M. and Fatica, M. and Melchionna, S. and Succi, S. and Kaxiras, E.},
- title = {A flexible high-performance lattice Boltzmann GPU code for the
+ title = {A flexible high-performance lattice {B}oltzmann {GPU} code for the
simulations of fluid flow in complex geometries},
journal = {Concurrency Computat.: Pract. Exper.},
year = {2010},
@Article{xian2011,
author = { Xian, W. and Takayuki, A.},
- title = {Multi-GPU performance of incompressible flow computation by
-lattice Boltzmann method on GPU cluster},
+ title = {Multi-{GPU} performance of incompressible flow computation by
+lattice {B}oltzmann method on {GPU} cluster},
journal = {Parallel Comput.},
year = {2011},
volume = {37},
@Article{feichtinger2011,
author = {Feichtinger, C. and Habich, J. and K\"ostler, H. and Hager, G. and R\"ude, U. and
Wellein, G.},
- title = {A flexible patch-based lattice Boltzmann parallelization approach for heterogeneous GPU-CPU clusters},
+ title = {A flexible patch-based lattice {B}oltzmann parallelization approach for heterogeneous {GPU-CPU} clusters},
journal = {Parallel Computing},
year = {2011},
volume = {37},
@Article{wellein2006,
author = {Wellein, G. and Zeiser, T. and Hager, G. and Donath, S.},
- title = {On the single processor performance of simple lattice Boltzmann kernels},
+ title = {On the single processor performance of simple lattice {B}oltzmann kernels},
journal = {Computers and Fluids},
year = {2006},
volume = {35},
@Article{pohl2003,
author = { Pohl, T. and Kowarschik, M. and Wilke, J. and Igelberger, K. and R\"ude, U.},
- title = {Optimization and profiling of the cache performance of parallel lattice Boltzmann code},
+ title = {Optimization and profiling of the cache performance of parallel lattice {B}oltzmann code},
journal = {Parallel Process Lett.},
year = {2003},
volume = {13},
@Article{mattila2007,
author = {Mattila, K. and Hyv\"aluoma, J. and Rossi, T. and Aspn\"as M. and Westerholm, J.},
- title = {An efficient swap algorithm for the lattice Boltzmann method},
+ title = {An efficient swap algorithm for the lattice {B}oltzmann method},
journal = {Comput. Phys. Comms.},
year = {2007},
volume = {176},
@Article{wittmann2012,
author = {Wittmann, M. and Zeiser, T. and Hager, G. and Wellein, G.},
- title = {Comparison of different propagation steps for lattice Boltzmann methods},
+ title = {Comparison of different propagation steps for lattice {B}oltzmann methods},
journal = {Comput. Math with Appl.},
year = {2012},
note = {doi:10.1016/j.camwa.2012.05.002},
@Article{walshsaar2012,
author = {Walsh, S. D. C. and Saar, M. O.},
- title = {Developing extensible lattice Boltzmann simulators for general-purpose graphics-processing units},
+ title = {Developing extensible lattice {B}oltzmann simulators for general-purpose graphics-processing units},
journal = {Comm. Comput. Phys.},
year = {2013},
volume = {13},
@InProceedings{williams2011,
author = {Williams, S. and Oliker, L. and Carter, J. and Shalf, J.},
- title = {Extracting ultra-scale lattice Boltzmann performance via
+ title = {Extracting ultra-scale lattice {B}oltzmann performance via
hierarchical and distributed auto-tuning},
booktitle = {International Conference for High Performance Computing, Networking, Storage and Analysis (SC) },
pages = {1--12},
@Article{ch14:stratford-jsp2005,
author = {Stratford, K. and Adhikari, R. and Pagonabarraga, I. and Desplat, J.-C.},
- title = {Lattice Boltzmann for Binary Fluids with Suspended Colloids},
+ title = {Lattice {B}oltzmann for Binary Fluids with Suspended Colloids},
journal = {J. Stat. Phys.},
year = {2005},
volume = {121},
@Article{ladd1994,
author = {Ladd, A. J. C.},
- title = {Numerical simulations of particle suspensions via a discretized Boltzmann equation. Part 1. Theoretical foundation and Part II. Numerical results},
+ title = {Numerical simulations of particle suspensions via a discretized {B}oltzmann equation. {P}art 1. Theoretical foundation and {P}art II. {N}umerical results},
journal = {J. Fluid Mech.},
year = {1994},
volume = {271},
@Article{nguyen2002,
author = {Nguyen, N.-Q. and Ladd, A. J. C.},
- title = {Lubrication corrections for lattice Boltzmann simulations of particle suspensions},
+ title = {Lubrication corrections for lattice {B}oltzmann simulations of particle suspensions},
journal = {Phys. Rev. E},
year = {2002},
volume = {66},
@Article{ch14:immersed-lb,
author = {Feng, Z.-G. and Michaelides, E. E},
- title = {The immersed boundary-lattice Boltzmann method for solving
+ title = {The immersed boundary-lattice {B}oltzmann method for solving
fluid-particles interaction problem},
journal = {J. Comp. Phys.},
year = {2004},
dynamics problems. It provides a way to solve the incompressible
isothermal Navier-Stokes equations and has the attractive features of
being both explicit in time and local in space. This makes the LB
-method wellsuited to parallel computation. Many efficient parallel
+method well suited to parallel computation. Many efficient parallel
implementations of the LB method have been undertaken, typically using
a combination of distributed domain decomposition and the Message
Passing Interface (MPI). However, the potential
performance benefits offered by GPUs has motivated a new ``mixed-mode''
approach to address very large problems. Here, fine-grained
parallelism is implemented on the GPU, while MPI is reserved for
-largerscale parallelism. This mixed mode is of increasing interest
+larger scale parallelism. This mixed mode is of increasing interest
to application programmers at a time when many supercomputing services
are moving
toward clusters of GPU accelerated nodes. The design questions which
${\cal M}_{ij}f_j$ is used to transform the distributions into the
hydrodynamic quantities, where ${\cal M}_{ij}$ is a constant 19x19
matrix related to the choice of
-$\mathbf{c}_i$. The nonconserved hydrodynamic quantities are then
+$\mathbf{c}_i$. The non-conserved hydrodynamic quantities are then
relaxed toward their (known) equilibrium values and are transformed
-back to new postcollision distributions via the inverse transformation
+back to new post-collision distributions via the inverse transformation
${\cal M}^{-1}_{ij}$. This gives rise to the need for a minimum of $2\times 19^2$
floating point multiplications per lattice site. (Additional operations are
required to implement, for example, the force $\mathbf{f}(\mathbf{r})$.)
consecutive memory addresses allows the prefetcher to engage fully.
(The temporary scalar \texttt{a\_tmp} allows caching of the
intermediate accumulated value in the innermost loop.) A variety of
-standard sequential optimisations are relevant for the collision stage
+standard sequential optimizations are relevant for the collision stage
(loop unrolling, inclusion of SIMD operations, and so on
\cite{wellein2006}). For example, the collision stage in
\textit{Ludwig} includes explicit SIMD code, which is useful if the
-compiler on a given platform cannot identify it. The propagation
-stage is separate and is organised as a ``pull'' (again, various
+compiler on a given platform cannot identify the appropriate vectorization.
+The propagation
+stage is separate and is organized as a ``pull'' (again, various
optimizations have been considered, e.g.,
\cite{pohl2003,mattila2007,wittmann2012}). No further optimization is
done here beyond ensuring that the ordering of the discrete velocities
optimizations are important, it should be remembered that for some
complex fluid problems, the hydrodynamics embodied in the LB
calculation is a relatively small part of the total computational cost
-(at the 10\%level in some cases). This means optimization effort may
+(at the 10\% level in some cases). This means optimization effort may
be better concentrated elsewhere.
For the D3Q19 model, this reduces the volume of data traffic from 19 to
5 of the $f_i(\mathbf{r};t)$ per lattice site at each edge. In the CPU
version, the necessary transfers are implemented in place using
-a vector of appropriately strided MPI datatypes for each direction.
+a vector of MPI datatypes with appropriate stride for each direction.
\section{Single GPU implementation}\label{ch14:sec:singlegpu}
assigned to the fast {\it constant} on-chip device memory.
For the propagation stage, the GPU implementation adds a second time
-level of distribution values. The datadependencies inherent in the
+level of distribution values. The data dependencies inherent in the
propagation mean that the in-place propagation of the CPU version
cannot be parallelized effectively without the additional time level.
As both time levels may remain resident on the GPU, this is not a
through the host (in the future, direct GPU-to-GPU data transfers via
MPI may be possible, obviating the need for these steps). This
means host MPI sends must be preceded by appropriate device-to-host
-transfers and host MPI receives must be followed by corresponding host-to-device transfers.
+transfers and host MPI receives must be followed by corresponding
+host-to-device transfers.
%To execute on multiple GPUs in parallel we decompose the problem using
%the existing high-level parallelisaton framework: each MPI task acts
hosts at the same time as kernels for packing and retrieving of
data for the second coordinate direction are
executed. This
-overlapping must respect the synchronisation required to ensure
+overlapping must respect the synchronization required to ensure
that data values at the corners of the subdomain are transferred
correctly.
We use a separate CUDA stream for each coordinate direction:
\begin{figure}[t]
\centering
%\includegraphics[width=10cm]{Chapters/chapter14/figures/bbl}
-\includegraphics[width=10cm]{Chapters/chapter14/figures/colloid_new}
+\includegraphics[width=10cm]{Chapters/chapter14/figures/colloid_new_gray}
\caption[A two-dimensional schematic picture of spherical particles on the lattice.]{
A two-dimensional schematic picture of spherical particles on the lattice.
Left: a particle is allowed
In this section, we give a brief overview of the essential features
of the CPU implementation, and how the considerations raised in the
previous sections---maximizing parallelism and minimising both
-host-to-device and GPU to GPU data movement---shape the design decisions
+host-to-device and GPU-to-GPU data movement---shape the design decisions
for a GPU implementation. We restrict our discussion to a simple fluid
in this section; additional solid-fluid boundary conditions (e.g.,
wetting at a fluid-fluid-solid contact line) usually arise elsewhere
-in the calculation broadly independent of the hydrodynamic boundary
+in the calculation and are broadly independent of the hydrodynamic boundary
conditions which are described in what follows.
Moving solid particles (here, spheres) are defined by a center position
intercept or cut the spherical shell (see Fig.~\ref{ch14:fig:bbl}).
Hydrodynamic boundary conditions are then implemented via the standard
approach of bounce-back on links \cite{ladd1994, nguyen2002}, where the
-relevant postcollision distribution values are reversed at the
+relevant post-collision distribution values are reversed at the
propagation stage with an appropriate correction to allow for the
solid body motion. The exchange of momentum at each link must then be
accumulated around the entire particle surface to provide the net
molecular dynamics-like step.
For the CPU implementation, a number of additional MPI communications
-are required: (1) to exchange the center position, radius, etc of each
-particle as it moves and (2), to allow the accumulation of the force
+are required: (1) to exchange the center position, radius, etc.\ of each
+particle as it moves and (2) to allow the accumulation of the force
and torque for each particle (the links of which may be distributed
between up to 8 MPI tasks in three dimensions). Appropriate marshaling
of these data can provide an effective parallelization for relatively
concerns which are relevant to a GPU implementation:
\begin{enumerate}
\item
-Minimisation of host-device data transfer would argue for moving the
+Minimization of host-device data transfer would argue for moving the
entire particle code to the GPU. However, the code in question involves
largely conditional logic (e.g., identifying cut surface links) and
irregular memory accesses (e.g., access to distribution elements around
We have implemented the second option as follows. For each subdomain,
-a list of boundary-cutting links is assembled on the CPU which includes
-the identity of the relevant element of the distribution. This list,
+a list of boundary-cutting links is assembled on the CPU; the list includes
+the identity of the relevant element of the distribution in each case.
+This list,
together with the particle information required to compute the correct
-bounce-back term, are transferred to the GPU. The updates to the relevant
+bounce-back term, is transferred to the GPU. The updates to the relevant
elements of the distribution can then take place on the GPU. The
corresponding information to compute the update of the particle dynamics
is returned
possible. For each data structure, such as the distribution, a separate
analogue is maintained in both the CPU and GPU memory spaces. However,
the GPU copy does not include the complete CPU structure: in
-particular, nonintrinsic datatypes such as MPI datatypes are not
+particular, non-intrinsic datatypes such as MPI datatypes are not
required on the GPU. Functions to marshal data between CPU and GPU
are provided for each data structure, abstracting the underlying
CUDA implementation. (This reasonably lightweight abstraction layer
% use section* for acknowledgement
-\section*{Acknowledgments}
+\section{Acknowledgments}
AG was supported by the CRESTA project, which has received
funding from the European Community's Seventh Framework Programme
(ICT-2011.9.13) under Grant Agreement no. 287703. KS was supported
