fMerge branch 'master' of ssh://bilbo.iut-bm.univ-fcomte.fr/GMRES2stage

[GMRES2stage.git] / paper.tex

diff --git a/paper.tex b/paper.tex
index 54e35d38f3a810d26fbe5585e02b1c3a8d48f89b..1d4cac09f311bc2942f5bc0278957528aa0c19a3 100644 (file)
--- a/paper.tex
+++ b/paper.tex
@@ -364,12 +364,13 @@
 \algnewcommand\Output{\item[\algorithmicoutput]}
 
 \newtheorem{proposition}{Proposition}
 \algnewcommand\Output{\item[\algorithmicoutput]}
 
 \newtheorem{proposition}{Proposition}

+\newtheorem{proof}{Proof}

 
 \begin{document}
 %
 % paper title
 % can use linebreaks \\ within to get better formatting as desired
 
 \begin{document}
 %
 % paper title
 % can use linebreaks \\ within to get better formatting as desired

-\title{TSIRM: A Two-Stage Iteration with least-square Residual Minimization algorithm to solve large sparse linear systems}
+\title{TSIRM: A Two-Stage Iteration with least-squares Residual Minimization algorithm to solve large sparse linear systems}

 
 
 
 
 
 


@@ -380,7 +381,7 @@
 % use a multiple column layout for up to two different
 % affiliations
 
 % use a multiple column layout for up to two different
 % affiliations
 

-\author{\IEEEauthorblockN{Rapha\"el Couturier\IEEEauthorrefmark{1}, Lilia Ziane Khodja \IEEEauthorrefmark{2}, and Christophe Guyeux\IEEEauthorrefmark{1}}
+\author{\IEEEauthorblockN{Rapha\"el Couturier\IEEEauthorrefmark{1}, Lilia Ziane Khodja\IEEEauthorrefmark{2}, and Christophe Guyeux\IEEEauthorrefmark{1}}

 \IEEEauthorblockA{\IEEEauthorrefmark{1} Femto-ST Institute, University of Franche Comte, France\\
 Email: \{raphael.couturier,christophe.guyeux\}@univ-fcomte.fr}
 \IEEEauthorblockA{\IEEEauthorrefmark{2} INRIA Bordeaux Sud-Ouest, France\\
 \IEEEauthorblockA{\IEEEauthorrefmark{1} Femto-ST Institute, University of Franche Comte, France\\
 Email: \{raphael.couturier,christophe.guyeux\}@univ-fcomte.fr}
 \IEEEauthorblockA{\IEEEauthorrefmark{2} INRIA Bordeaux Sud-Ouest, France\\


@@ -425,16 +426,17 @@ Email: lilia.ziane@inria.fr}
 
 
 \begin{abstract}
 
 
 \begin{abstract}

-In  this article,  a  two-stage  iterative algorithm is proposed to improve  the
-convergence of Krylov based iterative methods,  typically those of GMRES variants. The
-principle of  the proposed approach  is to  build an external  iteration over  the Krylov
-method, and to  frequently store its current  residual   (at  each
-GMRES restart for instance). After a given number of outer iterations, a minimization
-step  is applied  on the  matrix composed by the  saved residuals,  in  order to
-compute a better solution and to make  new iterations if required.  It is proven that
-the proposal has  the same convergence properties than the  inner embedded method itself. 
-Experiments using up  to 16,394 cores also show that the proposed algorithm
-runs around 5 or 7 times faster than GMRES.
+In  this article, a  two-stage iterative  algorithm is  proposed to  improve the
+convergence  of  Krylov  based  iterative  methods,  typically  those  of  GMRES
+variants.  The  principle of  the  proposed approach  is  to  build an  external
+iteration over the  Krylov method, and to frequently  store its current residual
+(at each GMRES restart for instance).  After a given number of outer iterations,
+a least-squares minimization step is applied on the matrix composed by the saved
+residuals, in order  to compute a better solution and to  make new iterations if
+required.  It  is proven that the  proposal has the  same convergence properties
+than the  inner embedded  method itself.  Experiments  using up to  16,394 cores
+also  show that the  proposed algorithm  runs around  5 or  7 times  faster than
+GMRES.

 \end{abstract}
 
 \begin{IEEEkeywords}
 \end{abstract}
 
 \begin{IEEEkeywords}


@@ -581,7 +583,7 @@ performances.
 
 The present  article is  organized as follows.   Related works are  presented in
 Section~\ref{sec:02}. Section~\ref{sec:03} details the two-stage algorithm using
 
 The present  article is  organized as follows.   Related works are  presented in
 Section~\ref{sec:02}. Section~\ref{sec:03} details the two-stage algorithm using

-a  least-square  residual   minimization,  while  Section~\ref{sec:04}  provides
+a  least-squares  residual   minimization,  while  Section~\ref{sec:04}  provides

 convergence  results  regarding this  method.   Section~\ref{sec:05} shows  some
 experimental  results  obtained  on  large  clusters  using  routines  of  PETSc
 toolkit. This research work ends by  a conclusion section, in which the proposal
 convergence  results  regarding this  method.   Section~\ref{sec:05} shows  some
 experimental  results  obtained  on  large  clusters  using  routines  of  PETSc
 toolkit. This research work ends by  a conclusion section, in which the proposal


@@ -604,7 +606,7 @@ is summarized while intended perspectives are provided.
 
 %%%*********************************************************
 %%%*********************************************************
 
 %%%*********************************************************
 %%%*********************************************************

-\section{Two-stage iteration with least-square residuals minimization algorithm}
+\section{Two-stage iteration with least-squares residuals minimization algorithm}

 \label{sec:03}
 A two-stage algorithm is proposed  to solve large  sparse linear systems  of the
 form  $Ax=b$,  where  $A\in\mathbb{R}^{n\times   n}$  is  a  sparse  and  square
 \label{sec:03}
 A two-stage algorithm is proposed  to solve large  sparse linear systems  of the
 form  $Ax=b$,  where  $A\in\mathbb{R}^{n\times   n}$  is  a  sparse  and  square


@@ -615,7 +617,7 @@ inner-outer iteration  solver based  on iterative Krylov  methods. The  main
 key-points of the proposed solver are given in Algorithm~\ref{algo:01}.
 It can be summarized as follows: the
 inner solver is a Krylov based one. In order to accelerate its convergence, the 
 key-points of the proposed solver are given in Algorithm~\ref{algo:01}.
 It can be summarized as follows: the
 inner solver is a Krylov based one. In order to accelerate its convergence, the 

-outer solver periodically applies a least-square minimization  on the residuals computed by  the inner one. %Tsolver which does not required to be changed.
+outer solver periodically applies a least-squares minimization  on the residuals computed by  the inner one. %Tsolver which does not required to be changed.

 
 At each outer iteration, the sparse linear system $Ax=b$ is partially 
 solved using only $m$
 
 At each outer iteration, the sparse linear system $Ax=b$ is partially 
 solved using only $m$


@@ -626,8 +628,8 @@ inner solver. The current approximation of the Krylov method is then stored insi
 $S$ composed by the successive solutions that are computed during inner iterations.
 
 At each $s$ iterations, the minimization step is applied in order to
 $S$ composed by the successive solutions that are computed during inner iterations.
 
 At each $s$ iterations, the minimization step is applied in order to

-compute a new  solution $x$. For that, the previous  residuals are computed with
-$(b-AS)$. The minimization of the residuals is obtained by 
+compute a new  solution $x$. For that, the previous  residuals of $Ax=b$ are computed by
+the inner iterations with $(b-AS)$. The minimization of the residuals is obtained by  

 \begin{equation}
    \underset{\alpha\in\mathbb{R}^{s}}{min}\|b-R\alpha\|_2
 \label{eq:01}
 \begin{equation}
    \underset{\alpha\in\mathbb{R}^{s}}{min}\|b-R\alpha\|_2
 \label{eq:01}


@@ -636,7 +638,7 @@ with $R=AS$. Then the new solution $x$ is computed with $x=S\alpha$.
 
 
 In  practice, $R$  is a  dense rectangular  matrix belonging in  $\mathbb{R}^{n\times s}$,
 
 
 In  practice, $R$  is a  dense rectangular  matrix belonging in  $\mathbb{R}^{n\times s}$,

-with $s\ll n$.   In order  to minimize~\eqref{eq:01}, a  least-square method  such as
+with $s\ll n$.   In order  to minimize~\eqref{eq:01}, a  least-squares method  such as

 CGLS ~\cite{Hestenes52}  or LSQR~\cite{Paige82} is used. Remark that these  methods are more
 appropriate than a single direct method in a parallel context.
 
 CGLS ~\cite{Hestenes52}  or LSQR~\cite{Paige82} is used. Remark that these  methods are more
 appropriate than a single direct method in a parallel context.
 


@@ -647,15 +649,15 @@ appropriate than a single direct method in a parallel context.
 \begin{algorithmic}[1]
   \Input $A$ (sparse matrix), $b$ (right-hand side)
   \Output $x$ (solution vector)\vspace{0.2cm}
 \begin{algorithmic}[1]
   \Input $A$ (sparse matrix), $b$ (right-hand side)
   \Output $x$ (solution vector)\vspace{0.2cm}

-  \State Set the initial guess $x^0$
+  \State Set the initial guess $x_0$

   \For {$k=1,2,3,\ldots$ until convergence (error$<\epsilon_{tsirm}$)} \label{algo:conv}
   \For {$k=1,2,3,\ldots$ until convergence (error$<\epsilon_{tsirm}$)} \label{algo:conv}

-    \State  $x^k=Solve(A,b,x^{k-1},max\_iter_{kryl})$   \label{algo:solve}
+    \State  $x_k=Solve(A,b,x_{k-1},max\_iter_{kryl})$   \label{algo:solve}

     \State retrieve error
     \State retrieve error

-    \State $S_{k \mod s}=x^k$ \label{algo:store}
+    \State $S_{k \mod s}=x_k$ \label{algo:store}

     \If {$k \mod s=0$ {\bf and} error$>\epsilon_{kryl}$}
       \State $R=AS$ \Comment{compute dense matrix} \label{algo:matrix_mul}
     \If {$k \mod s=0$ {\bf and} error$>\epsilon_{kryl}$}
       \State $R=AS$ \Comment{compute dense matrix} \label{algo:matrix_mul}

-            \State Solve least-square problem $\underset{\alpha\in\mathbb{R}^{s}}{min}\|b-R\alpha\|_2$ \label{algo:}
-      \State $x^k=S\alpha$  \Comment{compute new solution}
+            \State $\alpha=Least\_Squares(R,b,max\_iter_{ls})$ \label{algo:}
+      \State $x_k=S\alpha$  \Comment{compute new solution}

     \EndIf
   \EndFor
 \end{algorithmic}
     \EndIf
   \EndFor
 \end{algorithmic}


@@ -680,19 +682,19 @@ Let us summarize the most important parameters of TSIRM:
 \item $\epsilon_{tsirm}$: the threshold to stop the TSIRM method;
 \item $max\_iter_{kryl}$: the maximum number of iterations for the Krylov method;
 \item $s$: the number of outer iterations before applying the minimization step;
 \item $\epsilon_{tsirm}$: the threshold to stop the TSIRM method;
 \item $max\_iter_{kryl}$: the maximum number of iterations for the Krylov method;
 \item $s$: the number of outer iterations before applying the minimization step;

-\item $max\_iter_{ls}$: the maximum number of iterations for the iterative least-square method;
-\item $\epsilon_{ls}$: the threshold used to stop the least-square method.
+\item $max\_iter_{ls}$: the maximum number of iterations for the iterative least-squares method;
+\item $\epsilon_{ls}$: the threshold used to stop the least-squares method.

 \end{itemize}
 
 
 The  parallelisation  of  TSIRM  relies   on  the  parallelization  of  all  its
 \end{itemize}
 
 
 The  parallelisation  of  TSIRM  relies   on  the  parallelization  of  all  its

-parts. More  precisely, except  the least-square step,  all the other  parts are
+parts. More  precisely, except  the least-squares step,  all the other  parts are

 obvious to  achieve out in parallel. In  order to develop a  parallel version of
 our   code,   we   have   chosen  to   use   PETSc~\cite{petsc-web-page}.    For
 line~\ref{algo:matrix_mul} the  matrix-matrix multiplication is  implemented and
 efficient since the  matrix $A$ is sparse and since the  matrix $S$ contains few
 colums in  practice. As explained  previously, at least  two methods seem  to be
 obvious to  achieve out in parallel. In  order to develop a  parallel version of
 our   code,   we   have   chosen  to   use   PETSc~\cite{petsc-web-page}.    For
 line~\ref{algo:matrix_mul} the  matrix-matrix multiplication is  implemented and
 efficient since the  matrix $A$ is sparse and since the  matrix $S$ contains few
 colums in  practice. As explained  previously, at least  two methods seem  to be

-interesting to solve the least-square minimization, CGLS and LSQR.
+interesting to solve the least-squares minimization, CGLS and LSQR.

 
 In the following  we remind the CGLS algorithm. The LSQR  method follows more or
 less the same principle but it takes more place, so we briefly explain the parallelization of CGLS which is similar to LSQR.
 
 In the following  we remind the CGLS algorithm. The LSQR  method follows more or
 less the same principle but it takes more place, so we briefly explain the parallelization of CGLS which is similar to LSQR.


@@ -702,19 +704,21 @@ less the same principle but it takes more place, so we briefly explain the paral
 \begin{algorithmic}[1]
   \Input $A$ (matrix), $b$ (right-hand side)
   \Output $x$ (solution vector)\vspace{0.2cm}
 \begin{algorithmic}[1]
   \Input $A$ (matrix), $b$ (right-hand side)
   \Output $x$ (solution vector)\vspace{0.2cm}

-  \State $r=b-Ax$
-  \State $p=A'r$
-  \State $s=p$
-  \State $g=||s||^2_2$
-  \For {$k=1,2,3,\ldots$ until convergence (g$<\epsilon_{ls}$)} \label{algo2:conv}
-    \State $q=Ap$
-    \State $\alpha=g/||q||^2_2$
-    \State $x=x+alpha*p$
-    \State $r=r-alpha*q$
-    \State $s=A'*r$
-    \State $g_{old}=g$
-    \State $g=||s||^2_2$
-    \State $\beta=g/g_{old}$
+  \State Let $x_0$ be an initial approximation
+  \State $r_0=b-Ax_0$
+  \State $p_1=A^Tr_0$
+  \State $s_0=p_1$
+  \State $\gamma=||s_0||^2_2$
+  \For {$k=1,2,3,\ldots$ until convergence ($\gamma<\epsilon_{ls}$)} \label{algo2:conv}
+    \State $q_k=Ap_k$
+    \State $\alpha_k=\gamma/||q_k||^2_2$
+    \State $x_k=x_{k-1}+\alpha_kp_k$
+    \State $r_k=r_{k-1}-\alpha_kq_k$
+    \State $s_k=A^Tr_k$
+    \State $\gamma_{old}=\gamma$
+    \State $\gamma=||s_k||^2_2$
+    \State $\beta_k=\gamma/\gamma_{old}$
+    \State $p_{k+1}=s_k+\beta_kp_k$

   \EndFor
 \end{algorithmic}
 \label{algo:02}
   \EndFor
 \end{algorithmic}
 \label{algo:02}


@@ -738,12 +742,23 @@ Suppose that $A$ is a positive real matrix with symmetric part $M$. Then the res
 \begin{equation}
 ||r_m|| \leqslant \left(1-\dfrac{\alpha}{\beta}\right)^{\frac{m}{2}} ||r_0|| ,
 \end{equation}
 \begin{equation}
 ||r_m|| \leqslant \left(1-\dfrac{\alpha}{\beta}\right)^{\frac{m}{2}} ||r_0|| ,
 \end{equation}

-where $\alpha = \lambda_min(M)^2$ and $\beta = \lambda_max(A^T A)$, which proves 
+where $\alpha = \lambda_{min}(M)^2$ and $\beta = \lambda_{max}(A^T A)$, which proves 

 the convergence of GMRES($m$) for all $m$ under that assumption regarding $A$.
 \end{proposition}
 
 the convergence of GMRES($m$) for all $m$ under that assumption regarding $A$.
 \end{proposition}
 

+We can now claim that,
+\begin{proposition}
+If $A$ is a positive real matrix, then the TSIRM algorithm is convergent.
+\end{proposition}
+
+\begin{proof}
+Let $r_k = b-Ax_k$, where $x_k$ is the approximation of the solution after the
+$k$-th iterate of TSIRM.
+We will prove that $r_k \rightarrow 0$ when $k \rightarrow +\infty$.

 
 
 
 

+\end{proof}
+

 %%%*********************************************************
 %%%*********************************************************
 \section{Experiments using PETSc}
 %%%*********************************************************
 %%%*********************************************************
 \section{Experiments using PETSc}


@@ -754,7 +769,7 @@ In order to see the influence of our algorithm with only one processor, we first
 show  a comparison  with the  standard version  of GMRES  and our  algorithm. In
 Table~\ref{tab:01},  we  show  the  matrices  we  have used  and  some  of  them
 characteristics. For all  the matrices, the name, the field,  the number of rows
 show  a comparison  with the  standard version  of GMRES  and our  algorithm. In
 Table~\ref{tab:01},  we  show  the  matrices  we  have used  and  some  of  them
 characteristics. For all  the matrices, the name, the field,  the number of rows

-and the number of nonzero elements is given.
+and the number of nonzero elements are given.

 
 \begin{table}[htbp]
 \begin{center}
 
 \begin{table}[htbp]
 \begin{center}


@@ -777,7 +792,7 @@ torso3             & 2D/3D problem & 259,156 & 4,429,042 \\
 
 The following  parameters have been chosen  for our experiments.   As by default
 the restart  of GMRES is performed every  30 iterations, we have  chosen to stop
 
 The following  parameters have been chosen  for our experiments.   As by default
 the restart  of GMRES is performed every  30 iterations, we have  chosen to stop

-the GMRES every 30 iterations, $max\_iter_{kryl}=30$).  $s$ is set to 8. CGLS is
+the GMRES every 30 iterations (\emph{i.e.} $max\_iter_{kryl}=30$).  $s$ is set to 8. CGLS is

 chosen  to minimize  the least-squares  problem with  the  following parameters:
 $\epsilon_{ls}=1e-40$ and $max\_iter_{ls}=20$.  The external precision is set to
 $\epsilon_{tsirm}=1e-10$.  Those  experiments have been performed  on a Intel(R)
 chosen  to minimize  the least-squares  problem with  the  following parameters:
 $\epsilon_{ls}=1e-40$ and $max\_iter_{ls}=20$.  The external precision is set to
 $\epsilon_{tsirm}=1e-10$.  Those  experiments have been performed  on a Intel(R)


@@ -789,7 +804,7 @@ systems obtained with the previous matrices  with a GMRES variant and with out 2
 stage algorithm are  given. In the second column, it can  be noticed that either
 gmres or fgmres is used to  solve the linear system.  According to the matrices,
 different  preconditioner is used.   With TSIRM,  the same  solver and  the same
 stage algorithm are  given. In the second column, it can  be noticed that either
 gmres or fgmres is used to  solve the linear system.  According to the matrices,
 different  preconditioner is used.   With TSIRM,  the same  solver and  the same

-preconditionner is used.  This Table shows that TSIRM can drastically reduce the
+preconditionner are used.  This Table shows that TSIRM can drastically reduce the

 number of iterations to reach the  convergence when the number of iterations for
 the normal GMRES is more or less  greater than 500. In fact this also depends on
 tow  parameters: the  number  of iterations  to  stop GMRES  and  the number  of
 number of iterations to reach the  convergence when the number of iterations for
 the normal GMRES is more or less  greater than 500. In fact this also depends on
 tow  parameters: the  number  of iterations  to  stop GMRES  and  the number  of


@@ -823,28 +838,27 @@ torso3             & fgmres / sor  & 37.70 & 565 & 34.97 & 510 \\
 
 
 
 
 
 

-In order to perform larger  experiments, we have tested some example application
+In order to perform larger  experiments, we have tested some example applications

 of PETSc. Those  applications are available in the ksp part  which is suited for
 scalable linear equations solvers:
 \begin{itemize}
 \item ex15  is an example  which solves in  parallel an operator using  a finite
 of PETSc. Those  applications are available in the ksp part  which is suited for
 scalable linear equations solvers:
 \begin{itemize}
 \item ex15  is an example  which solves in  parallel an operator using  a finite

-  difference  scheme.   The  diagonal  is  equals to  4  and  4  extra-diagonals
-  representing the neighbors in each directions  is equal to -1. This example is
+  difference  scheme.   The  diagonal  is  equal to  4  and  4  extra-diagonals
+  representing the neighbors in each directions  are equal to -1. This example is

   used  in many  physical phenomena, for  example, heat  and fluid  flow, wave
   used  in many  physical phenomena, for  example, heat  and fluid  flow, wave

-  propagation...
+  propagation, etc.

 \item ex54 is another example based on 2D problem discretized with quadrilateral
   finite elements. For this example, the user can define the scaling of material
 \item ex54 is another example based on 2D problem discretized with quadrilateral
   finite elements. For this example, the user can define the scaling of material

-  coefficient in embedded circle, it is called $\alpha$.
+  coefficient in embedded circle called $\alpha$.

 \end{itemize}
 \end{itemize}

-For more technical details on  these applications, interested reader are invited
-to  read the  codes available  in the  PETSc sources.   Those problem  have been
-chosen because they  are scalable with many cores. We  have tested other problem
-but they are not scalable with many cores.
+For more technical details on  these applications, interested readers are invited
+to  read the  codes available  in the  PETSc sources.   Those problems  have been
+chosen because they  are scalable with many cores which is not the case of other problems that we have tested.

 
 In the following larger experiments are described on two large scale architectures: Curie and Juqeen... {\bf description...}\\
 
 
 
 In the following larger experiments are described on two large scale architectures: Curie and Juqeen... {\bf description...}\\
 
 

-{\bf Description of preconditioners}
+{\bf Description of preconditioners}\\

 
 \begin{table*}[htbp]
 \begin{center}
 
 \begin{table*}[htbp]
 \begin{center}


@@ -865,15 +879,15 @@ In the following larger experiments are described on two large scale architectur
 \hline
 
 \end{tabular}
 \hline
 
 \end{tabular}

-\caption{Comparison of FGMRES and TSIRM with FGMRES for example ex15 of PETSc with two preconditioner (mg and sor) with 25,000 components per core on Juqueen (threshold 1e-3, restart=30, s=12),  time is expressed in seconds.}
+\caption{Comparison of FGMRES and TSIRM with FGMRES for example ex15 of PETSc with two preconditioners (mg and sor) with 25,000 components per core on Juqueen (threshold 1e-3, restart=30, s=12),  time is expressed in seconds.}

 \label{tab:03}
 \end{center}
 \end{table*}
 
 Table~\ref{tab:03} shows  the execution  times and the  number of  iterations of
 \label{tab:03}
 \end{center}
 \end{table*}
 
 Table~\ref{tab:03} shows  the execution  times and the  number of  iterations of

-example ex15  of PETSc on the  Juqueen architecture. Differents  number of cores
-are  studied rangin  from  2,048  upto 16,383.   Two  preconditioners have  been
-tested.   For those experiments,  the number  of components  (or unknown  of the
+example ex15  of PETSc on the  Juqueen architecture. Different  numbers of cores
+are  studied ranging  from  2,048  up-to 16,383.   Two  preconditioners have  been
+tested: {\it mg} and {\it sor}.   For those experiments,  the number  of components  (or unknowns  of the

 problems)  per processor  is fixed  to 25,000,  also called  weak  scaling. This
 number can seem relatively small. In fact, for some applications that need a lot
 of  memory, the  number of  components per  processor requires  sometimes  to be
 problems)  per processor  is fixed  to 25,000,  also called  weak  scaling. This
 number can seem relatively small. In fact, for some applications that need a lot
 of  memory, the  number of  components per  processor requires  sometimes  to be


@@ -881,11 +895,11 @@ small.
 
 
 
 
 
 

-In this Table, we  can notice that TSIRM is always faster  than FGMRES. The last
+In Table~\ref{tab:03}, we  can notice that TSIRM is always faster  than FGMRES. The last

 column shows the ratio between FGMRES and the best version of TSIRM according to
 the minimization  procedure: CGLS or  LSQR. Even if  we have computed  the worst
 column shows the ratio between FGMRES and the best version of TSIRM according to
 the minimization  procedure: CGLS or  LSQR. Even if  we have computed  the worst

-case  between CGLS  and LSQR,  it is  clear that  TSIRM is  alsways  faster than
-FGMRES. For this example, the  multigrid preconditionner is faster than SOR. The
+case  between CGLS  and LSQR,  it is  clear that  TSIRM is  always  faster than
+FGMRES. For this example, the  multigrid preconditioner is faster than SOR. The

 gain  between   TSIRM  and  FGMRES  is   more  or  less  similar   for  the  two
 preconditioners.  Looking at the number  of iterations to reach the convergence,
 it is  obvious that TSIRM allows the  reduction of the number  of iterations. It
 gain  between   TSIRM  and  FGMRES  is   more  or  less  similar   for  the  two
 preconditioners.  Looking at the number  of iterations to reach the convergence,
 it is  obvious that TSIRM allows the  reduction of the number  of iterations. It


@@ -897,7 +911,7 @@ corresponds to 30*12, there are $max\_iter_{ls}$ which corresponds to 15.
 \begin{figure}[htbp]
 \centering
   \includegraphics[width=0.45\textwidth]{nb_iter_sec_ex15_juqueen}
 \begin{figure}[htbp]
 \centering
   \includegraphics[width=0.45\textwidth]{nb_iter_sec_ex15_juqueen}

-\caption{Number of iterations per second with ex15 and the same parameters than in Table~\ref{tab:03}}
+\caption{Number of iterations per second with ex15 and the same parameters than in Table~\ref{tab:03} (weak scaling)}

 \label{fig:01}
 \end{figure}
 
 \label{fig:01}
 \end{figure}
 


@@ -940,7 +954,7 @@ the number of iterations. So, the overall benefit of using TSIRM is interesting.
 \end{table*}
 
 
 \end{table*}
 
 

-In Table~\ref{tab:04}, some experiments with example ex54 on the Curie architecture are reported
+In Table~\ref{tab:04}, some experiments with example ex54 on the Curie architecture are reported.

 
 
 \begin{table*}[htbp]
 
 
 \begin{table*}[htbp]


@@ -965,6 +979,13 @@ In Table~\ref{tab:04}, some experiments with example ex54 on the Curie architect
 \end{center}
 \end{table*}
 
 \end{center}
 \end{table*}
 

+\begin{figure}[htbp]
+\centering
+  \includegraphics[width=0.45\textwidth]{nb_iter_sec_ex54_curie}
+\caption{Number of iterations per second with ex54 and the same parameters than in Table~\ref{tab:05} (strong scaling)}
+\label{fig:02}
+\end{figure}
+

 %%%*********************************************************
 %%%*********************************************************
 
 %%%*********************************************************
 %%%*********************************************************
 


@@ -1039,5 +1060,3 @@ Curie and Juqueen respectively based in France and Germany.
 
 % that's all folks
 \end{document}
 
 % that's all folks
 \end{document}

-
-